Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * copyfromparse.c
4 : : * Parse CSV/text/binary format for COPY FROM.
5 : : *
6 : : * This file contains routines to parse the text, CSV and binary input
7 : : * formats. The main entry point is NextCopyFrom(), which parses the
8 : : * next input line and returns it as Datums.
9 : : *
10 : : * In text/CSV mode, the parsing happens in multiple stages:
11 : : *
12 : : * [data source] --> raw_buf --> input_buf --> line_buf --> attribute_buf
13 : : * 1. 2. 3. 4.
14 : : *
15 : : * 1. CopyLoadRawBuf() reads raw data from the input file or client, and
16 : : * places it into 'raw_buf'.
17 : : *
18 : : * 2. CopyConvertBuf() calls the encoding conversion function to convert
19 : : * the data in 'raw_buf' from client to server encoding, placing the
20 : : * converted result in 'input_buf'.
21 : : *
22 : : * 3. CopyReadLine() parses the data in 'input_buf', one line at a time.
23 : : * It is responsible for finding the next newline marker, taking quote and
24 : : * escape characters into account according to the COPY options. The line
25 : : * is copied into 'line_buf', with quotes and escape characters still
26 : : * intact.
27 : : *
28 : : * 4. CopyReadAttributesText/CSV() function takes the input line from
29 : : * 'line_buf', and splits it into fields, unescaping the data as required.
30 : : * The fields are stored in 'attribute_buf', and 'raw_fields' array holds
31 : : * pointers to each field.
32 : : *
33 : : * If encoding conversion is not required, a shortcut is taken in step 2 to
34 : : * avoid copying the data unnecessarily. The 'input_buf' pointer is set to
35 : : * point directly to 'raw_buf', so that CopyLoadRawBuf() loads the raw data
36 : : * directly into 'input_buf'. CopyConvertBuf() then merely validates that
37 : : * the data is valid in the current encoding.
38 : : *
39 : : * In binary mode, the pipeline is much simpler. Input is loaded into
40 : : * 'raw_buf', and encoding conversion is done in the datatype-specific
41 : : * receive functions, if required. 'input_buf' and 'line_buf' are not used,
42 : : * but 'attribute_buf' is used as a temporary buffer to hold one attribute's
43 : : * data when it's passed the receive function.
44 : : *
45 : : * 'raw_buf' is always 64 kB in size (RAW_BUF_SIZE). 'input_buf' is also
46 : : * 64 kB (INPUT_BUF_SIZE), if encoding conversion is required. 'line_buf'
47 : : * and 'attribute_buf' are expanded on demand, to hold the longest line
48 : : * encountered so far.
49 : : *
50 : : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
51 : : * Portions Copyright (c) 1994, Regents of the University of California
52 : : *
53 : : *
54 : : * IDENTIFICATION
55 : : * src/backend/commands/copyfromparse.c
56 : : *
57 : : *-------------------------------------------------------------------------
58 : : */
59 : : #include "postgres.h"
60 : :
61 : : #include <ctype.h>
62 : : #include <unistd.h>
63 : : #include <sys/stat.h>
64 : :
65 : : #include "commands/copyapi.h"
66 : : #include "commands/copyfrom_internal.h"
67 : : #include "commands/progress.h"
68 : : #include "executor/executor.h"
69 : : #include "libpq/libpq.h"
70 : : #include "libpq/pqformat.h"
71 : : #include "mb/pg_wchar.h"
72 : : #include "miscadmin.h"
73 : : #include "pgstat.h"
74 : : #include "port/pg_bitutils.h"
75 : : #include "port/pg_bswap.h"
76 : : #include "port/simd.h"
77 : : #include "utils/builtins.h"
78 : : #include "utils/rel.h"
79 : : #include "utils/wait_event.h"
80 : :
81 : : #define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
82 : : #define OCTVALUE(c) ((c) - '0')
83 : :
84 : : /*
85 : : * These macros centralize code used to process line_buf and input_buf buffers.
86 : : * They are macros because they often do continue/break control and to avoid
87 : : * function call overhead in tight COPY loops.
88 : : *
89 : : * We must use "if (1)" because the usual "do {...} while(0)" wrapper would
90 : : * prevent the continue/break processing from working. We end the "if (1)"
91 : : * with "else ((void) 0)" to ensure the "if" does not unintentionally match
92 : : * any "else" in the calling code, and to avoid any compiler warnings about
93 : : * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
94 : : */
95 : :
96 : : /*
97 : : * This keeps the character read at the top of the loop in the buffer
98 : : * even if there is more than one read-ahead.
99 : : */
100 : : #define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
101 : : if (1) \
102 : : { \
103 : : if (input_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
104 : : { \
105 : : input_buf_ptr = prev_raw_ptr; /* undo fetch */ \
106 : : need_data = true; \
107 : : continue; \
108 : : } \
109 : : } else ((void) 0)
110 : :
111 : : /* This consumes the remainder of the buffer and breaks */
112 : : #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
113 : : if (1) \
114 : : { \
115 : : if (input_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
116 : : { \
117 : : if (extralen) \
118 : : input_buf_ptr = copy_buf_len; /* consume the partial character */ \
119 : : /* backslash just before EOF, treat as data char */ \
120 : : result = true; \
121 : : break; \
122 : : } \
123 : : } else ((void) 0)
124 : :
125 : : /*
126 : : * Transfer any approved data to line_buf; must do this to be sure
127 : : * there is some room in input_buf.
128 : : */
129 : : #define REFILL_LINEBUF \
130 : : if (1) \
131 : : { \
132 : : if (input_buf_ptr > cstate->input_buf_index) \
133 : : { \
134 : : appendBinaryStringInfo(&cstate->line_buf, \
135 : : cstate->input_buf + cstate->input_buf_index, \
136 : : input_buf_ptr - cstate->input_buf_index); \
137 : : cstate->input_buf_index = input_buf_ptr; \
138 : : } \
139 : : } else ((void) 0)
140 : :
141 : : /* NOTE: there's a copy of this in copyto.c */
142 : : static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0";
143 : :
144 : :
145 : : /* non-export function prototypes */
146 : : static bool CopyReadLine(CopyFromState cstate, bool is_csv);
147 : : static pg_always_inline bool CopyReadLineText(CopyFromState cstate,
148 : : bool is_csv);
149 : : static int CopyReadAttributesText(CopyFromState cstate);
150 : : static int CopyReadAttributesCSV(CopyFromState cstate);
151 : : static Datum CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
152 : : Oid typioparam, int32 typmod,
153 : : bool *isnull);
154 : : static pg_always_inline bool CopyFromTextLikeOneRow(CopyFromState cstate,
155 : : ExprContext *econtext,
156 : : Datum *values,
157 : : bool *nulls,
158 : : bool is_csv);
159 : : static pg_always_inline bool NextCopyFromRawFieldsInternal(CopyFromState cstate,
160 : : char ***fields,
161 : : int *nfields,
162 : : bool is_csv);
163 : :
164 : :
165 : : /* Low-level communications functions */
166 : : static int CopyGetData(CopyFromState cstate, void *databuf,
167 : : int minread, int maxread);
168 : : static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
169 : : static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
170 : : static void CopyLoadInputBuf(CopyFromState cstate, bool speculative);
171 : : static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
172 : :
173 : : void
2127 heikki.linnakangas@i 174 :CBC 691 : ReceiveCopyBegin(CopyFromState cstate)
175 : : {
176 : : StringInfoData buf;
2026 177 : 691 : int natts = list_length(cstate->attnumlist);
188 andrew@dunslane.net 178 : 691 : int16 format = (cstate->opts.format == COPY_FORMAT_BINARY ? 1 : 0);
179 : : int i;
180 : :
1125 nathan@postgresql.or 181 : 691 : pq_beginmessage(&buf, PqMsg_CopyInResponse);
2026 heikki.linnakangas@i 182 : 691 : pq_sendbyte(&buf, format); /* overall format */
183 : 691 : pq_sendint16(&buf, natts);
184 [ + + ]: 2477 : for (i = 0; i < natts; i++)
185 : 1786 : pq_sendint16(&buf, format); /* per-column formats */
186 : 691 : pq_endmessage(&buf);
187 : 691 : cstate->copy_src = COPY_FRONTEND;
188 : 691 : cstate->fe_msgbuf = makeStringInfo();
189 : : /* We *must* flush here to ensure FE knows it can send. */
2127 190 : 691 : pq_flush();
191 : 691 : }
192 : :
193 : : void
194 : 8 : ReceiveCopyBinaryHeader(CopyFromState cstate)
195 : : {
196 : : char readSig[11];
197 : : int32 tmp;
198 : :
199 : : /* Signature */
200 [ + - ]: 8 : if (CopyReadBinaryData(cstate, readSig, 11) != 11 ||
201 [ - + ]: 8 : memcmp(readSig, BinarySignature, 11) != 0)
2127 heikki.linnakangas@i 202 [ # # ]:UBC 0 : ereport(ERROR,
203 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
204 : : errmsg("COPY file signature not recognized")));
205 : : /* Flags field */
2127 heikki.linnakangas@i 206 [ - + ]:CBC 8 : if (!CopyGetInt32(cstate, &tmp))
2127 heikki.linnakangas@i 207 [ # # ]:UBC 0 : ereport(ERROR,
208 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
209 : : errmsg("invalid COPY file header (missing flags)")));
2127 heikki.linnakangas@i 210 [ - + ]:CBC 8 : if ((tmp & (1 << 16)) != 0)
2127 heikki.linnakangas@i 211 [ # # ]:UBC 0 : ereport(ERROR,
212 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
213 : : errmsg("invalid COPY file header (WITH OIDS)")));
2127 heikki.linnakangas@i 214 :CBC 8 : tmp &= ~(1 << 16);
215 [ - + ]: 8 : if ((tmp >> 16) != 0)
2127 heikki.linnakangas@i 216 [ # # ]:UBC 0 : ereport(ERROR,
217 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
218 : : errmsg("unrecognized critical flags in COPY file header")));
219 : : /* Header extension length */
2127 heikki.linnakangas@i 220 [ + - ]:CBC 8 : if (!CopyGetInt32(cstate, &tmp) ||
221 [ - + ]: 8 : tmp < 0)
2127 heikki.linnakangas@i 222 [ # # ]:UBC 0 : ereport(ERROR,
223 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
224 : : errmsg("invalid COPY file header (missing length)")));
225 : : /* Skip extension header, if present */
2127 heikki.linnakangas@i 226 [ - + ]:CBC 8 : while (tmp-- > 0)
227 : : {
2127 heikki.linnakangas@i 228 [ # # ]:UBC 0 : if (CopyReadBinaryData(cstate, readSig, 1) != 1)
229 [ # # ]: 0 : ereport(ERROR,
230 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
231 : : errmsg("invalid COPY file header (wrong length)")));
232 : : }
2127 heikki.linnakangas@i 233 :CBC 8 : }
234 : :
235 : : /*
236 : : * CopyGetData reads data from the source (file or frontend)
237 : : *
238 : : * We attempt to read at least minread, and at most maxread, bytes from
239 : : * the source. The actual number of bytes read is returned; if this is
240 : : * less than minread, EOF was detected.
241 : : *
242 : : * Note: when copying from the frontend, we expect a proper EOF mark per
243 : : * protocol; if the frontend simply drops the connection, we raise error.
244 : : * It seems unwise to allow the COPY IN to complete normally in that case.
245 : : *
246 : : * NB: no data conversion is applied here.
247 : : */
248 : : static int
249 : 217577 : CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
250 : : {
251 : 217577 : int bytesread = 0;
252 : :
253 [ + + + - ]: 217577 : switch (cstate->copy_src)
254 : : {
255 : 713 : case COPY_FILE:
229 michael@paquier.xyz 256 : 713 : pgstat_report_wait_start(WAIT_EVENT_COPY_FROM_READ);
2127 heikki.linnakangas@i 257 : 713 : bytesread = fread(databuf, 1, maxread, cstate->copy_file);
229 michael@paquier.xyz 258 : 713 : pgstat_report_wait_end();
2127 heikki.linnakangas@i 259 [ - + ]: 713 : if (ferror(cstate->copy_file))
2127 heikki.linnakangas@i 260 [ # # ]:UBC 0 : ereport(ERROR,
261 : : (errcode_for_file_access(),
262 : : errmsg("could not read from COPY file: %m")));
2127 heikki.linnakangas@i 263 [ + + ]:CBC 713 : if (bytesread == 0)
1998 264 : 281 : cstate->raw_reached_eof = true;
2127 265 : 713 : break;
2026 266 : 201784 : case COPY_FRONTEND:
1998 267 [ + - + + : 402543 : while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof)
+ + ]
268 : : {
269 : : int avail;
270 : :
2127 271 [ + + ]: 402061 : while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
272 : : {
273 : : /* Try to receive another message */
274 : : int mtype;
275 : : int maxmsglen;
276 : :
277 : 201302 : readmessage:
278 : 201302 : HOLD_CANCEL_INTERRUPTS();
279 : 201302 : pq_startmsgread();
280 : 201302 : mtype = pq_getbyte();
281 [ - + ]: 201302 : if (mtype == EOF)
2127 heikki.linnakangas@i 282 [ # # ]:UBC 0 : ereport(ERROR,
283 : : (errcode(ERRCODE_CONNECTION_FAILURE),
284 : : errmsg("unexpected EOF on client connection with an open transaction")));
285 : : /* Validate message type and set packet size limit */
1971 tgl@sss.pgh.pa.us 286 [ + + + ]:CBC 201302 : switch (mtype)
287 : : {
1125 nathan@postgresql.or 288 : 200759 : case PqMsg_CopyData:
1971 tgl@sss.pgh.pa.us 289 : 200759 : maxmsglen = PQ_LARGE_MESSAGE_LIMIT;
290 : 200759 : break;
1125 nathan@postgresql.or 291 : 541 : case PqMsg_CopyDone:
292 : : case PqMsg_CopyFail:
293 : : case PqMsg_Flush:
294 : : case PqMsg_Sync:
1971 tgl@sss.pgh.pa.us 295 : 541 : maxmsglen = PQ_SMALL_MESSAGE_LIMIT;
296 : 541 : break;
297 : 2 : default:
298 [ + - ]: 2 : ereport(ERROR,
299 : : (errcode(ERRCODE_PROTOCOL_VIOLATION),
300 : : errmsg("unexpected message type 0x%02X during COPY from stdin",
301 : : mtype)));
302 : : maxmsglen = 0; /* keep compiler quiet */
303 : : break;
304 : : }
305 : : /* Now collect the message body */
306 [ - + ]: 201300 : if (pq_getmessage(cstate->fe_msgbuf, maxmsglen))
2127 heikki.linnakangas@i 307 [ # # ]:UBC 0 : ereport(ERROR,
308 : : (errcode(ERRCODE_CONNECTION_FAILURE),
309 : : errmsg("unexpected EOF on client connection with an open transaction")));
2127 heikki.linnakangas@i 310 [ - + ]:CBC 201300 : RESUME_CANCEL_INTERRUPTS();
311 : : /* ... and process it */
312 [ + + - - : 201300 : switch (mtype)
- ]
313 : : {
1125 nathan@postgresql.or 314 : 200759 : case PqMsg_CopyData:
2127 heikki.linnakangas@i 315 : 200759 : break;
1125 nathan@postgresql.or 316 : 541 : case PqMsg_CopyDone:
317 : : /* COPY IN correctly terminated by frontend */
1998 heikki.linnakangas@i 318 : 541 : cstate->raw_reached_eof = true;
2127 319 : 541 : return bytesread;
1125 nathan@postgresql.or 320 :UBC 0 : case PqMsg_CopyFail:
2127 heikki.linnakangas@i 321 [ # # ]: 0 : ereport(ERROR,
322 : : (errcode(ERRCODE_QUERY_CANCELED),
323 : : errmsg("COPY from stdin failed: %s",
324 : : pq_getmsgstring(cstate->fe_msgbuf))));
325 : : break;
1125 nathan@postgresql.or 326 : 0 : case PqMsg_Flush:
327 : : case PqMsg_Sync:
328 : :
329 : : /*
330 : : * Ignore Flush/Sync for the convenience of client
331 : : * libraries (such as libpq) that may send those
332 : : * without noticing that the command they just
333 : : * sent was COPY.
334 : : */
2127 heikki.linnakangas@i 335 : 0 : goto readmessage;
336 : 0 : default:
1971 tgl@sss.pgh.pa.us 337 : 0 : Assert(false); /* NOT REACHED */
338 : : }
339 : : }
2127 heikki.linnakangas@i 340 :CBC 200759 : avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
341 [ - + ]: 200759 : if (avail > maxread)
2127 heikki.linnakangas@i 342 :UBC 0 : avail = maxread;
2127 heikki.linnakangas@i 343 :CBC 200759 : pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
303 peter@eisentraut.org 344 : 200759 : databuf = (char *) databuf + avail;
2127 heikki.linnakangas@i 345 : 200759 : maxread -= avail;
346 : 200759 : bytesread += avail;
347 : : }
348 : 201241 : break;
349 : 15080 : case COPY_CALLBACK:
350 : 15080 : bytesread = cstate->data_source_cb(databuf, minread, maxread);
351 : 15080 : break;
352 : : }
353 : :
354 : 217034 : return bytesread;
355 : : }
356 : :
357 : :
358 : : /*
359 : : * These functions do apply some data conversion
360 : : */
361 : :
362 : : /*
363 : : * CopyGetInt32 reads an int32 that appears in network byte order
364 : : *
365 : : * Returns true if OK, false if EOF
366 : : */
367 : : static inline bool
368 : 116 : CopyGetInt32(CopyFromState cstate, int32 *val)
369 : : {
370 : : uint32 buf;
371 : :
372 [ - + ]: 116 : if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
373 : : {
2127 heikki.linnakangas@i 374 :UBC 0 : *val = 0; /* suppress compiler warning */
375 : 0 : return false;
376 : : }
2127 heikki.linnakangas@i 377 :CBC 116 : *val = (int32) pg_ntoh32(buf);
378 : 116 : return true;
379 : : }
380 : :
381 : : /*
382 : : * CopyGetInt16 reads an int16 that appears in network byte order
383 : : */
384 : : static inline bool
385 : 25 : CopyGetInt16(CopyFromState cstate, int16 *val)
386 : : {
387 : : uint16 buf;
388 : :
389 [ - + ]: 25 : if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
390 : : {
2127 heikki.linnakangas@i 391 :UBC 0 : *val = 0; /* suppress compiler warning */
392 : 0 : return false;
393 : : }
2127 heikki.linnakangas@i 394 :CBC 25 : *val = (int16) pg_ntoh16(buf);
395 : 25 : return true;
396 : : }
397 : :
398 : :
399 : : /*
400 : : * Perform encoding conversion on data in 'raw_buf', writing the converted
401 : : * data into 'input_buf'.
402 : : *
403 : : * On entry, there must be some data to convert in 'raw_buf'.
404 : : */
405 : : static void
1998 406 : 434353 : CopyConvertBuf(CopyFromState cstate)
407 : : {
408 : : /*
409 : : * If the file and server encoding are the same, no encoding conversion is
410 : : * required. However, we still need to verify that the input is valid for
411 : : * the encoding.
412 : : */
413 [ + + ]: 434353 : if (!cstate->need_transcoding)
414 : : {
415 : : /*
416 : : * When conversion is not required, input_buf and raw_buf are the
417 : : * same. raw_buf_len is the total number of bytes in the buffer, and
418 : : * input_buf_len tracks how many of those bytes have already been
419 : : * verified.
420 : : */
421 : 434265 : int preverifiedlen = cstate->input_buf_len;
422 : 434265 : int unverifiedlen = cstate->raw_buf_len - cstate->input_buf_len;
423 : : int nverified;
424 : :
425 [ + + ]: 434265 : if (unverifiedlen == 0)
426 : : {
427 : : /*
428 : : * If no more raw data is coming, report the EOF to the caller.
429 : : */
430 [ + + ]: 218242 : if (cstate->raw_reached_eof)
431 : 1262 : cstate->input_reached_eof = true;
432 : 218242 : return;
433 : : }
434 : :
435 : : /*
436 : : * Verify the new data, including any residual unverified bytes from
437 : : * previous round.
438 : : */
439 : 216023 : nverified = pg_encoding_verifymbstr(cstate->file_encoding,
440 : 216023 : cstate->raw_buf + preverifiedlen,
441 : : unverifiedlen);
442 [ - + ]: 216023 : if (nverified == 0)
443 : : {
444 : : /*
445 : : * Could not verify anything.
446 : : *
447 : : * If there is no more raw input data coming, it means that there
448 : : * was an incomplete multi-byte sequence at the end. Also, if
449 : : * there's "enough" input left, we should be able to verify at
450 : : * least one character, and a failure to do so means that we've
451 : : * hit an invalid byte sequence.
452 : : */
1575 heikki.linnakangas@i 453 [ # # # # ]:UBC 0 : if (cstate->raw_reached_eof || unverifiedlen >= pg_encoding_max_length(cstate->file_encoding))
1998 454 : 0 : cstate->input_reached_error = true;
455 : 0 : return;
456 : : }
1998 heikki.linnakangas@i 457 :CBC 216023 : cstate->input_buf_len += nverified;
458 : : }
459 : : else
460 : : {
461 : : /*
462 : : * Encoding conversion is needed.
463 : : */
464 : : int nbytes;
465 : : unsigned char *src;
466 : : int srclen;
467 : : unsigned char *dst;
468 : : int dstlen;
469 : : int convertedlen;
470 : :
471 [ + + ]: 88 : if (RAW_BUF_BYTES(cstate) == 0)
472 : : {
473 : : /*
474 : : * If no more raw data is coming, report the EOF to the caller.
475 : : */
476 [ + + ]: 56 : if (cstate->raw_reached_eof)
477 : 16 : cstate->input_reached_eof = true;
478 : 56 : return;
479 : : }
480 : :
481 : : /*
482 : : * First, copy down any unprocessed data.
483 : : */
484 : 32 : nbytes = INPUT_BUF_BYTES(cstate);
485 [ - + - - ]: 32 : if (nbytes > 0 && cstate->input_buf_index > 0)
1998 heikki.linnakangas@i 486 :UBC 0 : memmove(cstate->input_buf, cstate->input_buf + cstate->input_buf_index,
487 : : nbytes);
1998 heikki.linnakangas@i 488 :CBC 32 : cstate->input_buf_index = 0;
489 : 32 : cstate->input_buf_len = nbytes;
490 : 32 : cstate->input_buf[nbytes] = '\0';
491 : :
492 : 32 : src = (unsigned char *) cstate->raw_buf + cstate->raw_buf_index;
493 : 32 : srclen = cstate->raw_buf_len - cstate->raw_buf_index;
494 : 32 : dst = (unsigned char *) cstate->input_buf + cstate->input_buf_len;
495 : 32 : dstlen = INPUT_BUF_SIZE - cstate->input_buf_len + 1;
496 : :
497 : : /*
498 : : * Do the conversion. This might stop short, if there is an invalid
499 : : * byte sequence in the input. We'll convert as much as we can in
500 : : * that case.
501 : : *
502 : : * Note: Even if we hit an invalid byte sequence, we don't report the
503 : : * error until all the valid bytes have been consumed. The input
504 : : * might contain an end-of-input marker (\.), and we don't want to
505 : : * report an error if the invalid byte sequence is after the
506 : : * end-of-input marker. We might unnecessarily convert some data
507 : : * after the end-of-input marker as long as it's valid for the
508 : : * encoding, but that's harmless.
509 : : */
510 : 32 : convertedlen = pg_do_encoding_conversion_buf(cstate->conversion_proc,
511 : : cstate->file_encoding,
512 : : GetDatabaseEncoding(),
513 : : src, srclen,
514 : : dst, dstlen,
515 : : true);
516 [ + + ]: 32 : if (convertedlen == 0)
517 : : {
518 : : /*
519 : : * Could not convert anything. If there is no more raw input data
520 : : * coming, it means that there was an incomplete multi-byte
521 : : * sequence at the end. Also, if there is plenty of input left,
522 : : * we should be able to convert at least one character, so a
523 : : * failure to do so must mean that we've hit a byte sequence
524 : : * that's invalid.
525 : : */
526 [ + + - + ]: 16 : if (cstate->raw_reached_eof || srclen >= MAX_CONVERSION_INPUT_LENGTH)
527 : 8 : cstate->input_reached_error = true;
528 : 16 : return;
529 : : }
530 : 16 : cstate->raw_buf_index += convertedlen;
531 : 16 : cstate->input_buf_len += strlen((char *) dst);
532 : : }
533 : : }
534 : :
535 : : /*
536 : : * Report an encoding or conversion error.
537 : : */
538 : : static void
539 : 8 : CopyConversionError(CopyFromState cstate)
540 : : {
541 [ - + ]: 8 : Assert(cstate->raw_buf_len > 0);
542 [ - + ]: 8 : Assert(cstate->input_reached_error);
543 : :
544 [ - + ]: 8 : if (!cstate->need_transcoding)
545 : : {
546 : : /*
547 : : * Everything up to input_buf_len was successfully verified, and
548 : : * input_buf_len points to the invalid or incomplete character.
549 : : */
1998 heikki.linnakangas@i 550 :UBC 0 : report_invalid_encoding(cstate->file_encoding,
551 : 0 : cstate->raw_buf + cstate->input_buf_len,
552 : 0 : cstate->raw_buf_len - cstate->input_buf_len);
553 : : }
554 : : else
555 : : {
556 : : /*
557 : : * raw_buf_index points to the invalid or untranslatable character. We
558 : : * let the conversion routine report the error, because it can provide
559 : : * a more specific error message than we could here. An earlier call
560 : : * to the conversion routine in CopyConvertBuf() detected that there
561 : : * is an error, now we call the conversion routine again with
562 : : * noError=false, to have it throw the error.
563 : : */
564 : : unsigned char *src;
565 : : int srclen;
566 : : unsigned char *dst;
567 : : int dstlen;
568 : :
1998 heikki.linnakangas@i 569 :CBC 8 : src = (unsigned char *) cstate->raw_buf + cstate->raw_buf_index;
570 : 8 : srclen = cstate->raw_buf_len - cstate->raw_buf_index;
571 : 8 : dst = (unsigned char *) cstate->input_buf + cstate->input_buf_len;
572 : 8 : dstlen = INPUT_BUF_SIZE - cstate->input_buf_len + 1;
573 : :
574 : 8 : (void) pg_do_encoding_conversion_buf(cstate->conversion_proc,
575 : : cstate->file_encoding,
576 : : GetDatabaseEncoding(),
577 : : src, srclen,
578 : : dst, dstlen,
579 : : false);
580 : :
581 : : /*
582 : : * The conversion routine should have reported an error, so this
583 : : * should not be reached.
584 : : */
1998 heikki.linnakangas@i 585 [ # # ]:UBC 0 : elog(ERROR, "encoding conversion failed without error");
586 : : }
587 : : }
588 : :
589 : : /*
590 : : * Load more data from data source to raw_buf.
591 : : *
592 : : * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the
593 : : * beginning of the buffer, and we load new data after that.
594 : : */
595 : : static void
2127 heikki.linnakangas@i 596 :CBC 217048 : CopyLoadRawBuf(CopyFromState cstate)
597 : : {
598 : : int nbytes;
599 : : int inbytes;
600 : :
601 : : /*
602 : : * In text mode, if encoding conversion is not required, raw_buf and
603 : : * input_buf point to the same buffer. Their len/index better agree, too.
604 : : */
1998 605 [ + + ]: 217048 : if (cstate->raw_buf == cstate->input_buf)
606 : : {
607 [ - + ]: 216980 : Assert(!cstate->need_transcoding);
608 [ - + ]: 216980 : Assert(cstate->raw_buf_index == cstate->input_buf_index);
609 [ - + ]: 216980 : Assert(cstate->input_buf_len <= cstate->raw_buf_len);
610 : : }
611 : :
612 : : /*
613 : : * Copy down the unprocessed data if any.
614 : : */
615 : 217048 : nbytes = RAW_BUF_BYTES(cstate);
616 [ + + + + ]: 217048 : if (nbytes > 0 && cstate->raw_buf_index > 0)
2127 617 : 607 : memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
618 : : nbytes);
1998 619 : 217048 : cstate->raw_buf_len -= cstate->raw_buf_index;
620 : 217048 : cstate->raw_buf_index = 0;
621 : :
622 : : /*
623 : : * If raw_buf and input_buf are in fact the same buffer, adjust the
624 : : * input_buf variables, too.
625 : : */
626 [ + + ]: 217048 : if (cstate->raw_buf == cstate->input_buf)
627 : : {
628 : 216980 : cstate->input_buf_len -= cstate->input_buf_index;
629 : 216980 : cstate->input_buf_index = 0;
630 : : }
631 : :
632 : : /* Load more data */
633 : 217048 : inbytes = CopyGetData(cstate, cstate->raw_buf + cstate->raw_buf_len,
634 : 217048 : 1, RAW_BUF_SIZE - cstate->raw_buf_len);
2127 635 : 217046 : nbytes += inbytes;
636 : 217046 : cstate->raw_buf[nbytes] = '\0';
637 : 217046 : cstate->raw_buf_len = nbytes;
638 : :
2054 639 : 217046 : cstate->bytes_processed += inbytes;
2083 tomas.vondra@postgre 640 : 217046 : pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
641 : :
1998 heikki.linnakangas@i 642 [ + + ]: 217046 : if (inbytes == 0)
643 : 986 : cstate->raw_reached_eof = true;
644 : 217046 : }
645 : :
646 : : /*
647 : : * CopyLoadInputBuf loads some more data into input_buf
648 : : *
649 : : * On return, at least one more input character is loaded into
650 : : * input_buf, or input_reached_eof is set.
651 : : *
652 : : * If INPUT_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
653 : : * of the buffer and then we load more data after that.
654 : : *
655 : : * If "speculative" is true, this function skips reporting any encoding or
656 : : * conversion errors, provided there are still data for the caller to process.
657 : : * It also won't read past a backslash in text mode, since that might begin an
658 : : * end-of-copy marker (in which case there's no point in waiting for more data,
659 : : * which might not materialize anyway). Such callers must be prepared for this
660 : : * function to return without loading anything.
661 : : */
662 : : static void
19 nathan@postgresql.or 663 : 217339 : CopyLoadInputBuf(CopyFromState cstate, bool speculative)
664 : : {
1998 heikki.linnakangas@i 665 : 217339 : int nbytes = INPUT_BUF_BYTES(cstate);
666 : :
667 : : /*
668 : : * If "speculative" is true and we're in text mode, refuse to wait for
669 : : * more input if there's a backslash in the buffer that the caller still
670 : : * needs to process. That might be the start of an end-of-copy marker. If
671 : : * it _is_ an end-of-copy marker, we don't need any more data, and more
672 : : * data might not show up, anyway (e.g., from a pipe that was left open).
673 : : */
11 nathan@postgresql.or 674 [ + + + + ]: 217339 : if (speculative && cstate->opts.format == COPY_FORMAT_TEXT &&
675 [ + + ]: 216485 : memchr(cstate->input_buf + cstate->input_buf_index, '\\',
676 : : nbytes) != NULL)
677 : 12 : return;
678 : :
679 : : /*
680 : : * The caller has updated input_buf_index to indicate how much of the
681 : : * input has been consumed and isn't needed anymore. If input_buf is the
682 : : * same physical area as raw_buf, update raw_buf_index accordingly.
683 : : */
1998 heikki.linnakangas@i 684 [ + + ]: 217327 : if (cstate->raw_buf == cstate->input_buf)
685 : : {
686 [ - + ]: 217287 : Assert(!cstate->need_transcoding);
687 [ - + ]: 217287 : Assert(cstate->input_buf_index >= cstate->raw_buf_index);
688 : 217287 : cstate->raw_buf_index = cstate->input_buf_index;
689 : : }
690 : :
691 : : for (;;)
692 : : {
693 : : /* If we now have some unconverted data, try to convert it */
694 : 434353 : CopyConvertBuf(cstate);
695 : :
696 : : /* If we now have some more input bytes ready, return them */
697 [ + + ]: 434353 : if (INPUT_BUF_BYTES(cstate) > nbytes)
698 : 216039 : return;
699 : :
700 : : /*
701 : : * If we reached an invalid byte sequence, or we're at an incomplete
702 : : * multi-byte character but there is no more raw input data, report
703 : : * conversion error. As an exception, if "speculative" is true and
704 : : * there are still data for the caller to process, just return
705 : : * instead.
706 : : */
707 [ + + ]: 218314 : if (cstate->input_reached_error)
708 : : {
19 nathan@postgresql.or 709 [ + - - + ]: 8 : if (speculative && INPUT_BUF_BYTES(cstate) > 0)
19 nathan@postgresql.or 710 :UBC 0 : return;
1998 heikki.linnakangas@i 711 :CBC 8 : CopyConversionError(cstate);
712 : : }
713 : :
714 : : /* no more input, and everything has been converted */
715 [ + + ]: 218306 : if (cstate->input_reached_eof)
716 : 1278 : break;
717 : :
718 : : /* Try to load more raw data */
719 [ - + ]: 217028 : Assert(!cstate->raw_reached_eof);
720 : 217028 : CopyLoadRawBuf(cstate);
721 : : }
722 : : }
723 : :
724 : : /*
725 : : * CopyReadBinaryData
726 : : *
727 : : * Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf
728 : : * and writes them to 'dest'. Returns the number of bytes read (which
729 : : * would be less than 'nbytes' only if we reach EOF).
730 : : */
731 : : static int
2127 732 : 236 : CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
733 : : {
734 : 236 : int copied_bytes = 0;
735 : :
736 [ + + ]: 236 : if (RAW_BUF_BYTES(cstate) >= nbytes)
737 : : {
738 : : /* Enough bytes are present in the buffer. */
739 : 216 : memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, nbytes);
740 : 216 : cstate->raw_buf_index += nbytes;
741 : 216 : copied_bytes = nbytes;
742 : : }
743 : : else
744 : : {
745 : : /*
746 : : * Not enough bytes in the buffer, so must read from the file. Need
747 : : * to loop since 'nbytes' could be larger than the buffer size.
748 : : */
749 : : do
750 : : {
751 : : int copy_bytes;
752 : :
753 : : /* Load more data if buffer is empty. */
754 [ + - ]: 20 : if (RAW_BUF_BYTES(cstate) == 0)
755 : : {
1998 756 : 20 : CopyLoadRawBuf(cstate);
757 [ + + ]: 20 : if (cstate->raw_reached_eof)
2127 758 : 7 : break; /* EOF */
759 : : }
760 : :
761 : : /* Transfer some bytes. */
762 : 13 : copy_bytes = Min(nbytes - copied_bytes, RAW_BUF_BYTES(cstate));
763 : 13 : memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, copy_bytes);
764 : 13 : cstate->raw_buf_index += copy_bytes;
765 : 13 : dest += copy_bytes;
766 : 13 : copied_bytes += copy_bytes;
767 [ - + ]: 13 : } while (copied_bytes < nbytes);
768 : : }
769 : :
770 : 236 : return copied_bytes;
771 : : }
772 : :
773 : : /*
774 : : * This function is exposed for use by extensions that read raw fields in the
775 : : * next line. See NextCopyFromRawFieldsInternal() for details.
776 : : */
777 : : bool
569 msawada@postgresql.o 778 :UBC 0 : NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields)
779 : : {
780 : 0 : return NextCopyFromRawFieldsInternal(cstate, fields, nfields,
188 andrew@dunslane.net 781 : 0 : cstate->opts.format == COPY_FORMAT_CSV);
782 : : }
783 : :
784 : : /*
785 : : * Workhorse for NextCopyFromRawFields().
786 : : *
787 : : * Read raw fields in the next line for COPY FROM in text or csv mode. Return
788 : : * false if no more lines.
789 : : *
790 : : * An internal temporary buffer is returned via 'fields'. It is valid until
791 : : * the next call of the function. Since the function returns all raw fields
792 : : * in the input file, 'nfields' could be different from the number of columns
793 : : * in the relation.
794 : : *
795 : : * NOTE: force_not_null option are not applied to the returned fields.
796 : : *
797 : : * We use pg_always_inline to reduce function call overhead
798 : : * and to help compilers to optimize away the 'is_csv' condition when called
799 : : * by internal functions such as CopyFromTextLikeOneRow().
800 : : */
801 : : static pg_always_inline bool
569 msawada@postgresql.o 802 :CBC 755027 : NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields, bool is_csv)
803 : : {
804 : : int fldct;
444 fujii@postgresql.org 805 : 755027 : bool done = false;
806 : :
807 : : /* only available for text or csv input */
188 andrew@dunslane.net 808 [ + + - + ]: 755027 : Assert(cstate->opts.format == COPY_FORMAT_TEXT ||
809 : : cstate->opts.format == COPY_FORMAT_CSV);
810 : :
811 : : /* on input check that the header line is correct if needed */
444 fujii@postgresql.org 812 [ + + + + ]: 755027 : if (cstate->cur_lineno == 0 && cstate->opts.header_line != COPY_HEADER_FALSE)
813 : : {
814 : : ListCell *cur;
815 : : TupleDesc tupDesc;
816 : 93 : int lines_to_skip = cstate->opts.header_line;
817 : :
818 : : /* If set to "match", one header line is skipped */
819 [ + + ]: 93 : if (cstate->opts.header_line == COPY_HEADER_MATCH)
820 : 50 : lines_to_skip = 1;
821 : :
1635 peter@eisentraut.org 822 : 93 : tupDesc = RelationGetDescr(cstate->rel);
823 : :
444 fujii@postgresql.org 824 [ + + ]: 218 : for (int i = 0; i < lines_to_skip; i++)
825 : : {
826 : 130 : cstate->cur_lineno++;
827 [ + + ]: 130 : if ((done = CopyReadLine(cstate, is_csv)))
828 : 5 : break;
829 : : }
830 : :
1635 peter@eisentraut.org 831 [ + + ]: 93 : if (cstate->opts.header_line == COPY_HEADER_MATCH)
832 : : {
833 : : int fldnum;
834 : :
569 msawada@postgresql.o 835 [ + + ]: 50 : if (is_csv)
949 michael@paquier.xyz 836 : 6 : fldct = CopyReadAttributesCSV(cstate);
837 : : else
838 : 44 : fldct = CopyReadAttributesText(cstate);
839 : :
1635 peter@eisentraut.org 840 [ + + ]: 50 : if (fldct != list_length(cstate->attnumlist))
841 [ + - ]: 16 : ereport(ERROR,
842 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
843 : : errmsg("wrong number of fields in header line: got %d, expected %d",
844 : : fldct, list_length(cstate->attnumlist))));
845 : :
846 : 34 : fldnum = 0;
847 [ + - + + : 104 : foreach(cur, cstate->attnumlist)
+ + ]
848 : : {
849 : 83 : int attnum = lfirst_int(cur);
850 : : char *colName;
851 : 83 : Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
852 : :
1550 michael@paquier.xyz 853 [ - + ]: 83 : Assert(fldnum < cstate->max_fields);
854 : :
855 : 83 : colName = cstate->raw_fields[fldnum++];
1635 peter@eisentraut.org 856 [ + + ]: 83 : if (colName == NULL)
857 [ + - ]: 4 : ereport(ERROR,
858 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
859 : : errmsg("column name mismatch in header line field %d: got null value (\"%s\"), expected \"%s\"",
860 : : fldnum, cstate->opts.null_print, NameStr(attr->attname))));
861 : :
1592 tgl@sss.pgh.pa.us 862 [ + + ]: 79 : if (namestrcmp(&attr->attname, colName) != 0)
863 : : {
1635 peter@eisentraut.org 864 [ + - ]: 9 : ereport(ERROR,
865 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
866 : : errmsg("column name mismatch in header line field %d: got \"%s\", expected \"%s\"",
867 : : fldnum, colName, NameStr(attr->attname))));
868 : : }
869 : : }
870 : : }
871 : :
872 [ + + ]: 64 : if (done)
873 : 5 : return false;
874 : : }
875 : :
2127 heikki.linnakangas@i 876 : 754993 : cstate->cur_lineno++;
877 : :
878 : : /* Actually read the line into memory here */
569 msawada@postgresql.o 879 : 754993 : done = CopyReadLine(cstate, is_csv);
880 : :
881 : : /*
882 : : * EOF at start of line means we're done. If we see EOF after some
883 : : * characters, we act as though it was newline followed by EOF, ie,
884 : : * process the line and then exit loop on next iteration.
885 : : */
2127 heikki.linnakangas@i 886 [ + + + - ]: 754975 : if (done && cstate->line_buf.len == 0)
887 : 996 : return false;
888 : :
889 : : /* Parse the line into de-escaped field values */
569 msawada@postgresql.o 890 [ + + ]: 753979 : if (is_csv)
949 michael@paquier.xyz 891 : 312 : fldct = CopyReadAttributesCSV(cstate);
892 : : else
893 : 753667 : fldct = CopyReadAttributesText(cstate);
894 : :
2127 heikki.linnakangas@i 895 : 753971 : *fields = cstate->raw_fields;
896 : 753971 : *nfields = fldct;
897 : 753971 : return true;
898 : : }
899 : :
900 : : /*
901 : : * Read next tuple from file for COPY FROM. Return false if no more tuples.
902 : : *
903 : : * 'econtext' is used to evaluate default expression for each column that is
904 : : * either not read from the file or is using the DEFAULT option of COPY FROM.
905 : : * It can be NULL when no default values are used, i.e. when all columns are
906 : : * read from the file, and DEFAULT option is unset.
907 : : *
908 : : * 'values' and 'nulls' arrays must be the same length as columns of the
909 : : * relation passed to BeginCopyFrom. This function fills the arrays.
910 : : */
911 : : bool
912 : 755052 : NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
913 : : Datum *values, bool *nulls)
914 : : {
915 : : TupleDesc tupDesc;
916 : : AttrNumber num_phys_attrs,
917 : 755052 : num_defaults = cstate->num_defaults;
918 : : int i;
919 : 755052 : int *defmap = cstate->defmap;
920 : 755052 : ExprState **defexprs = cstate->defexprs;
921 : :
922 : 755052 : tupDesc = RelationGetDescr(cstate->rel);
923 : 755052 : num_phys_attrs = tupDesc->natts;
924 : :
925 : : /* Initialize all values for row to NULL */
926 [ + - + - : 3535379 : MemSet(values, 0, num_phys_attrs * sizeof(Datum));
+ - + - +
+ ]
927 [ + - + + : 755052 : MemSet(nulls, true, num_phys_attrs * sizeof(bool));
- + - - -
- ]
1151 drowley@postgresql.o 928 [ + - + + : 851124 : MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool));
+ - + - +
+ ]
929 : :
930 : : /* Get one row from source */
569 msawada@postgresql.o 931 [ + + ]: 755052 : if (!cstate->routine->CopyFromOneRow(cstate, econtext, values, nulls))
932 : 1008 : return false;
933 : :
934 : : /*
935 : : * Now compute and insert any defaults available for the columns not
936 : : * provided by the input data. Anything not processed here or above will
937 : : * remain NULL.
938 : : */
939 [ + + ]: 794260 : for (i = 0; i < num_defaults; i++)
940 : : {
941 : : /*
942 : : * The caller must supply econtext and have switched into the
943 : : * per-tuple memory context in it.
944 : : */
945 [ - + ]: 40345 : Assert(econtext != NULL);
946 [ - + ]: 40345 : Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
947 : :
948 : 40345 : values[defmap[i]] = ExecEvalExpr(defexprs[defmap[i]], econtext,
949 : 40345 : &nulls[defmap[i]]);
950 : : }
951 : :
952 : 753915 : return true;
953 : : }
954 : :
955 : : /* Implementation of the per-row callback for text format */
956 : : bool
957 : 754557 : CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
958 : : bool *nulls)
959 : : {
960 : 754557 : return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, false);
961 : : }
962 : :
963 : : /* Implementation of the per-row callback for CSV format */
964 : : bool
965 : 470 : CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
966 : : bool *nulls)
967 : : {
968 : 470 : return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, true);
969 : : }
970 : :
971 : : /*
972 : : * Workhorse for CopyFromTextOneRow() and CopyFromCSVOneRow().
973 : : *
974 : : * We use pg_always_inline to reduce function call overhead
975 : : * and to help compilers to optimize away the 'is_csv' condition.
976 : : */
977 : : static pg_always_inline bool
978 : 755027 : CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
979 : : Datum *values, bool *nulls, bool is_csv)
980 : : {
981 : : TupleDesc tupDesc;
982 : : AttrNumber attr_count;
983 : 755027 : FmgrInfo *in_functions = cstate->in_functions;
984 : 755027 : Oid *typioparams = cstate->typioparams;
985 : 755027 : ExprState **defexprs = cstate->defexprs;
986 : : char **field_strings;
987 : : ListCell *cur;
988 : : int fldct;
989 : : int fieldno;
990 : : char *string;
201 peter@eisentraut.org 991 : 755027 : bool current_row_erroneous = false;
992 : :
569 msawada@postgresql.o 993 : 755027 : tupDesc = RelationGetDescr(cstate->rel);
994 : 755027 : attr_count = list_length(cstate->attnumlist);
995 : :
996 : : /* read raw fields in the next line */
997 [ + + ]: 755027 : if (!NextCopyFromRawFieldsInternal(cstate, &field_strings, &fldct, is_csv))
998 : 1001 : return false;
999 : :
1000 : : /* check for overflowing fields */
1001 [ + + + + ]: 753971 : if (attr_count > 0 && fldct > attr_count)
1002 [ + - ]: 16 : ereport(ERROR,
1003 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1004 : : errmsg("extra data after last expected column")));
1005 : :
1006 : 753955 : fieldno = 0;
1007 : :
1008 : : /* Loop to read the user attributes on the line. */
1009 [ + + + + : 3447638 : foreach(cur, cstate->attnumlist)
+ + ]
1010 : : {
1011 : 2693822 : int attnum = lfirst_int(cur);
1012 : 2693822 : int m = attnum - 1;
1013 : 2693822 : Form_pg_attribute att = TupleDescAttr(tupDesc, m);
1014 : :
1015 [ + + ]: 2693822 : if (fieldno >= fldct)
2127 heikki.linnakangas@i 1016 [ + - ]: 16 : ereport(ERROR,
1017 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1018 : : errmsg("missing data for column \"%s\"",
1019 : : NameStr(att->attname))));
569 msawada@postgresql.o 1020 : 2693806 : string = field_strings[fieldno++];
1021 : :
1022 [ + + ]: 2693806 : if (cstate->convert_select_flags &&
1023 [ + + ]: 10 : !cstate->convert_select_flags[m])
1024 : : {
1025 : : /* ignore input field, leaving column as NULL */
1026 : 5 : continue;
1027 : : }
1028 : :
1029 [ + + ]: 2693801 : if (is_csv)
1030 : : {
1031 [ + + ]: 619 : if (string == NULL &&
1032 [ + + ]: 27 : cstate->opts.force_notnull_flags[m])
1033 : : {
1034 : : /*
1035 : : * FORCE_NOT_NULL option is set and column is NULL - convert
1036 : : * it to the NULL string.
1037 : : */
1038 : 18 : string = cstate->opts.null_print;
1039 : : }
1040 [ + + + + ]: 601 : else if (string != NULL && cstate->opts.force_null_flags[m]
1041 [ + + ]: 32 : && strcmp(string, cstate->opts.null_print) == 0)
1042 : : {
1043 : : /*
1044 : : * FORCE_NULL option is set and column matches the NULL
1045 : : * string. It must have been quoted, or otherwise the string
1046 : : * would already have been set to NULL. Convert it to NULL as
1047 : : * specified.
1048 : : */
1049 : 17 : string = NULL;
1050 : : }
1051 : : }
1052 : :
1053 : 2693801 : cstate->cur_attname = NameStr(att->attname);
1054 : 2693801 : cstate->cur_attval = string;
1055 : :
1056 [ + + ]: 2693801 : if (string != NULL)
1057 : 2691045 : nulls[m] = false;
1058 : :
1059 [ + + ]: 2693801 : if (cstate->defaults[m])
1060 : : {
1061 : : /* We must have switched into the per-tuple memory context */
1062 [ - + ]: 38 : Assert(econtext != NULL);
1063 [ - + ]: 38 : Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
1064 : :
1065 : 38 : values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]);
1066 : : }
1067 : :
1068 : : /*
1069 : : * If ON_ERROR is specified, handle the different options
1070 : : */
1071 [ + + ]: 2693738 : else if (!InputFunctionCallSafe(&in_functions[m],
1072 : : string,
1073 : 2693763 : typioparams[m],
1074 : : att->atttypmod,
1075 : 2693763 : (Node *) cstate->escontext,
1076 : 2693763 : &values[m]))
1077 : : {
1078 [ - + ]: 116 : Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP);
1079 : :
201 peter@eisentraut.org 1080 [ + + ]: 116 : if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE)
1081 : 82 : cstate->num_errors++;
1082 [ + - ]: 34 : else if (cstate->opts.on_error == COPY_ON_ERROR_SET_NULL)
1083 : : {
1084 : : /*
1085 : : * Reset error state so the subsequent InputFunctionCallSafe
1086 : : * call (for domain constraint check) can properly report
1087 : : * whether it succeeded or failed.
1088 : : */
1089 : 34 : cstate->escontext->error_occurred = false;
1090 : :
1091 [ - + ]: 34 : Assert(cstate->domain_with_constraint != NULL);
1092 : :
1093 : : /*
1094 : : * For constrained domains, we need an additional
1095 : : * InputFunctionCallSafe() to ensure that an error is thrown
1096 : : * if the domain constraint rejects null values.
1097 : : */
1098 [ + + + + ]: 58 : if (!cstate->domain_with_constraint[m] ||
1099 : 24 : InputFunctionCallSafe(&in_functions[m],
1100 : : NULL,
1101 : 24 : typioparams[m],
1102 : : att->atttypmod,
1103 : 24 : (Node *) cstate->escontext,
1104 : 24 : &values[m]))
1105 : : {
1106 : 18 : nulls[m] = true;
1107 : 18 : values[m] = (Datum) 0;
1108 : : }
1109 : : else
1110 [ + - ]: 16 : ereport(ERROR,
1111 : : errcode(ERRCODE_NOT_NULL_VIOLATION),
1112 : : errmsg("domain %s does not allow null values",
1113 : : format_type_be(typioparams[m])),
1114 : : errdetail("ON_ERROR SET_NULL cannot be applied because column \"%s\" (domain %s) does not accept null values.",
1115 : : cstate->cur_attname,
1116 : : format_type_be(typioparams[m])),
1117 : : errdatatype(typioparams[m]));
1118 : :
1119 : : /*
1120 : : * We count only the number of rows (not fields) where
1121 : : * ON_ERROR SET_NULL was applied.
1122 : : */
1123 [ + + ]: 18 : if (!current_row_erroneous)
1124 : : {
1125 : 14 : current_row_erroneous = true;
1126 : 14 : cstate->num_errors++;
1127 : : }
1128 : : }
1129 : :
569 msawada@postgresql.o 1130 [ + + ]: 100 : if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE)
1131 : : {
1132 : : /*
1133 : : * Since we emit line number and column info in the below
1134 : : * notice message, we suppress error context information other
1135 : : * than the relation name.
1136 : : */
1137 [ - + ]: 44 : Assert(!cstate->relname_only);
1138 : 44 : cstate->relname_only = true;
1139 : :
1140 [ + + ]: 44 : if (cstate->cur_attval)
1141 : : {
1142 : : char *attval;
1143 : :
1144 : 40 : attval = CopyLimitPrintoutLength(cstate->cur_attval);
1145 : :
201 peter@eisentraut.org 1146 [ + + ]: 40 : if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE)
1147 [ + - ]: 24 : ereport(NOTICE,
1148 : : errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"",
1149 : : cstate->cur_lineno,
1150 : : cstate->cur_attname,
1151 : : attval));
1152 [ + - ]: 16 : else if (cstate->opts.on_error == COPY_ON_ERROR_SET_NULL)
1153 [ + - ]: 16 : ereport(NOTICE,
1154 : : errmsg("setting to null due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"",
1155 : : cstate->cur_lineno,
1156 : : cstate->cur_attname,
1157 : : attval));
569 msawada@postgresql.o 1158 : 40 : pfree(attval);
1159 : : }
1160 : : else
1161 : : {
201 peter@eisentraut.org 1162 [ + - ]: 4 : if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE)
1163 [ + - ]: 4 : ereport(NOTICE,
1164 : : errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": null input",
1165 : : cstate->cur_lineno,
1166 : : cstate->cur_attname));
1167 : : }
1168 : : /* reset relname_only */
569 msawada@postgresql.o 1169 : 44 : cstate->relname_only = false;
1170 : : }
1171 : :
201 peter@eisentraut.org 1172 [ + + ]: 100 : if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE)
1173 : 82 : return true;
1174 [ + - ]: 18 : else if (cstate->opts.on_error == COPY_ON_ERROR_SET_NULL)
1175 : 18 : continue;
1176 : : }
1177 : :
569 msawada@postgresql.o 1178 : 2693660 : cstate->cur_attname = NULL;
1179 : 2693660 : cstate->cur_attval = NULL;
1180 : : }
1181 : :
1182 [ - + ]: 753816 : Assert(fieldno == attr_count);
1183 : :
1184 : 753816 : return true;
1185 : : }
1186 : :
1187 : : /* Implementation of the per-row callback for binary format */
1188 : : bool
1189 : 25 : CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
1190 : : bool *nulls)
1191 : : {
1192 : : TupleDesc tupDesc;
1193 : : AttrNumber attr_count;
1194 : 25 : FmgrInfo *in_functions = cstate->in_functions;
1195 : 25 : Oid *typioparams = cstate->typioparams;
1196 : : int16 fld_count;
1197 : : ListCell *cur;
1198 : :
1199 : 25 : tupDesc = RelationGetDescr(cstate->rel);
1200 : 25 : attr_count = list_length(cstate->attnumlist);
1201 : :
1202 : 25 : cstate->cur_lineno++;
1203 : :
1204 [ - + ]: 25 : if (!CopyGetInt16(cstate, &fld_count))
1205 : : {
1206 : : /* EOF detected (end of file, or protocol-level EOF) */
569 msawada@postgresql.o 1207 :UBC 0 : return false;
1208 : : }
1209 : :
569 msawada@postgresql.o 1210 [ + + ]:CBC 25 : if (fld_count == -1)
1211 : : {
1212 : : /*
1213 : : * Received EOF marker. Wait for the protocol-level EOF, and complain
1214 : : * if it doesn't come immediately. In COPY FROM STDIN, this ensures
1215 : : * that we correctly handle CopyFail, if client chooses to send that
1216 : : * now. When copying from file, we could ignore the rest of the file
1217 : : * like in text mode, but we choose to be consistent with the COPY
1218 : : * FROM STDIN case.
1219 : : */
1220 : : char dummy;
1221 : :
1222 [ - + ]: 7 : if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
569 msawada@postgresql.o 1223 [ # # ]:UBC 0 : ereport(ERROR,
1224 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1225 : : errmsg("received copy data after EOF marker")));
569 msawada@postgresql.o 1226 :CBC 7 : return false;
1227 : : }
1228 : :
1229 [ - + ]: 18 : if (fld_count != attr_count)
569 msawada@postgresql.o 1230 [ # # ]:UBC 0 : ereport(ERROR,
1231 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1232 : : errmsg("row field count is %d, expected %d",
1233 : : fld_count, attr_count)));
1234 : :
569 msawada@postgresql.o 1235 [ + - + + :CBC 117 : foreach(cur, cstate->attnumlist)
+ + ]
1236 : : {
1237 : 100 : int attnum = lfirst_int(cur);
1238 : 100 : int m = attnum - 1;
1239 : 100 : Form_pg_attribute att = TupleDescAttr(tupDesc, m);
1240 : :
1241 : 100 : cstate->cur_attname = NameStr(att->attname);
1242 : 199 : values[m] = CopyReadBinaryAttribute(cstate,
1243 : 100 : &in_functions[m],
1244 : 100 : typioparams[m],
1245 : : att->atttypmod,
1246 : : &nulls[m]);
1247 : 99 : cstate->cur_attname = NULL;
1248 : : }
1249 : :
2127 heikki.linnakangas@i 1250 : 17 : return true;
1251 : : }
1252 : :
1253 : : /*
1254 : : * Read the next input line and stash it in line_buf.
1255 : : *
1256 : : * Result is true if read was terminated by EOF, false if terminated
1257 : : * by newline. The terminating newline or EOF marker is not included
1258 : : * in the final value of line_buf.
1259 : : */
1260 : : static bool
569 msawada@postgresql.o 1261 : 755123 : CopyReadLine(CopyFromState cstate, bool is_csv)
1262 : : {
1263 : : bool result;
1264 : :
2127 heikki.linnakangas@i 1265 : 755123 : resetStringInfo(&cstate->line_buf);
1998 1266 : 755123 : cstate->line_buf_valid = false;
1267 : :
1268 : : /*
1269 : : * Parse data and transfer into line_buf.
1270 : : *
1271 : : * Because this is performance critical, we inline CopyReadLineText() and
1272 : : * pass the boolean parameters as constants to allow the compiler to emit
1273 : : * specialized code with fewer branches.
1274 : : */
212 nathan@postgresql.or 1275 [ + + ]: 755123 : if (is_csv)
1276 : 546 : result = CopyReadLineText(cstate, true);
1277 : : else
1278 : 754577 : result = CopyReadLineText(cstate, false);
1279 : :
2127 heikki.linnakangas@i 1280 [ + + ]: 755105 : if (result)
1281 : : {
1282 : : /*
1283 : : * Reached EOF. In protocol version 3, we should ignore anything
1284 : : * after \. up to the protocol end of copy data. (XXX maybe better
1285 : : * not to treat \. as special?)
1286 : : */
2026 1287 [ + + ]: 1001 : if (cstate->copy_src == COPY_FRONTEND)
1288 : : {
1289 : : int inbytes;
1290 : :
1291 : : do
1292 : : {
1998 1293 : 529 : inbytes = CopyGetData(cstate, cstate->input_buf,
1294 : : 1, INPUT_BUF_SIZE);
1295 [ - + ]: 529 : } while (inbytes > 0);
1296 : 529 : cstate->input_buf_index = 0;
1297 : 529 : cstate->input_buf_len = 0;
1298 : 529 : cstate->raw_buf_index = 0;
1299 : 529 : cstate->raw_buf_len = 0;
1300 : : }
1301 : : }
1302 : : else
1303 : : {
1304 : : /*
1305 : : * If we didn't hit EOF, then we must have transferred the EOL marker
1306 : : * to line_buf along with the data. Get rid of it.
1307 : : */
2127 1308 [ + - - - : 754104 : switch (cstate->eol_type)
- ]
1309 : : {
1310 : 754104 : case EOL_NL:
1311 [ - + ]: 754104 : Assert(cstate->line_buf.len >= 1);
1312 [ - + ]: 754104 : Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
1313 : 754104 : cstate->line_buf.len--;
1314 : 754104 : cstate->line_buf.data[cstate->line_buf.len] = '\0';
1315 : 754104 : break;
2127 heikki.linnakangas@i 1316 :UBC 0 : case EOL_CR:
1317 [ # # ]: 0 : Assert(cstate->line_buf.len >= 1);
1318 [ # # ]: 0 : Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
1319 : 0 : cstate->line_buf.len--;
1320 : 0 : cstate->line_buf.data[cstate->line_buf.len] = '\0';
1321 : 0 : break;
1322 : 0 : case EOL_CRNL:
1323 [ # # ]: 0 : Assert(cstate->line_buf.len >= 2);
1324 [ # # ]: 0 : Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
1325 [ # # ]: 0 : Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
1326 : 0 : cstate->line_buf.len -= 2;
1327 : 0 : cstate->line_buf.data[cstate->line_buf.len] = '\0';
1328 : 0 : break;
1329 : 0 : case EOL_UNKNOWN:
1330 : : /* shouldn't get here */
1331 : 0 : Assert(false);
1332 : : break;
1333 : : }
1334 : : }
1335 : :
1336 : : /* Now it's safe to use the buffer in error messages */
1998 heikki.linnakangas@i 1337 :CBC 755105 : cstate->line_buf_valid = true;
1338 : :
2127 1339 : 755105 : return result;
1340 : : }
1341 : :
1342 : : #ifndef USE_NO_SIMD
1343 : : /*
1344 : : * Helper function for CopyReadLineText() that uses SIMD instructions to scan
1345 : : * the input buffer for special characters. This can be much faster.
1346 : : *
1347 : : * Note that we disable SIMD for the remainder of the COPY FROM command upon
1348 : : * encountering a special character (except for end-of-line characters) or a
1349 : : * short line. This is perhaps too conservative, but it should help avoid
1350 : : * regressions. It could probably be made more lenient in the future via
1351 : : * fine-tuned heuristics.
1352 : : */
1353 : : static bool
191 nathan@postgresql.or 1354 : 332573 : CopyReadLineTextSIMDHelper(CopyFromState cstate, bool is_csv,
1355 : : bool *hit_eof_p, int *input_buf_ptr_p)
1356 : : {
1357 : : char *copy_input_buf;
1358 : : int input_buf_ptr;
1359 : : int copy_buf_len;
1360 : : bool unique_esc_char; /* for csv, do quote/esc chars differ? */
1361 : 332573 : bool first = true;
1362 : 332573 : bool result = false;
1363 : 332573 : const Vector8 nl_vec = vector8_broadcast('\n');
1364 : 332573 : const Vector8 cr_vec = vector8_broadcast('\r');
1365 : : Vector8 bs_or_quote_vec; /* '\' for text, quote for csv */
1366 : : Vector8 esc_vec; /* only for csv */
1367 : :
1368 [ + + ]: 332573 : if (is_csv)
1369 : : {
1370 : 392 : char quote = cstate->opts.quote[0];
1371 : 392 : char esc = cstate->opts.escape[0];
1372 : :
1373 : 392 : bs_or_quote_vec = vector8_broadcast(quote);
1374 : 392 : esc_vec = vector8_broadcast(esc);
1375 : 392 : unique_esc_char = (quote != esc);
1376 : : }
1377 : : else
1378 : : {
1379 : 332181 : bs_or_quote_vec = vector8_broadcast('\\');
1380 : 332181 : unique_esc_char = false;
1381 : : }
1382 : :
1383 : : /*
1384 : : * For a little extra speed within the loop, we copy some state members
1385 : : * into local variables. Note that we need to use a separate local
1386 : : * variable for input_buf_ptr so that the REFILL_LINEBUF macro works. We
1387 : : * copy its value into the input_buf_ptr_p argument before returning.
1388 : : */
1389 : 332573 : copy_input_buf = cstate->input_buf;
1390 : 332573 : input_buf_ptr = cstate->input_buf_index;
1391 : 332573 : copy_buf_len = cstate->input_buf_len;
1392 : :
1393 : : /*
1394 : : * See the corresponding loop in CopyReadLineText() for more information
1395 : : * about the purpose of this loop. This one does the same thing using
1396 : : * SIMD instructions, although we are quick to bail out to the scalar path
1397 : : * if we encounter a special character.
1398 : : */
1399 : : for (;;)
1400 : 391669 : {
1401 : : Vector8 chunk;
1402 : : Vector8 match;
1403 : :
1404 : : /* Load more data if needed. */
1405 [ + + ]: 724242 : if (copy_buf_len - input_buf_ptr < sizeof(Vector8))
1406 : : {
1407 [ + + ]: 216853 : REFILL_LINEBUF;
1408 : :
19 1409 : 216853 : CopyLoadInputBuf(cstate, true);
1410 : : /* update our local variables */
191 1411 : 216843 : *hit_eof_p = cstate->input_reached_eof;
1412 : 216843 : input_buf_ptr = cstate->input_buf_index;
1413 : 216843 : copy_buf_len = cstate->input_buf_len;
1414 : :
1415 : : /*
1416 : : * If we are completely out of data, break out of the loop,
1417 : : * reporting EOF.
1418 : : */
1419 [ + + ]: 216843 : if (INPUT_BUF_BYTES(cstate) <= 0)
1420 : : {
1421 : 602 : result = true;
1422 : 602 : break;
1423 : : }
1424 : : }
1425 : :
1426 : : /*
1427 : : * If we still don't have enough data for the SIMD path, fall back to
1428 : : * the scalar code. Note that this doesn't necessarily mean we
1429 : : * encountered a short line, so we leave cstate->simd_enabled set to
1430 : : * true.
1431 : : */
1432 [ + + ]: 723630 : if (copy_buf_len - input_buf_ptr < sizeof(Vector8))
1433 : 215353 : break;
1434 : :
1435 : : /*
1436 : : * If we made it here, we have at least enough data to fit in a
1437 : : * Vector8, so we can use SIMD instructions to scan for special
1438 : : * characters.
1439 : : */
1440 : 508277 : vector8_load(&chunk, (const uint8 *) ©_input_buf[input_buf_ptr]);
1441 : :
1442 : : /*
1443 : : * Check for \n, \r, \\ (for text), quotes (for csv), and escapes (for
1444 : : * csv, if different from quotes).
1445 : : */
1446 : 508277 : match = vector8_eq(chunk, nl_vec);
1447 : 508277 : match = vector8_or(match, vector8_eq(chunk, cr_vec));
1448 : 508277 : match = vector8_or(match, vector8_eq(chunk, bs_or_quote_vec));
1449 [ + + ]: 508277 : if (unique_esc_char)
1450 : 21 : match = vector8_or(match, vector8_eq(chunk, esc_vec));
1451 : :
1452 : : /*
1453 : : * If we found a special character, advance to it and hand off to the
1454 : : * scalar path. Except for end-of-line characters, we also disable
1455 : : * SIMD processing for the remainder of the COPY FROM command.
1456 : : */
1457 [ + + ]: 508277 : if (vector8_is_highbit_set(match))
1458 : : {
1459 : : uint32 mask;
1460 : : char c;
1461 : :
1462 : 116608 : mask = vector8_highbit_mask(match);
1463 : 116608 : input_buf_ptr += pg_rightmost_one_pos32(mask);
1464 : :
1465 : : /*
1466 : : * Don't disable SIMD if we found \n or \r, else we'd stop using
1467 : : * SIMD instructions after the first line. As an exception, we do
1468 : : * disable it if this is the first vector we processed, as that
1469 : : * means the line is too short for SIMD.
1470 : : */
1471 : 116608 : c = copy_input_buf[input_buf_ptr];
1472 [ + + + + : 116608 : if (first || (c != '\n' && c != '\r'))
+ - ]
1473 : 389 : cstate->simd_enabled = false;
1474 : :
1475 : 116608 : break;
1476 : : }
1477 : :
1478 : : /* That chunk was clear of special characters, so we can skip it. */
1479 : 391669 : input_buf_ptr += sizeof(Vector8);
1480 : 391669 : first = false;
1481 : : }
1482 : :
1483 : 332563 : *input_buf_ptr_p = input_buf_ptr;
1484 : 332563 : return result;
1485 : : }
1486 : : #endif /* ! USE_NO_SIMD */
1487 : :
1488 : : /*
1489 : : * CopyReadLineText - inner loop of CopyReadLine for text mode
1490 : : */
1491 : : static pg_always_inline bool
569 msawada@postgresql.o 1492 : 755123 : CopyReadLineText(CopyFromState cstate, bool is_csv)
1493 : : {
1494 : : char *copy_input_buf;
1495 : : int input_buf_ptr;
1496 : : int copy_buf_len;
2127 heikki.linnakangas@i 1497 : 755123 : bool need_data = false;
1498 : 755123 : bool hit_eof = false;
1499 : 755123 : bool result = false;
1500 : :
1501 : : /* CSV variables */
1502 : 755123 : bool in_quote = false,
1503 : 755123 : last_was_esc = false;
1504 : 755123 : char quotec = '\0';
1505 : 755123 : char escapec = '\0';
1506 : :
569 msawada@postgresql.o 1507 [ + + ]: 755123 : if (is_csv)
1508 : : {
2127 heikki.linnakangas@i 1509 : 546 : quotec = cstate->opts.quote[0];
1510 : 546 : escapec = cstate->opts.escape[0];
1511 : : /* ignore special escape processing if it's the same as quotec */
1512 [ + + ]: 546 : if (quotec == escapec)
1513 : 438 : escapec = '\0';
1514 : : }
1515 : :
1516 : : /*
1517 : : * The objective of this loop is to transfer the entire next input line
1518 : : * into line_buf. Hence, we only care for detecting newlines (\r and/or
1519 : : * \n) and the end-of-copy marker (\.).
1520 : : *
1521 : : * In CSV mode, \r and \n inside a quoted field are just part of the data
1522 : : * value and are put in line_buf. We keep just enough state to know if we
1523 : : * are currently in a quoted field or not.
1524 : : *
1525 : : * The input has already been converted to the database encoding. All
1526 : : * supported server encodings have the property that all bytes in a
1527 : : * multi-byte sequence have the high bit set, so a multibyte character
1528 : : * cannot contain any newline or escape characters embedded in the
1529 : : * multibyte sequence. Therefore, we can process the input byte-by-byte,
1530 : : * regardless of the encoding.
1531 : : *
1532 : : * For speed, we try to move data from input_buf to line_buf in chunks
1533 : : * rather than one character at a time. input_buf_ptr points to the next
1534 : : * character to examine; any characters from input_buf_index to
1535 : : * input_buf_ptr have been determined to be part of the line, but not yet
1536 : : * transferred to line_buf.
1537 : : *
1538 : : * For a little extra speed within the loop, we copy some state
1539 : : * information into local variables. input_buf_ptr could be changed in
1540 : : * the SIMD path, so we must set that one before it. The others are set
1541 : : * afterwards.
1542 : : */
1998 1543 : 755123 : input_buf_ptr = cstate->input_buf_index;
1544 : :
1545 : : /*
1546 : : * We first try to use SIMD for the task described above, falling back to
1547 : : * the scalar path (i.e., the loop below) if needed.
1548 : : */
1549 : : #ifndef USE_NO_SIMD
191 nathan@postgresql.or 1550 [ + + ]: 755123 : if (cstate->simd_enabled)
1551 : : {
1552 : : /*
1553 : : * Using temporary variables seems to encourage the compiler to keep
1554 : : * them in a register, which is beneficial for performance.
1555 : : */
1556 : 332573 : bool tmp_hit_eof = false;
1557 : 332573 : int tmp_input_buf_ptr = 0; /* silence compiler warning */
1558 : :
1559 : 332573 : result = CopyReadLineTextSIMDHelper(cstate, is_csv, &tmp_hit_eof,
1560 : : &tmp_input_buf_ptr);
1561 : 332563 : hit_eof = tmp_hit_eof;
1562 : 332563 : input_buf_ptr = tmp_input_buf_ptr;
1563 : :
1564 [ + + ]: 332563 : if (result)
1565 : : {
1566 : : /* Transfer any still-uncopied data to line_buf. */
1567 [ - + ]: 602 : REFILL_LINEBUF;
1568 : :
1569 : 602 : return result;
1570 : : }
1571 : : }
1572 : : #endif /* ! USE_NO_SIMD */
1573 : :
1574 : 754511 : copy_input_buf = cstate->input_buf;
1998 heikki.linnakangas@i 1575 : 754511 : copy_buf_len = cstate->input_buf_len;
1576 : :
1577 : : for (;;)
2127 1578 : 8712967 : {
1579 : : int prev_raw_ptr;
1580 : : char c;
1581 : :
1582 : : /*
1583 : : * Load more data if needed.
1584 : : *
1585 : : * TODO: We could just force four bytes of read-ahead and avoid the
1586 : : * many calls to IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(). That was
1587 : : * unsafe with the old v2 COPY protocol, but we don't support that
1588 : : * anymore.
1589 : : */
1998 1590 [ + + - + ]: 9467478 : if (input_buf_ptr >= copy_buf_len || need_data)
1591 : : {
2127 1592 [ + + ]: 486 : REFILL_LINEBUF;
1593 : :
19 nathan@postgresql.or 1594 : 486 : CopyLoadInputBuf(cstate, false);
1595 : : /* update our local variables */
1998 heikki.linnakangas@i 1596 : 486 : hit_eof = cstate->input_reached_eof;
1597 : 486 : input_buf_ptr = cstate->input_buf_index;
1598 : 486 : copy_buf_len = cstate->input_buf_len;
1599 : :
1600 : : /*
1601 : : * If we are completely out of data, break out of the loop,
1602 : : * reporting EOF.
1603 : : */
1604 [ + + ]: 486 : if (INPUT_BUF_BYTES(cstate) <= 0)
1605 : : {
2127 1606 : 352 : result = true;
1607 : 352 : break;
1608 : : }
1609 : 134 : need_data = false;
1610 : : }
1611 : :
1612 : : /* OK to fetch a character */
1998 1613 : 9467126 : prev_raw_ptr = input_buf_ptr;
1614 : 9467126 : c = copy_input_buf[input_buf_ptr++];
1615 : :
569 msawada@postgresql.o 1616 [ + + ]: 9467126 : if (is_csv)
1617 : : {
1618 : : /*
1619 : : * If character is '\r', we may need to look ahead below. Force
1620 : : * fetch of the next character if we don't already have it. We
1621 : : * need to do this before changing CSV state, in case '\r' is also
1622 : : * the quote or escape character.
1623 : : */
720 tgl@sss.pgh.pa.us 1624 [ + + ]: 2615 : if (c == '\r')
1625 : : {
2127 heikki.linnakangas@i 1626 [ - + - - ]: 24 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1627 : : }
1628 : :
1629 : : /*
1630 : : * Dealing with quotes and escapes here is mildly tricky. If the
1631 : : * quote char is also the escape char, there's no problem - we
1632 : : * just use the char as a toggle. If they are different, we need
1633 : : * to ensure that we only take account of an escape inside a
1634 : : * quoted field and immediately preceding a quote char, and not
1635 : : * the second in an escape-escape sequence.
1636 : : */
1637 [ + + + + ]: 2615 : if (in_quote && c == escapec)
1638 : 32 : last_was_esc = !last_was_esc;
1639 [ + + + - ]: 2615 : if (c == quotec && !last_was_esc)
1640 : 308 : in_quote = !in_quote;
1641 [ + + ]: 2615 : if (c != escapec)
1642 : 2579 : last_was_esc = false;
1643 : :
1644 : : /*
1645 : : * Updating the line count for embedded CR and/or LF chars is
1646 : : * necessarily a little fragile - this test is probably about the
1647 : : * best we can do. (XXX it's arguable whether we should do this
1648 : : * at all --- is cur_lineno a physical or logical count?)
1649 : : */
1650 [ + + + + : 2615 : if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
+ + ]
1651 : 24 : cstate->cur_lineno++;
1652 : : }
1653 : :
1654 : : /* Process \r */
569 msawada@postgresql.o 1655 [ + + + - : 9467126 : if (c == '\r' && (!is_csv || !in_quote))
- + ]
1656 : : {
1657 : : /* Check for \r\n on first line, _and_ handle \r\n. */
2127 heikki.linnakangas@i 1658 [ # # ]:UBC 0 : if (cstate->eol_type == EOL_UNKNOWN ||
1659 [ # # ]: 0 : cstate->eol_type == EOL_CRNL)
1660 : : {
1661 : : /*
1662 : : * If need more data, go back to loop top to load it.
1663 : : *
1664 : : * Note that if we are at EOF, c will wind up as '\0' because
1665 : : * of the guaranteed pad of input_buf.
1666 : : */
1667 [ # # # # ]: 0 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1668 : :
1669 : : /* get next char */
1998 1670 : 0 : c = copy_input_buf[input_buf_ptr];
1671 : :
2127 1672 [ # # ]: 0 : if (c == '\n')
1673 : : {
1998 1674 : 0 : input_buf_ptr++; /* eat newline */
2127 1675 : 0 : cstate->eol_type = EOL_CRNL; /* in case not set yet */
1676 : : }
1677 : : else
1678 : : {
1679 : : /* found \r, but no \n */
1680 [ # # ]: 0 : if (cstate->eol_type == EOL_CRNL)
1681 [ # # # # : 0 : ereport(ERROR,
# # ]
1682 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1683 : : !is_csv ?
1684 : : errmsg("literal carriage return found in data") :
1685 : : errmsg("unquoted carriage return found in data"),
1686 : : !is_csv ?
1687 : : errhint("Use \"\\r\" to represent carriage return.") :
1688 : : errhint("Use quoted CSV field to represent carriage return.")));
1689 : :
1690 : : /*
1691 : : * if we got here, it is the first line and we didn't find
1692 : : * \n, so don't consume the peeked character
1693 : : */
1694 : 0 : cstate->eol_type = EOL_CR;
1695 : : }
1696 : : }
1697 [ # # ]: 0 : else if (cstate->eol_type == EOL_NL)
1698 [ # # # # : 0 : ereport(ERROR,
# # ]
1699 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1700 : : !is_csv ?
1701 : : errmsg("literal carriage return found in data") :
1702 : : errmsg("unquoted carriage return found in data"),
1703 : : !is_csv ?
1704 : : errhint("Use \"\\r\" to represent carriage return.") :
1705 : : errhint("Use quoted CSV field to represent carriage return.")));
1706 : : /* If reach here, we have found the line terminator */
1707 : 0 : break;
1708 : : }
1709 : :
1710 : : /* Process \n */
569 msawada@postgresql.o 1711 [ + + + + :CBC 9467126 : if (c == '\n' && (!is_csv || !in_quote))
+ + ]
1712 : : {
2127 heikki.linnakangas@i 1713 [ + - - + ]: 754104 : if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
2127 heikki.linnakangas@i 1714 [ # # # # :UBC 0 : ereport(ERROR,
# # ]
1715 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1716 : : !is_csv ?
1717 : : errmsg("literal newline found in data") :
1718 : : errmsg("unquoted newline found in data"),
1719 : : !is_csv ?
1720 : : errhint("Use \"\\n\" to represent newline.") :
1721 : : errhint("Use quoted CSV field to represent newline.")));
2127 heikki.linnakangas@i 1722 :CBC 754104 : cstate->eol_type = EOL_NL; /* in case not set yet */
1723 : : /* If reach here, we have found the line terminator */
1724 : 754104 : break;
1725 : : }
1726 : :
1727 : : /*
1728 : : * Process backslash, except in CSV mode where backslash is a normal
1729 : : * character.
1730 : : */
569 msawada@postgresql.o 1731 [ + + + + ]: 8713022 : if (c == '\\' && !is_csv)
1732 : : {
1733 : : char c2;
1734 : :
2127 heikki.linnakangas@i 1735 [ - + - - ]: 4908 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1736 [ - + - - ]: 4908 : IF_NEED_REFILL_AND_EOF_BREAK(0);
1737 : :
1738 : : /* -----
1739 : : * get next character
1740 : : * Note: we do not change c so if it isn't \., we can fall
1741 : : * through and continue processing.
1742 : : * -----
1743 : : */
1998 1744 : 4908 : c2 = copy_input_buf[input_buf_ptr];
1745 : :
2127 1746 [ + + ]: 4908 : if (c2 == '.')
1747 : : {
1998 1748 : 55 : input_buf_ptr++; /* consume the '.' */
2127 1749 [ - + ]: 55 : if (cstate->eol_type == EOL_CRNL)
1750 : : {
1751 : : /* Get the next character */
2127 heikki.linnakangas@i 1752 [ # # # # ]:UBC 0 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1753 : : /* if hit_eof, c2 will become '\0' */
1998 1754 : 0 : c2 = copy_input_buf[input_buf_ptr++];
1755 : :
2127 1756 [ # # ]: 0 : if (c2 == '\n')
720 tgl@sss.pgh.pa.us 1757 [ # # ]: 0 : ereport(ERROR,
1758 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1759 : : errmsg("end-of-copy marker does not match previous newline style")));
2127 heikki.linnakangas@i 1760 [ # # ]: 0 : else if (c2 != '\r')
720 tgl@sss.pgh.pa.us 1761 [ # # ]: 0 : ereport(ERROR,
1762 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1763 : : errmsg("end-of-copy marker is not alone on its line")));
1764 : : }
1765 : :
1766 : : /* Get the next character */
2127 heikki.linnakangas@i 1767 [ - + - - ]:CBC 55 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1768 : : /* if hit_eof, c2 will become '\0' */
1998 1769 : 55 : c2 = copy_input_buf[input_buf_ptr++];
1770 : :
2127 1771 [ + - + + ]: 55 : if (c2 != '\r' && c2 != '\n')
720 tgl@sss.pgh.pa.us 1772 [ + - ]: 4 : ereport(ERROR,
1773 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1774 : : errmsg("end-of-copy marker is not alone on its line")));
1775 : :
2127 heikki.linnakangas@i 1776 [ + + + - ]: 51 : if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
1777 [ - + - - ]: 51 : (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
1778 [ - + - - ]: 51 : (cstate->eol_type == EOL_CR && c2 != '\r'))
2127 heikki.linnakangas@i 1779 [ # # ]:UBC 0 : ereport(ERROR,
1780 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1781 : : errmsg("end-of-copy marker does not match previous newline style")));
1782 : :
1783 : : /*
1784 : : * If there is any data on this line before the \., complain.
1785 : : */
719 tgl@sss.pgh.pa.us 1786 [ + - ]:CBC 51 : if (cstate->line_buf.len > 0 ||
1787 [ + + ]: 51 : prev_raw_ptr > cstate->input_buf_index)
1788 [ + - ]: 4 : ereport(ERROR,
1789 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1790 : : errmsg("end-of-copy marker is not alone on its line")));
1791 : :
1792 : : /*
1793 : : * Discard the \. and newline, then report EOF.
1794 : : */
1998 heikki.linnakangas@i 1795 : 47 : cstate->input_buf_index = input_buf_ptr;
2127 1796 : 47 : result = true; /* report EOF */
1797 : 47 : break;
1798 : : }
1799 : : else
1800 : : {
1801 : : /*
1802 : : * If we are here, it means we found a backslash followed by
1803 : : * something other than a period. In non-CSV mode, anything
1804 : : * after a backslash is special, so we skip over that second
1805 : : * character too. If we didn't do that \\. would be
1806 : : * considered an eof-of copy, while in non-CSV mode it is a
1807 : : * literal backslash followed by a period.
1808 : : */
1998 1809 : 4853 : input_buf_ptr++;
1810 : : }
1811 : : }
1812 : : } /* end of outer loop */
1813 : :
1814 : : /*
1815 : : * Transfer any still-uncopied data to line_buf.
1816 : : */
2127 1817 [ + + ]: 754503 : REFILL_LINEBUF;
1818 : :
1819 : 754503 : return result;
1820 : : }
1821 : :
1822 : : /*
1823 : : * Return decimal value for a hexadecimal digit
1824 : : */
1825 : : static int
2127 heikki.linnakangas@i 1826 :UBC 0 : GetDecimalFromHex(char hex)
1827 : : {
1828 [ # # ]: 0 : if (isdigit((unsigned char) hex))
1829 : 0 : return hex - '0';
1830 : : else
446 jdavis@postgresql.or 1831 : 0 : return pg_ascii_tolower((unsigned char) hex) - 'a' + 10;
1832 : : }
1833 : :
1834 : : /*
1835 : : * Parse the current line into separate attributes (fields),
1836 : : * performing de-escaping as needed.
1837 : : *
1838 : : * The input is in line_buf. We use attribute_buf to hold the result
1839 : : * strings. cstate->raw_fields[k] is set to point to the k'th attribute
1840 : : * string, or NULL when the input matches the null marker string.
1841 : : * This array is expanded as necessary.
1842 : : *
1843 : : * (Note that the caller cannot check for nulls since the returned
1844 : : * string would be the post-de-escaping equivalent, which may look
1845 : : * the same as some valid data string.)
1846 : : *
1847 : : * delim is the column delimiter string (must be just one byte for now).
1848 : : * null_print is the null marker string. Note that this is compared to
1849 : : * the pre-de-escaped input string.
1850 : : *
1851 : : * The return value is the number of fields actually read.
1852 : : */
1853 : : static int
2127 heikki.linnakangas@i 1854 :CBC 753711 : CopyReadAttributesText(CopyFromState cstate)
1855 : : {
1856 : 753711 : char delimc = cstate->opts.delim[0];
1857 : : int fieldno;
1858 : : char *output_ptr;
1859 : : char *cur_ptr;
1860 : : char *line_end_ptr;
1861 : :
1862 : : /*
1863 : : * We need a special case for zero-column tables: check that the input
1864 : : * line is empty, and return.
1865 : : */
1866 [ + + ]: 753711 : if (cstate->max_fields <= 0)
1867 : : {
1868 [ - + ]: 4 : if (cstate->line_buf.len != 0)
2127 heikki.linnakangas@i 1869 [ # # ]:UBC 0 : ereport(ERROR,
1870 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1871 : : errmsg("extra data after last expected column")));
2127 heikki.linnakangas@i 1872 :CBC 4 : return 0;
1873 : : }
1874 : :
1875 : 753707 : resetStringInfo(&cstate->attribute_buf);
1876 : :
1877 : : /*
1878 : : * The de-escaped attributes will certainly not be longer than the input
1879 : : * data line, so we can just force attribute_buf to be large enough and
1880 : : * then transfer data without any checks for enough space. We need to do
1881 : : * it this way because enlarging attribute_buf mid-stream would invalidate
1882 : : * pointers already stored into cstate->raw_fields[].
1883 : : */
1884 [ + + ]: 753707 : if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
1885 : 4 : enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
1886 : 753707 : output_ptr = cstate->attribute_buf.data;
1887 : :
1888 : : /* set pointer variables for loop */
1889 : 753707 : cur_ptr = cstate->line_buf.data;
1890 : 753707 : line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
1891 : :
1892 : : /* Outer loop iterates over fields */
1893 : 753707 : fieldno = 0;
1894 : : for (;;)
1895 : 1939767 : {
1896 : 2693474 : bool found_delim = false;
1897 : : char *start_ptr;
1898 : : char *end_ptr;
1899 : : int input_len;
1900 : 2693474 : bool saw_non_ascii = false;
1901 : :
1902 : : /* Make sure there is enough space for the next value */
1903 [ + + ]: 2693474 : if (fieldno >= cstate->max_fields)
1904 : : {
1905 : 28 : cstate->max_fields *= 2;
1906 : 28 : cstate->raw_fields =
34 michael@paquier.xyz 1907 :GNC 28 : repalloc_array(cstate->raw_fields, char *, cstate->max_fields);
1908 : : }
1909 : :
1910 : : /* Remember start of field on both input and output sides */
2127 heikki.linnakangas@i 1911 :CBC 2693474 : start_ptr = cur_ptr;
1912 : 2693474 : cstate->raw_fields[fieldno] = output_ptr;
1913 : :
1914 : : /*
1915 : : * Scan data for field.
1916 : : *
1917 : : * Note that in this loop, we are scanning to locate the end of field
1918 : : * and also speculatively performing de-escaping. Once we find the
1919 : : * end-of-field, we can match the raw field contents against the null
1920 : : * marker string. Only after that comparison fails do we know that
1921 : : * de-escaping is actually the right thing to do; therefore we *must
1922 : : * not* throw any syntax errors before we've done the null-marker
1923 : : * check.
1924 : : */
1925 : : for (;;)
1926 : 13828883 : {
1927 : : char c;
1928 : :
1929 : 16522357 : end_ptr = cur_ptr;
1930 [ + + ]: 16522357 : if (cur_ptr >= line_end_ptr)
1931 : 753703 : break;
1932 : 15768654 : c = *cur_ptr++;
1933 [ + + ]: 15768654 : if (c == delimc)
1934 : : {
1935 : 1939771 : found_delim = true;
1936 : 1939771 : break;
1937 : : }
1938 [ + + ]: 13828883 : if (c == '\\')
1939 : : {
1940 [ - + ]: 4853 : if (cur_ptr >= line_end_ptr)
2127 heikki.linnakangas@i 1941 :UBC 0 : break;
2127 heikki.linnakangas@i 1942 :CBC 4853 : c = *cur_ptr++;
1943 [ + + - - : 4853 : switch (c)
+ - - -
+ ]
1944 : : {
1945 : 8 : case '0':
1946 : : case '1':
1947 : : case '2':
1948 : : case '3':
1949 : : case '4':
1950 : : case '5':
1951 : : case '6':
1952 : : case '7':
1953 : : {
1954 : : /* handle \013 */
1955 : : int val;
1956 : :
1957 : 8 : val = OCTVALUE(c);
1958 [ + + ]: 8 : if (cur_ptr < line_end_ptr)
1959 : : {
1960 : 4 : c = *cur_ptr;
1961 [ - + - - ]: 4 : if (ISOCTAL(c))
1962 : : {
2127 heikki.linnakangas@i 1963 :UBC 0 : cur_ptr++;
1964 : 0 : val = (val << 3) + OCTVALUE(c);
1965 [ # # ]: 0 : if (cur_ptr < line_end_ptr)
1966 : : {
1967 : 0 : c = *cur_ptr;
1968 [ # # # # ]: 0 : if (ISOCTAL(c))
1969 : : {
1970 : 0 : cur_ptr++;
1971 : 0 : val = (val << 3) + OCTVALUE(c);
1972 : : }
1973 : : }
1974 : : }
1975 : : }
2127 heikki.linnakangas@i 1976 :CBC 8 : c = val & 0377;
1977 [ - + - - ]: 8 : if (c == '\0' || IS_HIGHBIT_SET(c))
1978 : 8 : saw_non_ascii = true;
1979 : : }
1980 : 8 : break;
1981 : 8 : case 'x':
1982 : : /* Handle \x3F */
1983 [ + + ]: 8 : if (cur_ptr < line_end_ptr)
1984 : : {
1985 : 4 : char hexchar = *cur_ptr;
1986 : :
1987 [ - + ]: 4 : if (isxdigit((unsigned char) hexchar))
1988 : : {
2127 heikki.linnakangas@i 1989 :UBC 0 : int val = GetDecimalFromHex(hexchar);
1990 : :
1991 : 0 : cur_ptr++;
1992 [ # # ]: 0 : if (cur_ptr < line_end_ptr)
1993 : : {
1994 : 0 : hexchar = *cur_ptr;
1995 [ # # ]: 0 : if (isxdigit((unsigned char) hexchar))
1996 : : {
1997 : 0 : cur_ptr++;
1998 : 0 : val = (val << 4) + GetDecimalFromHex(hexchar);
1999 : : }
2000 : : }
2001 : 0 : c = val & 0xff;
2002 [ # # # # ]: 0 : if (c == '\0' || IS_HIGHBIT_SET(c))
2003 : 0 : saw_non_ascii = true;
2004 : : }
2005 : : }
2127 heikki.linnakangas@i 2006 :CBC 8 : break;
2127 heikki.linnakangas@i 2007 :UBC 0 : case 'b':
2008 : 0 : c = '\b';
2009 : 0 : break;
2010 : 0 : case 'f':
2011 : 0 : c = '\f';
2012 : 0 : break;
2127 heikki.linnakangas@i 2013 :CBC 2033 : case 'n':
2014 : 2033 : c = '\n';
2015 : 2033 : break;
2127 heikki.linnakangas@i 2016 :UBC 0 : case 'r':
2017 : 0 : c = '\r';
2018 : 0 : break;
2019 : 0 : case 't':
2020 : 0 : c = '\t';
2021 : 0 : break;
2022 : 0 : case 'v':
2023 : 0 : c = '\v';
2024 : 0 : break;
2025 : :
2026 : : /*
2027 : : * in all other cases, take the char after '\'
2028 : : * literally
2029 : : */
2030 : : }
2031 : : }
2032 : :
2033 : : /* Add c to output string */
2127 heikki.linnakangas@i 2034 :CBC 13828883 : *output_ptr++ = c;
2035 : : }
2036 : :
2037 : : /* Check whether raw input matched null marker */
2038 : 2693474 : input_len = end_ptr - start_ptr;
2039 [ + + ]: 2693474 : if (input_len == cstate->opts.null_print_len &&
2040 [ + + ]: 162364 : strncmp(start_ptr, cstate->opts.null_print, input_len) == 0)
2041 : 2734 : cstate->raw_fields[fieldno] = NULL;
2042 : : /* Check whether raw input matched default marker */
1285 andrew@dunslane.net 2043 [ + + ]: 2690740 : else if (fieldno < list_length(cstate->attnumlist) &&
2044 [ + + ]: 2690708 : cstate->opts.default_print &&
1287 2045 [ + + ]: 76 : input_len == cstate->opts.default_print_len &&
2046 [ + - ]: 20 : strncmp(start_ptr, cstate->opts.default_print, input_len) == 0)
2047 : 16 : {
2048 : : /* fieldno is 0-indexed and attnum is 1-indexed */
2049 : 20 : int m = list_nth_int(cstate->attnumlist, fieldno) - 1;
2050 : :
2051 [ + + ]: 20 : if (cstate->defexprs[m] != NULL)
2052 : : {
2053 : : /* defaults contain entries for all physical attributes */
2054 : 16 : cstate->defaults[m] = true;
2055 : : }
2056 : : else
2057 : : {
2058 : 4 : TupleDesc tupDesc = RelationGetDescr(cstate->rel);
2059 : 4 : Form_pg_attribute att = TupleDescAttr(tupDesc, m);
2060 : :
2061 [ + - ]: 4 : ereport(ERROR,
2062 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2063 : : errmsg("unexpected default marker in COPY data"),
2064 : : errdetail("Column \"%s\" has no default value.",
2065 : : NameStr(att->attname))));
2066 : : }
2067 : : }
2068 : : else
2069 : : {
2070 : : /*
2071 : : * At this point we know the field is supposed to contain data.
2072 : : *
2073 : : * If we de-escaped any non-7-bit-ASCII chars, make sure the
2074 : : * resulting string is valid data for the db encoding.
2075 : : */
2127 heikki.linnakangas@i 2076 [ - + ]: 2690720 : if (saw_non_ascii)
2077 : : {
2127 heikki.linnakangas@i 2078 :UBC 0 : char *fld = cstate->raw_fields[fieldno];
2079 : :
2080 : 0 : pg_verifymbstr(fld, output_ptr - fld, false);
2081 : : }
2082 : : }
2083 : :
2084 : : /* Terminate attribute value in output area */
2127 heikki.linnakangas@i 2085 :CBC 2693470 : *output_ptr++ = '\0';
2086 : :
2087 : 2693470 : fieldno++;
2088 : : /* Done if we hit EOL instead of a delim */
2089 [ + + ]: 2693470 : if (!found_delim)
2090 : 753703 : break;
2091 : : }
2092 : :
2093 : : /* Clean up state of attribute_buf */
2094 : 753703 : output_ptr--;
2095 [ - + ]: 753703 : Assert(*output_ptr == '\0');
2096 : 753703 : cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
2097 : :
2098 : 753703 : return fieldno;
2099 : : }
2100 : :
2101 : : /*
2102 : : * Parse the current line into separate attributes (fields),
2103 : : * performing de-escaping as needed. This has exactly the same API as
2104 : : * CopyReadAttributesText, except we parse the fields according to
2105 : : * "standard" (i.e. common) CSV usage.
2106 : : */
2107 : : static int
2108 : 318 : CopyReadAttributesCSV(CopyFromState cstate)
2109 : : {
2110 : 318 : char delimc = cstate->opts.delim[0];
2111 : 318 : char quotec = cstate->opts.quote[0];
2112 : 318 : char escapec = cstate->opts.escape[0];
2113 : : int fieldno;
2114 : : char *output_ptr;
2115 : : char *cur_ptr;
2116 : : char *line_end_ptr;
2117 : :
2118 : : /*
2119 : : * We need a special case for zero-column tables: check that the input
2120 : : * line is empty, and return.
2121 : : */
2122 [ - + ]: 318 : if (cstate->max_fields <= 0)
2123 : : {
2127 heikki.linnakangas@i 2124 [ # # ]:UBC 0 : if (cstate->line_buf.len != 0)
2125 [ # # ]: 0 : ereport(ERROR,
2126 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2127 : : errmsg("extra data after last expected column")));
2128 : 0 : return 0;
2129 : : }
2130 : :
2127 heikki.linnakangas@i 2131 :CBC 318 : resetStringInfo(&cstate->attribute_buf);
2132 : :
2133 : : /*
2134 : : * The de-escaped attributes will certainly not be longer than the input
2135 : : * data line, so we can just force attribute_buf to be large enough and
2136 : : * then transfer data without any checks for enough space. We need to do
2137 : : * it this way because enlarging attribute_buf mid-stream would invalidate
2138 : : * pointers already stored into cstate->raw_fields[].
2139 : : */
2140 [ - + ]: 318 : if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
2127 heikki.linnakangas@i 2141 :UBC 0 : enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
2127 heikki.linnakangas@i 2142 :CBC 318 : output_ptr = cstate->attribute_buf.data;
2143 : :
2144 : : /* set pointer variables for loop */
2145 : 318 : cur_ptr = cstate->line_buf.data;
2146 : 318 : line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
2147 : :
2148 : : /* Outer loop iterates over fields */
2149 : 318 : fieldno = 0;
2150 : : for (;;)
2151 : 326 : {
2152 : 644 : bool found_delim = false;
2153 : 644 : bool saw_quote = false;
2154 : : char *start_ptr;
2155 : : char *end_ptr;
2156 : : int input_len;
2157 : :
2158 : : /* Make sure there is enough space for the next value */
2159 [ - + ]: 644 : if (fieldno >= cstate->max_fields)
2160 : : {
2127 heikki.linnakangas@i 2161 :UBC 0 : cstate->max_fields *= 2;
2162 : 0 : cstate->raw_fields =
34 michael@paquier.xyz 2163 :UNC 0 : repalloc_array(cstate->raw_fields, char *, cstate->max_fields);
2164 : : }
2165 : :
2166 : : /* Remember start of field on both input and output sides */
2127 heikki.linnakangas@i 2167 :CBC 644 : start_ptr = cur_ptr;
2168 : 644 : cstate->raw_fields[fieldno] = output_ptr;
2169 : :
2170 : : /*
2171 : : * Scan data for field,
2172 : : *
2173 : : * The loop starts in "not quote" mode and then toggles between that
2174 : : * and "in quote" mode. The loop exits normally if it is in "not
2175 : : * quote" mode and a delimiter or line end is seen.
2176 : : */
2177 : : for (;;)
2178 : 137 : {
2179 : : char c;
2180 : :
2181 : : /* Not in quote */
2182 : : for (;;)
2183 : : {
2184 : 2045 : end_ptr = cur_ptr;
2185 [ + + ]: 2045 : if (cur_ptr >= line_end_ptr)
2186 : 314 : goto endfield;
2187 : 1731 : c = *cur_ptr++;
2188 : : /* unquoted field delimiter */
2189 [ + + ]: 1731 : if (c == delimc)
2190 : : {
2191 : 330 : found_delim = true;
2192 : 330 : goto endfield;
2193 : : }
2194 : : /* start of quoted field (or part of field) */
2195 [ + + ]: 1401 : if (c == quotec)
2196 : : {
2197 : 137 : saw_quote = true;
2198 : 137 : break;
2199 : : }
2200 : : /* Add c to output string */
2201 : 1264 : *output_ptr++ = c;
2202 : : }
2203 : :
2204 : : /* In quote */
2205 : : for (;;)
2206 : : {
2207 : 852 : end_ptr = cur_ptr;
2208 [ - + ]: 852 : if (cur_ptr >= line_end_ptr)
2127 heikki.linnakangas@i 2209 [ # # ]:UBC 0 : ereport(ERROR,
2210 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2211 : : errmsg("unterminated CSV quoted field")));
2212 : :
2127 heikki.linnakangas@i 2213 :CBC 852 : c = *cur_ptr++;
2214 : :
2215 : : /* escape within a quoted field */
2216 [ + + ]: 852 : if (c == escapec)
2217 : : {
2218 : : /*
2219 : : * peek at the next char if available, and escape it if it
2220 : : * is an escape char or a quote char
2221 : : */
2222 [ + + ]: 81 : if (cur_ptr < line_end_ptr)
2223 : : {
2224 : 47 : char nextc = *cur_ptr;
2225 : :
2226 [ + + - + ]: 47 : if (nextc == escapec || nextc == quotec)
2227 : : {
2228 : 16 : *output_ptr++ = nextc;
2229 : 16 : cur_ptr++;
2230 : 16 : continue;
2231 : : }
2232 : : }
2233 : : }
2234 : :
2235 : : /*
2236 : : * end of quoted field. Must do this test after testing for
2237 : : * escape in case quote char and escape char are the same
2238 : : * (which is the common case).
2239 : : */
2240 [ + + ]: 836 : if (c == quotec)
2241 : 137 : break;
2242 : :
2243 : : /* Add c to output string */
2244 : 699 : *output_ptr++ = c;
2245 : : }
2246 : : }
2247 : 644 : endfield:
2248 : :
2249 : : /* Terminate attribute value in output area */
2250 : 644 : *output_ptr++ = '\0';
2251 : :
2252 : : /* Check whether raw input matched null marker */
2253 : 644 : input_len = end_ptr - start_ptr;
2254 [ + + + + ]: 644 : if (!saw_quote && input_len == cstate->opts.null_print_len &&
2255 [ + - ]: 27 : strncmp(start_ptr, cstate->opts.null_print, input_len) == 0)
2256 : 27 : cstate->raw_fields[fieldno] = NULL;
2257 : : /* Check whether raw input matched default marker */
1285 andrew@dunslane.net 2258 [ + - ]: 617 : else if (fieldno < list_length(cstate->attnumlist) &&
2259 [ + + ]: 617 : cstate->opts.default_print &&
1287 2260 [ + + ]: 94 : input_len == cstate->opts.default_print_len &&
2261 [ + - ]: 26 : strncmp(start_ptr, cstate->opts.default_print, input_len) == 0)
2262 : : {
2263 : : /* fieldno is 0-index and attnum is 1-index */
2264 : 26 : int m = list_nth_int(cstate->attnumlist, fieldno) - 1;
2265 : :
2266 [ + + ]: 26 : if (cstate->defexprs[m] != NULL)
2267 : : {
2268 : : /* defaults contain entries for all physical attributes */
2269 : 22 : cstate->defaults[m] = true;
2270 : : }
2271 : : else
2272 : : {
2273 : 4 : TupleDesc tupDesc = RelationGetDescr(cstate->rel);
2274 : 4 : Form_pg_attribute att = TupleDescAttr(tupDesc, m);
2275 : :
2276 [ + - ]: 4 : ereport(ERROR,
2277 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2278 : : errmsg("unexpected default marker in COPY data"),
2279 : : errdetail("Column \"%s\" has no default value.",
2280 : : NameStr(att->attname))));
2281 : : }
2282 : : }
2283 : :
2127 heikki.linnakangas@i 2284 : 640 : fieldno++;
2285 : : /* Done if we hit EOL instead of a delim */
2286 [ + + ]: 640 : if (!found_delim)
2287 : 314 : break;
2288 : : }
2289 : :
2290 : : /* Clean up state of attribute_buf */
2291 : 314 : output_ptr--;
2292 [ - + ]: 314 : Assert(*output_ptr == '\0');
2293 : 314 : cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
2294 : :
2295 : 314 : return fieldno;
2296 : : }
2297 : :
2298 : :
2299 : : /*
2300 : : * Read a binary attribute
2301 : : */
2302 : : static Datum
2303 : 100 : CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
2304 : : Oid typioparam, int32 typmod,
2305 : : bool *isnull)
2306 : : {
2307 : : int32 fld_size;
2308 : : Datum result;
2309 : :
2310 [ - + ]: 100 : if (!CopyGetInt32(cstate, &fld_size))
2127 heikki.linnakangas@i 2311 [ # # ]:UBC 0 : ereport(ERROR,
2312 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2313 : : errmsg("unexpected EOF in COPY data")));
2127 heikki.linnakangas@i 2314 [ + + ]:CBC 100 : if (fld_size == -1)
2315 : : {
2316 : 20 : *isnull = true;
2317 : 20 : return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
2318 : : }
2319 [ - + ]: 80 : if (fld_size < 0)
2127 heikki.linnakangas@i 2320 [ # # ]:UBC 0 : ereport(ERROR,
2321 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2322 : : errmsg("invalid field size")));
2323 : :
2324 : : /* reset attribute_buf to empty, and load raw data in it */
2127 heikki.linnakangas@i 2325 :CBC 80 : resetStringInfo(&cstate->attribute_buf);
2326 : :
2327 : 80 : enlargeStringInfo(&cstate->attribute_buf, fld_size);
2328 : 80 : if (CopyReadBinaryData(cstate, cstate->attribute_buf.data,
2329 [ - + ]: 80 : fld_size) != fld_size)
2127 heikki.linnakangas@i 2330 [ # # ]:UBC 0 : ereport(ERROR,
2331 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2332 : : errmsg("unexpected EOF in COPY data")));
2333 : :
2127 heikki.linnakangas@i 2334 :CBC 80 : cstate->attribute_buf.len = fld_size;
2335 : 80 : cstate->attribute_buf.data[fld_size] = '\0';
2336 : :
2337 : : /* Call the column type's binary input converter */
2338 : 80 : result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
2339 : : typioparam, typmod);
2340 : :
2341 : : /* Trouble if it didn't eat the whole buffer */
2342 [ + + ]: 80 : if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
2343 [ + - ]: 1 : ereport(ERROR,
2344 : : (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
2345 : : errmsg("incorrect binary data format")));
2346 : :
2347 : 79 : *isnull = false;
2348 : 79 : return result;
2349 : : }
|
