Branch data Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * copyfromparse.c
4 : : * Parse CSV/text/binary format for COPY FROM.
5 : : *
6 : : * This file contains routines to parse the text, CSV and binary input
7 : : * formats. The main entry point is NextCopyFrom(), which parses the
8 : : * next input line and returns it as Datums.
9 : : *
10 : : * In text/CSV mode, the parsing happens in multiple stages:
11 : : *
12 : : * [data source] --> raw_buf --> input_buf --> line_buf --> attribute_buf
13 : : * 1. 2. 3. 4.
14 : : *
15 : : * 1. CopyLoadRawBuf() reads raw data from the input file or client, and
16 : : * places it into 'raw_buf'.
17 : : *
18 : : * 2. CopyConvertBuf() calls the encoding conversion function to convert
19 : : * the data in 'raw_buf' from client to server encoding, placing the
20 : : * converted result in 'input_buf'.
21 : : *
22 : : * 3. CopyReadLine() parses the data in 'input_buf', one line at a time.
23 : : * It is responsible for finding the next newline marker, taking quote and
24 : : * escape characters into account according to the COPY options. The line
25 : : * is copied into 'line_buf', with quotes and escape characters still
26 : : * intact.
27 : : *
28 : : * 4. CopyReadAttributesText/CSV() function takes the input line from
29 : : * 'line_buf', and splits it into fields, unescaping the data as required.
30 : : * The fields are stored in 'attribute_buf', and 'raw_fields' array holds
31 : : * pointers to each field.
32 : : *
33 : : * If encoding conversion is not required, a shortcut is taken in step 2 to
34 : : * avoid copying the data unnecessarily. The 'input_buf' pointer is set to
35 : : * point directly to 'raw_buf', so that CopyLoadRawBuf() loads the raw data
36 : : * directly into 'input_buf'. CopyConvertBuf() then merely validates that
37 : : * the data is valid in the current encoding.
38 : : *
39 : : * In binary mode, the pipeline is much simpler. Input is loaded into
40 : : * 'raw_buf', and encoding conversion is done in the datatype-specific
41 : : * receive functions, if required. 'input_buf' and 'line_buf' are not used,
42 : : * but 'attribute_buf' is used as a temporary buffer to hold one attribute's
43 : : * data when it's passed the receive function.
44 : : *
45 : : * 'raw_buf' is always 64 kB in size (RAW_BUF_SIZE). 'input_buf' is also
46 : : * 64 kB (INPUT_BUF_SIZE), if encoding conversion is required. 'line_buf'
47 : : * and 'attribute_buf' are expanded on demand, to hold the longest line
48 : : * encountered so far.
49 : : *
50 : : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
51 : : * Portions Copyright (c) 1994, Regents of the University of California
52 : : *
53 : : *
54 : : * IDENTIFICATION
55 : : * src/backend/commands/copyfromparse.c
56 : : *
57 : : *-------------------------------------------------------------------------
58 : : */
59 : : #include "postgres.h"
60 : :
61 : : #include <ctype.h>
62 : : #include <unistd.h>
63 : : #include <sys/stat.h>
64 : :
65 : : #include "commands/copyapi.h"
66 : : #include "commands/copyfrom_internal.h"
67 : : #include "commands/progress.h"
68 : : #include "executor/executor.h"
69 : : #include "libpq/libpq.h"
70 : : #include "libpq/pqformat.h"
71 : : #include "mb/pg_wchar.h"
72 : : #include "miscadmin.h"
73 : : #include "pgstat.h"
74 : : #include "port/pg_bitutils.h"
75 : : #include "port/pg_bswap.h"
76 : : #include "port/simd.h"
77 : : #include "utils/builtins.h"
78 : : #include "utils/rel.h"
79 : : #include "utils/wait_event.h"
80 : :
81 : : #define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
82 : : #define OCTVALUE(c) ((c) - '0')
83 : :
84 : : /*
85 : : * These macros centralize code used to process line_buf and input_buf buffers.
86 : : * They are macros because they often do continue/break control and to avoid
87 : : * function call overhead in tight COPY loops.
88 : : *
89 : : * We must use "if (1)" because the usual "do {...} while(0)" wrapper would
90 : : * prevent the continue/break processing from working. We end the "if (1)"
91 : : * with "else ((void) 0)" to ensure the "if" does not unintentionally match
92 : : * any "else" in the calling code, and to avoid any compiler warnings about
93 : : * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
94 : : */
95 : :
96 : : /*
97 : : * This keeps the character read at the top of the loop in the buffer
98 : : * even if there is more than one read-ahead.
99 : : */
100 : : #define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
101 : : if (1) \
102 : : { \
103 : : if (input_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
104 : : { \
105 : : input_buf_ptr = prev_raw_ptr; /* undo fetch */ \
106 : : need_data = true; \
107 : : continue; \
108 : : } \
109 : : } else ((void) 0)
110 : :
111 : : /* This consumes the remainder of the buffer and breaks */
112 : : #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
113 : : if (1) \
114 : : { \
115 : : if (input_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
116 : : { \
117 : : if (extralen) \
118 : : input_buf_ptr = copy_buf_len; /* consume the partial character */ \
119 : : /* backslash just before EOF, treat as data char */ \
120 : : result = true; \
121 : : break; \
122 : : } \
123 : : } else ((void) 0)
124 : :
125 : : /*
126 : : * Transfer any approved data to line_buf; must do this to be sure
127 : : * there is some room in input_buf.
128 : : */
129 : : #define REFILL_LINEBUF \
130 : : if (1) \
131 : : { \
132 : : if (input_buf_ptr > cstate->input_buf_index) \
133 : : { \
134 : : appendBinaryStringInfo(&cstate->line_buf, \
135 : : cstate->input_buf + cstate->input_buf_index, \
136 : : input_buf_ptr - cstate->input_buf_index); \
137 : : cstate->input_buf_index = input_buf_ptr; \
138 : : } \
139 : : } else ((void) 0)
140 : :
141 : : /* NOTE: there's a copy of this in copyto.c */
142 : : static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0";
143 : :
144 : :
145 : : /* non-export function prototypes */
146 : : static bool CopyReadLine(CopyFromState cstate, bool is_csv);
147 : : static pg_always_inline bool CopyReadLineText(CopyFromState cstate,
148 : : bool is_csv);
149 : : static int CopyReadAttributesText(CopyFromState cstate);
150 : : static int CopyReadAttributesCSV(CopyFromState cstate);
151 : : static Datum CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
152 : : Oid typioparam, int32 typmod,
153 : : bool *isnull);
154 : : static pg_always_inline bool CopyFromTextLikeOneRow(CopyFromState cstate,
155 : : ExprContext *econtext,
156 : : Datum *values,
157 : : bool *nulls,
158 : : bool is_csv);
159 : : static pg_always_inline bool NextCopyFromRawFieldsInternal(CopyFromState cstate,
160 : : char ***fields,
161 : : int *nfields,
162 : : bool is_csv);
163 : :
164 : :
165 : : /* Low-level communications functions */
166 : : static int CopyGetData(CopyFromState cstate, void *databuf,
167 : : int minread, int maxread);
168 : : static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
169 : : static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
170 : : static void CopyLoadInputBuf(CopyFromState cstate, bool speculative);
171 : : static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
172 : :
173 : : void
174 : 691 : ReceiveCopyBegin(CopyFromState cstate)
175 : : {
176 : : StringInfoData buf;
177 : 691 : int natts = list_length(cstate->attnumlist);
178 : 691 : int16 format = (cstate->opts.format == COPY_FORMAT_BINARY ? 1 : 0);
179 : : int i;
180 : :
181 : 691 : pq_beginmessage(&buf, PqMsg_CopyInResponse);
182 : 691 : pq_sendbyte(&buf, format); /* overall format */
183 : 691 : pq_sendint16(&buf, natts);
184 [ + + ]: 2477 : for (i = 0; i < natts; i++)
185 : 1786 : pq_sendint16(&buf, format); /* per-column formats */
186 : 691 : pq_endmessage(&buf);
187 : 691 : cstate->copy_src = COPY_FRONTEND;
188 : 691 : cstate->fe_msgbuf = makeStringInfo();
189 : : /* We *must* flush here to ensure FE knows it can send. */
190 : 691 : pq_flush();
191 : 691 : }
192 : :
193 : : void
194 : 8 : ReceiveCopyBinaryHeader(CopyFromState cstate)
195 : : {
196 : : char readSig[11];
197 : : int32 tmp;
198 : :
199 : : /* Signature */
200 [ + - ]: 8 : if (CopyReadBinaryData(cstate, readSig, 11) != 11 ||
201 [ - + ]: 8 : memcmp(readSig, BinarySignature, 11) != 0)
202 [ # # ]: 0 : ereport(ERROR,
203 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
204 : : errmsg("COPY file signature not recognized")));
205 : : /* Flags field */
206 [ - + ]: 8 : if (!CopyGetInt32(cstate, &tmp))
207 [ # # ]: 0 : ereport(ERROR,
208 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
209 : : errmsg("invalid COPY file header (missing flags)")));
210 [ - + ]: 8 : if ((tmp & (1 << 16)) != 0)
211 [ # # ]: 0 : ereport(ERROR,
212 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
213 : : errmsg("invalid COPY file header (WITH OIDS)")));
214 : 8 : tmp &= ~(1 << 16);
215 [ - + ]: 8 : if ((tmp >> 16) != 0)
216 [ # # ]: 0 : ereport(ERROR,
217 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
218 : : errmsg("unrecognized critical flags in COPY file header")));
219 : : /* Header extension length */
220 [ + - ]: 8 : if (!CopyGetInt32(cstate, &tmp) ||
221 [ - + ]: 8 : tmp < 0)
222 [ # # ]: 0 : ereport(ERROR,
223 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
224 : : errmsg("invalid COPY file header (missing length)")));
225 : : /* Skip extension header, if present */
226 [ - + ]: 8 : while (tmp-- > 0)
227 : : {
228 [ # # ]: 0 : if (CopyReadBinaryData(cstate, readSig, 1) != 1)
229 [ # # ]: 0 : ereport(ERROR,
230 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
231 : : errmsg("invalid COPY file header (wrong length)")));
232 : : }
233 : 8 : }
234 : :
235 : : /*
236 : : * CopyGetData reads data from the source (file or frontend)
237 : : *
238 : : * We attempt to read at least minread, and at most maxread, bytes from
239 : : * the source. The actual number of bytes read is returned; if this is
240 : : * less than minread, EOF was detected.
241 : : *
242 : : * Note: when copying from the frontend, we expect a proper EOF mark per
243 : : * protocol; if the frontend simply drops the connection, we raise error.
244 : : * It seems unwise to allow the COPY IN to complete normally in that case.
245 : : *
246 : : * NB: no data conversion is applied here.
247 : : */
248 : : static int
249 : 214604 : CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
250 : : {
251 : 214604 : int bytesread = 0;
252 : :
253 [ + + + - ]: 214604 : switch (cstate->copy_src)
254 : : {
255 : 715 : case COPY_FILE:
256 : 715 : pgstat_report_wait_start(WAIT_EVENT_COPY_FROM_READ);
257 : 715 : bytesread = fread(databuf, 1, maxread, cstate->copy_file);
258 : 715 : pgstat_report_wait_end();
259 [ - + ]: 715 : if (ferror(cstate->copy_file))
260 [ # # ]: 0 : ereport(ERROR,
261 : : (errcode_for_file_access(),
262 : : errmsg("could not read from COPY file: %m")));
263 [ + + ]: 715 : if (bytesread == 0)
264 : 282 : cstate->raw_reached_eof = true;
265 : 715 : break;
266 : 201788 : case COPY_FRONTEND:
267 [ + - + + : 402547 : while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof)
+ + ]
268 : : {
269 : : int avail;
270 : :
271 [ + + ]: 402061 : while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
272 : : {
273 : : /* Try to receive another message */
274 : : int mtype;
275 : : int maxmsglen;
276 : :
277 : 201302 : readmessage:
278 : 201302 : HOLD_CANCEL_INTERRUPTS();
279 : 201302 : pq_startmsgread();
280 : 201302 : mtype = pq_getbyte();
281 [ - + ]: 201302 : if (mtype == EOF)
282 [ # # ]: 0 : ereport(ERROR,
283 : : (errcode(ERRCODE_CONNECTION_FAILURE),
284 : : errmsg("unexpected EOF on client connection with an open transaction")));
285 : : /* Validate message type and set packet size limit */
286 [ + + + ]: 201302 : switch (mtype)
287 : : {
288 : 200759 : case PqMsg_CopyData:
289 : 200759 : maxmsglen = PQ_LARGE_MESSAGE_LIMIT;
290 : 200759 : break;
291 : 541 : case PqMsg_CopyDone:
292 : : case PqMsg_CopyFail:
293 : : case PqMsg_Flush:
294 : : case PqMsg_Sync:
295 : 541 : maxmsglen = PQ_SMALL_MESSAGE_LIMIT;
296 : 541 : break;
297 : 2 : default:
298 [ + - ]: 2 : ereport(ERROR,
299 : : (errcode(ERRCODE_PROTOCOL_VIOLATION),
300 : : errmsg("unexpected message type 0x%02X during COPY from stdin",
301 : : mtype)));
302 : : maxmsglen = 0; /* keep compiler quiet */
303 : : break;
304 : : }
305 : : /* Now collect the message body */
306 [ - + ]: 201300 : if (pq_getmessage(cstate->fe_msgbuf, maxmsglen))
307 [ # # ]: 0 : ereport(ERROR,
308 : : (errcode(ERRCODE_CONNECTION_FAILURE),
309 : : errmsg("unexpected EOF on client connection with an open transaction")));
310 : 201300 : RESUME_CANCEL_INTERRUPTS();
311 : : /* ... and process it */
312 [ + + - - : 201300 : switch (mtype)
- ]
313 : : {
314 : 200759 : case PqMsg_CopyData:
315 : 200759 : break;
316 : 541 : case PqMsg_CopyDone:
317 : : /* COPY IN correctly terminated by frontend */
318 : 541 : cstate->raw_reached_eof = true;
319 : 541 : return bytesread;
320 : 0 : case PqMsg_CopyFail:
321 [ # # ]: 0 : ereport(ERROR,
322 : : (errcode(ERRCODE_QUERY_CANCELED),
323 : : errmsg("COPY from stdin failed: %s",
324 : : pq_getmsgstring(cstate->fe_msgbuf))));
325 : : break;
326 : 0 : case PqMsg_Flush:
327 : : case PqMsg_Sync:
328 : :
329 : : /*
330 : : * Ignore Flush/Sync for the convenience of client
331 : : * libraries (such as libpq) that may send those
332 : : * without noticing that the command they just
333 : : * sent was COPY.
334 : : */
335 : 0 : goto readmessage;
336 : 200759 : default:
337 : : Assert(false); /* NOT REACHED */
338 : : }
339 : : }
340 : 200759 : avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
341 [ - + ]: 200759 : if (avail > maxread)
342 : 0 : avail = maxread;
343 : 200759 : pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
344 : 200759 : databuf = (char *) databuf + avail;
345 : 200759 : maxread -= avail;
346 : 200759 : bytesread += avail;
347 : : }
348 : 201245 : break;
349 : 12101 : case COPY_CALLBACK:
350 : 12101 : bytesread = cstate->data_source_cb(databuf, minread, maxread);
351 : 12101 : break;
352 : : }
353 : :
354 : 214061 : return bytesread;
355 : : }
356 : :
357 : :
358 : : /*
359 : : * These functions do apply some data conversion
360 : : */
361 : :
362 : : /*
363 : : * CopyGetInt32 reads an int32 that appears in network byte order
364 : : *
365 : : * Returns true if OK, false if EOF
366 : : */
367 : : static inline bool
368 : 116 : CopyGetInt32(CopyFromState cstate, int32 *val)
369 : : {
370 : : uint32 buf;
371 : :
372 [ - + ]: 116 : if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
373 : : {
374 : 0 : *val = 0; /* suppress compiler warning */
375 : 0 : return false;
376 : : }
377 : 116 : *val = (int32) pg_ntoh32(buf);
378 : 116 : return true;
379 : : }
380 : :
381 : : /*
382 : : * CopyGetInt16 reads an int16 that appears in network byte order
383 : : */
384 : : static inline bool
385 : 25 : CopyGetInt16(CopyFromState cstate, int16 *val)
386 : : {
387 : : uint16 buf;
388 : :
389 [ - + ]: 25 : if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
390 : : {
391 : 0 : *val = 0; /* suppress compiler warning */
392 : 0 : return false;
393 : : }
394 : 25 : *val = (int16) pg_ntoh16(buf);
395 : 25 : return true;
396 : : }
397 : :
398 : :
399 : : /*
400 : : * Perform encoding conversion on data in 'raw_buf', writing the converted
401 : : * data into 'input_buf'.
402 : : *
403 : : * On entry, there must be some data to convert in 'raw_buf'.
404 : : */
405 : : static void
406 : 428416 : CopyConvertBuf(CopyFromState cstate)
407 : : {
408 : : /*
409 : : * If the file and server encoding are the same, no encoding conversion is
410 : : * required. However, we still need to verify that the input is valid for
411 : : * the encoding.
412 : : */
413 [ + + ]: 428416 : if (!cstate->need_transcoding)
414 : : {
415 : : /*
416 : : * When conversion is not required, input_buf and raw_buf are the
417 : : * same. raw_buf_len is the total number of bytes in the buffer, and
418 : : * input_buf_len tracks how many of those bytes have already been
419 : : * verified.
420 : : */
421 : 428328 : int preverifiedlen = cstate->input_buf_len;
422 : 428328 : int unverifiedlen = cstate->raw_buf_len - cstate->input_buf_len;
423 : : int nverified;
424 : :
425 [ + + ]: 428328 : if (unverifiedlen == 0)
426 : : {
427 : : /*
428 : : * If no more raw data is coming, report the EOF to the caller.
429 : : */
430 [ + + ]: 215283 : if (cstate->raw_reached_eof)
431 : 1276 : cstate->input_reached_eof = true;
432 : 215283 : return;
433 : : }
434 : :
435 : : /*
436 : : * Verify the new data, including any residual unverified bytes from
437 : : * previous round.
438 : : */
439 : 213045 : nverified = pg_encoding_verifymbstr(cstate->file_encoding,
440 : 213045 : cstate->raw_buf + preverifiedlen,
441 : : unverifiedlen);
442 [ - + ]: 213045 : if (nverified == 0)
443 : : {
444 : : /*
445 : : * Could not verify anything.
446 : : *
447 : : * If there is no more raw input data coming, it means that there
448 : : * was an incomplete multi-byte sequence at the end. Also, if
449 : : * there's "enough" input left, we should be able to verify at
450 : : * least one character, and a failure to do so means that we've
451 : : * hit an invalid byte sequence.
452 : : */
453 [ # # # # ]: 0 : if (cstate->raw_reached_eof || unverifiedlen >= pg_encoding_max_length(cstate->file_encoding))
454 : 0 : cstate->input_reached_error = true;
455 : 0 : return;
456 : : }
457 : 213045 : cstate->input_buf_len += nverified;
458 : : }
459 : : else
460 : : {
461 : : /*
462 : : * Encoding conversion is needed.
463 : : */
464 : : int nbytes;
465 : : unsigned char *src;
466 : : int srclen;
467 : : unsigned char *dst;
468 : : int dstlen;
469 : : int convertedlen;
470 : :
471 [ + + ]: 88 : if (RAW_BUF_BYTES(cstate) == 0)
472 : : {
473 : : /*
474 : : * If no more raw data is coming, report the EOF to the caller.
475 : : */
476 [ + + ]: 56 : if (cstate->raw_reached_eof)
477 : 16 : cstate->input_reached_eof = true;
478 : 56 : return;
479 : : }
480 : :
481 : : /*
482 : : * First, copy down any unprocessed data.
483 : : */
484 : 32 : nbytes = INPUT_BUF_BYTES(cstate);
485 [ - + - - ]: 32 : if (nbytes > 0 && cstate->input_buf_index > 0)
486 : 0 : memmove(cstate->input_buf, cstate->input_buf + cstate->input_buf_index,
487 : : nbytes);
488 : 32 : cstate->input_buf_index = 0;
489 : 32 : cstate->input_buf_len = nbytes;
490 : 32 : cstate->input_buf[nbytes] = '\0';
491 : :
492 : 32 : src = (unsigned char *) cstate->raw_buf + cstate->raw_buf_index;
493 : 32 : srclen = cstate->raw_buf_len - cstate->raw_buf_index;
494 : 32 : dst = (unsigned char *) cstate->input_buf + cstate->input_buf_len;
495 : 32 : dstlen = INPUT_BUF_SIZE - cstate->input_buf_len + 1;
496 : :
497 : : /*
498 : : * Do the conversion. This might stop short, if there is an invalid
499 : : * byte sequence in the input. We'll convert as much as we can in
500 : : * that case.
501 : : *
502 : : * Note: Even if we hit an invalid byte sequence, we don't report the
503 : : * error until all the valid bytes have been consumed. The input
504 : : * might contain an end-of-input marker (\.), and we don't want to
505 : : * report an error if the invalid byte sequence is after the
506 : : * end-of-input marker. We might unnecessarily convert some data
507 : : * after the end-of-input marker as long as it's valid for the
508 : : * encoding, but that's harmless.
509 : : */
510 : 32 : convertedlen = pg_do_encoding_conversion_buf(cstate->conversion_proc,
511 : : cstate->file_encoding,
512 : : GetDatabaseEncoding(),
513 : : src, srclen,
514 : : dst, dstlen,
515 : : true);
516 [ + + ]: 32 : if (convertedlen == 0)
517 : : {
518 : : /*
519 : : * Could not convert anything. If there is no more raw input data
520 : : * coming, it means that there was an incomplete multi-byte
521 : : * sequence at the end. Also, if there is plenty of input left,
522 : : * we should be able to convert at least one character, so a
523 : : * failure to do so must mean that we've hit a byte sequence
524 : : * that's invalid.
525 : : */
526 [ + + - + ]: 16 : if (cstate->raw_reached_eof || srclen >= MAX_CONVERSION_INPUT_LENGTH)
527 : 8 : cstate->input_reached_error = true;
528 : 16 : return;
529 : : }
530 : 16 : cstate->raw_buf_index += convertedlen;
531 : 16 : cstate->input_buf_len += strlen((char *) dst);
532 : : }
533 : : }
534 : :
535 : : /*
536 : : * Report an encoding or conversion error.
537 : : */
538 : : static void
539 : 8 : CopyConversionError(CopyFromState cstate)
540 : : {
541 : : Assert(cstate->raw_buf_len > 0);
542 : : Assert(cstate->input_reached_error);
543 : :
544 [ - + ]: 8 : if (!cstate->need_transcoding)
545 : : {
546 : : /*
547 : : * Everything up to input_buf_len was successfully verified, and
548 : : * input_buf_len points to the invalid or incomplete character.
549 : : */
550 : 0 : report_invalid_encoding(cstate->file_encoding,
551 : 0 : cstate->raw_buf + cstate->input_buf_len,
552 : 0 : cstate->raw_buf_len - cstate->input_buf_len);
553 : : }
554 : : else
555 : : {
556 : : /*
557 : : * raw_buf_index points to the invalid or untranslatable character. We
558 : : * let the conversion routine report the error, because it can provide
559 : : * a more specific error message than we could here. An earlier call
560 : : * to the conversion routine in CopyConvertBuf() detected that there
561 : : * is an error, now we call the conversion routine again with
562 : : * noError=false, to have it throw the error.
563 : : */
564 : : unsigned char *src;
565 : : int srclen;
566 : : unsigned char *dst;
567 : : int dstlen;
568 : :
569 : 8 : src = (unsigned char *) cstate->raw_buf + cstate->raw_buf_index;
570 : 8 : srclen = cstate->raw_buf_len - cstate->raw_buf_index;
571 : 8 : dst = (unsigned char *) cstate->input_buf + cstate->input_buf_len;
572 : 8 : dstlen = INPUT_BUF_SIZE - cstate->input_buf_len + 1;
573 : :
574 : 8 : (void) pg_do_encoding_conversion_buf(cstate->conversion_proc,
575 : : cstate->file_encoding,
576 : : GetDatabaseEncoding(),
577 : : src, srclen,
578 : : dst, dstlen,
579 : : false);
580 : :
581 : : /*
582 : : * The conversion routine should have reported an error, so this
583 : : * should not be reached.
584 : : */
585 [ # # ]: 0 : elog(ERROR, "encoding conversion failed without error");
586 : : }
587 : : }
588 : :
589 : : /*
590 : : * Load more data from data source to raw_buf.
591 : : *
592 : : * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the
593 : : * beginning of the buffer, and we load new data after that.
594 : : */
595 : : static void
596 : 214075 : CopyLoadRawBuf(CopyFromState cstate)
597 : : {
598 : : int nbytes;
599 : : int inbytes;
600 : :
601 : : /*
602 : : * In text mode, if encoding conversion is not required, raw_buf and
603 : : * input_buf point to the same buffer. Their len/index better agree, too.
604 : : */
605 : 214075 : if (cstate->raw_buf == cstate->input_buf)
606 : : {
607 : : Assert(!cstate->need_transcoding);
608 : : Assert(cstate->raw_buf_index == cstate->input_buf_index);
609 : : Assert(cstate->input_buf_len <= cstate->raw_buf_len);
610 : : }
611 : :
612 : : /*
613 : : * Copy down the unprocessed data if any.
614 : : */
615 : 214075 : nbytes = RAW_BUF_BYTES(cstate);
616 [ + + + + ]: 214075 : if (nbytes > 0 && cstate->raw_buf_index > 0)
617 : 617 : memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
618 : : nbytes);
619 : 214075 : cstate->raw_buf_len -= cstate->raw_buf_index;
620 : 214075 : cstate->raw_buf_index = 0;
621 : :
622 : : /*
623 : : * If raw_buf and input_buf are in fact the same buffer, adjust the
624 : : * input_buf variables, too.
625 : : */
626 [ + + ]: 214075 : if (cstate->raw_buf == cstate->input_buf)
627 : : {
628 : 214007 : cstate->input_buf_len -= cstate->input_buf_index;
629 : 214007 : cstate->input_buf_index = 0;
630 : : }
631 : :
632 : : /* Load more data */
633 : 214075 : inbytes = CopyGetData(cstate, cstate->raw_buf + cstate->raw_buf_len,
634 : 214075 : 1, RAW_BUF_SIZE - cstate->raw_buf_len);
635 : 214073 : nbytes += inbytes;
636 : 214073 : cstate->raw_buf[nbytes] = '\0';
637 : 214073 : cstate->raw_buf_len = nbytes;
638 : :
639 : 214073 : cstate->bytes_processed += inbytes;
640 : 214073 : pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
641 : :
642 [ + + ]: 214073 : if (inbytes == 0)
643 : 991 : cstate->raw_reached_eof = true;
644 : 214073 : }
645 : :
646 : : /*
647 : : * CopyLoadInputBuf loads some more data into input_buf
648 : : *
649 : : * On return, at least one more input character is loaded into
650 : : * input_buf, or input_reached_eof is set.
651 : : *
652 : : * If INPUT_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
653 : : * of the buffer and then we load more data after that.
654 : : *
655 : : * If "speculative" is true, this function skips reporting any encoding or
656 : : * conversion errors, provided there are still data for the caller to process.
657 : : * Such callers must be prepared for this function to return without loading
658 : : * anything.
659 : : */
660 : : static void
661 : 214363 : CopyLoadInputBuf(CopyFromState cstate, bool speculative)
662 : : {
663 : 214363 : int nbytes = INPUT_BUF_BYTES(cstate);
664 : :
665 : : /*
666 : : * The caller has updated input_buf_index to indicate how much of the
667 : : * input has been consumed and isn't needed anymore. If input_buf is the
668 : : * same physical area as raw_buf, update raw_buf_index accordingly.
669 : : */
670 [ + + ]: 214363 : if (cstate->raw_buf == cstate->input_buf)
671 : : {
672 : : Assert(!cstate->need_transcoding);
673 : : Assert(cstate->input_buf_index >= cstate->raw_buf_index);
674 : 214323 : cstate->raw_buf_index = cstate->input_buf_index;
675 : : }
676 : :
677 : : for (;;)
678 : : {
679 : : /* If we now have some unconverted data, try to convert it */
680 : 428416 : CopyConvertBuf(cstate);
681 : :
682 : : /* If we now have some more input bytes ready, return them */
683 [ + + ]: 428416 : if (INPUT_BUF_BYTES(cstate) > nbytes)
684 : 213061 : return;
685 : :
686 : : /*
687 : : * If we reached an invalid byte sequence, or we're at an incomplete
688 : : * multi-byte character but there is no more raw input data, report
689 : : * conversion error. As an exception, if "speculative" is true and
690 : : * there are still data for the caller to process, just return
691 : : * instead.
692 : : */
693 [ + + ]: 215355 : if (cstate->input_reached_error)
694 : : {
695 [ + - - + ]: 8 : if (speculative && INPUT_BUF_BYTES(cstate) > 0)
696 : 0 : return;
697 : 8 : CopyConversionError(cstate);
698 : : }
699 : :
700 : : /* no more input, and everything has been converted */
701 [ + + ]: 215347 : if (cstate->input_reached_eof)
702 : 1292 : break;
703 : :
704 : : /* Try to load more raw data */
705 : : Assert(!cstate->raw_reached_eof);
706 : 214055 : CopyLoadRawBuf(cstate);
707 : : }
708 : : }
709 : :
710 : : /*
711 : : * CopyReadBinaryData
712 : : *
713 : : * Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf
714 : : * and writes them to 'dest'. Returns the number of bytes read (which
715 : : * would be less than 'nbytes' only if we reach EOF).
716 : : */
717 : : static int
718 : 236 : CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
719 : : {
720 : 236 : int copied_bytes = 0;
721 : :
722 [ + + ]: 236 : if (RAW_BUF_BYTES(cstate) >= nbytes)
723 : : {
724 : : /* Enough bytes are present in the buffer. */
725 : 216 : memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, nbytes);
726 : 216 : cstate->raw_buf_index += nbytes;
727 : 216 : copied_bytes = nbytes;
728 : : }
729 : : else
730 : : {
731 : : /*
732 : : * Not enough bytes in the buffer, so must read from the file. Need
733 : : * to loop since 'nbytes' could be larger than the buffer size.
734 : : */
735 : : do
736 : : {
737 : : int copy_bytes;
738 : :
739 : : /* Load more data if buffer is empty. */
740 [ + - ]: 20 : if (RAW_BUF_BYTES(cstate) == 0)
741 : : {
742 : 20 : CopyLoadRawBuf(cstate);
743 [ + + ]: 20 : if (cstate->raw_reached_eof)
744 : 7 : break; /* EOF */
745 : : }
746 : :
747 : : /* Transfer some bytes. */
748 : 13 : copy_bytes = Min(nbytes - copied_bytes, RAW_BUF_BYTES(cstate));
749 : 13 : memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, copy_bytes);
750 : 13 : cstate->raw_buf_index += copy_bytes;
751 : 13 : dest += copy_bytes;
752 : 13 : copied_bytes += copy_bytes;
753 [ - + ]: 13 : } while (copied_bytes < nbytes);
754 : : }
755 : :
756 : 236 : return copied_bytes;
757 : : }
758 : :
759 : : /*
760 : : * This function is exposed for use by extensions that read raw fields in the
761 : : * next line. See NextCopyFromRawFieldsInternal() for details.
762 : : */
763 : : bool
764 : 0 : NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields)
765 : : {
766 : 0 : return NextCopyFromRawFieldsInternal(cstate, fields, nfields,
767 : 0 : cstate->opts.format == COPY_FORMAT_CSV);
768 : : }
769 : :
770 : : /*
771 : : * Workhorse for NextCopyFromRawFields().
772 : : *
773 : : * Read raw fields in the next line for COPY FROM in text or csv mode. Return
774 : : * false if no more lines.
775 : : *
776 : : * An internal temporary buffer is returned via 'fields'. It is valid until
777 : : * the next call of the function. Since the function returns all raw fields
778 : : * in the input file, 'nfields' could be different from the number of columns
779 : : * in the relation.
780 : : *
781 : : * NOTE: force_not_null option are not applied to the returned fields.
782 : : *
783 : : * We use pg_always_inline to reduce function call overhead
784 : : * and to help compilers to optimize away the 'is_csv' condition when called
785 : : * by internal functions such as CopyFromTextLikeOneRow().
786 : : */
787 : : static pg_always_inline bool
788 : 755948 : NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields, bool is_csv)
789 : : {
790 : : int fldct;
791 : 755948 : bool done = false;
792 : :
793 : : /* only available for text or csv input */
794 : : Assert(cstate->opts.format == COPY_FORMAT_TEXT ||
795 : : cstate->opts.format == COPY_FORMAT_CSV);
796 : :
797 : : /* on input check that the header line is correct if needed */
798 [ + + + + ]: 755948 : if (cstate->cur_lineno == 0 && cstate->opts.header_line != COPY_HEADER_FALSE)
799 : : {
800 : : ListCell *cur;
801 : : TupleDesc tupDesc;
802 : 93 : int lines_to_skip = cstate->opts.header_line;
803 : :
804 : : /* If set to "match", one header line is skipped */
805 [ + + ]: 93 : if (cstate->opts.header_line == COPY_HEADER_MATCH)
806 : 50 : lines_to_skip = 1;
807 : :
808 : 93 : tupDesc = RelationGetDescr(cstate->rel);
809 : :
810 [ + + ]: 218 : for (int i = 0; i < lines_to_skip; i++)
811 : : {
812 : 130 : cstate->cur_lineno++;
813 [ + + ]: 130 : if ((done = CopyReadLine(cstate, is_csv)))
814 : 5 : break;
815 : : }
816 : :
817 [ + + ]: 93 : if (cstate->opts.header_line == COPY_HEADER_MATCH)
818 : : {
819 : : int fldnum;
820 : :
821 [ + + ]: 50 : if (is_csv)
822 : 6 : fldct = CopyReadAttributesCSV(cstate);
823 : : else
824 : 44 : fldct = CopyReadAttributesText(cstate);
825 : :
826 [ + + ]: 50 : if (fldct != list_length(cstate->attnumlist))
827 [ + - ]: 16 : ereport(ERROR,
828 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
829 : : errmsg("wrong number of fields in header line: got %d, expected %d",
830 : : fldct, list_length(cstate->attnumlist))));
831 : :
832 : 34 : fldnum = 0;
833 [ + - + + : 104 : foreach(cur, cstate->attnumlist)
+ + ]
834 : : {
835 : 83 : int attnum = lfirst_int(cur);
836 : : char *colName;
837 : 83 : Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
838 : :
839 : : Assert(fldnum < cstate->max_fields);
840 : :
841 : 83 : colName = cstate->raw_fields[fldnum++];
842 [ + + ]: 83 : if (colName == NULL)
843 [ + - ]: 4 : ereport(ERROR,
844 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
845 : : errmsg("column name mismatch in header line field %d: got null value (\"%s\"), expected \"%s\"",
846 : : fldnum, cstate->opts.null_print, NameStr(attr->attname))));
847 : :
848 [ + + ]: 79 : if (namestrcmp(&attr->attname, colName) != 0)
849 : : {
850 [ + - ]: 9 : ereport(ERROR,
851 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
852 : : errmsg("column name mismatch in header line field %d: got \"%s\", expected \"%s\"",
853 : : fldnum, colName, NameStr(attr->attname))));
854 : : }
855 : : }
856 : : }
857 : :
858 [ + + ]: 64 : if (done)
859 : 5 : return false;
860 : : }
861 : :
862 : 755914 : cstate->cur_lineno++;
863 : :
864 : : /* Actually read the line into memory here */
865 : 755914 : done = CopyReadLine(cstate, is_csv);
866 : :
867 : : /*
868 : : * EOF at start of line means we're done. If we see EOF after some
869 : : * characters, we act as though it was newline followed by EOF, ie,
870 : : * process the line and then exit loop on next iteration.
871 : : */
872 [ + + + + ]: 755896 : if (done && cstate->line_buf.len == 0)
873 : 997 : return false;
874 : :
875 : : /* Parse the line into de-escaped field values */
876 [ + + ]: 754899 : if (is_csv)
877 : 312 : fldct = CopyReadAttributesCSV(cstate);
878 : : else
879 : 754587 : fldct = CopyReadAttributesText(cstate);
880 : :
881 : 754891 : *fields = cstate->raw_fields;
882 : 754891 : *nfields = fldct;
883 : 754891 : return true;
884 : : }
885 : :
886 : : /*
887 : : * Read next tuple from file for COPY FROM. Return false if no more tuples.
888 : : *
889 : : * 'econtext' is used to evaluate default expression for each column that is
890 : : * either not read from the file or is using the DEFAULT option of COPY FROM.
891 : : * It can be NULL when no default values are used, i.e. when all columns are
892 : : * read from the file, and DEFAULT option is unset.
893 : : *
894 : : * 'values' and 'nulls' arrays must be the same length as columns of the
895 : : * relation passed to BeginCopyFrom. This function fills the arrays.
896 : : */
897 : : bool
898 : 755973 : NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
899 : : Datum *values, bool *nulls)
900 : : {
901 : : TupleDesc tupDesc;
902 : : AttrNumber num_phys_attrs,
903 : 755973 : num_defaults = cstate->num_defaults;
904 : : int i;
905 : 755973 : int *defmap = cstate->defmap;
906 : 755973 : ExprState **defexprs = cstate->defexprs;
907 : :
908 : 755973 : tupDesc = RelationGetDescr(cstate->rel);
909 : 755973 : num_phys_attrs = tupDesc->natts;
910 : :
911 : : /* Initialize all values for row to NULL */
912 [ + - + - : 3560559 : MemSet(values, 0, num_phys_attrs * sizeof(Datum));
+ - + - +
+ ]
913 [ + - + + : 755973 : MemSet(nulls, true, num_phys_attrs * sizeof(bool));
- + - - -
- ]
914 [ + - + + : 852045 : MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool));
+ - + - +
+ ]
915 : :
916 : : /* Get one row from source */
917 [ + + ]: 755973 : if (!cstate->routine->CopyFromOneRow(cstate, econtext, values, nulls))
918 : 1009 : return false;
919 : :
920 : : /*
921 : : * Now compute and insert any defaults available for the columns not
922 : : * provided by the input data. Anything not processed here or above will
923 : : * remain NULL.
924 : : */
925 [ + + ]: 795180 : for (i = 0; i < num_defaults; i++)
926 : : {
927 : : /*
928 : : * The caller must supply econtext and have switched into the
929 : : * per-tuple memory context in it.
930 : : */
931 : : Assert(econtext != NULL);
932 : : Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
933 : :
934 : 40345 : values[defmap[i]] = ExecEvalExpr(defexprs[defmap[i]], econtext,
935 : 40345 : &nulls[defmap[i]]);
936 : : }
937 : :
938 : 754835 : return true;
939 : : }
940 : :
941 : : /* Implementation of the per-row callback for text format */
942 : : bool
943 : 755478 : CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
944 : : bool *nulls)
945 : : {
946 : 755478 : return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, false);
947 : : }
948 : :
949 : : /* Implementation of the per-row callback for CSV format */
950 : : bool
951 : 470 : CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
952 : : bool *nulls)
953 : : {
954 : 470 : return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, true);
955 : : }
956 : :
957 : : /*
958 : : * Workhorse for CopyFromTextOneRow() and CopyFromCSVOneRow().
959 : : *
960 : : * We use pg_always_inline to reduce function call overhead
961 : : * and to help compilers to optimize away the 'is_csv' condition.
962 : : */
963 : : static pg_always_inline bool
964 : 755948 : CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
965 : : Datum *values, bool *nulls, bool is_csv)
966 : : {
967 : : TupleDesc tupDesc;
968 : : AttrNumber attr_count;
969 : 755948 : FmgrInfo *in_functions = cstate->in_functions;
970 : 755948 : Oid *typioparams = cstate->typioparams;
971 : 755948 : ExprState **defexprs = cstate->defexprs;
972 : : char **field_strings;
973 : : ListCell *cur;
974 : : int fldct;
975 : : int fieldno;
976 : : char *string;
977 : 755948 : bool current_row_erroneous = false;
978 : :
979 : 755948 : tupDesc = RelationGetDescr(cstate->rel);
980 : 755948 : attr_count = list_length(cstate->attnumlist);
981 : :
982 : : /* read raw fields in the next line */
983 [ + + ]: 755948 : if (!NextCopyFromRawFieldsInternal(cstate, &field_strings, &fldct, is_csv))
984 : 1002 : return false;
985 : :
986 : : /* check for overflowing fields */
987 [ + + + + ]: 754891 : if (attr_count > 0 && fldct > attr_count)
988 [ + - ]: 16 : ereport(ERROR,
989 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
990 : : errmsg("extra data after last expected column")));
991 : :
992 : 754875 : fieldno = 0;
993 : :
994 : : /* Loop to read the user attributes on the line. */
995 [ + + + + : 3468936 : foreach(cur, cstate->attnumlist)
+ + ]
996 : : {
997 : 2714200 : int attnum = lfirst_int(cur);
998 : 2714200 : int m = attnum - 1;
999 : 2714200 : Form_pg_attribute att = TupleDescAttr(tupDesc, m);
1000 : :
1001 [ + + ]: 2714200 : if (fieldno >= fldct)
1002 [ + - ]: 16 : ereport(ERROR,
1003 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1004 : : errmsg("missing data for column \"%s\"",
1005 : : NameStr(att->attname))));
1006 : 2714184 : string = field_strings[fieldno++];
1007 : :
1008 [ + + ]: 2714184 : if (cstate->convert_select_flags &&
1009 [ + + ]: 10 : !cstate->convert_select_flags[m])
1010 : : {
1011 : : /* ignore input field, leaving column as NULL */
1012 : 5 : continue;
1013 : : }
1014 : :
1015 [ + + ]: 2714179 : if (is_csv)
1016 : : {
1017 [ + + ]: 619 : if (string == NULL &&
1018 [ + + ]: 27 : cstate->opts.force_notnull_flags[m])
1019 : : {
1020 : : /*
1021 : : * FORCE_NOT_NULL option is set and column is NULL - convert
1022 : : * it to the NULL string.
1023 : : */
1024 : 18 : string = cstate->opts.null_print;
1025 : : }
1026 [ + + + + ]: 601 : else if (string != NULL && cstate->opts.force_null_flags[m]
1027 [ + + ]: 32 : && strcmp(string, cstate->opts.null_print) == 0)
1028 : : {
1029 : : /*
1030 : : * FORCE_NULL option is set and column matches the NULL
1031 : : * string. It must have been quoted, or otherwise the string
1032 : : * would already have been set to NULL. Convert it to NULL as
1033 : : * specified.
1034 : : */
1035 : 17 : string = NULL;
1036 : : }
1037 : : }
1038 : :
1039 : 2714179 : cstate->cur_attname = NameStr(att->attname);
1040 : 2714179 : cstate->cur_attval = string;
1041 : :
1042 [ + + ]: 2714179 : if (string != NULL)
1043 : 2711423 : nulls[m] = false;
1044 : :
1045 [ + + ]: 2714179 : if (cstate->defaults[m])
1046 : : {
1047 : : /* We must have switched into the per-tuple memory context */
1048 : : Assert(econtext != NULL);
1049 : : Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
1050 : :
1051 : 38 : values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]);
1052 : : }
1053 : :
1054 : : /*
1055 : : * If ON_ERROR is specified, handle the different options
1056 : : */
1057 [ + + ]: 2714116 : else if (!InputFunctionCallSafe(&in_functions[m],
1058 : : string,
1059 : 2714141 : typioparams[m],
1060 : : att->atttypmod,
1061 : 2714141 : (Node *) cstate->escontext,
1062 : 2714141 : &values[m]))
1063 : : {
1064 : : Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP);
1065 : :
1066 [ + + ]: 116 : if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE)
1067 : 82 : cstate->num_errors++;
1068 [ + - ]: 34 : else if (cstate->opts.on_error == COPY_ON_ERROR_SET_NULL)
1069 : : {
1070 : : /*
1071 : : * Reset error state so the subsequent InputFunctionCallSafe
1072 : : * call (for domain constraint check) can properly report
1073 : : * whether it succeeded or failed.
1074 : : */
1075 : 34 : cstate->escontext->error_occurred = false;
1076 : :
1077 : : Assert(cstate->domain_with_constraint != NULL);
1078 : :
1079 : : /*
1080 : : * For constrained domains, we need an additional
1081 : : * InputFunctionCallSafe() to ensure that an error is thrown
1082 : : * if the domain constraint rejects null values.
1083 : : */
1084 [ + + + + ]: 58 : if (!cstate->domain_with_constraint[m] ||
1085 : 24 : InputFunctionCallSafe(&in_functions[m],
1086 : : NULL,
1087 : 24 : typioparams[m],
1088 : : att->atttypmod,
1089 : 24 : (Node *) cstate->escontext,
1090 : 24 : &values[m]))
1091 : : {
1092 : 18 : nulls[m] = true;
1093 : 18 : values[m] = (Datum) 0;
1094 : : }
1095 : : else
1096 [ + - ]: 16 : ereport(ERROR,
1097 : : errcode(ERRCODE_NOT_NULL_VIOLATION),
1098 : : errmsg("domain %s does not allow null values",
1099 : : format_type_be(typioparams[m])),
1100 : : errdetail("ON_ERROR SET_NULL cannot be applied because column \"%s\" (domain %s) does not accept null values.",
1101 : : cstate->cur_attname,
1102 : : format_type_be(typioparams[m])),
1103 : : errdatatype(typioparams[m]));
1104 : :
1105 : : /*
1106 : : * We count only the number of rows (not fields) where
1107 : : * ON_ERROR SET_NULL was applied.
1108 : : */
1109 [ + + ]: 18 : if (!current_row_erroneous)
1110 : : {
1111 : 14 : current_row_erroneous = true;
1112 : 14 : cstate->num_errors++;
1113 : : }
1114 : : }
1115 : :
1116 [ + + ]: 100 : if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE)
1117 : : {
1118 : : /*
1119 : : * Since we emit line number and column info in the below
1120 : : * notice message, we suppress error context information other
1121 : : * than the relation name.
1122 : : */
1123 : : Assert(!cstate->relname_only);
1124 : 44 : cstate->relname_only = true;
1125 : :
1126 [ + + ]: 44 : if (cstate->cur_attval)
1127 : : {
1128 : : char *attval;
1129 : :
1130 : 40 : attval = CopyLimitPrintoutLength(cstate->cur_attval);
1131 : :
1132 [ + + ]: 40 : if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE)
1133 [ + - ]: 24 : ereport(NOTICE,
1134 : : errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"",
1135 : : cstate->cur_lineno,
1136 : : cstate->cur_attname,
1137 : : attval));
1138 [ + - ]: 16 : else if (cstate->opts.on_error == COPY_ON_ERROR_SET_NULL)
1139 [ + - ]: 16 : ereport(NOTICE,
1140 : : errmsg("setting to null due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"",
1141 : : cstate->cur_lineno,
1142 : : cstate->cur_attname,
1143 : : attval));
1144 : 40 : pfree(attval);
1145 : : }
1146 : : else
1147 : : {
1148 [ + - ]: 4 : if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE)
1149 [ + - ]: 4 : ereport(NOTICE,
1150 : : errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": null input",
1151 : : cstate->cur_lineno,
1152 : : cstate->cur_attname));
1153 : : }
1154 : : /* reset relname_only */
1155 : 44 : cstate->relname_only = false;
1156 : : }
1157 : :
1158 [ + + ]: 100 : if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE)
1159 : 82 : return true;
1160 [ + - ]: 18 : else if (cstate->opts.on_error == COPY_ON_ERROR_SET_NULL)
1161 : 18 : continue;
1162 : : }
1163 : :
1164 : 2714038 : cstate->cur_attname = NULL;
1165 : 2714038 : cstate->cur_attval = NULL;
1166 : : }
1167 : :
1168 : : Assert(fieldno == attr_count);
1169 : :
1170 : 754736 : return true;
1171 : : }
1172 : :
1173 : : /* Implementation of the per-row callback for binary format */
1174 : : bool
1175 : 25 : CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
1176 : : bool *nulls)
1177 : : {
1178 : : TupleDesc tupDesc;
1179 : : AttrNumber attr_count;
1180 : 25 : FmgrInfo *in_functions = cstate->in_functions;
1181 : 25 : Oid *typioparams = cstate->typioparams;
1182 : : int16 fld_count;
1183 : : ListCell *cur;
1184 : :
1185 : 25 : tupDesc = RelationGetDescr(cstate->rel);
1186 : 25 : attr_count = list_length(cstate->attnumlist);
1187 : :
1188 : 25 : cstate->cur_lineno++;
1189 : :
1190 [ - + ]: 25 : if (!CopyGetInt16(cstate, &fld_count))
1191 : : {
1192 : : /* EOF detected (end of file, or protocol-level EOF) */
1193 : 0 : return false;
1194 : : }
1195 : :
1196 [ + + ]: 25 : if (fld_count == -1)
1197 : : {
1198 : : /*
1199 : : * Received EOF marker. Wait for the protocol-level EOF, and complain
1200 : : * if it doesn't come immediately. In COPY FROM STDIN, this ensures
1201 : : * that we correctly handle CopyFail, if client chooses to send that
1202 : : * now. When copying from file, we could ignore the rest of the file
1203 : : * like in text mode, but we choose to be consistent with the COPY
1204 : : * FROM STDIN case.
1205 : : */
1206 : : char dummy;
1207 : :
1208 [ - + ]: 7 : if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
1209 [ # # ]: 0 : ereport(ERROR,
1210 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1211 : : errmsg("received copy data after EOF marker")));
1212 : 7 : return false;
1213 : : }
1214 : :
1215 [ - + ]: 18 : if (fld_count != attr_count)
1216 [ # # ]: 0 : ereport(ERROR,
1217 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1218 : : errmsg("row field count is %d, expected %d",
1219 : : fld_count, attr_count)));
1220 : :
1221 [ + - + + : 117 : foreach(cur, cstate->attnumlist)
+ + ]
1222 : : {
1223 : 100 : int attnum = lfirst_int(cur);
1224 : 100 : int m = attnum - 1;
1225 : 100 : Form_pg_attribute att = TupleDescAttr(tupDesc, m);
1226 : :
1227 : 100 : cstate->cur_attname = NameStr(att->attname);
1228 : 199 : values[m] = CopyReadBinaryAttribute(cstate,
1229 : 100 : &in_functions[m],
1230 : 100 : typioparams[m],
1231 : : att->atttypmod,
1232 : : &nulls[m]);
1233 : 99 : cstate->cur_attname = NULL;
1234 : : }
1235 : :
1236 : 17 : return true;
1237 : : }
1238 : :
1239 : : /*
1240 : : * Read the next input line and stash it in line_buf.
1241 : : *
1242 : : * Result is true if read was terminated by EOF, false if terminated
1243 : : * by newline. The terminating newline or EOF marker is not included
1244 : : * in the final value of line_buf.
1245 : : */
1246 : : static bool
1247 : 756044 : CopyReadLine(CopyFromState cstate, bool is_csv)
1248 : : {
1249 : : bool result;
1250 : :
1251 : 756044 : resetStringInfo(&cstate->line_buf);
1252 : 756044 : cstate->line_buf_valid = false;
1253 : :
1254 : : /*
1255 : : * Parse data and transfer into line_buf.
1256 : : *
1257 : : * Because this is performance critical, we inline CopyReadLineText() and
1258 : : * pass the boolean parameters as constants to allow the compiler to emit
1259 : : * specialized code with fewer branches.
1260 : : */
1261 [ + + ]: 756044 : if (is_csv)
1262 : 546 : result = CopyReadLineText(cstate, true);
1263 : : else
1264 : 755498 : result = CopyReadLineText(cstate, false);
1265 : :
1266 [ + + ]: 756026 : if (result)
1267 : : {
1268 : : /*
1269 : : * Reached EOF. In protocol version 3, we should ignore anything
1270 : : * after \. up to the protocol end of copy data. (XXX maybe better
1271 : : * not to treat \. as special?)
1272 : : */
1273 [ + + ]: 1005 : if (cstate->copy_src == COPY_FRONTEND)
1274 : : {
1275 : : int inbytes;
1276 : :
1277 : : do
1278 : : {
1279 : 529 : inbytes = CopyGetData(cstate, cstate->input_buf,
1280 : : 1, INPUT_BUF_SIZE);
1281 [ - + ]: 529 : } while (inbytes > 0);
1282 : 529 : cstate->input_buf_index = 0;
1283 : 529 : cstate->input_buf_len = 0;
1284 : 529 : cstate->raw_buf_index = 0;
1285 : 529 : cstate->raw_buf_len = 0;
1286 : : }
1287 : : }
1288 : : else
1289 : : {
1290 : : /*
1291 : : * If we didn't hit EOF, then we must have transferred the EOL marker
1292 : : * to line_buf along with the data. Get rid of it.
1293 : : */
1294 [ + - - - : 755021 : switch (cstate->eol_type)
- ]
1295 : : {
1296 : 755021 : case EOL_NL:
1297 : : Assert(cstate->line_buf.len >= 1);
1298 : : Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
1299 : 755021 : cstate->line_buf.len--;
1300 : 755021 : cstate->line_buf.data[cstate->line_buf.len] = '\0';
1301 : 755021 : break;
1302 : 0 : case EOL_CR:
1303 : : Assert(cstate->line_buf.len >= 1);
1304 : : Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
1305 : 0 : cstate->line_buf.len--;
1306 : 0 : cstate->line_buf.data[cstate->line_buf.len] = '\0';
1307 : 0 : break;
1308 : 0 : case EOL_CRNL:
1309 : : Assert(cstate->line_buf.len >= 2);
1310 : : Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
1311 : : Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
1312 : 0 : cstate->line_buf.len -= 2;
1313 : 0 : cstate->line_buf.data[cstate->line_buf.len] = '\0';
1314 : 0 : break;
1315 : 0 : case EOL_UNKNOWN:
1316 : : /* shouldn't get here */
1317 : : Assert(false);
1318 : 0 : break;
1319 : : }
1320 : : }
1321 : :
1322 : : /* Now it's safe to use the buffer in error messages */
1323 : 756026 : cstate->line_buf_valid = true;
1324 : :
1325 : 756026 : return result;
1326 : : }
1327 : :
1328 : : #ifndef USE_NO_SIMD
1329 : : /*
1330 : : * Helper function for CopyReadLineText() that uses SIMD instructions to scan
1331 : : * the input buffer for special characters. This can be much faster.
1332 : : *
1333 : : * Note that we disable SIMD for the remainder of the COPY FROM command upon
1334 : : * encountering a special character (except for end-of-line characters) or a
1335 : : * short line. This is perhaps too conservative, but it should help avoid
1336 : : * regressions. It could probably be made more lenient in the future via
1337 : : * fine-tuned heuristics.
1338 : : */
1339 : : static bool
1340 : 333485 : CopyReadLineTextSIMDHelper(CopyFromState cstate, bool is_csv,
1341 : : bool *hit_eof_p, int *input_buf_ptr_p)
1342 : : {
1343 : : char *copy_input_buf;
1344 : : int input_buf_ptr;
1345 : : int copy_buf_len;
1346 : : bool unique_esc_char; /* for csv, do quote/esc chars differ? */
1347 : 333485 : bool first = true;
1348 : 333485 : bool result = false;
1349 : 333485 : const Vector8 nl_vec = vector8_broadcast('\n');
1350 : 333485 : const Vector8 cr_vec = vector8_broadcast('\r');
1351 : : Vector8 bs_or_quote_vec; /* '\' for text, quote for csv */
1352 : : Vector8 esc_vec; /* only for csv */
1353 : :
1354 [ + + ]: 333485 : if (is_csv)
1355 : : {
1356 : 392 : char quote = cstate->opts.quote[0];
1357 : 392 : char esc = cstate->opts.escape[0];
1358 : :
1359 : 392 : bs_or_quote_vec = vector8_broadcast(quote);
1360 : 392 : esc_vec = vector8_broadcast(esc);
1361 : 392 : unique_esc_char = (quote != esc);
1362 : : }
1363 : : else
1364 : : {
1365 : 333093 : bs_or_quote_vec = vector8_broadcast('\\');
1366 : 333093 : unique_esc_char = false;
1367 : : }
1368 : :
1369 : : /*
1370 : : * For a little extra speed within the loop, we copy some state members
1371 : : * into local variables. Note that we need to use a separate local
1372 : : * variable for input_buf_ptr so that the REFILL_LINEBUF macro works. We
1373 : : * copy its value into the input_buf_ptr_p argument before returning.
1374 : : */
1375 : 333485 : copy_input_buf = cstate->input_buf;
1376 : 333485 : input_buf_ptr = cstate->input_buf_index;
1377 : 333485 : copy_buf_len = cstate->input_buf_len;
1378 : :
1379 : : /*
1380 : : * See the corresponding loop in CopyReadLineText() for more information
1381 : : * about the purpose of this loop. This one does the same thing using
1382 : : * SIMD instructions, although we are quick to bail out to the scalar path
1383 : : * if we encounter a special character.
1384 : : */
1385 : : for (;;)
1386 : 400471 : {
1387 : : Vector8 chunk;
1388 : : Vector8 match;
1389 : :
1390 : : /* Load more data if needed. */
1391 [ + + ]: 733956 : if (copy_buf_len - input_buf_ptr < sizeof(Vector8))
1392 : : {
1393 [ + + ]: 213871 : REFILL_LINEBUF;
1394 : :
1395 : 213871 : CopyLoadInputBuf(cstate, true);
1396 : : /* update our local variables */
1397 : 213861 : *hit_eof_p = cstate->input_reached_eof;
1398 : 213861 : input_buf_ptr = cstate->input_buf_index;
1399 : 213861 : copy_buf_len = cstate->input_buf_len;
1400 : :
1401 : : /*
1402 : : * If we are completely out of data, break out of the loop,
1403 : : * reporting EOF.
1404 : : */
1405 [ + + ]: 213861 : if (INPUT_BUF_BYTES(cstate) <= 0)
1406 : : {
1407 : 600 : result = true;
1408 : 600 : break;
1409 : : }
1410 : : }
1411 : :
1412 : : /*
1413 : : * If we still don't have enough data for the SIMD path, fall back to
1414 : : * the scalar code. Note that this doesn't necessarily mean we
1415 : : * encountered a short line, so we leave cstate->simd_enabled set to
1416 : : * true.
1417 : : */
1418 [ + + ]: 733346 : if (copy_buf_len - input_buf_ptr < sizeof(Vector8))
1419 : 212372 : break;
1420 : :
1421 : : /*
1422 : : * If we made it here, we have at least enough data to fit in a
1423 : : * Vector8, so we can use SIMD instructions to scan for special
1424 : : * characters.
1425 : : */
1426 : 520974 : vector8_load(&chunk, (const uint8 *) ©_input_buf[input_buf_ptr]);
1427 : :
1428 : : /*
1429 : : * Check for \n, \r, \\ (for text), quotes (for csv), and escapes (for
1430 : : * csv, if different from quotes).
1431 : : */
1432 : 520974 : match = vector8_eq(chunk, nl_vec);
1433 : 520974 : match = vector8_or(match, vector8_eq(chunk, cr_vec));
1434 : 520974 : match = vector8_or(match, vector8_eq(chunk, bs_or_quote_vec));
1435 [ + + ]: 520974 : if (unique_esc_char)
1436 : 21 : match = vector8_or(match, vector8_eq(chunk, esc_vec));
1437 : :
1438 : : /*
1439 : : * If we found a special character, advance to it and hand off to the
1440 : : * scalar path. Except for end-of-line characters, we also disable
1441 : : * SIMD processing for the remainder of the COPY FROM command.
1442 : : */
1443 [ + + ]: 520974 : if (vector8_is_highbit_set(match))
1444 : : {
1445 : : uint32 mask;
1446 : : char c;
1447 : :
1448 : 120503 : mask = vector8_highbit_mask(match);
1449 : 120503 : input_buf_ptr += pg_rightmost_one_pos32(mask);
1450 : :
1451 : : /*
1452 : : * Don't disable SIMD if we found \n or \r, else we'd stop using
1453 : : * SIMD instructions after the first line. As an exception, we do
1454 : : * disable it if this is the first vector we processed, as that
1455 : : * means the line is too short for SIMD.
1456 : : */
1457 : 120503 : c = copy_input_buf[input_buf_ptr];
1458 [ + + + + : 120503 : if (first || (c != '\n' && c != '\r'))
+ - ]
1459 : 392 : cstate->simd_enabled = false;
1460 : :
1461 : 120503 : break;
1462 : : }
1463 : :
1464 : : /* That chunk was clear of special characters, so we can skip it. */
1465 : 400471 : input_buf_ptr += sizeof(Vector8);
1466 : 400471 : first = false;
1467 : : }
1468 : :
1469 : 333475 : *input_buf_ptr_p = input_buf_ptr;
1470 : 333475 : return result;
1471 : : }
1472 : : #endif /* ! USE_NO_SIMD */
1473 : :
1474 : : /*
1475 : : * CopyReadLineText - inner loop of CopyReadLine for text mode
1476 : : */
1477 : : static pg_always_inline bool
1478 : 756044 : CopyReadLineText(CopyFromState cstate, bool is_csv)
1479 : : {
1480 : : char *copy_input_buf;
1481 : : int input_buf_ptr;
1482 : : int copy_buf_len;
1483 : 756044 : bool need_data = false;
1484 : 756044 : bool hit_eof = false;
1485 : 756044 : bool result = false;
1486 : :
1487 : : /* CSV variables */
1488 : 756044 : bool in_quote = false,
1489 : 756044 : last_was_esc = false;
1490 : 756044 : char quotec = '\0';
1491 : 756044 : char escapec = '\0';
1492 : :
1493 [ + + ]: 756044 : if (is_csv)
1494 : : {
1495 : 546 : quotec = cstate->opts.quote[0];
1496 : 546 : escapec = cstate->opts.escape[0];
1497 : : /* ignore special escape processing if it's the same as quotec */
1498 [ + + ]: 546 : if (quotec == escapec)
1499 : 438 : escapec = '\0';
1500 : : }
1501 : :
1502 : : /*
1503 : : * The objective of this loop is to transfer the entire next input line
1504 : : * into line_buf. Hence, we only care for detecting newlines (\r and/or
1505 : : * \n) and the end-of-copy marker (\.).
1506 : : *
1507 : : * In CSV mode, \r and \n inside a quoted field are just part of the data
1508 : : * value and are put in line_buf. We keep just enough state to know if we
1509 : : * are currently in a quoted field or not.
1510 : : *
1511 : : * The input has already been converted to the database encoding. All
1512 : : * supported server encodings have the property that all bytes in a
1513 : : * multi-byte sequence have the high bit set, so a multibyte character
1514 : : * cannot contain any newline or escape characters embedded in the
1515 : : * multibyte sequence. Therefore, we can process the input byte-by-byte,
1516 : : * regardless of the encoding.
1517 : : *
1518 : : * For speed, we try to move data from input_buf to line_buf in chunks
1519 : : * rather than one character at a time. input_buf_ptr points to the next
1520 : : * character to examine; any characters from input_buf_index to
1521 : : * input_buf_ptr have been determined to be part of the line, but not yet
1522 : : * transferred to line_buf.
1523 : : *
1524 : : * For a little extra speed within the loop, we copy some state
1525 : : * information into local variables. input_buf_ptr could be changed in
1526 : : * the SIMD path, so we must set that one before it. The others are set
1527 : : * afterwards.
1528 : : */
1529 : 756044 : input_buf_ptr = cstate->input_buf_index;
1530 : :
1531 : : /*
1532 : : * We first try to use SIMD for the task described above, falling back to
1533 : : * the scalar path (i.e., the loop below) if needed.
1534 : : */
1535 : : #ifndef USE_NO_SIMD
1536 [ + + ]: 756044 : if (cstate->simd_enabled)
1537 : : {
1538 : : /*
1539 : : * Using temporary variables seems to encourage the compiler to keep
1540 : : * them in a register, which is beneficial for performance.
1541 : : */
1542 : 333485 : bool tmp_hit_eof = false;
1543 : 333485 : int tmp_input_buf_ptr = 0; /* silence compiler warning */
1544 : :
1545 : 333485 : result = CopyReadLineTextSIMDHelper(cstate, is_csv, &tmp_hit_eof,
1546 : : &tmp_input_buf_ptr);
1547 : 333475 : hit_eof = tmp_hit_eof;
1548 : 333475 : input_buf_ptr = tmp_input_buf_ptr;
1549 : :
1550 [ + + ]: 333475 : if (result)
1551 : : {
1552 : : /* Transfer any still-uncopied data to line_buf. */
1553 [ - + ]: 600 : REFILL_LINEBUF;
1554 : :
1555 : 600 : return result;
1556 : : }
1557 : : }
1558 : : #endif /* ! USE_NO_SIMD */
1559 : :
1560 : 755434 : copy_input_buf = cstate->input_buf;
1561 : 755434 : copy_buf_len = cstate->input_buf_len;
1562 : :
1563 : : for (;;)
1564 : 8702144 : {
1565 : : int prev_raw_ptr;
1566 : : char c;
1567 : :
1568 : : /*
1569 : : * Load more data if needed.
1570 : : *
1571 : : * TODO: We could just force four bytes of read-ahead and avoid the
1572 : : * many calls to IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(). That was
1573 : : * unsafe with the old v2 COPY protocol, but we don't support that
1574 : : * anymore.
1575 : : */
1576 [ + + - + ]: 9457578 : if (input_buf_ptr >= copy_buf_len || need_data)
1577 : : {
1578 [ + + ]: 492 : REFILL_LINEBUF;
1579 : :
1580 : 492 : CopyLoadInputBuf(cstate, false);
1581 : : /* update our local variables */
1582 : 492 : hit_eof = cstate->input_reached_eof;
1583 : 492 : input_buf_ptr = cstate->input_buf_index;
1584 : 492 : copy_buf_len = cstate->input_buf_len;
1585 : :
1586 : : /*
1587 : : * If we are completely out of data, break out of the loop,
1588 : : * reporting EOF.
1589 : : */
1590 [ + + ]: 492 : if (INPUT_BUF_BYTES(cstate) <= 0)
1591 : : {
1592 : 358 : result = true;
1593 : 358 : break;
1594 : : }
1595 : 134 : need_data = false;
1596 : : }
1597 : :
1598 : : /* OK to fetch a character */
1599 : 9457220 : prev_raw_ptr = input_buf_ptr;
1600 : 9457220 : c = copy_input_buf[input_buf_ptr++];
1601 : :
1602 [ + + ]: 9457220 : if (is_csv)
1603 : : {
1604 : : /*
1605 : : * If character is '\r', we may need to look ahead below. Force
1606 : : * fetch of the next character if we don't already have it. We
1607 : : * need to do this before changing CSV state, in case '\r' is also
1608 : : * the quote or escape character.
1609 : : */
1610 [ + + ]: 2615 : if (c == '\r')
1611 : : {
1612 [ - + - - ]: 24 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1613 : : }
1614 : :
1615 : : /*
1616 : : * Dealing with quotes and escapes here is mildly tricky. If the
1617 : : * quote char is also the escape char, there's no problem - we
1618 : : * just use the char as a toggle. If they are different, we need
1619 : : * to ensure that we only take account of an escape inside a
1620 : : * quoted field and immediately preceding a quote char, and not
1621 : : * the second in an escape-escape sequence.
1622 : : */
1623 [ + + + + ]: 2615 : if (in_quote && c == escapec)
1624 : 32 : last_was_esc = !last_was_esc;
1625 [ + + + - ]: 2615 : if (c == quotec && !last_was_esc)
1626 : 308 : in_quote = !in_quote;
1627 [ + + ]: 2615 : if (c != escapec)
1628 : 2579 : last_was_esc = false;
1629 : :
1630 : : /*
1631 : : * Updating the line count for embedded CR and/or LF chars is
1632 : : * necessarily a little fragile - this test is probably about the
1633 : : * best we can do. (XXX it's arguable whether we should do this
1634 : : * at all --- is cur_lineno a physical or logical count?)
1635 : : */
1636 [ + + + + : 2615 : if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
+ + ]
1637 : 24 : cstate->cur_lineno++;
1638 : : }
1639 : :
1640 : : /* Process \r */
1641 [ + + + - : 9457220 : if (c == '\r' && (!is_csv || !in_quote))
- + ]
1642 : : {
1643 : : /* Check for \r\n on first line, _and_ handle \r\n. */
1644 [ # # ]: 0 : if (cstate->eol_type == EOL_UNKNOWN ||
1645 [ # # ]: 0 : cstate->eol_type == EOL_CRNL)
1646 : : {
1647 : : /*
1648 : : * If need more data, go back to loop top to load it.
1649 : : *
1650 : : * Note that if we are at EOF, c will wind up as '\0' because
1651 : : * of the guaranteed pad of input_buf.
1652 : : */
1653 [ # # # # ]: 0 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1654 : :
1655 : : /* get next char */
1656 : 0 : c = copy_input_buf[input_buf_ptr];
1657 : :
1658 [ # # ]: 0 : if (c == '\n')
1659 : : {
1660 : 0 : input_buf_ptr++; /* eat newline */
1661 : 0 : cstate->eol_type = EOL_CRNL; /* in case not set yet */
1662 : : }
1663 : : else
1664 : : {
1665 : : /* found \r, but no \n */
1666 [ # # ]: 0 : if (cstate->eol_type == EOL_CRNL)
1667 [ # # # # : 0 : ereport(ERROR,
# # ]
1668 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1669 : : !is_csv ?
1670 : : errmsg("literal carriage return found in data") :
1671 : : errmsg("unquoted carriage return found in data"),
1672 : : !is_csv ?
1673 : : errhint("Use \"\\r\" to represent carriage return.") :
1674 : : errhint("Use quoted CSV field to represent carriage return.")));
1675 : :
1676 : : /*
1677 : : * if we got here, it is the first line and we didn't find
1678 : : * \n, so don't consume the peeked character
1679 : : */
1680 : 0 : cstate->eol_type = EOL_CR;
1681 : : }
1682 : : }
1683 [ # # ]: 0 : else if (cstate->eol_type == EOL_NL)
1684 [ # # # # : 0 : ereport(ERROR,
# # ]
1685 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1686 : : !is_csv ?
1687 : : errmsg("literal carriage return found in data") :
1688 : : errmsg("unquoted carriage return found in data"),
1689 : : !is_csv ?
1690 : : errhint("Use \"\\r\" to represent carriage return.") :
1691 : : errhint("Use quoted CSV field to represent carriage return.")));
1692 : : /* If reach here, we have found the line terminator */
1693 : 0 : break;
1694 : : }
1695 : :
1696 : : /* Process \n */
1697 [ + + + + : 9457220 : if (c == '\n' && (!is_csv || !in_quote))
+ + ]
1698 : : {
1699 [ + - - + ]: 755021 : if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
1700 [ # # # # : 0 : ereport(ERROR,
# # ]
1701 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1702 : : !is_csv ?
1703 : : errmsg("literal newline found in data") :
1704 : : errmsg("unquoted newline found in data"),
1705 : : !is_csv ?
1706 : : errhint("Use \"\\n\" to represent newline.") :
1707 : : errhint("Use quoted CSV field to represent newline.")));
1708 : 755021 : cstate->eol_type = EOL_NL; /* in case not set yet */
1709 : : /* If reach here, we have found the line terminator */
1710 : 755021 : break;
1711 : : }
1712 : :
1713 : : /*
1714 : : * Process backslash, except in CSV mode where backslash is a normal
1715 : : * character.
1716 : : */
1717 [ + + + + ]: 8702199 : if (c == '\\' && !is_csv)
1718 : : {
1719 : : char c2;
1720 : :
1721 [ - + - - ]: 4908 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1722 [ - + - - ]: 4908 : IF_NEED_REFILL_AND_EOF_BREAK(0);
1723 : :
1724 : : /* -----
1725 : : * get next character
1726 : : * Note: we do not change c so if it isn't \., we can fall
1727 : : * through and continue processing.
1728 : : * -----
1729 : : */
1730 : 4908 : c2 = copy_input_buf[input_buf_ptr];
1731 : :
1732 [ + + ]: 4908 : if (c2 == '.')
1733 : : {
1734 : 55 : input_buf_ptr++; /* consume the '.' */
1735 [ - + ]: 55 : if (cstate->eol_type == EOL_CRNL)
1736 : : {
1737 : : /* Get the next character */
1738 [ # # # # ]: 0 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1739 : : /* if hit_eof, c2 will become '\0' */
1740 : 0 : c2 = copy_input_buf[input_buf_ptr++];
1741 : :
1742 [ # # ]: 0 : if (c2 == '\n')
1743 [ # # ]: 0 : ereport(ERROR,
1744 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1745 : : errmsg("end-of-copy marker does not match previous newline style")));
1746 [ # # ]: 0 : else if (c2 != '\r')
1747 [ # # ]: 0 : ereport(ERROR,
1748 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1749 : : errmsg("end-of-copy marker is not alone on its line")));
1750 : : }
1751 : :
1752 : : /* Get the next character */
1753 [ - + - - ]: 55 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1754 : : /* if hit_eof, c2 will become '\0' */
1755 : 55 : c2 = copy_input_buf[input_buf_ptr++];
1756 : :
1757 [ + - + + ]: 55 : if (c2 != '\r' && c2 != '\n')
1758 [ + - ]: 4 : ereport(ERROR,
1759 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1760 : : errmsg("end-of-copy marker is not alone on its line")));
1761 : :
1762 [ + + + - ]: 51 : if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
1763 [ - + - - ]: 51 : (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
1764 [ - + - - ]: 51 : (cstate->eol_type == EOL_CR && c2 != '\r'))
1765 [ # # ]: 0 : ereport(ERROR,
1766 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1767 : : errmsg("end-of-copy marker does not match previous newline style")));
1768 : :
1769 : : /*
1770 : : * If there is any data on this line before the \., complain.
1771 : : */
1772 [ + - ]: 51 : if (cstate->line_buf.len > 0 ||
1773 [ + + ]: 51 : prev_raw_ptr > cstate->input_buf_index)
1774 [ + - ]: 4 : ereport(ERROR,
1775 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1776 : : errmsg("end-of-copy marker is not alone on its line")));
1777 : :
1778 : : /*
1779 : : * Discard the \. and newline, then report EOF.
1780 : : */
1781 : 47 : cstate->input_buf_index = input_buf_ptr;
1782 : 47 : result = true; /* report EOF */
1783 : 47 : break;
1784 : : }
1785 : : else
1786 : : {
1787 : : /*
1788 : : * If we are here, it means we found a backslash followed by
1789 : : * something other than a period. In non-CSV mode, anything
1790 : : * after a backslash is special, so we skip over that second
1791 : : * character too. If we didn't do that \\. would be
1792 : : * considered an eof-of copy, while in non-CSV mode it is a
1793 : : * literal backslash followed by a period.
1794 : : */
1795 : 4853 : input_buf_ptr++;
1796 : : }
1797 : : }
1798 : : } /* end of outer loop */
1799 : :
1800 : : /*
1801 : : * Transfer any still-uncopied data to line_buf.
1802 : : */
1803 [ + + ]: 755426 : REFILL_LINEBUF;
1804 : :
1805 : 755426 : return result;
1806 : : }
1807 : :
1808 : : /*
1809 : : * Return decimal value for a hexadecimal digit
1810 : : */
1811 : : static int
1812 : 0 : GetDecimalFromHex(char hex)
1813 : : {
1814 [ # # ]: 0 : if (isdigit((unsigned char) hex))
1815 : 0 : return hex - '0';
1816 : : else
1817 : 0 : return pg_ascii_tolower((unsigned char) hex) - 'a' + 10;
1818 : : }
1819 : :
1820 : : /*
1821 : : * Parse the current line into separate attributes (fields),
1822 : : * performing de-escaping as needed.
1823 : : *
1824 : : * The input is in line_buf. We use attribute_buf to hold the result
1825 : : * strings. cstate->raw_fields[k] is set to point to the k'th attribute
1826 : : * string, or NULL when the input matches the null marker string.
1827 : : * This array is expanded as necessary.
1828 : : *
1829 : : * (Note that the caller cannot check for nulls since the returned
1830 : : * string would be the post-de-escaping equivalent, which may look
1831 : : * the same as some valid data string.)
1832 : : *
1833 : : * delim is the column delimiter string (must be just one byte for now).
1834 : : * null_print is the null marker string. Note that this is compared to
1835 : : * the pre-de-escaped input string.
1836 : : *
1837 : : * The return value is the number of fields actually read.
1838 : : */
1839 : : static int
1840 : 754631 : CopyReadAttributesText(CopyFromState cstate)
1841 : : {
1842 : 754631 : char delimc = cstate->opts.delim[0];
1843 : : int fieldno;
1844 : : char *output_ptr;
1845 : : char *cur_ptr;
1846 : : char *line_end_ptr;
1847 : :
1848 : : /*
1849 : : * We need a special case for zero-column tables: check that the input
1850 : : * line is empty, and return.
1851 : : */
1852 [ + + ]: 754631 : if (cstate->max_fields <= 0)
1853 : : {
1854 [ - + ]: 4 : if (cstate->line_buf.len != 0)
1855 [ # # ]: 0 : ereport(ERROR,
1856 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1857 : : errmsg("extra data after last expected column")));
1858 : 4 : return 0;
1859 : : }
1860 : :
1861 : 754627 : resetStringInfo(&cstate->attribute_buf);
1862 : :
1863 : : /*
1864 : : * The de-escaped attributes will certainly not be longer than the input
1865 : : * data line, so we can just force attribute_buf to be large enough and
1866 : : * then transfer data without any checks for enough space. We need to do
1867 : : * it this way because enlarging attribute_buf mid-stream would invalidate
1868 : : * pointers already stored into cstate->raw_fields[].
1869 : : */
1870 [ + + ]: 754627 : if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
1871 : 4 : enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
1872 : 754627 : output_ptr = cstate->attribute_buf.data;
1873 : :
1874 : : /* set pointer variables for loop */
1875 : 754627 : cur_ptr = cstate->line_buf.data;
1876 : 754627 : line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
1877 : :
1878 : : /* Outer loop iterates over fields */
1879 : 754627 : fieldno = 0;
1880 : : for (;;)
1881 : 1959225 : {
1882 : 2713852 : bool found_delim = false;
1883 : : char *start_ptr;
1884 : : char *end_ptr;
1885 : : int input_len;
1886 : 2713852 : bool saw_non_ascii = false;
1887 : :
1888 : : /* Make sure there is enough space for the next value */
1889 [ + + ]: 2713852 : if (fieldno >= cstate->max_fields)
1890 : : {
1891 : 28 : cstate->max_fields *= 2;
1892 : 28 : cstate->raw_fields =
1893 : 28 : repalloc_array(cstate->raw_fields, char *, cstate->max_fields);
1894 : : }
1895 : :
1896 : : /* Remember start of field on both input and output sides */
1897 : 2713852 : start_ptr = cur_ptr;
1898 : 2713852 : cstate->raw_fields[fieldno] = output_ptr;
1899 : :
1900 : : /*
1901 : : * Scan data for field.
1902 : : *
1903 : : * Note that in this loop, we are scanning to locate the end of field
1904 : : * and also speculatively performing de-escaping. Once we find the
1905 : : * end-of-field, we can match the raw field contents against the null
1906 : : * marker string. Only after that comparison fails do we know that
1907 : : * de-escaping is actually the right thing to do; therefore we *must
1908 : : * not* throw any syntax errors before we've done the null-marker
1909 : : * check.
1910 : : */
1911 : : for (;;)
1912 : 13972613 : {
1913 : : char c;
1914 : :
1915 : 16686465 : end_ptr = cur_ptr;
1916 [ + + ]: 16686465 : if (cur_ptr >= line_end_ptr)
1917 : 754623 : break;
1918 : 15931842 : c = *cur_ptr++;
1919 [ + + ]: 15931842 : if (c == delimc)
1920 : : {
1921 : 1959229 : found_delim = true;
1922 : 1959229 : break;
1923 : : }
1924 [ + + ]: 13972613 : if (c == '\\')
1925 : : {
1926 [ - + ]: 4853 : if (cur_ptr >= line_end_ptr)
1927 : 0 : break;
1928 : 4853 : c = *cur_ptr++;
1929 [ + + - - : 4853 : switch (c)
+ - - -
+ ]
1930 : : {
1931 : 8 : case '0':
1932 : : case '1':
1933 : : case '2':
1934 : : case '3':
1935 : : case '4':
1936 : : case '5':
1937 : : case '6':
1938 : : case '7':
1939 : : {
1940 : : /* handle \013 */
1941 : : int val;
1942 : :
1943 : 8 : val = OCTVALUE(c);
1944 [ + + ]: 8 : if (cur_ptr < line_end_ptr)
1945 : : {
1946 : 4 : c = *cur_ptr;
1947 [ - + - - ]: 4 : if (ISOCTAL(c))
1948 : : {
1949 : 0 : cur_ptr++;
1950 : 0 : val = (val << 3) + OCTVALUE(c);
1951 [ # # ]: 0 : if (cur_ptr < line_end_ptr)
1952 : : {
1953 : 0 : c = *cur_ptr;
1954 [ # # # # ]: 0 : if (ISOCTAL(c))
1955 : : {
1956 : 0 : cur_ptr++;
1957 : 0 : val = (val << 3) + OCTVALUE(c);
1958 : : }
1959 : : }
1960 : : }
1961 : : }
1962 : 8 : c = val & 0377;
1963 [ - + - - ]: 8 : if (c == '\0' || IS_HIGHBIT_SET(c))
1964 : 8 : saw_non_ascii = true;
1965 : : }
1966 : 8 : break;
1967 : 8 : case 'x':
1968 : : /* Handle \x3F */
1969 [ + + ]: 8 : if (cur_ptr < line_end_ptr)
1970 : : {
1971 : 4 : char hexchar = *cur_ptr;
1972 : :
1973 [ - + ]: 4 : if (isxdigit((unsigned char) hexchar))
1974 : : {
1975 : 0 : int val = GetDecimalFromHex(hexchar);
1976 : :
1977 : 0 : cur_ptr++;
1978 [ # # ]: 0 : if (cur_ptr < line_end_ptr)
1979 : : {
1980 : 0 : hexchar = *cur_ptr;
1981 [ # # ]: 0 : if (isxdigit((unsigned char) hexchar))
1982 : : {
1983 : 0 : cur_ptr++;
1984 : 0 : val = (val << 4) + GetDecimalFromHex(hexchar);
1985 : : }
1986 : : }
1987 : 0 : c = val & 0xff;
1988 [ # # # # ]: 0 : if (c == '\0' || IS_HIGHBIT_SET(c))
1989 : 0 : saw_non_ascii = true;
1990 : : }
1991 : : }
1992 : 8 : break;
1993 : 0 : case 'b':
1994 : 0 : c = '\b';
1995 : 0 : break;
1996 : 0 : case 'f':
1997 : 0 : c = '\f';
1998 : 0 : break;
1999 : 2033 : case 'n':
2000 : 2033 : c = '\n';
2001 : 2033 : break;
2002 : 0 : case 'r':
2003 : 0 : c = '\r';
2004 : 0 : break;
2005 : 0 : case 't':
2006 : 0 : c = '\t';
2007 : 0 : break;
2008 : 0 : case 'v':
2009 : 0 : c = '\v';
2010 : 0 : break;
2011 : :
2012 : : /*
2013 : : * in all other cases, take the char after '\'
2014 : : * literally
2015 : : */
2016 : : }
2017 : : }
2018 : :
2019 : : /* Add c to output string */
2020 : 13972613 : *output_ptr++ = c;
2021 : : }
2022 : :
2023 : : /* Check whether raw input matched null marker */
2024 : 2713852 : input_len = end_ptr - start_ptr;
2025 [ + + ]: 2713852 : if (input_len == cstate->opts.null_print_len &&
2026 [ + + ]: 165425 : strncmp(start_ptr, cstate->opts.null_print, input_len) == 0)
2027 : 2734 : cstate->raw_fields[fieldno] = NULL;
2028 : : /* Check whether raw input matched default marker */
2029 [ + + ]: 2711118 : else if (fieldno < list_length(cstate->attnumlist) &&
2030 [ + + ]: 2711086 : cstate->opts.default_print &&
2031 [ + + ]: 76 : input_len == cstate->opts.default_print_len &&
2032 [ + - ]: 20 : strncmp(start_ptr, cstate->opts.default_print, input_len) == 0)
2033 : 16 : {
2034 : : /* fieldno is 0-indexed and attnum is 1-indexed */
2035 : 20 : int m = list_nth_int(cstate->attnumlist, fieldno) - 1;
2036 : :
2037 [ + + ]: 20 : if (cstate->defexprs[m] != NULL)
2038 : : {
2039 : : /* defaults contain entries for all physical attributes */
2040 : 16 : cstate->defaults[m] = true;
2041 : : }
2042 : : else
2043 : : {
2044 : 4 : TupleDesc tupDesc = RelationGetDescr(cstate->rel);
2045 : 4 : Form_pg_attribute att = TupleDescAttr(tupDesc, m);
2046 : :
2047 [ + - ]: 4 : ereport(ERROR,
2048 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2049 : : errmsg("unexpected default marker in COPY data"),
2050 : : errdetail("Column \"%s\" has no default value.",
2051 : : NameStr(att->attname))));
2052 : : }
2053 : : }
2054 : : else
2055 : : {
2056 : : /*
2057 : : * At this point we know the field is supposed to contain data.
2058 : : *
2059 : : * If we de-escaped any non-7-bit-ASCII chars, make sure the
2060 : : * resulting string is valid data for the db encoding.
2061 : : */
2062 [ - + ]: 2711098 : if (saw_non_ascii)
2063 : : {
2064 : 0 : char *fld = cstate->raw_fields[fieldno];
2065 : :
2066 : 0 : pg_verifymbstr(fld, output_ptr - fld, false);
2067 : : }
2068 : : }
2069 : :
2070 : : /* Terminate attribute value in output area */
2071 : 2713848 : *output_ptr++ = '\0';
2072 : :
2073 : 2713848 : fieldno++;
2074 : : /* Done if we hit EOL instead of a delim */
2075 [ + + ]: 2713848 : if (!found_delim)
2076 : 754623 : break;
2077 : : }
2078 : :
2079 : : /* Clean up state of attribute_buf */
2080 : 754623 : output_ptr--;
2081 : : Assert(*output_ptr == '\0');
2082 : 754623 : cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
2083 : :
2084 : 754623 : return fieldno;
2085 : : }
2086 : :
2087 : : /*
2088 : : * Parse the current line into separate attributes (fields),
2089 : : * performing de-escaping as needed. This has exactly the same API as
2090 : : * CopyReadAttributesText, except we parse the fields according to
2091 : : * "standard" (i.e. common) CSV usage.
2092 : : */
2093 : : static int
2094 : 318 : CopyReadAttributesCSV(CopyFromState cstate)
2095 : : {
2096 : 318 : char delimc = cstate->opts.delim[0];
2097 : 318 : char quotec = cstate->opts.quote[0];
2098 : 318 : char escapec = cstate->opts.escape[0];
2099 : : int fieldno;
2100 : : char *output_ptr;
2101 : : char *cur_ptr;
2102 : : char *line_end_ptr;
2103 : :
2104 : : /*
2105 : : * We need a special case for zero-column tables: check that the input
2106 : : * line is empty, and return.
2107 : : */
2108 [ - + ]: 318 : if (cstate->max_fields <= 0)
2109 : : {
2110 [ # # ]: 0 : if (cstate->line_buf.len != 0)
2111 [ # # ]: 0 : ereport(ERROR,
2112 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2113 : : errmsg("extra data after last expected column")));
2114 : 0 : return 0;
2115 : : }
2116 : :
2117 : 318 : resetStringInfo(&cstate->attribute_buf);
2118 : :
2119 : : /*
2120 : : * The de-escaped attributes will certainly not be longer than the input
2121 : : * data line, so we can just force attribute_buf to be large enough and
2122 : : * then transfer data without any checks for enough space. We need to do
2123 : : * it this way because enlarging attribute_buf mid-stream would invalidate
2124 : : * pointers already stored into cstate->raw_fields[].
2125 : : */
2126 [ - + ]: 318 : if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
2127 : 0 : enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
2128 : 318 : output_ptr = cstate->attribute_buf.data;
2129 : :
2130 : : /* set pointer variables for loop */
2131 : 318 : cur_ptr = cstate->line_buf.data;
2132 : 318 : line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
2133 : :
2134 : : /* Outer loop iterates over fields */
2135 : 318 : fieldno = 0;
2136 : : for (;;)
2137 : 326 : {
2138 : 644 : bool found_delim = false;
2139 : 644 : bool saw_quote = false;
2140 : : char *start_ptr;
2141 : : char *end_ptr;
2142 : : int input_len;
2143 : :
2144 : : /* Make sure there is enough space for the next value */
2145 [ - + ]: 644 : if (fieldno >= cstate->max_fields)
2146 : : {
2147 : 0 : cstate->max_fields *= 2;
2148 : 0 : cstate->raw_fields =
2149 : 0 : repalloc_array(cstate->raw_fields, char *, cstate->max_fields);
2150 : : }
2151 : :
2152 : : /* Remember start of field on both input and output sides */
2153 : 644 : start_ptr = cur_ptr;
2154 : 644 : cstate->raw_fields[fieldno] = output_ptr;
2155 : :
2156 : : /*
2157 : : * Scan data for field,
2158 : : *
2159 : : * The loop starts in "not quote" mode and then toggles between that
2160 : : * and "in quote" mode. The loop exits normally if it is in "not
2161 : : * quote" mode and a delimiter or line end is seen.
2162 : : */
2163 : : for (;;)
2164 : 137 : {
2165 : : char c;
2166 : :
2167 : : /* Not in quote */
2168 : : for (;;)
2169 : : {
2170 : 2045 : end_ptr = cur_ptr;
2171 [ + + ]: 2045 : if (cur_ptr >= line_end_ptr)
2172 : 314 : goto endfield;
2173 : 1731 : c = *cur_ptr++;
2174 : : /* unquoted field delimiter */
2175 [ + + ]: 1731 : if (c == delimc)
2176 : : {
2177 : 330 : found_delim = true;
2178 : 330 : goto endfield;
2179 : : }
2180 : : /* start of quoted field (or part of field) */
2181 [ + + ]: 1401 : if (c == quotec)
2182 : : {
2183 : 137 : saw_quote = true;
2184 : 137 : break;
2185 : : }
2186 : : /* Add c to output string */
2187 : 1264 : *output_ptr++ = c;
2188 : : }
2189 : :
2190 : : /* In quote */
2191 : : for (;;)
2192 : : {
2193 : 852 : end_ptr = cur_ptr;
2194 [ - + ]: 852 : if (cur_ptr >= line_end_ptr)
2195 [ # # ]: 0 : ereport(ERROR,
2196 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2197 : : errmsg("unterminated CSV quoted field")));
2198 : :
2199 : 852 : c = *cur_ptr++;
2200 : :
2201 : : /* escape within a quoted field */
2202 [ + + ]: 852 : if (c == escapec)
2203 : : {
2204 : : /*
2205 : : * peek at the next char if available, and escape it if it
2206 : : * is an escape char or a quote char
2207 : : */
2208 [ + + ]: 81 : if (cur_ptr < line_end_ptr)
2209 : : {
2210 : 47 : char nextc = *cur_ptr;
2211 : :
2212 [ + + - + ]: 47 : if (nextc == escapec || nextc == quotec)
2213 : : {
2214 : 16 : *output_ptr++ = nextc;
2215 : 16 : cur_ptr++;
2216 : 16 : continue;
2217 : : }
2218 : : }
2219 : : }
2220 : :
2221 : : /*
2222 : : * end of quoted field. Must do this test after testing for
2223 : : * escape in case quote char and escape char are the same
2224 : : * (which is the common case).
2225 : : */
2226 [ + + ]: 836 : if (c == quotec)
2227 : 137 : break;
2228 : :
2229 : : /* Add c to output string */
2230 : 699 : *output_ptr++ = c;
2231 : : }
2232 : : }
2233 : 644 : endfield:
2234 : :
2235 : : /* Terminate attribute value in output area */
2236 : 644 : *output_ptr++ = '\0';
2237 : :
2238 : : /* Check whether raw input matched null marker */
2239 : 644 : input_len = end_ptr - start_ptr;
2240 [ + + + + ]: 644 : if (!saw_quote && input_len == cstate->opts.null_print_len &&
2241 [ + - ]: 27 : strncmp(start_ptr, cstate->opts.null_print, input_len) == 0)
2242 : 27 : cstate->raw_fields[fieldno] = NULL;
2243 : : /* Check whether raw input matched default marker */
2244 [ + - ]: 617 : else if (fieldno < list_length(cstate->attnumlist) &&
2245 [ + + ]: 617 : cstate->opts.default_print &&
2246 [ + + ]: 94 : input_len == cstate->opts.default_print_len &&
2247 [ + - ]: 26 : strncmp(start_ptr, cstate->opts.default_print, input_len) == 0)
2248 : : {
2249 : : /* fieldno is 0-index and attnum is 1-index */
2250 : 26 : int m = list_nth_int(cstate->attnumlist, fieldno) - 1;
2251 : :
2252 [ + + ]: 26 : if (cstate->defexprs[m] != NULL)
2253 : : {
2254 : : /* defaults contain entries for all physical attributes */
2255 : 22 : cstate->defaults[m] = true;
2256 : : }
2257 : : else
2258 : : {
2259 : 4 : TupleDesc tupDesc = RelationGetDescr(cstate->rel);
2260 : 4 : Form_pg_attribute att = TupleDescAttr(tupDesc, m);
2261 : :
2262 [ + - ]: 4 : ereport(ERROR,
2263 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2264 : : errmsg("unexpected default marker in COPY data"),
2265 : : errdetail("Column \"%s\" has no default value.",
2266 : : NameStr(att->attname))));
2267 : : }
2268 : : }
2269 : :
2270 : 640 : fieldno++;
2271 : : /* Done if we hit EOL instead of a delim */
2272 [ + + ]: 640 : if (!found_delim)
2273 : 314 : break;
2274 : : }
2275 : :
2276 : : /* Clean up state of attribute_buf */
2277 : 314 : output_ptr--;
2278 : : Assert(*output_ptr == '\0');
2279 : 314 : cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
2280 : :
2281 : 314 : return fieldno;
2282 : : }
2283 : :
2284 : :
2285 : : /*
2286 : : * Read a binary attribute
2287 : : */
2288 : : static Datum
2289 : 100 : CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
2290 : : Oid typioparam, int32 typmod,
2291 : : bool *isnull)
2292 : : {
2293 : : int32 fld_size;
2294 : : Datum result;
2295 : :
2296 [ - + ]: 100 : if (!CopyGetInt32(cstate, &fld_size))
2297 [ # # ]: 0 : ereport(ERROR,
2298 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2299 : : errmsg("unexpected EOF in COPY data")));
2300 [ + + ]: 100 : if (fld_size == -1)
2301 : : {
2302 : 20 : *isnull = true;
2303 : 20 : return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
2304 : : }
2305 [ - + ]: 80 : if (fld_size < 0)
2306 [ # # ]: 0 : ereport(ERROR,
2307 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2308 : : errmsg("invalid field size")));
2309 : :
2310 : : /* reset attribute_buf to empty, and load raw data in it */
2311 : 80 : resetStringInfo(&cstate->attribute_buf);
2312 : :
2313 : 80 : enlargeStringInfo(&cstate->attribute_buf, fld_size);
2314 : 80 : if (CopyReadBinaryData(cstate, cstate->attribute_buf.data,
2315 [ - + ]: 80 : fld_size) != fld_size)
2316 [ # # ]: 0 : ereport(ERROR,
2317 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2318 : : errmsg("unexpected EOF in COPY data")));
2319 : :
2320 : 80 : cstate->attribute_buf.len = fld_size;
2321 : 80 : cstate->attribute_buf.data[fld_size] = '\0';
2322 : :
2323 : : /* Call the column type's binary input converter */
2324 : 80 : result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
2325 : : typioparam, typmod);
2326 : :
2327 : : /* Trouble if it didn't eat the whole buffer */
2328 [ + + ]: 80 : if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
2329 [ + - ]: 1 : ereport(ERROR,
2330 : : (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
2331 : : errmsg("incorrect binary data format")));
2332 : :
2333 : 79 : *isnull = false;
2334 : 79 : return result;
2335 : : }
|
