Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * copyfromparse.c
4 : * Parse CSV/text/binary format for COPY FROM.
5 : *
6 : * This file contains routines to parse the text, CSV and binary input
7 : * formats. The main entry point is NextCopyFrom(), which parses the
8 : * next input line and returns it as Datums.
9 : *
10 : * In text/CSV mode, the parsing happens in multiple stages:
11 : *
12 : * [data source] --> raw_buf --> input_buf --> line_buf --> attribute_buf
13 : * 1. 2. 3. 4.
14 : *
15 : * 1. CopyLoadRawBuf() reads raw data from the input file or client, and
16 : * places it into 'raw_buf'.
17 : *
18 : * 2. CopyConvertBuf() calls the encoding conversion function to convert
19 : * the data in 'raw_buf' from client to server encoding, placing the
20 : * converted result in 'input_buf'.
21 : *
22 : * 3. CopyReadLine() parses the data in 'input_buf', one line at a time.
23 : * It is responsible for finding the next newline marker, taking quote and
24 : * escape characters into account according to the COPY options. The line
25 : * is copied into 'line_buf', with quotes and escape characters still
26 : * intact.
27 : *
28 : * 4. CopyReadAttributesText/CSV() function takes the input line from
29 : * 'line_buf', and splits it into fields, unescaping the data as required.
30 : * The fields are stored in 'attribute_buf', and 'raw_fields' array holds
31 : * pointers to each field.
32 : *
33 : * If encoding conversion is not required, a shortcut is taken in step 2 to
34 : * avoid copying the data unnecessarily. The 'input_buf' pointer is set to
35 : * point directly to 'raw_buf', so that CopyLoadRawBuf() loads the raw data
36 : * directly into 'input_buf'. CopyConvertBuf() then merely validates that
37 : * the data is valid in the current encoding.
38 : *
39 : * In binary mode, the pipeline is much simpler. Input is loaded into
40 : * 'raw_buf', and encoding conversion is done in the datatype-specific
41 : * receive functions, if required. 'input_buf' and 'line_buf' are not used,
42 : * but 'attribute_buf' is used as a temporary buffer to hold one attribute's
43 : * data when it's passed the receive function.
44 : *
45 : * 'raw_buf' is always 64 kB in size (RAW_BUF_SIZE). 'input_buf' is also
46 : * 64 kB (INPUT_BUF_SIZE), if encoding conversion is required. 'line_buf'
47 : * and 'attribute_buf' are expanded on demand, to hold the longest line
48 : * encountered so far.
49 : *
50 : * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
51 : * Portions Copyright (c) 1994, Regents of the University of California
52 : *
53 : *
54 : * IDENTIFICATION
55 : * src/backend/commands/copyfromparse.c
56 : *
57 : *-------------------------------------------------------------------------
58 : */
59 : #include "postgres.h"
60 :
61 : #include <ctype.h>
62 : #include <unistd.h>
63 : #include <sys/stat.h>
64 :
65 : #include "commands/copyapi.h"
66 : #include "commands/copyfrom_internal.h"
67 : #include "commands/progress.h"
68 : #include "executor/executor.h"
69 : #include "libpq/libpq.h"
70 : #include "libpq/pqformat.h"
71 : #include "mb/pg_wchar.h"
72 : #include "miscadmin.h"
73 : #include "pgstat.h"
74 : #include "port/pg_bswap.h"
75 : #include "utils/builtins.h"
76 : #include "utils/rel.h"
77 :
78 : #define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
79 : #define OCTVALUE(c) ((c) - '0')
80 :
81 : /*
82 : * These macros centralize code used to process line_buf and input_buf buffers.
83 : * They are macros because they often do continue/break control and to avoid
84 : * function call overhead in tight COPY loops.
85 : *
86 : * We must use "if (1)" because the usual "do {...} while(0)" wrapper would
87 : * prevent the continue/break processing from working. We end the "if (1)"
88 : * with "else ((void) 0)" to ensure the "if" does not unintentionally match
89 : * any "else" in the calling code, and to avoid any compiler warnings about
90 : * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
91 : */
92 :
93 : /*
94 : * This keeps the character read at the top of the loop in the buffer
95 : * even if there is more than one read-ahead.
96 : */
97 : #define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
98 : if (1) \
99 : { \
100 : if (input_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
101 : { \
102 : input_buf_ptr = prev_raw_ptr; /* undo fetch */ \
103 : need_data = true; \
104 : continue; \
105 : } \
106 : } else ((void) 0)
107 :
108 : /* This consumes the remainder of the buffer and breaks */
109 : #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
110 : if (1) \
111 : { \
112 : if (input_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
113 : { \
114 : if (extralen) \
115 : input_buf_ptr = copy_buf_len; /* consume the partial character */ \
116 : /* backslash just before EOF, treat as data char */ \
117 : result = true; \
118 : break; \
119 : } \
120 : } else ((void) 0)
121 :
122 : /*
123 : * Transfer any approved data to line_buf; must do this to be sure
124 : * there is some room in input_buf.
125 : */
126 : #define REFILL_LINEBUF \
127 : if (1) \
128 : { \
129 : if (input_buf_ptr > cstate->input_buf_index) \
130 : { \
131 : appendBinaryStringInfo(&cstate->line_buf, \
132 : cstate->input_buf + cstate->input_buf_index, \
133 : input_buf_ptr - cstate->input_buf_index); \
134 : cstate->input_buf_index = input_buf_ptr; \
135 : } \
136 : } else ((void) 0)
137 :
138 : /* NOTE: there's a copy of this in copyto.c */
139 : static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0";
140 :
141 :
142 : /* non-export function prototypes */
143 : static bool CopyReadLine(CopyFromState cstate, bool is_csv);
144 : static bool CopyReadLineText(CopyFromState cstate, bool is_csv);
145 : static int CopyReadAttributesText(CopyFromState cstate);
146 : static int CopyReadAttributesCSV(CopyFromState cstate);
147 : static Datum CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
148 : Oid typioparam, int32 typmod,
149 : bool *isnull);
150 : static pg_attribute_always_inline bool CopyFromTextLikeOneRow(CopyFromState cstate,
151 : ExprContext *econtext,
152 : Datum *values,
153 : bool *nulls,
154 : bool is_csv);
155 : static pg_attribute_always_inline bool NextCopyFromRawFieldsInternal(CopyFromState cstate,
156 : char ***fields,
157 : int *nfields,
158 : bool is_csv);
159 :
160 :
161 : /* Low-level communications functions */
162 : static int CopyGetData(CopyFromState cstate, void *databuf,
163 : int minread, int maxread);
164 : static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
165 : static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
166 : static void CopyLoadInputBuf(CopyFromState cstate);
167 : static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
168 :
169 : void
170 2228 : ReceiveCopyBegin(CopyFromState cstate)
171 : {
172 : StringInfoData buf;
173 2228 : int natts = list_length(cstate->attnumlist);
174 2228 : int16 format = (cstate->opts.binary ? 1 : 0);
175 : int i;
176 :
177 2228 : pq_beginmessage(&buf, PqMsg_CopyInResponse);
178 2228 : pq_sendbyte(&buf, format); /* overall format */
179 2228 : pq_sendint16(&buf, natts);
180 9778 : for (i = 0; i < natts; i++)
181 7550 : pq_sendint16(&buf, format); /* per-column formats */
182 2228 : pq_endmessage(&buf);
183 2228 : cstate->copy_src = COPY_FRONTEND;
184 2228 : cstate->fe_msgbuf = makeStringInfo();
185 : /* We *must* flush here to ensure FE knows it can send. */
186 2228 : pq_flush();
187 2228 : }
188 :
189 : void
190 14 : ReceiveCopyBinaryHeader(CopyFromState cstate)
191 : {
192 : char readSig[11];
193 : int32 tmp;
194 :
195 : /* Signature */
196 14 : if (CopyReadBinaryData(cstate, readSig, 11) != 11 ||
197 14 : memcmp(readSig, BinarySignature, 11) != 0)
198 0 : ereport(ERROR,
199 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
200 : errmsg("COPY file signature not recognized")));
201 : /* Flags field */
202 14 : if (!CopyGetInt32(cstate, &tmp))
203 0 : ereport(ERROR,
204 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
205 : errmsg("invalid COPY file header (missing flags)")));
206 14 : if ((tmp & (1 << 16)) != 0)
207 0 : ereport(ERROR,
208 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
209 : errmsg("invalid COPY file header (WITH OIDS)")));
210 14 : tmp &= ~(1 << 16);
211 14 : if ((tmp >> 16) != 0)
212 0 : ereport(ERROR,
213 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
214 : errmsg("unrecognized critical flags in COPY file header")));
215 : /* Header extension length */
216 14 : if (!CopyGetInt32(cstate, &tmp) ||
217 14 : tmp < 0)
218 0 : ereport(ERROR,
219 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
220 : errmsg("invalid COPY file header (missing length)")));
221 : /* Skip extension header, if present */
222 14 : while (tmp-- > 0)
223 : {
224 0 : if (CopyReadBinaryData(cstate, readSig, 1) != 1)
225 0 : ereport(ERROR,
226 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
227 : errmsg("invalid COPY file header (wrong length)")));
228 : }
229 14 : }
230 :
231 : /*
232 : * CopyGetData reads data from the source (file or frontend)
233 : *
234 : * We attempt to read at least minread, and at most maxread, bytes from
235 : * the source. The actual number of bytes read is returned; if this is
236 : * less than minread, EOF was detected.
237 : *
238 : * Note: when copying from the frontend, we expect a proper EOF mark per
239 : * protocol; if the frontend simply drops the connection, we raise error.
240 : * It seems unwise to allow the COPY IN to complete normally in that case.
241 : *
242 : * NB: no data conversion is applied here.
243 : */
244 : static int
245 440114 : CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
246 : {
247 440114 : int bytesread = 0;
248 :
249 440114 : switch (cstate->copy_src)
250 : {
251 1080 : case COPY_FILE:
252 1080 : bytesread = fread(databuf, 1, maxread, cstate->copy_file);
253 1080 : if (ferror(cstate->copy_file))
254 0 : ereport(ERROR,
255 : (errcode_for_file_access(),
256 : errmsg("could not read from COPY file: %m")));
257 1080 : if (bytesread == 0)
258 422 : cstate->raw_reached_eof = true;
259 1080 : break;
260 410990 : case COPY_FRONTEND:
261 819206 : while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof)
262 : {
263 : int avail;
264 :
265 410242 : while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
266 : {
267 : /* Try to receive another message */
268 : int mtype;
269 : int maxmsglen;
270 :
271 410242 : readmessage:
272 410278 : HOLD_CANCEL_INTERRUPTS();
273 410278 : pq_startmsgread();
274 410278 : mtype = pq_getbyte();
275 410278 : if (mtype == EOF)
276 0 : ereport(ERROR,
277 : (errcode(ERRCODE_CONNECTION_FAILURE),
278 : errmsg("unexpected EOF on client connection with an open transaction")));
279 : /* Validate message type and set packet size limit */
280 : switch (mtype)
281 : {
282 408216 : case PqMsg_CopyData:
283 408216 : maxmsglen = PQ_LARGE_MESSAGE_LIMIT;
284 408216 : break;
285 2062 : case PqMsg_CopyDone:
286 : case PqMsg_CopyFail:
287 : case PqMsg_Flush:
288 : case PqMsg_Sync:
289 2062 : maxmsglen = PQ_SMALL_MESSAGE_LIMIT;
290 2062 : break;
291 0 : default:
292 0 : ereport(ERROR,
293 : (errcode(ERRCODE_PROTOCOL_VIOLATION),
294 : errmsg("unexpected message type 0x%02X during COPY from stdin",
295 : mtype)));
296 : maxmsglen = 0; /* keep compiler quiet */
297 : break;
298 : }
299 : /* Now collect the message body */
300 410278 : if (pq_getmessage(cstate->fe_msgbuf, maxmsglen))
301 0 : ereport(ERROR,
302 : (errcode(ERRCODE_CONNECTION_FAILURE),
303 : errmsg("unexpected EOF on client connection with an open transaction")));
304 410278 : RESUME_CANCEL_INTERRUPTS();
305 : /* ... and process it */
306 : switch (mtype)
307 : {
308 408216 : case PqMsg_CopyData:
309 408216 : break;
310 2026 : case PqMsg_CopyDone:
311 : /* COPY IN correctly terminated by frontend */
312 2026 : cstate->raw_reached_eof = true;
313 2026 : return bytesread;
314 0 : case PqMsg_CopyFail:
315 0 : ereport(ERROR,
316 : (errcode(ERRCODE_QUERY_CANCELED),
317 : errmsg("COPY from stdin failed: %s",
318 : pq_getmsgstring(cstate->fe_msgbuf))));
319 : break;
320 36 : case PqMsg_Flush:
321 : case PqMsg_Sync:
322 :
323 : /*
324 : * Ignore Flush/Sync for the convenience of client
325 : * libraries (such as libpq) that may send those
326 : * without noticing that the command they just
327 : * sent was COPY.
328 : */
329 36 : goto readmessage;
330 818458 : default:
331 : Assert(false); /* NOT REACHED */
332 : }
333 : }
334 408216 : avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
335 408216 : if (avail > maxread)
336 0 : avail = maxread;
337 408216 : pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
338 408216 : databuf = (void *) ((char *) databuf + avail);
339 408216 : maxread -= avail;
340 408216 : bytesread += avail;
341 : }
342 408964 : break;
343 28044 : case COPY_CALLBACK:
344 28044 : bytesread = cstate->data_source_cb(databuf, minread, maxread);
345 28044 : break;
346 : }
347 :
348 438088 : return bytesread;
349 : }
350 :
351 :
352 : /*
353 : * These functions do apply some data conversion
354 : */
355 :
356 : /*
357 : * CopyGetInt32 reads an int32 that appears in network byte order
358 : *
359 : * Returns true if OK, false if EOF
360 : */
361 : static inline bool
362 186 : CopyGetInt32(CopyFromState cstate, int32 *val)
363 : {
364 : uint32 buf;
365 :
366 186 : if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
367 : {
368 0 : *val = 0; /* suppress compiler warning */
369 0 : return false;
370 : }
371 186 : *val = (int32) pg_ntoh32(buf);
372 186 : return true;
373 : }
374 :
375 : /*
376 : * CopyGetInt16 reads an int16 that appears in network byte order
377 : */
378 : static inline bool
379 42 : CopyGetInt16(CopyFromState cstate, int16 *val)
380 : {
381 : uint16 buf;
382 :
383 42 : if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
384 : {
385 0 : *val = 0; /* suppress compiler warning */
386 0 : return false;
387 : }
388 42 : *val = (int16) pg_ntoh16(buf);
389 42 : return true;
390 : }
391 :
392 :
393 : /*
394 : * Perform encoding conversion on data in 'raw_buf', writing the converted
395 : * data into 'input_buf'.
396 : *
397 : * On entry, there must be some data to convert in 'raw_buf'.
398 : */
399 : static void
400 876092 : CopyConvertBuf(CopyFromState cstate)
401 : {
402 : /*
403 : * If the file and server encoding are the same, no encoding conversion is
404 : * required. However, we still need to verify that the input is valid for
405 : * the encoding.
406 : */
407 876092 : if (!cstate->need_transcoding)
408 : {
409 : /*
410 : * When conversion is not required, input_buf and raw_buf are the
411 : * same. raw_buf_len is the total number of bytes in the buffer, and
412 : * input_buf_len tracks how many of those bytes have already been
413 : * verified.
414 : */
415 876008 : int preverifiedlen = cstate->input_buf_len;
416 876008 : int unverifiedlen = cstate->raw_buf_len - cstate->input_buf_len;
417 : int nverified;
418 :
419 876008 : if (unverifiedlen == 0)
420 : {
421 : /*
422 : * If no more raw data is coming, report the EOF to the caller.
423 : */
424 439516 : if (cstate->raw_reached_eof)
425 1512 : cstate->input_reached_eof = true;
426 439516 : return;
427 : }
428 :
429 : /*
430 : * Verify the new data, including any residual unverified bytes from
431 : * previous round.
432 : */
433 436492 : nverified = pg_encoding_verifymbstr(cstate->file_encoding,
434 436492 : cstate->raw_buf + preverifiedlen,
435 : unverifiedlen);
436 436492 : if (nverified == 0)
437 : {
438 : /*
439 : * Could not verify anything.
440 : *
441 : * If there is no more raw input data coming, it means that there
442 : * was an incomplete multi-byte sequence at the end. Also, if
443 : * there's "enough" input left, we should be able to verify at
444 : * least one character, and a failure to do so means that we've
445 : * hit an invalid byte sequence.
446 : */
447 0 : if (cstate->raw_reached_eof || unverifiedlen >= pg_encoding_max_length(cstate->file_encoding))
448 0 : cstate->input_reached_error = true;
449 0 : return;
450 : }
451 436492 : cstate->input_buf_len += nverified;
452 : }
453 : else
454 : {
455 : /*
456 : * Encoding conversion is needed.
457 : */
458 : int nbytes;
459 : unsigned char *src;
460 : int srclen;
461 : unsigned char *dst;
462 : int dstlen;
463 : int convertedlen;
464 :
465 84 : if (RAW_BUF_BYTES(cstate) == 0)
466 : {
467 : /*
468 : * If no more raw data is coming, report the EOF to the caller.
469 : */
470 48 : if (cstate->raw_reached_eof)
471 12 : cstate->input_reached_eof = true;
472 48 : return;
473 : }
474 :
475 : /*
476 : * First, copy down any unprocessed data.
477 : */
478 36 : nbytes = INPUT_BUF_BYTES(cstate);
479 36 : if (nbytes > 0 && cstate->input_buf_index > 0)
480 0 : memmove(cstate->input_buf, cstate->input_buf + cstate->input_buf_index,
481 : nbytes);
482 36 : cstate->input_buf_index = 0;
483 36 : cstate->input_buf_len = nbytes;
484 36 : cstate->input_buf[nbytes] = '\0';
485 :
486 36 : src = (unsigned char *) cstate->raw_buf + cstate->raw_buf_index;
487 36 : srclen = cstate->raw_buf_len - cstate->raw_buf_index;
488 36 : dst = (unsigned char *) cstate->input_buf + cstate->input_buf_len;
489 36 : dstlen = INPUT_BUF_SIZE - cstate->input_buf_len + 1;
490 :
491 : /*
492 : * Do the conversion. This might stop short, if there is an invalid
493 : * byte sequence in the input. We'll convert as much as we can in
494 : * that case.
495 : *
496 : * Note: Even if we hit an invalid byte sequence, we don't report the
497 : * error until all the valid bytes have been consumed. The input
498 : * might contain an end-of-input marker (\.), and we don't want to
499 : * report an error if the invalid byte sequence is after the
500 : * end-of-input marker. We might unnecessarily convert some data
501 : * after the end-of-input marker as long as it's valid for the
502 : * encoding, but that's harmless.
503 : */
504 36 : convertedlen = pg_do_encoding_conversion_buf(cstate->conversion_proc,
505 : cstate->file_encoding,
506 : GetDatabaseEncoding(),
507 : src, srclen,
508 : dst, dstlen,
509 : true);
510 36 : if (convertedlen == 0)
511 : {
512 : /*
513 : * Could not convert anything. If there is no more raw input data
514 : * coming, it means that there was an incomplete multi-byte
515 : * sequence at the end. Also, if there is plenty of input left,
516 : * we should be able to convert at least one character, so a
517 : * failure to do so must mean that we've hit a byte sequence
518 : * that's invalid.
519 : */
520 24 : if (cstate->raw_reached_eof || srclen >= MAX_CONVERSION_INPUT_LENGTH)
521 12 : cstate->input_reached_error = true;
522 24 : return;
523 : }
524 12 : cstate->raw_buf_index += convertedlen;
525 12 : cstate->input_buf_len += strlen((char *) dst);
526 : }
527 : }
528 :
529 : /*
530 : * Report an encoding or conversion error.
531 : */
532 : static void
533 12 : CopyConversionError(CopyFromState cstate)
534 : {
535 : Assert(cstate->raw_buf_len > 0);
536 : Assert(cstate->input_reached_error);
537 :
538 12 : if (!cstate->need_transcoding)
539 : {
540 : /*
541 : * Everything up to input_buf_len was successfully verified, and
542 : * input_buf_len points to the invalid or incomplete character.
543 : */
544 0 : report_invalid_encoding(cstate->file_encoding,
545 0 : cstate->raw_buf + cstate->input_buf_len,
546 0 : cstate->raw_buf_len - cstate->input_buf_len);
547 : }
548 : else
549 : {
550 : /*
551 : * raw_buf_index points to the invalid or untranslatable character. We
552 : * let the conversion routine report the error, because it can provide
553 : * a more specific error message than we could here. An earlier call
554 : * to the conversion routine in CopyConvertBuf() detected that there
555 : * is an error, now we call the conversion routine again with
556 : * noError=false, to have it throw the error.
557 : */
558 : unsigned char *src;
559 : int srclen;
560 : unsigned char *dst;
561 : int dstlen;
562 :
563 12 : src = (unsigned char *) cstate->raw_buf + cstate->raw_buf_index;
564 12 : srclen = cstate->raw_buf_len - cstate->raw_buf_index;
565 12 : dst = (unsigned char *) cstate->input_buf + cstate->input_buf_len;
566 12 : dstlen = INPUT_BUF_SIZE - cstate->input_buf_len + 1;
567 :
568 12 : (void) pg_do_encoding_conversion_buf(cstate->conversion_proc,
569 : cstate->file_encoding,
570 : GetDatabaseEncoding(),
571 : src, srclen,
572 : dst, dstlen,
573 : false);
574 :
575 : /*
576 : * The conversion routine should have reported an error, so this
577 : * should not be reached.
578 : */
579 0 : elog(ERROR, "encoding conversion failed without error");
580 : }
581 : }
582 :
583 : /*
584 : * Load more data from data source to raw_buf.
585 : *
586 : * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the
587 : * beginning of the buffer, and we load new data after that.
588 : */
589 : static void
590 438088 : CopyLoadRawBuf(CopyFromState cstate)
591 : {
592 : int nbytes;
593 : int inbytes;
594 :
595 : /*
596 : * In text mode, if encoding conversion is not required, raw_buf and
597 : * input_buf point to the same buffer. Their len/index better agree, too.
598 : */
599 438088 : if (cstate->raw_buf == cstate->input_buf)
600 : {
601 : Assert(!cstate->need_transcoding);
602 : Assert(cstate->raw_buf_index == cstate->input_buf_index);
603 : Assert(cstate->input_buf_len <= cstate->raw_buf_len);
604 : }
605 :
606 : /*
607 : * Copy down the unprocessed data if any.
608 : */
609 438088 : nbytes = RAW_BUF_BYTES(cstate);
610 438088 : if (nbytes > 0 && cstate->raw_buf_index > 0)
611 36 : memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
612 : nbytes);
613 438088 : cstate->raw_buf_len -= cstate->raw_buf_index;
614 438088 : cstate->raw_buf_index = 0;
615 :
616 : /*
617 : * If raw_buf and input_buf are in fact the same buffer, adjust the
618 : * input_buf variables, too.
619 : */
620 438088 : if (cstate->raw_buf == cstate->input_buf)
621 : {
622 438004 : cstate->input_buf_len -= cstate->input_buf_index;
623 438004 : cstate->input_buf_index = 0;
624 : }
625 :
626 : /* Load more data */
627 438088 : inbytes = CopyGetData(cstate, cstate->raw_buf + cstate->raw_buf_len,
628 438088 : 1, RAW_BUF_SIZE - cstate->raw_buf_len);
629 438088 : nbytes += inbytes;
630 438088 : cstate->raw_buf[nbytes] = '\0';
631 438088 : cstate->raw_buf_len = nbytes;
632 :
633 438088 : cstate->bytes_processed += inbytes;
634 438088 : pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
635 :
636 438088 : if (inbytes == 0)
637 1548 : cstate->raw_reached_eof = true;
638 438088 : }
639 :
640 : /*
641 : * CopyLoadInputBuf loads some more data into input_buf
642 : *
643 : * On return, at least one more input character is loaded into
644 : * input_buf, or input_reached_eof is set.
645 : *
646 : * If INPUT_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
647 : * of the buffer and then we load more data after that.
648 : */
649 : static void
650 438040 : CopyLoadInputBuf(CopyFromState cstate)
651 : {
652 438040 : int nbytes = INPUT_BUF_BYTES(cstate);
653 :
654 : /*
655 : * The caller has updated input_buf_index to indicate how much of the
656 : * input has been consumed and isn't needed anymore. If input_buf is the
657 : * same physical area as raw_buf, update raw_buf_index accordingly.
658 : */
659 438040 : if (cstate->raw_buf == cstate->input_buf)
660 : {
661 : Assert(!cstate->need_transcoding);
662 : Assert(cstate->input_buf_index >= cstate->raw_buf_index);
663 438004 : cstate->raw_buf_index = cstate->input_buf_index;
664 : }
665 :
666 : for (;;)
667 : {
668 : /* If we now have some unconverted data, try to convert it */
669 876092 : CopyConvertBuf(cstate);
670 :
671 : /* If we now have some more input bytes ready, return them */
672 876092 : if (INPUT_BUF_BYTES(cstate) > nbytes)
673 436504 : return;
674 :
675 : /*
676 : * If we reached an invalid byte sequence, or we're at an incomplete
677 : * multi-byte character but there is no more raw input data, report
678 : * conversion error.
679 : */
680 439588 : if (cstate->input_reached_error)
681 12 : CopyConversionError(cstate);
682 :
683 : /* no more input, and everything has been converted */
684 439576 : if (cstate->input_reached_eof)
685 1524 : break;
686 :
687 : /* Try to load more raw data */
688 : Assert(!cstate->raw_reached_eof);
689 438052 : CopyLoadRawBuf(cstate);
690 : }
691 : }
692 :
693 : /*
694 : * CopyReadBinaryData
695 : *
696 : * Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf
697 : * and writes them to 'dest'. Returns the number of bytes read (which
698 : * would be less than 'nbytes' only if we reach EOF).
699 : */
700 : static int
701 382 : CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
702 : {
703 382 : int copied_bytes = 0;
704 :
705 382 : if (RAW_BUF_BYTES(cstate) >= nbytes)
706 : {
707 : /* Enough bytes are present in the buffer. */
708 346 : memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, nbytes);
709 346 : cstate->raw_buf_index += nbytes;
710 346 : copied_bytes = nbytes;
711 : }
712 : else
713 : {
714 : /*
715 : * Not enough bytes in the buffer, so must read from the file. Need
716 : * to loop since 'nbytes' could be larger than the buffer size.
717 : */
718 : do
719 : {
720 : int copy_bytes;
721 :
722 : /* Load more data if buffer is empty. */
723 36 : if (RAW_BUF_BYTES(cstate) == 0)
724 : {
725 36 : CopyLoadRawBuf(cstate);
726 36 : if (cstate->raw_reached_eof)
727 12 : break; /* EOF */
728 : }
729 :
730 : /* Transfer some bytes. */
731 24 : copy_bytes = Min(nbytes - copied_bytes, RAW_BUF_BYTES(cstate));
732 24 : memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, copy_bytes);
733 24 : cstate->raw_buf_index += copy_bytes;
734 24 : dest += copy_bytes;
735 24 : copied_bytes += copy_bytes;
736 24 : } while (copied_bytes < nbytes);
737 : }
738 :
739 382 : return copied_bytes;
740 : }
741 :
742 : /*
743 : * This function is exposed for use by extensions that read raw fields in the
744 : * next line. See NextCopyFromRawFieldsInternal() for details.
745 : */
746 : bool
747 0 : NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields)
748 : {
749 0 : return NextCopyFromRawFieldsInternal(cstate, fields, nfields,
750 0 : cstate->opts.csv_mode);
751 : }
752 :
753 : /*
754 : * Workhorse for NextCopyFromRawFields().
755 : *
756 : * Read raw fields in the next line for COPY FROM in text or csv mode. Return
757 : * false if no more lines.
758 : *
759 : * An internal temporary buffer is returned via 'fields'. It is valid until
760 : * the next call of the function. Since the function returns all raw fields
761 : * in the input file, 'nfields' could be different from the number of columns
762 : * in the relation.
763 : *
764 : * NOTE: force_not_null option are not applied to the returned fields.
765 : *
766 : * We use pg_attribute_always_inline to reduce function call overhead
767 : * and to help compilers to optimize away the 'is_csv' condition when called
768 : * by internal functions such as CopyFromTextLikeOneRow().
769 : */
770 : static pg_attribute_always_inline bool
771 2101090 : NextCopyFromRawFieldsInternal(CopyFromState cstate, char ***fields, int *nfields, bool is_csv)
772 : {
773 : int fldct;
774 : bool done;
775 :
776 : /* only available for text or csv input */
777 : Assert(!cstate->opts.binary);
778 :
779 : /* on input check that the header line is correct if needed */
780 2101090 : if (cstate->cur_lineno == 0 && cstate->opts.header_line)
781 : {
782 : ListCell *cur;
783 : TupleDesc tupDesc;
784 :
785 120 : tupDesc = RelationGetDescr(cstate->rel);
786 :
787 120 : cstate->cur_lineno++;
788 120 : done = CopyReadLine(cstate, is_csv);
789 :
790 120 : if (cstate->opts.header_line == COPY_HEADER_MATCH)
791 : {
792 : int fldnum;
793 :
794 76 : if (is_csv)
795 10 : fldct = CopyReadAttributesCSV(cstate);
796 : else
797 66 : fldct = CopyReadAttributesText(cstate);
798 :
799 76 : if (fldct != list_length(cstate->attnumlist))
800 24 : ereport(ERROR,
801 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
802 : errmsg("wrong number of fields in header line: got %d, expected %d",
803 : fldct, list_length(cstate->attnumlist))));
804 :
805 52 : fldnum = 0;
806 158 : foreach(cur, cstate->attnumlist)
807 : {
808 126 : int attnum = lfirst_int(cur);
809 : char *colName;
810 126 : Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
811 :
812 : Assert(fldnum < cstate->max_fields);
813 :
814 126 : colName = cstate->raw_fields[fldnum++];
815 126 : if (colName == NULL)
816 6 : ereport(ERROR,
817 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
818 : errmsg("column name mismatch in header line field %d: got null value (\"%s\"), expected \"%s\"",
819 : fldnum, cstate->opts.null_print, NameStr(attr->attname))));
820 :
821 120 : if (namestrcmp(&attr->attname, colName) != 0)
822 : {
823 14 : ereport(ERROR,
824 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
825 : errmsg("column name mismatch in header line field %d: got \"%s\", expected \"%s\"",
826 : fldnum, colName, NameStr(attr->attname))));
827 : }
828 : }
829 : }
830 :
831 76 : if (done)
832 0 : return false;
833 : }
834 :
835 2101046 : cstate->cur_lineno++;
836 :
837 : /* Actually read the line into memory here */
838 2101046 : done = CopyReadLine(cstate, is_csv);
839 :
840 : /*
841 : * EOF at start of line means we're done. If we see EOF after some
842 : * characters, we act as though it was newline followed by EOF, ie,
843 : * process the line and then exit loop on next iteration.
844 : */
845 2101022 : if (done && cstate->line_buf.len == 0)
846 2802 : return false;
847 :
848 : /* Parse the line into de-escaped field values */
849 2098220 : if (is_csv)
850 464 : fldct = CopyReadAttributesCSV(cstate);
851 : else
852 2097756 : fldct = CopyReadAttributesText(cstate);
853 :
854 2098208 : *fields = cstate->raw_fields;
855 2098208 : *nfields = fldct;
856 2098208 : return true;
857 : }
858 :
859 : /*
860 : * Read next tuple from file for COPY FROM. Return false if no more tuples.
861 : *
862 : * 'econtext' is used to evaluate default expression for each column that is
863 : * either not read from the file or is using the DEFAULT option of COPY FROM.
864 : * It can be NULL when no default values are used, i.e. when all columns are
865 : * read from the file, and DEFAULT option is unset.
866 : *
867 : * 'values' and 'nulls' arrays must be the same length as columns of the
868 : * relation passed to BeginCopyFrom. This function fills the arrays.
869 : */
870 : bool
871 2101132 : NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
872 : Datum *values, bool *nulls)
873 : {
874 : TupleDesc tupDesc;
875 : AttrNumber num_phys_attrs,
876 2101132 : num_defaults = cstate->num_defaults;
877 : int i;
878 2101132 : int *defmap = cstate->defmap;
879 2101132 : ExprState **defexprs = cstate->defexprs;
880 :
881 2101132 : tupDesc = RelationGetDescr(cstate->rel);
882 2101132 : num_phys_attrs = tupDesc->natts;
883 :
884 : /* Initialize all values for row to NULL */
885 9201042 : MemSet(values, 0, num_phys_attrs * sizeof(Datum));
886 2101132 : MemSet(nulls, true, num_phys_attrs * sizeof(bool));
887 2343336 : MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool));
888 :
889 : /* Get one row from source */
890 2101132 : if (!cstate->routine->CopyFromOneRow(cstate, econtext, values, nulls))
891 2814 : return false;
892 :
893 : /*
894 : * Now compute and insert any defaults available for the columns not
895 : * provided by the input data. Anything not processed here or above will
896 : * remain NULL.
897 : */
898 2158692 : for (i = 0; i < num_defaults; i++)
899 : {
900 : /*
901 : * The caller must supply econtext and have switched into the
902 : * per-tuple memory context in it.
903 : */
904 : Assert(econtext != NULL);
905 : Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
906 :
907 60530 : values[defmap[i]] = ExecEvalExpr(defexprs[defmap[i]], econtext,
908 60530 : &nulls[defmap[i]]);
909 : }
910 :
911 2098162 : return true;
912 : }
913 :
914 : /* Implementation of the per-row callback for text format */
915 : bool
916 2100416 : CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
917 : bool *nulls)
918 : {
919 2100416 : return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, false);
920 : }
921 :
922 : /* Implementation of the per-row callback for CSV format */
923 : bool
924 674 : CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
925 : bool *nulls)
926 : {
927 674 : return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, true);
928 : }
929 :
930 : /*
931 : * Workhorse for CopyFromTextOneRow() and CopyFromCSVOneRow().
932 : *
933 : * We use pg_attribute_always_inline to reduce function call overhead
934 : * and to help compilers to optimize away the 'is_csv' condition.
935 : */
936 : static pg_attribute_always_inline bool
937 2101090 : CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext,
938 : Datum *values, bool *nulls, bool is_csv)
939 : {
940 : TupleDesc tupDesc;
941 : AttrNumber attr_count;
942 2101090 : FmgrInfo *in_functions = cstate->in_functions;
943 2101090 : Oid *typioparams = cstate->typioparams;
944 2101090 : ExprState **defexprs = cstate->defexprs;
945 : char **field_strings;
946 : ListCell *cur;
947 : int fldct;
948 : int fieldno;
949 : char *string;
950 :
951 2101090 : tupDesc = RelationGetDescr(cstate->rel);
952 2101090 : attr_count = list_length(cstate->attnumlist);
953 :
954 : /* read raw fields in the next line */
955 2101090 : if (!NextCopyFromRawFieldsInternal(cstate, &field_strings, &fldct, is_csv))
956 2802 : return false;
957 :
958 : /* check for overflowing fields */
959 2098208 : if (attr_count > 0 && fldct > attr_count)
960 18 : ereport(ERROR,
961 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
962 : errmsg("extra data after last expected column")));
963 :
964 2098190 : fieldno = 0;
965 :
966 : /* Loop to read the user attributes on the line. */
967 9060984 : foreach(cur, cstate->attnumlist)
968 : {
969 6962978 : int attnum = lfirst_int(cur);
970 6962978 : int m = attnum - 1;
971 6962978 : Form_pg_attribute att = TupleDescAttr(tupDesc, m);
972 :
973 6962978 : if (fieldno >= fldct)
974 18 : ereport(ERROR,
975 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
976 : errmsg("missing data for column \"%s\"",
977 : NameStr(att->attname))));
978 6962960 : string = field_strings[fieldno++];
979 :
980 6962960 : if (cstate->convert_select_flags &&
981 20 : !cstate->convert_select_flags[m])
982 : {
983 : /* ignore input field, leaving column as NULL */
984 10 : continue;
985 : }
986 :
987 6962950 : if (is_csv)
988 : {
989 962 : if (string == NULL &&
990 44 : cstate->opts.force_notnull_flags[m])
991 : {
992 : /*
993 : * FORCE_NOT_NULL option is set and column is NULL - convert
994 : * it to the NULL string.
995 : */
996 28 : string = cstate->opts.null_print;
997 : }
998 934 : else if (string != NULL && cstate->opts.force_null_flags[m]
999 50 : && strcmp(string, cstate->opts.null_print) == 0)
1000 : {
1001 : /*
1002 : * FORCE_NULL option is set and column matches the NULL
1003 : * string. It must have been quoted, or otherwise the string
1004 : * would already have been set to NULL. Convert it to NULL as
1005 : * specified.
1006 : */
1007 26 : string = NULL;
1008 : }
1009 : }
1010 :
1011 6962950 : cstate->cur_attname = NameStr(att->attname);
1012 6962950 : cstate->cur_attval = string;
1013 :
1014 6962950 : if (string != NULL)
1015 6829568 : nulls[m] = false;
1016 :
1017 6962950 : if (cstate->defaults[m])
1018 : {
1019 : /* We must have switched into the per-tuple memory context */
1020 : Assert(econtext != NULL);
1021 : Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
1022 :
1023 60 : values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]);
1024 : }
1025 :
1026 : /*
1027 : * If ON_ERROR is specified with IGNORE, skip rows with soft errors
1028 : */
1029 6962852 : else if (!InputFunctionCallSafe(&in_functions[m],
1030 : string,
1031 6962890 : typioparams[m],
1032 : att->atttypmod,
1033 6962890 : (Node *) cstate->escontext,
1034 6962890 : &values[m]))
1035 : {
1036 : Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP);
1037 :
1038 128 : cstate->num_errors++;
1039 :
1040 128 : if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE)
1041 : {
1042 : /*
1043 : * Since we emit line number and column info in the below
1044 : * notice message, we suppress error context information other
1045 : * than the relation name.
1046 : */
1047 : Assert(!cstate->relname_only);
1048 42 : cstate->relname_only = true;
1049 :
1050 42 : if (cstate->cur_attval)
1051 : {
1052 : char *attval;
1053 :
1054 36 : attval = CopyLimitPrintoutLength(cstate->cur_attval);
1055 36 : ereport(NOTICE,
1056 : errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"",
1057 : cstate->cur_lineno,
1058 : cstate->cur_attname,
1059 : attval));
1060 36 : pfree(attval);
1061 : }
1062 : else
1063 6 : ereport(NOTICE,
1064 : errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": null input",
1065 : cstate->cur_lineno,
1066 : cstate->cur_attname));
1067 :
1068 : /* reset relname_only */
1069 42 : cstate->relname_only = false;
1070 : }
1071 :
1072 128 : return true;
1073 : }
1074 :
1075 6962784 : cstate->cur_attname = NULL;
1076 6962784 : cstate->cur_attval = NULL;
1077 : }
1078 :
1079 : Assert(fieldno == attr_count);
1080 :
1081 2098006 : return true;
1082 : }
1083 :
1084 : /* Implementation of the per-row callback for binary format */
1085 : bool
1086 42 : CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
1087 : bool *nulls)
1088 : {
1089 : TupleDesc tupDesc;
1090 : AttrNumber attr_count;
1091 42 : FmgrInfo *in_functions = cstate->in_functions;
1092 42 : Oid *typioparams = cstate->typioparams;
1093 : int16 fld_count;
1094 : ListCell *cur;
1095 :
1096 42 : tupDesc = RelationGetDescr(cstate->rel);
1097 42 : attr_count = list_length(cstate->attnumlist);
1098 :
1099 42 : cstate->cur_lineno++;
1100 :
1101 42 : if (!CopyGetInt16(cstate, &fld_count))
1102 : {
1103 : /* EOF detected (end of file, or protocol-level EOF) */
1104 0 : return false;
1105 : }
1106 :
1107 42 : if (fld_count == -1)
1108 : {
1109 : /*
1110 : * Received EOF marker. Wait for the protocol-level EOF, and complain
1111 : * if it doesn't come immediately. In COPY FROM STDIN, this ensures
1112 : * that we correctly handle CopyFail, if client chooses to send that
1113 : * now. When copying from file, we could ignore the rest of the file
1114 : * like in text mode, but we choose to be consistent with the COPY
1115 : * FROM STDIN case.
1116 : */
1117 : char dummy;
1118 :
1119 12 : if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
1120 0 : ereport(ERROR,
1121 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1122 : errmsg("received copy data after EOF marker")));
1123 12 : return false;
1124 : }
1125 :
1126 30 : if (fld_count != attr_count)
1127 0 : ereport(ERROR,
1128 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1129 : errmsg("row field count is %d, expected %d",
1130 : (int) fld_count, attr_count)));
1131 :
1132 186 : foreach(cur, cstate->attnumlist)
1133 : {
1134 158 : int attnum = lfirst_int(cur);
1135 158 : int m = attnum - 1;
1136 158 : Form_pg_attribute att = TupleDescAttr(tupDesc, m);
1137 :
1138 158 : cstate->cur_attname = NameStr(att->attname);
1139 314 : values[m] = CopyReadBinaryAttribute(cstate,
1140 158 : &in_functions[m],
1141 158 : typioparams[m],
1142 : att->atttypmod,
1143 : &nulls[m]);
1144 156 : cstate->cur_attname = NULL;
1145 : }
1146 :
1147 28 : return true;
1148 : }
1149 :
1150 : /*
1151 : * Read the next input line and stash it in line_buf.
1152 : *
1153 : * Result is true if read was terminated by EOF, false if terminated
1154 : * by newline. The terminating newline or EOF marker is not included
1155 : * in the final value of line_buf.
1156 : */
1157 : static bool
1158 2101166 : CopyReadLine(CopyFromState cstate, bool is_csv)
1159 : {
1160 : bool result;
1161 :
1162 2101166 : resetStringInfo(&cstate->line_buf);
1163 2101166 : cstate->line_buf_valid = false;
1164 :
1165 : /* Parse data and transfer into line_buf */
1166 2101166 : result = CopyReadLineText(cstate, is_csv);
1167 :
1168 2101142 : if (result)
1169 : {
1170 : /*
1171 : * Reached EOF. In protocol version 3, we should ignore anything
1172 : * after \. up to the protocol end of copy data. (XXX maybe better
1173 : * not to treat \. as special?)
1174 : */
1175 2802 : if (cstate->copy_src == COPY_FRONTEND)
1176 : {
1177 : int inbytes;
1178 :
1179 : do
1180 : {
1181 2026 : inbytes = CopyGetData(cstate, cstate->input_buf,
1182 : 1, INPUT_BUF_SIZE);
1183 2026 : } while (inbytes > 0);
1184 2026 : cstate->input_buf_index = 0;
1185 2026 : cstate->input_buf_len = 0;
1186 2026 : cstate->raw_buf_index = 0;
1187 2026 : cstate->raw_buf_len = 0;
1188 : }
1189 : }
1190 : else
1191 : {
1192 : /*
1193 : * If we didn't hit EOF, then we must have transferred the EOL marker
1194 : * to line_buf along with the data. Get rid of it.
1195 : */
1196 2098340 : switch (cstate->eol_type)
1197 : {
1198 2098340 : case EOL_NL:
1199 : Assert(cstate->line_buf.len >= 1);
1200 : Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
1201 2098340 : cstate->line_buf.len--;
1202 2098340 : cstate->line_buf.data[cstate->line_buf.len] = '\0';
1203 2098340 : break;
1204 0 : case EOL_CR:
1205 : Assert(cstate->line_buf.len >= 1);
1206 : Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
1207 0 : cstate->line_buf.len--;
1208 0 : cstate->line_buf.data[cstate->line_buf.len] = '\0';
1209 0 : break;
1210 0 : case EOL_CRNL:
1211 : Assert(cstate->line_buf.len >= 2);
1212 : Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
1213 : Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
1214 0 : cstate->line_buf.len -= 2;
1215 0 : cstate->line_buf.data[cstate->line_buf.len] = '\0';
1216 0 : break;
1217 0 : case EOL_UNKNOWN:
1218 : /* shouldn't get here */
1219 : Assert(false);
1220 0 : break;
1221 : }
1222 2101142 : }
1223 :
1224 : /* Now it's safe to use the buffer in error messages */
1225 2101142 : cstate->line_buf_valid = true;
1226 :
1227 2101142 : return result;
1228 : }
1229 :
1230 : /*
1231 : * CopyReadLineText - inner loop of CopyReadLine for text mode
1232 : */
1233 : static bool
1234 2101166 : CopyReadLineText(CopyFromState cstate, bool is_csv)
1235 : {
1236 : char *copy_input_buf;
1237 : int input_buf_ptr;
1238 : int copy_buf_len;
1239 2101166 : bool need_data = false;
1240 2101166 : bool hit_eof = false;
1241 2101166 : bool result = false;
1242 :
1243 : /* CSV variables */
1244 2101166 : bool in_quote = false,
1245 2101166 : last_was_esc = false;
1246 2101166 : char quotec = '\0';
1247 2101166 : char escapec = '\0';
1248 :
1249 2101166 : if (is_csv)
1250 : {
1251 720 : quotec = cstate->opts.quote[0];
1252 720 : escapec = cstate->opts.escape[0];
1253 : /* ignore special escape processing if it's the same as quotec */
1254 720 : if (quotec == escapec)
1255 526 : escapec = '\0';
1256 : }
1257 :
1258 : /*
1259 : * The objective of this loop is to transfer the entire next input line
1260 : * into line_buf. Hence, we only care for detecting newlines (\r and/or
1261 : * \n) and the end-of-copy marker (\.).
1262 : *
1263 : * In CSV mode, \r and \n inside a quoted field are just part of the data
1264 : * value and are put in line_buf. We keep just enough state to know if we
1265 : * are currently in a quoted field or not.
1266 : *
1267 : * The input has already been converted to the database encoding. All
1268 : * supported server encodings have the property that all bytes in a
1269 : * multi-byte sequence have the high bit set, so a multibyte character
1270 : * cannot contain any newline or escape characters embedded in the
1271 : * multibyte sequence. Therefore, we can process the input byte-by-byte,
1272 : * regardless of the encoding.
1273 : *
1274 : * For speed, we try to move data from input_buf to line_buf in chunks
1275 : * rather than one character at a time. input_buf_ptr points to the next
1276 : * character to examine; any characters from input_buf_index to
1277 : * input_buf_ptr have been determined to be part of the line, but not yet
1278 : * transferred to line_buf.
1279 : *
1280 : * For a little extra speed within the loop, we copy input_buf and
1281 : * input_buf_len into local variables.
1282 : */
1283 2101166 : copy_input_buf = cstate->input_buf;
1284 2101166 : input_buf_ptr = cstate->input_buf_index;
1285 2101166 : copy_buf_len = cstate->input_buf_len;
1286 :
1287 : for (;;)
1288 48789366 : {
1289 : int prev_raw_ptr;
1290 : char c;
1291 :
1292 : /*
1293 : * Load more data if needed.
1294 : *
1295 : * TODO: We could just force four bytes of read-ahead and avoid the
1296 : * many calls to IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(). That was
1297 : * unsafe with the old v2 COPY protocol, but we don't support that
1298 : * anymore.
1299 : */
1300 50890532 : if (input_buf_ptr >= copy_buf_len || need_data)
1301 : {
1302 438040 : REFILL_LINEBUF;
1303 :
1304 438040 : CopyLoadInputBuf(cstate);
1305 : /* update our local variables */
1306 438028 : hit_eof = cstate->input_reached_eof;
1307 438028 : input_buf_ptr = cstate->input_buf_index;
1308 438028 : copy_buf_len = cstate->input_buf_len;
1309 :
1310 : /*
1311 : * If we are completely out of data, break out of the loop,
1312 : * reporting EOF.
1313 : */
1314 438028 : if (INPUT_BUF_BYTES(cstate) <= 0)
1315 : {
1316 1524 : result = true;
1317 1524 : break;
1318 : }
1319 436504 : need_data = false;
1320 : }
1321 :
1322 : /* OK to fetch a character */
1323 50888996 : prev_raw_ptr = input_buf_ptr;
1324 50888996 : c = copy_input_buf[input_buf_ptr++];
1325 :
1326 50888996 : if (is_csv)
1327 : {
1328 : /*
1329 : * If character is '\r', we may need to look ahead below. Force
1330 : * fetch of the next character if we don't already have it. We
1331 : * need to do this before changing CSV state, in case '\r' is also
1332 : * the quote or escape character.
1333 : */
1334 5418 : if (c == '\r')
1335 : {
1336 36 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1337 : }
1338 :
1339 : /*
1340 : * Dealing with quotes and escapes here is mildly tricky. If the
1341 : * quote char is also the escape char, there's no problem - we
1342 : * just use the char as a toggle. If they are different, we need
1343 : * to ensure that we only take account of an escape inside a
1344 : * quoted field and immediately preceding a quote char, and not
1345 : * the second in an escape-escape sequence.
1346 : */
1347 5418 : if (in_quote && c == escapec)
1348 48 : last_was_esc = !last_was_esc;
1349 5418 : if (c == quotec && !last_was_esc)
1350 508 : in_quote = !in_quote;
1351 5418 : if (c != escapec)
1352 5364 : last_was_esc = false;
1353 :
1354 : /*
1355 : * Updating the line count for embedded CR and/or LF chars is
1356 : * necessarily a little fragile - this test is probably about the
1357 : * best we can do. (XXX it's arguable whether we should do this
1358 : * at all --- is cur_lineno a physical or logical count?)
1359 : */
1360 5418 : if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
1361 36 : cstate->cur_lineno++;
1362 : }
1363 :
1364 : /* Process \r */
1365 50888996 : if (c == '\r' && (!is_csv || !in_quote))
1366 : {
1367 : /* Check for \r\n on first line, _and_ handle \r\n. */
1368 0 : if (cstate->eol_type == EOL_UNKNOWN ||
1369 0 : cstate->eol_type == EOL_CRNL)
1370 : {
1371 : /*
1372 : * If need more data, go back to loop top to load it.
1373 : *
1374 : * Note that if we are at EOF, c will wind up as '\0' because
1375 : * of the guaranteed pad of input_buf.
1376 : */
1377 0 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1378 :
1379 : /* get next char */
1380 0 : c = copy_input_buf[input_buf_ptr];
1381 :
1382 0 : if (c == '\n')
1383 : {
1384 0 : input_buf_ptr++; /* eat newline */
1385 0 : cstate->eol_type = EOL_CRNL; /* in case not set yet */
1386 : }
1387 : else
1388 : {
1389 : /* found \r, but no \n */
1390 0 : if (cstate->eol_type == EOL_CRNL)
1391 0 : ereport(ERROR,
1392 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1393 : !is_csv ?
1394 : errmsg("literal carriage return found in data") :
1395 : errmsg("unquoted carriage return found in data"),
1396 : !is_csv ?
1397 : errhint("Use \"\\r\" to represent carriage return.") :
1398 : errhint("Use quoted CSV field to represent carriage return.")));
1399 :
1400 : /*
1401 : * if we got here, it is the first line and we didn't find
1402 : * \n, so don't consume the peeked character
1403 : */
1404 0 : cstate->eol_type = EOL_CR;
1405 : }
1406 : }
1407 0 : else if (cstate->eol_type == EOL_NL)
1408 0 : ereport(ERROR,
1409 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1410 : !is_csv ?
1411 : errmsg("literal carriage return found in data") :
1412 : errmsg("unquoted carriage return found in data"),
1413 : !is_csv ?
1414 : errhint("Use \"\\r\" to represent carriage return.") :
1415 : errhint("Use quoted CSV field to represent carriage return.")));
1416 : /* If reach here, we have found the line terminator */
1417 0 : break;
1418 : }
1419 :
1420 : /* Process \n */
1421 50888996 : if (c == '\n' && (!is_csv || !in_quote))
1422 : {
1423 2098340 : if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
1424 0 : ereport(ERROR,
1425 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1426 : !is_csv ?
1427 : errmsg("literal newline found in data") :
1428 : errmsg("unquoted newline found in data"),
1429 : !is_csv ?
1430 : errhint("Use \"\\n\" to represent newline.") :
1431 : errhint("Use quoted CSV field to represent newline.")));
1432 2098340 : cstate->eol_type = EOL_NL; /* in case not set yet */
1433 : /* If reach here, we have found the line terminator */
1434 2098340 : break;
1435 : }
1436 :
1437 : /*
1438 : * Process backslash, except in CSV mode where backslash is a normal
1439 : * character.
1440 : */
1441 48790656 : if (c == '\\' && !is_csv)
1442 : {
1443 : char c2;
1444 :
1445 139576 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1446 139540 : IF_NEED_REFILL_AND_EOF_BREAK(0);
1447 :
1448 : /* -----
1449 : * get next character
1450 : * Note: we do not change c so if it isn't \., we can fall
1451 : * through and continue processing.
1452 : * -----
1453 : */
1454 139540 : c2 = copy_input_buf[input_buf_ptr];
1455 :
1456 139540 : if (c2 == '.')
1457 : {
1458 1290 : input_buf_ptr++; /* consume the '.' */
1459 1290 : if (cstate->eol_type == EOL_CRNL)
1460 : {
1461 : /* Get the next character */
1462 0 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1463 : /* if hit_eof, c2 will become '\0' */
1464 0 : c2 = copy_input_buf[input_buf_ptr++];
1465 :
1466 0 : if (c2 == '\n')
1467 0 : ereport(ERROR,
1468 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1469 : errmsg("end-of-copy marker does not match previous newline style")));
1470 0 : else if (c2 != '\r')
1471 0 : ereport(ERROR,
1472 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1473 : errmsg("end-of-copy marker is not alone on its line")));
1474 : }
1475 :
1476 : /* Get the next character */
1477 1290 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1478 : /* if hit_eof, c2 will become '\0' */
1479 1290 : c2 = copy_input_buf[input_buf_ptr++];
1480 :
1481 1290 : if (c2 != '\r' && c2 != '\n')
1482 6 : ereport(ERROR,
1483 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1484 : errmsg("end-of-copy marker is not alone on its line")));
1485 :
1486 1284 : if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
1487 1284 : (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
1488 1284 : (cstate->eol_type == EOL_CR && c2 != '\r'))
1489 0 : ereport(ERROR,
1490 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1491 : errmsg("end-of-copy marker does not match previous newline style")));
1492 :
1493 : /*
1494 : * If there is any data on this line before the \., complain.
1495 : */
1496 1284 : if (cstate->line_buf.len > 0 ||
1497 1284 : prev_raw_ptr > cstate->input_buf_index)
1498 6 : ereport(ERROR,
1499 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1500 : errmsg("end-of-copy marker is not alone on its line")));
1501 :
1502 : /*
1503 : * Discard the \. and newline, then report EOF.
1504 : */
1505 1278 : cstate->input_buf_index = input_buf_ptr;
1506 1278 : result = true; /* report EOF */
1507 1278 : break;
1508 : }
1509 : else
1510 : {
1511 : /*
1512 : * If we are here, it means we found a backslash followed by
1513 : * something other than a period. In non-CSV mode, anything
1514 : * after a backslash is special, so we skip over that second
1515 : * character too. If we didn't do that \\. would be
1516 : * considered an eof-of copy, while in non-CSV mode it is a
1517 : * literal backslash followed by a period.
1518 : */
1519 138250 : input_buf_ptr++;
1520 : }
1521 : }
1522 : } /* end of outer loop */
1523 :
1524 : /*
1525 : * Transfer any still-uncopied data to line_buf.
1526 : */
1527 2101142 : REFILL_LINEBUF;
1528 :
1529 2101142 : return result;
1530 : }
1531 :
1532 : /*
1533 : * Return decimal value for a hexadecimal digit
1534 : */
1535 : static int
1536 0 : GetDecimalFromHex(char hex)
1537 : {
1538 0 : if (isdigit((unsigned char) hex))
1539 0 : return hex - '0';
1540 : else
1541 0 : return tolower((unsigned char) hex) - 'a' + 10;
1542 : }
1543 :
1544 : /*
1545 : * Parse the current line into separate attributes (fields),
1546 : * performing de-escaping as needed.
1547 : *
1548 : * The input is in line_buf. We use attribute_buf to hold the result
1549 : * strings. cstate->raw_fields[k] is set to point to the k'th attribute
1550 : * string, or NULL when the input matches the null marker string.
1551 : * This array is expanded as necessary.
1552 : *
1553 : * (Note that the caller cannot check for nulls since the returned
1554 : * string would be the post-de-escaping equivalent, which may look
1555 : * the same as some valid data string.)
1556 : *
1557 : * delim is the column delimiter string (must be just one byte for now).
1558 : * null_print is the null marker string. Note that this is compared to
1559 : * the pre-de-escaped input string.
1560 : *
1561 : * The return value is the number of fields actually read.
1562 : */
1563 : static int
1564 2097822 : CopyReadAttributesText(CopyFromState cstate)
1565 : {
1566 2097822 : char delimc = cstate->opts.delim[0];
1567 : int fieldno;
1568 : char *output_ptr;
1569 : char *cur_ptr;
1570 : char *line_end_ptr;
1571 :
1572 : /*
1573 : * We need a special case for zero-column tables: check that the input
1574 : * line is empty, and return.
1575 : */
1576 2097822 : if (cstate->max_fields <= 0)
1577 : {
1578 10 : if (cstate->line_buf.len != 0)
1579 0 : ereport(ERROR,
1580 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1581 : errmsg("extra data after last expected column")));
1582 10 : return 0;
1583 : }
1584 :
1585 2097812 : resetStringInfo(&cstate->attribute_buf);
1586 :
1587 : /*
1588 : * The de-escaped attributes will certainly not be longer than the input
1589 : * data line, so we can just force attribute_buf to be large enough and
1590 : * then transfer data without any checks for enough space. We need to do
1591 : * it this way because enlarging attribute_buf mid-stream would invalidate
1592 : * pointers already stored into cstate->raw_fields[].
1593 : */
1594 2097812 : if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
1595 30 : enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
1596 2097812 : output_ptr = cstate->attribute_buf.data;
1597 :
1598 : /* set pointer variables for loop */
1599 2097812 : cur_ptr = cstate->line_buf.data;
1600 2097812 : line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
1601 :
1602 : /* Outer loop iterates over fields */
1603 2097812 : fieldno = 0;
1604 : for (;;)
1605 4864554 : {
1606 6962366 : bool found_delim = false;
1607 : char *start_ptr;
1608 : char *end_ptr;
1609 : int input_len;
1610 6962366 : bool saw_non_ascii = false;
1611 :
1612 : /* Make sure there is enough space for the next value */
1613 6962366 : if (fieldno >= cstate->max_fields)
1614 : {
1615 36 : cstate->max_fields *= 2;
1616 36 : cstate->raw_fields =
1617 36 : repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
1618 : }
1619 :
1620 : /* Remember start of field on both input and output sides */
1621 6962366 : start_ptr = cur_ptr;
1622 6962366 : cstate->raw_fields[fieldno] = output_ptr;
1623 :
1624 : /*
1625 : * Scan data for field.
1626 : *
1627 : * Note that in this loop, we are scanning to locate the end of field
1628 : * and also speculatively performing de-escaping. Once we find the
1629 : * end-of-field, we can match the raw field contents against the null
1630 : * marker string. Only after that comparison fails do we know that
1631 : * de-escaping is actually the right thing to do; therefore we *must
1632 : * not* throw any syntax errors before we've done the null-marker
1633 : * check.
1634 : */
1635 : for (;;)
1636 43919360 : {
1637 : char c;
1638 :
1639 50881726 : end_ptr = cur_ptr;
1640 50881726 : if (cur_ptr >= line_end_ptr)
1641 2097806 : break;
1642 48783920 : c = *cur_ptr++;
1643 48783920 : if (c == delimc)
1644 : {
1645 4864560 : found_delim = true;
1646 4864560 : break;
1647 : }
1648 43919360 : if (c == '\\')
1649 : {
1650 138250 : if (cur_ptr >= line_end_ptr)
1651 0 : break;
1652 138250 : c = *cur_ptr++;
1653 138250 : switch (c)
1654 : {
1655 12 : case '0':
1656 : case '1':
1657 : case '2':
1658 : case '3':
1659 : case '4':
1660 : case '5':
1661 : case '6':
1662 : case '7':
1663 : {
1664 : /* handle \013 */
1665 : int val;
1666 :
1667 12 : val = OCTVALUE(c);
1668 12 : if (cur_ptr < line_end_ptr)
1669 : {
1670 6 : c = *cur_ptr;
1671 6 : if (ISOCTAL(c))
1672 : {
1673 0 : cur_ptr++;
1674 0 : val = (val << 3) + OCTVALUE(c);
1675 0 : if (cur_ptr < line_end_ptr)
1676 : {
1677 0 : c = *cur_ptr;
1678 0 : if (ISOCTAL(c))
1679 : {
1680 0 : cur_ptr++;
1681 0 : val = (val << 3) + OCTVALUE(c);
1682 : }
1683 : }
1684 : }
1685 : }
1686 12 : c = val & 0377;
1687 12 : if (c == '\0' || IS_HIGHBIT_SET(c))
1688 12 : saw_non_ascii = true;
1689 : }
1690 12 : break;
1691 12 : case 'x':
1692 : /* Handle \x3F */
1693 12 : if (cur_ptr < line_end_ptr)
1694 : {
1695 6 : char hexchar = *cur_ptr;
1696 :
1697 6 : if (isxdigit((unsigned char) hexchar))
1698 : {
1699 0 : int val = GetDecimalFromHex(hexchar);
1700 :
1701 0 : cur_ptr++;
1702 0 : if (cur_ptr < line_end_ptr)
1703 : {
1704 0 : hexchar = *cur_ptr;
1705 0 : if (isxdigit((unsigned char) hexchar))
1706 : {
1707 0 : cur_ptr++;
1708 0 : val = (val << 4) + GetDecimalFromHex(hexchar);
1709 : }
1710 : }
1711 0 : c = val & 0xff;
1712 0 : if (c == '\0' || IS_HIGHBIT_SET(c))
1713 0 : saw_non_ascii = true;
1714 : }
1715 : }
1716 12 : break;
1717 0 : case 'b':
1718 0 : c = '\b';
1719 0 : break;
1720 0 : case 'f':
1721 0 : c = '\f';
1722 0 : break;
1723 4182 : case 'n':
1724 4182 : c = '\n';
1725 4182 : break;
1726 0 : case 'r':
1727 0 : c = '\r';
1728 0 : break;
1729 0 : case 't':
1730 0 : c = '\t';
1731 0 : break;
1732 0 : case 'v':
1733 0 : c = '\v';
1734 0 : break;
1735 :
1736 : /*
1737 : * in all other cases, take the char after '\'
1738 : * literally
1739 : */
1740 : }
1741 43781110 : }
1742 :
1743 : /* Add c to output string */
1744 43919360 : *output_ptr++ = c;
1745 : }
1746 :
1747 : /* Check whether raw input matched null marker */
1748 6962366 : input_len = end_ptr - start_ptr;
1749 6962366 : if (input_len == cstate->opts.null_print_len &&
1750 783578 : strncmp(start_ptr, cstate->opts.null_print, input_len) == 0)
1751 133346 : cstate->raw_fields[fieldno] = NULL;
1752 : /* Check whether raw input matched default marker */
1753 6829020 : else if (fieldno < list_length(cstate->attnumlist) &&
1754 6828978 : cstate->opts.default_print &&
1755 114 : input_len == cstate->opts.default_print_len &&
1756 30 : strncmp(start_ptr, cstate->opts.default_print, input_len) == 0)
1757 24 : {
1758 : /* fieldno is 0-indexed and attnum is 1-indexed */
1759 30 : int m = list_nth_int(cstate->attnumlist, fieldno) - 1;
1760 :
1761 30 : if (cstate->defexprs[m] != NULL)
1762 : {
1763 : /* defaults contain entries for all physical attributes */
1764 24 : cstate->defaults[m] = true;
1765 : }
1766 : else
1767 : {
1768 6 : TupleDesc tupDesc = RelationGetDescr(cstate->rel);
1769 6 : Form_pg_attribute att = TupleDescAttr(tupDesc, m);
1770 :
1771 6 : ereport(ERROR,
1772 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1773 : errmsg("unexpected default marker in COPY data"),
1774 : errdetail("Column \"%s\" has no default value.",
1775 : NameStr(att->attname))));
1776 : }
1777 : }
1778 : else
1779 : {
1780 : /*
1781 : * At this point we know the field is supposed to contain data.
1782 : *
1783 : * If we de-escaped any non-7-bit-ASCII chars, make sure the
1784 : * resulting string is valid data for the db encoding.
1785 : */
1786 6828990 : if (saw_non_ascii)
1787 : {
1788 0 : char *fld = cstate->raw_fields[fieldno];
1789 :
1790 0 : pg_verifymbstr(fld, output_ptr - fld, false);
1791 : }
1792 : }
1793 :
1794 : /* Terminate attribute value in output area */
1795 6962360 : *output_ptr++ = '\0';
1796 :
1797 6962360 : fieldno++;
1798 : /* Done if we hit EOL instead of a delim */
1799 6962360 : if (!found_delim)
1800 2097806 : break;
1801 : }
1802 :
1803 : /* Clean up state of attribute_buf */
1804 2097806 : output_ptr--;
1805 : Assert(*output_ptr == '\0');
1806 2097806 : cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
1807 :
1808 2097806 : return fieldno;
1809 : }
1810 :
1811 : /*
1812 : * Parse the current line into separate attributes (fields),
1813 : * performing de-escaping as needed. This has exactly the same API as
1814 : * CopyReadAttributesText, except we parse the fields according to
1815 : * "standard" (i.e. common) CSV usage.
1816 : */
1817 : static int
1818 474 : CopyReadAttributesCSV(CopyFromState cstate)
1819 : {
1820 474 : char delimc = cstate->opts.delim[0];
1821 474 : char quotec = cstate->opts.quote[0];
1822 474 : char escapec = cstate->opts.escape[0];
1823 : int fieldno;
1824 : char *output_ptr;
1825 : char *cur_ptr;
1826 : char *line_end_ptr;
1827 :
1828 : /*
1829 : * We need a special case for zero-column tables: check that the input
1830 : * line is empty, and return.
1831 : */
1832 474 : if (cstate->max_fields <= 0)
1833 : {
1834 0 : if (cstate->line_buf.len != 0)
1835 0 : ereport(ERROR,
1836 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1837 : errmsg("extra data after last expected column")));
1838 0 : return 0;
1839 : }
1840 :
1841 474 : resetStringInfo(&cstate->attribute_buf);
1842 :
1843 : /*
1844 : * The de-escaped attributes will certainly not be longer than the input
1845 : * data line, so we can just force attribute_buf to be large enough and
1846 : * then transfer data without any checks for enough space. We need to do
1847 : * it this way because enlarging attribute_buf mid-stream would invalidate
1848 : * pointers already stored into cstate->raw_fields[].
1849 : */
1850 474 : if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
1851 0 : enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
1852 474 : output_ptr = cstate->attribute_buf.data;
1853 :
1854 : /* set pointer variables for loop */
1855 474 : cur_ptr = cstate->line_buf.data;
1856 474 : line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
1857 :
1858 : /* Outer loop iterates over fields */
1859 474 : fieldno = 0;
1860 : for (;;)
1861 530 : {
1862 1004 : bool found_delim = false;
1863 1004 : bool saw_quote = false;
1864 : char *start_ptr;
1865 : char *end_ptr;
1866 : int input_len;
1867 :
1868 : /* Make sure there is enough space for the next value */
1869 1004 : if (fieldno >= cstate->max_fields)
1870 : {
1871 0 : cstate->max_fields *= 2;
1872 0 : cstate->raw_fields =
1873 0 : repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
1874 : }
1875 :
1876 : /* Remember start of field on both input and output sides */
1877 1004 : start_ptr = cur_ptr;
1878 1004 : cstate->raw_fields[fieldno] = output_ptr;
1879 :
1880 : /*
1881 : * Scan data for field,
1882 : *
1883 : * The loop starts in "not quote" mode and then toggles between that
1884 : * and "in quote" mode. The loop exits normally if it is in "not
1885 : * quote" mode and a delimiter or line end is seen.
1886 : */
1887 : for (;;)
1888 222 : {
1889 : char c;
1890 :
1891 : /* Not in quote */
1892 : for (;;)
1893 : {
1894 3230 : end_ptr = cur_ptr;
1895 3230 : if (cur_ptr >= line_end_ptr)
1896 468 : goto endfield;
1897 2762 : c = *cur_ptr++;
1898 : /* unquoted field delimiter */
1899 2762 : if (c == delimc)
1900 : {
1901 536 : found_delim = true;
1902 536 : goto endfield;
1903 : }
1904 : /* start of quoted field (or part of field) */
1905 2226 : if (c == quotec)
1906 : {
1907 222 : saw_quote = true;
1908 222 : break;
1909 : }
1910 : /* Add c to output string */
1911 2004 : *output_ptr++ = c;
1912 : }
1913 :
1914 : /* In quote */
1915 : for (;;)
1916 : {
1917 1390 : end_ptr = cur_ptr;
1918 1390 : if (cur_ptr >= line_end_ptr)
1919 0 : ereport(ERROR,
1920 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1921 : errmsg("unterminated CSV quoted field")));
1922 :
1923 1390 : c = *cur_ptr++;
1924 :
1925 : /* escape within a quoted field */
1926 1390 : if (c == escapec)
1927 : {
1928 : /*
1929 : * peek at the next char if available, and escape it if it
1930 : * is an escape char or a quote char
1931 : */
1932 118 : if (cur_ptr < line_end_ptr)
1933 : {
1934 72 : char nextc = *cur_ptr;
1935 :
1936 72 : if (nextc == escapec || nextc == quotec)
1937 : {
1938 24 : *output_ptr++ = nextc;
1939 24 : cur_ptr++;
1940 24 : continue;
1941 : }
1942 : }
1943 : }
1944 :
1945 : /*
1946 : * end of quoted field. Must do this test after testing for
1947 : * escape in case quote char and escape char are the same
1948 : * (which is the common case).
1949 : */
1950 1366 : if (c == quotec)
1951 222 : break;
1952 :
1953 : /* Add c to output string */
1954 1144 : *output_ptr++ = c;
1955 : }
1956 : }
1957 1004 : endfield:
1958 :
1959 : /* Terminate attribute value in output area */
1960 1004 : *output_ptr++ = '\0';
1961 :
1962 : /* Check whether raw input matched null marker */
1963 1004 : input_len = end_ptr - start_ptr;
1964 1004 : if (!saw_quote && input_len == cstate->opts.null_print_len &&
1965 44 : strncmp(start_ptr, cstate->opts.null_print, input_len) == 0)
1966 44 : cstate->raw_fields[fieldno] = NULL;
1967 : /* Check whether raw input matched default marker */
1968 960 : else if (fieldno < list_length(cstate->attnumlist) &&
1969 960 : cstate->opts.default_print &&
1970 150 : input_len == cstate->opts.default_print_len &&
1971 42 : strncmp(start_ptr, cstate->opts.default_print, input_len) == 0)
1972 : {
1973 : /* fieldno is 0-index and attnum is 1-index */
1974 42 : int m = list_nth_int(cstate->attnumlist, fieldno) - 1;
1975 :
1976 42 : if (cstate->defexprs[m] != NULL)
1977 : {
1978 : /* defaults contain entries for all physical attributes */
1979 36 : cstate->defaults[m] = true;
1980 : }
1981 : else
1982 : {
1983 6 : TupleDesc tupDesc = RelationGetDescr(cstate->rel);
1984 6 : Form_pg_attribute att = TupleDescAttr(tupDesc, m);
1985 :
1986 6 : ereport(ERROR,
1987 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1988 : errmsg("unexpected default marker in COPY data"),
1989 : errdetail("Column \"%s\" has no default value.",
1990 : NameStr(att->attname))));
1991 : }
1992 : }
1993 :
1994 998 : fieldno++;
1995 : /* Done if we hit EOL instead of a delim */
1996 998 : if (!found_delim)
1997 468 : break;
1998 : }
1999 :
2000 : /* Clean up state of attribute_buf */
2001 468 : output_ptr--;
2002 : Assert(*output_ptr == '\0');
2003 468 : cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
2004 :
2005 468 : return fieldno;
2006 : }
2007 :
2008 :
2009 : /*
2010 : * Read a binary attribute
2011 : */
2012 : static Datum
2013 158 : CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
2014 : Oid typioparam, int32 typmod,
2015 : bool *isnull)
2016 : {
2017 : int32 fld_size;
2018 : Datum result;
2019 :
2020 158 : if (!CopyGetInt32(cstate, &fld_size))
2021 0 : ereport(ERROR,
2022 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2023 : errmsg("unexpected EOF in COPY data")));
2024 158 : if (fld_size == -1)
2025 : {
2026 30 : *isnull = true;
2027 30 : return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
2028 : }
2029 128 : if (fld_size < 0)
2030 0 : ereport(ERROR,
2031 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2032 : errmsg("invalid field size")));
2033 :
2034 : /* reset attribute_buf to empty, and load raw data in it */
2035 128 : resetStringInfo(&cstate->attribute_buf);
2036 :
2037 128 : enlargeStringInfo(&cstate->attribute_buf, fld_size);
2038 128 : if (CopyReadBinaryData(cstate, cstate->attribute_buf.data,
2039 128 : fld_size) != fld_size)
2040 0 : ereport(ERROR,
2041 : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2042 : errmsg("unexpected EOF in COPY data")));
2043 :
2044 128 : cstate->attribute_buf.len = fld_size;
2045 128 : cstate->attribute_buf.data[fld_size] = '\0';
2046 :
2047 : /* Call the column type's binary input converter */
2048 128 : result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
2049 : typioparam, typmod);
2050 :
2051 : /* Trouble if it didn't eat the whole buffer */
2052 128 : if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
2053 2 : ereport(ERROR,
2054 : (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
2055 : errmsg("incorrect binary data format")));
2056 :
2057 126 : *isnull = false;
2058 126 : return result;
2059 : }
|
