LCOV - code coverage report
Current view: top level - src/backend/commands - copyfromparse.c (source / functions) Hit Total Coverage
Test: PostgreSQL 17devel Lines: 488 639 76.4 %
Date: 2024-04-22 22:11:44 Functions: 16 18 88.9 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * copyfromparse.c
       4             :  *      Parse CSV/text/binary format for COPY FROM.
       5             :  *
       6             :  * This file contains routines to parse the text, CSV and binary input
       7             :  * formats.  The main entry point is NextCopyFrom(), which parses the
       8             :  * next input line and returns it as Datums.
       9             :  *
      10             :  * In text/CSV mode, the parsing happens in multiple stages:
      11             :  *
      12             :  * [data source] --> raw_buf --> input_buf --> line_buf --> attribute_buf
      13             :  *                1.          2.            3.           4.
      14             :  *
      15             :  * 1. CopyLoadRawBuf() reads raw data from the input file or client, and
      16             :  *    places it into 'raw_buf'.
      17             :  *
      18             :  * 2. CopyConvertBuf() calls the encoding conversion function to convert
      19             :  *    the data in 'raw_buf' from client to server encoding, placing the
      20             :  *    converted result in 'input_buf'.
      21             :  *
      22             :  * 3. CopyReadLine() parses the data in 'input_buf', one line at a time.
      23             :  *    It is responsible for finding the next newline marker, taking quote and
      24             :  *    escape characters into account according to the COPY options.  The line
      25             :  *    is copied into 'line_buf', with quotes and escape characters still
      26             :  *    intact.
      27             :  *
      28             :  * 4. CopyReadAttributesText/CSV() function takes the input line from
      29             :  *    'line_buf', and splits it into fields, unescaping the data as required.
      30             :  *    The fields are stored in 'attribute_buf', and 'raw_fields' array holds
      31             :  *    pointers to each field.
      32             :  *
      33             :  * If encoding conversion is not required, a shortcut is taken in step 2 to
      34             :  * avoid copying the data unnecessarily.  The 'input_buf' pointer is set to
      35             :  * point directly to 'raw_buf', so that CopyLoadRawBuf() loads the raw data
      36             :  * directly into 'input_buf'.  CopyConvertBuf() then merely validates that
      37             :  * the data is valid in the current encoding.
      38             :  *
      39             :  * In binary mode, the pipeline is much simpler.  Input is loaded into
      40             :  * 'raw_buf', and encoding conversion is done in the datatype-specific
      41             :  * receive functions, if required.  'input_buf' and 'line_buf' are not used,
      42             :  * but 'attribute_buf' is used as a temporary buffer to hold one attribute's
      43             :  * data when it's passed the receive function.
      44             :  *
      45             :  * 'raw_buf' is always 64 kB in size (RAW_BUF_SIZE).  'input_buf' is also
      46             :  * 64 kB (INPUT_BUF_SIZE), if encoding conversion is required.  'line_buf'
      47             :  * and 'attribute_buf' are expanded on demand, to hold the longest line
      48             :  * encountered so far.
      49             :  *
      50             :  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
      51             :  * Portions Copyright (c) 1994, Regents of the University of California
      52             :  *
      53             :  *
      54             :  * IDENTIFICATION
      55             :  *    src/backend/commands/copyfromparse.c
      56             :  *
      57             :  *-------------------------------------------------------------------------
      58             :  */
      59             : #include "postgres.h"
      60             : 
      61             : #include <ctype.h>
      62             : #include <unistd.h>
      63             : #include <sys/stat.h>
      64             : 
      65             : #include "commands/copy.h"
      66             : #include "commands/copyfrom_internal.h"
      67             : #include "commands/progress.h"
      68             : #include "executor/executor.h"
      69             : #include "libpq/libpq.h"
      70             : #include "libpq/pqformat.h"
      71             : #include "mb/pg_wchar.h"
      72             : #include "miscadmin.h"
      73             : #include "nodes/miscnodes.h"
      74             : #include "pgstat.h"
      75             : #include "port/pg_bswap.h"
      76             : #include "utils/builtins.h"
      77             : #include "utils/rel.h"
      78             : 
      79             : #define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
      80             : #define OCTVALUE(c) ((c) - '0')
      81             : 
      82             : /*
      83             :  * These macros centralize code used to process line_buf and input_buf buffers.
      84             :  * They are macros because they often do continue/break control and to avoid
      85             :  * function call overhead in tight COPY loops.
      86             :  *
      87             :  * We must use "if (1)" because the usual "do {...} while(0)" wrapper would
      88             :  * prevent the continue/break processing from working.  We end the "if (1)"
      89             :  * with "else ((void) 0)" to ensure the "if" does not unintentionally match
      90             :  * any "else" in the calling code, and to avoid any compiler warnings about
      91             :  * empty statements.  See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
      92             :  */
      93             : 
      94             : /*
      95             :  * This keeps the character read at the top of the loop in the buffer
      96             :  * even if there is more than one read-ahead.
      97             :  */
      98             : #define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
      99             : if (1) \
     100             : { \
     101             :     if (input_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
     102             :     { \
     103             :         input_buf_ptr = prev_raw_ptr; /* undo fetch */ \
     104             :         need_data = true; \
     105             :         continue; \
     106             :     } \
     107             : } else ((void) 0)
     108             : 
     109             : /* This consumes the remainder of the buffer and breaks */
     110             : #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
     111             : if (1) \
     112             : { \
     113             :     if (input_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
     114             :     { \
     115             :         if (extralen) \
     116             :             input_buf_ptr = copy_buf_len; /* consume the partial character */ \
     117             :         /* backslash just before EOF, treat as data char */ \
     118             :         result = true; \
     119             :         break; \
     120             :     } \
     121             : } else ((void) 0)
     122             : 
     123             : /*
     124             :  * Transfer any approved data to line_buf; must do this to be sure
     125             :  * there is some room in input_buf.
     126             :  */
     127             : #define REFILL_LINEBUF \
     128             : if (1) \
     129             : { \
     130             :     if (input_buf_ptr > cstate->input_buf_index) \
     131             :     { \
     132             :         appendBinaryStringInfo(&cstate->line_buf, \
     133             :                              cstate->input_buf + cstate->input_buf_index, \
     134             :                                input_buf_ptr - cstate->input_buf_index); \
     135             :         cstate->input_buf_index = input_buf_ptr; \
     136             :     } \
     137             : } else ((void) 0)
     138             : 
     139             : /* Undo any read-ahead and jump out of the block. */
     140             : #define NO_END_OF_COPY_GOTO \
     141             : if (1) \
     142             : { \
     143             :     input_buf_ptr = prev_raw_ptr + 1; \
     144             :     goto not_end_of_copy; \
     145             : } else ((void) 0)
     146             : 
     147             : /* NOTE: there's a copy of this in copyto.c */
     148             : static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0";
     149             : 
     150             : 
     151             : /* non-export function prototypes */
     152             : static bool CopyReadLine(CopyFromState cstate);
     153             : static bool CopyReadLineText(CopyFromState cstate);
     154             : static int  CopyReadAttributesText(CopyFromState cstate);
     155             : static int  CopyReadAttributesCSV(CopyFromState cstate);
     156             : static Datum CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
     157             :                                      Oid typioparam, int32 typmod,
     158             :                                      bool *isnull);
     159             : 
     160             : 
     161             : /* Low-level communications functions */
     162             : static int  CopyGetData(CopyFromState cstate, void *databuf,
     163             :                         int minread, int maxread);
     164             : static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
     165             : static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
     166             : static void CopyLoadInputBuf(CopyFromState cstate);
     167             : static int  CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
     168             : 
     169             : void
     170         890 : ReceiveCopyBegin(CopyFromState cstate)
     171             : {
     172             :     StringInfoData buf;
     173         890 :     int         natts = list_length(cstate->attnumlist);
     174         890 :     int16       format = (cstate->opts.binary ? 1 : 0);
     175             :     int         i;
     176             : 
     177         890 :     pq_beginmessage(&buf, PqMsg_CopyInResponse);
     178         890 :     pq_sendbyte(&buf, format);  /* overall format */
     179         890 :     pq_sendint16(&buf, natts);
     180        3000 :     for (i = 0; i < natts; i++)
     181        2110 :         pq_sendint16(&buf, format); /* per-column formats */
     182         890 :     pq_endmessage(&buf);
     183         890 :     cstate->copy_src = COPY_FRONTEND;
     184         890 :     cstate->fe_msgbuf = makeStringInfo();
     185             :     /* We *must* flush here to ensure FE knows it can send. */
     186         890 :     pq_flush();
     187         890 : }
     188             : 
     189             : void
     190          14 : ReceiveCopyBinaryHeader(CopyFromState cstate)
     191             : {
     192             :     char        readSig[11];
     193             :     int32       tmp;
     194             : 
     195             :     /* Signature */
     196          14 :     if (CopyReadBinaryData(cstate, readSig, 11) != 11 ||
     197          14 :         memcmp(readSig, BinarySignature, 11) != 0)
     198           0 :         ereport(ERROR,
     199             :                 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
     200             :                  errmsg("COPY file signature not recognized")));
     201             :     /* Flags field */
     202          14 :     if (!CopyGetInt32(cstate, &tmp))
     203           0 :         ereport(ERROR,
     204             :                 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
     205             :                  errmsg("invalid COPY file header (missing flags)")));
     206          14 :     if ((tmp & (1 << 16)) != 0)
     207           0 :         ereport(ERROR,
     208             :                 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
     209             :                  errmsg("invalid COPY file header (WITH OIDS)")));
     210          14 :     tmp &= ~(1 << 16);
     211          14 :     if ((tmp >> 16) != 0)
     212           0 :         ereport(ERROR,
     213             :                 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
     214             :                  errmsg("unrecognized critical flags in COPY file header")));
     215             :     /* Header extension length */
     216          14 :     if (!CopyGetInt32(cstate, &tmp) ||
     217          14 :         tmp < 0)
     218           0 :         ereport(ERROR,
     219             :                 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
     220             :                  errmsg("invalid COPY file header (missing length)")));
     221             :     /* Skip extension header, if present */
     222          14 :     while (tmp-- > 0)
     223             :     {
     224           0 :         if (CopyReadBinaryData(cstate, readSig, 1) != 1)
     225           0 :             ereport(ERROR,
     226             :                     (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
     227             :                      errmsg("invalid COPY file header (wrong length)")));
     228             :     }
     229          14 : }
     230             : 
     231             : /*
     232             :  * CopyGetData reads data from the source (file or frontend)
     233             :  *
     234             :  * We attempt to read at least minread, and at most maxread, bytes from
     235             :  * the source.  The actual number of bytes read is returned; if this is
     236             :  * less than minread, EOF was detected.
     237             :  *
     238             :  * Note: when copying from the frontend, we expect a proper EOF mark per
     239             :  * protocol; if the frontend simply drops the connection, we raise error.
     240             :  * It seems unwise to allow the COPY IN to complete normally in that case.
     241             :  *
     242             :  * NB: no data conversion is applied here.
     243             :  */
     244             : static int
     245      430746 : CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
     246             : {
     247      430746 :     int         bytesread = 0;
     248             : 
     249      430746 :     switch (cstate->copy_src)
     250             :     {
     251         970 :         case COPY_FILE:
     252         970 :             bytesread = fread(databuf, 1, maxread, cstate->copy_file);
     253         970 :             if (ferror(cstate->copy_file))
     254           0 :                 ereport(ERROR,
     255             :                         (errcode_for_file_access(),
     256             :                          errmsg("could not read from COPY file: %m")));
     257         970 :             if (bytesread == 0)
     258         368 :                 cstate->raw_reached_eof = true;
     259         970 :             break;
     260      401858 :         case COPY_FRONTEND:
     261      802924 :             while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof)
     262             :             {
     263             :                 int         avail;
     264             : 
     265      401778 :                 while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
     266             :                 {
     267             :                     /* Try to receive another message */
     268             :                     int         mtype;
     269             :                     int         maxmsglen;
     270             : 
     271      401778 :             readmessage:
     272      401778 :                     HOLD_CANCEL_INTERRUPTS();
     273      401778 :                     pq_startmsgread();
     274      401778 :                     mtype = pq_getbyte();
     275      401778 :                     if (mtype == EOF)
     276           0 :                         ereport(ERROR,
     277             :                                 (errcode(ERRCODE_CONNECTION_FAILURE),
     278             :                                  errmsg("unexpected EOF on client connection with an open transaction")));
     279             :                     /* Validate message type and set packet size limit */
     280             :                     switch (mtype)
     281             :                     {
     282      401066 :                         case PqMsg_CopyData:
     283      401066 :                             maxmsglen = PQ_LARGE_MESSAGE_LIMIT;
     284      401066 :                             break;
     285         712 :                         case PqMsg_CopyDone:
     286             :                         case PqMsg_CopyFail:
     287             :                         case PqMsg_Flush:
     288             :                         case PqMsg_Sync:
     289         712 :                             maxmsglen = PQ_SMALL_MESSAGE_LIMIT;
     290         712 :                             break;
     291           0 :                         default:
     292           0 :                             ereport(ERROR,
     293             :                                     (errcode(ERRCODE_PROTOCOL_VIOLATION),
     294             :                                      errmsg("unexpected message type 0x%02X during COPY from stdin",
     295             :                                             mtype)));
     296             :                             maxmsglen = 0;  /* keep compiler quiet */
     297             :                             break;
     298             :                     }
     299             :                     /* Now collect the message body */
     300      401778 :                     if (pq_getmessage(cstate->fe_msgbuf, maxmsglen))
     301           0 :                         ereport(ERROR,
     302             :                                 (errcode(ERRCODE_CONNECTION_FAILURE),
     303             :                                  errmsg("unexpected EOF on client connection with an open transaction")));
     304      401778 :                     RESUME_CANCEL_INTERRUPTS();
     305             :                     /* ... and process it */
     306             :                     switch (mtype)
     307             :                     {
     308      401066 :                         case PqMsg_CopyData:
     309      401066 :                             break;
     310         712 :                         case PqMsg_CopyDone:
     311             :                             /* COPY IN correctly terminated by frontend */
     312         712 :                             cstate->raw_reached_eof = true;
     313         712 :                             return bytesread;
     314           0 :                         case PqMsg_CopyFail:
     315           0 :                             ereport(ERROR,
     316             :                                     (errcode(ERRCODE_QUERY_CANCELED),
     317             :                                      errmsg("COPY from stdin failed: %s",
     318             :                                             pq_getmsgstring(cstate->fe_msgbuf))));
     319             :                             break;
     320           0 :                         case PqMsg_Flush:
     321             :                         case PqMsg_Sync:
     322             : 
     323             :                             /*
     324             :                              * Ignore Flush/Sync for the convenience of client
     325             :                              * libraries (such as libpq) that may send those
     326             :                              * without noticing that the command they just
     327             :                              * sent was COPY.
     328             :                              */
     329           0 :                             goto readmessage;
     330      802844 :                         default:
     331             :                             Assert(false);  /* NOT REACHED */
     332             :                     }
     333             :                 }
     334      401066 :                 avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
     335      401066 :                 if (avail > maxread)
     336           0 :                     avail = maxread;
     337      401066 :                 pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
     338      401066 :                 databuf = (void *) ((char *) databuf + avail);
     339      401066 :                 maxread -= avail;
     340      401066 :                 bytesread += avail;
     341             :             }
     342      401146 :             break;
     343       27918 :         case COPY_CALLBACK:
     344       27918 :             bytesread = cstate->data_source_cb(databuf, minread, maxread);
     345       27918 :             break;
     346             :     }
     347             : 
     348      430034 :     return bytesread;
     349             : }
     350             : 
     351             : 
     352             : /*
     353             :  * These functions do apply some data conversion
     354             :  */
     355             : 
     356             : /*
     357             :  * CopyGetInt32 reads an int32 that appears in network byte order
     358             :  *
     359             :  * Returns true if OK, false if EOF
     360             :  */
     361             : static inline bool
     362         186 : CopyGetInt32(CopyFromState cstate, int32 *val)
     363             : {
     364             :     uint32      buf;
     365             : 
     366         186 :     if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
     367             :     {
     368           0 :         *val = 0;               /* suppress compiler warning */
     369           0 :         return false;
     370             :     }
     371         186 :     *val = (int32) pg_ntoh32(buf);
     372         186 :     return true;
     373             : }
     374             : 
     375             : /*
     376             :  * CopyGetInt16 reads an int16 that appears in network byte order
     377             :  */
     378             : static inline bool
     379          42 : CopyGetInt16(CopyFromState cstate, int16 *val)
     380             : {
     381             :     uint16      buf;
     382             : 
     383          42 :     if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
     384             :     {
     385           0 :         *val = 0;               /* suppress compiler warning */
     386           0 :         return false;
     387             :     }
     388          42 :     *val = (int16) pg_ntoh16(buf);
     389          42 :     return true;
     390             : }
     391             : 
     392             : 
     393             : /*
     394             :  * Perform encoding conversion on data in 'raw_buf', writing the converted
     395             :  * data into 'input_buf'.
     396             :  *
     397             :  * On entry, there must be some data to convert in 'raw_buf'.
     398             :  */
     399             : static void
     400      859996 : CopyConvertBuf(CopyFromState cstate)
     401             : {
     402             :     /*
     403             :      * If the file and server encoding are the same, no encoding conversion is
     404             :      * required.  However, we still need to verify that the input is valid for
     405             :      * the encoding.
     406             :      */
     407      859996 :     if (!cstate->need_transcoding)
     408             :     {
     409             :         /*
     410             :          * When conversion is not required, input_buf and raw_buf are the
     411             :          * same.  raw_buf_len is the total number of bytes in the buffer, and
     412             :          * input_buf_len tracks how many of those bytes have already been
     413             :          * verified.
     414             :          */
     415      859996 :         int         preverifiedlen = cstate->input_buf_len;
     416      859996 :         int         unverifiedlen = cstate->raw_buf_len - cstate->input_buf_len;
     417             :         int         nverified;
     418             : 
     419      859996 :         if (unverifiedlen == 0)
     420             :         {
     421             :             /*
     422             :              * If no more raw data is coming, report the EOF to the caller.
     423             :              */
     424      430768 :             if (cstate->raw_reached_eof)
     425         770 :                 cstate->input_reached_eof = true;
     426      430768 :             return;
     427             :         }
     428             : 
     429             :         /*
     430             :          * Verify the new data, including any residual unverified bytes from
     431             :          * previous round.
     432             :          */
     433      429228 :         nverified = pg_encoding_verifymbstr(cstate->file_encoding,
     434      429228 :                                             cstate->raw_buf + preverifiedlen,
     435             :                                             unverifiedlen);
     436      429228 :         if (nverified == 0)
     437             :         {
     438             :             /*
     439             :              * Could not verify anything.
     440             :              *
     441             :              * If there is no more raw input data coming, it means that there
     442             :              * was an incomplete multi-byte sequence at the end.  Also, if
     443             :              * there's "enough" input left, we should be able to verify at
     444             :              * least one character, and a failure to do so means that we've
     445             :              * hit an invalid byte sequence.
     446             :              */
     447           0 :             if (cstate->raw_reached_eof || unverifiedlen >= pg_encoding_max_length(cstate->file_encoding))
     448           0 :                 cstate->input_reached_error = true;
     449           0 :             return;
     450             :         }
     451      429228 :         cstate->input_buf_len += nverified;
     452             :     }
     453             :     else
     454             :     {
     455             :         /*
     456             :          * Encoding conversion is needed.
     457             :          */
     458             :         int         nbytes;
     459             :         unsigned char *src;
     460             :         int         srclen;
     461             :         unsigned char *dst;
     462             :         int         dstlen;
     463             :         int         convertedlen;
     464             : 
     465           0 :         if (RAW_BUF_BYTES(cstate) == 0)
     466             :         {
     467             :             /*
     468             :              * If no more raw data is coming, report the EOF to the caller.
     469             :              */
     470           0 :             if (cstate->raw_reached_eof)
     471           0 :                 cstate->input_reached_eof = true;
     472           0 :             return;
     473             :         }
     474             : 
     475             :         /*
     476             :          * First, copy down any unprocessed data.
     477             :          */
     478           0 :         nbytes = INPUT_BUF_BYTES(cstate);
     479           0 :         if (nbytes > 0 && cstate->input_buf_index > 0)
     480           0 :             memmove(cstate->input_buf, cstate->input_buf + cstate->input_buf_index,
     481             :                     nbytes);
     482           0 :         cstate->input_buf_index = 0;
     483           0 :         cstate->input_buf_len = nbytes;
     484           0 :         cstate->input_buf[nbytes] = '\0';
     485             : 
     486           0 :         src = (unsigned char *) cstate->raw_buf + cstate->raw_buf_index;
     487           0 :         srclen = cstate->raw_buf_len - cstate->raw_buf_index;
     488           0 :         dst = (unsigned char *) cstate->input_buf + cstate->input_buf_len;
     489           0 :         dstlen = INPUT_BUF_SIZE - cstate->input_buf_len + 1;
     490             : 
     491             :         /*
     492             :          * Do the conversion.  This might stop short, if there is an invalid
     493             :          * byte sequence in the input.  We'll convert as much as we can in
     494             :          * that case.
     495             :          *
     496             :          * Note: Even if we hit an invalid byte sequence, we don't report the
     497             :          * error until all the valid bytes have been consumed.  The input
     498             :          * might contain an end-of-input marker (\.), and we don't want to
     499             :          * report an error if the invalid byte sequence is after the
     500             :          * end-of-input marker.  We might unnecessarily convert some data
     501             :          * after the end-of-input marker as long as it's valid for the
     502             :          * encoding, but that's harmless.
     503             :          */
     504           0 :         convertedlen = pg_do_encoding_conversion_buf(cstate->conversion_proc,
     505             :                                                      cstate->file_encoding,
     506             :                                                      GetDatabaseEncoding(),
     507             :                                                      src, srclen,
     508             :                                                      dst, dstlen,
     509             :                                                      true);
     510           0 :         if (convertedlen == 0)
     511             :         {
     512             :             /*
     513             :              * Could not convert anything.  If there is no more raw input data
     514             :              * coming, it means that there was an incomplete multi-byte
     515             :              * sequence at the end.  Also, if there is plenty of input left,
     516             :              * we should be able to convert at least one character, so a
     517             :              * failure to do so must mean that we've hit a byte sequence
     518             :              * that's invalid.
     519             :              */
     520           0 :             if (cstate->raw_reached_eof || srclen >= MAX_CONVERSION_INPUT_LENGTH)
     521           0 :                 cstate->input_reached_error = true;
     522           0 :             return;
     523             :         }
     524           0 :         cstate->raw_buf_index += convertedlen;
     525           0 :         cstate->input_buf_len += strlen((char *) dst);
     526             :     }
     527             : }
     528             : 
     529             : /*
     530             :  * Report an encoding or conversion error.
     531             :  */
     532             : static void
     533           0 : CopyConversionError(CopyFromState cstate)
     534             : {
     535             :     Assert(cstate->raw_buf_len > 0);
     536             :     Assert(cstate->input_reached_error);
     537             : 
     538           0 :     if (!cstate->need_transcoding)
     539             :     {
     540             :         /*
     541             :          * Everything up to input_buf_len was successfully verified, and
     542             :          * input_buf_len points to the invalid or incomplete character.
     543             :          */
     544           0 :         report_invalid_encoding(cstate->file_encoding,
     545           0 :                                 cstate->raw_buf + cstate->input_buf_len,
     546           0 :                                 cstate->raw_buf_len - cstate->input_buf_len);
     547             :     }
     548             :     else
     549             :     {
     550             :         /*
     551             :          * raw_buf_index points to the invalid or untranslatable character. We
     552             :          * let the conversion routine report the error, because it can provide
     553             :          * a more specific error message than we could here.  An earlier call
     554             :          * to the conversion routine in CopyConvertBuf() detected that there
     555             :          * is an error, now we call the conversion routine again with
     556             :          * noError=false, to have it throw the error.
     557             :          */
     558             :         unsigned char *src;
     559             :         int         srclen;
     560             :         unsigned char *dst;
     561             :         int         dstlen;
     562             : 
     563           0 :         src = (unsigned char *) cstate->raw_buf + cstate->raw_buf_index;
     564           0 :         srclen = cstate->raw_buf_len - cstate->raw_buf_index;
     565           0 :         dst = (unsigned char *) cstate->input_buf + cstate->input_buf_len;
     566           0 :         dstlen = INPUT_BUF_SIZE - cstate->input_buf_len + 1;
     567             : 
     568           0 :         (void) pg_do_encoding_conversion_buf(cstate->conversion_proc,
     569             :                                              cstate->file_encoding,
     570             :                                              GetDatabaseEncoding(),
     571             :                                              src, srclen,
     572             :                                              dst, dstlen,
     573             :                                              false);
     574             : 
     575             :         /*
     576             :          * The conversion routine should have reported an error, so this
     577             :          * should not be reached.
     578             :          */
     579           0 :         elog(ERROR, "encoding conversion failed without error");
     580             :     }
     581             : }
     582             : 
     583             : /*
     584             :  * Load more data from data source to raw_buf.
     585             :  *
     586             :  * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the
     587             :  * beginning of the buffer, and we load new data after that.
     588             :  */
     589             : static void
     590      430034 : CopyLoadRawBuf(CopyFromState cstate)
     591             : {
     592             :     int         nbytes;
     593             :     int         inbytes;
     594             : 
     595             :     /*
     596             :      * In text mode, if encoding conversion is not required, raw_buf and
     597             :      * input_buf point to the same buffer.  Their len/index better agree, too.
     598             :      */
     599      430034 :     if (cstate->raw_buf == cstate->input_buf)
     600             :     {
     601             :         Assert(!cstate->need_transcoding);
     602             :         Assert(cstate->raw_buf_index == cstate->input_buf_index);
     603             :         Assert(cstate->input_buf_len <= cstate->raw_buf_len);
     604             :     }
     605             : 
     606             :     /*
     607             :      * Copy down the unprocessed data if any.
     608             :      */
     609      430034 :     nbytes = RAW_BUF_BYTES(cstate);
     610      430034 :     if (nbytes > 0 && cstate->raw_buf_index > 0)
     611           0 :         memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
     612             :                 nbytes);
     613      430034 :     cstate->raw_buf_len -= cstate->raw_buf_index;
     614      430034 :     cstate->raw_buf_index = 0;
     615             : 
     616             :     /*
     617             :      * If raw_buf and input_buf are in fact the same buffer, adjust the
     618             :      * input_buf variables, too.
     619             :      */
     620      430034 :     if (cstate->raw_buf == cstate->input_buf)
     621             :     {
     622      429998 :         cstate->input_buf_len -= cstate->input_buf_index;
     623      429998 :         cstate->input_buf_index = 0;
     624             :     }
     625             : 
     626             :     /* Load more data */
     627      430034 :     inbytes = CopyGetData(cstate, cstate->raw_buf + cstate->raw_buf_len,
     628      430034 :                           1, RAW_BUF_SIZE - cstate->raw_buf_len);
     629      430034 :     nbytes += inbytes;
     630      430034 :     cstate->raw_buf[nbytes] = '\0';
     631      430034 :     cstate->raw_buf_len = nbytes;
     632             : 
     633      430034 :     cstate->bytes_processed += inbytes;
     634      430034 :     pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
     635             : 
     636      430034 :     if (inbytes == 0)
     637         782 :         cstate->raw_reached_eof = true;
     638      430034 : }
     639             : 
     640             : /*
     641             :  * CopyLoadInputBuf loads some more data into input_buf
     642             :  *
     643             :  * On return, at least one more input character is loaded into
     644             :  * input_buf, or input_reached_eof is set.
     645             :  *
     646             :  * If INPUT_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
     647             :  * of the buffer and then we load more data after that.
     648             :  */
     649             : static void
     650      429998 : CopyLoadInputBuf(CopyFromState cstate)
     651             : {
     652      429998 :     int         nbytes = INPUT_BUF_BYTES(cstate);
     653             : 
     654             :     /*
     655             :      * The caller has updated input_buf_index to indicate how much of the
     656             :      * input has been consumed and isn't needed anymore.  If input_buf is the
     657             :      * same physical area as raw_buf, update raw_buf_index accordingly.
     658             :      */
     659      429998 :     if (cstate->raw_buf == cstate->input_buf)
     660             :     {
     661             :         Assert(!cstate->need_transcoding);
     662             :         Assert(cstate->input_buf_index >= cstate->raw_buf_index);
     663      429998 :         cstate->raw_buf_index = cstate->input_buf_index;
     664             :     }
     665             : 
     666             :     for (;;)
     667             :     {
     668             :         /* If we now have some unconverted data, try to convert it */
     669      859996 :         CopyConvertBuf(cstate);
     670             : 
     671             :         /* If we now have some more input bytes ready, return them */
     672      859996 :         if (INPUT_BUF_BYTES(cstate) > nbytes)
     673      429228 :             return;
     674             : 
     675             :         /*
     676             :          * If we reached an invalid byte sequence, or we're at an incomplete
     677             :          * multi-byte character but there is no more raw input data, report
     678             :          * conversion error.
     679             :          */
     680      430768 :         if (cstate->input_reached_error)
     681           0 :             CopyConversionError(cstate);
     682             : 
     683             :         /* no more input, and everything has been converted */
     684      430768 :         if (cstate->input_reached_eof)
     685         770 :             break;
     686             : 
     687             :         /* Try to load more raw data */
     688             :         Assert(!cstate->raw_reached_eof);
     689      429998 :         CopyLoadRawBuf(cstate);
     690             :     }
     691             : }
     692             : 
     693             : /*
     694             :  * CopyReadBinaryData
     695             :  *
     696             :  * Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf
     697             :  * and writes them to 'dest'.  Returns the number of bytes read (which
     698             :  * would be less than 'nbytes' only if we reach EOF).
     699             :  */
     700             : static int
     701         382 : CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
     702             : {
     703         382 :     int         copied_bytes = 0;
     704             : 
     705         382 :     if (RAW_BUF_BYTES(cstate) >= nbytes)
     706             :     {
     707             :         /* Enough bytes are present in the buffer. */
     708         346 :         memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, nbytes);
     709         346 :         cstate->raw_buf_index += nbytes;
     710         346 :         copied_bytes = nbytes;
     711             :     }
     712             :     else
     713             :     {
     714             :         /*
     715             :          * Not enough bytes in the buffer, so must read from the file.  Need
     716             :          * to loop since 'nbytes' could be larger than the buffer size.
     717             :          */
     718             :         do
     719             :         {
     720             :             int         copy_bytes;
     721             : 
     722             :             /* Load more data if buffer is empty. */
     723          36 :             if (RAW_BUF_BYTES(cstate) == 0)
     724             :             {
     725          36 :                 CopyLoadRawBuf(cstate);
     726          36 :                 if (cstate->raw_reached_eof)
     727          12 :                     break;      /* EOF */
     728             :             }
     729             : 
     730             :             /* Transfer some bytes. */
     731          24 :             copy_bytes = Min(nbytes - copied_bytes, RAW_BUF_BYTES(cstate));
     732          24 :             memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, copy_bytes);
     733          24 :             cstate->raw_buf_index += copy_bytes;
     734          24 :             dest += copy_bytes;
     735          24 :             copied_bytes += copy_bytes;
     736          24 :         } while (copied_bytes < nbytes);
     737             :     }
     738             : 
     739         382 :     return copied_bytes;
     740             : }
     741             : 
     742             : /*
     743             :  * Read raw fields in the next line for COPY FROM in text or csv mode.
     744             :  * Return false if no more lines.
     745             :  *
     746             :  * An internal temporary buffer is returned via 'fields'. It is valid until
     747             :  * the next call of the function. Since the function returns all raw fields
     748             :  * in the input file, 'nfields' could be different from the number of columns
     749             :  * in the relation.
     750             :  *
     751             :  * NOTE: force_not_null option are not applied to the returned fields.
     752             :  */
     753             : bool
     754     1250134 : NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields)
     755             : {
     756             :     int         fldct;
     757             :     bool        done;
     758             : 
     759             :     /* only available for text or csv input */
     760             :     Assert(!cstate->opts.binary);
     761             : 
     762             :     /* on input check that the header line is correct if needed */
     763     1250134 :     if (cstate->cur_lineno == 0 && cstate->opts.header_line)
     764             :     {
     765             :         ListCell   *cur;
     766             :         TupleDesc   tupDesc;
     767             : 
     768         110 :         tupDesc = RelationGetDescr(cstate->rel);
     769             : 
     770         110 :         cstate->cur_lineno++;
     771         110 :         done = CopyReadLine(cstate);
     772             : 
     773         110 :         if (cstate->opts.header_line == COPY_HEADER_MATCH)
     774             :         {
     775             :             int         fldnum;
     776             : 
     777          76 :             if (cstate->opts.csv_mode)
     778          10 :                 fldct = CopyReadAttributesCSV(cstate);
     779             :             else
     780          66 :                 fldct = CopyReadAttributesText(cstate);
     781             : 
     782          76 :             if (fldct != list_length(cstate->attnumlist))
     783          24 :                 ereport(ERROR,
     784             :                         (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
     785             :                          errmsg("wrong number of fields in header line: got %d, expected %d",
     786             :                                 fldct, list_length(cstate->attnumlist))));
     787             : 
     788          52 :             fldnum = 0;
     789         158 :             foreach(cur, cstate->attnumlist)
     790             :             {
     791         126 :                 int         attnum = lfirst_int(cur);
     792             :                 char       *colName;
     793         126 :                 Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
     794             : 
     795             :                 Assert(fldnum < cstate->max_fields);
     796             : 
     797         126 :                 colName = cstate->raw_fields[fldnum++];
     798         126 :                 if (colName == NULL)
     799           6 :                     ereport(ERROR,
     800             :                             (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
     801             :                              errmsg("column name mismatch in header line field %d: got null value (\"%s\"), expected \"%s\"",
     802             :                                     fldnum, cstate->opts.null_print, NameStr(attr->attname))));
     803             : 
     804         120 :                 if (namestrcmp(&attr->attname, colName) != 0)
     805             :                 {
     806          14 :                     ereport(ERROR,
     807             :                             (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
     808             :                              errmsg("column name mismatch in header line field %d: got \"%s\", expected \"%s\"",
     809             :                                     fldnum, colName, NameStr(attr->attname))));
     810             :                 }
     811             :             }
     812             :         }
     813             : 
     814          66 :         if (done)
     815           0 :             return false;
     816             :     }
     817             : 
     818     1250090 :     cstate->cur_lineno++;
     819             : 
     820             :     /* Actually read the line into memory here */
     821     1250090 :     done = CopyReadLine(cstate);
     822             : 
     823             :     /*
     824             :      * EOF at start of line means we're done.  If we see EOF after some
     825             :      * characters, we act as though it was newline followed by EOF, ie,
     826             :      * process the line and then exit loop on next iteration.
     827             :      */
     828     1250090 :     if (done && cstate->line_buf.len == 0)
     829        1402 :         return false;
     830             : 
     831             :     /* Parse the line into de-escaped field values */
     832     1248688 :     if (cstate->opts.csv_mode)
     833         394 :         fldct = CopyReadAttributesCSV(cstate);
     834             :     else
     835     1248294 :         fldct = CopyReadAttributesText(cstate);
     836             : 
     837     1248676 :     *fields = cstate->raw_fields;
     838     1248676 :     *nfields = fldct;
     839     1248676 :     return true;
     840             : }
     841             : 
     842             : /*
     843             :  * Read next tuple from file for COPY FROM. Return false if no more tuples.
     844             :  *
     845             :  * 'econtext' is used to evaluate default expression for each column that is
     846             :  * either not read from the file or is using the DEFAULT option of COPY FROM.
     847             :  * It can be NULL when no default values are used, i.e. when all columns are
     848             :  * read from the file, and DEFAULT option is unset.
     849             :  *
     850             :  * 'values' and 'nulls' arrays must be the same length as columns of the
     851             :  * relation passed to BeginCopyFrom. This function fills the arrays.
     852             :  */
     853             : bool
     854     1250176 : NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
     855             :              Datum *values, bool *nulls)
     856             : {
     857             :     TupleDesc   tupDesc;
     858             :     AttrNumber  num_phys_attrs,
     859             :                 attr_count,
     860     1250176 :                 num_defaults = cstate->num_defaults;
     861     1250176 :     FmgrInfo   *in_functions = cstate->in_functions;
     862     1250176 :     Oid        *typioparams = cstate->typioparams;
     863             :     int         i;
     864     1250176 :     int        *defmap = cstate->defmap;
     865     1250176 :     ExprState **defexprs = cstate->defexprs;
     866             : 
     867     1250176 :     tupDesc = RelationGetDescr(cstate->rel);
     868     1250176 :     num_phys_attrs = tupDesc->natts;
     869     1250176 :     attr_count = list_length(cstate->attnumlist);
     870             : 
     871             :     /* Initialize all values for row to NULL */
     872     5801536 :     MemSet(values, 0, num_phys_attrs * sizeof(Datum));
     873     1250176 :     MemSet(nulls, true, num_phys_attrs * sizeof(bool));
     874     1394284 :     MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool));
     875             : 
     876     1250176 :     if (!cstate->opts.binary)
     877             :     {
     878             :         char      **field_strings;
     879             :         ListCell   *cur;
     880             :         int         fldct;
     881             :         int         fieldno;
     882             :         char       *string;
     883             : 
     884             :         /* read raw fields in the next line */
     885     1250134 :         if (!NextCopyFromRawFields(cstate, &field_strings, &fldct))
     886        1444 :             return false;
     887             : 
     888             :         /* check for overflowing fields */
     889     1248676 :         if (attr_count > 0 && fldct > attr_count)
     890          18 :             ereport(ERROR,
     891             :                     (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
     892             :                      errmsg("extra data after last expected column")));
     893             : 
     894     1248658 :         fieldno = 0;
     895             : 
     896             :         /* Loop to read the user attributes on the line. */
     897     5676976 :         foreach(cur, cstate->attnumlist)
     898             :         {
     899     4428416 :             int         attnum = lfirst_int(cur);
     900     4428416 :             int         m = attnum - 1;
     901     4428416 :             Form_pg_attribute att = TupleDescAttr(tupDesc, m);
     902             : 
     903     4428416 :             if (fieldno >= fldct)
     904          18 :                 ereport(ERROR,
     905             :                         (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
     906             :                          errmsg("missing data for column \"%s\"",
     907             :                                 NameStr(att->attname))));
     908     4428398 :             string = field_strings[fieldno++];
     909             : 
     910     4428398 :             if (cstate->convert_select_flags &&
     911          20 :                 !cstate->convert_select_flags[m])
     912             :             {
     913             :                 /* ignore input field, leaving column as NULL */
     914          10 :                 continue;
     915             :             }
     916             : 
     917     4428388 :             if (cstate->opts.csv_mode)
     918             :             {
     919         852 :                 if (string == NULL &&
     920          44 :                     cstate->opts.force_notnull_flags[m])
     921             :                 {
     922             :                     /*
     923             :                      * FORCE_NOT_NULL option is set and column is NULL -
     924             :                      * convert it to the NULL string.
     925             :                      */
     926          28 :                     string = cstate->opts.null_print;
     927             :                 }
     928         824 :                 else if (string != NULL && cstate->opts.force_null_flags[m]
     929          50 :                          && strcmp(string, cstate->opts.null_print) == 0)
     930             :                 {
     931             :                     /*
     932             :                      * FORCE_NULL option is set and column matches the NULL
     933             :                      * string. It must have been quoted, or otherwise the
     934             :                      * string would already have been set to NULL. Convert it
     935             :                      * to NULL as specified.
     936             :                      */
     937          26 :                     string = NULL;
     938             :                 }
     939             :             }
     940             : 
     941     4428388 :             cstate->cur_attname = NameStr(att->attname);
     942     4428388 :             cstate->cur_attval = string;
     943             : 
     944     4428388 :             if (string != NULL)
     945     4423550 :                 nulls[m] = false;
     946             : 
     947     4428388 :             if (cstate->defaults[m])
     948             :             {
     949             :                 /*
     950             :                  * The caller must supply econtext and have switched into the
     951             :                  * per-tuple memory context in it.
     952             :                  */
     953             :                 Assert(econtext != NULL);
     954             :                 Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
     955             : 
     956          60 :                 values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]);
     957             :             }
     958             : 
     959             :             /*
     960             :              * If ON_ERROR is specified with IGNORE, skip rows with soft
     961             :              * errors
     962             :              */
     963     4428290 :             else if (!InputFunctionCallSafe(&in_functions[m],
     964             :                                             string,
     965     4428328 :                                             typioparams[m],
     966             :                                             att->atttypmod,
     967     4428328 :                                             (Node *) cstate->escontext,
     968     4428328 :                                             &values[m]))
     969             :             {
     970             :                 Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP);
     971             : 
     972          42 :                 cstate->num_errors++;
     973             : 
     974          42 :                 if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE)
     975             :                 {
     976             :                     /*
     977             :                      * Since we emit line number and column info in the below
     978             :                      * notice message, we suppress error context information
     979             :                      * other than the relation name.
     980             :                      */
     981             :                     Assert(!cstate->relname_only);
     982          42 :                     cstate->relname_only = true;
     983             : 
     984          42 :                     if (cstate->cur_attval)
     985             :                     {
     986             :                         char       *attval;
     987             : 
     988          36 :                         attval = CopyLimitPrintoutLength(cstate->cur_attval);
     989          36 :                         ereport(NOTICE,
     990             :                                 errmsg("skipping row due to data type incompatibility at line %llu for column %s: \"%s\"",
     991             :                                        (unsigned long long) cstate->cur_lineno,
     992             :                                        cstate->cur_attname,
     993             :                                        attval));
     994          36 :                         pfree(attval);
     995             :                     }
     996             :                     else
     997           6 :                         ereport(NOTICE,
     998             :                                 errmsg("skipping row due to data type incompatibility at line %llu for column %s: null input",
     999             :                                        (unsigned long long) cstate->cur_lineno,
    1000             :                                        cstate->cur_attname));
    1001             : 
    1002             :                     /* reset relname_only */
    1003          42 :                     cstate->relname_only = false;
    1004             :                 }
    1005             : 
    1006          42 :                 return true;
    1007             :             }
    1008             : 
    1009     4428308 :             cstate->cur_attname = NULL;
    1010     4428308 :             cstate->cur_attval = NULL;
    1011             :         }
    1012             : 
    1013             :         Assert(fieldno == attr_count);
    1014             :     }
    1015             :     else
    1016             :     {
    1017             :         /* binary */
    1018             :         int16       fld_count;
    1019             :         ListCell   *cur;
    1020             : 
    1021          42 :         cstate->cur_lineno++;
    1022             : 
    1023          42 :         if (!CopyGetInt16(cstate, &fld_count))
    1024             :         {
    1025             :             /* EOF detected (end of file, or protocol-level EOF) */
    1026          12 :             return false;
    1027             :         }
    1028             : 
    1029          42 :         if (fld_count == -1)
    1030             :         {
    1031             :             /*
    1032             :              * Received EOF marker.  Wait for the protocol-level EOF, and
    1033             :              * complain if it doesn't come immediately.  In COPY FROM STDIN,
    1034             :              * this ensures that we correctly handle CopyFail, if client
    1035             :              * chooses to send that now.  When copying from file, we could
    1036             :              * ignore the rest of the file like in text mode, but we choose to
    1037             :              * be consistent with the COPY FROM STDIN case.
    1038             :              */
    1039             :             char        dummy;
    1040             : 
    1041          12 :             if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
    1042           0 :                 ereport(ERROR,
    1043             :                         (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    1044             :                          errmsg("received copy data after EOF marker")));
    1045          12 :             return false;
    1046             :         }
    1047             : 
    1048          30 :         if (fld_count != attr_count)
    1049           0 :             ereport(ERROR,
    1050             :                     (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    1051             :                      errmsg("row field count is %d, expected %d",
    1052             :                             (int) fld_count, attr_count)));
    1053             : 
    1054         186 :         foreach(cur, cstate->attnumlist)
    1055             :         {
    1056         158 :             int         attnum = lfirst_int(cur);
    1057         158 :             int         m = attnum - 1;
    1058         158 :             Form_pg_attribute att = TupleDescAttr(tupDesc, m);
    1059             : 
    1060         158 :             cstate->cur_attname = NameStr(att->attname);
    1061         314 :             values[m] = CopyReadBinaryAttribute(cstate,
    1062         158 :                                                 &in_functions[m],
    1063         158 :                                                 typioparams[m],
    1064             :                                                 att->atttypmod,
    1065             :                                                 &nulls[m]);
    1066         156 :             cstate->cur_attname = NULL;
    1067             :         }
    1068             :     }
    1069             : 
    1070             :     /*
    1071             :      * Now compute and insert any defaults available for the columns not
    1072             :      * provided by the input data.  Anything not processed here or above will
    1073             :      * remain NULL.
    1074             :      */
    1075     1309118 :     for (i = 0; i < num_defaults; i++)
    1076             :     {
    1077             :         /*
    1078             :          * The caller must supply econtext and have switched into the
    1079             :          * per-tuple memory context in it.
    1080             :          */
    1081             :         Assert(econtext != NULL);
    1082             :         Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
    1083             : 
    1084       60530 :         values[defmap[i]] = ExecEvalExpr(defexprs[defmap[i]], econtext,
    1085       60530 :                                          &nulls[defmap[i]]);
    1086             :     }
    1087             : 
    1088     1248588 :     return true;
    1089             : }
    1090             : 
    1091             : /*
    1092             :  * Read the next input line and stash it in line_buf.
    1093             :  *
    1094             :  * Result is true if read was terminated by EOF, false if terminated
    1095             :  * by newline.  The terminating newline or EOF marker is not included
    1096             :  * in the final value of line_buf.
    1097             :  */
    1098             : static bool
    1099     1250200 : CopyReadLine(CopyFromState cstate)
    1100             : {
    1101             :     bool        result;
    1102             : 
    1103     1250200 :     resetStringInfo(&cstate->line_buf);
    1104     1250200 :     cstate->line_buf_valid = false;
    1105             : 
    1106             :     /* Parse data and transfer into line_buf */
    1107     1250200 :     result = CopyReadLineText(cstate);
    1108             : 
    1109     1250200 :     if (result)
    1110             :     {
    1111             :         /*
    1112             :          * Reached EOF.  In protocol version 3, we should ignore anything
    1113             :          * after \. up to the protocol end of copy data.  (XXX maybe better
    1114             :          * not to treat \. as special?)
    1115             :          */
    1116        1402 :         if (cstate->copy_src == COPY_FRONTEND)
    1117             :         {
    1118             :             int         inbytes;
    1119             : 
    1120             :             do
    1121             :             {
    1122         712 :                 inbytes = CopyGetData(cstate, cstate->input_buf,
    1123             :                                       1, INPUT_BUF_SIZE);
    1124         712 :             } while (inbytes > 0);
    1125         712 :             cstate->input_buf_index = 0;
    1126         712 :             cstate->input_buf_len = 0;
    1127         712 :             cstate->raw_buf_index = 0;
    1128         712 :             cstate->raw_buf_len = 0;
    1129             :         }
    1130             :     }
    1131             :     else
    1132             :     {
    1133             :         /*
    1134             :          * If we didn't hit EOF, then we must have transferred the EOL marker
    1135             :          * to line_buf along with the data.  Get rid of it.
    1136             :          */
    1137     1248798 :         switch (cstate->eol_type)
    1138             :         {
    1139     1248798 :             case EOL_NL:
    1140             :                 Assert(cstate->line_buf.len >= 1);
    1141             :                 Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
    1142     1248798 :                 cstate->line_buf.len--;
    1143     1248798 :                 cstate->line_buf.data[cstate->line_buf.len] = '\0';
    1144     1248798 :                 break;
    1145           0 :             case EOL_CR:
    1146             :                 Assert(cstate->line_buf.len >= 1);
    1147             :                 Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
    1148           0 :                 cstate->line_buf.len--;
    1149           0 :                 cstate->line_buf.data[cstate->line_buf.len] = '\0';
    1150           0 :                 break;
    1151           0 :             case EOL_CRNL:
    1152             :                 Assert(cstate->line_buf.len >= 2);
    1153             :                 Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
    1154             :                 Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
    1155           0 :                 cstate->line_buf.len -= 2;
    1156           0 :                 cstate->line_buf.data[cstate->line_buf.len] = '\0';
    1157           0 :                 break;
    1158           0 :             case EOL_UNKNOWN:
    1159             :                 /* shouldn't get here */
    1160             :                 Assert(false);
    1161           0 :                 break;
    1162             :         }
    1163     1250200 :     }
    1164             : 
    1165             :     /* Now it's safe to use the buffer in error messages */
    1166     1250200 :     cstate->line_buf_valid = true;
    1167             : 
    1168     1250200 :     return result;
    1169             : }
    1170             : 
    1171             : /*
    1172             :  * CopyReadLineText - inner loop of CopyReadLine for text mode
    1173             :  */
    1174             : static bool
    1175     1250200 : CopyReadLineText(CopyFromState cstate)
    1176             : {
    1177             :     char       *copy_input_buf;
    1178             :     int         input_buf_ptr;
    1179             :     int         copy_buf_len;
    1180     1250200 :     bool        need_data = false;
    1181     1250200 :     bool        hit_eof = false;
    1182     1250200 :     bool        result = false;
    1183             : 
    1184             :     /* CSV variables */
    1185     1250200 :     bool        first_char_in_line = true;
    1186     1250200 :     bool        in_quote = false,
    1187     1250200 :                 last_was_esc = false;
    1188     1250200 :     char        quotec = '\0';
    1189     1250200 :     char        escapec = '\0';
    1190             : 
    1191     1250200 :     if (cstate->opts.csv_mode)
    1192             :     {
    1193         602 :         quotec = cstate->opts.quote[0];
    1194         602 :         escapec = cstate->opts.escape[0];
    1195             :         /* ignore special escape processing if it's the same as quotec */
    1196         602 :         if (quotec == escapec)
    1197         466 :             escapec = '\0';
    1198             :     }
    1199             : 
    1200             :     /*
    1201             :      * The objective of this loop is to transfer the entire next input line
    1202             :      * into line_buf.  Hence, we only care for detecting newlines (\r and/or
    1203             :      * \n) and the end-of-copy marker (\.).
    1204             :      *
    1205             :      * In CSV mode, \r and \n inside a quoted field are just part of the data
    1206             :      * value and are put in line_buf.  We keep just enough state to know if we
    1207             :      * are currently in a quoted field or not.
    1208             :      *
    1209             :      * The input has already been converted to the database encoding.  All
    1210             :      * supported server encodings have the property that all bytes in a
    1211             :      * multi-byte sequence have the high bit set, so a multibyte character
    1212             :      * cannot contain any newline or escape characters embedded in the
    1213             :      * multibyte sequence.  Therefore, we can process the input byte-by-byte,
    1214             :      * regardless of the encoding.
    1215             :      *
    1216             :      * For speed, we try to move data from input_buf to line_buf in chunks
    1217             :      * rather than one character at a time.  input_buf_ptr points to the next
    1218             :      * character to examine; any characters from input_buf_index to
    1219             :      * input_buf_ptr have been determined to be part of the line, but not yet
    1220             :      * transferred to line_buf.
    1221             :      *
    1222             :      * For a little extra speed within the loop, we copy input_buf and
    1223             :      * input_buf_len into local variables.
    1224             :      */
    1225     1250200 :     copy_input_buf = cstate->input_buf;
    1226     1250200 :     input_buf_ptr = cstate->input_buf_index;
    1227     1250200 :     copy_buf_len = cstate->input_buf_len;
    1228             : 
    1229             :     for (;;)
    1230    24753344 :     {
    1231             :         int         prev_raw_ptr;
    1232             :         char        c;
    1233             : 
    1234             :         /*
    1235             :          * Load more data if needed.
    1236             :          *
    1237             :          * TODO: We could just force four bytes of read-ahead and avoid the
    1238             :          * many calls to IF_NEED_REFILL_AND_NOT_EOF_CONTINUE().  That was
    1239             :          * unsafe with the old v2 COPY protocol, but we don't support that
    1240             :          * anymore.
    1241             :          */
    1242    26003544 :         if (input_buf_ptr >= copy_buf_len || need_data)
    1243             :         {
    1244      429998 :             REFILL_LINEBUF;
    1245             : 
    1246      429998 :             CopyLoadInputBuf(cstate);
    1247             :             /* update our local variables */
    1248      429998 :             hit_eof = cstate->input_reached_eof;
    1249      429998 :             input_buf_ptr = cstate->input_buf_index;
    1250      429998 :             copy_buf_len = cstate->input_buf_len;
    1251             : 
    1252             :             /*
    1253             :              * If we are completely out of data, break out of the loop,
    1254             :              * reporting EOF.
    1255             :              */
    1256      429998 :             if (INPUT_BUF_BYTES(cstate) <= 0)
    1257             :             {
    1258         770 :                 result = true;
    1259         770 :                 break;
    1260             :             }
    1261      429228 :             need_data = false;
    1262             :         }
    1263             : 
    1264             :         /* OK to fetch a character */
    1265    26002774 :         prev_raw_ptr = input_buf_ptr;
    1266    26002774 :         c = copy_input_buf[input_buf_ptr++];
    1267             : 
    1268    26002774 :         if (cstate->opts.csv_mode)
    1269             :         {
    1270             :             /*
    1271             :              * If character is '\\' or '\r', we may need to look ahead below.
    1272             :              * Force fetch of the next character if we don't already have it.
    1273             :              * We need to do this before changing CSV state, in case one of
    1274             :              * these characters is also the quote or escape character.
    1275             :              */
    1276        4858 :             if (c == '\\' || c == '\r')
    1277             :             {
    1278         318 :                 IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
    1279             :             }
    1280             : 
    1281             :             /*
    1282             :              * Dealing with quotes and escapes here is mildly tricky. If the
    1283             :              * quote char is also the escape char, there's no problem - we
    1284             :              * just use the char as a toggle. If they are different, we need
    1285             :              * to ensure that we only take account of an escape inside a
    1286             :              * quoted field and immediately preceding a quote char, and not
    1287             :              * the second in an escape-escape sequence.
    1288             :              */
    1289        4858 :             if (in_quote && c == escapec)
    1290          48 :                 last_was_esc = !last_was_esc;
    1291        4858 :             if (c == quotec && !last_was_esc)
    1292         408 :                 in_quote = !in_quote;
    1293        4858 :             if (c != escapec)
    1294        4804 :                 last_was_esc = false;
    1295             : 
    1296             :             /*
    1297             :              * Updating the line count for embedded CR and/or LF chars is
    1298             :              * necessarily a little fragile - this test is probably about the
    1299             :              * best we can do.  (XXX it's arguable whether we should do this
    1300             :              * at all --- is cur_lineno a physical or logical count?)
    1301             :              */
    1302        4858 :             if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
    1303          36 :                 cstate->cur_lineno++;
    1304             :         }
    1305             : 
    1306             :         /* Process \r */
    1307    26002774 :         if (c == '\r' && (!cstate->opts.csv_mode || !in_quote))
    1308             :         {
    1309             :             /* Check for \r\n on first line, _and_ handle \r\n. */
    1310           0 :             if (cstate->eol_type == EOL_UNKNOWN ||
    1311           0 :                 cstate->eol_type == EOL_CRNL)
    1312             :             {
    1313             :                 /*
    1314             :                  * If need more data, go back to loop top to load it.
    1315             :                  *
    1316             :                  * Note that if we are at EOF, c will wind up as '\0' because
    1317             :                  * of the guaranteed pad of input_buf.
    1318             :                  */
    1319           0 :                 IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
    1320             : 
    1321             :                 /* get next char */
    1322           0 :                 c = copy_input_buf[input_buf_ptr];
    1323             : 
    1324           0 :                 if (c == '\n')
    1325             :                 {
    1326           0 :                     input_buf_ptr++;    /* eat newline */
    1327           0 :                     cstate->eol_type = EOL_CRNL; /* in case not set yet */
    1328             :                 }
    1329             :                 else
    1330             :                 {
    1331             :                     /* found \r, but no \n */
    1332           0 :                     if (cstate->eol_type == EOL_CRNL)
    1333           0 :                         ereport(ERROR,
    1334             :                                 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    1335             :                                  !cstate->opts.csv_mode ?
    1336             :                                  errmsg("literal carriage return found in data") :
    1337             :                                  errmsg("unquoted carriage return found in data"),
    1338             :                                  !cstate->opts.csv_mode ?
    1339             :                                  errhint("Use \"\\r\" to represent carriage return.") :
    1340             :                                  errhint("Use quoted CSV field to represent carriage return.")));
    1341             : 
    1342             :                     /*
    1343             :                      * if we got here, it is the first line and we didn't find
    1344             :                      * \n, so don't consume the peeked character
    1345             :                      */
    1346           0 :                     cstate->eol_type = EOL_CR;
    1347             :                 }
    1348             :             }
    1349           0 :             else if (cstate->eol_type == EOL_NL)
    1350           0 :                 ereport(ERROR,
    1351             :                         (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    1352             :                          !cstate->opts.csv_mode ?
    1353             :                          errmsg("literal carriage return found in data") :
    1354             :                          errmsg("unquoted carriage return found in data"),
    1355             :                          !cstate->opts.csv_mode ?
    1356             :                          errhint("Use \"\\r\" to represent carriage return.") :
    1357             :                          errhint("Use quoted CSV field to represent carriage return.")));
    1358             :             /* If reach here, we have found the line terminator */
    1359           0 :             break;
    1360             :         }
    1361             : 
    1362             :         /* Process \n */
    1363    26002774 :         if (c == '\n' && (!cstate->opts.csv_mode || !in_quote))
    1364             :         {
    1365     1248798 :             if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
    1366           0 :                 ereport(ERROR,
    1367             :                         (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    1368             :                          !cstate->opts.csv_mode ?
    1369             :                          errmsg("literal newline found in data") :
    1370             :                          errmsg("unquoted newline found in data"),
    1371             :                          !cstate->opts.csv_mode ?
    1372             :                          errhint("Use \"\\n\" to represent newline.") :
    1373             :                          errhint("Use quoted CSV field to represent newline.")));
    1374     1248798 :             cstate->eol_type = EOL_NL;   /* in case not set yet */
    1375             :             /* If reach here, we have found the line terminator */
    1376     1248798 :             break;
    1377             :         }
    1378             : 
    1379             :         /*
    1380             :          * In CSV mode, we only recognize \. alone on a line.  This is because
    1381             :          * \. is a valid CSV data value.
    1382             :          */
    1383    24753976 :         if (c == '\\' && (!cstate->opts.csv_mode || first_char_in_line))
    1384             :         {
    1385             :             char        c2;
    1386             : 
    1387        8632 :             IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
    1388        8632 :             IF_NEED_REFILL_AND_EOF_BREAK(0);
    1389             : 
    1390             :             /* -----
    1391             :              * get next character
    1392             :              * Note: we do not change c so if it isn't \., we can fall
    1393             :              * through and continue processing.
    1394             :              * -----
    1395             :              */
    1396        8632 :             c2 = copy_input_buf[input_buf_ptr];
    1397             : 
    1398        8632 :             if (c2 == '.')
    1399             :             {
    1400         638 :                 input_buf_ptr++;    /* consume the '.' */
    1401             : 
    1402             :                 /*
    1403             :                  * Note: if we loop back for more data here, it does not
    1404             :                  * matter that the CSV state change checks are re-executed; we
    1405             :                  * will come back here with no important state changed.
    1406             :                  */
    1407         638 :                 if (cstate->eol_type == EOL_CRNL)
    1408             :                 {
    1409             :                     /* Get the next character */
    1410           0 :                     IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
    1411             :                     /* if hit_eof, c2 will become '\0' */
    1412           0 :                     c2 = copy_input_buf[input_buf_ptr++];
    1413             : 
    1414           0 :                     if (c2 == '\n')
    1415             :                     {
    1416           0 :                         if (!cstate->opts.csv_mode)
    1417           0 :                             ereport(ERROR,
    1418             :                                     (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    1419             :                                      errmsg("end-of-copy marker does not match previous newline style")));
    1420             :                         else
    1421           0 :                             NO_END_OF_COPY_GOTO;
    1422             :                     }
    1423           0 :                     else if (c2 != '\r')
    1424             :                     {
    1425           0 :                         if (!cstate->opts.csv_mode)
    1426           0 :                             ereport(ERROR,
    1427             :                                     (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    1428             :                                      errmsg("end-of-copy marker corrupt")));
    1429             :                         else
    1430           0 :                             NO_END_OF_COPY_GOTO;
    1431             :                     }
    1432             :                 }
    1433             : 
    1434             :                 /* Get the next character */
    1435         638 :                 IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
    1436             :                 /* if hit_eof, c2 will become '\0' */
    1437         638 :                 c2 = copy_input_buf[input_buf_ptr++];
    1438             : 
    1439         638 :                 if (c2 != '\r' && c2 != '\n')
    1440             :                 {
    1441           6 :                     if (!cstate->opts.csv_mode)
    1442           0 :                         ereport(ERROR,
    1443             :                                 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    1444             :                                  errmsg("end-of-copy marker corrupt")));
    1445             :                     else
    1446           6 :                         NO_END_OF_COPY_GOTO;
    1447             :                 }
    1448             : 
    1449         632 :                 if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
    1450         632 :                     (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
    1451         632 :                     (cstate->eol_type == EOL_CR && c2 != '\r'))
    1452             :                 {
    1453           0 :                     ereport(ERROR,
    1454             :                             (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    1455             :                              errmsg("end-of-copy marker does not match previous newline style")));
    1456             :                 }
    1457             : 
    1458             :                 /*
    1459             :                  * Transfer only the data before the \. into line_buf, then
    1460             :                  * discard the data and the \. sequence.
    1461             :                  */
    1462         632 :                 if (prev_raw_ptr > cstate->input_buf_index)
    1463           0 :                     appendBinaryStringInfo(&cstate->line_buf,
    1464           0 :                                            cstate->input_buf + cstate->input_buf_index,
    1465           0 :                                            prev_raw_ptr - cstate->input_buf_index);
    1466         632 :                 cstate->input_buf_index = input_buf_ptr;
    1467         632 :                 result = true;  /* report EOF */
    1468         632 :                 break;
    1469             :             }
    1470        7994 :             else if (!cstate->opts.csv_mode)
    1471             :             {
    1472             :                 /*
    1473             :                  * If we are here, it means we found a backslash followed by
    1474             :                  * something other than a period.  In non-CSV mode, anything
    1475             :                  * after a backslash is special, so we skip over that second
    1476             :                  * character too.  If we didn't do that \\. would be
    1477             :                  * considered an eof-of copy, while in non-CSV mode it is a
    1478             :                  * literal backslash followed by a period.  In CSV mode,
    1479             :                  * backslashes are not special, so we want to process the
    1480             :                  * character after the backslash just like a normal character,
    1481             :                  * so we don't increment in those cases.
    1482             :                  */
    1483        7988 :                 input_buf_ptr++;
    1484             :             }
    1485             :         }
    1486             : 
    1487             :         /*
    1488             :          * This label is for CSV cases where \. appears at the start of a
    1489             :          * line, but there is more text after it, meaning it was a data value.
    1490             :          * We are more strict for \. in CSV mode because \. could be a data
    1491             :          * value, while in non-CSV mode, \. cannot be a data value.
    1492             :          */
    1493    24745350 : not_end_of_copy:
    1494    24753344 :         first_char_in_line = false;
    1495             :     }                           /* end of outer loop */
    1496             : 
    1497             :     /*
    1498             :      * Transfer any still-uncopied data to line_buf.
    1499             :      */
    1500     1250200 :     REFILL_LINEBUF;
    1501             : 
    1502     1250200 :     return result;
    1503             : }
    1504             : 
    1505             : /*
    1506             :  *  Return decimal value for a hexadecimal digit
    1507             :  */
    1508             : static int
    1509           0 : GetDecimalFromHex(char hex)
    1510             : {
    1511           0 :     if (isdigit((unsigned char) hex))
    1512           0 :         return hex - '0';
    1513             :     else
    1514           0 :         return tolower((unsigned char) hex) - 'a' + 10;
    1515             : }
    1516             : 
    1517             : /*
    1518             :  * Parse the current line into separate attributes (fields),
    1519             :  * performing de-escaping as needed.
    1520             :  *
    1521             :  * The input is in line_buf.  We use attribute_buf to hold the result
    1522             :  * strings.  cstate->raw_fields[k] is set to point to the k'th attribute
    1523             :  * string, or NULL when the input matches the null marker string.
    1524             :  * This array is expanded as necessary.
    1525             :  *
    1526             :  * (Note that the caller cannot check for nulls since the returned
    1527             :  * string would be the post-de-escaping equivalent, which may look
    1528             :  * the same as some valid data string.)
    1529             :  *
    1530             :  * delim is the column delimiter string (must be just one byte for now).
    1531             :  * null_print is the null marker string.  Note that this is compared to
    1532             :  * the pre-de-escaped input string.
    1533             :  *
    1534             :  * The return value is the number of fields actually read.
    1535             :  */
    1536             : static int
    1537     1248360 : CopyReadAttributesText(CopyFromState cstate)
    1538             : {
    1539     1248360 :     char        delimc = cstate->opts.delim[0];
    1540             :     int         fieldno;
    1541             :     char       *output_ptr;
    1542             :     char       *cur_ptr;
    1543             :     char       *line_end_ptr;
    1544             : 
    1545             :     /*
    1546             :      * We need a special case for zero-column tables: check that the input
    1547             :      * line is empty, and return.
    1548             :      */
    1549     1248360 :     if (cstate->max_fields <= 0)
    1550             :     {
    1551           8 :         if (cstate->line_buf.len != 0)
    1552           0 :             ereport(ERROR,
    1553             :                     (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    1554             :                      errmsg("extra data after last expected column")));
    1555           8 :         return 0;
    1556             :     }
    1557             : 
    1558     1248352 :     resetStringInfo(&cstate->attribute_buf);
    1559             : 
    1560             :     /*
    1561             :      * The de-escaped attributes will certainly not be longer than the input
    1562             :      * data line, so we can just force attribute_buf to be large enough and
    1563             :      * then transfer data without any checks for enough space.  We need to do
    1564             :      * it this way because enlarging attribute_buf mid-stream would invalidate
    1565             :      * pointers already stored into cstate->raw_fields[].
    1566             :      */
    1567     1248352 :     if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
    1568           8 :         enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
    1569     1248352 :     output_ptr = cstate->attribute_buf.data;
    1570             : 
    1571             :     /* set pointer variables for loop */
    1572     1248352 :     cur_ptr = cstate->line_buf.data;
    1573     1248352 :     line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
    1574             : 
    1575             :     /* Outer loop iterates over fields */
    1576     1248352 :     fieldno = 0;
    1577             :     for (;;)
    1578     3179490 :     {
    1579     4427842 :         bool        found_delim = false;
    1580             :         char       *start_ptr;
    1581             :         char       *end_ptr;
    1582             :         int         input_len;
    1583     4427842 :         bool        saw_non_ascii = false;
    1584             : 
    1585             :         /* Make sure there is enough space for the next value */
    1586     4427842 :         if (fieldno >= cstate->max_fields)
    1587             :         {
    1588          36 :             cstate->max_fields *= 2;
    1589          36 :             cstate->raw_fields =
    1590          36 :                 repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
    1591             :         }
    1592             : 
    1593             :         /* Remember start of field on both input and output sides */
    1594     4427842 :         start_ptr = cur_ptr;
    1595     4427842 :         cstate->raw_fields[fieldno] = output_ptr;
    1596             : 
    1597             :         /*
    1598             :          * Scan data for field.
    1599             :          *
    1600             :          * Note that in this loop, we are scanning to locate the end of field
    1601             :          * and also speculatively performing de-escaping.  Once we find the
    1602             :          * end-of-field, we can match the raw field contents against the null
    1603             :          * marker string.  Only after that comparison fails do we know that
    1604             :          * de-escaping is actually the right thing to do; therefore we *must
    1605             :          * not* throw any syntax errors before we've done the null-marker
    1606             :          * check.
    1607             :          */
    1608             :         for (;;)
    1609    21569050 :         {
    1610             :             char        c;
    1611             : 
    1612    25996892 :             end_ptr = cur_ptr;
    1613    25996892 :             if (cur_ptr >= line_end_ptr)
    1614     1248346 :                 break;
    1615    24748546 :             c = *cur_ptr++;
    1616    24748546 :             if (c == delimc)
    1617             :             {
    1618     3179496 :                 found_delim = true;
    1619     3179496 :                 break;
    1620             :             }
    1621    21569050 :             if (c == '\\')
    1622             :             {
    1623        7988 :                 if (cur_ptr >= line_end_ptr)
    1624           0 :                     break;
    1625        7988 :                 c = *cur_ptr++;
    1626        7988 :                 switch (c)
    1627             :                 {
    1628          12 :                     case '0':
    1629             :                     case '1':
    1630             :                     case '2':
    1631             :                     case '3':
    1632             :                     case '4':
    1633             :                     case '5':
    1634             :                     case '6':
    1635             :                     case '7':
    1636             :                         {
    1637             :                             /* handle \013 */
    1638             :                             int         val;
    1639             : 
    1640          12 :                             val = OCTVALUE(c);
    1641          12 :                             if (cur_ptr < line_end_ptr)
    1642             :                             {
    1643           6 :                                 c = *cur_ptr;
    1644           6 :                                 if (ISOCTAL(c))
    1645             :                                 {
    1646           0 :                                     cur_ptr++;
    1647           0 :                                     val = (val << 3) + OCTVALUE(c);
    1648           0 :                                     if (cur_ptr < line_end_ptr)
    1649             :                                     {
    1650           0 :                                         c = *cur_ptr;
    1651           0 :                                         if (ISOCTAL(c))
    1652             :                                         {
    1653           0 :                                             cur_ptr++;
    1654           0 :                                             val = (val << 3) + OCTVALUE(c);
    1655             :                                         }
    1656             :                                     }
    1657             :                                 }
    1658             :                             }
    1659          12 :                             c = val & 0377;
    1660          12 :                             if (c == '\0' || IS_HIGHBIT_SET(c))
    1661          12 :                                 saw_non_ascii = true;
    1662             :                         }
    1663          12 :                         break;
    1664          12 :                     case 'x':
    1665             :                         /* Handle \x3F */
    1666          12 :                         if (cur_ptr < line_end_ptr)
    1667             :                         {
    1668           6 :                             char        hexchar = *cur_ptr;
    1669             : 
    1670           6 :                             if (isxdigit((unsigned char) hexchar))
    1671             :                             {
    1672           0 :                                 int         val = GetDecimalFromHex(hexchar);
    1673             : 
    1674           0 :                                 cur_ptr++;
    1675           0 :                                 if (cur_ptr < line_end_ptr)
    1676             :                                 {
    1677           0 :                                     hexchar = *cur_ptr;
    1678           0 :                                     if (isxdigit((unsigned char) hexchar))
    1679             :                                     {
    1680           0 :                                         cur_ptr++;
    1681           0 :                                         val = (val << 4) + GetDecimalFromHex(hexchar);
    1682             :                                     }
    1683             :                                 }
    1684           0 :                                 c = val & 0xff;
    1685           0 :                                 if (c == '\0' || IS_HIGHBIT_SET(c))
    1686           0 :                                     saw_non_ascii = true;
    1687             :                             }
    1688             :                         }
    1689          12 :                         break;
    1690           0 :                     case 'b':
    1691           0 :                         c = '\b';
    1692           0 :                         break;
    1693           0 :                     case 'f':
    1694           0 :                         c = '\f';
    1695           0 :                         break;
    1696        3050 :                     case 'n':
    1697        3050 :                         c = '\n';
    1698        3050 :                         break;
    1699           0 :                     case 'r':
    1700           0 :                         c = '\r';
    1701           0 :                         break;
    1702           0 :                     case 't':
    1703           0 :                         c = '\t';
    1704           0 :                         break;
    1705           0 :                     case 'v':
    1706           0 :                         c = '\v';
    1707           0 :                         break;
    1708             : 
    1709             :                         /*
    1710             :                          * in all other cases, take the char after '\'
    1711             :                          * literally
    1712             :                          */
    1713             :                 }
    1714    21561062 :             }
    1715             : 
    1716             :             /* Add c to output string */
    1717    21569050 :             *output_ptr++ = c;
    1718             :         }
    1719             : 
    1720             :         /* Check whether raw input matched null marker */
    1721     4427842 :         input_len = end_ptr - start_ptr;
    1722     4427842 :         if (input_len == cstate->opts.null_print_len &&
    1723      241400 :             strncmp(start_ptr, cstate->opts.null_print, input_len) == 0)
    1724        4802 :             cstate->raw_fields[fieldno] = NULL;
    1725             :         /* Check whether raw input matched default marker */
    1726     4423040 :         else if (fieldno < list_length(cstate->attnumlist) &&
    1727     4422998 :                  cstate->opts.default_print &&
    1728         114 :                  input_len == cstate->opts.default_print_len &&
    1729          30 :                  strncmp(start_ptr, cstate->opts.default_print, input_len) == 0)
    1730          24 :         {
    1731             :             /* fieldno is 0-indexed and attnum is 1-indexed */
    1732          30 :             int         m = list_nth_int(cstate->attnumlist, fieldno) - 1;
    1733             : 
    1734          30 :             if (cstate->defexprs[m] != NULL)
    1735             :             {
    1736             :                 /* defaults contain entries for all physical attributes */
    1737          24 :                 cstate->defaults[m] = true;
    1738             :             }
    1739             :             else
    1740             :             {
    1741           6 :                 TupleDesc   tupDesc = RelationGetDescr(cstate->rel);
    1742           6 :                 Form_pg_attribute att = TupleDescAttr(tupDesc, m);
    1743             : 
    1744           6 :                 ereport(ERROR,
    1745             :                         (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    1746             :                          errmsg("unexpected default marker in COPY data"),
    1747             :                          errdetail("Column \"%s\" has no default value.",
    1748             :                                    NameStr(att->attname))));
    1749             :             }
    1750             :         }
    1751             :         else
    1752             :         {
    1753             :             /*
    1754             :              * At this point we know the field is supposed to contain data.
    1755             :              *
    1756             :              * If we de-escaped any non-7-bit-ASCII chars, make sure the
    1757             :              * resulting string is valid data for the db encoding.
    1758             :              */
    1759     4423010 :             if (saw_non_ascii)
    1760             :             {
    1761           0 :                 char       *fld = cstate->raw_fields[fieldno];
    1762             : 
    1763           0 :                 pg_verifymbstr(fld, output_ptr - fld, false);
    1764             :             }
    1765             :         }
    1766             : 
    1767             :         /* Terminate attribute value in output area */
    1768     4427836 :         *output_ptr++ = '\0';
    1769             : 
    1770     4427836 :         fieldno++;
    1771             :         /* Done if we hit EOL instead of a delim */
    1772     4427836 :         if (!found_delim)
    1773     1248346 :             break;
    1774             :     }
    1775             : 
    1776             :     /* Clean up state of attribute_buf */
    1777     1248346 :     output_ptr--;
    1778             :     Assert(*output_ptr == '\0');
    1779     1248346 :     cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
    1780             : 
    1781     1248346 :     return fieldno;
    1782             : }
    1783             : 
    1784             : /*
    1785             :  * Parse the current line into separate attributes (fields),
    1786             :  * performing de-escaping as needed.  This has exactly the same API as
    1787             :  * CopyReadAttributesText, except we parse the fields according to
    1788             :  * "standard" (i.e. common) CSV usage.
    1789             :  */
    1790             : static int
    1791         404 : CopyReadAttributesCSV(CopyFromState cstate)
    1792             : {
    1793         404 :     char        delimc = cstate->opts.delim[0];
    1794         404 :     char        quotec = cstate->opts.quote[0];
    1795         404 :     char        escapec = cstate->opts.escape[0];
    1796             :     int         fieldno;
    1797             :     char       *output_ptr;
    1798             :     char       *cur_ptr;
    1799             :     char       *line_end_ptr;
    1800             : 
    1801             :     /*
    1802             :      * We need a special case for zero-column tables: check that the input
    1803             :      * line is empty, and return.
    1804             :      */
    1805         404 :     if (cstate->max_fields <= 0)
    1806             :     {
    1807           0 :         if (cstate->line_buf.len != 0)
    1808           0 :             ereport(ERROR,
    1809             :                     (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    1810             :                      errmsg("extra data after last expected column")));
    1811           0 :         return 0;
    1812             :     }
    1813             : 
    1814         404 :     resetStringInfo(&cstate->attribute_buf);
    1815             : 
    1816             :     /*
    1817             :      * The de-escaped attributes will certainly not be longer than the input
    1818             :      * data line, so we can just force attribute_buf to be large enough and
    1819             :      * then transfer data without any checks for enough space.  We need to do
    1820             :      * it this way because enlarging attribute_buf mid-stream would invalidate
    1821             :      * pointers already stored into cstate->raw_fields[].
    1822             :      */
    1823         404 :     if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
    1824           0 :         enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
    1825         404 :     output_ptr = cstate->attribute_buf.data;
    1826             : 
    1827             :     /* set pointer variables for loop */
    1828         404 :     cur_ptr = cstate->line_buf.data;
    1829         404 :     line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
    1830             : 
    1831             :     /* Outer loop iterates over fields */
    1832         404 :     fieldno = 0;
    1833             :     for (;;)
    1834         490 :     {
    1835         894 :         bool        found_delim = false;
    1836         894 :         bool        saw_quote = false;
    1837             :         char       *start_ptr;
    1838             :         char       *end_ptr;
    1839             :         int         input_len;
    1840             : 
    1841             :         /* Make sure there is enough space for the next value */
    1842         894 :         if (fieldno >= cstate->max_fields)
    1843             :         {
    1844           0 :             cstate->max_fields *= 2;
    1845           0 :             cstate->raw_fields =
    1846           0 :                 repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
    1847             :         }
    1848             : 
    1849             :         /* Remember start of field on both input and output sides */
    1850         894 :         start_ptr = cur_ptr;
    1851         894 :         cstate->raw_fields[fieldno] = output_ptr;
    1852             : 
    1853             :         /*
    1854             :          * Scan data for field,
    1855             :          *
    1856             :          * The loop starts in "not quote" mode and then toggles between that
    1857             :          * and "in quote" mode. The loop exits normally if it is in "not
    1858             :          * quote" mode and a delimiter or line end is seen.
    1859             :          */
    1860             :         for (;;)
    1861         182 :         {
    1862             :             char        c;
    1863             : 
    1864             :             /* Not in quote */
    1865             :             for (;;)
    1866             :             {
    1867        2866 :                 end_ptr = cur_ptr;
    1868        2866 :                 if (cur_ptr >= line_end_ptr)
    1869         398 :                     goto endfield;
    1870        2468 :                 c = *cur_ptr++;
    1871             :                 /* unquoted field delimiter */
    1872        2468 :                 if (c == delimc)
    1873             :                 {
    1874         496 :                     found_delim = true;
    1875         496 :                     goto endfield;
    1876             :                 }
    1877             :                 /* start of quoted field (or part of field) */
    1878        1972 :                 if (c == quotec)
    1879             :                 {
    1880         182 :                     saw_quote = true;
    1881         182 :                     break;
    1882             :                 }
    1883             :                 /* Add c to output string */
    1884        1790 :                 *output_ptr++ = c;
    1885             :             }
    1886             : 
    1887             :             /* In quote */
    1888             :             for (;;)
    1889             :             {
    1890        1170 :                 end_ptr = cur_ptr;
    1891        1170 :                 if (cur_ptr >= line_end_ptr)
    1892           0 :                     ereport(ERROR,
    1893             :                             (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    1894             :                              errmsg("unterminated CSV quoted field")));
    1895             : 
    1896        1170 :                 c = *cur_ptr++;
    1897             : 
    1898             :                 /* escape within a quoted field */
    1899        1170 :                 if (c == escapec)
    1900             :                 {
    1901             :                     /*
    1902             :                      * peek at the next char if available, and escape it if it
    1903             :                      * is an escape char or a quote char
    1904             :                      */
    1905         118 :                     if (cur_ptr < line_end_ptr)
    1906             :                     {
    1907          72 :                         char        nextc = *cur_ptr;
    1908             : 
    1909          72 :                         if (nextc == escapec || nextc == quotec)
    1910             :                         {
    1911          24 :                             *output_ptr++ = nextc;
    1912          24 :                             cur_ptr++;
    1913          24 :                             continue;
    1914             :                         }
    1915             :                     }
    1916             :                 }
    1917             : 
    1918             :                 /*
    1919             :                  * end of quoted field. Must do this test after testing for
    1920             :                  * escape in case quote char and escape char are the same
    1921             :                  * (which is the common case).
    1922             :                  */
    1923        1146 :                 if (c == quotec)
    1924         182 :                     break;
    1925             : 
    1926             :                 /* Add c to output string */
    1927         964 :                 *output_ptr++ = c;
    1928             :             }
    1929             :         }
    1930         894 : endfield:
    1931             : 
    1932             :         /* Terminate attribute value in output area */
    1933         894 :         *output_ptr++ = '\0';
    1934             : 
    1935             :         /* Check whether raw input matched null marker */
    1936         894 :         input_len = end_ptr - start_ptr;
    1937         894 :         if (!saw_quote && input_len == cstate->opts.null_print_len &&
    1938          44 :             strncmp(start_ptr, cstate->opts.null_print, input_len) == 0)
    1939          44 :             cstate->raw_fields[fieldno] = NULL;
    1940             :         /* Check whether raw input matched default marker */
    1941         850 :         else if (fieldno < list_length(cstate->attnumlist) &&
    1942         850 :                  cstate->opts.default_print &&
    1943         150 :                  input_len == cstate->opts.default_print_len &&
    1944          42 :                  strncmp(start_ptr, cstate->opts.default_print, input_len) == 0)
    1945             :         {
    1946             :             /* fieldno is 0-index and attnum is 1-index */
    1947          42 :             int         m = list_nth_int(cstate->attnumlist, fieldno) - 1;
    1948             : 
    1949          42 :             if (cstate->defexprs[m] != NULL)
    1950             :             {
    1951             :                 /* defaults contain entries for all physical attributes */
    1952          36 :                 cstate->defaults[m] = true;
    1953             :             }
    1954             :             else
    1955             :             {
    1956           6 :                 TupleDesc   tupDesc = RelationGetDescr(cstate->rel);
    1957           6 :                 Form_pg_attribute att = TupleDescAttr(tupDesc, m);
    1958             : 
    1959           6 :                 ereport(ERROR,
    1960             :                         (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    1961             :                          errmsg("unexpected default marker in COPY data"),
    1962             :                          errdetail("Column \"%s\" has no default value.",
    1963             :                                    NameStr(att->attname))));
    1964             :             }
    1965             :         }
    1966             : 
    1967         888 :         fieldno++;
    1968             :         /* Done if we hit EOL instead of a delim */
    1969         888 :         if (!found_delim)
    1970         398 :             break;
    1971             :     }
    1972             : 
    1973             :     /* Clean up state of attribute_buf */
    1974         398 :     output_ptr--;
    1975             :     Assert(*output_ptr == '\0');
    1976         398 :     cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
    1977             : 
    1978         398 :     return fieldno;
    1979             : }
    1980             : 
    1981             : 
    1982             : /*
    1983             :  * Read a binary attribute
    1984             :  */
    1985             : static Datum
    1986         158 : CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
    1987             :                         Oid typioparam, int32 typmod,
    1988             :                         bool *isnull)
    1989             : {
    1990             :     int32       fld_size;
    1991             :     Datum       result;
    1992             : 
    1993         158 :     if (!CopyGetInt32(cstate, &fld_size))
    1994           0 :         ereport(ERROR,
    1995             :                 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    1996             :                  errmsg("unexpected EOF in COPY data")));
    1997         158 :     if (fld_size == -1)
    1998             :     {
    1999          30 :         *isnull = true;
    2000          30 :         return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
    2001             :     }
    2002         128 :     if (fld_size < 0)
    2003           0 :         ereport(ERROR,
    2004             :                 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    2005             :                  errmsg("invalid field size")));
    2006             : 
    2007             :     /* reset attribute_buf to empty, and load raw data in it */
    2008         128 :     resetStringInfo(&cstate->attribute_buf);
    2009             : 
    2010         128 :     enlargeStringInfo(&cstate->attribute_buf, fld_size);
    2011         128 :     if (CopyReadBinaryData(cstate, cstate->attribute_buf.data,
    2012         128 :                            fld_size) != fld_size)
    2013           0 :         ereport(ERROR,
    2014             :                 (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
    2015             :                  errmsg("unexpected EOF in COPY data")));
    2016             : 
    2017         128 :     cstate->attribute_buf.len = fld_size;
    2018         128 :     cstate->attribute_buf.data[fld_size] = '\0';
    2019             : 
    2020             :     /* Call the column type's binary input converter */
    2021         128 :     result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
    2022             :                                  typioparam, typmod);
    2023             : 
    2024             :     /* Trouble if it didn't eat the whole buffer */
    2025         128 :     if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
    2026           2 :         ereport(ERROR,
    2027             :                 (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
    2028             :                  errmsg("incorrect binary data format")));
    2029             : 
    2030         126 :     *isnull = false;
    2031         126 :     return result;
    2032             : }

Generated by: LCOV version 1.14