LCOV - code coverage report
Current view: top level - src/bin/pg_upgrade - file.c (source / functions) Hit Total Coverage
Test: PostgreSQL 17devel Lines: 15 129 11.6 %
Date: 2023-10-02 08:10:26 Functions: 1 6 16.7 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*
       2             :  *  file.c
       3             :  *
       4             :  *  file system operations
       5             :  *
       6             :  *  Copyright (c) 2010-2023, PostgreSQL Global Development Group
       7             :  *  src/bin/pg_upgrade/file.c
       8             :  */
       9             : 
      10             : #include "postgres_fe.h"
      11             : 
      12             : #include <sys/stat.h>
      13             : #include <fcntl.h>
      14             : #ifdef HAVE_COPYFILE_H
      15             : #include <copyfile.h>
      16             : #endif
      17             : #ifdef __linux__
      18             : #include <sys/ioctl.h>
      19             : #include <linux/fs.h>
      20             : #endif
      21             : 
      22             : #include "access/visibilitymapdefs.h"
      23             : #include "common/file_perm.h"
      24             : #include "pg_upgrade.h"
      25             : #include "storage/bufpage.h"
      26             : #include "storage/checksum.h"
      27             : #include "storage/checksum_impl.h"
      28             : 
      29             : 
      30             : /*
      31             :  * cloneFile()
      32             :  *
      33             :  * Clones/reflinks a relation file from src to dst.
      34             :  *
      35             :  * schemaName/relName are relation's SQL name (used for error messages only).
      36             :  */
      37             : void
      38           0 : cloneFile(const char *src, const char *dst,
      39             :           const char *schemaName, const char *relName)
      40             : {
      41             : #if defined(HAVE_COPYFILE) && defined(COPYFILE_CLONE_FORCE)
      42             :     if (copyfile(src, dst, NULL, COPYFILE_CLONE_FORCE) < 0)
      43             :         pg_fatal("error while cloning relation \"%s.%s\" (\"%s\" to \"%s\"): %s",
      44             :                  schemaName, relName, src, dst, strerror(errno));
      45             : #elif defined(__linux__) && defined(FICLONE)
      46             :     int         src_fd;
      47             :     int         dest_fd;
      48             : 
      49           0 :     if ((src_fd = open(src, O_RDONLY | PG_BINARY, 0)) < 0)
      50           0 :         pg_fatal("error while cloning relation \"%s.%s\": could not open file \"%s\": %s",
      51           0 :                  schemaName, relName, src, strerror(errno));
      52             : 
      53           0 :     if ((dest_fd = open(dst, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
      54             :                         pg_file_create_mode)) < 0)
      55           0 :         pg_fatal("error while cloning relation \"%s.%s\": could not create file \"%s\": %s",
      56           0 :                  schemaName, relName, dst, strerror(errno));
      57             : 
      58           0 :     if (ioctl(dest_fd, FICLONE, src_fd) < 0)
      59             :     {
      60           0 :         int         save_errno = errno;
      61             : 
      62           0 :         unlink(dst);
      63             : 
      64           0 :         pg_fatal("error while cloning relation \"%s.%s\" (\"%s\" to \"%s\"): %s",
      65             :                  schemaName, relName, src, dst, strerror(save_errno));
      66             :     }
      67             : 
      68           0 :     close(src_fd);
      69           0 :     close(dest_fd);
      70             : #endif
      71           0 : }
      72             : 
      73             : 
      74             : /*
      75             :  * copyFile()
      76             :  *
      77             :  * Copies a relation file from src to dst.
      78             :  * schemaName/relName are relation's SQL name (used for error messages only).
      79             :  */
      80             : void
      81        3132 : copyFile(const char *src, const char *dst,
      82             :          const char *schemaName, const char *relName)
      83             : {
      84             : #ifndef WIN32
      85             :     int         src_fd;
      86             :     int         dest_fd;
      87             :     char       *buffer;
      88             : 
      89        3132 :     if ((src_fd = open(src, O_RDONLY | PG_BINARY, 0)) < 0)
      90           0 :         pg_fatal("error while copying relation \"%s.%s\": could not open file \"%s\": %s",
      91           0 :                  schemaName, relName, src, strerror(errno));
      92             : 
      93        3132 :     if ((dest_fd = open(dst, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
      94             :                         pg_file_create_mode)) < 0)
      95           0 :         pg_fatal("error while copying relation \"%s.%s\": could not create file \"%s\": %s",
      96           0 :                  schemaName, relName, dst, strerror(errno));
      97             : 
      98             :     /* copy in fairly large chunks for best efficiency */
      99             : #define COPY_BUF_SIZE (50 * BLCKSZ)
     100             : 
     101        3132 :     buffer = (char *) pg_malloc(COPY_BUF_SIZE);
     102             : 
     103             :     /* perform data copying i.e read src source, write to destination */
     104             :     while (true)
     105        2584 :     {
     106        5716 :         ssize_t     nbytes = read(src_fd, buffer, COPY_BUF_SIZE);
     107             : 
     108        5716 :         if (nbytes < 0)
     109           0 :             pg_fatal("error while copying relation \"%s.%s\": could not read file \"%s\": %s",
     110           0 :                      schemaName, relName, src, strerror(errno));
     111             : 
     112        5716 :         if (nbytes == 0)
     113        3132 :             break;
     114             : 
     115        2584 :         errno = 0;
     116        2584 :         if (write(dest_fd, buffer, nbytes) != nbytes)
     117             :         {
     118             :             /* if write didn't set errno, assume problem is no disk space */
     119           0 :             if (errno == 0)
     120           0 :                 errno = ENOSPC;
     121           0 :             pg_fatal("error while copying relation \"%s.%s\": could not write file \"%s\": %s",
     122           0 :                      schemaName, relName, dst, strerror(errno));
     123             :         }
     124             :     }
     125             : 
     126        3132 :     pg_free(buffer);
     127        3132 :     close(src_fd);
     128        3132 :     close(dest_fd);
     129             : 
     130             : #else                           /* WIN32 */
     131             : 
     132             :     if (CopyFile(src, dst, true) == 0)
     133             :     {
     134             :         _dosmaperr(GetLastError());
     135             :         pg_fatal("error while copying relation \"%s.%s\" (\"%s\" to \"%s\"): %s",
     136             :                  schemaName, relName, src, dst, strerror(errno));
     137             :     }
     138             : 
     139             : #endif                          /* WIN32 */
     140        3132 : }
     141             : 
     142             : 
     143             : /*
     144             :  * linkFile()
     145             :  *
     146             :  * Hard-links a relation file from src to dst.
     147             :  * schemaName/relName are relation's SQL name (used for error messages only).
     148             :  */
     149             : void
     150           0 : linkFile(const char *src, const char *dst,
     151             :          const char *schemaName, const char *relName)
     152             : {
     153           0 :     if (link(src, dst) < 0)
     154           0 :         pg_fatal("error while creating link for relation \"%s.%s\" (\"%s\" to \"%s\"): %s",
     155           0 :                  schemaName, relName, src, dst, strerror(errno));
     156           0 : }
     157             : 
     158             : 
     159             : /*
     160             :  * rewriteVisibilityMap()
     161             :  *
     162             :  * Transform a visibility map file, copying from src to dst.
     163             :  * schemaName/relName are relation's SQL name (used for error messages only).
     164             :  *
     165             :  * In versions of PostgreSQL prior to catversion 201603011, PostgreSQL's
     166             :  * visibility map included one bit per heap page; it now includes two.
     167             :  * When upgrading a cluster from before that time to a current PostgreSQL
     168             :  * version, we could refuse to copy visibility maps from the old cluster
     169             :  * to the new cluster; the next VACUUM would recreate them, but at the
     170             :  * price of scanning the entire table.  So, instead, we rewrite the old
     171             :  * visibility maps in the new format.  That way, the all-visible bits
     172             :  * remain set for the pages for which they were set previously.  The
     173             :  * all-frozen bits are never set by this conversion; we leave that to VACUUM.
     174             :  */
     175             : void
     176           0 : rewriteVisibilityMap(const char *fromfile, const char *tofile,
     177             :                      const char *schemaName, const char *relName)
     178             : {
     179             :     int         src_fd;
     180             :     int         dst_fd;
     181             :     PGIOAlignedBlock buffer;
     182             :     PGIOAlignedBlock new_vmbuf;
     183           0 :     ssize_t     totalBytesRead = 0;
     184             :     ssize_t     src_filesize;
     185             :     int         rewriteVmBytesPerPage;
     186           0 :     BlockNumber new_blkno = 0;
     187             :     struct stat statbuf;
     188             : 
     189             :     /* Compute number of old-format bytes per new page */
     190           0 :     rewriteVmBytesPerPage = (BLCKSZ - SizeOfPageHeaderData) / 2;
     191             : 
     192           0 :     if ((src_fd = open(fromfile, O_RDONLY | PG_BINARY, 0)) < 0)
     193           0 :         pg_fatal("error while copying relation \"%s.%s\": could not open file \"%s\": %s",
     194           0 :                  schemaName, relName, fromfile, strerror(errno));
     195             : 
     196           0 :     if (fstat(src_fd, &statbuf) != 0)
     197           0 :         pg_fatal("error while copying relation \"%s.%s\": could not stat file \"%s\": %s",
     198           0 :                  schemaName, relName, fromfile, strerror(errno));
     199             : 
     200           0 :     if ((dst_fd = open(tofile, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
     201             :                        pg_file_create_mode)) < 0)
     202           0 :         pg_fatal("error while copying relation \"%s.%s\": could not create file \"%s\": %s",
     203           0 :                  schemaName, relName, tofile, strerror(errno));
     204             : 
     205             :     /* Save old file size */
     206           0 :     src_filesize = statbuf.st_size;
     207             : 
     208             :     /*
     209             :      * Turn each visibility map page into 2 pages one by one. Each new page
     210             :      * has the same page header as the old one.  If the last section of the
     211             :      * last page is empty, we skip it, mostly to avoid turning one-page
     212             :      * visibility maps for small relations into two pages needlessly.
     213             :      */
     214           0 :     while (totalBytesRead < src_filesize)
     215             :     {
     216             :         ssize_t     bytesRead;
     217             :         char       *old_cur;
     218             :         char       *old_break;
     219             :         char       *old_blkend;
     220             :         PageHeaderData pageheader;
     221             :         bool        old_lastblk;
     222             : 
     223           0 :         if ((bytesRead = read(src_fd, buffer.data, BLCKSZ)) != BLCKSZ)
     224             :         {
     225           0 :             if (bytesRead < 0)
     226           0 :                 pg_fatal("error while copying relation \"%s.%s\": could not read file \"%s\": %s",
     227           0 :                          schemaName, relName, fromfile, strerror(errno));
     228             :             else
     229           0 :                 pg_fatal("error while copying relation \"%s.%s\": partial page found in file \"%s\"",
     230             :                          schemaName, relName, fromfile);
     231             :         }
     232             : 
     233           0 :         totalBytesRead += BLCKSZ;
     234           0 :         old_lastblk = (totalBytesRead == src_filesize);
     235             : 
     236             :         /* Save the page header data */
     237           0 :         memcpy(&pageheader, buffer.data, SizeOfPageHeaderData);
     238             : 
     239             :         /*
     240             :          * These old_* variables point to old visibility map page. old_cur
     241             :          * points to current position on old page. old_blkend points to end of
     242             :          * old block.  old_break is the end+1 position on the old page for the
     243             :          * data that will be transferred to the current new page.
     244             :          */
     245           0 :         old_cur = buffer.data + SizeOfPageHeaderData;
     246           0 :         old_blkend = buffer.data + bytesRead;
     247           0 :         old_break = old_cur + rewriteVmBytesPerPage;
     248             : 
     249           0 :         while (old_break <= old_blkend)
     250             :         {
     251             :             char       *new_cur;
     252           0 :             bool        empty = true;
     253             :             bool        old_lastpart;
     254             : 
     255             :             /* First, copy old page header to new page */
     256           0 :             memcpy(new_vmbuf.data, &pageheader, SizeOfPageHeaderData);
     257             : 
     258             :             /* Rewriting the last part of the last old page? */
     259           0 :             old_lastpart = old_lastblk && (old_break == old_blkend);
     260             : 
     261           0 :             new_cur = new_vmbuf.data + SizeOfPageHeaderData;
     262             : 
     263             :             /* Process old page bytes one by one, and turn it into new page. */
     264           0 :             while (old_cur < old_break)
     265             :             {
     266           0 :                 uint8       byte = *(uint8 *) old_cur;
     267           0 :                 uint16      new_vmbits = 0;
     268             :                 int         i;
     269             : 
     270             :                 /* Generate new format bits while keeping old information */
     271           0 :                 for (i = 0; i < BITS_PER_BYTE; i++)
     272             :                 {
     273           0 :                     if (byte & (1 << i))
     274             :                     {
     275           0 :                         empty = false;
     276           0 :                         new_vmbits |=
     277           0 :                             VISIBILITYMAP_ALL_VISIBLE << (BITS_PER_HEAPBLOCK * i);
     278             :                     }
     279             :                 }
     280             : 
     281             :                 /* Copy new visibility map bytes to new-format page */
     282           0 :                 new_cur[0] = (char) (new_vmbits & 0xFF);
     283           0 :                 new_cur[1] = (char) (new_vmbits >> 8);
     284             : 
     285           0 :                 old_cur++;
     286           0 :                 new_cur += BITS_PER_HEAPBLOCK;
     287             :             }
     288             : 
     289             :             /* If the last part of the last page is empty, skip writing it */
     290           0 :             if (old_lastpart && empty)
     291           0 :                 break;
     292             : 
     293             :             /* Set new checksum for visibility map page, if enabled */
     294           0 :             if (new_cluster.controldata.data_checksum_version != 0)
     295           0 :                 ((PageHeader) new_vmbuf.data)->pd_checksum =
     296           0 :                     pg_checksum_page(new_vmbuf.data, new_blkno);
     297             : 
     298           0 :             errno = 0;
     299           0 :             if (write(dst_fd, new_vmbuf.data, BLCKSZ) != BLCKSZ)
     300             :             {
     301             :                 /* if write didn't set errno, assume problem is no disk space */
     302           0 :                 if (errno == 0)
     303           0 :                     errno = ENOSPC;
     304           0 :                 pg_fatal("error while copying relation \"%s.%s\": could not write file \"%s\": %s",
     305           0 :                          schemaName, relName, tofile, strerror(errno));
     306             :             }
     307             : 
     308             :             /* Advance for next new page */
     309           0 :             old_break += rewriteVmBytesPerPage;
     310           0 :             new_blkno++;
     311             :         }
     312             :     }
     313             : 
     314             :     /* Clean up */
     315           0 :     close(dst_fd);
     316           0 :     close(src_fd);
     317           0 : }
     318             : 
     319             : void
     320           0 : check_file_clone(void)
     321             : {
     322             :     char        existing_file[MAXPGPATH];
     323             :     char        new_link_file[MAXPGPATH];
     324             : 
     325           0 :     snprintf(existing_file, sizeof(existing_file), "%s/PG_VERSION", old_cluster.pgdata);
     326           0 :     snprintf(new_link_file, sizeof(new_link_file), "%s/PG_VERSION.clonetest", new_cluster.pgdata);
     327           0 :     unlink(new_link_file);      /* might fail */
     328             : 
     329             : #if defined(HAVE_COPYFILE) && defined(COPYFILE_CLONE_FORCE)
     330             :     if (copyfile(existing_file, new_link_file, NULL, COPYFILE_CLONE_FORCE) < 0)
     331             :         pg_fatal("could not clone file between old and new data directories: %s",
     332             :                  strerror(errno));
     333             : #elif defined(__linux__) && defined(FICLONE)
     334             :     {
     335             :         int         src_fd;
     336             :         int         dest_fd;
     337             : 
     338           0 :         if ((src_fd = open(existing_file, O_RDONLY | PG_BINARY, 0)) < 0)
     339           0 :             pg_fatal("could not open file \"%s\": %s",
     340           0 :                      existing_file, strerror(errno));
     341             : 
     342           0 :         if ((dest_fd = open(new_link_file, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
     343             :                             pg_file_create_mode)) < 0)
     344           0 :             pg_fatal("could not create file \"%s\": %s",
     345           0 :                      new_link_file, strerror(errno));
     346             : 
     347           0 :         if (ioctl(dest_fd, FICLONE, src_fd) < 0)
     348           0 :             pg_fatal("could not clone file between old and new data directories: %s",
     349           0 :                      strerror(errno));
     350             : 
     351           0 :         close(src_fd);
     352           0 :         close(dest_fd);
     353             :     }
     354             : #else
     355             :     pg_fatal("file cloning not supported on this platform");
     356             : #endif
     357             : 
     358           0 :     unlink(new_link_file);
     359           0 : }
     360             : 
     361             : void
     362           0 : check_hard_link(void)
     363             : {
     364             :     char        existing_file[MAXPGPATH];
     365             :     char        new_link_file[MAXPGPATH];
     366             : 
     367           0 :     snprintf(existing_file, sizeof(existing_file), "%s/PG_VERSION", old_cluster.pgdata);
     368           0 :     snprintf(new_link_file, sizeof(new_link_file), "%s/PG_VERSION.linktest", new_cluster.pgdata);
     369           0 :     unlink(new_link_file);      /* might fail */
     370             : 
     371           0 :     if (link(existing_file, new_link_file) < 0)
     372           0 :         pg_fatal("could not create hard link between old and new data directories: %s\n"
     373             :                  "In link mode the old and new data directories must be on the same file system.",
     374           0 :                  strerror(errno));
     375             : 
     376           0 :     unlink(new_link_file);
     377           0 : }

Generated by: LCOV version 1.14