LCOV - code coverage report
Current view: top level - src/bin/pg_rewind - file_ops.c (source / functions) Hit Total Coverage
Test: PostgreSQL 14devel Lines: 121 171 70.8 %
Date: 2021-01-26 03:06:49 Functions: 13 15 86.7 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * file_ops.c
       4             :  *    Helper functions for operating on files.
       5             :  *
       6             :  * Most of the functions in this file are helper functions for writing to
       7             :  * the target data directory. The functions check the --dry-run flag, and
       8             :  * do nothing if it's enabled. You should avoid accessing the target files
       9             :  * directly but if you do, make sure you honor the --dry-run mode!
      10             :  *
      11             :  * Portions Copyright (c) 2013-2021, PostgreSQL Global Development Group
      12             :  *
      13             :  *-------------------------------------------------------------------------
      14             :  */
      15             : #include "postgres_fe.h"
      16             : 
      17             : #include <sys/stat.h>
      18             : #include <dirent.h>
      19             : #include <fcntl.h>
      20             : #include <unistd.h>
      21             : 
      22             : #include "common/file_perm.h"
      23             : #include "common/file_utils.h"
      24             : #include "file_ops.h"
      25             : #include "filemap.h"
      26             : #include "pg_rewind.h"
      27             : 
      28             : /*
      29             :  * Currently open target file.
      30             :  */
      31             : static int  dstfd = -1;
      32             : static char dstpath[MAXPGPATH] = "";
      33             : 
      34             : static void create_target_dir(const char *path);
      35             : static void remove_target_dir(const char *path);
      36             : static void create_target_symlink(const char *path, const char *link);
      37             : static void remove_target_symlink(const char *path);
      38             : 
      39             : static void recurse_dir(const char *datadir, const char *parentpath,
      40             :                         process_file_callback_t callback);
      41             : 
      42             : /*
      43             :  * Open a target file for writing. If 'trunc' is true and the file already
      44             :  * exists, it will be truncated.
      45             :  */
      46             : void
      47       16118 : open_target_file(const char *path, bool trunc)
      48             : {
      49             :     int         mode;
      50             : 
      51       16118 :     if (dry_run)
      52        1036 :         return;
      53             : 
      54       15082 :     if (dstfd != -1 && !trunc &&
      55        7834 :         strcmp(path, &dstpath[strlen(datadir_target) + 1]) == 0)
      56        3970 :         return;                 /* already open */
      57             : 
      58       11112 :     close_target_file();
      59             : 
      60       11112 :     snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
      61             : 
      62       11112 :     mode = O_WRONLY | O_CREAT | PG_BINARY;
      63       11112 :     if (trunc)
      64        7248 :         mode |= O_TRUNC;
      65       11112 :     dstfd = open(dstpath, mode, pg_file_create_mode);
      66       11112 :     if (dstfd < 0)
      67           0 :         pg_fatal("could not open target file \"%s\": %m",
      68             :                  dstpath);
      69             : }
      70             : 
      71             : /*
      72             :  * Close target file, if it's open.
      73             :  */
      74             : void
      75       11160 : close_target_file(void)
      76             : {
      77       11160 :     if (dstfd == -1)
      78          48 :         return;
      79             : 
      80       11112 :     if (close(dstfd) != 0)
      81           0 :         pg_fatal("could not close target file \"%s\": %m",
      82             :                  dstpath);
      83             : 
      84       11112 :     dstfd = -1;
      85             : }
      86             : 
      87             : void
      88       82864 : write_target_range(char *buf, off_t begin, size_t size)
      89             : {
      90             :     size_t      writeleft;
      91             :     char       *p;
      92             : 
      93             :     /* update progress report */
      94       82864 :     fetch_done += size;
      95       82864 :     progress_report(false);
      96             : 
      97       82864 :     if (dry_run)
      98       13442 :         return;
      99             : 
     100       69422 :     if (lseek(dstfd, begin, SEEK_SET) == -1)
     101           0 :         pg_fatal("could not seek in target file \"%s\": %m",
     102             :                  dstpath);
     103             : 
     104       69422 :     writeleft = size;
     105       69422 :     p = buf;
     106      138844 :     while (writeleft > 0)
     107             :     {
     108             :         ssize_t     writelen;
     109             : 
     110       69422 :         errno = 0;
     111       69422 :         writelen = write(dstfd, p, writeleft);
     112       69422 :         if (writelen < 0)
     113             :         {
     114             :             /* if write didn't set errno, assume problem is no disk space */
     115           0 :             if (errno == 0)
     116           0 :                 errno = ENOSPC;
     117           0 :             pg_fatal("could not write file \"%s\": %m",
     118             :                      dstpath);
     119             :         }
     120             : 
     121       69422 :         p += writelen;
     122       69422 :         writeleft -= writelen;
     123             :     }
     124             : 
     125             :     /* keep the file open, in case we need to copy more blocks in it */
     126             : }
     127             : 
     128             : 
     129             : void
     130        1370 : remove_target(file_entry_t *entry)
     131             : {
     132             :     Assert(entry->action == FILE_ACTION_REMOVE);
     133             :     Assert(entry->target_exists);
     134             : 
     135        1370 :     switch (entry->target_type)
     136             :     {
     137          32 :         case FILE_TYPE_DIRECTORY:
     138          32 :             remove_target_dir(entry->path);
     139          32 :             break;
     140             : 
     141        1338 :         case FILE_TYPE_REGULAR:
     142        1338 :             remove_target_file(entry->path, false);
     143        1338 :             break;
     144             : 
     145           0 :         case FILE_TYPE_SYMLINK:
     146           0 :             remove_target_symlink(entry->path);
     147           0 :             break;
     148             : 
     149           0 :         case FILE_TYPE_UNDEFINED:
     150           0 :             pg_fatal("undefined file type for \"%s\"", entry->path);
     151             :             break;
     152             :     }
     153        1370 : }
     154             : 
     155             : void
     156          14 : create_target(file_entry_t *entry)
     157             : {
     158             :     Assert(entry->action == FILE_ACTION_CREATE);
     159             :     Assert(!entry->target_exists);
     160             : 
     161          14 :     switch (entry->source_type)
     162             :     {
     163          14 :         case FILE_TYPE_DIRECTORY:
     164          14 :             create_target_dir(entry->path);
     165          14 :             break;
     166             : 
     167           0 :         case FILE_TYPE_SYMLINK:
     168           0 :             create_target_symlink(entry->path, entry->source_link_target);
     169           0 :             break;
     170             : 
     171           0 :         case FILE_TYPE_REGULAR:
     172             :             /* can't happen. Regular files are created with open_target_file. */
     173           0 :             pg_fatal("invalid action (CREATE) for regular file");
     174             :             break;
     175             : 
     176           0 :         case FILE_TYPE_UNDEFINED:
     177           0 :             pg_fatal("undefined file type for \"%s\"", entry->path);
     178             :             break;
     179             :     }
     180          14 : }
     181             : 
     182             : /*
     183             :  * Remove a file from target data directory.  If missing_ok is true, it
     184             :  * is fine for the target file to not exist.
     185             :  */
     186             : void
     187        1338 : remove_target_file(const char *path, bool missing_ok)
     188             : {
     189             :     char        dstpath[MAXPGPATH];
     190             : 
     191        1338 :     if (dry_run)
     192          14 :         return;
     193             : 
     194        1324 :     snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
     195        1324 :     if (unlink(dstpath) != 0)
     196             :     {
     197           0 :         if (errno == ENOENT && missing_ok)
     198           0 :             return;
     199             : 
     200           0 :         pg_fatal("could not remove file \"%s\": %m",
     201             :                  dstpath);
     202             :     }
     203             : }
     204             : 
     205             : void
     206           8 : truncate_target_file(const char *path, off_t newsize)
     207             : {
     208             :     char        dstpath[MAXPGPATH];
     209             :     int         fd;
     210             : 
     211           8 :     if (dry_run)
     212           2 :         return;
     213             : 
     214           6 :     snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
     215             : 
     216           6 :     fd = open(dstpath, O_WRONLY, pg_file_create_mode);
     217           6 :     if (fd < 0)
     218           0 :         pg_fatal("could not open file \"%s\" for truncation: %m",
     219             :                  dstpath);
     220             : 
     221           6 :     if (ftruncate(fd, newsize) != 0)
     222           0 :         pg_fatal("could not truncate file \"%s\" to %u: %m",
     223             :                  dstpath, (unsigned int) newsize);
     224             : 
     225           6 :     close(fd);
     226             : }
     227             : 
     228             : static void
     229          14 : create_target_dir(const char *path)
     230             : {
     231             :     char        dstpath[MAXPGPATH];
     232             : 
     233          14 :     if (dry_run)
     234           0 :         return;
     235             : 
     236          14 :     snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
     237          14 :     if (mkdir(dstpath, pg_dir_create_mode) != 0)
     238           0 :         pg_fatal("could not create directory \"%s\": %m",
     239             :                  dstpath);
     240             : }
     241             : 
     242             : static void
     243          32 : remove_target_dir(const char *path)
     244             : {
     245             :     char        dstpath[MAXPGPATH];
     246             : 
     247          32 :     if (dry_run)
     248           2 :         return;
     249             : 
     250          30 :     snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
     251          30 :     if (rmdir(dstpath) != 0)
     252           0 :         pg_fatal("could not remove directory \"%s\": %m",
     253             :                  dstpath);
     254             : }
     255             : 
     256             : static void
     257           0 : create_target_symlink(const char *path, const char *link)
     258             : {
     259             :     char        dstpath[MAXPGPATH];
     260             : 
     261           0 :     if (dry_run)
     262           0 :         return;
     263             : 
     264           0 :     snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
     265           0 :     if (symlink(link, dstpath) != 0)
     266           0 :         pg_fatal("could not create symbolic link at \"%s\": %m",
     267             :                  dstpath);
     268             : }
     269             : 
     270             : static void
     271           0 : remove_target_symlink(const char *path)
     272             : {
     273             :     char        dstpath[MAXPGPATH];
     274             : 
     275           0 :     if (dry_run)
     276           0 :         return;
     277             : 
     278           0 :     snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
     279           0 :     if (unlink(dstpath) != 0)
     280           0 :         pg_fatal("could not remove symbolic link \"%s\": %m",
     281             :                  dstpath);
     282             : }
     283             : 
     284             : /*
     285             :  * Sync target data directory to ensure that modifications are safely on disk.
     286             :  *
     287             :  * We do this once, for the whole data directory, for performance reasons.  At
     288             :  * the end of pg_rewind's run, the kernel is likely to already have flushed
     289             :  * most dirty buffers to disk.  Additionally fsync_pgdata uses a two-pass
     290             :  * approach (only initiating writeback in the first pass), which often reduces
     291             :  * the overall amount of IO noticeably.
     292             :  */
     293             : void
     294          24 : sync_target_dir(void)
     295             : {
     296          24 :     if (!do_sync || dry_run)
     297          22 :         return;
     298             : 
     299           2 :     fsync_pgdata(datadir_target, PG_VERSION_NUM);
     300             : }
     301             : 
     302             : 
     303             : /*
     304             :  * Read a file into memory. The file to be read is <datadir>/<path>.
     305             :  * The file contents are returned in a malloc'd buffer, and *filesize
     306             :  * is set to the length of the file.
     307             :  *
     308             :  * The returned buffer is always zero-terminated; the size of the returned
     309             :  * buffer is actually *filesize + 1. That's handy when reading a text file.
     310             :  * This function can be used to read binary files as well, you can just
     311             :  * ignore the zero-terminator in that case.
     312             :  */
     313             : char *
     314          96 : slurpFile(const char *datadir, const char *path, size_t *filesize)
     315             : {
     316             :     int         fd;
     317             :     char       *buffer;
     318             :     struct stat statbuf;
     319             :     char        fullpath[MAXPGPATH];
     320             :     int         len;
     321             :     int         r;
     322             : 
     323          96 :     snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path);
     324             : 
     325          96 :     if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) == -1)
     326           0 :         pg_fatal("could not open file \"%s\" for reading: %m",
     327             :                  fullpath);
     328             : 
     329          96 :     if (fstat(fd, &statbuf) < 0)
     330           0 :         pg_fatal("could not open file \"%s\" for reading: %m",
     331             :                  fullpath);
     332             : 
     333          96 :     len = statbuf.st_size;
     334             : 
     335          96 :     buffer = pg_malloc(len + 1);
     336             : 
     337          96 :     r = read(fd, buffer, len);
     338          96 :     if (r != len)
     339             :     {
     340           0 :         if (r < 0)
     341           0 :             pg_fatal("could not read file \"%s\": %m",
     342             :                      fullpath);
     343             :         else
     344           0 :             pg_fatal("could not read file \"%s\": read %d of %zu",
     345             :                      fullpath, r, (Size) len);
     346             :     }
     347          96 :     close(fd);
     348             : 
     349             :     /* Zero-terminate the buffer. */
     350          96 :     buffer[len] = '\0';
     351             : 
     352          96 :     if (filesize)
     353          80 :         *filesize = len;
     354          96 :     return buffer;
     355             : }
     356             : 
     357             : /*
     358             :  * Traverse through all files in a data directory, calling 'callback'
     359             :  * for each file.
     360             :  */
     361             : void
     362          36 : traverse_datadir(const char *datadir, process_file_callback_t callback)
     363             : {
     364          36 :     recurse_dir(datadir, NULL, callback);
     365          36 : }
     366             : 
     367             : /*
     368             :  * recursive part of traverse_datadir
     369             :  *
     370             :  * parentpath is the current subdirectory's path relative to datadir,
     371             :  * or NULL at the top level.
     372             :  */
     373             : static void
     374         996 : recurse_dir(const char *datadir, const char *parentpath,
     375             :             process_file_callback_t callback)
     376             : {
     377             :     DIR        *xldir;
     378             :     struct dirent *xlde;
     379             :     char        fullparentpath[MAXPGPATH];
     380             : 
     381         996 :     if (parentpath)
     382         960 :         snprintf(fullparentpath, MAXPGPATH, "%s/%s", datadir, parentpath);
     383             :     else
     384          36 :         snprintf(fullparentpath, MAXPGPATH, "%s", datadir);
     385             : 
     386         996 :     xldir = opendir(fullparentpath);
     387         996 :     if (xldir == NULL)
     388           0 :         pg_fatal("could not open directory \"%s\": %m",
     389             :                  fullparentpath);
     390             : 
     391       43496 :     while (errno = 0, (xlde = readdir(xldir)) != NULL)
     392             :     {
     393             :         struct stat fst;
     394             :         char        fullpath[MAXPGPATH * 2];
     395             :         char        path[MAXPGPATH * 2];
     396             : 
     397       42500 :         if (strcmp(xlde->d_name, ".") == 0 ||
     398       41504 :             strcmp(xlde->d_name, "..") == 0)
     399        1992 :             continue;
     400             : 
     401       40508 :         snprintf(fullpath, sizeof(fullpath), "%s/%s", fullparentpath, xlde->d_name);
     402             : 
     403       40508 :         if (lstat(fullpath, &fst) < 0)
     404             :         {
     405           0 :             if (errno == ENOENT)
     406             :             {
     407             :                 /*
     408             :                  * File doesn't exist anymore. This is ok, if the new primary
     409             :                  * is running and the file was just removed. If it was a data
     410             :                  * file, there should be a WAL record of the removal. If it
     411             :                  * was something else, it couldn't have been anyway.
     412             :                  *
     413             :                  * TODO: But complain if we're processing the target dir!
     414             :                  */
     415             :             }
     416             :             else
     417           0 :                 pg_fatal("could not stat file \"%s\": %m",
     418             :                          fullpath);
     419             :         }
     420             : 
     421       40508 :         if (parentpath)
     422       39632 :             snprintf(path, sizeof(path), "%s/%s", parentpath, xlde->d_name);
     423             :         else
     424         876 :             snprintf(path, sizeof(path), "%s", xlde->d_name);
     425             : 
     426       40508 :         if (S_ISREG(fst.st_mode))
     427       39548 :             callback(path, FILE_TYPE_REGULAR, fst.st_size, NULL);
     428         960 :         else if (S_ISDIR(fst.st_mode))
     429             :         {
     430         956 :             callback(path, FILE_TYPE_DIRECTORY, 0, NULL);
     431             :             /* recurse to handle subdirectories */
     432         956 :             recurse_dir(datadir, path, callback);
     433             :         }
     434             : #ifndef WIN32
     435           4 :         else if (S_ISLNK(fst.st_mode))
     436             : #else
     437             :         else if (pgwin32_is_junction(fullpath))
     438             : #endif
     439             :         {
     440             : #if defined(HAVE_READLINK) || defined(WIN32)
     441             :             char        link_target[MAXPGPATH];
     442             :             int         len;
     443             : 
     444           4 :             len = readlink(fullpath, link_target, sizeof(link_target));
     445           4 :             if (len < 0)
     446           0 :                 pg_fatal("could not read symbolic link \"%s\": %m",
     447             :                          fullpath);
     448           4 :             if (len >= sizeof(link_target))
     449           0 :                 pg_fatal("symbolic link \"%s\" target is too long",
     450             :                          fullpath);
     451           4 :             link_target[len] = '\0';
     452             : 
     453           4 :             callback(path, FILE_TYPE_SYMLINK, 0, link_target);
     454             : 
     455             :             /*
     456             :              * If it's a symlink within pg_tblspc, we need to recurse into it,
     457             :              * to process all the tablespaces.  We also follow a symlink if
     458             :              * it's for pg_wal.  Symlinks elsewhere are ignored.
     459             :              */
     460           4 :             if ((parentpath && strcmp(parentpath, "pg_tblspc") == 0) ||
     461           4 :                 strcmp(path, "pg_wal") == 0)
     462           4 :                 recurse_dir(datadir, path, callback);
     463             : #else
     464             :             pg_fatal("\"%s\" is a symbolic link, but symbolic links are not supported on this platform",
     465             :                      fullpath);
     466             : #endif                          /* HAVE_READLINK */
     467             :         }
     468             :     }
     469             : 
     470         996 :     if (errno)
     471           0 :         pg_fatal("could not read directory \"%s\": %m",
     472             :                  fullparentpath);
     473             : 
     474         996 :     if (closedir(xldir))
     475           0 :         pg_fatal("could not close directory \"%s\": %m",
     476             :                  fullparentpath);
     477         996 : }

Generated by: LCOV version 1.13