Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * file_ops.c
4 : * Helper functions for operating on files.
5 : *
6 : * Most of the functions in this file are helper functions for writing to
7 : * the target data directory. The functions check the --dry-run flag, and
8 : * do nothing if it's enabled. You should avoid accessing the target files
9 : * directly but if you do, make sure you honor the --dry-run mode!
10 : *
11 : * Portions Copyright (c) 2013-2024, PostgreSQL Global Development Group
12 : *
13 : *-------------------------------------------------------------------------
14 : */
15 : #include "postgres_fe.h"
16 :
17 : #include <sys/stat.h>
18 : #include <dirent.h>
19 : #include <fcntl.h>
20 : #include <unistd.h>
21 :
22 : #include "common/file_perm.h"
23 : #include "common/file_utils.h"
24 : #include "file_ops.h"
25 : #include "filemap.h"
26 : #include "pg_rewind.h"
27 :
28 : /*
29 : * Currently open target file.
30 : */
31 : static int dstfd = -1;
32 : static char dstpath[MAXPGPATH] = "";
33 :
34 : static void create_target_dir(const char *path);
35 : static void remove_target_dir(const char *path);
36 : static void create_target_symlink(const char *path, const char *link);
37 : static void remove_target_symlink(const char *path);
38 :
39 : static void recurse_dir(const char *datadir, const char *parentpath,
40 : process_file_callback_t callback);
41 :
42 : /*
43 : * Open a target file for writing. If 'trunc' is true and the file already
44 : * exists, it will be truncated.
45 : */
46 : void
47 15468 : open_target_file(const char *path, bool trunc)
48 : {
49 : int mode;
50 :
51 15468 : if (dry_run)
52 540 : return;
53 :
54 14928 : if (dstfd != -1 && !trunc &&
55 6540 : strcmp(path, &dstpath[strlen(datadir_target) + 1]) == 0)
56 1798 : return; /* already open */
57 :
58 13130 : close_target_file();
59 :
60 13130 : snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
61 :
62 13130 : mode = O_WRONLY | O_CREAT | PG_BINARY;
63 13130 : if (trunc)
64 8388 : mode |= O_TRUNC;
65 13130 : dstfd = open(dstpath, mode, pg_file_create_mode);
66 13130 : if (dstfd < 0)
67 0 : pg_fatal("could not open target file \"%s\": %m",
68 : dstpath);
69 : }
70 :
71 : /*
72 : * Close target file, if it's open.
73 : */
74 : void
75 13182 : close_target_file(void)
76 : {
77 13182 : if (dstfd == -1)
78 54 : return;
79 :
80 13128 : if (close(dstfd) != 0)
81 0 : pg_fatal("could not close target file \"%s\": %m",
82 : dstpath);
83 :
84 13128 : dstfd = -1;
85 : }
86 :
87 : void
88 119744 : write_target_range(char *buf, off_t begin, size_t size)
89 : {
90 : size_t writeleft;
91 : char *p;
92 :
93 : /* update progress report */
94 119744 : fetch_done += size;
95 119744 : progress_report(false);
96 :
97 119744 : if (dry_run)
98 13462 : return;
99 :
100 106282 : if (lseek(dstfd, begin, SEEK_SET) == -1)
101 0 : pg_fatal("could not seek in target file \"%s\": %m",
102 : dstpath);
103 :
104 106282 : writeleft = size;
105 106282 : p = buf;
106 212450 : while (writeleft > 0)
107 : {
108 : ssize_t writelen;
109 :
110 106168 : errno = 0;
111 106168 : writelen = write(dstfd, p, writeleft);
112 106168 : if (writelen < 0)
113 : {
114 : /* if write didn't set errno, assume problem is no disk space */
115 0 : if (errno == 0)
116 0 : errno = ENOSPC;
117 0 : pg_fatal("could not write file \"%s\": %m",
118 : dstpath);
119 : }
120 :
121 106168 : p += writelen;
122 106168 : writeleft -= writelen;
123 : }
124 :
125 : /* keep the file open, in case we need to copy more blocks in it */
126 : }
127 :
128 :
129 : void
130 1428 : remove_target(file_entry_t *entry)
131 : {
132 : Assert(entry->action == FILE_ACTION_REMOVE);
133 : Assert(entry->target_exists);
134 :
135 1428 : switch (entry->target_type)
136 : {
137 34 : case FILE_TYPE_DIRECTORY:
138 34 : remove_target_dir(entry->path);
139 34 : break;
140 :
141 1394 : case FILE_TYPE_REGULAR:
142 1394 : remove_target_file(entry->path, false);
143 1394 : break;
144 :
145 0 : case FILE_TYPE_SYMLINK:
146 0 : remove_target_symlink(entry->path);
147 0 : break;
148 :
149 0 : case FILE_TYPE_UNDEFINED:
150 0 : pg_fatal("undefined file type for \"%s\"", entry->path);
151 : break;
152 : }
153 1428 : }
154 :
155 : void
156 18 : create_target(file_entry_t *entry)
157 : {
158 : Assert(entry->action == FILE_ACTION_CREATE);
159 : Assert(!entry->target_exists);
160 :
161 18 : switch (entry->source_type)
162 : {
163 18 : case FILE_TYPE_DIRECTORY:
164 18 : create_target_dir(entry->path);
165 18 : break;
166 :
167 0 : case FILE_TYPE_SYMLINK:
168 0 : create_target_symlink(entry->path, entry->source_link_target);
169 0 : break;
170 :
171 0 : case FILE_TYPE_REGULAR:
172 : /* can't happen. Regular files are created with open_target_file. */
173 0 : pg_fatal("invalid action (CREATE) for regular file");
174 : break;
175 :
176 0 : case FILE_TYPE_UNDEFINED:
177 0 : pg_fatal("undefined file type for \"%s\"", entry->path);
178 : break;
179 : }
180 18 : }
181 :
182 : /*
183 : * Remove a file from target data directory. If missing_ok is true, it
184 : * is fine for the target file to not exist.
185 : */
186 : void
187 1394 : remove_target_file(const char *path, bool missing_ok)
188 : {
189 : char dstpath[MAXPGPATH];
190 :
191 1394 : if (dry_run)
192 14 : return;
193 :
194 1380 : snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
195 1380 : if (unlink(dstpath) != 0)
196 : {
197 0 : if (errno == ENOENT && missing_ok)
198 0 : return;
199 :
200 0 : pg_fatal("could not remove file \"%s\": %m",
201 : dstpath);
202 : }
203 : }
204 :
205 : void
206 8 : truncate_target_file(const char *path, off_t newsize)
207 : {
208 : char dstpath[MAXPGPATH];
209 : int fd;
210 :
211 8 : if (dry_run)
212 2 : return;
213 :
214 6 : snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
215 :
216 6 : fd = open(dstpath, O_WRONLY, pg_file_create_mode);
217 6 : if (fd < 0)
218 0 : pg_fatal("could not open file \"%s\" for truncation: %m",
219 : dstpath);
220 :
221 6 : if (ftruncate(fd, newsize) != 0)
222 0 : pg_fatal("could not truncate file \"%s\" to %u: %m",
223 : dstpath, (unsigned int) newsize);
224 :
225 6 : close(fd);
226 : }
227 :
228 : static void
229 18 : create_target_dir(const char *path)
230 : {
231 : char dstpath[MAXPGPATH];
232 :
233 18 : if (dry_run)
234 0 : return;
235 :
236 18 : snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
237 18 : if (mkdir(dstpath, pg_dir_create_mode) != 0)
238 0 : pg_fatal("could not create directory \"%s\": %m",
239 : dstpath);
240 : }
241 :
242 : static void
243 34 : remove_target_dir(const char *path)
244 : {
245 : char dstpath[MAXPGPATH];
246 :
247 34 : if (dry_run)
248 2 : return;
249 :
250 32 : snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
251 32 : if (rmdir(dstpath) != 0)
252 0 : pg_fatal("could not remove directory \"%s\": %m",
253 : dstpath);
254 : }
255 :
256 : static void
257 0 : create_target_symlink(const char *path, const char *link)
258 : {
259 : char dstpath[MAXPGPATH];
260 :
261 0 : if (dry_run)
262 0 : return;
263 :
264 0 : snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
265 0 : if (symlink(link, dstpath) != 0)
266 0 : pg_fatal("could not create symbolic link at \"%s\": %m",
267 : dstpath);
268 : }
269 :
270 : static void
271 0 : remove_target_symlink(const char *path)
272 : {
273 : char dstpath[MAXPGPATH];
274 :
275 0 : if (dry_run)
276 0 : return;
277 :
278 0 : snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
279 0 : if (unlink(dstpath) != 0)
280 0 : pg_fatal("could not remove symbolic link \"%s\": %m",
281 : dstpath);
282 : }
283 :
284 : /*
285 : * Sync target data directory to ensure that modifications are safely on disk.
286 : *
287 : * We do this once, for the whole data directory, for performance reasons. At
288 : * the end of pg_rewind's run, the kernel is likely to already have flushed
289 : * most dirty buffers to disk. Additionally sync_pgdata uses a two-pass
290 : * approach when fsync is specified (only initiating writeback in the first
291 : * pass), which often reduces the overall amount of IO noticeably.
292 : */
293 : void
294 26 : sync_target_dir(void)
295 : {
296 26 : if (!do_sync || dry_run)
297 24 : return;
298 :
299 2 : sync_pgdata(datadir_target, PG_VERSION_NUM, sync_method);
300 : }
301 :
302 :
303 : /*
304 : * Read a file into memory. The file to be read is <datadir>/<path>.
305 : * The file contents are returned in a malloc'd buffer, and *filesize
306 : * is set to the length of the file.
307 : *
308 : * The returned buffer is always zero-terminated; the size of the returned
309 : * buffer is actually *filesize + 1. That's handy when reading a text file.
310 : * This function can be used to read binary files as well, you can just
311 : * ignore the zero-terminator in that case.
312 : */
313 : char *
314 110 : slurpFile(const char *datadir, const char *path, size_t *filesize)
315 : {
316 : int fd;
317 : char *buffer;
318 : struct stat statbuf;
319 : char fullpath[MAXPGPATH];
320 : int len;
321 : int r;
322 :
323 110 : snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path);
324 :
325 110 : if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) == -1)
326 0 : pg_fatal("could not open file \"%s\" for reading: %m",
327 : fullpath);
328 :
329 110 : if (fstat(fd, &statbuf) < 0)
330 0 : pg_fatal("could not open file \"%s\" for reading: %m",
331 : fullpath);
332 :
333 110 : len = statbuf.st_size;
334 :
335 110 : buffer = pg_malloc(len + 1);
336 :
337 110 : r = read(fd, buffer, len);
338 110 : if (r != len)
339 : {
340 0 : if (r < 0)
341 0 : pg_fatal("could not read file \"%s\": %m",
342 : fullpath);
343 : else
344 0 : pg_fatal("could not read file \"%s\": read %d of %zu",
345 : fullpath, r, (Size) len);
346 : }
347 110 : close(fd);
348 :
349 : /* Zero-terminate the buffer. */
350 110 : buffer[len] = '\0';
351 :
352 110 : if (filesize)
353 90 : *filesize = len;
354 110 : return buffer;
355 : }
356 :
357 : /*
358 : * Traverse through all files in a data directory, calling 'callback'
359 : * for each file.
360 : */
361 : void
362 44 : traverse_datadir(const char *datadir, process_file_callback_t callback)
363 : {
364 44 : recurse_dir(datadir, NULL, callback);
365 44 : }
366 :
367 : /*
368 : * recursive part of traverse_datadir
369 : *
370 : * parentpath is the current subdirectory's path relative to datadir,
371 : * or NULL at the top level.
372 : */
373 : static void
374 1294 : recurse_dir(const char *datadir, const char *parentpath,
375 : process_file_callback_t callback)
376 : {
377 : DIR *xldir;
378 : struct dirent *xlde;
379 : char fullparentpath[MAXPGPATH];
380 :
381 1294 : if (parentpath)
382 1250 : snprintf(fullparentpath, MAXPGPATH, "%s/%s", datadir, parentpath);
383 : else
384 44 : snprintf(fullparentpath, MAXPGPATH, "%s", datadir);
385 :
386 1294 : xldir = opendir(fullparentpath);
387 1294 : if (xldir == NULL)
388 0 : pg_fatal("could not open directory \"%s\": %m",
389 : fullparentpath);
390 :
391 53936 : while (errno = 0, (xlde = readdir(xldir)) != NULL)
392 : {
393 : struct stat fst;
394 : char fullpath[MAXPGPATH * 2];
395 : char path[MAXPGPATH * 2];
396 :
397 52642 : if (strcmp(xlde->d_name, ".") == 0 ||
398 51348 : strcmp(xlde->d_name, "..") == 0)
399 2588 : continue;
400 :
401 50054 : snprintf(fullpath, sizeof(fullpath), "%s/%s", fullparentpath, xlde->d_name);
402 :
403 50054 : if (lstat(fullpath, &fst) < 0)
404 : {
405 0 : if (errno == ENOENT)
406 : {
407 : /*
408 : * File doesn't exist anymore. This is ok, if the new primary
409 : * is running and the file was just removed. If it was a data
410 : * file, there should be a WAL record of the removal. If it
411 : * was something else, it couldn't have been anyway.
412 : *
413 : * TODO: But complain if we're processing the target dir!
414 : */
415 : }
416 : else
417 0 : pg_fatal("could not stat file \"%s\": %m",
418 : fullpath);
419 : }
420 :
421 50054 : if (parentpath)
422 48982 : snprintf(path, sizeof(path), "%s/%s", parentpath, xlde->d_name);
423 : else
424 1072 : snprintf(path, sizeof(path), "%s", xlde->d_name);
425 :
426 50054 : if (S_ISREG(fst.st_mode))
427 48804 : callback(path, FILE_TYPE_REGULAR, fst.st_size, NULL);
428 1250 : else if (S_ISDIR(fst.st_mode))
429 : {
430 1246 : callback(path, FILE_TYPE_DIRECTORY, 0, NULL);
431 : /* recurse to handle subdirectories */
432 1246 : recurse_dir(datadir, path, callback);
433 : }
434 4 : else if (S_ISLNK(fst.st_mode))
435 : {
436 : char link_target[MAXPGPATH];
437 : int len;
438 :
439 4 : len = readlink(fullpath, link_target, sizeof(link_target));
440 4 : if (len < 0)
441 0 : pg_fatal("could not read symbolic link \"%s\": %m",
442 : fullpath);
443 4 : if (len >= sizeof(link_target))
444 0 : pg_fatal("symbolic link \"%s\" target is too long",
445 : fullpath);
446 4 : link_target[len] = '\0';
447 :
448 4 : callback(path, FILE_TYPE_SYMLINK, 0, link_target);
449 :
450 : /*
451 : * If it's a symlink within pg_tblspc, we need to recurse into it,
452 : * to process all the tablespaces. We also follow a symlink if
453 : * it's for pg_wal. Symlinks elsewhere are ignored.
454 : */
455 4 : if ((parentpath && strcmp(parentpath, PG_TBLSPC_DIR) == 0) ||
456 4 : strcmp(path, "pg_wal") == 0)
457 4 : recurse_dir(datadir, path, callback);
458 : }
459 : }
460 :
461 1294 : if (errno)
462 0 : pg_fatal("could not read directory \"%s\": %m",
463 : fullparentpath);
464 :
465 1294 : if (closedir(xldir))
466 0 : pg_fatal("could not close directory \"%s\": %m",
467 : fullparentpath);
468 1294 : }
|
