Sync pg_dump and pg_dumpall output

Before exiting any files are fsync'ed. A --no-sync option is also
provided for a faster exit if desired.

Michael Paquier.

Reviewed by Albe Laurenz

Discussion: https://postgr.es/m/CAB7nPqS1uZ=Ov+UruW6jr3vB-S_DLVMPc0dQpV-fTDjmm0ZQMg@mail.gmail.com

diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index a1e03c481d1107891c4d80654fa697fb80920976..bb32fb12e0b4c46e582a098d35cabbd4db69eb65 100644 (file)
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -859,6 +859,20 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>--no-sync</option></term>
+      <listitem>
+       <para>
+        By default, <command>pg_dump</command> will wait for all files
+        to be written safely to disk.  This option causes
+        <command>pg_dump</command> to return without waiting, which is
+        faster, but means that a subsequent operating system crash can leave
+        the dump corrupt.  Generally, this option is useful for testing
+        but should not be used when dumping data from production installation.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>--quote-all-identifiers</></term>
       <listitem>

diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index afbadce2476453efb2f8e1644976504f710cea14..070b902487201358462f2826990f3fafed656eab 100644 (file)
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -354,6 +354,20 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>--no-sync</option></term>
+      <listitem>
+       <para>
+        By default, <command>pg_dumpall</command> will wait for all files
+        to be written safely to disk.  This option causes
+        <command>pg_dumpall</command> to return without waiting, which is
+        faster, but means that a subsequent operating system crash can leave
+        the dump corrupt.  Generally, this option is useful for testing
+        but should not be used when dumping data from production installation.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>--no-tablespaces</option></term>
       <listitem>

diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 983a999fcdca6fd5983abeea394ac72801300ec3..610bed531c4b0544571e02c6f9bbf94a12b79c19 100644 (file)
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -276,7 +276,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
 
 /* Create a new archive */
 extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
-                         const int compression, ArchiveMode mode,
+                         const int compression, bool dosync, ArchiveMode mode,
                          SetupWorkerPtr setupDumpWorker);
 
 /* The --list option */

diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 734373beaa5090c6781dc6e25e45df7f59500e49..dd0892539a3ccf00634901b3c2a1e918e4068255 100644 (file)
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -54,7 +54,8 @@ static const char *modulename = gettext_noop("archiver");
 
 
 static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
-        const int compression, ArchiveMode mode, SetupWorkerPtr setupWorkerPtr);
+        const int compression, bool dosync, ArchiveMode mode,
+        SetupWorkerPtr setupWorkerPtr);
 static void _getObjectDescription(PQExpBuffer buf, TocEntry *te,
                                          ArchiveHandle *AH);
 static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData, bool acl_pass);
@@ -202,10 +203,12 @@ setupRestoreWorker(Archive *AHX)
 /* Public */
 Archive *
 CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
-        const int compression, ArchiveMode mode, SetupWorkerPtr setupDumpWorker)
+                         const int compression, bool dosync, ArchiveMode mode,
+                         SetupWorkerPtr setupDumpWorker)
 
 {
-       ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, mode, setupDumpWorker);
+       ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync,
+                                                                mode, setupDumpWorker);
 
        return (Archive *) AH;
 }
@@ -215,7 +218,7 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
 Archive *
 OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
 {
-       ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, archModeRead, setupRestoreWorker);
+       ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
 
        return (Archive *) AH;
 }
@@ -2269,7 +2272,8 @@ _discoverArchiveFormat(ArchiveHandle *AH)
  */
 static ArchiveHandle *
 _allocAH(const char *FileSpec, const ArchiveFormat fmt,
-         const int compression, ArchiveMode mode, SetupWorkerPtr setupWorkerPtr)
+                const int compression, bool dosync, ArchiveMode mode,
+                SetupWorkerPtr setupWorkerPtr)
 {
        ArchiveHandle *AH;
 
@@ -2323,6 +2327,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
 
        AH->mode = mode;
        AH->compression = compression;
+       AH->dosync = dosync;
 
        memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
 

diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a44e16ee45a787e573dfc52baf68520ebae22d08..b00a7ede97316392d49b7d80db2247d7625c10d2 100644 (file)
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -312,6 +312,7 @@ struct _archiveHandle
                                                                 * values for compression: -1
                                                                 * Z_DEFAULT_COMPRESSION 0      COMPRESSION_NONE
                                                                 * 1-9 levels for gzip compression */
+       bool            dosync;                 /* data requested to be synced on sight */
        ArchiveMode mode;                       /* File mode - r or w */
        void       *formatData;         /* Header data specific to file format */
 

diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 5737608f9e13ab4f812995525f47b04ea9ce66c4..a1f4cb1fea534d6c06f3072353ec965bfcca6bbc 100644 (file)
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -28,6 +28,7 @@
 #include "compress_io.h"
 #include "parallel.h"
 #include "pg_backup_utils.h"
+#include "common/file_utils.h"
 
 /*--------
  * Routines in the format interface
@@ -721,6 +722,10 @@ _CloseArchive(ArchiveHandle *AH)
        if (fclose(AH->FH) != 0)
                exit_horribly(modulename, "could not close archive file: %s\n", strerror(errno));
 
+       /* Sync the output file if one is defined */
+       if (AH->dosync && AH->mode == archModeWrite && AH->fSpec)
+               (void) fsync_fname(AH->fSpec, false, progname);
+
        AH->FH = NULL;
 }
 

diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 0d7322f73a7dd5e905a7a2406472abe599779053..79922da8ba36b2feee69f12ac27a757aff83d6e1 100644 (file)
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -37,6 +37,7 @@
 #include "compress_io.h"
 #include "parallel.h"
 #include "pg_backup_utils.h"
+#include "common/file_utils.h"
 
 #include <dirent.h>
 #include <sys/stat.h>
@@ -593,6 +594,13 @@ _CloseArchive(ArchiveHandle *AH)
                WriteDataChunks(AH, ctx->pstate);
 
                ParallelBackupEnd(AH, ctx->pstate);
+
+               /*
+                * In directory mode, there is no need to sync all the entries
+                * individually. Just recurse once through all the files generated.
+                */
+               if (AH->dosync)
+                       fsync_dir_recurse(ctx->directory, progname);
        }
        AH->FH = NULL;
 }

diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 9cadd0c4a459b714742e6cf512e09e61a6febf9c..a2b320f3710ce249d89c197ec0bd283b5a59f606 100644 (file)
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -33,6 +33,7 @@
 #include "pg_backup_tar.h"
 #include "pg_backup_utils.h"
 #include "pgtar.h"
+#include "common/file_utils.h"
 #include "fe_utils/string_utils.h"
 
 #include <sys/stat.h>
@@ -901,6 +902,10 @@ _CloseArchive(ArchiveHandle *AH)
                        if (fputc(0, ctx->tarFH) == EOF)
                                WRITE_ERROR_EXIT;
                }
+
+               /* Sync the output file if one is defined */
+               if (AH->dosync && AH->fSpec)
+                       (void) fsync_fname(AH->fSpec, false, progname);
        }
 
        AH->FH = NULL;

diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 52fa6f33e3d1fb570c192e2d584394bfdab9545a..2b5a52656c96ea987030fcdbbe15caf4beb86ca5 100644 (file)
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -89,6 +89,8 @@ typedef enum OidOptions
 /* global decls */
 bool           g_verbose;                      /* User wants verbose narration of our
                                                                 * activities. */
+static bool dosync = true;             /* Issue fsync() to make dump durable
+                                                                * on disk. */
 
 /* subquery used to convert user ID (eg, datdba) to user name */
 static const char *username_subquery;
@@ -353,6 +355,7 @@ main(int argc, char **argv)
                {"no-security-labels", no_argument, &dopt.no_security_labels, 1},
                {"no-synchronized-snapshots", no_argument, &dopt.no_synchronized_snapshots, 1},
                {"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1},
+               {"no-sync", no_argument, NULL, 7},
 
                {NULL, 0, NULL, 0}
        };
@@ -533,6 +536,10 @@ main(int argc, char **argv)
                                dumpsnapshot = pg_strdup(optarg);
                                break;
 
+                       case 7:                         /* no-sync */
+                               dosync = false;
+                               break;
+
                        default:
                                fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
                                exit_nicely(1);
@@ -632,8 +639,8 @@ main(int argc, char **argv)
                exit_horribly(NULL, "parallel backup only supported by the directory format\n");
 
        /* Open the output file */
-       fout = CreateArchive(filename, archiveFormat, compressLevel, archiveMode,
-                                                setupDumpWorker);
+       fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+                                                archiveMode, setupDumpWorker);
 
        /* Make dump options accessible right away */
        SetArchiveOptions(fout, &dopt, NULL);
@@ -914,6 +921,7 @@ help(const char *progname)
        printf(_("  -V, --version                output version information, then exit\n"));
        printf(_("  -Z, --compress=0-9           compression level for compressed formats\n"));
        printf(_("  --lock-wait-timeout=TIMEOUT  fail after waiting TIMEOUT for a table lock\n"));
+       printf(_("  --no-sync                    do not wait for changes to be written safely to disk\n"));
        printf(_("  -?, --help                   show this help, then exit\n"));
 
        printf(_("\nOptions controlling the output content:\n"));

diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index 81ed924b9f3f1eedcf092379bd60e7a6605ac417..d598d100163e705b968517e8c366bfa81b1f9cc5 100644 (file)
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -22,6 +22,7 @@
 
 #include "dumputils.h"
 #include "pg_backup.h"
+#include "common/file_utils.h"
 #include "fe_utils/string_utils.h"
 
 /* version string we expect back from pg_dump */
@@ -63,6 +64,7 @@ static PQExpBuffer pgdumpopts;
 static char *connstr = "";
 static bool skip_acls = false;
 static bool verbose = false;
+static bool dosync = true;
 
 static int     binary_upgrade = 0;
 static int     column_inserts = 0;
@@ -127,6 +129,7 @@ main(int argc, char *argv[])
                {"role", required_argument, NULL, 3},
                {"use-set-session-authorization", no_argument, &use_setsessauth, 1},
                {"no-security-labels", no_argument, &no_security_labels, 1},
+               {"no-sync", no_argument, NULL, 4},
                {"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1},
                {"no-role-passwords", no_argument, &no_role_passwords, 1},
 
@@ -297,6 +300,11 @@ main(int argc, char *argv[])
                                appendShellString(pgdumpopts, use_role);
                                break;
 
+                       case 4:
+                               dosync = false;
+                               appendPQExpBufferStr(pgdumpopts, " --no-sync");
+                               break;
+
                        default:
                                fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
                                exit_nicely(1);
@@ -549,8 +557,14 @@ main(int argc, char *argv[])
        fprintf(OPF, "--\n-- PostgreSQL database cluster dump complete\n--\n\n");
 
        if (filename)
+       {
                fclose(OPF);
 
+               /* sync the resulting file, errors are not fatal */
+               if (dosync)
+                       (void) fsync_fname(filename, false, progname);
+       }
+
        exit_nicely(0);
 }
 
@@ -586,6 +600,7 @@ help(void)
        printf(_("  --if-exists                  use IF EXISTS when dropping objects\n"));
        printf(_("  --inserts                    dump data as INSERT commands, rather than COPY\n"));
        printf(_("  --no-security-labels         do not dump security label assignments\n"));
+       printf(_("  --no-sync                    do not wait for changes to be written safely to disk\n"));
        printf(_("  --no-tablespaces             do not dump tablespace assignments\n"));
        printf(_("  --no-unlogged-table-data     do not dump unlogged table data\n"));
        printf(_("  --no-role-passwords          do not dump passwords for roles\n"));

diff --git a/src/common/file_utils.c b/src/common/file_utils.c
index a978e64f5a15e66713c895e4b91ec590378706cf..72b0565c7100eb17d961c3a092d014421fc53cf8 100644 (file)
--- a/src/common/file_utils.c
+++ b/src/common/file_utils.c
@@ -115,6 +115,25 @@ fsync_pgdata(const char *pg_data,
        walkdir(pg_tblspc, fsync_fname, true, progname);
 }
 
+/*
+ * Issue fsync recursively on the given directory and all its contents.
+ *
+ * This is a convenient wrapper on top of walkdir().
+ */
+void
+fsync_dir_recurse(const char *dir, const char *progname)
+{
+       /*
+        * If possible, hint to the kernel that we're soon going to fsync the data
+        * directory and its contents.
+        */
+#ifdef PG_FLUSH_DATA_WORKS
+       walkdir(dir, pre_sync_fname, false, progname);
+#endif
+
+       walkdir(dir, fsync_fname, false, progname);
+}
+
 /*
  * walkdir: recursively walk a directory, applying the action to each
  * regular file and directory (including the named directory itself).

diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h
index 07c25c244d025a6150b00d10c096279d0d90e85b..48cc97a4096e29d17678930a481e9344c6bbd857 100644 (file)
--- a/src/include/common/file_utils.h
+++ b/src/include/common/file_utils.h
@@ -19,6 +19,7 @@ extern int fsync_fname(const char *fname, bool isdir,
                                           const char *progname);
 extern void fsync_pgdata(const char *pg_data, const char *progname,
                                                 int serverVersion);
+extern void fsync_dir_recurse(const char *dir, const char *progname);
 extern int durable_rename(const char *oldfile, const char *newfile,
                                                  const char *progname);
 extern int fsync_parent_path(const char *fname, const char *progname);