#include "access/xlog_internal.h" /* for pg_start/stop_backup */
#include "catalog/pg_type.h"
#include "common/file_perm.h"
+#include "common/sha2.h"
#include "lib/stringinfo.h"
#include "libpq/libpq.h"
#include "libpq/pqformat.h"
static int64 sendDir(const char *path, int basepathlen, bool sizeonly,
- List *tablespaces, bool sendtblspclinks);
+ List *tablespaces, bool sendtblspclinks,
+ StringInfo manifest, const char *tsoid);
static bool sendFile(const char *readfilename, const char *tarfilename,
- struct stat *statbuf, bool missing_ok, Oid dboid);
-static void sendFileWithContent(const char *filename, const char *content);
+ struct stat *statbuf, bool missing_ok, Oid dboid,
+ StringInfo manifest, const char *tsoid);
+static void sendFileWithContent(const char *filename, const char *content,
+ StringInfo manifest);
static int64 _tarWriteHeader(const char *filename, const char *linktarget,
struct stat *statbuf, bool sizeonly);
static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf,
bool sizeonly);
static void send_int8_string(StringInfoData *buf, int64 intval);
static void SendBackupHeader(List *tablespaces);
+static void InitializeManifest(StringInfo manifest);
+static void AddFileToManifest(StringInfo manifest, const char *tsoid,
+ const char *filename, size_t size, time_t mtime,
+ uint8 *shabuf);
+static void SendBackupManifest(StringInfo manifest);
+static char *escape_field_for_manifest(const char *s);
static void base_backup_cleanup(int code, Datum arg);
static void perform_base_backup(basebackup_options *opt);
static void parse_basebackup_options(List *options, basebackup_options *opt);
TimeLineID endtli;
StringInfo labelfile;
StringInfo tblspc_map_file = NULL;
+ StringInfo manifest;
int datadirpathlen;
List *tablespaces = NIL;
labelfile = makeStringInfo();
tblspc_map_file = makeStringInfo();
+ manifest = makeStringInfo();
+ InitializeManifest(manifest);
total_checksum_failures = 0;
/* Add a node for the base directory at the end */
ti = palloc0(sizeof(tablespaceinfo));
- ti->size = opt->progress ? sendDir(".", 1, true, tablespaces, true) : -1;
+ if (opt->progress)
+ ti->size = sendDir(".", 1, true, tablespaces, true, NULL, NULL);
+ else
+ ti->size = -1;
tablespaces = lappend(tablespaces, ti);
/* Send tablespace header */
struct stat statbuf;
/* In the main tar, include the backup_label first... */
- sendFileWithContent(BACKUP_LABEL_FILE, labelfile->data);
+ sendFileWithContent(BACKUP_LABEL_FILE, labelfile->data,
+ manifest);
/*
* Send tablespace_map file if required and then the bulk of
*/
if (tblspc_map_file && opt->sendtblspcmapfile)
{
- sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data);
- sendDir(".", 1, false, tablespaces, false);
+ sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data,
+ manifest);
+ sendDir(".", 1, false, tablespaces, false, manifest, NULL);
}
else
- sendDir(".", 1, false, tablespaces, true);
+ sendDir(".", 1, false, tablespaces, true, manifest, NULL);
/* ... and pg_control after everything else. */
if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
(errcode_for_file_access(),
errmsg("could not stat file \"%s\": %m",
XLOG_CONTROL_FILE)));
- sendFile(XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf, false, InvalidOid);
+ sendFile(XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf,
+ false, InvalidOid, manifest, NULL);
}
else
- sendTablespace(ti->path, false);
+ sendTablespace(ti->path, ti->oid, false, manifest);
/*
* If we're including WAL, and this is the main data directory we
* complete segment.
*/
StatusFilePath(pathbuf, walFileName, ".done");
- sendFileWithContent(pathbuf, "");
+ sendFileWithContent(pathbuf, "", manifest);
}
/*
(errcode_for_file_access(),
errmsg("could not stat file \"%s\": %m", pathbuf)));
- sendFile(pathbuf, pathbuf, &statbuf, false, InvalidOid);
+ sendFile(pathbuf, pathbuf, &statbuf, false, InvalidOid, manifest,
+ NULL);
/* unconditionally mark file as archived */
StatusFilePath(pathbuf, fname, ".done");
- sendFileWithContent(pathbuf, "");
+ sendFileWithContent(pathbuf, "", manifest);
}
/* Send CopyDone message for the last tar file */
pq_putemptymessage('c');
}
+
+ SendBackupManifest(manifest);
+
SendXlogRecPtrResult(endptr, endtli);
if (total_checksum_failures)
pq_puttextmessage('C', "SELECT");
}
+static void
+InitializeManifest(StringInfo manifest)
+{
+ appendStringInfoString(manifest, "PostgreSQL-Backup-Manifest-Version 1\n");
+}
+
+/*
+ * Add an entry to the backup manifest for a file.
+ */
+static void
+AddFileToManifest(StringInfo manifest, const char *tsoid,
+ const char *filename, size_t size, time_t mtime,
+ uint8 *shabuf)
+{
+ char pathbuf[MAXPGPATH];
+ char *escaped_filename;
+ static char timebuf[128];
+ static char shatextbuf[PG_SHA256_DIGEST_LENGTH * 2 + 1];
+ int shatextlen;
+
+ /*
+ * If this file is part of a tablespace, the filename passed to this
+ * function will be relative to the tar file that contains it. We want
+ * the pathname relative to the data directory (ignoring the intermediate
+ * symlink traversal).
+ */
+ if (tsoid != NULL)
+ {
+ snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%s/%s", tsoid, filename);
+ filename = pathbuf;
+ }
+
+ /* Escape filename, if necessary. */
+ escaped_filename = escape_field_for_manifest(filename);
+
+ /*
+ * Convert time to a string. Since it's not clear what time zone to use
+ * and since time zone definitions can change, possibly causing confusion,
+ * use GMT always.
+ */
+ pg_strftime(timebuf, sizeof(timebuf), "%Y-%m-%d %H:%M:%S %Z",
+ pg_gmtime(&mtime));
+
+ /* Convert checksum to hexadecimal. */
+ shatextlen =
+ hex_encode((char *) shabuf, PG_SHA256_DIGEST_LENGTH, shatextbuf);
+ Assert(shatextlen + 1 == sizeof(shatextbuf));
+ shatextbuf[shatextlen] = '\0';
+
+ /* Add to manifest. */
+ appendStringInfo(manifest, "File\t%s\t%zu\t%s\t%s\n",
+ escaped_filename == NULL ? filename : escaped_filename,
+ size, timebuf, shatextbuf);
+
+ /* Avoid leaking memory. */
+ if (escaped_filename != NULL)
+ pfree(escaped_filename);
+}
+
+/*
+ * Finalize the backup manifest, and send it to the client.
+ */
+static void
+SendBackupManifest(StringInfo manifest)
+{
+ pg_sha256_ctx sha256_ctx;
+ uint8 shabuf[PG_SHA256_DIGEST_LENGTH];
+ StringInfoData protobuf;
+ int shastringlen;
+
+ /* Checksum the manifest. */
+ pg_sha256_init(&sha256_ctx);
+ pg_sha256_update(&sha256_ctx, (uint8 *) manifest->data, manifest->len);
+ pg_sha256_final(&sha256_ctx, shabuf);
+ appendStringInfoString(manifest, "Manifest-Checksum\t");
+ shastringlen = PG_SHA256_DIGEST_LENGTH * 2;
+ enlargeStringInfo(manifest, shastringlen);
+ shastringlen = hex_encode((char *) shabuf, PG_SHA256_DIGEST_LENGTH,
+ manifest->data + manifest->len);
+ Assert(shastringlen == PG_SHA256_DIGEST_LENGTH * 2);
+ manifest->len += shastringlen;
+ appendStringInfoChar(manifest, '\n');
+
+ /* Send CopyOutResponse message */
+ pq_beginmessage(&protobuf, 'H');
+ pq_sendbyte(&protobuf, 0); /* overall format */
+ pq_sendint16(&protobuf, 0); /* natts */
+ pq_endmessage(&protobuf);
+
+ /* Send CopyData message */
+ pq_putmessage('d', manifest->data, manifest->len);
+
+ /* And finally CopyDone message */
+ pq_putemptymessage('c');
+}
+
+/*
+ * Escape a field for inclusion in a manifest.
+ *
+ * We use the following escaping rule: If a field contains \t, \r, or \n,
+ * the field must be surrounded by double-quotes, and any internal double
+ * quotes must be doubled. Otherwise, no escaping is required.
+ *
+ * The return value is a new palloc'd string with escaping added, or NULL
+ * if no escaping is required.
+ */
+static char *
+escape_field_for_manifest(const char *s)
+{
+ bool escaping_required = false;
+ int escaped_length = 2;
+ const char *t;
+ char *result;
+ char *r;
+
+ for (t = s; *t != '\0'; ++t)
+ {
+ if (*t == '\t' || *t == '\r' || *t == '\n')
+ escaping_required = true;
+ if (*t == '"')
+ ++escaped_length;
+ ++escaped_length;
+ }
+
+ if (!escaping_required)
+ return NULL;
+
+ result = palloc(escaped_length + 1);
+ result[0] = '"';
+ result[escaped_length - 1] = '"';
+ result[escaped_length] = '\0';
+ r = result + 1;
+
+ for (t = s; *t != '\0'; ++t)
+ {
+ *(r++) = *t;
+ if (*t == '"')
+ *(r++) = *t;
+ }
+
+ Assert(r == &result[escaped_length - 1]);
+
+ return result;
+}
+
/*
* Send a single resultset containing just a single
* XLogRecPtr record (in text format)
* Inject a file with given name and content in the output tar stream.
*/
static void
-sendFileWithContent(const char *filename, const char *content)
+sendFileWithContent(const char *filename, const char *content,
+ StringInfo manifest)
{
struct stat statbuf;
int pad,
len;
+ pg_sha256_ctx sha256_ctx;
+ uint8 shabuf[PG_SHA256_DIGEST_LENGTH];
+
+ pg_sha256_init(&sha256_ctx);
len = strlen(content);
MemSet(buf, 0, pad);
pq_putmessage('d', buf, pad);
}
+
+ pg_sha256_update(&sha256_ctx, (uint8 *) content, len);
+ pg_sha256_final(&sha256_ctx, shabuf);
+ AddFileToManifest(manifest, NULL, filename, len, statbuf.st_mtime,
+ shabuf);
}
/*
* Only used to send auxiliary tablespaces, not PGDATA.
*/
int64
-sendTablespace(char *path, bool sizeonly)
+sendTablespace(char *path, char *oid, bool sizeonly, StringInfo manifest)
{
int64 size;
char pathbuf[MAXPGPATH];
sizeonly);
/* Send all the files in the tablespace version directory */
- size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true);
+ size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true, manifest, oid);
return size;
}
*/
static int64
sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces,
- bool sendtblspclinks)
+ bool sendtblspclinks, StringInfo manifest, const char *tsoid)
{
DIR *dir;
struct dirent *de;
skip_this_dir = true;
if (!skip_this_dir)
- size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces, sendtblspclinks);
+ size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces,
+ sendtblspclinks, manifest, tsoid);
}
else if (S_ISREG(statbuf.st_mode))
{
if (!sizeonly)
sent = sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf,
- true, isDbDir ? pg_atoi(lastDir + 1, sizeof(Oid), 0) : InvalidOid);
+ true, isDbDir ? pg_atoi(lastDir + 1, sizeof(Oid), 0) : InvalidOid,
+ manifest, tsoid);
if (sent || sizeonly)
{
* and the file did not exist.
*/
static bool
-sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf,
- bool missing_ok, Oid dboid)
+sendFile(const char *readfilename, const char *tarfilename,
+ struct stat *statbuf, bool missing_ok, Oid dboid,
+ StringInfo manifest, const char *tsoid)
{
FILE *fp;
BlockNumber blkno = 0;
int segmentno = 0;
char *segmentpath;
bool verify_checksum = false;
+ pg_sha256_ctx sha256_ctx;
+ uint8 shabuf[PG_SHA256_DIGEST_LENGTH];
+
+ pg_sha256_init(&sha256_ctx);
fp = AllocateFile(readfilename, "rb");
if (fp == NULL)
ereport(ERROR,
(errmsg("base backup could not send data, aborting backup")));
+ /* Also feed it to the checksum machinery. */
+ pg_sha256_update(&sha256_ctx, (uint8 *) buf, cnt);
+
len += cnt;
throttle(cnt);
{
cnt = Min(sizeof(buf), statbuf->st_size - len);
pq_putmessage('d', buf, cnt);
+ pg_sha256_update(&sha256_ctx, (uint8 *) buf, cnt);
len += cnt;
throttle(cnt);
}
/*
* Pad to 512 byte boundary, per tar format requirements. (This small
- * piece of data is probably not worth throttling.)
+ * piece of data is probably not worth throttling, and is not checksummed
+ * because it's not actually part of the file.)
*/
pad = ((len + 511) & ~511) - len;
if (pad > 0)
total_checksum_failures += checksum_failures;
+ pg_sha256_final(&sha256_ctx, shabuf);
+ AddFileToManifest(manifest, tsoid, tarfilename, statbuf->st_size,
+ statbuf->st_mtime, shabuf);
+
return true;
}
FILE *file;
} UnpackTarState;
+typedef struct WriteManifestState
+{
+ char filename[MAXPGPATH];
+ FILE *file;
+} WriteManifestState;
+
typedef void (*WriteDataCallback) (size_t nbytes, char *buf,
void *callback_data);
static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum);
static void ReceiveTarAndUnpackCopyChunk(size_t r, char *copybuf,
void *callback_data);
+static void ReceiveBackupManifest(PGconn *conn);
+static void ReceiveBackupManifestChunk(size_t r, char *copybuf,
+ void *callback_data);
+static void ReceiveBackupManifestInMemory(PGconn *conn, PQExpBuffer buf);
+static void ReceiveBackupManifestInMemoryChunk(size_t r, char *copybuf,
+ void *callback_data);
static void BaseBackup(void);
static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline,
res = PQgetResult(conn);
if (PQresultStatus(res) != PGRES_COPY_OUT)
{
- pg_log_error("could not get COPY data stream: %s",
- PQerrorMessage(conn));
+ pg_log_error("could not get COPY data stream: %s [%s]",
+ PQerrorMessage(conn), PQresStatus(PQresultStatus(res)));
exit(1);
}
PQclear(res);
}
}
+ /*
+ * Normally, we emit the backup manifest as a separate file, but when
+ * we're writing a tarfile to stdout, we don't have that option, so
+ * include it in the one tarfile we've got.
+ */
+ if (strcmp(basedir, "-") == 0)
+ {
+ char header[512];
+ PQExpBufferData buf;
+
+ initPQExpBuffer(&buf);
+ ReceiveBackupManifestInMemory(conn, &buf);
+ if (PQExpBufferBroken(&buf))
+ {
+ pg_log_error("out of memory");
+ exit(1);
+ }
+ tarCreateHeader(header, "backup_manifest", NULL, buf.len,
+ pg_file_create_mode, 04000, 02000,
+ time(NULL));
+ writeTarData(&state, header, sizeof(header));
+ writeTarData(&state, buf.data, buf.len);
+ termPQExpBuffer(&buf);
+ }
+
/* 2 * 512 bytes empty data at end of file */
writeTarData(&state, zerobuf, sizeof(zerobuf));
} /* continuing data in existing file */
}
+/*
+ * Receive the backup manifest file and write it out to a file.
+ */
+static void
+ReceiveBackupManifest(PGconn *conn)
+{
+ WriteManifestState state;
+
+ snprintf(state.filename, sizeof(state.filename),
+ "%s/backup_manifest", basedir);
+ state.file = fopen(state.filename, "wb");
+ if (state.file == NULL)
+ {
+ pg_log_error("could not create file \"%s\": %m", state.filename);
+ exit(1);
+ }
+
+ ReceiveCopyData(conn, ReceiveBackupManifestChunk, &state);
+
+ fclose(state.file);
+}
+
+/*
+ * Receive one chunk of the backup manifest file and write it out to a file.
+ */
+static void
+ReceiveBackupManifestChunk(size_t r, char *copybuf, void *callback_data)
+{
+ WriteManifestState *state = callback_data;
+
+ if (fwrite(copybuf, r, 1, state->file) != 1)
+ {
+ pg_log_error("could not write to file \"%s\": %m", state->filename);
+ exit(1);
+ }
+}
+
+/*
+ * Receive the backup manifest file and write it out to a file.
+ */
+static void
+ReceiveBackupManifestInMemory(PGconn *conn, PQExpBuffer buf)
+{
+ ReceiveCopyData(conn, ReceiveBackupManifestInMemoryChunk, buf);
+}
+
+/*
+ * Receive one chunk of the backup manifest file and write it out to a file.
+ */
+static void
+ReceiveBackupManifestInMemoryChunk(size_t r, char *copybuf,
+ void *callback_data)
+{
+ PQExpBuffer buf = callback_data;
+
+ appendPQExpBuffer(buf, copybuf, r);
+}
static void
BaseBackup(void)
maxServerMajor;
int serverVersion,
serverMajor;
+ int writing_to_stdout;
Assert(conn != NULL);
/*
* When writing to stdout, require a single tablespace
*/
- if (format == 't' && strcmp(basedir, "-") == 0 && PQntuples(res) > 1)
+ writing_to_stdout = format == 't' && strcmp(basedir, "-") == 0;
+ if (writing_to_stdout && PQntuples(res) > 1)
{
pg_log_error("can only write single tablespace to stdout, database has %d",
PQntuples(res));
ReceiveAndUnpackTarFile(conn, res, i);
} /* Loop over all tablespaces */
+ /*
+ * Now receive backup manifest, if appropriate.
+ *
+ * If we're writing a tarfile to stdout, ReceiveTarFile will have already
+ * processed the backup manifest and included it in the output tarfile.
+ * Such a configuration doesn't allow for writing multiple files.
+ *
+ * If we're talking to an older server, it won't send a backup manifest,
+ * so don't try to receive one.
+ */
+ if (!writing_to_stdout && serverMajor >= 1300)
+ ReceiveBackupManifest(conn);
+
if (showprogress)
{
progress_report(PQntuples(res), NULL, true);
projects / users / rhaas / postgres.git / commitdiff