To support external compression of archived WAL data, add a flag bit to

WAL records that shows whether it is safe to remove full-page images
(ie, whether or not an on-line backup was in progress when the WAL entry
was made).  Also make provision for an XLOG_NOOP record type that can be
used to fill in the extra space when decompressing the data for restore.

This is the portion of Koichi Suzuki's "full page writes" patch that
has to go into the core database.  The remainder of that work is two
external compression and decompression programs, which for the time being
will undergo separate development on pgfoundry.  Per discussion.

Also, twiddle the handling of BTREE_SPLIT records to ensure it'll be
possible to compress them (the previous coding caused essential info
to be omitted).  The other commonly-used record types seem OK already,
with the possible exception of GIN and GIST WAL records, which I don't
understand well enough to opine on.

diff --git a/src/backend/access/nbtree/nbtinsert.c b/src/backend/access/nbtree/nbtinsert.c
index 41cc01888c95312ab40ea33c56d9e0afc55a6bed..2a67494b745799b5af96bf3f76b2ddaa8f02d47f 100644 (file)
--- a/src/backend/access/nbtree/nbtinsert.c
+++ b/src/backend/access/nbtree/nbtinsert.c
@@ -1034,21 +1034,23 @@ _bt_split(Relation rel, Buffer buf, OffsetNumber firstright,
                 * Log the new item and its offset, if it was inserted on the left
                 * page. (If it was put on the right page, we don't need to explicitly
                 * WAL log it because it's included with all the other items on the
-                * right page.) Show these as belonging to the left page buffer,
-                * so that they are not stored if XLogInsert decides it needs a
-                * full-page image of the left page.
+                * right page.) Show the new item as belonging to the left page buffer,
+                * so that it is not stored if XLogInsert decides it needs a full-page
+                * image of the left page.  We store the offset anyway, though, to
+                * support archive compression of these records.
                 */
                if (newitemonleft)
                {
                        lastrdata->next = lastrdata + 1;
                        lastrdata++;
+
                        lastrdata->data = (char *) &newitemoff;
                        lastrdata->len = sizeof(OffsetNumber);
-                       lastrdata->buffer = buf;                /* backup block 1 */
-                       lastrdata->buffer_std = true;
+                       lastrdata->buffer = InvalidBuffer;
 
                        lastrdata->next = lastrdata + 1;
                        lastrdata++;
+
                        lastrdata->data = (char *) newitem;
                        lastrdata->len = MAXALIGN(newitemsz);
                        lastrdata->buffer = buf;                /* backup block 1 */
@@ -1064,6 +1066,7 @@ _bt_split(Relation rel, Buffer buf, OffsetNumber firstright,
                         */
                        lastrdata->next = lastrdata + 1;
                        lastrdata++;
+
                        lastrdata->data = NULL;
                        lastrdata->len = 0;
                        lastrdata->buffer = buf;                /* backup block 1 */

diff --git a/src/backend/access/nbtree/nbtxlog.c b/src/backend/access/nbtree/nbtxlog.c
index 103b275c5f818cdffaa6400d19f413dbe809d9da..0c09ee8d44d2c901f6e542f6a2356343c1d6d101 100644 (file)
--- a/src/backend/access/nbtree/nbtxlog.c
+++ b/src/backend/access/nbtree/nbtxlog.c
@@ -292,14 +292,17 @@ btree_xlog_split(bool onleft, bool isroot,
        }
 
        /* Extract newitem and newitemoff, if present */
-       if (onleft && !(record->xl_info & XLR_BKP_BLOCK_1))
+       if (onleft)
        {
-               IndexTupleData itupdata;
-
                /* Extract the offset (still assuming 16-bit alignment) */
                memcpy(&newitemoff, datapos, sizeof(OffsetNumber));
                datapos += sizeof(OffsetNumber);
                datalen -= sizeof(OffsetNumber);
+       }
+
+       if (onleft && !(record->xl_info & XLR_BKP_BLOCK_1))
+       {
+               IndexTupleData itupdata;
 
                /*
                 * We need to copy the tuple header to apply IndexTupleDSize, because

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 9a82abfef6b32dcd784e2df032ded9bc43798dc1..5b13822e6e4937e63f8ee3814743faff7ed07aae 100644 (file)
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -783,6 +783,19 @@ begin:;
                }
        }
 
+       /*
+        * If we backed up any full blocks and online backup is not in progress,
+        * mark the backup blocks as removable.  This allows the WAL archiver to
+        * know whether it is safe to compress archived WAL data by transforming
+        * full-block records into the non-full-block format.
+        *
+        * Note: we could just set the flag whenever !forcePageWrites, but
+        * defining it like this leaves the info bit free for some potential
+        * other use in records without any backup blocks.
+        */
+       if ((info & XLR_BKP_BLOCK_MASK) && !Insert->forcePageWrites)
+               info |= XLR_BKP_REMOVABLE;
+
        /*
         * If there isn't enough space on the current XLOG page for a record
         * header, advance to the next page (leaving the unused space as zeroes).
@@ -5868,6 +5881,10 @@ xlog_redo(XLogRecPtr lsn, XLogRecord *record)
 
                RecoveryRestartPoint(&checkPoint);
        }
+       else if (info == XLOG_NOOP)
+       {
+               /* nothing to do here */
+       }
        else if (info == XLOG_SWITCH)
        {
                /* nothing to do here */
@@ -5894,6 +5911,10 @@ xlog_desc(StringInfo buf, uint8 xl_info, char *rec)
                                                 checkpoint->nextMultiOffset,
                                 (info == XLOG_CHECKPOINT_SHUTDOWN) ? "shutdown" : "online");
        }
+       else if (info == XLOG_NOOP)
+       {
+               appendStringInfo(buf, "xlog no-op");
+       }
        else if (info == XLOG_NEXTOID)
        {
                Oid                     nextOid;

diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index c3d086902416a82df71541eba470110bd33082e3..dd40c8ebfff98c21048fd509a4879ebdd6573e65 100644 (file)
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -66,8 +66,7 @@ typedef struct XLogRecord
 /*
  * If we backed up any disk blocks with the XLOG record, we use flag bits in
  * xl_info to signal it.  We support backup of up to 3 disk blocks per XLOG
- * record.     (Could support 4 if we cared to dedicate all the xl_info bits for
- * this purpose; currently bit 0 of xl_info is unused and available.)
+ * record.
  */
 #define XLR_BKP_BLOCK_MASK             0x0E    /* all info bits used for bkp blocks */
 #define XLR_MAX_BKP_BLOCKS             3
@@ -76,6 +75,15 @@ typedef struct XLogRecord
 #define XLR_BKP_BLOCK_2                        XLR_SET_BKP_BLOCK(1)    /* 0x04 */
 #define XLR_BKP_BLOCK_3                        XLR_SET_BKP_BLOCK(2)    /* 0x02 */
 
+/*
+ * Bit 0 of xl_info is set if the backed-up blocks could safely be removed
+ * from a compressed version of XLOG (that is, they are backed up only to
+ * prevent partial-page-write problems, and not to ensure consistency of PITR
+ * recovery).  The compression algorithm would need to extract data from the
+ * blocks to create an equivalent non-full-page XLOG record.
+ */
+#define XLR_BKP_REMOVABLE              0x01
+
 /*
  * Sometimes we log records which are out of transaction control.
  * Rmgr may "or" XLOG_NO_TRAN into info passed to XLogInsert to indicate this.

diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 6a7618937734178b91c0519bab64a3eede36bd56..8f50bbb92a56c42af114716611e4e5932500995e 100644 (file)
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -71,7 +71,7 @@ typedef struct XLogContRecord
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD061 /* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD062 /* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {

diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index fb834ee1823040a438b2b39f5f0da37a4c5aa88c..d1a783371f636f2d235f20cb42c568ee2caf0980 100644 (file)
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -44,6 +44,7 @@ typedef struct CheckPoint
 /* XLOG info values for XLOG rmgr */
 #define XLOG_CHECKPOINT_SHUTDOWN               0x00
 #define XLOG_CHECKPOINT_ONLINE                 0x10
+#define XLOG_NOOP                                              0x20
 #define XLOG_NEXTOID                                   0x30
 #define XLOG_SWITCH                                            0x40