Switch buffile.c/h to use pgoff_t instead of off_t

off_t was previously used for offsets, which is 4 bytes on Windows,
hence limiting the backend code to a hard limit for files longer than
2GB.  This leads to some simplification in these files, removing some
casts based on long, also 4 bytes on Windows.

This commit removes one comment introduced in db3c4c3a2d98, not relevant
anymore as pgoff_t is a safe 8-byte alternative on Windows.

This change is surprisingly not invasive, as the callers of
BufFileTell(), BufFileSeek() and BufFileTruncateFileSet() (worker.c,
tuplestore.c, etc.) track offsets in local structures that just to
switch from off_t to pgoff_t for the most part.

The file is still relying on a maximum file size of
MAX_PHYSICAL_FILESIZE (1GB).  This change allows the code to make this
maximum potentially larger in the future, or larger on a per-demand
basis.

Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/aUStrqoOCDRFAq1M@paquier.xyz

diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index fc64476a9ef13118aecf1369c92466c800e36257..d1ee0261c646fad65c21d8fb733f1d41cab2ee6f 100644 (file)
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -530,7 +530,7 @@ typedef struct SubXactInfo
 {
    TransactionId xid;          /* XID of the subxact */
    int         fileno;         /* file number in the buffile */
-   off_t       offset;         /* offset in the file */
+   pgoff_t     offset;         /* offset in the file */
 } SubXactInfo;
 
 /* Sub-transaction data for the current streaming transaction */
@@ -2226,12 +2226,12 @@ apply_handle_stream_abort(StringInfo s)
  */
 static void
 ensure_last_message(FileSet *stream_fileset, TransactionId xid, int fileno,
-                   off_t offset)
+                   pgoff_t offset)
 {
    char        path[MAXPGPATH];
    BufFile    *fd;
    int         last_fileno;
-   off_t       last_offset;
+   pgoff_t     last_offset;
 
    Assert(!IsTransactionState());
 
@@ -2266,7 +2266,7 @@ apply_spooled_messages(FileSet *stream_fileset, TransactionId xid,
    MemoryContext oldcxt;
    ResourceOwner oldowner;
    int         fileno;
-   off_t       offset;
+   pgoff_t     offset;
 
    if (!am_parallel_apply_worker())
        maybe_start_skipping_changes(lsn);

diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c
index 4e520065ae0001c883f97a7f2921ea628b3847a6..85b316d879dc0a8a6f7e9542ed22d5e8cec2d2d7 100644 (file)
--- a/src/backend/storage/file/buffile.c
+++ b/src/backend/storage/file/buffile.c
@@ -92,7 +92,7 @@ struct BufFile
     * Position as seen by user of BufFile is (curFile, curOffset + pos).
     */
    int         curFile;        /* file index (0..n) part of current pos */
-   off_t       curOffset;      /* offset part of current pos */
+   pgoff_t     curOffset;      /* offset part of current pos */
    int         pos;            /* next read/write position in buffer */
    int         nbytes;         /* total # of valid bytes in buffer */
 
@@ -503,7 +503,7 @@ BufFileDumpBuffer(BufFile *file)
     */
    while (wpos < file->nbytes)
    {
-       off_t       availbytes;
+       pgoff_t     availbytes;
        instr_time  io_start;
        instr_time  io_time;
 
@@ -524,7 +524,7 @@ BufFileDumpBuffer(BufFile *file)
        bytestowrite = file->nbytes - wpos;
        availbytes = MAX_PHYSICAL_FILESIZE - file->curOffset;
 
-       if ((off_t) bytestowrite > availbytes)
+       if ((pgoff_t) bytestowrite > availbytes)
            bytestowrite = (int) availbytes;
 
        thisfile = file->files[file->curFile];
@@ -729,7 +729,7 @@ BufFileFlush(BufFile *file)
  * BufFileSeek
  *
  * Like fseek(), except that target position needs two values in order to
- * work when logical filesize exceeds maximum value representable by off_t.
+ * work when logical filesize exceeds maximum value representable by pgoff_t.
  * We do not support relative seeks across more than that, however.
  * I/O errors are reported by ereport().
  *
@@ -737,10 +737,10 @@ BufFileFlush(BufFile *file)
  * impossible seek is attempted.
  */
 int
-BufFileSeek(BufFile *file, int fileno, off_t offset, int whence)
+BufFileSeek(BufFile *file, int fileno, pgoff_t offset, int whence)
 {
    int         newFile;
-   off_t       newOffset;
+   pgoff_t     newOffset;
 
    switch (whence)
    {
@@ -754,8 +754,7 @@ BufFileSeek(BufFile *file, int fileno, off_t offset, int whence)
 
            /*
             * Relative seek considers only the signed offset, ignoring
-            * fileno. Note that large offsets (> 1 GB) risk overflow in this
-            * add, unless we have 64-bit off_t.
+            * fileno.
             */
            newFile = file->curFile;
            newOffset = (file->curOffset + file->pos) + offset;
@@ -830,7 +829,7 @@ BufFileSeek(BufFile *file, int fileno, off_t offset, int whence)
 }
 
 void
-BufFileTell(BufFile *file, int *fileno, off_t *offset)
+BufFileTell(BufFile *file, int *fileno, pgoff_t *offset)
 {
    *fileno = file->curFile;
    *offset = file->curOffset + file->pos;
@@ -852,7 +851,7 @@ BufFileSeekBlock(BufFile *file, int64 blknum)
 {
    return BufFileSeek(file,
                       (int) (blknum / BUFFILE_SEG_SIZE),
-                      (off_t) (blknum % BUFFILE_SEG_SIZE) * BLCKSZ,
+                      (pgoff_t) (blknum % BUFFILE_SEG_SIZE) * BLCKSZ,
                       SEEK_SET);
 }
 
@@ -925,11 +924,11 @@ BufFileAppend(BufFile *target, BufFile *source)
  * and the offset.
  */
 void
-BufFileTruncateFileSet(BufFile *file, int fileno, off_t offset)
+BufFileTruncateFileSet(BufFile *file, int fileno, pgoff_t offset)
 {
    int         numFiles = file->numFiles;
    int         newFile = fileno;
-   off_t       newOffset = file->curOffset;
+   pgoff_t     newOffset = file->curOffset;
    char        segment_name[MAXPGPATH];
    int         i;
 

diff --git a/src/backend/utils/sort/tuplestore.c b/src/backend/utils/sort/tuplestore.c
index def945b04541ad6f5f03df23f6ec796f8c927939..9701b8133602bb04b528f173da22b44cd0790015 100644 (file)
--- a/src/backend/utils/sort/tuplestore.c
+++ b/src/backend/utils/sort/tuplestore.c
@@ -94,7 +94,7 @@ typedef struct
    bool        eof_reached;    /* read has reached EOF */
    int         current;        /* next array index to read */
    int         file;           /* temp file# */
-   off_t       offset;         /* byte offset in file */
+   pgoff_t     offset;         /* byte offset in file */
 } TSReadPointer;
 
 /*
@@ -179,7 +179,7 @@ struct Tuplestorestate
    int         readptrsize;    /* allocated length of readptrs array */
 
    int         writepos_file;  /* file# (valid if READFILE state) */
-   off_t       writepos_offset;    /* offset (valid if READFILE state) */
+   pgoff_t     writepos_offset;    /* offset (valid if READFILE state) */
 };
 
 #define COPYTUP(state,tup) ((*(state)->copytup) (state, tup))
@@ -1051,7 +1051,7 @@ tuplestore_gettuple(Tuplestorestate *state, bool forward,
             * Back up to fetch previously-returned tuple's ending length
             * word. If seek fails, assume we are at start of file.
             */
-           if (BufFileSeek(state->myfile, 0, -(long) sizeof(unsigned int),
+           if (BufFileSeek(state->myfile, 0, -(pgoff_t) sizeof(unsigned int),
                            SEEK_CUR) != 0)
            {
                /* even a failed backwards fetch gets you out of eof state */
@@ -1072,7 +1072,7 @@ tuplestore_gettuple(Tuplestorestate *state, bool forward,
                 * Back up to get ending length word of tuple before it.
                 */
                if (BufFileSeek(state->myfile, 0,
-                               -(long) (tuplen + 2 * sizeof(unsigned int)),
+                               -(pgoff_t) (tuplen + 2 * sizeof(unsigned int)),
                                SEEK_CUR) != 0)
                {
                    /*
@@ -1082,7 +1082,7 @@ tuplestore_gettuple(Tuplestorestate *state, bool forward,
                     * what in-memory case does).
                     */
                    if (BufFileSeek(state->myfile, 0,
-                                   -(long) (tuplen + sizeof(unsigned int)),
+                                   -(pgoff_t) (tuplen + sizeof(unsigned int)),
                                    SEEK_CUR) != 0)
                        ereport(ERROR,
                                (errcode_for_file_access(),
@@ -1099,7 +1099,7 @@ tuplestore_gettuple(Tuplestorestate *state, bool forward,
             * length word of the tuple, so back up to that point.
             */
            if (BufFileSeek(state->myfile, 0,
-                           -(long) tuplen,
+                           -(pgoff_t) tuplen,
                            SEEK_CUR) != 0)
                ereport(ERROR,
                        (errcode_for_file_access(),

diff --git a/src/include/storage/buffile.h b/src/include/storage/buffile.h
index a2f4821f240b9cf6d3db3ce50aecf2d36c21a067..4908dc56c7be27ec0621c8538f01df228260be2f 100644 (file)
--- a/src/include/storage/buffile.h
+++ b/src/include/storage/buffile.h
@@ -42,8 +42,8 @@ pg_nodiscard extern size_t BufFileRead(BufFile *file, void *ptr, size_t size);
 extern void BufFileReadExact(BufFile *file, void *ptr, size_t size);
 extern size_t BufFileReadMaybeEOF(BufFile *file, void *ptr, size_t size, bool eofOK);
 extern void BufFileWrite(BufFile *file, const void *ptr, size_t size);
-extern int BufFileSeek(BufFile *file, int fileno, off_t offset, int whence);
-extern void BufFileTell(BufFile *file, int *fileno, off_t *offset);
+extern int BufFileSeek(BufFile *file, int fileno, pgoff_t offset, int whence);
+extern void BufFileTell(BufFile *file, int *fileno, pgoff_t *offset);
 extern int BufFileSeekBlock(BufFile *file, int64 blknum);
 extern int64 BufFileSize(BufFile *file);
 extern int64 BufFileAppend(BufFile *target, BufFile *source);
@@ -54,6 +54,6 @@ extern BufFile *BufFileOpenFileSet(FileSet *fileset, const char *name,
                                   int mode, bool missing_ok);
 extern void BufFileDeleteFileSet(FileSet *fileset, const char *name,
                                 bool missing_ok);
-extern void BufFileTruncateFileSet(BufFile *file, int fileno, off_t offset);
+extern void BufFileTruncateFileSet(BufFile *file, int fileno, pgoff_t offset);
 
 #endif                         /* BUFFILE_H */