Install defenses against overflow in BuildTupleHashTable().

The planner can sometimes compute very large values for numGroups, and in
cases where we have no alternative to building a hashtable, such a value
will get fed directly to BuildTupleHashTable as its nbuckets parameter.
There were two ways in which that could go bad.  First, BuildTupleHashTable
declared the parameter as "int" but most callers were passing "long"s,
so on 64-bit machines undetected overflow could occur leading to a bogus
negative value.  The obvious fix for that is to change the parameter to
"long", which is what I've done in HEAD.  In the back branches that seems a
bit risky, though, since third-party code might be calling this function.
So for them, just put in a kluge to treat negative inputs as INT_MAX.
Second, hash_create can go nuts with extremely large requested table sizes
(notably, my_log2 becomes an infinite loop for inputs larger than
LONG_MAX/2).  What seems most appropriate to avoid that is to bound the
initial table size request to work_mem.

This fixes bug #6035 reported by Daniel Schreiber.  Although the reported
case only occurs back to 8.4 since it involves WITH RECURSIVE, I think
it's a good idea to install the defenses in all supported branches.

diff --git a/src/backend/executor/execGrouping.c b/src/backend/executor/execGrouping.c
index bfb8cbb303c6cb3635e7f925c0eefb4f9a8d570d..17098efa670e3964d044cc7da39cbaffa3a7e3c0 100644 (file)
--- a/src/backend/executor/execGrouping.c
+++ b/src/backend/executor/execGrouping.c
@@ -19,6 +19,7 @@
 #include "postgres.h"
 
 #include "executor/executor.h"
+#include "miscadmin.h"
 #include "parser/parse_oper.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
@@ -276,7 +277,7 @@ TupleHashTable
 BuildTupleHashTable(int numCols, AttrNumber *keyColIdx,
                    FmgrInfo *eqfunctions,
                    FmgrInfo *hashfunctions,
-                   int nbuckets, Size entrysize,
+                   long nbuckets, Size entrysize,
                    MemoryContext tablecxt, MemoryContext tempcxt)
 {
    TupleHashTable hashtable;
@@ -285,6 +286,9 @@ BuildTupleHashTable(int numCols, AttrNumber *keyColIdx,
    Assert(nbuckets > 0);
    Assert(entrysize >= sizeof(TupleHashEntryData));
 
+   /* Limit initial table size request to not more than work_mem */
+   nbuckets = Min(nbuckets, (long) ((work_mem * 1024L) / entrysize));
+
    hashtable = (TupleHashTable) MemoryContextAlloc(tablecxt,
                                                 sizeof(TupleHashTableData));
 
@@ -306,7 +310,7 @@ BuildTupleHashTable(int numCols, AttrNumber *keyColIdx,
    hash_ctl.hash = TupleHashTableHash;
    hash_ctl.match = TupleHashTableMatch;
    hash_ctl.hcxt = tablecxt;
-   hashtable->hashtab = hash_create("TupleHashTable", (long) nbuckets,
+   hashtable->hashtab = hash_create("TupleHashTable", nbuckets,
                                     &hash_ctl,
                    HASH_ELEM | HASH_FUNCTION | HASH_COMPARE | HASH_CONTEXT);
 

diff --git a/src/backend/executor/nodeSubplan.c b/src/backend/executor/nodeSubplan.c
index 4fcb66bbb03b86d6f67a4c1bc8a21d8b90231796..0e12bb5afbdb2f39363348f4303da9af2fe04f75 100644 (file)
--- a/src/backend/executor/nodeSubplan.c
+++ b/src/backend/executor/nodeSubplan.c
@@ -432,7 +432,7 @@ buildSubPlanHash(SubPlanState *node, ExprContext *econtext)
    int         ncols = list_length(subplan->paramIds);
    ExprContext *innerecontext = node->innerecontext;
    MemoryContext oldcontext;
-   int         nbuckets;
+   long        nbuckets;
    TupleTableSlot *slot;
 
    Assert(subplan->subLinkType == ANY_SUBLINK);
@@ -458,7 +458,7 @@ buildSubPlanHash(SubPlanState *node, ExprContext *econtext)
    node->havehashrows = false;
    node->havenullrows = false;
 
-   nbuckets = (int) ceil(planstate->plan->plan_rows);
+   nbuckets = (long) Min(planstate->plan->plan_rows, (double) LONG_MAX);
    if (nbuckets < 1)
        nbuckets = 1;
 

diff --git a/src/include/executor/executor.h b/src/include/executor/executor.h
index 1d651dd40819275a4410ad4448116bf4887585dc..bdd499bea662d405fc6522c036c53955c0a2e3f9 100644 (file)
--- a/src/include/executor/executor.h
+++ b/src/include/executor/executor.h
@@ -131,7 +131,7 @@ extern void execTuplesHashPrepare(int numCols,
 extern TupleHashTable BuildTupleHashTable(int numCols, AttrNumber *keyColIdx,
                    FmgrInfo *eqfunctions,
                    FmgrInfo *hashfunctions,
-                   int nbuckets, Size entrysize,
+                   long nbuckets, Size entrysize,
                    MemoryContext tablecxt,
                    MemoryContext tempcxt);
 extern TupleHashEntry LookupTupleHashEntry(TupleHashTable hashtable,