From: Shigeru Hanada Date: Tue, 5 Oct 2010 08:03:28 +0000 (+0900) Subject: Add EstimateCost() API to FdwRoutine, which is used to estimate X-Git-Url: http://waps.l3s.uni-hannover.de/gitweb/?a=commitdiff_plain;h=0c27c0536f15e46d939aecbeb0e5ca568b7d6730;p=users%2Fhanada%2Fpostgres.git Add EstimateCost() API to FdwRoutine, which is used to estimate the costs of a ForeignScan. Also implement pgEstimateCost() for postgresql_fdw, which estimate the costs as sum of: * costs of sequencial scan * connection cost (if connection_cost was set in the server generic option) * transfer cost (if transfer_cost, cost-per-width, was set in the server generic option) --- diff --git a/contrib/dblink/dblink.c b/contrib/dblink/dblink.c index ce86829210..47b99fe70a 100644 --- a/contrib/dblink/dblink.c +++ b/contrib/dblink/dblink.c @@ -2329,6 +2329,9 @@ get_connect_string(const char *servername) { DefElem *def = lfirst(cell); + if (!is_libpq_connection_option(def->defname)) + continue; + appendStringInfo(buf, "%s='%s' ", def->defname, escape_param_str(strVal(def->arg))); } @@ -2337,6 +2340,9 @@ get_connect_string(const char *servername) { DefElem *def = lfirst(cell); + if (!is_libpq_connection_option(def->defname)) + continue; + appendStringInfo(buf, "%s='%s' ", def->defname, escape_param_str(strVal(def->arg))); } @@ -2346,6 +2352,9 @@ get_connect_string(const char *servername) DefElem *def = lfirst(cell); + if (!is_libpq_connection_option(def->defname)) + continue; + appendStringInfo(buf, "%s='%s' ", def->defname, escape_param_str(strVal(def->arg))); } diff --git a/contrib/dblink/expected/dblink.out b/contrib/dblink/expected/dblink.out index c59a67c737..ca37d5c880 100644 --- a/contrib/dblink/expected/dblink.out +++ b/contrib/dblink/expected/dblink.out @@ -796,7 +796,7 @@ SELECT dblink_disconnect('dtest1'); -- test foreign data wrapper functionality CREATE USER dblink_regression_test; CREATE FOREIGN DATA WRAPPER postgresql; -CREATE SERVER fdtest FOREIGN DATA WRAPPER postgresql OPTIONS (dbname 'contrib_regression'); +CREATE SERVER fdtest FOREIGN DATA WRAPPER postgresql OPTIONS (dbname 'contrib_regression', connect_cost '100.0'); CREATE USER MAPPING FOR public SERVER fdtest; GRANT USAGE ON FOREIGN SERVER fdtest TO dblink_regression_test; GRANT EXECUTE ON FUNCTION dblink_connect_u(text, text) TO dblink_regression_test; diff --git a/contrib/dblink/sql/dblink.sql b/contrib/dblink/sql/dblink.sql index a6d7811bfc..53ad837218 100644 --- a/contrib/dblink/sql/dblink.sql +++ b/contrib/dblink/sql/dblink.sql @@ -375,7 +375,7 @@ SELECT dblink_disconnect('dtest1'); CREATE USER dblink_regression_test; CREATE FOREIGN DATA WRAPPER postgresql; -CREATE SERVER fdtest FOREIGN DATA WRAPPER postgresql OPTIONS (dbname 'contrib_regression'); +CREATE SERVER fdtest FOREIGN DATA WRAPPER postgresql OPTIONS (dbname 'contrib_regression', connect_cost '100.0'); CREATE USER MAPPING FOR public SERVER fdtest; GRANT USAGE ON FOREIGN SERVER fdtest TO dblink_regression_test; GRANT EXECUTE ON FUNCTION dblink_connect_u(text, text) TO dblink_regression_test; diff --git a/contrib/postgresql_fdw/expected/postgresql_fdw.out b/contrib/postgresql_fdw/expected/postgresql_fdw.out index 984d0a3cb8..0acedda5b4 100644 --- a/contrib/postgresql_fdw/expected/postgresql_fdw.out +++ b/contrib/postgresql_fdw/expected/postgresql_fdw.out @@ -53,16 +53,17 @@ COMMIT; -- create FOREIGN DATA WRAPPER for PostgresSQL CREATE FOREIGN DATA WRAPPER contrib_regression_wrapper HANDLER postgresql_fdw_handler - VALIDATOR postgresql_fdw_validator; + VALIDATOR postgresql_fdw_validator; -- create FOREIGN SERVER for remote database 1 CREATE SERVER contrib_regression_srv_1 FOREIGN DATA WRAPPER contrib_regression_wrapper - OPTIONS (host 'localhost', dbname 'contrib_regression_f1'); + OPTIONS (host 'localhost', dbname 'contrib_regression_f1', + connection_cost '100.0', transfer_cost '0.01'); CREATE USER MAPPING FOR PUBLIC SERVER contrib_regression_srv_1; -- create FOREIGN SERVER for remote database 2 CREATE SERVER contrib_regression_srv_2 FOREIGN DATA WRAPPER contrib_regression_wrapper - OPTIONS (host 'localhost', dbname 'contrib_regression_f2'); + OPTIONS (host 'localhost', dbname 'contrib_regression_f2'); CREATE USER MAPPING FOR PUBLIC SERVER contrib_regression_srv_2; -- Check ALTER FOREIGN TABLE OWNER TO before create various test tables CREATE FOREIGN TABLE ft1 (c1 integer) SERVER contrib_regression_srv_2; diff --git a/contrib/postgresql_fdw/postgresql_fdw.c b/contrib/postgresql_fdw/postgresql_fdw.c index a287177d8c..7193e756f7 100644 --- a/contrib/postgresql_fdw/postgresql_fdw.c +++ b/contrib/postgresql_fdw/postgresql_fdw.c @@ -18,9 +18,12 @@ #include "libpq-fe.h" #include "mb/pg_wchar.h" #include "miscadmin.h" -#include "nodes/nodeFuncs.h" #include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "nodes/relation.h" #include "optimizer/clauses.h" +#include "optimizer/cost.h" +#include "parser/parsetree.h" #include "parser/scansup.h" #include "utils/builtins.h" #include "utils/lsyscache.h" @@ -53,10 +56,11 @@ static void pgClose(ForeignScanState *scanstate); static void pgReOpen(ForeignScanState *scanstate); static void pgAnalyze(Relation rel, VacuumStmt *vacstmt, bool update_reltuples, bool inh); +static void pgEstimateCost(Path *path, PlannerInfo *root, RelOptInfo *baserel); /* deparse SQL from the request */ static bool is_immutable_func(Oid funcid); -static bool is_foreign_qual(ExprState *state); +static bool is_foreign_qual(Expr *expr); static bool foreign_qual_walker(Node *node, void *context); static char *deparseSql(ForeignScanState *scanstate); static int flatten_deflist(List *options, @@ -85,6 +89,7 @@ FdwRoutine postgresql_fdw_routine = pgClose, pgReOpen, pgAnalyze, + pgEstimateCost, }; /* @@ -105,25 +110,39 @@ static FSConnection * pgConnectServer(ForeignServer *server, UserMapping *user) { PGconn *conn; + const char **all_keywords; + const char **all_values; const char **keywords; const char **values; int n; + int i, j; /* - * construct connection params from options of ForeignDataWrapper, - * ForeignServer and UserMapping. Assuming all GENERIC OPTIONS are - * conneciton information. - * - * TODO: allow non-connection options and ignore them during constructing - * connection string. + * Construct connection params from generic options of ForeignServer and + * UserMapping. Generic options might not be a one of connection options. */ n = list_length(server->options) + list_length(user->options) + 1; + all_keywords = (const char **) palloc(sizeof(char *) * n); + all_values = (const char **) palloc(sizeof(char *) * n); keywords = (const char **) palloc(sizeof(char *) * n); values = (const char **) palloc(sizeof(char *) * n); n = 0; - n += flatten_deflist(server->options, keywords + n, values + n); - n += flatten_deflist(user->options, keywords + n, values + n); - keywords[n] = values[n] = NULL; + n += flatten_deflist(server->options, all_keywords + n, all_values + n); + n += flatten_deflist(user->options, all_keywords + n, all_values + n); + all_keywords[n] = all_values[n] = NULL; + + for (i = 0, j = 0; all_keywords[i]; i++) + { + /* Use only libpq connection options. */ + if (!is_libpq_connection_option(all_keywords[i])) + continue; + keywords[j] = all_keywords[i]; + values[j] = all_values[i]; + j++; + } + keywords[j] = values[j] = NULL; + pfree(all_keywords); + pfree(all_values); /* verify connection parameters and do connect */ check_conn_params(keywords, values); @@ -205,9 +224,9 @@ is_immutable_func(Oid funcid) * - scalar array operator (ANY/ALL) */ static bool -is_foreign_qual(ExprState *state) +is_foreign_qual(Expr *expr) { - return !foreign_qual_walker((Node *) state->expr, NULL); + return !foreign_qual_walker((Node *) expr, NULL); } /* @@ -398,7 +417,7 @@ deparseSql(ForeignScanState *scanstate) { ExprState *state = lfirst(lc); - if (is_foreign_qual(state)) + if (is_foreign_qual(state->expr)) { elog(DEBUG1, "foreign qual: %s", nodeToString(state->expr)); foreign_qual = lappend(foreign_qual, state); @@ -750,3 +769,68 @@ pgAnalyze(Relation rel, VacuumStmt *vacstmt, bool update_reltuples, bool inh) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("analyze of remote PostgreSQL table not implemented"))); } + +/* + * Estimate costs of scanning on a foreign table. + * + * baserel->baserestrictinfo can be used to examine quals on the relation. + */ +static void +pgEstimateCost(Path *path, PlannerInfo *root, RelOptInfo *baserel) +{ + RangeTblEntry *rte; + ForeignTable *table; + ForeignServer *server; + int n; + const char **keywords; + const char **values; + int i; + + /* Use the costs of sequential scan as approximate values. */ + cost_seqscan(path, root, baserel); + + /* + * Retrieve generic options from the target table and its server to correct + * costs. + */ + rte = planner_rt_fetch(baserel->relid, root); + table = GetForeignTable(rte->relid); + server = GetForeignServer(table->serverid); + n = list_length(table->options) + list_length(server->options) + 1; + keywords = (const char **) palloc(sizeof(char *) * n); + values = (const char **) palloc(sizeof(char *) * n); + n = 0; + n += flatten_deflist(server->options, keywords + n, values + n); + n += flatten_deflist(table->options, keywords + n, values + n); + keywords[n] = values[n] = NULL; + + /* + * Current costs in Path have been estimated as sequencial scan on a local + * table with cost_seqscan() above. If any cost factor option was set in a + * generic option of the foreign server or foreign table, we corrent the + * costs with the factor. Because currently the postgresql_fdw retrieve + * all result tuples at a time, first call of Iterate(), both startup_cost + * and total_cost will be corrected. + */ + for (i = 0; keywords[i]; i++) + { + if (pg_strcasecmp(keywords[i], "connection_cost") == 0) + { + double cost; + cost = strtod(values[i], NULL); + path->startup_cost += cost; + path->total_cost += cost; + } + else if (pg_strcasecmp(keywords[i], "transfer_cost") == 0) + { + double cost; + cost = strtod(values[i], NULL); + cost *= path->parent->width * path->parent->rows; + path->startup_cost += cost; + path->total_cost += cost; + } + } + + pfree(keywords); + pfree(values); +} diff --git a/contrib/postgresql_fdw/sql/postgresql_fdw.sql b/contrib/postgresql_fdw/sql/postgresql_fdw.sql index 3c3245602a..a23a1c15b5 100644 --- a/contrib/postgresql_fdw/sql/postgresql_fdw.sql +++ b/contrib/postgresql_fdw/sql/postgresql_fdw.sql @@ -51,18 +51,19 @@ COMMIT; -- create FOREIGN DATA WRAPPER for PostgresSQL CREATE FOREIGN DATA WRAPPER contrib_regression_wrapper HANDLER postgresql_fdw_handler - VALIDATOR postgresql_fdw_validator; + VALIDATOR postgresql_fdw_validator; -- create FOREIGN SERVER for remote database 1 CREATE SERVER contrib_regression_srv_1 FOREIGN DATA WRAPPER contrib_regression_wrapper - OPTIONS (host 'localhost', dbname 'contrib_regression_f1'); + OPTIONS (host 'localhost', dbname 'contrib_regression_f1', + connection_cost '100.0', transfer_cost '0.01'); CREATE USER MAPPING FOR PUBLIC SERVER contrib_regression_srv_1; -- create FOREIGN SERVER for remote database 2 CREATE SERVER contrib_regression_srv_2 FOREIGN DATA WRAPPER contrib_regression_wrapper - OPTIONS (host 'localhost', dbname 'contrib_regression_f2'); + OPTIONS (host 'localhost', dbname 'contrib_regression_f2'); CREATE USER MAPPING FOR PUBLIC SERVER contrib_regression_srv_2; -- Check ALTER FOREIGN TABLE OWNER TO before create various test tables diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c index ce9923026b..6c31b9ec7a 100644 --- a/src/backend/foreign/foreign.c +++ b/src/backend/foreign/foreign.c @@ -330,6 +330,7 @@ struct PgFdwOption { const char *optname; Oid optcontext; /* Oid of catalog in which option may appear */ + bool is_conn_opt; /* True if the option is a connection option */ }; /* @@ -339,32 +340,35 @@ struct PgFdwOption */ static struct PgFdwOption valid_options[] = { /* Connection Options */ - {"authtype", ForeignServerRelationId}, - {"service", ForeignServerRelationId}, - {"user", UserMappingRelationId}, - {"password", UserMappingRelationId}, - {"connect_timeout", ForeignServerRelationId}, - {"dbname", ForeignServerRelationId}, - {"host", ForeignServerRelationId}, - {"hostaddr", ForeignServerRelationId}, - {"port", ForeignServerRelationId}, - {"tty", ForeignServerRelationId}, - {"options", ForeignServerRelationId}, - {"requiressl", ForeignServerRelationId}, - {"sslmode", ForeignServerRelationId}, - {"gsslib", ForeignServerRelationId}, - - /* Other options */ - {"nspname", ForeignTableRelationId}, - {"relname", ForeignTableRelationId}, + {"authtype", ForeignServerRelationId, true}, + {"service", ForeignServerRelationId, true}, + {"user", UserMappingRelationId, true}, + {"password", UserMappingRelationId, true}, + {"connect_timeout", ForeignServerRelationId, true}, + {"dbname", ForeignServerRelationId, true}, + {"host", ForeignServerRelationId, true}, + {"hostaddr", ForeignServerRelationId, true}, + {"port", ForeignServerRelationId, true}, + {"tty", ForeignServerRelationId, true}, + {"options", ForeignServerRelationId, true}, + {"requiressl", ForeignServerRelationId, true}, + {"sslmode", ForeignServerRelationId, true}, + {"gsslib", ForeignServerRelationId, true}, + + /* Catalog options */ + {"nspname", ForeignTableRelationId, false}, + {"relname", ForeignTableRelationId, false}, + + /* Planner cost options */ + {"connection_cost", ForeignServerRelationId, false}, + {"transfer_cost", ForeignServerRelationId, false}, /* Centinel */ - {NULL, InvalidOid} + {NULL, InvalidOid, false} }; - /* - * Check if the provided option is one of libpq conninfo options. + * Check if the provided option is one of valid options. * context is the Oid of the catalog the option came from, or 0 if we * don't care. */ @@ -379,9 +383,24 @@ is_valid_option(const char *option, Oid context) return false; } +/* + * Check if the provided option is one of libpq conninfo options. + * XXX: Should be moved to interface/libpq or backend/libpq? + */ +bool +is_libpq_connection_option(const char *option) +{ + struct PgFdwOption *opt; + + for (opt = valid_options; opt->optname; opt++) + if (opt->is_conn_opt && strcmp(opt->optname, option) == 0) + return true; + return false; +} /* - * Validate the generic option given to SERVER or USER MAPPING. + * Validate the generic option given to FOREIGN DATA WRAPPER, SERVER, USER + * MAPPING or FOREIGN TABLE. * Raise an ERROR if the option or its value is considered * invalid. * diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 9930abdac3..924ce621b7 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -1029,13 +1029,28 @@ void cost_foreignscan(Path *path, PlannerInfo *root, RelOptInfo *baserel) { + RangeTblEntry *rte; + ForeignTable *table; + ForeignServer *server; + ForeignDataWrapper *wrapper; + FdwRoutine *routine; + /* Should only be applied to base relations */ Assert(baserel->relid > 0); Assert(baserel->rtekind == RTE_RELATION); - /* XXX estimate cost to scan the foreign table */ - path->startup_cost = 10000.0; - path->total_cost = 10000.0; + /* Have the wrapper handler estimate the cost of this scan. */ + rte = planner_rt_fetch(baserel->relid, root); + table = GetForeignTable(rte->relid); + server = GetForeignServer(table->serverid); + wrapper = GetForeignDataWrapper(server->fdwid); + routine = GetFdwRoutine(wrapper->fdwhandler); + + /* Use cost_seqscan() instead if the wrapper did't support this API. */ + if (routine->EstimateCost != NULL) + routine->EstimateCost(path, root, baserel); + else + cost_seqscan(path, root, baserel); } /* @@ -2447,11 +2462,6 @@ cost_rescan(PlannerInfo *root, Path *path, *rescan_total_cost = run_cost; } break; - case T_ForeignScan: - /* XXX estimate cost to scan foreign table via FDW */ - *rescan_startup_cost = path->startup_cost; - *rescan_total_cost = path->total_cost; - break; default: *rescan_startup_cost = path->startup_cost; *rescan_total_cost = path->total_cost; diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 92666e9479..d2734907c3 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -1670,8 +1670,6 @@ create_foreignscan_plan(PlannerInfo *root, Path *best_path, scan_relid); copy_path_costsize(&scan_plan->scan.plan, best_path); - /* XXX override estimated rows to emulate big-result */ - scan_plan->scan.plan.plan_rows = 1000; return scan_plan; } diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 9102f42872..e83e50f65b 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -440,9 +440,6 @@ estimate_rel_size(Relation rel, int32 *attr_widths, *pages = 1; *tuples = 1; break; - case RELKIND_FOREIGN_TABLE: - /* XXX: estimate with FDW or use pg_class.reltuples/relpages ? */ - break; default: /* else it has no disk storage; probably shouldn't get here? */ *pages = 0; diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h index b9cd134f6c..343d316840 100644 --- a/src/include/foreign/foreign.h +++ b/src/include/foreign/foreign.h @@ -73,6 +73,7 @@ typedef struct FdwRoutine FdwRoutine; typedef struct FSConnection FSConnection; typedef struct FdwReply FdwReply; +/* catalog manipulation */ extern ForeignServer *GetForeignServer(Oid serverid); extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok); extern Oid GetForeignServerOidByName(const char *name, bool missing_ok); @@ -87,6 +88,7 @@ extern bool IsForeignTable(Oid relid); /* ALTER FOREIGN TABLE ... OPTIONS (...) handlers */ extern void ATExecGenericOptions(Relation rel, List *options); +extern bool is_libpq_connection_option(const char *option); /* foreign server connections */ extern FSConnection *ConnectToForeignServer(FdwRoutine *routine, diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index e7254f7ac4..3c580d70a7 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1380,7 +1380,7 @@ typedef struct ForeignScanState /* * Common interface routines of FDW baed on SQL/MED standard. - * Each FDW must implement these routines. + * A foreign-data wrapper must implement these routines. */ struct FdwRoutine { @@ -1422,10 +1422,20 @@ struct FdwRoutine * foreign table. Statistics to be updated are: * - all columns of pg_statistics * - reltuples, relpages of pg_class (only if update_reltuples was true) + * * If this API was not implemented (== NULL), ANALYZE skips that relation. */ void (*Analyze)(Relation rel, VacuumStmt *vacstmt, bool update_reltuples, bool inh); + + /* + * Estimate costs of scanning on the foreign table. + * FDW should update startup_cost and total_cost in the Path. + * + * If this API was not implemented (== NULL), cost_seqscan() will be used + * instead. + */ + void (*EstimateCost)(Path *path, PlannerInfo *root, RelOptInfo *baserel); }; /* ---------------------------------------------------------------- diff --git a/src/test/regress/expected/foreign_data.out b/src/test/regress/expected/foreign_data.out index 8837c691f3..99272ee783 100644 --- a/src/test/regress/expected/foreign_data.out +++ b/src/test/regress/expected/foreign_data.out @@ -285,7 +285,7 @@ CREATE SERVER s6 VERSION '16.0' FOREIGN DATA WRAPPER foo OPTIONS (host 'a', dbna CREATE SERVER s7 TYPE 'oracle' VERSION '17.0' FOREIGN DATA WRAPPER foo OPTIONS (host 'a', dbname 'b'); CREATE SERVER s8 FOREIGN DATA WRAPPER postgresql OPTIONS (foo '1'); -- ERROR ERROR: invalid option "foo" -HINT: Valid options in this context are: authtype, service, connect_timeout, dbname, host, hostaddr, port, tty, options, requiressl, sslmode, gsslib +HINT: Valid options in this context are: authtype, service, connect_timeout, dbname, host, hostaddr, port, tty, options, requiressl, sslmode, gsslib, connection_cost, transfer_cost CREATE SERVER s8 FOREIGN DATA WRAPPER postgresql OPTIONS (host 'localhost', dbname 's8db'); \des+ List of foreign servers