From: Teodor Sigaev Date: Wed, 9 Nov 2016 17:01:17 +0000 (+0300) Subject: Use local cache for tracking changes to reduce reading file X-Git-Url: http://sigaev.ru/git/gitweb.cgi?a=commitdiff_plain;h=223c8eb8b8e8fc1beefc31fad8f8d62b2bba4848;p=online_analyze.git Use local cache for tracking changes to reduce reading file --- diff --git a/README.online_analyze b/README.online_analyze index 3bb942c..bb4491c 100644 --- a/README.online_analyze +++ b/README.online_analyze @@ -1,7 +1,8 @@ Module makes an analyze call immediately after INSERT/UPDATE/DELETE/SELECT INTO for affected table(s). -Supported versions of PostgreSQL: 8.4.*, 9.0.*, 9.1.*, 9.2.*, 9.3.*, 9.4* +Supported versions of PostgreSQL: 8.4.*, 9.0.*, 9.1.*, 9.2.*, 9.3.*, 9.4*, 9.5*, + 9.6* Usage: LOAD 'online_analyze'; diff --git a/online_analyze.c b/online_analyze.c index 25a1ecf..d16b233 100644 --- a/online_analyze.c +++ b/online_analyze.c @@ -37,6 +37,8 @@ #include "nodes/parsenodes.h" #include "storage/bufmgr.h" #include "utils/builtins.h" +#include "utils/hsearch.h" +#include "utils/memutils.h" #include "utils/lsyscache.h" #include "utils/guc.h" #if PG_VERSION_NUM >= 90200 @@ -94,6 +96,17 @@ typedef struct TableList { static TableList excludeTables = {0, NULL, NULL}; static TableList includeTables = {0, NULL, NULL}; +typedef struct OnlineAnalyzeTableStat { + Oid tableid; + bool rereadStat; + PgStat_Counter n_tuples; + PgStat_Counter changes_since_analyze; + TimestampTz autovac_analyze_timestamp; + TimestampTz analyze_timestamp; +} OnlineAnalyzeTableStat; + +static HTAB *relstats = NULL; + static int oid_cmp(const void *a, const void *b) { @@ -321,10 +334,13 @@ makeRangeVarFromOid(Oid relOid) static void makeAnalyze(Oid relOid, CmdType operation, int32 naffected) { - PgStat_StatTabEntry *tabentry; TimestampTz now = GetCurrentTimestamp(); Relation rel; OnlineAnalyzeTableType reltype; + bool found, + newTable = false; + OnlineAnalyzeTableStat *rstat; + PgStat_StatTabEntry *tabentry = NULL; if (relOid == InvalidOid) return; @@ -385,26 +401,50 @@ makeAnalyze(Oid relOid, CmdType operation, int32 naffected) break; } - tabentry = pgstat_fetch_stat_tabentry(relOid); + rstat = hash_search(relstats, &relOid, HASH_ENTER, &found); + + if (found == false || rstat->rereadStat == true || naffected == 0) + { + if (!found) + { + MemSet(rstat, 0, sizeof(*rstat)); + rstat->tableid = relOid; + } + Assert(rstat->tableid == relOid); + + tabentry = pgstat_fetch_stat_tabentry(relOid); + + if (tabentry) + { + rstat->n_tuples = tabentry->n_dead_tuples + tabentry->n_live_tuples; + rstat->changes_since_analyze = #if PG_VERSION_NUM >= 90000 -#define changes_since_analyze(t) ((t)->changes_since_analyze) + tabentry->changes_since_analyze; #else -#define changes_since_analyze(t) ((t)->n_live_tuples + (t)->n_dead_tuples - (t)->last_anl_tuples) -#endif - - if ( - tabentry == NULL /* a new table */ || - ( - /* do not analyze too often, if both stamps are exceeded the go */ - TimestampDifferenceExceeds(tabentry->analyze_timestamp, now, online_analyze_min_interval) && - TimestampDifferenceExceeds(tabentry->autovac_analyze_timestamp, now, online_analyze_min_interval) && - /* be in sync with relation_needs_vacanalyze */ - ((double)(changes_since_analyze(tabentry) + naffected)) >= - online_analyze_scale_factor * ((double)(tabentry->n_dead_tuples + tabentry->n_live_tuples)) + - (double)online_analyze_threshold - ) - ) + tabentry->n_live_tuples + tabentry->n_dead_tuples - + tabentry->last_anl_tuples; +#endif + rstat->autovac_analyze_timestamp = + tabentry->autovac_analyze_timestamp; + rstat->analyze_timestamp = tabentry->analyze_timestamp; + rstat->rereadStat = false; + } + else + { + newTable = true; + rstat->rereadStat = true; + } + } + + if (newTable || ( + /* do not analyze too often, if both stamps are exceeded the go */ + TimestampDifferenceExceeds(rstat->analyze_timestamp, now, online_analyze_min_interval) && + TimestampDifferenceExceeds(rstat->autovac_analyze_timestamp, now, online_analyze_min_interval) && + /* be in sync with relation_needs_vacanalyze */ + ((double)(rstat->changes_since_analyze + naffected)) >= + online_analyze_scale_factor * ((double)rstat->n_tuples) + + (double)online_analyze_threshold)) { #if PG_VERSION_NUM < 90500 VacuumStmt vacstmt; @@ -413,6 +453,7 @@ makeAnalyze(Oid relOid, CmdType operation, int32 naffected) #endif TimestampTz startStamp, endStamp; + memset(&startStamp, 0, sizeof(startStamp)); /* keep compiler quiet */ memset(&vacstmt, 0, sizeof(vacstmt)); @@ -471,23 +512,28 @@ makeAnalyze(Oid relOid, CmdType operation, int32 naffected) ((double)secs) + ((double)microsecs)/1.0e6); } - if (tabentry == NULL) - { - /* new table */ - pgstat_clear_snapshot(); - } - else - { - /* update last analyze timestamp in local memory of backend */ + rstat->autovac_analyze_timestamp = now; + rstat->changes_since_analyze = 0; + rstat->rereadStat = true; + + /* update last analyze timestamp in local memory of backend */ + if (tabentry) tabentry->analyze_timestamp = now; - } + +#if 0 + /* force reload stat for new table */ + if (newTable) + pgstat_clear_snapshot(); +#endif } -#if PG_VERSION_NUM >= 90000 - else if (tabentry != NULL) + else { - tabentry->changes_since_analyze += naffected; - } +#if PG_VERSION_NUM >= 90000 + if (tabentry != NULL) + tabentry->changes_since_analyze += naffected; #endif + rstat->changes_since_analyze += naffected; + } } extern PGDLLIMPORT void onlineAnalyzeHooker(QueryDesc *queryDesc); @@ -579,10 +625,36 @@ onlineAnalyzeHookerUtility(Node *parsetree, const char *queryString, } #endif +static void +relstatsInit() +{ + HASHCTL hash_ctl; + int flags = 0; + + MemSet(&hash_ctl, 0, sizeof(hash_ctl)); + + hash_ctl.hash = oid_hash; + flags |= HASH_FUNCTION; + + hash_ctl.hcxt = AllocSetContextCreate(CacheMemoryContext, + "online_analyze storage context", + ALLOCSET_DEFAULT_SIZES); + flags |= HASH_CONTEXT; + + hash_ctl.keysize = sizeof(Oid); + + hash_ctl.entrysize = sizeof(OnlineAnalyzeTableStat); + flags |= HASH_ELEM; + + relstats = hash_create("online_analyze storage", 1024, &hash_ctl, flags); +} + void _PG_init(void); void _PG_init(void) { + relstatsInit(); + oldExecutorEndHook = ExecutorEnd_hook; ExecutorEnd_hook = onlineAnalyzeHooker;