2 * Copyright (c) 2011 Teodor Sigaev <teodor@sigaev.ru>
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. Neither the name of the author nor the names of any co-contributors
14 * may be used to endorse or promote products derived from this software
15 * without specific prior written permission.
17 * THIS SOFTWARE IS PROVIDED BY CONTRIBUTORS ``AS IS'' AND ANY EXPRESS
18 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL CONTRIBUTORS BE LIABLE FOR ANY
21 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 #include "catalog/namespace.h"
34 #include "commands/vacuum.h"
35 #include "executor/executor.h"
36 #include "nodes/nodes.h"
37 #include "nodes/parsenodes.h"
38 #include "storage/bufmgr.h"
39 #include "utils/builtins.h"
40 #include "utils/hsearch.h"
41 #include "utils/memutils.h"
42 #include "utils/lsyscache.h"
43 #include "utils/guc.h"
44 #if PG_VERSION_NUM >= 90200
45 #include "catalog/pg_class.h"
46 #include "nodes/primnodes.h"
47 #include "tcop/utility.h"
48 #include "utils/rel.h"
49 #include "utils/relcache.h"
50 #include "utils/timestamp.h"
51 #if PG_VERSION_NUM >= 90500
52 #include "nodes/makefuncs.h"
53 #if PG_VERSION_NUM >= 100000
54 #include "utils/varlena.h"
55 #include "utils/regproc.h"
60 #ifdef PG_MODULE_MAGIC
64 static bool online_analyze_enable = true;
65 static bool online_analyze_verbose = true;
66 static double online_analyze_scale_factor = 0.1;
67 static int online_analyze_threshold = 50;
68 static int online_analyze_capacity_threshold = 100000;
69 static double online_analyze_min_interval = 10000;
71 static ExecutorEnd_hook_type oldExecutorEndHook = NULL;
72 #if PG_VERSION_NUM >= 90200
73 static ProcessUtility_hook_type oldProcessUtilityHook = NULL;
79 OATT_PERSISTENT = 0x01,
80 OATT_TEMPORARY = 0x02,
82 } OnlineAnalyzeTableType;
84 static const struct config_enum_entry online_analyze_table_type_options[] =
86 {"all", OATT_ALL, false},
87 {"persistent", OATT_PERSISTENT, false},
88 {"temporary", OATT_TEMPORARY, false},
89 {"none", OATT_NONE, false},
93 static int online_analyze_table_type = (int)OATT_ALL;
95 typedef struct TableList {
101 static TableList excludeTables = {0, NULL, NULL};
102 static TableList includeTables = {0, NULL, NULL};
104 typedef struct OnlineAnalyzeTableStat {
107 PgStat_Counter n_tuples;
108 PgStat_Counter changes_since_analyze;
109 TimestampTz autovac_analyze_timestamp;
110 TimestampTz analyze_timestamp;
111 } OnlineAnalyzeTableStat;
113 static MemoryContext onlineAnalyzeMemoryContext = NULL;
114 static HTAB *relstats = NULL;
116 static void relstatsInit(void);
118 #if PG_VERSION_NUM < 100000
120 oid_cmp(const void *a, const void *b)
122 if (*(Oid*)a == *(Oid*)b)
124 return (*(Oid*)a > *(Oid*)b) ? 1 : -1;
129 tableListAssign(const char * newval, bool doit, TableList *tbl)
138 rawname = pstrdup(newval);
140 if (!SplitIdentifierString(rawname, ',', &namelist))
145 nOids = list_length(namelist);
146 newOids = malloc(sizeof(Oid) * (nOids+1));
148 elog(ERROR,"could not allocate %d bytes",
149 (int)(sizeof(Oid) * (nOids+1)));
154 char *curname = (char *) lfirst(l);
155 #if PG_VERSION_NUM >= 90200
156 Oid relOid = RangeVarGetRelid(makeRangeVarFromNameList(
157 stringToQualifiedNameList(curname)), NoLock, true);
159 Oid relOid = RangeVarGetRelid(makeRangeVarFromNameList(
160 stringToQualifiedNameList(curname)), true);
163 if (relOid == InvalidOid)
165 #if PG_VERSION_NUM >= 90100
168 elog(WARNING,"'%s' does not exist", curname);
171 else if ( get_rel_relkind(relOid) != RELKIND_RELATION )
173 #if PG_VERSION_NUM >= 90100
176 elog(WARNING,"'%s' is not an table", curname);
181 newOids[i++] = relOid;
190 tbl->tables = newOids;
191 if (tbl->nTables > 1)
192 qsort(tbl->tables, tbl->nTables, sizeof(tbl->tables[0]), oid_cmp);
208 #if PG_VERSION_NUM >= 90100
210 excludeTablesCheck(char **newval, void **extra, GucSource source)
214 val = (char*)tableListAssign(*newval, false, &excludeTables);
226 excludeTablesAssign(const char *newval, void *extra)
228 tableListAssign(newval, true, &excludeTables);
232 includeTablesCheck(char **newval, void **extra, GucSource source)
236 val = (char*)tableListAssign(*newval, false, &includeTables);
248 includeTablesAssign(const char *newval, void *extra)
250 tableListAssign(newval, true, &excludeTables);
253 #else /* PG_VERSION_NUM < 90100 */
256 excludeTablesAssign(const char * newval, bool doit, GucSource source)
258 return tableListAssign(newval, doit, &excludeTables);
262 includeTablesAssign(const char * newval, bool doit, GucSource source)
264 return tableListAssign(newval, doit, &includeTables);
270 tableListShow(TableList *tbl)
276 len = 1 /* \0 */ + tbl->nTables * (2 * NAMEDATALEN + 2 /* ', ' */ + 1 /* . */);
277 ptr = val = palloc(len);
279 for(i=0; i<tbl->nTables; i++)
281 char *relname = get_rel_name(tbl->tables[i]);
282 Oid nspOid = get_rel_namespace(tbl->tables[i]);
283 char *nspname = get_namespace_name(nspOid);
285 if ( relname == NULL || nspOid == InvalidOid || nspname == NULL )
288 ptr += snprintf(ptr, len - (ptr - val), "%s%s.%s",
297 excludeTablesShow(void)
299 return tableListShow(&excludeTables);
303 includeTablesShow(void)
305 return tableListShow(&includeTables);
309 matchOid(TableList *tbl, Oid oid)
311 Oid *StopLow = tbl->tables,
312 *StopHigh = tbl->tables + tbl->nTables,
315 /* Loop invariant: StopLow <= val < StopHigh */
316 while (StopLow < StopHigh)
318 StopMiddle = StopLow + ((StopHigh - StopLow) >> 1);
320 if (*StopMiddle == oid)
322 else if (*StopMiddle < oid)
323 StopLow = StopMiddle + 1;
325 StopHigh = StopMiddle;
331 #if PG_VERSION_NUM >= 90500
333 makeRangeVarFromOid(Oid relOid)
336 get_namespace_name(get_rel_namespace(relOid)),
337 get_rel_name(relOid),
345 makeAnalyze(Oid relOid, CmdType operation, int32 naffected)
347 TimestampTz now = GetCurrentTimestamp();
349 OnlineAnalyzeTableType reltype;
352 OnlineAnalyzeTableStat *rstat,
354 PgStat_StatTabEntry *tabentry = NULL;
356 if (relOid == InvalidOid)
360 /* return if there is no changes */
362 else if (naffected < 0)
363 /* number if affected rows is unknown */
366 rel = RelationIdGetRelation(relOid);
367 if (rel->rd_rel->relkind != RELKIND_RELATION)
374 #if PG_VERSION_NUM >= 90100
375 (rel->rd_rel->relpersistence == RELPERSISTENCE_TEMP)
377 (rel->rd_istemp || rel->rd_islocaltemp)
379 ? OATT_TEMPORARY : OATT_PERSISTENT;
384 * includeTables overwrites excludeTables
386 switch(online_analyze_table_type)
389 if (get_rel_relkind(relOid) != RELKIND_RELATION ||
390 (matchOid(&excludeTables, relOid) == true &&
391 matchOid(&includeTables, relOid) == false))
395 if (get_rel_relkind(relOid) != RELKIND_RELATION ||
396 matchOid(&includeTables, relOid) == false)
400 case OATT_PERSISTENT:
403 * skip analyze if relation's type doesn't not match
404 * online_analyze_table_type
406 if ((online_analyze_table_type & reltype) == 0 ||
407 matchOid(&excludeTables, relOid) == true)
409 if (matchOid(&includeTables, relOid) == false)
416 * Do not store data about persistent table in local memory because we
417 * could not track changes of them: they could be changed by another
418 * backends. So always get a pgstat table entry.
420 if (reltype == OATT_TEMPORARY)
421 rstat = hash_search(relstats, &relOid, HASH_ENTER, &found);
423 rstat = &dummyrstat; /* found == false for following if */
425 if (found == false || rstat->rereadStat == true || naffected == 0)
430 MemSet(rstat, 0, sizeof(*rstat));
431 rstat->tableid = relOid;
433 Assert(rstat->tableid == relOid);
435 tabentry = pgstat_fetch_stat_tabentry(relOid);
439 rstat->n_tuples = tabentry->n_dead_tuples + tabentry->n_live_tuples;
440 rstat->changes_since_analyze =
441 #if PG_VERSION_NUM >= 90000
442 tabentry->changes_since_analyze;
444 tabentry->n_live_tuples + tabentry->n_dead_tuples -
445 tabentry->last_anl_tuples;
447 rstat->autovac_analyze_timestamp =
448 tabentry->autovac_analyze_timestamp;
449 rstat->analyze_timestamp = tabentry->analyze_timestamp;
450 rstat->rereadStat = false;
455 rstat->rereadStat = true;
460 /* do not analyze too often, if both stamps are exceeded the go */
461 TimestampDifferenceExceeds(rstat->analyze_timestamp, now, online_analyze_min_interval) &&
462 TimestampDifferenceExceeds(rstat->autovac_analyze_timestamp, now, online_analyze_min_interval) &&
463 /* be in sync with relation_needs_vacanalyze */
464 ((double)(rstat->changes_since_analyze + naffected)) >=
465 online_analyze_scale_factor * ((double)rstat->n_tuples) +
466 (double)online_analyze_threshold))
468 #if PG_VERSION_NUM < 90500
471 VacuumParams vacstmt;
473 TimestampTz startStamp, endStamp;
476 memset(&startStamp, 0, sizeof(startStamp)); /* keep compiler quiet */
478 memset(&vacstmt, 0, sizeof(vacstmt));
480 vacstmt.freeze_min_age = -1;
481 vacstmt.freeze_table_age = -1; /* ??? */
483 #if PG_VERSION_NUM < 90500
484 vacstmt.type = T_VacuumStmt;
485 vacstmt.relation = NULL;
486 vacstmt.va_cols = NIL;
487 #if PG_VERSION_NUM >= 90000
488 vacstmt.options = VACOPT_ANALYZE;
489 if (online_analyze_verbose)
490 vacstmt.options |= VACOPT_VERBOSE;
492 vacstmt.vacuum = vacstmt.full = false;
493 vacstmt.analyze = true;
494 vacstmt.verbose = online_analyze_verbose;
497 vacstmt.multixact_freeze_min_age = -1;
498 vacstmt.multixact_freeze_table_age = -1;
499 vacstmt.log_min_duration = -1;
502 if (online_analyze_verbose)
503 startStamp = GetCurrentTimestamp();
506 #if PG_VERSION_NUM < 90500
508 #if PG_VERSION_NUM >= 90018
511 , GetAccessStrategy(BAS_VACUUM)
512 #if (PG_VERSION_NUM >= 90000) && (PG_VERSION_NUM < 90004)
516 makeRangeVarFromOid(relOid),
517 VACOPT_ANALYZE | ((online_analyze_verbose) ? VACOPT_VERBOSE : 0),
518 &vacstmt, NULL, true, GetAccessStrategy(BAS_VACUUM)
522 if (online_analyze_verbose)
527 endStamp = GetCurrentTimestamp();
528 TimestampDifference(startStamp, endStamp, &secs, µsecs);
529 elog(INFO, "analyze \"%s\" took %.02f seconds",
530 get_rel_name(relOid),
531 ((double)secs) + ((double)microsecs)/1.0e6);
534 rstat->autovac_analyze_timestamp = now;
535 rstat->changes_since_analyze = 0;
536 rstat->rereadStat = true;
538 /* update last analyze timestamp in local memory of backend */
541 tabentry->analyze_timestamp = now;
542 tabentry->changes_since_analyze = 0;
545 /* force reload stat for new table */
547 pgstat_clear_snapshot();
552 #if PG_VERSION_NUM >= 90000
554 tabentry->changes_since_analyze += naffected;
556 rstat->changes_since_analyze += naffected;
559 /* Reset local cache if we are over limit */
560 if (hash_get_num_entries(relstats) > online_analyze_capacity_threshold)
564 extern PGDLLIMPORT void onlineAnalyzeHooker(QueryDesc *queryDesc);
566 onlineAnalyzeHooker(QueryDesc *queryDesc)
568 uint32 naffected = -1;
570 if (queryDesc->estate)
571 naffected = queryDesc->estate->es_processed;
573 if (online_analyze_enable && queryDesc->plannedstmt &&
574 (queryDesc->operation == CMD_INSERT ||
575 queryDesc->operation == CMD_UPDATE ||
576 queryDesc->operation == CMD_DELETE
577 #if PG_VERSION_NUM < 90200
578 || (queryDesc->operation == CMD_SELECT &&
579 queryDesc->plannedstmt->intoClause)
583 #if PG_VERSION_NUM < 90200
584 if (queryDesc->operation == CMD_SELECT)
586 Oid relOid = RangeVarGetRelid(queryDesc->plannedstmt->intoClause->rel, true);
588 makeAnalyze(relOid, queryDesc->operation, naffected);
592 if (queryDesc->plannedstmt->resultRelations &&
593 queryDesc->plannedstmt->rtable)
597 foreach(l, queryDesc->plannedstmt->resultRelations)
599 int n = lfirst_int(l);
600 RangeTblEntry *rte = list_nth(queryDesc->plannedstmt->rtable, n-1);
602 if (rte->rtekind == RTE_RELATION)
603 makeAnalyze(rte->relid, queryDesc->operation, naffected);
608 if (oldExecutorEndHook)
609 oldExecutorEndHook(queryDesc);
611 standard_ExecutorEnd(queryDesc);
614 #if PG_VERSION_NUM >= 90200
616 onlineAnalyzeHookerUtility(
617 #if PG_VERSION_NUM >= 100000
622 const char *queryString,
623 #if PG_VERSION_NUM >= 90300
624 ProcessUtilityContext context, ParamListInfo params,
625 #if PG_VERSION_NUM >= 100000
626 QueryEnvironment *queryEnv,
629 ParamListInfo params, bool isTopLevel,
631 DestReceiver *dest, char *completionTag) {
632 RangeVar *tblname = NULL;
633 #if PG_VERSION_NUM >= 100000
634 Node *parsetree = NULL;
636 if (pstmt->commandType == CMD_UTILITY)
637 parsetree = pstmt->utilityStmt;
640 if (parsetree && IsA(parsetree, CreateTableAsStmt) &&
641 ((CreateTableAsStmt*)parsetree)->into &&
642 online_analyze_enable)
643 tblname = (RangeVar*)copyObject(((CreateTableAsStmt*)parsetree)->into->rel);
645 #if PG_VERSION_NUM >= 100000
646 #define parsetree pstmt
649 if (oldProcessUtilityHook)
650 oldProcessUtilityHook(parsetree, queryString,
651 #if PG_VERSION_NUM >= 90300
653 #if PG_VERSION_NUM >= 100000
659 dest, completionTag);
661 standard_ProcessUtility(parsetree, queryString,
662 #if PG_VERSION_NUM >= 90300
664 #if PG_VERSION_NUM >= 100000
670 dest, completionTag);
672 #if PG_VERSION_NUM >= 100000
677 Oid tblOid = RangeVarGetRelid(tblname, NoLock, true);
679 makeAnalyze(tblOid, CMD_INSERT, -1);
690 MemSet(&hash_ctl, 0, sizeof(hash_ctl));
692 hash_ctl.hash = oid_hash;
693 flags |= HASH_FUNCTION;
695 if (onlineAnalyzeMemoryContext)
697 Assert(relstats != NULL);
698 MemoryContextReset(onlineAnalyzeMemoryContext);
702 Assert(relstats == NULL);
703 onlineAnalyzeMemoryContext =
704 AllocSetContextCreate(CacheMemoryContext,
705 "online_analyze storage context",
706 #if PG_VERSION_NUM < 90600
707 ALLOCSET_DEFAULT_MINSIZE,
708 ALLOCSET_DEFAULT_INITSIZE,
709 ALLOCSET_DEFAULT_MAXSIZE
711 ALLOCSET_DEFAULT_SIZES
716 hash_ctl.hcxt = onlineAnalyzeMemoryContext;
717 flags |= HASH_CONTEXT;
719 hash_ctl.keysize = sizeof(Oid);
721 hash_ctl.entrysize = sizeof(OnlineAnalyzeTableStat);
724 relstats = hash_create("online_analyze storage", 1024, &hash_ctl, flags);
733 oldExecutorEndHook = ExecutorEnd_hook;
735 ExecutorEnd_hook = onlineAnalyzeHooker;
737 #if PG_VERSION_NUM >= 90200
738 oldProcessUtilityHook = ProcessUtility_hook;
740 ProcessUtility_hook = onlineAnalyzeHookerUtility;
744 DefineCustomBoolVariable(
745 "online_analyze.enable",
746 "Enable on-line analyze",
747 "Enables analyze of table directly after insert/update/delete/select into",
748 &online_analyze_enable,
749 #if PG_VERSION_NUM >= 80400
750 online_analyze_enable,
753 #if PG_VERSION_NUM >= 80400
755 #if PG_VERSION_NUM >= 90100
763 DefineCustomBoolVariable(
764 "online_analyze.verbose",
765 "Verbosity of on-line analyze",
766 "Make ANALYZE VERBOSE after table's changes",
767 &online_analyze_verbose,
768 #if PG_VERSION_NUM >= 80400
769 online_analyze_verbose,
772 #if PG_VERSION_NUM >= 80400
774 #if PG_VERSION_NUM >= 90100
782 DefineCustomRealVariable(
783 "online_analyze.scale_factor",
784 "fraction of table size to start on-line analyze",
785 "fraction of table size to start on-line analyze",
786 &online_analyze_scale_factor,
787 #if PG_VERSION_NUM >= 80400
788 online_analyze_scale_factor,
793 #if PG_VERSION_NUM >= 80400
795 #if PG_VERSION_NUM >= 90100
803 DefineCustomIntVariable(
804 "online_analyze.threshold",
805 "min number of row updates before on-line analyze",
806 "min number of row updates before on-line analyze",
807 &online_analyze_threshold,
808 #if PG_VERSION_NUM >= 80400
809 online_analyze_threshold,
814 #if PG_VERSION_NUM >= 80400
816 #if PG_VERSION_NUM >= 90100
824 DefineCustomIntVariable(
825 "online_analyze.capacity_threshold",
826 "Max local cache table capacity",
827 "Max local cache table capacity",
828 &online_analyze_capacity_threshold,
829 #if PG_VERSION_NUM >= 80400
830 online_analyze_capacity_threshold,
835 #if PG_VERSION_NUM >= 80400
837 #if PG_VERSION_NUM >= 90100
845 DefineCustomRealVariable(
846 "online_analyze.min_interval",
847 "minimum time interval between analyze call (in milliseconds)",
848 "minimum time interval between analyze call (in milliseconds)",
849 &online_analyze_min_interval,
850 #if PG_VERSION_NUM >= 80400
851 online_analyze_min_interval,
856 #if PG_VERSION_NUM >= 80400
858 #if PG_VERSION_NUM >= 90100
866 DefineCustomEnumVariable(
867 "online_analyze.table_type",
868 "Type(s) of table for online analyze: all(default), persistent, temporary, none",
870 &online_analyze_table_type,
871 #if PG_VERSION_NUM >= 80400
872 online_analyze_table_type,
874 online_analyze_table_type_options,
876 #if PG_VERSION_NUM >= 80400
878 #if PG_VERSION_NUM >= 90100
886 DefineCustomStringVariable(
887 "online_analyze.exclude_tables",
888 "List of tables which will not online analyze",
890 &excludeTables.tableStr,
891 #if PG_VERSION_NUM >= 80400
896 #if PG_VERSION_NUM >= 90100
905 DefineCustomStringVariable(
906 "online_analyze.include_tables",
907 "List of tables which will online analyze",
909 &includeTables.tableStr,
910 #if PG_VERSION_NUM >= 80400
915 #if PG_VERSION_NUM >= 90100
929 ExecutorEnd_hook = oldExecutorEndHook;
930 #if PG_VERSION_NUM >= 90200
931 ProcessUtility_hook = oldProcessUtilityHook;
934 if (excludeTables.tables)
935 free(excludeTables.tables);
936 if (includeTables.tables)
937 free(includeTables.tables);
939 excludeTables.tables = includeTables.tables = NULL;
940 excludeTables.nTables = includeTables.nTables = 0;