2 * Copyright (c) 2011 Teodor Sigaev <teodor@sigaev.ru>
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. Neither the name of the author nor the names of any co-contributors
14 * may be used to endorse or promote products derived from this software
15 * without specific prior written permission.
17 * THIS SOFTWARE IS PROVIDED BY CONTRIBUTORS ``AS IS'' AND ANY EXPRESS
18 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL CONTRIBUTORS BE LIABLE FOR ANY
21 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 #include "catalog/namespace.h"
34 #include "commands/vacuum.h"
35 #include "executor/executor.h"
36 #include "nodes/nodes.h"
37 #include "nodes/parsenodes.h"
38 #include "storage/bufmgr.h"
39 #include "utils/builtins.h"
40 #include "utils/hsearch.h"
41 #include "utils/memutils.h"
42 #include "utils/lsyscache.h"
43 #include "utils/guc.h"
44 #if PG_VERSION_NUM >= 90200
45 #include "catalog/pg_class.h"
46 #include "nodes/primnodes.h"
47 #include "tcop/utility.h"
48 #include "utils/rel.h"
49 #include "utils/relcache.h"
50 #include "utils/timestamp.h"
51 #if PG_VERSION_NUM >= 90500
52 #include "nodes/makefuncs.h"
53 #if PG_VERSION_NUM >= 100000
54 #include "utils/varlena.h"
55 #include "utils/regproc.h"
60 #ifdef PG_MODULE_MAGIC
64 static bool online_analyze_enable = true;
65 static bool online_analyze_verbose = true;
66 static double online_analyze_scale_factor = 0.1;
67 static int online_analyze_threshold = 50;
68 static int online_analyze_capacity_threshold = 100000;
69 static double online_analyze_min_interval = 10000;
70 static int online_analyze_lower_limit = 0;
72 static ExecutorEnd_hook_type oldExecutorEndHook = NULL;
73 #if PG_VERSION_NUM >= 90200
74 static ProcessUtility_hook_type oldProcessUtilityHook = NULL;
79 CK_SELECT = CMD_SELECT,
80 CK_UPDATE = CMD_UPDATE,
81 CK_INSERT = CMD_INSERT,
82 CK_DELETE = CMD_DELETE,
90 OATT_PERSISTENT = 0x01,
91 OATT_TEMPORARY = 0x02,
93 } OnlineAnalyzeTableType;
95 static const struct config_enum_entry online_analyze_table_type_options[] =
97 {"all", OATT_ALL, false},
98 {"persistent", OATT_PERSISTENT, false},
99 {"temporary", OATT_TEMPORARY, false},
100 {"none", OATT_NONE, false},
104 static int online_analyze_table_type = (int)OATT_ALL;
106 typedef struct TableList {
112 static TableList excludeTables = {0, NULL, NULL};
113 static TableList includeTables = {0, NULL, NULL};
115 typedef struct OnlineAnalyzeTableStat {
118 PgStat_Counter n_tuples;
119 PgStat_Counter changes_since_analyze;
120 TimestampTz autovac_analyze_timestamp;
121 TimestampTz analyze_timestamp;
122 } OnlineAnalyzeTableStat;
124 static MemoryContext onlineAnalyzeMemoryContext = NULL;
125 static HTAB *relstats = NULL;
127 static void relstatsInit(void);
129 #if PG_VERSION_NUM < 100000
131 oid_cmp(const void *a, const void *b)
133 if (*(Oid*)a == *(Oid*)b)
135 return (*(Oid*)a > *(Oid*)b) ? 1 : -1;
140 tableListAssign(const char * newval, bool doit, TableList *tbl)
149 rawname = pstrdup(newval);
151 if (!SplitIdentifierString(rawname, ',', &namelist))
156 nOids = list_length(namelist);
157 newOids = malloc(sizeof(Oid) * (nOids+1));
159 elog(ERROR,"could not allocate %d bytes",
160 (int)(sizeof(Oid) * (nOids+1)));
165 char *curname = (char *) lfirst(l);
166 #if PG_VERSION_NUM >= 90200
167 Oid relOid = RangeVarGetRelid(makeRangeVarFromNameList(
168 stringToQualifiedNameList(curname)), NoLock, true);
170 Oid relOid = RangeVarGetRelid(makeRangeVarFromNameList(
171 stringToQualifiedNameList(curname)), true);
174 if (relOid == InvalidOid)
176 #if PG_VERSION_NUM >= 90100
179 elog(WARNING,"'%s' does not exist", curname);
182 else if ( get_rel_relkind(relOid) != RELKIND_RELATION )
184 #if PG_VERSION_NUM >= 90100
187 elog(WARNING,"'%s' is not an table", curname);
192 newOids[i++] = relOid;
201 tbl->tables = newOids;
202 if (tbl->nTables > 1)
203 qsort(tbl->tables, tbl->nTables, sizeof(tbl->tables[0]), oid_cmp);
219 #if PG_VERSION_NUM >= 90100
221 excludeTablesCheck(char **newval, void **extra, GucSource source)
225 val = (char*)tableListAssign(*newval, false, &excludeTables);
237 excludeTablesAssign(const char *newval, void *extra)
239 tableListAssign(newval, true, &excludeTables);
243 includeTablesCheck(char **newval, void **extra, GucSource source)
247 val = (char*)tableListAssign(*newval, false, &includeTables);
259 includeTablesAssign(const char *newval, void *extra)
261 tableListAssign(newval, true, &excludeTables);
264 #else /* PG_VERSION_NUM < 90100 */
267 excludeTablesAssign(const char * newval, bool doit, GucSource source)
269 return tableListAssign(newval, doit, &excludeTables);
273 includeTablesAssign(const char * newval, bool doit, GucSource source)
275 return tableListAssign(newval, doit, &includeTables);
281 tableListShow(TableList *tbl)
287 len = 1 /* \0 */ + tbl->nTables * (2 * NAMEDATALEN + 2 /* ', ' */ + 1 /* . */);
288 ptr = val = palloc(len);
290 for(i=0; i<tbl->nTables; i++)
292 char *relname = get_rel_name(tbl->tables[i]);
293 Oid nspOid = get_rel_namespace(tbl->tables[i]);
294 char *nspname = get_namespace_name(nspOid);
296 if ( relname == NULL || nspOid == InvalidOid || nspname == NULL )
299 ptr += snprintf(ptr, len - (ptr - val), "%s%s.%s",
308 excludeTablesShow(void)
310 return tableListShow(&excludeTables);
314 includeTablesShow(void)
316 return tableListShow(&includeTables);
320 matchOid(TableList *tbl, Oid oid)
322 Oid *StopLow = tbl->tables,
323 *StopHigh = tbl->tables + tbl->nTables,
326 /* Loop invariant: StopLow <= val < StopHigh */
327 while (StopLow < StopHigh)
329 StopMiddle = StopLow + ((StopHigh - StopLow) >> 1);
331 if (*StopMiddle == oid)
333 else if (*StopMiddle < oid)
334 StopLow = StopMiddle + 1;
336 StopHigh = StopMiddle;
342 #if PG_VERSION_NUM >= 90500
344 makeRangeVarFromOid(Oid relOid)
347 get_namespace_name(get_rel_namespace(relOid)),
348 get_rel_name(relOid),
356 makeAnalyze(Oid relOid, CmdKind operation, int64 naffected)
358 TimestampTz now = GetCurrentTimestamp();
360 OnlineAnalyzeTableType reltype;
363 OnlineAnalyzeTableStat *rstat,
365 PgStat_StatTabEntry *tabentry = NULL;
367 if (relOid == InvalidOid)
371 /* return if there is no changes */
373 else if (naffected < 0)
374 /* number if affected rows is unknown */
377 rel = RelationIdGetRelation(relOid);
378 if (rel->rd_rel->relkind != RELKIND_RELATION)
385 #if PG_VERSION_NUM >= 90100
386 (rel->rd_rel->relpersistence == RELPERSISTENCE_TEMP)
388 (rel->rd_istemp || rel->rd_islocaltemp)
390 ? OATT_TEMPORARY : OATT_PERSISTENT;
395 * includeTables overwrites excludeTables
397 switch(online_analyze_table_type)
400 if (get_rel_relkind(relOid) != RELKIND_RELATION ||
401 (matchOid(&excludeTables, relOid) == true &&
402 matchOid(&includeTables, relOid) == false))
406 if (get_rel_relkind(relOid) != RELKIND_RELATION ||
407 matchOid(&includeTables, relOid) == false)
411 case OATT_PERSISTENT:
414 * skip analyze if relation's type doesn't not match
415 * online_analyze_table_type
417 if ((online_analyze_table_type & reltype) == 0 ||
418 matchOid(&excludeTables, relOid) == true)
420 if (matchOid(&includeTables, relOid) == false)
427 * Do not store data about persistent table in local memory because we
428 * could not track changes of them: they could be changed by another
429 * backends. So always get a pgstat table entry.
431 if (reltype == OATT_TEMPORARY)
432 rstat = hash_search(relstats, &relOid, HASH_ENTER, &found);
434 rstat = &dummyrstat; /* found == false for following if */
438 MemSet(rstat, 0, sizeof(*rstat));
439 rstat->tableid = relOid;
443 Assert(rstat->tableid == relOid);
445 elog(NOTICE,"makeAnalyze op:%d %u", operation, naffected);
447 if (operation != CK_TRUNCATE &&
448 (found == false || rstat->rereadStat == true))
451 rstat->rereadStat = false;
453 tabentry = pgstat_fetch_stat_tabentry(relOid);
457 rstat->n_tuples = tabentry->n_dead_tuples + tabentry->n_live_tuples;
458 rstat->changes_since_analyze =
459 #if PG_VERSION_NUM >= 90000
460 tabentry->changes_since_analyze;
462 tabentry->n_live_tuples + tabentry->n_dead_tuples -
463 tabentry->last_anl_tuples;
465 rstat->autovac_analyze_timestamp =
466 tabentry->autovac_analyze_timestamp;
467 rstat->analyze_timestamp = tabentry->analyze_timestamp;
472 /* force analyze from after truncate */
473 operation == CK_TRUNCATE || (
474 /* do not analyze too often, if both stamps are exceeded the go */
475 TimestampDifferenceExceeds(rstat->analyze_timestamp, now, online_analyze_min_interval) &&
476 TimestampDifferenceExceeds(rstat->autovac_analyze_timestamp, now, online_analyze_min_interval) &&
477 /* do not analyze too small tables */
478 rstat->n_tuples + rstat->changes_since_analyze + naffected > online_analyze_lower_limit &&
479 /* be in sync with relation_needs_vacanalyze */
480 ((double)(rstat->changes_since_analyze + naffected)) >=
481 online_analyze_scale_factor * ((double)rstat->n_tuples) +
482 (double)online_analyze_threshold))
484 #if PG_VERSION_NUM < 90500
487 VacuumParams vacstmt;
489 TimestampTz startStamp, endStamp;
492 memset(&startStamp, 0, sizeof(startStamp)); /* keep compiler quiet */
494 memset(&vacstmt, 0, sizeof(vacstmt));
496 vacstmt.freeze_min_age = -1;
497 vacstmt.freeze_table_age = -1; /* ??? */
499 #if PG_VERSION_NUM < 90500
500 vacstmt.type = T_VacuumStmt;
501 vacstmt.relation = NULL;
502 vacstmt.va_cols = NIL;
503 #if PG_VERSION_NUM >= 90000
504 vacstmt.options = VACOPT_ANALYZE;
505 if (online_analyze_verbose)
506 vacstmt.options |= VACOPT_VERBOSE;
508 vacstmt.vacuum = vacstmt.full = false;
509 vacstmt.analyze = true;
510 vacstmt.verbose = online_analyze_verbose;
513 vacstmt.multixact_freeze_min_age = -1;
514 vacstmt.multixact_freeze_table_age = -1;
515 vacstmt.log_min_duration = -1;
518 if (online_analyze_verbose)
519 startStamp = GetCurrentTimestamp();
522 #if PG_VERSION_NUM < 90500
524 #if PG_VERSION_NUM >= 90018
527 , GetAccessStrategy(BAS_VACUUM)
528 #if (PG_VERSION_NUM >= 90000) && (PG_VERSION_NUM < 90004)
532 makeRangeVarFromOid(relOid),
533 VACOPT_ANALYZE | ((online_analyze_verbose) ? VACOPT_VERBOSE : 0),
534 &vacstmt, NULL, true, GetAccessStrategy(BAS_VACUUM)
538 if (online_analyze_verbose)
543 endStamp = GetCurrentTimestamp();
544 TimestampDifference(startStamp, endStamp, &secs, µsecs);
545 elog(INFO, "analyze \"%s\" took %.02f seconds",
546 get_rel_name(relOid),
547 ((double)secs) + ((double)microsecs)/1.0e6);
550 rstat->autovac_analyze_timestamp = now;
551 rstat->changes_since_analyze = 0;
556 rstat->n_tuples += naffected;
557 rstat->rereadStat = false;
560 rstat->n_tuples += naffected;
561 rstat->rereadStat = true;
564 rstat->rereadStat = true;
568 rstat->rereadStat = false;
574 /* update last analyze timestamp in local memory of backend */
577 tabentry->analyze_timestamp = now;
578 tabentry->changes_since_analyze = 0;
581 /* force reload stat for new table */
583 pgstat_clear_snapshot();
588 #if PG_VERSION_NUM >= 90000
590 tabentry->changes_since_analyze += naffected;
595 rstat->changes_since_analyze += naffected;
596 rstat->n_tuples += naffected;
599 rstat->changes_since_analyze += 2 * naffected;
600 rstat->n_tuples += naffected;
602 rstat->changes_since_analyze += naffected;
605 rstat->changes_since_analyze = rstat->n_tuples;
613 /* Reset local cache if we are over limit */
614 if (hash_get_num_entries(relstats) > online_analyze_capacity_threshold)
618 extern PGDLLIMPORT void onlineAnalyzeHooker(QueryDesc *queryDesc);
620 onlineAnalyzeHooker(QueryDesc *queryDesc)
622 int64 naffected = -1;
624 if (queryDesc->estate)
625 naffected = queryDesc->estate->es_processed;
627 if (online_analyze_enable && queryDesc->plannedstmt &&
628 (queryDesc->operation == CMD_INSERT ||
629 queryDesc->operation == CMD_UPDATE ||
630 queryDesc->operation == CMD_DELETE
631 #if PG_VERSION_NUM < 90200
632 || (queryDesc->operation == CMD_SELECT &&
633 queryDesc->plannedstmt->intoClause)
637 #if PG_VERSION_NUM < 90200
638 if (queryDesc->operation == CMD_SELECT)
640 Oid relOid = RangeVarGetRelid(queryDesc->plannedstmt->intoClause->rel, true);
642 makeAnalyze(relOid, queryDesc->operation, naffected);
646 if (queryDesc->plannedstmt->resultRelations &&
647 queryDesc->plannedstmt->rtable)
651 foreach(l, queryDesc->plannedstmt->resultRelations)
653 int n = lfirst_int(l);
654 RangeTblEntry *rte = list_nth(queryDesc->plannedstmt->rtable, n-1);
656 if (rte->rtekind == RTE_RELATION)
657 makeAnalyze(rte->relid, (CmdKind)queryDesc->operation, naffected);
662 if (oldExecutorEndHook)
663 oldExecutorEndHook(queryDesc);
665 standard_ExecutorEnd(queryDesc);
668 #if PG_VERSION_NUM >= 90200
670 onlineAnalyzeHookerUtility(
671 #if PG_VERSION_NUM >= 100000
676 const char *queryString,
677 #if PG_VERSION_NUM >= 90300
678 ProcessUtilityContext context, ParamListInfo params,
679 #if PG_VERSION_NUM >= 100000
680 QueryEnvironment *queryEnv,
683 ParamListInfo params, bool isTopLevel,
685 DestReceiver *dest, char *completionTag) {
686 List *tblnames = NIL;
687 CmdKind op = CK_INSERT;
688 #if PG_VERSION_NUM >= 100000
689 Node *parsetree = NULL;
691 if (pstmt->commandType == CMD_UTILITY)
692 parsetree = pstmt->utilityStmt;
695 if (parsetree && online_analyze_enable)
697 if (IsA(parsetree, CreateTableAsStmt) &&
698 ((CreateTableAsStmt*)parsetree)->into)
701 list_make1((RangeVar*)copyObject(((CreateTableAsStmt*)parsetree)->into->rel));
704 else if (IsA(parsetree, TruncateStmt))
706 tblnames = list_copy(((TruncateStmt*)parsetree)->relations);
711 #if PG_VERSION_NUM >= 100000
712 #define parsetree pstmt
715 if (oldProcessUtilityHook)
716 oldProcessUtilityHook(parsetree, queryString,
717 #if PG_VERSION_NUM >= 90300
719 #if PG_VERSION_NUM >= 100000
725 dest, completionTag);
727 standard_ProcessUtility(parsetree, queryString,
728 #if PG_VERSION_NUM >= 90300
730 #if PG_VERSION_NUM >= 100000
736 dest, completionTag);
738 #if PG_VERSION_NUM >= 100000
747 RangeVar *tblname = (RangeVar*)lfirst(l);
748 Oid tblOid = RangeVarGetRelid(tblname, NoLock, true);
750 makeAnalyze(tblOid, op, -1);
762 MemSet(&hash_ctl, 0, sizeof(hash_ctl));
764 hash_ctl.hash = oid_hash;
765 flags |= HASH_FUNCTION;
767 if (onlineAnalyzeMemoryContext)
769 Assert(relstats != NULL);
770 MemoryContextReset(onlineAnalyzeMemoryContext);
774 Assert(relstats == NULL);
775 onlineAnalyzeMemoryContext =
776 AllocSetContextCreate(CacheMemoryContext,
777 "online_analyze storage context",
778 #if PG_VERSION_NUM < 90600
779 ALLOCSET_DEFAULT_MINSIZE,
780 ALLOCSET_DEFAULT_INITSIZE,
781 ALLOCSET_DEFAULT_MAXSIZE
783 ALLOCSET_DEFAULT_SIZES
788 hash_ctl.hcxt = onlineAnalyzeMemoryContext;
789 flags |= HASH_CONTEXT;
791 hash_ctl.keysize = sizeof(Oid);
793 hash_ctl.entrysize = sizeof(OnlineAnalyzeTableStat);
796 relstats = hash_create("online_analyze storage", 1024, &hash_ctl, flags);
805 oldExecutorEndHook = ExecutorEnd_hook;
807 ExecutorEnd_hook = onlineAnalyzeHooker;
809 #if PG_VERSION_NUM >= 90200
810 oldProcessUtilityHook = ProcessUtility_hook;
812 ProcessUtility_hook = onlineAnalyzeHookerUtility;
816 DefineCustomBoolVariable(
817 "online_analyze.enable",
818 "Enable on-line analyze",
819 "Enables analyze of table directly after insert/update/delete/select into",
820 &online_analyze_enable,
821 #if PG_VERSION_NUM >= 80400
822 online_analyze_enable,
825 #if PG_VERSION_NUM >= 80400
827 #if PG_VERSION_NUM >= 90100
835 DefineCustomBoolVariable(
836 "online_analyze.verbose",
837 "Verbosity of on-line analyze",
838 "Make ANALYZE VERBOSE after table's changes",
839 &online_analyze_verbose,
840 #if PG_VERSION_NUM >= 80400
841 online_analyze_verbose,
844 #if PG_VERSION_NUM >= 80400
846 #if PG_VERSION_NUM >= 90100
854 DefineCustomRealVariable(
855 "online_analyze.scale_factor",
856 "fraction of table size to start on-line analyze",
857 "fraction of table size to start on-line analyze",
858 &online_analyze_scale_factor,
859 #if PG_VERSION_NUM >= 80400
860 online_analyze_scale_factor,
865 #if PG_VERSION_NUM >= 80400
867 #if PG_VERSION_NUM >= 90100
875 DefineCustomIntVariable(
876 "online_analyze.threshold",
877 "min number of row updates before on-line analyze",
878 "min number of row updates before on-line analyze",
879 &online_analyze_threshold,
880 #if PG_VERSION_NUM >= 80400
881 online_analyze_threshold,
886 #if PG_VERSION_NUM >= 80400
888 #if PG_VERSION_NUM >= 90100
896 DefineCustomIntVariable(
897 "online_analyze.capacity_threshold",
898 "Max local cache table capacity",
899 "Max local cache table capacity",
900 &online_analyze_capacity_threshold,
901 #if PG_VERSION_NUM >= 80400
902 online_analyze_capacity_threshold,
907 #if PG_VERSION_NUM >= 80400
909 #if PG_VERSION_NUM >= 90100
917 DefineCustomRealVariable(
918 "online_analyze.min_interval",
919 "minimum time interval between analyze call (in milliseconds)",
920 "minimum time interval between analyze call (in milliseconds)",
921 &online_analyze_min_interval,
922 #if PG_VERSION_NUM >= 80400
923 online_analyze_min_interval,
928 #if PG_VERSION_NUM >= 80400
930 #if PG_VERSION_NUM >= 90100
938 DefineCustomEnumVariable(
939 "online_analyze.table_type",
940 "Type(s) of table for online analyze: all(default), persistent, temporary, none",
942 &online_analyze_table_type,
943 #if PG_VERSION_NUM >= 80400
944 online_analyze_table_type,
946 online_analyze_table_type_options,
948 #if PG_VERSION_NUM >= 80400
950 #if PG_VERSION_NUM >= 90100
958 DefineCustomStringVariable(
959 "online_analyze.exclude_tables",
960 "List of tables which will not online analyze",
962 &excludeTables.tableStr,
963 #if PG_VERSION_NUM >= 80400
968 #if PG_VERSION_NUM >= 90100
977 DefineCustomStringVariable(
978 "online_analyze.include_tables",
979 "List of tables which will online analyze",
981 &includeTables.tableStr,
982 #if PG_VERSION_NUM >= 80400
987 #if PG_VERSION_NUM >= 90100
996 DefineCustomIntVariable(
997 "online_analyze.lower_limit",
998 "min number of rows in table to analyze",
999 "min number of rows in table to analyze",
1000 &online_analyze_lower_limit,
1001 #if PG_VERSION_NUM >= 80400
1002 online_analyze_lower_limit,
1007 #if PG_VERSION_NUM >= 80400
1009 #if PG_VERSION_NUM >= 90100
1019 void _PG_fini(void);
1023 ExecutorEnd_hook = oldExecutorEndHook;
1024 #if PG_VERSION_NUM >= 90200
1025 ProcessUtility_hook = oldProcessUtilityHook;
1028 if (excludeTables.tables)
1029 free(excludeTables.tables);
1030 if (includeTables.tables)
1031 free(includeTables.tables);
1033 excludeTables.tables = includeTables.tables = NULL;
1034 excludeTables.nTables = includeTables.nTables = 0;