2 * Copyright (c) 2011 Teodor Sigaev <teodor@sigaev.ru>
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. Neither the name of the author nor the names of any co-contributors
14 * may be used to endorse or promote products derived from this software
15 * without specific prior written permission.
17 * THIS SOFTWARE IS PROVIDED BY CONTRIBUTORS ``AS IS'' AND ANY EXPRESS
18 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL CONTRIBUTORS BE LIABLE FOR ANY
21 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 #include "catalog/namespace.h"
34 #include "commands/vacuum.h"
35 #include "executor/executor.h"
36 #include "nodes/nodes.h"
37 #include "nodes/parsenodes.h"
38 #include "storage/bufmgr.h"
39 #include "utils/builtins.h"
40 #include "utils/hsearch.h"
41 #include "utils/memutils.h"
42 #include "utils/lsyscache.h"
43 #include "utils/guc.h"
44 #if PG_VERSION_NUM >= 90200
45 #include "catalog/pg_class.h"
46 #include "nodes/primnodes.h"
47 #include "tcop/utility.h"
48 #include "utils/rel.h"
49 #include "utils/relcache.h"
50 #include "utils/timestamp.h"
51 #if PG_VERSION_NUM >= 90500
52 #include "nodes/makefuncs.h"
56 #ifdef PG_MODULE_MAGIC
60 static bool online_analyze_enable = true;
61 static bool online_analyze_verbose = true;
62 static double online_analyze_scale_factor = 0.1;
63 static int online_analyze_threshold = 50;
64 static double online_analyze_min_interval = 10000;
66 static ExecutorEnd_hook_type oldExecutorEndHook = NULL;
67 #if PG_VERSION_NUM >= 90200
68 static ProcessUtility_hook_type oldProcessUtilityHook = NULL;
74 OATT_PERSISTENT = 0x01,
75 OATT_TEMPORARY = 0x02,
77 } OnlineAnalyzeTableType;
79 static const struct config_enum_entry online_analyze_table_type_options[] =
81 {"all", OATT_ALL, false},
82 {"persistent", OATT_PERSISTENT, false},
83 {"temporary", OATT_TEMPORARY, false},
84 {"none", OATT_NONE, false},
88 static int online_analyze_table_type = (int)OATT_ALL;
90 typedef struct TableList {
96 static TableList excludeTables = {0, NULL, NULL};
97 static TableList includeTables = {0, NULL, NULL};
99 typedef struct OnlineAnalyzeTableStat {
102 PgStat_Counter n_tuples;
103 PgStat_Counter changes_since_analyze;
104 TimestampTz autovac_analyze_timestamp;
105 TimestampTz analyze_timestamp;
106 } OnlineAnalyzeTableStat;
108 static HTAB *relstats = NULL;
111 oid_cmp(const void *a, const void *b)
113 if (*(Oid*)a == *(Oid*)b)
115 return (*(Oid*)a > *(Oid*)b) ? 1 : -1;
119 tableListAssign(const char * newval, bool doit, TableList *tbl)
128 rawname = pstrdup(newval);
130 if (!SplitIdentifierString(rawname, ',', &namelist))
135 nOids = list_length(namelist);
136 newOids = malloc(sizeof(Oid) * (nOids+1));
138 elog(ERROR,"could not allocate %d bytes",
139 (int)(sizeof(Oid) * (nOids+1)));
144 char *curname = (char *) lfirst(l);
145 #if PG_VERSION_NUM >= 90200
146 Oid relOid = RangeVarGetRelid(makeRangeVarFromNameList(
147 stringToQualifiedNameList(curname)), NoLock, true);
149 Oid relOid = RangeVarGetRelid(makeRangeVarFromNameList(
150 stringToQualifiedNameList(curname)), true);
153 if (relOid == InvalidOid)
155 #if PG_VERSION_NUM >= 90100
158 elog(WARNING,"'%s' does not exist", curname);
161 else if ( get_rel_relkind(relOid) != RELKIND_RELATION )
163 #if PG_VERSION_NUM >= 90100
166 elog(WARNING,"'%s' is not an table", curname);
171 newOids[i++] = relOid;
180 tbl->tables = newOids;
181 if (tbl->nTables > 1)
182 qsort(tbl->tables, tbl->nTables, sizeof(tbl->tables[0]), oid_cmp);
198 #if PG_VERSION_NUM >= 90100
200 excludeTablesCheck(char **newval, void **extra, GucSource source)
204 val = (char*)tableListAssign(*newval, false, &excludeTables);
216 excludeTablesAssign(const char *newval, void *extra)
218 tableListAssign(newval, true, &excludeTables);
222 includeTablesCheck(char **newval, void **extra, GucSource source)
226 val = (char*)tableListAssign(*newval, false, &includeTables);
238 includeTablesAssign(const char *newval, void *extra)
240 tableListAssign(newval, true, &excludeTables);
243 #else /* PG_VERSION_NUM < 90100 */
246 excludeTablesAssign(const char * newval, bool doit, GucSource source)
248 return tableListAssign(newval, doit, &excludeTables);
252 includeTablesAssign(const char * newval, bool doit, GucSource source)
254 return tableListAssign(newval, doit, &includeTables);
260 tableListShow(TableList *tbl)
266 len = 1 /* \0 */ + tbl->nTables * (2 * NAMEDATALEN + 2 /* ', ' */ + 1 /* . */);
267 ptr = val = palloc(len);
269 for(i=0; i<tbl->nTables; i++)
271 char *relname = get_rel_name(tbl->tables[i]);
272 Oid nspOid = get_rel_namespace(tbl->tables[i]);
273 char *nspname = get_namespace_name(nspOid);
275 if ( relname == NULL || nspOid == InvalidOid || nspname == NULL )
278 ptr += snprintf(ptr, len - (ptr - val), "%s%s.%s",
287 excludeTablesShow(void)
289 return tableListShow(&excludeTables);
293 includeTablesShow(void)
295 return tableListShow(&includeTables);
299 matchOid(TableList *tbl, Oid oid)
301 Oid *StopLow = tbl->tables,
302 *StopHigh = tbl->tables + tbl->nTables,
305 /* Loop invariant: StopLow <= val < StopHigh */
306 while (StopLow < StopHigh)
308 StopMiddle = StopLow + ((StopHigh - StopLow) >> 1);
310 if (*StopMiddle == oid)
312 else if (*StopMiddle < oid)
313 StopLow = StopMiddle + 1;
315 StopHigh = StopMiddle;
321 #if PG_VERSION_NUM >= 90500
323 makeRangeVarFromOid(Oid relOid)
326 get_namespace_name(get_rel_namespace(relOid)),
327 get_rel_name(relOid),
335 makeAnalyze(Oid relOid, CmdType operation, int32 naffected)
337 TimestampTz now = GetCurrentTimestamp();
339 OnlineAnalyzeTableType reltype;
342 OnlineAnalyzeTableStat *rstat;
343 PgStat_StatTabEntry *tabentry = NULL;
345 if (relOid == InvalidOid)
349 /* return if there is not changes */
351 else if (naffected < 0)
352 /* number if affected rows is unknown */
355 rel = RelationIdGetRelation(relOid);
356 if (rel->rd_rel->relkind != RELKIND_RELATION)
363 #if PG_VERSION_NUM >= 90100
364 (rel->rd_rel->relpersistence == RELPERSISTENCE_TEMP)
366 (rel->rd_istemp || rel->rd_islocaltemp)
368 ? OATT_TEMPORARY : OATT_PERSISTENT;
373 * includeTables overwrites excludeTables
375 switch(online_analyze_table_type)
378 if (get_rel_relkind(relOid) != RELKIND_RELATION ||
379 (matchOid(&excludeTables, relOid) == true &&
380 matchOid(&includeTables, relOid) == false))
384 if (get_rel_relkind(relOid) != RELKIND_RELATION ||
385 matchOid(&includeTables, relOid) == false)
389 case OATT_PERSISTENT:
392 * skip analyze if relation's type doesn't not match
393 * online_analyze_table_type
395 if ((online_analyze_table_type & reltype) == 0 ||
396 matchOid(&excludeTables, relOid) == true)
398 if (matchOid(&includeTables, relOid) == false)
404 rstat = hash_search(relstats, &relOid, HASH_ENTER, &found);
406 if (found == false || rstat->rereadStat == true || naffected == 0)
411 MemSet(rstat, 0, sizeof(*rstat));
412 rstat->tableid = relOid;
414 Assert(rstat->tableid == relOid);
416 tabentry = pgstat_fetch_stat_tabentry(relOid);
420 rstat->n_tuples = tabentry->n_dead_tuples + tabentry->n_live_tuples;
421 rstat->changes_since_analyze =
422 #if PG_VERSION_NUM >= 90000
423 tabentry->changes_since_analyze;
425 tabentry->n_live_tuples + tabentry->n_dead_tuples -
426 tabentry->last_anl_tuples;
428 rstat->autovac_analyze_timestamp =
429 tabentry->autovac_analyze_timestamp;
430 rstat->analyze_timestamp = tabentry->analyze_timestamp;
431 rstat->rereadStat = false;
436 rstat->rereadStat = true;
441 /* do not analyze too often, if both stamps are exceeded the go */
442 TimestampDifferenceExceeds(rstat->analyze_timestamp, now, online_analyze_min_interval) &&
443 TimestampDifferenceExceeds(rstat->autovac_analyze_timestamp, now, online_analyze_min_interval) &&
444 /* be in sync with relation_needs_vacanalyze */
445 ((double)(rstat->changes_since_analyze + naffected)) >=
446 online_analyze_scale_factor * ((double)rstat->n_tuples) +
447 (double)online_analyze_threshold))
449 #if PG_VERSION_NUM < 90500
452 VacuumParams vacstmt;
454 TimestampTz startStamp, endStamp;
457 memset(&startStamp, 0, sizeof(startStamp)); /* keep compiler quiet */
459 memset(&vacstmt, 0, sizeof(vacstmt));
461 vacstmt.freeze_min_age = -1;
462 vacstmt.freeze_table_age = -1; /* ??? */
464 #if PG_VERSION_NUM < 90500
465 vacstmt.type = T_VacuumStmt;
466 vacstmt.relation = NULL;
467 vacstmt.va_cols = NIL;
468 #if PG_VERSION_NUM >= 90000
469 vacstmt.options = VACOPT_ANALYZE;
470 if (online_analyze_verbose)
471 vacstmt.options |= VACOPT_VERBOSE;
473 vacstmt.vacuum = vacstmt.full = false;
474 vacstmt.analyze = true;
475 vacstmt.verbose = online_analyze_verbose;
478 vacstmt.multixact_freeze_min_age = -1;
479 vacstmt.multixact_freeze_table_age = -1;
480 vacstmt.log_min_duration = -1;
483 if (online_analyze_verbose)
484 startStamp = GetCurrentTimestamp();
487 #if PG_VERSION_NUM < 90500
489 #if PG_VERSION_NUM >= 90018
492 , GetAccessStrategy(BAS_VACUUM)
493 #if (PG_VERSION_NUM >= 90000) && (PG_VERSION_NUM < 90004)
497 makeRangeVarFromOid(relOid),
498 VACOPT_ANALYZE | ((online_analyze_verbose) ? VACOPT_VERBOSE : 0),
499 &vacstmt, NULL, true, GetAccessStrategy(BAS_VACUUM)
503 if (online_analyze_verbose)
508 endStamp = GetCurrentTimestamp();
509 TimestampDifference(startStamp, endStamp, &secs, µsecs);
510 elog(INFO, "analyze \"%s\" took %.02f seconds",
511 get_rel_name(relOid),
512 ((double)secs) + ((double)microsecs)/1.0e6);
515 rstat->autovac_analyze_timestamp = now;
516 rstat->changes_since_analyze = 0;
517 rstat->rereadStat = true;
519 /* update last analyze timestamp in local memory of backend */
521 tabentry->analyze_timestamp = now;
524 /* force reload stat for new table */
526 pgstat_clear_snapshot();
531 #if PG_VERSION_NUM >= 90000
532 if (tabentry != NULL)
533 tabentry->changes_since_analyze += naffected;
535 rstat->changes_since_analyze += naffected;
539 extern PGDLLIMPORT void onlineAnalyzeHooker(QueryDesc *queryDesc);
541 onlineAnalyzeHooker(QueryDesc *queryDesc)
543 uint32 naffected = -1;
545 if (queryDesc->estate)
546 naffected = queryDesc->estate->es_processed;
548 if (online_analyze_enable && queryDesc->plannedstmt &&
549 (queryDesc->operation == CMD_INSERT ||
550 queryDesc->operation == CMD_UPDATE ||
551 queryDesc->operation == CMD_DELETE
552 #if PG_VERSION_NUM < 90200
553 || (queryDesc->operation == CMD_SELECT &&
554 queryDesc->plannedstmt->intoClause)
558 #if PG_VERSION_NUM < 90200
559 if (queryDesc->operation == CMD_SELECT)
561 Oid relOid = RangeVarGetRelid(queryDesc->plannedstmt->intoClause->rel, true);
563 makeAnalyze(relOid, queryDesc->operation, naffected);
567 if (queryDesc->plannedstmt->resultRelations &&
568 queryDesc->plannedstmt->rtable)
572 foreach(l, queryDesc->plannedstmt->resultRelations)
574 int n = lfirst_int(l);
575 RangeTblEntry *rte = list_nth(queryDesc->plannedstmt->rtable, n-1);
577 if (rte->rtekind == RTE_RELATION)
578 makeAnalyze(rte->relid, queryDesc->operation, naffected);
583 if (oldExecutorEndHook)
584 oldExecutorEndHook(queryDesc);
586 standard_ExecutorEnd(queryDesc);
589 #if PG_VERSION_NUM >= 90200
591 onlineAnalyzeHookerUtility(Node *parsetree, const char *queryString,
592 #if PG_VERSION_NUM >= 90300
593 ProcessUtilityContext context, ParamListInfo params,
595 ParamListInfo params, bool isTopLevel,
597 DestReceiver *dest, char *completionTag) {
598 RangeVar *tblname = NULL;
600 if (IsA(parsetree, CreateTableAsStmt) && ((CreateTableAsStmt*)parsetree)->into)
601 tblname = (RangeVar*)copyObject(((CreateTableAsStmt*)parsetree)->into->rel);
603 if (oldProcessUtilityHook)
604 oldProcessUtilityHook(parsetree, queryString,
605 #if PG_VERSION_NUM >= 90300
610 dest, completionTag);
612 standard_ProcessUtility(parsetree, queryString,
613 #if PG_VERSION_NUM >= 90300
618 dest, completionTag);
621 Oid tblOid = RangeVarGetRelid(tblname, NoLock, true);
623 makeAnalyze(tblOid, CMD_INSERT, -1);
634 MemSet(&hash_ctl, 0, sizeof(hash_ctl));
636 hash_ctl.hash = oid_hash;
637 flags |= HASH_FUNCTION;
639 hash_ctl.hcxt = AllocSetContextCreate(CacheMemoryContext,
640 "online_analyze storage context",
641 ALLOCSET_DEFAULT_SIZES);
642 flags |= HASH_CONTEXT;
644 hash_ctl.keysize = sizeof(Oid);
646 hash_ctl.entrysize = sizeof(OnlineAnalyzeTableStat);
649 relstats = hash_create("online_analyze storage", 1024, &hash_ctl, flags);
658 oldExecutorEndHook = ExecutorEnd_hook;
660 ExecutorEnd_hook = onlineAnalyzeHooker;
662 #if PG_VERSION_NUM >= 90200
663 oldProcessUtilityHook = ProcessUtility_hook;
665 ProcessUtility_hook = onlineAnalyzeHookerUtility;
669 DefineCustomBoolVariable(
670 "online_analyze.enable",
671 "Enable on-line analyze",
672 "Enables analyze of table directly after insert/update/delete/select into",
673 &online_analyze_enable,
674 #if PG_VERSION_NUM >= 80400
675 online_analyze_enable,
678 #if PG_VERSION_NUM >= 80400
680 #if PG_VERSION_NUM >= 90100
688 DefineCustomBoolVariable(
689 "online_analyze.verbose",
690 "Verbosity of on-line analyze",
691 "Make ANALYZE VERBOSE after table's changes",
692 &online_analyze_verbose,
693 #if PG_VERSION_NUM >= 80400
694 online_analyze_verbose,
697 #if PG_VERSION_NUM >= 80400
699 #if PG_VERSION_NUM >= 90100
707 DefineCustomRealVariable(
708 "online_analyze.scale_factor",
709 "fraction of table size to start on-line analyze",
710 "fraction of table size to start on-line analyze",
711 &online_analyze_scale_factor,
712 #if PG_VERSION_NUM >= 80400
713 online_analyze_scale_factor,
718 #if PG_VERSION_NUM >= 80400
720 #if PG_VERSION_NUM >= 90100
728 DefineCustomIntVariable(
729 "online_analyze.threshold",
730 "min number of row updates before on-line analyze",
731 "min number of row updates before on-line analyze",
732 &online_analyze_threshold,
733 #if PG_VERSION_NUM >= 80400
734 online_analyze_threshold,
739 #if PG_VERSION_NUM >= 80400
741 #if PG_VERSION_NUM >= 90100
749 DefineCustomRealVariable(
750 "online_analyze.min_interval",
751 "minimum time interval between analyze call (in milliseconds)",
752 "minimum time interval between analyze call (in milliseconds)",
753 &online_analyze_min_interval,
754 #if PG_VERSION_NUM >= 80400
755 online_analyze_min_interval,
760 #if PG_VERSION_NUM >= 80400
762 #if PG_VERSION_NUM >= 90100
770 DefineCustomEnumVariable(
771 "online_analyze.table_type",
772 "Type(s) of table for online analyze: all(default), persistent, temporary, none",
774 &online_analyze_table_type,
775 #if PG_VERSION_NUM >= 80400
776 online_analyze_table_type,
778 online_analyze_table_type_options,
780 #if PG_VERSION_NUM >= 80400
782 #if PG_VERSION_NUM >= 90100
790 DefineCustomStringVariable(
791 "online_analyze.exclude_tables",
792 "List of tables which will not online analyze",
794 &excludeTables.tableStr,
795 #if PG_VERSION_NUM >= 80400
800 #if PG_VERSION_NUM >= 90100
809 DefineCustomStringVariable(
810 "online_analyze.include_tables",
811 "List of tables which will online analyze",
813 &includeTables.tableStr,
814 #if PG_VERSION_NUM >= 80400
819 #if PG_VERSION_NUM >= 90100
833 ExecutorEnd_hook = oldExecutorEndHook;
834 #if PG_VERSION_NUM >= 90200
835 ProcessUtility_hook = oldProcessUtilityHook;
838 if (excludeTables.tables)
839 free(excludeTables.tables);
840 if (includeTables.tables)
841 free(includeTables.tables);
843 excludeTables.tables = includeTables.tables = NULL;
844 excludeTables.nTables = includeTables.nTables = 0;