online_analyze.c

   1 /*
   2  * Copyright (c) 2011 Teodor Sigaev <teodor@sigaev.ru>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *        notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *        notice, this list of conditions and the following disclaimer in the
  12  *        documentation and/or other materials provided with the distribution.
  13  * 3. Neither the name of the author nor the names of any co-contributors
  14  *        may be used to endorse or promote products derived from this software
  15  *        without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY CONTRIBUTORS ``AS IS'' AND ANY EXPRESS
  18  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  19  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED. IN NO EVENT SHALL CONTRIBUTORS BE LIABLE FOR ANY
  21  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
  23  * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  25  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  26  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  27  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  */
  29
  30 #include "postgres.h"
  31
  32 #include "pgstat.h"
  33 #include "catalog/namespace.h"
  34 #include "commands/vacuum.h"
  35 #include "executor/executor.h"
  36 #include "nodes/nodes.h"
  37 #include "nodes/parsenodes.h"
  38 #include "storage/bufmgr.h"
  39 #include "utils/builtins.h"
  40 #include "utils/lsyscache.h"
  41 #include "utils/guc.h"
  42 #if PG_VERSION_NUM >= 90200
  43 #include "catalog/pg_class.h"
  44 #include "utils/rel.h"
  45 #include "utils/relcache.h"
  46 #include "utils/timestamp.h"
  47 #endif
  48
  49 #ifdef PG_MODULE_MAGIC
  50 PG_MODULE_MAGIC;
  51 #endif
  52
  53 static bool online_analyze_enable = true;
  54 static bool online_analyze_verbose = true;
  55 static double online_analyze_scale_factor = 0.1;
  56 static int online_analyze_threshold = 50;
  57 static double online_analyze_min_interval = 10000;
  58
  59 typedef enum
  60 {
  61         OATT_ALL                = 0x03,
  62         OATT_PERSISTENT = 0x01,
  63         OATT_TEMPORARY  = 0x02,
  64         OATT_NONE               = 0x00
  65 } OnlineAnalyzeTableType;
  66
  67 static const struct config_enum_entry online_analyze_table_type_options[] =
  68 {
  69         {"all", OATT_ALL, false},
  70         {"persistent", OATT_PERSISTENT, false},
  71         {"temporary", OATT_TEMPORARY, false},
  72         {"none", OATT_NONE, false},
  73         {NULL, 0, false},
  74 };
  75
  76 static int online_analyze_table_type = (int)OATT_ALL;
  77
  78 typedef struct TableList {
  79         int             nTables;
  80         Oid             *tables;
  81         char    *tableStr;
  82 } TableList;
  83
  84 static TableList excludeTables = {0, NULL, NULL};
  85 static TableList includeTables = {0, NULL, NULL};
  86
  87 static int
  88 oid_cmp(const void *a, const void *b)
  89 {
  90         if (*(Oid*)a == *(Oid*)b)
  91                 return 0;
  92         return (*(Oid*)a > *(Oid*)b) ? 1 : -1;
  93 }
  94
  95 static const char *
  96 tableListAssign(const char * newval, bool doit, TableList *tbl)
  97 {
  98         char       *rawname;
  99         List       *namelist;
 100         ListCell   *l;
 101         Oid         *newOids = NULL;
 102         int         nOids = 0,
 103                                 i = 0;
 104
 105         rawname = pstrdup(newval);
 106
 107         if (!SplitIdentifierString(rawname, ',', &namelist))
 108                 goto cleanup;
 109
 110         if (doit)
 111         {
 112                 nOids = list_length(namelist);
 113                 newOids = malloc(sizeof(Oid) * (nOids+1));
 114                 if (!newOids)
 115                         elog(ERROR,"could not allocate %d bytes", (int)(sizeof(Oid) * (nOids+1)));
 116         }
 117
 118         foreach(l, namelist)
 119         {
 120                 char        *curname = (char *) lfirst(l);
 121                 Oid         relOid = RangeVarGetRelid(makeRangeVarFromNameList(stringToQualifiedNameList(curname)),
 122 #if PG_VERSION_NUM >= 90200
 123                                                                                                 NoLock,
 124 #endif
 125                                                                                                 true);
 126
 127                 if (relOid == InvalidOid)
 128                 {
 129 #if PG_VERSION_NUM >= 90100
 130                         if (doit == false)
 131 #endif
 132                         elog(WARNING,"'%s' does not exist", curname);
 133                         continue;
 134                 }
 135                 else if ( get_rel_relkind(relOid) != RELKIND_RELATION )
 136                 {
 137 #if PG_VERSION_NUM >= 90100
 138                         if (doit == false)
 139 #endif
 140                                 elog(WARNING,"'%s' is not an table", curname);
 141                         continue;
 142                 }
 143                 else if (doit)
 144                 {
 145                         newOids[i++] = relOid;
 146                 }
 147         }
 148
 149         if (doit)
 150         {
 151                 tbl->nTables = i;
 152                 if (tbl->tables)
 153                         free(tbl->tables);
 154                 tbl->tables = newOids;
 155                 if (tbl->nTables > 1)
 156                         qsort(tbl->tables, tbl->nTables, sizeof(tbl->tables[0]), oid_cmp);
 157         }
 158
 159         pfree(rawname);
 160         list_free(namelist);
 161
 162         return newval;
 163
 164 cleanup:
 165         if (newOids)
 166                 free(newOids);
 167         pfree(rawname);
 168         list_free(namelist);
 169         return NULL;
 170 }
 171
 172 #if PG_VERSION_NUM >= 90100
 173 static bool
 174 excludeTablesCheck(char **newval, void **extra, GucSource source)
 175 {
 176         char *val;
 177
 178         val = (char*)tableListAssign(*newval, false, &excludeTables);
 179
 180         if (val)
 181         {
 182                 *newval = val;
 183                 return true;
 184         }
 185
 186         return false;
 187 }
 188
 189 static void
 190 excludeTablesAssign(const char *newval, void *extra)
 191 {
 192         tableListAssign(newval, true, &excludeTables);
 193 }
 194
 195 static bool
 196 includeTablesCheck(char **newval, void **extra, GucSource source)
 197 {
 198         char *val;
 199
 200         val = (char*)tableListAssign(*newval, false, &includeTables);
 201
 202         if (val)
 203         {
 204                 *newval = val;
 205                 return true;
 206         }
 207
 208         return false;
 209 }
 210
 211 static void
 212 includeTablesAssign(const char *newval, void *extra)
 213 {
 214         tableListAssign(newval, true, &excludeTables);
 215 }
 216
 217 #else /* PG_VERSION_NUM < 90100 */
 218
 219 static const char *
 220 excludeTablesAssign(const char * newval, bool doit, GucSource source)
 221 {
 222         return tableListAssign(newval, doit, &excludeTables);
 223 }
 224
 225 static const char *
 226 includeTablesAssign(const char * newval, bool doit, GucSource source)
 227 {
 228         return tableListAssign(newval, doit, &includeTables);
 229 }
 230
 231 #endif
 232
 233 static const char*
 234 tableListShow(TableList *tbl)
 235 {
 236         char    *val, *ptr;
 237         int     i,
 238                         len;
 239
 240         len = 1 /* \0 */ + tbl->nTables * (2 * NAMEDATALEN + 2 /* ', ' */ + 1 /* . */);
 241         ptr = val = palloc(len);
 242         *ptr ='\0';
 243         for(i=0; i<tbl->nTables; i++)
 244         {
 245                 char    *relname = get_rel_name(tbl->tables[i]);
 246                 Oid     nspOid = get_rel_namespace(tbl->tables[i]);
 247                 char    *nspname = get_namespace_name(nspOid);
 248
 249                 if ( relname == NULL || nspOid == InvalidOid || nspname == NULL )
 250                         continue;
 251
 252                 ptr += snprintf(ptr, len - (ptr - val), "%s%s.%s",
 253                                                                                                         (i==0) ? "" : ", ",
 254                                                                                                         nspname, relname);
 255         }
 256
 257         return val;
 258 }
 259
 260 static const char*
 261 excludeTablesShow(void)
 262 {
 263         return tableListShow(&excludeTables);
 264 }
 265
 266 static const char*
 267 includeTablesShow(void)
 268 {
 269         return tableListShow(&includeTables);
 270 }
 271
 272 static bool
 273 matchOid(TableList *tbl, Oid oid)
 274 {
 275         Oid     *StopLow = tbl->tables,
 276                 *StopHigh = tbl->tables + tbl->nTables,
 277                 *StopMiddle;
 278
 279         /* Loop invariant: StopLow <= val < StopHigh */
 280         while (StopLow < StopHigh)
 281         {
 282                 StopMiddle = StopLow + ((StopHigh - StopLow) >> 1);
 283
 284                 if (*StopMiddle == oid)
 285                         return true;
 286                 else  if (*StopMiddle < oid)
 287                         StopLow = StopMiddle + 1;
 288                 else
 289                         StopHigh = StopMiddle;
 290         }
 291
 292         return false;
 293 }
 294
 295 static ExecutorEnd_hook_type oldhook = NULL;
 296
 297 static void
 298 makeAnalyze(Oid relOid, CmdType operation, uint32 naffected)
 299 {
 300         PgStat_StatTabEntry             *tabentry;
 301         TimestampTz                     now = GetCurrentTimestamp();
 302
 303         if (relOid == InvalidOid)
 304                 return;
 305
 306         tabentry = pgstat_fetch_stat_tabentry(relOid);
 307
 308 #if PG_VERSION_NUM >= 90000
 309 #define changes_since_analyze(t)        ((t)->changes_since_analyze)
 310 #else
 311 #define changes_since_analyze(t)        ((t)->n_live_tuples + (t)->n_dead_tuples - (t)->last_anl_tuples)
 312 #endif
 313
 314         if (
 315                 tabentry == NULL /* a new table */ ||
 316                 (
 317                         /* do not analyze too often, if both stamps are exceeded the go */
 318                         TimestampDifferenceExceeds(tabentry->analyze_timestamp, now, online_analyze_min_interval) &&
 319                         TimestampDifferenceExceeds(tabentry->autovac_analyze_timestamp, now, online_analyze_min_interval) &&
 320                         /* be in sync with relation_needs_vacanalyze */
 321                         ((double)(changes_since_analyze(tabentry) + naffected)) >=
 322                                 online_analyze_scale_factor * ((double)(tabentry->n_dead_tuples + tabentry->n_live_tuples)) +
 323                                         (double)online_analyze_threshold
 324                 )
 325         )
 326         {
 327                 VacuumStmt                              vacstmt;
 328                 TimestampTz                             startStamp, endStamp;
 329
 330                 /*
 331                  * includeTables overwrites excludeTables
 332                  */
 333                 switch(online_analyze_table_type)
 334                 {
 335                         case OATT_ALL:
 336                                 if (matchOid(&excludeTables, relOid) == true && matchOid(&includeTables, relOid) == false)
 337                                         return;
 338                                 break;
 339                         case OATT_NONE:
 340                                 if (matchOid(&includeTables, relOid) == false)
 341                                         return;
 342                                 break;
 343                         case OATT_TEMPORARY:
 344                         case OATT_PERSISTENT:
 345                         default:
 346                                 {
 347                                         Relation                                rel;
 348                                         OnlineAnalyzeTableType  reltype;
 349
 350                                         rel = RelationIdGetRelation(relOid);
 351                                         reltype =
 352 #if PG_VERSION_NUM >= 90100
 353                                                 (rel->rd_rel->relpersistence == RELPERSISTENCE_TEMP)
 354 #else
 355                                                 (rel->rd_istemp || rel->rd_islocaltemp)
 356 #endif
 357                                                         ? OATT_TEMPORARY : OATT_PERSISTENT;
 358                                         RelationClose(rel);
 359
 360                                         /*
 361                                          * skip analyze if relation's type doesn't not match online_analyze_table_type
 362                                          */
 363                                         if ((online_analyze_table_type & reltype) == 0 || matchOid(&excludeTables, relOid) == true)
 364                                         {
 365                                                 if (matchOid(&includeTables, relOid) == false)
 366                                                         return;
 367                                         }
 368                                 }
 369                                 break;
 370                 }
 371
 372                 vacstmt.type = T_VacuumStmt;
 373                 vacstmt.freeze_min_age = -1;
 374                 vacstmt.freeze_table_age = -1; /* ??? */
 375                 vacstmt.relation = NULL;
 376                 vacstmt.va_cols = NIL;
 377
 378 #if PG_VERSION_NUM >= 90000
 379                 vacstmt.options = VACOPT_ANALYZE;
 380                 if (online_analyze_verbose)
 381                         vacstmt.options |= VACOPT_VERBOSE;
 382 #else
 383                 vacstmt.vacuum = vacstmt.full = false;
 384                 vacstmt.analyze = true;
 385                 vacstmt.verbose = online_analyze_verbose;
 386 #endif
 387
 388                 if (online_analyze_verbose)
 389                         startStamp = GetCurrentTimestamp();
 390
 391                 analyze_rel(relOid, &vacstmt, GetAccessStrategy(BAS_VACUUM)
 392 #if (PG_VERSION_NUM < 90004) && (PG_VERSION_NUM >= 90000)
 393                         , true
 394 #endif
 395                 );
 396
 397                 if (online_analyze_verbose)
 398                 {
 399                         long    secs;
 400                         int             microsecs;
 401
 402                         endStamp = GetCurrentTimestamp();
 403                         TimestampDifference(startStamp, endStamp, &secs, &microsecs);
 404                         elog(INFO, "analyze \"%s\" took %.02f seconds",
 405                                 get_rel_name(relOid), ((double)secs) + ((double)microsecs)/1.0e6);
 406                 }
 407
 408
 409                 if (tabentry == NULL)
 410                 {
 411                         /* new table */
 412                         pgstat_clear_snapshot();
 413                 }
 414                 else
 415                 {
 416                         /* update last analyze timestamp in local memory of backend */
 417                         tabentry->analyze_timestamp = now;
 418                 }
 419         }
 420 #if PG_VERSION_NUM >= 90000
 421         else if (tabentry != NULL)
 422         {
 423                 tabentry->changes_since_analyze += naffected;
 424         }
 425 #endif
 426 }
 427
 428 extern PGDLLIMPORT void onlineAnalyzeHooker(QueryDesc *queryDesc);
 429 void
 430 onlineAnalyzeHooker(QueryDesc *queryDesc)
 431 {
 432         uint32  naffected = 0;
 433
 434         if (queryDesc->estate)
 435                 naffected = queryDesc->estate->es_processed;
 436
 437         if (online_analyze_enable && queryDesc->plannedstmt &&
 438                         (queryDesc->operation == CMD_INSERT ||
 439                          queryDesc->operation == CMD_UPDATE ||
 440                          queryDesc->operation == CMD_DELETE ||
 441 #if PG_VERSION_NUM >= 90200
 442                          0 /* (queryDesc->operation == CMD_SELECT && queryDesc->dest && queryDesc->dest == DestIntoRel) */
 443 #else
 444                          (queryDesc->operation == CMD_SELECT && queryDesc->plannedstmt->intoClause)
 445 #endif
 446                          ))
 447         {
 448 #if PG_VERSION_NUM < 90200
 449                 if (queryDesc->operation == CMD_SELECT)
 450                 {
 451                         Oid     relOid = RangeVarGetRelid(queryDesc->plannedstmt->intoClause->rel, true);
 452
 453                         makeAnalyze(relOid, queryDesc->operation, naffected);
 454                 }
 455                 else
 456 #endif
 457                 if (queryDesc->plannedstmt->resultRelations &&
 458                                  queryDesc->plannedstmt->rtable)
 459                 {
 460                         ListCell        *l;
 461
 462                         foreach(l, queryDesc->plannedstmt->resultRelations)
 463                         {
 464                                 int                     n = lfirst_int(l);
 465                                 RangeTblEntry   *rte = list_nth(queryDesc->plannedstmt->rtable, n-1);
 466
 467                                 if (rte->rtekind == RTE_RELATION)
 468                                         makeAnalyze(rte->relid, queryDesc->operation, naffected);
 469                         }
 470                 }
 471         }
 472
 473         if (oldhook)
 474                 (*oldhook)(queryDesc);
 475         else
 476                 standard_ExecutorEnd(queryDesc);
 477 }
 478
 479 void _PG_init(void);
 480 void
 481 _PG_init(void)
 482 {
 483         oldhook = ExecutorEnd_hook;
 484
 485         ExecutorEnd_hook = onlineAnalyzeHooker;
 486
 487         DefineCustomBoolVariable(
 488                 "online_analyze.enable",
 489                 "Enable on-line analyze",
 490                 "Enables analyze of table directly after insert/update/delete/select into",
 491                 &online_analyze_enable,
 492 #if PG_VERSION_NUM >= 80400
 493                 online_analyze_enable,
 494 #endif
 495                 PGC_USERSET,
 496 #if PG_VERSION_NUM >= 80400
 497                 GUC_NOT_IN_SAMPLE,
 498 #if PG_VERSION_NUM >= 90100
 499                 NULL,
 500 #endif
 501 #endif
 502                 NULL,
 503                 NULL
 504         );
 505
 506         DefineCustomBoolVariable(
 507                 "online_analyze.verbose",
 508                 "Verbosity of on-line analyze",
 509                 "Make ANALYZE VERBOSE after table's changes",
 510                 &online_analyze_verbose,
 511 #if PG_VERSION_NUM >= 80400
 512                 online_analyze_verbose,
 513 #endif
 514                 PGC_USERSET,
 515 #if PG_VERSION_NUM >= 80400
 516                 GUC_NOT_IN_SAMPLE,
 517 #if PG_VERSION_NUM >= 90100
 518                 NULL,
 519 #endif
 520 #endif
 521                 NULL,
 522                 NULL
 523         );
 524
 525     DefineCustomRealVariable(
 526                 "online_analyze.scale_factor",
 527                 "fraction of table size to start on-line analyze",
 528                 "fraction of table size to start on-line analyze",
 529                 &online_analyze_scale_factor,
 530 #if PG_VERSION_NUM >= 80400
 531                 online_analyze_scale_factor,
 532 #endif
 533                 0.0,
 534                 1.0,
 535                 PGC_USERSET,
 536 #if PG_VERSION_NUM >= 80400
 537                 GUC_NOT_IN_SAMPLE,
 538 #if PG_VERSION_NUM >= 90100
 539                 NULL,
 540 #endif
 541 #endif
 542                 NULL,
 543                 NULL
 544         );
 545
 546     DefineCustomIntVariable(
 547                 "online_analyze.threshold",
 548                 "min number of row updates before on-line analyze",
 549                 "min number of row updates before on-line analyze",
 550                 &online_analyze_threshold,
 551 #if PG_VERSION_NUM >= 80400
 552                 online_analyze_threshold,
 553 #endif
 554                 0,
 555                 0x7fffffff,
 556                 PGC_USERSET,
 557 #if PG_VERSION_NUM >= 80400
 558                 GUC_NOT_IN_SAMPLE,
 559 #if PG_VERSION_NUM >= 90100
 560                 NULL,
 561 #endif
 562 #endif
 563                 NULL,
 564                 NULL
 565         );
 566
 567     DefineCustomRealVariable(
 568                 "online_analyze.min_interval",
 569                 "minimum time interval between analyze call (in milliseconds)",
 570                 "minimum time interval between analyze call (in milliseconds)",
 571                 &online_analyze_scale_factor,
 572 #if PG_VERSION_NUM >= 80400
 573                 online_analyze_min_interval,
 574 #endif
 575                 0.0,
 576                 1e30,
 577                 PGC_USERSET,
 578 #if PG_VERSION_NUM >= 80400
 579                 GUC_NOT_IN_SAMPLE,
 580 #if PG_VERSION_NUM >= 90100
 581                 NULL,
 582 #endif
 583 #endif
 584                 NULL,
 585                 NULL
 586         );
 587
 588 DefineCustomEnumVariable(
 589                 "online_analyze.table_type",
 590                 "Type(s) of table for online analyze: all(default), persistent, temporary, none",
 591                 NULL,
 592                 &online_analyze_table_type,
 593 #if PG_VERSION_NUM >= 80400
 594                 online_analyze_table_type,
 595 #endif
 596                 online_analyze_table_type_options,
 597                 PGC_USERSET,
 598 #if PG_VERSION_NUM >= 80400
 599         GUC_NOT_IN_SAMPLE,
 600 #if PG_VERSION_NUM >= 90100
 601                 NULL,
 602 #endif
 603 #endif
 604                 NULL,
 605                 NULL
 606         );
 607
 608     DefineCustomStringVariable(
 609                 "online_analyze.exclude_tables",
 610                 "List of tables which will not online analyze",
 611                 NULL,
 612                 &excludeTables.tableStr,
 613 #if PG_VERSION_NUM >= 80400
 614                 "",
 615 #endif
 616                 PGC_USERSET,
 617                 0,
 618 #if PG_VERSION_NUM >= 90100
 619                 excludeTablesCheck,
 620                 excludeTablesAssign,
 621 #else
 622                 excludeTablesAssign,
 623 #endif
 624                 excludeTablesShow
 625         );
 626
 627     DefineCustomStringVariable(
 628                 "online_analyze.include_tables",
 629                 "List of tables which will online analyze",
 630                 NULL,
 631                 &includeTables.tableStr,
 632 #if PG_VERSION_NUM >= 80400
 633                 "",
 634 #endif
 635                 PGC_USERSET,
 636                 0,
 637 #if PG_VERSION_NUM >= 90100
 638                 includeTablesCheck,
 639                 includeTablesAssign,
 640 #else
 641                 includeTablesAssign,
 642 #endif
 643                 includeTablesShow
 644         );
 645 }
 646
 647 void _PG_fini(void);
 648 void
 649 _PG_fini(void)
 650 {
 651         ExecutorEnd_hook = oldhook;
 652
 653         if (excludeTables.tables)
 654                 free(excludeTables.tables);
 655         if (includeTables.tables)
 656                 free(includeTables.tables);
 657
 658         excludeTables.tables = includeTables.tables = NULL;
 659         excludeTables.nTables = includeTables.nTables = 0;
 660 }