online_analyze.c

   1 /*
   2  * Copyright (c) 2011 Teodor Sigaev <teodor@sigaev.ru>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *              notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *              notice, this list of conditions and the following disclaimer in the
  12  *              documentation and/or other materials provided with the distribution.
  13  * 3. Neither the name of the author nor the names of any co-contributors
  14  *              may be used to endorse or promote products derived from this software
  15  *              without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY CONTRIBUTORS ``AS IS'' AND ANY EXPRESS
  18  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  19  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED. IN NO EVENT SHALL CONTRIBUTORS BE LIABLE FOR ANY
  21  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
  23  * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  25  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  26  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  27  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  */
  29
  30 #include "postgres.h"
  31
  32 #include "pgstat.h"
  33 #include "miscadmin.h"
  34 #include "access/transam.h"
  35 #include "access/xact.h"
  36 #include "catalog/namespace.h"
  37 #include "commands/vacuum.h"
  38 #include "executor/executor.h"
  39 #include "nodes/nodes.h"
  40 #include "nodes/parsenodes.h"
  41 #include "storage/bufmgr.h"
  42 #include "utils/builtins.h"
  43 #include "utils/hsearch.h"
  44 #include "utils/memutils.h"
  45 #include "utils/lsyscache.h"
  46 #include "utils/guc.h"
  47 #if PG_VERSION_NUM >= 90200
  48 #include "catalog/pg_class.h"
  49 #include "nodes/primnodes.h"
  50 #include "tcop/utility.h"
  51 #include "utils/rel.h"
  52 #include "utils/relcache.h"
  53 #include "utils/timestamp.h"
  54 #if PG_VERSION_NUM >= 90500
  55 #include "nodes/makefuncs.h"
  56 #if PG_VERSION_NUM >= 100000
  57 #include "utils/varlena.h"
  58 #include "utils/regproc.h"
  59 #if PG_VERSION_NUM >= 130000
  60 #include "common/hashfn.h"
  61 #endif
  62 #endif
  63 #endif
  64 #endif
  65
  66 #ifdef PG_MODULE_MAGIC
  67 PG_MODULE_MAGIC;
  68 #endif
  69
  70 static bool online_analyze_enable = true;
  71 static bool online_analyze_local_tracking = false;
  72 static bool online_analyze_verbose = true;
  73 static double online_analyze_scale_factor = 0.1;
  74 static int online_analyze_threshold = 50;
  75 static int online_analyze_capacity_threshold = 100000;
  76 static double online_analyze_min_interval = 10000;
  77 static int online_analyze_lower_limit = 0;
  78
  79 static ExecutorEnd_hook_type oldExecutorEndHook = NULL;
  80 #if PG_VERSION_NUM >= 90200
  81 static ProcessUtility_hook_type oldProcessUtilityHook = NULL;
  82 #endif
  83
  84 #if PG_VERSION_NUM >= 120000
  85 #define VACOPT_NOWAIT VACOPT_SKIP_LOCKED
  86 #endif
  87
  88 typedef enum CmdKind
  89 {
  90         CK_SELECT = CMD_SELECT,
  91         CK_UPDATE = CMD_UPDATE,
  92         CK_INSERT = CMD_INSERT,
  93         CK_DELETE = CMD_DELETE,
  94         CK_TRUNCATE,
  95         CK_FASTTRUNCATE,
  96         CK_CREATE,
  97         CK_ANALYZE,
  98         CK_VACUUM
  99 } CmdKind;
 100
 101
 102 typedef enum
 103 {
 104         OATT_ALL                = 0x03,
 105         OATT_PERSISTENT = 0x01,
 106         OATT_TEMPORARY  = 0x02,
 107         OATT_NONE               = 0x00
 108 } OnlineAnalyzeTableType;
 109
 110 static const struct config_enum_entry online_analyze_table_type_options[] =
 111 {
 112         {"all", OATT_ALL, false},
 113         {"persistent", OATT_PERSISTENT, false},
 114         {"temporary", OATT_TEMPORARY, false},
 115         {"none", OATT_NONE, false},
 116         {NULL, 0, false},
 117 };
 118
 119 static int online_analyze_table_type = (int)OATT_ALL;
 120
 121 typedef struct TableList {
 122         int             nTables;
 123         Oid             *tables;
 124         char    *tableStr;
 125         bool    inited;
 126 } TableList;
 127
 128 static TableList excludeTables = {0, NULL, NULL, false};
 129 static TableList includeTables = {0, NULL, NULL, false};
 130
 131 typedef struct OnlineAnalyzeTableStat {
 132         Oid                             tableid;
 133         bool                    rereadStat;
 134         PgStat_Counter  n_tuples;
 135         PgStat_Counter  mod_since_analyze;
 136         TimestampTz             last_autoanalyze_time;
 137         TimestampTz             last_analyze_time;
 138 } OnlineAnalyzeTableStat;
 139
 140 static  MemoryContext   onlineAnalyzeMemoryContext = NULL;
 141 static  HTAB    *relstats = NULL;
 142
 143 static void relstatsInit(void);
 144
 145 #if PG_VERSION_NUM < 100000
 146 static int
 147 oid_cmp(const void *a, const void *b)
 148 {
 149         if (*(Oid*)a == *(Oid*)b)
 150                 return 0;
 151         return (*(Oid*)a > *(Oid*)b) ? 1 : -1;
 152 }
 153 #endif
 154
 155 static const char *
 156 tableListAssign(const char * newval, bool doit, TableList *tbl)
 157 {
 158         char            *rawname;
 159         List            *namelist;
 160         ListCell        *l;
 161         Oid                     *newOids = NULL;
 162         int                     nOids = 0,
 163                                 i = 0;
 164
 165         rawname = pstrdup(newval);
 166
 167         if (!SplitIdentifierString(rawname, ',', &namelist))
 168                 goto cleanup;
 169
 170         /*
 171         * follow work could be done only in normal processing because of
 172         * accsess to system catalog
 173         */
 174         if (MyBackendId == InvalidBackendId || !IsUnderPostmaster ||
 175                 !IsTransactionState())
 176         {
 177                 includeTables.inited = false;
 178                 excludeTables.inited = false;
 179                 return newval;
 180         }
 181
 182         if (doit)
 183         {
 184                 nOids = list_length(namelist);
 185                 newOids = malloc(sizeof(Oid) * (nOids+1));
 186                 if (!newOids)
 187                         elog(ERROR,"could not allocate %d bytes",
 188                                  (int)(sizeof(Oid) * (nOids+1)));
 189         }
 190
 191         foreach(l, namelist)
 192         {
 193                 char    *curname = (char *) lfirst(l);
 194 #if PG_VERSION_NUM >= 160000
 195                 Oid             relOid = RangeVarGetRelid(makeRangeVarFromNameList(
 196                                                         stringToQualifiedNameList(curname, NULL)), NoLock, true);
 197 #elif PG_VERSION_NUM >= 90200
 198                 Oid             relOid = RangeVarGetRelid(makeRangeVarFromNameList(
 199                                                         stringToQualifiedNameList(curname)), NoLock, true);
 200 #else
 201                 Oid             relOid = RangeVarGetRelid(makeRangeVarFromNameList(
 202                                                         stringToQualifiedNameList(curname)), true);
 203 #endif
 204
 205                 if (relOid == InvalidOid)
 206                 {
 207 #if PG_VERSION_NUM >= 90100
 208                         if (doit == false)
 209 #endif
 210                         elog(WARNING,"'%s' does not exist", curname);
 211                         continue;
 212                 }
 213                 else if ( get_rel_relkind(relOid) != RELKIND_RELATION )
 214                 {
 215 #if PG_VERSION_NUM >= 90100
 216                         if (doit == false)
 217 #endif
 218                                 elog(WARNING,"'%s' is not an table", curname);
 219                         continue;
 220                 }
 221                 else if (doit)
 222                 {
 223                         newOids[i++] = relOid;
 224                 }
 225         }
 226
 227         if (doit)
 228         {
 229                 tbl->nTables = i;
 230                 if (tbl->tables)
 231                         free(tbl->tables);
 232                 tbl->tables = newOids;
 233                 if (tbl->nTables > 1)
 234                         qsort(tbl->tables, tbl->nTables, sizeof(tbl->tables[0]), oid_cmp);
 235         }
 236
 237         pfree(rawname);
 238         list_free(namelist);
 239
 240         return newval;
 241
 242 cleanup:
 243         if (newOids)
 244                 free(newOids);
 245         pfree(rawname);
 246         list_free(namelist);
 247         return NULL;
 248 }
 249
 250 #if PG_VERSION_NUM >= 90100
 251 static bool
 252 excludeTablesCheck(char **newval, void **extra, GucSource source)
 253 {
 254         char *val;
 255
 256         val = (char*)tableListAssign(*newval, false, &excludeTables);
 257
 258         if (val)
 259         {
 260                 *newval = val;
 261                 return true;
 262         }
 263
 264         return false;
 265 }
 266
 267 static void
 268 excludeTablesAssign(const char *newval, void *extra)
 269 {
 270         tableListAssign(newval, true, &excludeTables);
 271 }
 272
 273 static bool
 274 includeTablesCheck(char **newval, void **extra, GucSource source)
 275 {
 276         char *val;
 277
 278         val = (char*)tableListAssign(*newval, false, &includeTables);
 279
 280         if (val)
 281         {
 282                 *newval = val;
 283                 return true;
 284         }
 285
 286         return false;
 287 }
 288
 289 static void
 290 includeTablesAssign(const char *newval, void *extra)
 291 {
 292         tableListAssign(newval, true, &includeTables);
 293 }
 294
 295 #else /* PG_VERSION_NUM < 90100 */
 296
 297 static const char *
 298 excludeTablesAssign(const char * newval, bool doit, GucSource source)
 299 {
 300         return tableListAssign(newval, doit, &excludeTables);
 301 }
 302
 303 static const char *
 304 includeTablesAssign(const char * newval, bool doit, GucSource source)
 305 {
 306         return tableListAssign(newval, doit, &includeTables);
 307 }
 308
 309 #endif
 310
 311 static void
 312 lateInit()
 313 {
 314         TableList       *tl[] = {&includeTables, &excludeTables};
 315         int i;
 316
 317         if (MyBackendId == InvalidBackendId || !IsUnderPostmaster ||
 318                 !IsTransactionState())
 319                 return; /* we aren't in connected state */
 320
 321         for(i=0; i<lengthof(tl); i++)
 322         {
 323                 TableList       *tbl = tl[i];
 324
 325                 if (tbl->inited == false)
 326                         tableListAssign(tbl->tableStr, true, tbl);
 327                 tbl->inited = true;
 328         }
 329 }
 330
 331 static const char*
 332 tableListShow(TableList *tbl)
 333 {
 334         char    *val, *ptr;
 335         int             i,
 336                         len;
 337
 338         lateInit();
 339
 340         len = 1 /* \0 */ + tbl->nTables * (2 * NAMEDATALEN + 2 /* ', ' */ + 1 /* . */);
 341         ptr = val = palloc(len);
 342         *ptr ='\0';
 343         for(i=0; i<tbl->nTables; i++)
 344         {
 345                 char    *relname = get_rel_name(tbl->tables[i]);
 346                 Oid             nspOid = get_rel_namespace(tbl->tables[i]);
 347                 char    *nspname = get_namespace_name(nspOid);
 348
 349                 if ( relname == NULL || nspOid == InvalidOid || nspname == NULL )
 350                         continue;
 351
 352                 ptr += snprintf(ptr, len - (ptr - val), "%s%s.%s",
 353                                                                                                         (i==0) ? "" : ", ",
 354                                                                                                         nspname, relname);
 355         }
 356
 357         return val;
 358 }
 359
 360 static const char*
 361 excludeTablesShow(void)
 362 {
 363         return tableListShow(&excludeTables);
 364 }
 365
 366 static const char*
 367 includeTablesShow(void)
 368 {
 369         return tableListShow(&includeTables);
 370 }
 371
 372 static bool
 373 matchOid(TableList *tbl, Oid oid)
 374 {
 375         Oid     *StopLow = tbl->tables,
 376                 *StopHigh = tbl->tables + tbl->nTables,
 377                 *StopMiddle;
 378
 379         /* Loop invariant: StopLow <= val < StopHigh */
 380         while (StopLow < StopHigh)
 381         {
 382                 StopMiddle = StopLow + ((StopHigh - StopLow) >> 1);
 383
 384                 if (*StopMiddle == oid)
 385                         return true;
 386                 else  if (*StopMiddle < oid)
 387                         StopLow = StopMiddle + 1;
 388                 else
 389                         StopHigh = StopMiddle;
 390         }
 391
 392         return false;
 393 }
 394
 395 #if PG_VERSION_NUM >= 90500
 396 static RangeVar*
 397 makeRangeVarFromOid(Oid relOid)
 398 {
 399         return makeRangeVar(
 400                                 get_namespace_name(get_rel_namespace(relOid)),
 401                                 get_rel_name(relOid),
 402                                 -1
 403                         );
 404
 405 }
 406 #endif
 407
 408 static void
 409 makeAnalyze(Oid relOid, CmdKind operation, int64 naffected)
 410 {
 411         TimestampTz                             now = GetCurrentTimestamp();
 412         Relation                                rel;
 413         OnlineAnalyzeTableType  reltype;
 414         bool                                    found = false,
 415                                                         newTable = false;
 416         OnlineAnalyzeTableStat  *rstat,
 417                                                         dummyrstat;
 418         PgStat_StatTabEntry             *tabentry = NULL;
 419
 420         if (relOid == InvalidOid)
 421                 return;
 422
 423         if (naffected == 0)
 424                 /* return if there is no changes */
 425                 return;
 426         else if (naffected < 0)
 427                 /* number if affected rows is unknown */
 428                 naffected = 0;
 429
 430         rel = RelationIdGetRelation(relOid);
 431         if (rel->rd_rel->relkind != RELKIND_RELATION)
 432         {
 433                 RelationClose(rel);
 434                 return;
 435         }
 436
 437         reltype =
 438 #if PG_VERSION_NUM >= 90100
 439                 (rel->rd_rel->relpersistence == RELPERSISTENCE_TEMP)
 440 #else
 441                 (rel->rd_istemp || rel->rd_islocaltemp)
 442 #endif
 443                         ? OATT_TEMPORARY : OATT_PERSISTENT;
 444
 445         RelationClose(rel);
 446
 447         /*
 448          * includeTables overwrites excludeTables
 449          */
 450         switch(online_analyze_table_type)
 451         {
 452                 case OATT_ALL:
 453                         if (get_rel_relkind(relOid) != RELKIND_RELATION ||
 454                                 (matchOid(&excludeTables, relOid) == true &&
 455                                 matchOid(&includeTables, relOid) == false))
 456                                 return;
 457                         break;
 458                 case OATT_NONE:
 459                         if (get_rel_relkind(relOid) != RELKIND_RELATION ||
 460                                 matchOid(&includeTables, relOid) == false)
 461                                 return;
 462                         break;
 463                 case OATT_TEMPORARY:
 464                 case OATT_PERSISTENT:
 465                 default:
 466                         /*
 467                          * skip analyze if relation's type doesn't not match
 468                          * online_analyze_table_type
 469                          */
 470                         if ((online_analyze_table_type & reltype) == 0 ||
 471                                 matchOid(&excludeTables, relOid) == true)
 472                         {
 473                                 if (matchOid(&includeTables, relOid) == false)
 474                                         return;
 475                         }
 476                         break;
 477         }
 478
 479         /*
 480          * Do not store data about persistent table in local memory because we
 481          * could not track changes of them: they could be changed by another
 482          * backends. So always get a pgstat table entry.
 483          */
 484         if (reltype == OATT_TEMPORARY)
 485                 rstat = hash_search(relstats, &relOid, HASH_ENTER, &found);
 486         else
 487                 rstat = &dummyrstat; /* found == false for following if */
 488
 489         if (!found)
 490         {
 491                 MemSet(rstat, 0, sizeof(*rstat));
 492                 rstat->tableid = relOid;
 493                 newTable = true;
 494         }
 495
 496         if (operation == CK_VACUUM)
 497         {
 498                 /* force reread because vacuum could change n_tuples */
 499                 rstat->rereadStat = true;
 500                 return;
 501         }
 502         else if (operation == CK_ANALYZE)
 503         {
 504                 /* only analyze */
 505                 rstat->mod_since_analyze = 0;
 506                 rstat->last_analyze_time = now;
 507                 if (newTable)
 508                         rstat->rereadStat = true;
 509                 return;
 510         }
 511
 512         Assert(rstat->tableid == relOid);
 513
 514         if (
 515                 /* do not reread data if it was a truncation */
 516                 operation != CK_TRUNCATE && operation != CK_FASTTRUNCATE &&
 517                 /* read  for persistent table and for temp teble if it allowed */
 518                 (reltype == OATT_PERSISTENT || online_analyze_local_tracking == false) &&
 519                 /* read only for new table or we know that it's needed */
 520                 (newTable == true || rstat->rereadStat == true)
 521            )
 522         {
 523                 rstat->rereadStat = false;
 524
 525                 tabentry = pgstat_fetch_stat_tabentry(relOid);
 526
 527                 if (tabentry)
 528                 {
 529                         rstat->n_tuples =
 530 #if PG_VERSION_NUM >= 160000
 531                                 tabentry->dead_tuples + tabentry->live_tuples;
 532 #else
 533                                 tabentry->n_dead_tuples + tabentry->n_live_tuples;
 534 #endif
 535
 536                         rstat->mod_since_analyze =
 537 #if PG_VERSION_NUM >= 160000
 538                                 tabentry->mod_since_analyze;
 539 #elif PG_VERSION_NUM >= 90000
 540                                 tabentry->changes_since_analyze;
 541 #else
 542                                 tabentry->n_live_tuples + tabentry->n_dead_tuples -
 543                                         tabentry->last_anl_tuples;
 544 #endif
 545
 546                         rstat->last_autoanalyze_time =
 547 #if PG_VERSION_NUM >= 160000
 548                                 tabentry->last_autoanalyze_time;
 549 #else
 550                                 tabentry->autovac_analyze_timestamp;
 551 #endif
 552
 553                         rstat->last_analyze_time =
 554 #if PG_VERSION_NUM >= 160000
 555                                 tabentry->last_analyze_time;
 556 #else
 557                                 tabentry->analyze_timestamp;
 558 #endif
 559                 }
 560         }
 561
 562         if (newTable ||
 563                 /* force analyze after truncate, fasttruncate already did analyze */
 564                 operation == CK_TRUNCATE || (
 565                 /* do not analyze too often, if both stamps are exceeded the go */
 566                 TimestampDifferenceExceeds(rstat->last_analyze_time, now, online_analyze_min_interval) &&
 567                 TimestampDifferenceExceeds(rstat->last_autoanalyze_time, now, online_analyze_min_interval) &&
 568                 /* do not analyze too small tables */
 569                 rstat->n_tuples + rstat->mod_since_analyze + naffected > online_analyze_lower_limit &&
 570                 /* be in sync with relation_needs_vacanalyze */
 571                 ((double)(rstat->mod_since_analyze + naffected)) >=
 572                          online_analyze_scale_factor * ((double)rstat->n_tuples) +
 573                          (double)online_analyze_threshold))
 574         {
 575 #if PG_VERSION_NUM < 90500
 576                 VacuumStmt                              vacstmt;
 577 #else
 578                 VacuumParams                    vacstmt;
 579 #endif
 580                 TimestampTz                             startStamp, endStamp;
 581                 int                                             flags;
 582
 583 #ifdef PGPRO_EE
 584                 /* ATX is not compatible with online_analyze */
 585                 if (getNestLevelATX() != 0)
 586                         return;
 587 #endif
 588
 589                 memset(&startStamp, 0, sizeof(startStamp)); /* keep compiler quiet */
 590
 591                 memset(&vacstmt, 0, sizeof(vacstmt));
 592
 593                 vacstmt.freeze_min_age = -1;
 594                 vacstmt.freeze_table_age = -1; /* ??? */
 595
 596 #if PG_VERSION_NUM < 90500
 597                 vacstmt.type = T_VacuumStmt;
 598                 vacstmt.relation = NULL;
 599                 vacstmt.va_cols = NIL;
 600 #if PG_VERSION_NUM >= 90000
 601                 vacstmt.options = VACOPT_ANALYZE;
 602                 if (online_analyze_verbose)
 603                         vacstmt.options |= VACOPT_VERBOSE;
 604 #else
 605                 vacstmt.vacuum = vacstmt.full = false;
 606                 vacstmt.analyze = true;
 607                 vacstmt.verbose = online_analyze_verbose;
 608 #endif
 609 #else
 610                 vacstmt.multixact_freeze_min_age = -1;
 611                 vacstmt.multixact_freeze_table_age = -1;
 612                 vacstmt.log_min_duration = -1;
 613 #endif
 614
 615
 616                 if (online_analyze_verbose)
 617                         startStamp = GetCurrentTimestamp();
 618
 619                 flags = VACOPT_ANALYZE | VACOPT_NOWAIT |
 620                                         ((online_analyze_verbose) ?  VACOPT_VERBOSE : 0);
 621
 622 #if PG_VERSION_NUM >= 120000
 623                 vacstmt.options = flags;
 624 #endif
 625                 analyze_rel(relOid,
 626 #if PG_VERSION_NUM < 90500
 627                         &vacstmt
 628 #if PG_VERSION_NUM >= 90018
 629                         , true
 630 #endif
 631                         , GetAccessStrategy(BAS_VACUUM)
 632 #if (PG_VERSION_NUM >= 90000) && (PG_VERSION_NUM < 90004)
 633                         , true
 634 #endif
 635 #else
 636                         makeRangeVarFromOid(relOid),
 637 #if PG_VERSION_NUM < 120000
 638                         flags,
 639 #endif
 640                         &vacstmt, NULL, true, GetAccessStrategy(BAS_VACUUM)
 641 #endif
 642                 );
 643
 644                 /* Make changes visible to subsequent calls */
 645                 CommandCounterIncrement();
 646
 647                 if (online_analyze_verbose)
 648                 {
 649                         long    secs;
 650                         int             microsecs;
 651
 652                         endStamp = GetCurrentTimestamp();
 653                         TimestampDifference(startStamp, endStamp, &secs, &microsecs);
 654                         elog(INFO, "analyze \"%s\" took %.02f seconds",
 655                                 get_rel_name(relOid),
 656                                 ((double)secs) + ((double)microsecs)/1.0e6);
 657                 }
 658
 659                 rstat->last_autoanalyze_time = now;
 660                 rstat->mod_since_analyze = 0;
 661
 662                 switch(operation)
 663                 {
 664                         case CK_CREATE:
 665                         case CK_INSERT:
 666                         case CK_UPDATE:
 667                                 rstat->n_tuples += naffected;
 668                                 /* FALLTHROUGH */
 669                         case CK_DELETE:
 670                                 rstat->rereadStat = (reltype == OATT_PERSISTENT);
 671                                 break;
 672                         case CK_TRUNCATE:
 673                         case CK_FASTTRUNCATE:
 674                                 rstat->rereadStat = false;
 675                                 rstat->n_tuples = 0;
 676                                 break;
 677                         default:
 678                                 break;
 679                 }
 680
 681                 /* update last analyze timestamp in local memory of backend */
 682                 if (tabentry)
 683                 {
 684 #if PG_VERSION_NUM >= 160000
 685                         tabentry->last_analyze_time = now;
 686                         tabentry->mod_since_analyze = 0;
 687 #else
 688                         tabentry->analyze_timestamp = now;
 689                         tabentry->changes_since_analyze = 0;
 690 #endif
 691                 }
 692 #if 0
 693                 /* force reload stat for new table */
 694                 if (newTable)
 695                         pgstat_clear_snapshot();
 696 #endif
 697         }
 698         else
 699         {
 700 #if PG_VERSION_NUM >= 90000
 701                 if (tabentry)
 702 #if PG_VERSION_NUM >= 160000
 703                         tabentry->mod_since_analyze += naffected;
 704 #else
 705                         tabentry->changes_since_analyze += naffected;
 706 #endif
 707 #endif
 708                 switch(operation)
 709                 {
 710                         case CK_CREATE:
 711                         case CK_INSERT:
 712                                 rstat->mod_since_analyze += naffected;
 713                                 rstat->n_tuples += naffected;
 714                                 break;
 715                         case CK_UPDATE:
 716                                 rstat->mod_since_analyze += 2 * naffected;
 717                                 rstat->n_tuples += naffected;
 718                                 break;
 719                         case CK_DELETE:
 720                                 rstat->mod_since_analyze += naffected;
 721                                 break;
 722                         case CK_TRUNCATE:
 723                         case CK_FASTTRUNCATE:
 724                                 rstat->mod_since_analyze = 0;
 725                                 rstat->n_tuples = 0;
 726                                 break;
 727                         default:
 728                                 break;
 729                 }
 730         }
 731
 732         /* Reset local cache if we are over limit */
 733         if (hash_get_num_entries(relstats) > online_analyze_capacity_threshold)
 734                 relstatsInit();
 735 }
 736
 737 static Const*
 738 isFastTruncateCall(QueryDesc *queryDesc)
 739 {
 740         TargetEntry     *te;
 741         FuncExpr        *fe;
 742         Const           *constval;
 743
 744         if (!(
 745                   queryDesc->plannedstmt &&
 746                   queryDesc->operation == CMD_SELECT &&
 747                   queryDesc->plannedstmt->planTree &&
 748                   queryDesc->plannedstmt->planTree->targetlist &&
 749                   list_length(queryDesc->plannedstmt->planTree->targetlist) == 1
 750                  ))
 751                 return NULL;
 752
 753         te = linitial(queryDesc->plannedstmt->planTree->targetlist);
 754
 755         if (!IsA(te, TargetEntry))
 756                 return NULL;
 757
 758         fe = (FuncExpr*)te->expr;
 759
 760         if (!(
 761                   fe && IsA(fe, FuncExpr) &&
 762                   fe->funcid >= FirstNormalObjectId &&
 763                   fe->funcretset == false &&
 764                   fe->funcresulttype == VOIDOID &&
 765                   fe->funcvariadic == false &&
 766                   list_length(fe->args) == 1
 767                  ))
 768                 return NULL;
 769
 770         constval = linitial(fe->args);
 771
 772         if (!(
 773                   IsA(constval,Const) &&
 774                   constval->consttype == TEXTOID &&
 775                   strcmp(get_func_name(fe->funcid), "fasttruncate") == 0
 776                  ))
 777                 return NULL;
 778
 779         return constval;
 780 }
 781
 782
 783 extern PGDLLIMPORT void onlineAnalyzeHooker(QueryDesc *queryDesc);
 784 void
 785 onlineAnalyzeHooker(QueryDesc *queryDesc)
 786 {
 787         int64   naffected = -1;
 788         Const   *constval;
 789
 790         if (queryDesc->estate)
 791                 naffected = queryDesc->estate->es_processed;
 792
 793         lateInit();
 794
 795 #if PG_VERSION_NUM >= 90200
 796         if (online_analyze_enable &&
 797                 (constval = isFastTruncateCall(queryDesc)) != NULL)
 798         {
 799                 Datum           tblnamed = constval->constvalue;
 800                 char            *tblname = text_to_cstring(DatumGetTextP(tblnamed));
 801 #if PG_VERSION_NUM >= 160000
 802                 RangeVar        *tblvar =
 803                         makeRangeVarFromNameList(stringToQualifiedNameList(tblname, NULL));
 804 #else
 805                 RangeVar        *tblvar =
 806                         makeRangeVarFromNameList(stringToQualifiedNameList(tblname));
 807 #endif
 808
 809                 makeAnalyze(RangeVarGetRelid(tblvar,
 810                                                                          NoLock,
 811                                                                          false),
 812                                         CK_FASTTRUNCATE, -1);
 813         }
 814 #endif
 815
 816         if (online_analyze_enable && queryDesc->plannedstmt &&
 817                         (queryDesc->operation == CMD_INSERT ||
 818                          queryDesc->operation == CMD_UPDATE ||
 819                          queryDesc->operation == CMD_DELETE
 820 #if PG_VERSION_NUM < 90200
 821                          || (queryDesc->operation == CMD_SELECT &&
 822                                  queryDesc->plannedstmt->intoClause)
 823 #endif
 824                          ))
 825         {
 826 #if PG_VERSION_NUM < 90200
 827                 if (queryDesc->operation == CMD_SELECT)
 828                 {
 829                         Oid     relOid = RangeVarGetRelid(queryDesc->plannedstmt->intoClause->rel, true);
 830
 831                         makeAnalyze(relOid, queryDesc->operation, naffected);
 832                 }
 833                 else
 834 #endif
 835                 if (queryDesc->plannedstmt->resultRelations &&
 836                                  queryDesc->plannedstmt->rtable)
 837                 {
 838                         ListCell        *l;
 839
 840                         foreach(l, queryDesc->plannedstmt->resultRelations)
 841                         {
 842                                 int                             n = lfirst_int(l);
 843                                 RangeTblEntry   *rte = list_nth(queryDesc->plannedstmt->rtable, n-1);
 844
 845                                 if (rte->rtekind == RTE_RELATION)
 846                                         makeAnalyze(rte->relid, (CmdKind)queryDesc->operation, naffected);
 847                         }
 848                 }
 849         }
 850
 851         if (oldExecutorEndHook)
 852                 oldExecutorEndHook(queryDesc);
 853         else
 854                 standard_ExecutorEnd(queryDesc);
 855 }
 856
 857 static List             *toremove = NIL;
 858
 859 /*
 860  * removeTable called on transaction end, see call RegisterXactCallback() below
 861  */
 862 static void
 863 removeTable(XactEvent event, void *arg)
 864 {
 865         ListCell        *cell;
 866
 867         switch(event)
 868         {
 869                 case XACT_EVENT_COMMIT:
 870                         break;
 871                 case XACT_EVENT_ABORT:
 872                         toremove = NIL;
 873                 default:
 874                         return;
 875         }
 876
 877         foreach(cell, toremove)
 878         {
 879                 Oid     relOid = lfirst_oid(cell);
 880
 881                 hash_search(relstats, &relOid, HASH_REMOVE, NULL);
 882         }
 883
 884         toremove = NIL;
 885 }
 886
 887 #if PG_VERSION_NUM >= 120000
 888 static int
 889 parse_vacuum_opt(VacuumStmt *vacstmt)
 890 {
 891         int                     options = vacstmt->is_vacuumcmd ? VACOPT_VACUUM : VACOPT_ANALYZE;
 892         ListCell        *lc;
 893
 894         foreach(lc, vacstmt->options)
 895         {
 896                 DefElem *opt = (DefElem *) lfirst(lc);
 897
 898                 /* Parse common options for VACUUM and ANALYZE */
 899                 if (strcmp(opt->defname, "verbose") == 0)
 900                         options |= VACOPT_VERBOSE;
 901                 else if (strcmp(opt->defname, "skip_locked") == 0)
 902                         options |= VACOPT_SKIP_LOCKED;
 903                 else if (strcmp(opt->defname, "analyze") == 0)
 904                         options |= VACOPT_ANALYZE;
 905                 else if (strcmp(opt->defname, "freeze") == 0)
 906                         options |= VACOPT_FREEZE;
 907                 else if (strcmp(opt->defname, "full") == 0)
 908                         options |= VACOPT_FULL;
 909                 else if (strcmp(opt->defname, "disable_page_skipping") == 0)
 910                         options |= VACOPT_DISABLE_PAGE_SKIPPING;
 911         }
 912
 913         return options;
 914 }
 915 #endif
 916
 917
 918 #if PG_VERSION_NUM >= 90200
 919 static void
 920 onlineAnalyzeHookerUtility(
 921 #if PG_VERSION_NUM >= 100000
 922                                                    PlannedStmt *pstmt,
 923 #else
 924                                                    Node *parsetree,
 925 #endif
 926                                                    const char *queryString,
 927 #if PG_VERSION_NUM >= 140000
 928                                                    bool readOnlyTree,
 929 #endif
 930 #if PG_VERSION_NUM >= 90300
 931                                                         ProcessUtilityContext context, ParamListInfo params,
 932 #if PG_VERSION_NUM >= 100000
 933                                                         QueryEnvironment *queryEnv,
 934 #endif
 935 #else
 936                                                         ParamListInfo params, bool isTopLevel,
 937 #endif
 938                                                         DestReceiver *dest,
 939 #if  PG_VERSION_NUM >= 130000
 940                                                         QueryCompletion *completionTag
 941 #else
 942                                                         char *completionTag
 943 #endif
 944 ) {
 945         List            *tblnames = NIL;
 946         CmdKind         op = CK_INSERT;
 947 #if PG_VERSION_NUM >= 100000
 948         Node            *parsetree = NULL;
 949
 950         if (pstmt->commandType == CMD_UTILITY)
 951                 parsetree = pstmt->utilityStmt;
 952 #endif
 953
 954         lateInit();
 955
 956         if (parsetree && online_analyze_enable)
 957         {
 958                 if (IsA(parsetree, CreateTableAsStmt) &&
 959                         ((CreateTableAsStmt*)parsetree)->into)
 960                 {
 961                         tblnames =
 962                                 list_make1((RangeVar*)copyObject(((CreateTableAsStmt*)parsetree)->into->rel));
 963                         op = CK_CREATE;
 964                 }
 965                 else if (IsA(parsetree, TruncateStmt))
 966                 {
 967                         tblnames = list_copy(((TruncateStmt*)parsetree)->relations);
 968                         op = CK_TRUNCATE;
 969                 }
 970                 else if (IsA(parsetree, DropStmt) &&
 971                                  ((DropStmt*)parsetree)->removeType == OBJECT_TABLE)
 972                 {
 973                         ListCell        *cell;
 974
 975                         foreach(cell, ((DropStmt*)parsetree)->objects)
 976                         {
 977                                 List            *relname = (List *) lfirst(cell);
 978                                 RangeVar        *rel = makeRangeVarFromNameList(relname);
 979                                 Oid                     relOid = RangeVarGetRelid(rel, NoLock, true);
 980
 981                                 if (OidIsValid(relOid))
 982                                 {
 983                                         MemoryContext   ctx;
 984
 985                                         ctx = MemoryContextSwitchTo(TopTransactionContext);
 986                                         toremove = lappend_oid(toremove, relOid);
 987                                         MemoryContextSwitchTo(ctx);
 988                                 }
 989                         }
 990                 }
 991                 else if (IsA(parsetree, VacuumStmt))
 992                 {
 993                         VacuumStmt      *vac = (VacuumStmt*)parsetree;
 994                         int                     options =
 995 #if PG_VERSION_NUM >= 120000
 996                                                         parse_vacuum_opt(vac)
 997 #else
 998                                                         vac->options
 999 #endif
1000                                                         ;
1001
1002
1003 #if PG_VERSION_NUM >= 110000
1004                         tblnames = vac->rels;
1005 #else
1006                         if (vac->relation)
1007                                 tblnames = list_make1(vac->relation);
1008 #endif
1009
1010                         if (options & (VACOPT_VACUUM | VACOPT_FULL | VACOPT_FREEZE))
1011                         {
1012                                 /* optionally with analyze */
1013                                 op = CK_VACUUM;
1014
1015                                 /* drop all collected stat */
1016                                 if (tblnames == NIL)
1017                                         relstatsInit();
1018                         }
1019                         else if (options & VACOPT_ANALYZE)
1020                         {
1021                                 op = CK_ANALYZE;
1022
1023                                 /* should reset all counters */
1024                                 if (tblnames == NIL)
1025                                 {
1026                                         HASH_SEQ_STATUS                 hs;
1027                                         OnlineAnalyzeTableStat  *rstat;
1028                                         TimestampTz                             now = GetCurrentTimestamp();
1029
1030                                         hash_seq_init(&hs, relstats);
1031
1032                                         while((rstat = hash_seq_search(&hs)) != NULL)
1033                                         {
1034                                                 rstat->mod_since_analyze = 0;
1035                                                 rstat->last_analyze_time = now;
1036                                         }
1037                                 }
1038                         }
1039                         else
1040                                 tblnames = NIL;
1041                 }
1042         }
1043
1044 #if PG_VERSION_NUM >= 100000
1045 #define parsetree pstmt
1046 #endif
1047
1048         if (oldProcessUtilityHook)
1049                 oldProcessUtilityHook(parsetree, queryString,
1050 #if PG_VERSION_NUM >= 140000
1051                                                           readOnlyTree,
1052 #endif
1053 #if PG_VERSION_NUM >= 90300
1054                                                           context, params,
1055 #if PG_VERSION_NUM >= 100000
1056                                                           queryEnv,
1057 #endif
1058 #else
1059                                                           params, isTopLevel,
1060 #endif
1061                                                           dest, completionTag);
1062         else
1063                 standard_ProcessUtility(parsetree, queryString,
1064 #if PG_VERSION_NUM >= 140000
1065                                                                 readOnlyTree,
1066 #endif
1067 #if PG_VERSION_NUM >= 90300
1068                                                                 context, params,
1069 #if PG_VERSION_NUM >= 100000
1070                                                                 queryEnv,
1071 #endif
1072 #else
1073                                                                 params, isTopLevel,
1074 #endif
1075                                                                 dest, completionTag);
1076
1077 #if PG_VERSION_NUM >= 100000
1078 #undef parsetree
1079 #endif
1080
1081         if (tblnames) {
1082                 ListCell        *l;
1083
1084                 foreach(l, tblnames)
1085                 {
1086                         RangeVar        *tblname =
1087 #if PG_VERSION_NUM >= 110000
1088                                 (IsA(lfirst(l), VacuumRelation)) ?
1089                                         ((VacuumRelation*)lfirst(l))->relation :
1090 #endif
1091                                         (RangeVar*)lfirst(l);
1092                         Oid     tblOid;
1093
1094                         Assert(IsA(tblname, RangeVar));
1095
1096                         tblOid = RangeVarGetRelid(tblname, NoLock, true);
1097                         makeAnalyze(tblOid, op, -1);
1098                 }
1099         }
1100 }
1101 #endif
1102
1103
1104 static void
1105 relstatsInit(void)
1106 {
1107         HASHCTL hash_ctl;
1108         int             flags = 0;
1109
1110         MemSet(&hash_ctl, 0, sizeof(hash_ctl));
1111
1112         hash_ctl.hash = oid_hash;
1113         flags |= HASH_FUNCTION;
1114
1115         if (onlineAnalyzeMemoryContext)
1116         {
1117                 Assert(relstats != NULL);
1118                 MemoryContextReset(onlineAnalyzeMemoryContext);
1119         }
1120         else
1121         {
1122                 Assert(relstats == NULL);
1123
1124 #if PG_VERSION_NUM < 90600
1125                 onlineAnalyzeMemoryContext =
1126                         AllocSetContextCreate(CacheMemoryContext,
1127                         "online_analyze storage context",
1128                         ALLOCSET_DEFAULT_MINSIZE,
1129                         ALLOCSET_DEFAULT_INITSIZE,
1130                         ALLOCSET_DEFAULT_MAXSIZE
1131                         );
1132 #else
1133                 onlineAnalyzeMemoryContext =
1134                         AllocSetContextCreate(CacheMemoryContext,
1135                         "online_analyze storage context", ALLOCSET_DEFAULT_SIZES);
1136 #endif
1137         }
1138
1139         hash_ctl.hcxt = onlineAnalyzeMemoryContext;
1140         flags |= HASH_CONTEXT;
1141
1142         hash_ctl.keysize = sizeof(Oid);
1143
1144         hash_ctl.entrysize = sizeof(OnlineAnalyzeTableStat);
1145         flags |= HASH_ELEM;
1146
1147         relstats = hash_create("online_analyze storage", 1024, &hash_ctl, flags);
1148 }
1149
1150 void _PG_init(void);
1151 void
1152 _PG_init(void)
1153 {
1154         relstatsInit();
1155
1156         oldExecutorEndHook = ExecutorEnd_hook;
1157
1158         ExecutorEnd_hook = onlineAnalyzeHooker;
1159
1160 #if PG_VERSION_NUM >= 90200
1161         oldProcessUtilityHook = ProcessUtility_hook;
1162
1163         ProcessUtility_hook = onlineAnalyzeHookerUtility;
1164 #endif
1165
1166
1167         DefineCustomBoolVariable(
1168                 "online_analyze.enable",
1169                 "Enable on-line analyze",
1170                 "Enables analyze of table directly after insert/update/delete/select into",
1171                 &online_analyze_enable,
1172 #if PG_VERSION_NUM >= 80400
1173                 online_analyze_enable,
1174 #endif
1175                 PGC_USERSET,
1176 #if PG_VERSION_NUM >= 80400
1177                 GUC_NOT_IN_SAMPLE,
1178 #if PG_VERSION_NUM >= 90100
1179                 NULL,
1180 #endif
1181 #endif
1182                 NULL,
1183                 NULL
1184         );
1185
1186         DefineCustomBoolVariable(
1187                 "online_analyze.local_tracking",
1188                 "Per backend tracking",
1189                 "Per backend tracking for temp tables (do not use system statistic)",
1190                 &online_analyze_local_tracking,
1191 #if PG_VERSION_NUM >= 80400
1192                 online_analyze_local_tracking,
1193 #endif
1194                 PGC_USERSET,
1195 #if PG_VERSION_NUM >= 80400
1196                 GUC_NOT_IN_SAMPLE,
1197 #if PG_VERSION_NUM >= 90100
1198                 NULL,
1199 #endif
1200 #endif
1201                 NULL,
1202                 NULL
1203         );
1204
1205         DefineCustomBoolVariable(
1206                 "online_analyze.verbose",
1207                 "Verbosity of on-line analyze",
1208                 "Make ANALYZE VERBOSE after table's changes",
1209                 &online_analyze_verbose,
1210 #if PG_VERSION_NUM >= 80400
1211                 online_analyze_verbose,
1212 #endif
1213                 PGC_USERSET,
1214 #if PG_VERSION_NUM >= 80400
1215                 GUC_NOT_IN_SAMPLE,
1216 #if PG_VERSION_NUM >= 90100
1217                 NULL,
1218 #endif
1219 #endif
1220                 NULL,
1221                 NULL
1222         );
1223
1224         DefineCustomRealVariable(
1225                 "online_analyze.scale_factor",
1226                 "fraction of table size to start on-line analyze",
1227                 "fraction of table size to start on-line analyze",
1228                 &online_analyze_scale_factor,
1229 #if PG_VERSION_NUM >= 80400
1230                 online_analyze_scale_factor,
1231 #endif
1232                 0.0,
1233                 1.0,
1234                 PGC_USERSET,
1235 #if PG_VERSION_NUM >= 80400
1236                 GUC_NOT_IN_SAMPLE,
1237 #if PG_VERSION_NUM >= 90100
1238                 NULL,
1239 #endif
1240 #endif
1241                 NULL,
1242                 NULL
1243         );
1244
1245         DefineCustomIntVariable(
1246                 "online_analyze.threshold",
1247                 "min number of row updates before on-line analyze",
1248                 "min number of row updates before on-line analyze",
1249                 &online_analyze_threshold,
1250 #if PG_VERSION_NUM >= 80400
1251                 online_analyze_threshold,
1252 #endif
1253                 0,
1254                 0x7fffffff,
1255                 PGC_USERSET,
1256 #if PG_VERSION_NUM >= 80400
1257                 GUC_NOT_IN_SAMPLE,
1258 #if PG_VERSION_NUM >= 90100
1259                 NULL,
1260 #endif
1261 #endif
1262                 NULL,
1263                 NULL
1264         );
1265
1266         DefineCustomIntVariable(
1267                 "online_analyze.capacity_threshold",
1268                 "Max local cache table capacity",
1269                 "Max local cache table capacity",
1270                 &online_analyze_capacity_threshold,
1271 #if PG_VERSION_NUM >= 80400
1272                 online_analyze_capacity_threshold,
1273 #endif
1274                 0,
1275                 0x7fffffff,
1276                 PGC_USERSET,
1277 #if PG_VERSION_NUM >= 80400
1278                 GUC_NOT_IN_SAMPLE,
1279 #if PG_VERSION_NUM >= 90100
1280                 NULL,
1281 #endif
1282 #endif
1283                 NULL,
1284                 NULL
1285         );
1286
1287         DefineCustomRealVariable(
1288                 "online_analyze.min_interval",
1289                 "minimum time interval between analyze call (in milliseconds)",
1290                 "minimum time interval between analyze call (in milliseconds)",
1291                 &online_analyze_min_interval,
1292 #if PG_VERSION_NUM >= 80400
1293                 online_analyze_min_interval,
1294 #endif
1295                 0.0,
1296                 1e30,
1297                 PGC_USERSET,
1298 #if PG_VERSION_NUM >= 80400
1299                 GUC_NOT_IN_SAMPLE,
1300 #if PG_VERSION_NUM >= 90100
1301                 NULL,
1302 #endif
1303 #endif
1304                 NULL,
1305                 NULL
1306         );
1307
1308         DefineCustomEnumVariable(
1309                 "online_analyze.table_type",
1310                 "Type(s) of table for online analyze: all(default), persistent, temporary, none",
1311                 NULL,
1312                 &online_analyze_table_type,
1313 #if PG_VERSION_NUM >= 80400
1314                 online_analyze_table_type,
1315 #endif
1316                 online_analyze_table_type_options,
1317                 PGC_USERSET,
1318 #if PG_VERSION_NUM >= 80400
1319                 GUC_NOT_IN_SAMPLE,
1320 #if PG_VERSION_NUM >= 90100
1321                 NULL,
1322 #endif
1323 #endif
1324                 NULL,
1325                 NULL
1326         );
1327
1328         DefineCustomStringVariable(
1329                 "online_analyze.exclude_tables",
1330                 "List of tables which will not online analyze",
1331                 NULL,
1332                 &excludeTables.tableStr,
1333 #if PG_VERSION_NUM >= 80400
1334                 "",
1335 #endif
1336                 PGC_USERSET,
1337                 0,
1338 #if PG_VERSION_NUM >= 90100
1339                 excludeTablesCheck,
1340                 excludeTablesAssign,
1341 #else
1342                 excludeTablesAssign,
1343 #endif
1344                 excludeTablesShow
1345         );
1346
1347         DefineCustomStringVariable(
1348                 "online_analyze.include_tables",
1349                 "List of tables which will online analyze",
1350                 NULL,
1351                 &includeTables.tableStr,
1352 #if PG_VERSION_NUM >= 80400
1353                 "",
1354 #endif
1355                 PGC_USERSET,
1356                 0,
1357 #if PG_VERSION_NUM >= 90100
1358                 includeTablesCheck,
1359                 includeTablesAssign,
1360 #else
1361                 includeTablesAssign,
1362 #endif
1363                 includeTablesShow
1364         );
1365
1366         DefineCustomIntVariable(
1367                 "online_analyze.lower_limit",
1368                 "min number of rows in table to analyze",
1369                 "min number of rows in table to analyze",
1370                 &online_analyze_lower_limit,
1371 #if PG_VERSION_NUM >= 80400
1372                 online_analyze_lower_limit,
1373 #endif
1374                 0,
1375                 0x7fffffff,
1376                 PGC_USERSET,
1377 #if PG_VERSION_NUM >= 80400
1378                 GUC_NOT_IN_SAMPLE,
1379 #if PG_VERSION_NUM >= 90100
1380                 NULL,
1381 #endif
1382 #endif
1383                 NULL,
1384                 NULL
1385         );
1386
1387         RegisterXactCallback(removeTable, NULL);
1388 }
1389
1390 #if PG_VERSION_NUM < 150000
1391 void _PG_fini(void);
1392 void
1393 _PG_fini(void)
1394 {
1395         ExecutorEnd_hook = oldExecutorEndHook;
1396 #if PG_VERSION_NUM >= 90200
1397         ProcessUtility_hook = oldProcessUtilityHook;
1398 #endif
1399
1400         if (excludeTables.tables)
1401                 free(excludeTables.tables);
1402         if (includeTables.tables)
1403                 free(includeTables.tables);
1404
1405         excludeTables.tables = includeTables.tables = NULL;
1406         excludeTables.nTables = includeTables.nTables = 0;
1407 }
1408 #endif