fix GiST estimation
authorTeodor Sigaev <teodor@sigaev.ru>
Mon, 14 May 2012 20:45:37 +0000 (00:45 +0400)
committerTeodor Sigaev <teodor@sigaev.ru>
Mon, 14 May 2012 20:45:37 +0000 (00:45 +0400)
smlar_gist.c

index ce9c6d8..3520e41 100644 (file)
@@ -1210,7 +1210,7 @@ gsmlsign_consistent(PG_FUNCTION_ARGS)
                                                for(i=0; i<s->nelems; i++)
                                                        count += GETBIT(sign, HASHVAL(s->hash[i]));
                                
-                                               if ( s->nelems == count  || ((double)count) / ((double)(s->nelems)) >= GetSmlarLimit() )
+                                               if ( s->nelems == count  || sqrt(((double)count) / ((double)(s->nelems))) >= GetSmlarLimit() )
                                                        res = true;
                                        }
                                        break;