From ae374911b5ca5eab6df18522f9d3ec415b719b9d Mon Sep 17 00:00:00 2001 From: Teodor Sigaev Date: Tue, 15 May 2012 00:45:37 +0400 Subject: [PATCH] fix GiST estimation --- smlar_gist.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smlar_gist.c b/smlar_gist.c index ce9c6d8..3520e41 100644 --- a/smlar_gist.c +++ b/smlar_gist.c @@ -1210,7 +1210,7 @@ gsmlsign_consistent(PG_FUNCTION_ARGS) for(i=0; inelems; i++) count += GETBIT(sign, HASHVAL(s->hash[i])); - if ( s->nelems == count || ((double)count) / ((double)(s->nelems)) >= GetSmlarLimit() ) + if ( s->nelems == count || sqrt(((double)count) / ((double)(s->nelems))) >= GetSmlarLimit() ) res = true; } break; -- 2.37.3