From 616042000fa85135606f626940ca1c73d3a58b36 Mon Sep 17 00:00:00 2001 From: teodor Date: Thu, 19 Oct 2006 13:04:54 +0000 Subject: [PATCH] Add common c-utils, make flatdb aligned on disk --- expected/flatdb | 262 ++++++++++++++++++++++-------------------------- flatdb.c | 14 ++- flatdb.h | 3 +- tbtree.h | 6 +- tests/flatdb | 8 +- tmalloc.h | 6 +- tools.h | 15 +++ 7 files changed, 155 insertions(+), 159 deletions(-) diff --git a/expected/flatdb b/expected/flatdb index 2bdea6c..29e3016 100644 --- a/expected/flatdb +++ b/expected/flatdb @@ -1,150 +1,128 @@ Put: off:16 len:4 'ariz' Put: off:24 len:6 'ddk29f' -Put: off:34 len:1 's' -Put: off:39 len:4 'a99t' -Put: off:47 len:7 'suf8eee' -Put: off:58 len:4 'z0kv' -Put: off:66 len:3 'lw2' -Put: off:73 len:4 'twv4' -Put: off:81 len:4 '8co6' -Put: off:89 len:9 'u2nie4xqc' -Put: off:102 len:3 'qgw' -Put: off:109 len:6 'cnpebn' -Put: off:119 len:10 'mmlkvhe6ox' -Put: off:133 len:4 'qite' -Put: off:141 len:9 'nqzpq8aqj' -Put: off:154 len:9 'ill7niztn' -Put: off:167 len:9 'khpq87qew' -Put: off:180 len:1 'q' -Put: off:185 len:2 'wz' -Put: off:191 len:5 'wgrmy' -Put: off:200 len:8 '8ch2icos' -Put: off:212 len:4 'v4xc' -Put: off:220 len:10 'ljklkyz5ud' -Put: off:234 len:6 '72jzxf' -Put: off:244 len:7 'r4ysgda' -Put: off:255 len:2 '9r' -Put: off:261 len:1 'x' -Put: off:266 len:5 '7o8gh' -Put: off:275 len:2 'yi' -Put: off:281 len:10 'oxr8qbhgwl' -Put: off:295 len:6 'y4zkee' -Put: off:305 len:4 'yjin' -Put: off:313 len:5 'ycblv' -Put: off:322 len:1 'q' -Put: off:327 len:9 'nlbzzooco' -Put: off:340 len:4 '1flp' -Put: off:348 len:5 'xphng' -Put: off:357 len:10 'avjrii116x' -Put: off:371 len:9 'fyuxgehv4' -Put: off:384 len:2 'ia' +Put: off:36 len:1 's' +Put: off:44 len:4 'a99t' +Put: off:52 len:7 'suf8eee' +Put: off:64 len:4 'z0kv' +Put: off:72 len:3 'lw2' +Put: off:80 len:4 'twv4' +Put: off:88 len:4 '8co6' +Put: off:96 len:9 'u2nie4xqc' +Put: off:112 len:3 'qgw' +Put: off:120 len:6 'cnpebn' +Put: off:132 len:10 'mmlkvhe6ox' +Put: off:148 len:4 'qite' +Put: off:156 len:9 'nqzpq8aqj' +Put: off:172 len:9 'ill7niztn' +Put: off:188 len:9 'khpq87qew' +Put: off:204 len:1 'q' +Put: off:212 len:2 'wz' +Put: off:220 len:5 'wgrmy' +Put: off:232 len:8 '8ch2icos' +Put: off:244 len:4 'v4xc' +Put: off:252 len:10 'ljklkyz5ud' +Put: off:268 len:6 '72jzxf' +Put: off:280 len:7 'r4ysgda' +Put: off:292 len:2 '9r' +Put: off:300 len:1 'x' +Put: off:308 len:5 '7o8gh' +Put: off:320 len:2 'yi' +Put: off:328 len:10 'oxr8qbhgwl' +Put: off:344 len:6 'y4zkee' +Put: off:356 len:4 'yjin' +Put: off:364 len:5 'ycblv' +Put: off:376 len:1 'q' +Put: off:384 len:9 'nlbzzooco' +Put: off:400 len:4 '1flp' +Put: off:408 len:5 'xphng' +Put: off:420 len:10 'avjrii116x' +Put: off:436 len:9 'fyuxgehv4' +Put: off:452 len:2 'ia' Get: off:16 len:4 'ariz' -Get: off:322 len:1 'q' -Get: off:340 len:4 '1flp' -Get: off:348 len:5 'xphng' Get: off:24 len:6 'ddk29f' -Get: off:34 len:1 's' -Get: off:39 len:4 'a99t' -Get: off:47 len:7 'suf8eee' -Get: off:58 len:4 'z0kv' -Get: off:234 len:6 '72jzxf' -Get: off:244 len:7 'r4ysgda' -Get: off:119 len:10 'mmlkvhe6ox' -Get: off:133 len:4 'qite' -Get: off:141 len:9 'nqzpq8aqj' -Get: off:154 len:9 'ill7niztn' -Get: off:167 len:9 'khpq87qew' -Get: off:34 len:1 's' -Get: off:39 len:4 'a99t' -Get: off:47 len:7 'suf8eee' -Get: off:58 len:4 'z0kv' -Get: off:234 len:6 '72jzxf' -Get: off:244 len:7 'r4ysgda' -Get: off:119 len:10 'mmlkvhe6ox' -Get: off:180 len:1 'q' -Get: off:185 len:2 'wz' -Get: off:191 len:5 'wgrmy' -Get: off:200 len:8 '8ch2icos' -Get: off:327 len:9 'nlbzzooco' -Get: off:212 len:4 'v4xc' -Get: off:220 len:10 'ljklkyz5ud' -Get: off:266 len:5 '7o8gh' -Get: off:255 len:2 '9r' -Get: off:261 len:1 'x' -Get: off:66 len:3 'lw2' -Get: off:180 len:1 'q' -Get: off:185 len:2 'wz' -Get: off:191 len:5 'wgrmy' -Get: off:200 len:8 '8ch2icos' -Get: off:327 len:9 'nlbzzooco' -Get: off:73 len:4 'twv4' -Get: off:81 len:4 '8co6' -Get: off:89 len:9 'u2nie4xqc' -Get: off:102 len:3 'qgw' -Get: off:109 len:6 'cnpebn' -Get: off:275 len:2 'yi' -Get: off:281 len:10 'oxr8qbhgwl' -Get: off:295 len:6 'y4zkee' -Get: off:305 len:4 'yjin' -Get: off:313 len:5 'ycblv' -Get: off:357 len:10 'avjrii116x' -Get: off:371 len:9 'fyuxgehv4' -Get: off:384 len:2 'ia' +Get: off:36 len:1 's' +Get: off:44 len:4 'a99t' +Get: off:52 len:7 'suf8eee' +Get: off:64 len:4 'z0kv' +Get: off:72 len:3 'lw2' +Get: off:80 len:4 'twv4' +Get: off:88 len:4 '8co6' +Get: off:96 len:9 'u2nie4xqc' +Get: off:112 len:3 'qgw' +Get: off:120 len:6 'cnpebn' +Get: off:132 len:10 'mmlkvhe6ox' +Get: off:148 len:4 'qite' +Get: off:156 len:9 'nqzpq8aqj' +Get: off:172 len:9 'ill7niztn' +Get: off:188 len:9 'khpq87qew' +Get: off:204 len:1 'q' +Get: off:212 len:2 'wz' +Get: off:220 len:5 'wgrmy' +Get: off:232 len:8 '8ch2icos' +Get: off:244 len:4 'v4xc' +Get: off:252 len:10 'ljklkyz5ud' +Get: off:268 len:6 '72jzxf' +Get: off:280 len:7 'r4ysgda' +Get: off:292 len:2 '9r' +Get: off:300 len:1 'x' +Get: off:308 len:5 '7o8gh' +Get: off:320 len:2 'yi' +Get: off:328 len:10 'oxr8qbhgwl' +Get: off:344 len:6 'y4zkee' +Get: off:356 len:4 'yjin' +Get: off:364 len:5 'ycblv' +Get: off:376 len:1 'q' +Get: off:384 len:9 'nlbzzooco' +Get: off:400 len:4 '1flp' +Get: off:408 len:5 'xphng' +Get: off:420 len:10 'avjrii116x' +Get: off:436 len:9 'fyuxgehv4' +Get: off:452 len:2 'ia' Del: off:16 Del: off:24 -Del: off:34 -Del: off:266 -Del: off:47 -Del: off:185 +Del: off:36 +Del: off:308 +Del: off:52 +Del: off:212 Vacuum -Get: off:322 len:1 'q' -Get: off:340 len:4 '1flp' -Get: off:348 len:5 'xphng' -Get: off:39 len:4 'a99t' -Get: off:58 len:4 'z0kv' -Get: off:234 len:6 '72jzxf' -Get: off:244 len:7 'r4ysgda' -Get: off:119 len:10 'mmlkvhe6ox' -Get: off:133 len:4 'qite' -Get: off:141 len:9 'nqzpq8aqj' -Get: off:154 len:9 'ill7niztn' -Get: off:167 len:9 'khpq87qew' -Get: off:34 len:1 's' -Get: off:39 len:4 'a99t' -Get: off:58 len:4 'z0kv' -Get: off:234 len:6 '72jzxf' -Get: off:244 len:7 'r4ysgda' -Get: off:119 len:10 'mmlkvhe6ox' -Get: off:180 len:1 'q' -Get: off:191 len:5 'wgrmy' -Get: off:200 len:8 '8ch2icos' -Get: off:327 len:9 'nlbzzooco' -Get: off:212 len:4 'v4xc' -Get: off:220 len:10 'ljklkyz5ud' -Get: off:255 len:2 '9r' -Get: off:261 len:1 'x' -Get: off:66 len:3 'lw2' -Get: off:180 len:1 'q' -Get: off:185 len:2 'wz' -Get: off:191 len:5 'wgrmy' -Get: off:200 len:8 '8ch2icos' -Get: off:327 len:9 'nlbzzooco' -Get: off:73 len:4 'twv4' -Get: off:81 len:4 '8co6' -Get: off:89 len:9 'u2nie4xqc' -Get: off:102 len:3 'qgw' -Get: off:109 len:6 'cnpebn' -Get: off:275 len:2 'yi' -Get: off:281 len:10 'oxr8qbhgwl' -Get: off:295 len:6 'y4zkee' -Get: off:305 len:4 'yjin' -Get: off:313 len:5 'ycblv' -Get: off:357 len:10 'avjrii116x' -Get: off:371 len:9 'fyuxgehv4' -Get: off:384 len:2 'ia' +Get: off:36 len:1 's' +Get: off:44 len:4 'a99t' +Get: off:64 len:4 'z0kv' +Get: off:72 len:3 'lw2' +Get: off:80 len:4 'twv4' +Get: off:88 len:4 '8co6' +Get: off:96 len:9 'u2nie4xqc' +Get: off:112 len:3 'qgw' +Get: off:120 len:6 'cnpebn' +Get: off:132 len:10 'mmlkvhe6ox' +Get: off:148 len:4 'qite' +Get: off:156 len:9 'nqzpq8aqj' +Get: off:172 len:9 'ill7niztn' +Get: off:188 len:9 'khpq87qew' +Get: off:204 len:1 'q' +Get: off:220 len:5 'wgrmy' +Get: off:232 len:8 '8ch2icos' +Get: off:244 len:4 'v4xc' +Get: off:252 len:10 'ljklkyz5ud' +Get: off:268 len:6 '72jzxf' +Get: off:280 len:7 'r4ysgda' +Get: off:292 len:2 '9r' +Get: off:300 len:1 'x' +Get: off:320 len:2 'yi' +Get: off:328 len:10 'oxr8qbhgwl' +Get: off:344 len:6 'y4zkee' +Get: off:356 len:4 'yjin' +Get: off:364 len:5 'ycblv' +Get: off:376 len:1 'q' +Get: off:384 len:9 'nlbzzooco' +Get: off:400 len:4 '1flp' +Get: off:408 len:5 'xphng' +Get: off:420 len:10 'avjrii116x' +Get: off:436 len:9 'fyuxgehv4' +Get: off:452 len:2 'ia' List of free space: - off: 16 len: 23 - off: 47 len: 11 - off: 185 len: 6 - off: 266 len: 9 - off: 390 len: 60 + off: 16 len: 28 + off: 52 len: 12 + off: 212 len: 8 + off: 308 len: 12 + off: 460 len: 60 diff --git a/flatdb.c b/flatdb.c index 56d78eb..10b3848 100644 --- a/flatdb.c +++ b/flatdb.c @@ -210,6 +210,7 @@ FDBClose(FDB *db) { FDBVacuumFreeSpace(db); header.lenfreespace = sizeof(FDBFreeSpace)*db->listcur; + ptr = findFreeSpace( db, header.lenfreespace ); if ( ptr ) { @@ -265,7 +266,7 @@ FDBDelete(FDB *db, off_t offset, size_t length) { if ( readLen(db, offset, &length) != FDB_OK ) return FDB_ERROR; - addFreeSpace(db, offset, length); + addFreeSpace(db, offset, PTRALIGN(length)); return FDB_OK; } @@ -311,11 +312,14 @@ FDBGet(FDB *db, off_t offset, size_t length, FDBRecord **record) { int FDBPut(FDB *db, FDBRecord *record, off_t *offset ) { FDBFreeSpace *ptr; + size_t aligned; if ( db->readonly ) return FDB_ERROR; - ptr = findFreeSpace( db, record->length ); + aligned = PTRALIGN(record->length); + + ptr = findFreeSpace( db, aligned ); if ( ptr ) { *offset = ptr->offset; ptr->length -= record->length; @@ -334,9 +338,13 @@ FDBPut(FDB *db, FDBRecord *record, off_t *offset ) { if ( write(db->fd, record, record->length) != record->length ) tlog(TL_CRIT|TL_EXIT,"FDBPut: write failed: %s", strerror(errno)); + if ( record->length != aligned ) { + char buf[] = {0, 0, 0, 0, 0, 0, 0, 0}; + if ( write(db->fd, buf, aligned - record->length) != (aligned - record->length) ) + tlog(TL_CRIT|TL_EXIT,"FDBPut: write failed: %s", strerror(errno)); + } return FDB_OK; - } diff --git a/flatdb.h b/flatdb.h index 5c3a4da..c94618b 100644 --- a/flatdb.h +++ b/flatdb.h @@ -30,6 +30,7 @@ #define __FLAT_DB__ #include +#include "tools.h" typedef struct { off_t freespace; @@ -44,7 +45,7 @@ typedef struct { char data[1]; } FDBRecord; -#define RECHDRSZ (sizeof(size_t)) +#define RECHDRSZ (offsetof(FDBRecord, data)) typedef struct { off_t offset; diff --git a/tbtree.h b/tbtree.h index 3971fda..28d8837 100644 --- a/tbtree.h +++ b/tbtree.h @@ -32,11 +32,7 @@ #include - -/* C-utils */ -#ifndef offsetof -#define offsetof(type, field) ((int) &((type *)0)->field) -#endif /* offsetof */ +#include "tools.h" #define HASHSIZE(LEN) ( (LEN)<<1 ) /* end utils */ diff --git a/tests/flatdb b/tests/flatdb index 3f111c1..02f330e 100644 --- a/tests/flatdb +++ b/tests/flatdb @@ -1,12 +1,14 @@ [ -f temp/FLAT ] && rm -rf temp/FLAT ./flatdbtest -f temp/FLAT -p ariz -p ddk29f -p s -p a99t -p suf8eee -p z0kv -p lw2 -p twv4 -p 8co6 -p u2nie4xqc -p qgw -p cnpebn -p mmlkvhe6ox -p qite -p nqzpq8aqj -p ill7niztn -p khpq87qew -p q -p wz -p wgrmy -p 8ch2icos -p v4xc -p ljklkyz5ud -p 72jzxf -p r4ysgda -p 9r -p x -p 7o8gh -p yi -p oxr8qbhgwl -p y4zkee -p yjin -p ycblv -p q -p nlbzzooco -p 1flp -p xphng -p avjrii116x -p fyuxgehv4 -p ia || exit 1 -./flatdbtest -r -f temp/FLAT -g 16 -g 322 -g 340 -g 348 -g 24 -g 34 -g 39 -g 47 -g 58 -g 234 -g 244 -g 119 -g 133 -g 141 -g 154 -g 167 -g 34 -g 39 -g 47 -g 58 -g 234 -g 244 -g 119 -g 180 -g 185 -g 191 -g 200 -g 327 -g 212 -g 220 -g 266 -g 255 -g 261 -g 66 -g 180 -g 185 -g 191 -g 200 -g 327 -g 73 -g 81 -g 89 -g 102 -g 109 -g 275 -g 281 -g 295 -g 305 -g 313 -g 357 -g 371 -g 384 || exit 1 +./flatdbtest -r -f temp/FLAT -g 16 -g 24 -g 36 -g 44 -g 52 -g 64 -g 72 -g 80 -g 88 -g 96 -g 112 -g 120 -g 132 -g 148 -g 156 -g 172 -g 188 -g 204 -g 212 -g 220 -g 232 -g 244 -g 252 -g 268 -g 280 -g 292 -g 300 -g 308 -g 320 -g 328 -g 344 -g 356 -g 364 -g 376 -g 384 -g 400 -g 408 -g 420 -g 436 -g 452 || exit 1 -./flatdbtest -f temp/FLAT -d 16 -d 24 -d 34 -d 266 -d 47 -d 185 || exit 1 +./flatdbtest -f temp/FLAT -d 16 -d 24 -d 36 -d 308 -d 52 -d 212 || exit 1 ./flatdbtest -f temp/FLAT -v || exit 1 -./flatdbtest -r -f temp/FLAT -g 322 -g 340 -g 348 -g 39 -g 58 -g 234 -g 244 -g 119 -g 133 -g 141 -g 154 -g 167 -g 34 -g 39 -g 58 -g 234 -g 244 -g 119 -g 180 -g 191 -g 200 -g 327 -g 212 -g 220 -g 255 -g 261 -g 66 -g 180 -g 185 -g 191 -g 200 -g 327 -g 73 -g 81 -g 89 -g 102 -g 109 -g 275 -g 281 -g 295 -g 305 -g 313 -g 357 -g 371 -g 384 || exit 1 +./flatdbtest -r -f temp/FLAT -g 36 -g 44 -g 64 -g 72 -g 80 -g 88 -g 96 -g 112 -g 120 -g 132 -g 148 -g 156 -g 172 -g 188 -g 204 -g 220 -g 232 -g 244 -g 252 -g 268 -g 280 -g 292 -g 300 -g 320 -g 328 -g 344 -g 356 -g 364 -g 376 -g 384 -g 400 -g 408 -g 420 -g 436 -g 452 || exit 1 ./flatdbtest -f temp/FLAT -s || exit 1 + + diff --git a/tmalloc.h b/tmalloc.h index fce5dba..a39a3ff 100644 --- a/tmalloc.h +++ b/tmalloc.h @@ -31,6 +31,7 @@ #define __TMALLOC_H__ #include +#include "tools.h" void * tmalloc(size_t size); void * trealloc(void * ptr, size_t size); @@ -74,11 +75,6 @@ typedef struct { #define MCASHDRSZ ( sizeof(size_t) + sizeof(MemoryContext*) ) #define MCMAGICKNUMBER (0xFE0FBEEF) -#define TYPEALIGN(ALIGNVAL,LEN) \ - (((long) (LEN) + (ALIGNVAL-1)) & ~((long) (ALIGNVAL-1))) - -#define PTRALIGN(LEN) TYPEALIGN(sizeof(void*), (LEN)) - MemoryContext *allocMemoryContext(MemoryContext* parent, int flags); void freeMemoryContext(MemoryContext* cntx); void* mcalloc(MemoryContext *cntx, size_t size); diff --git a/tools.h b/tools.h index d70d14d..6db856d 100644 --- a/tools.h +++ b/tools.h @@ -39,4 +39,19 @@ char* xtostr(u_int32_t x); double timediff(struct timeval *begin, struct timeval *end); double elapsedtime(struct timeval *begin); + +#ifndef offsetof +#define offsetof(type, field) ((int) &((type *)0)->field) +#endif /* offsetof */ + +#ifndef TYPEALIGN +#define TYPEALIGN(ALIGNVAL,LEN) \ + (((long) (LEN) + ((ALIGNVAL) - 1)) & ~((long) ((ALIGNVAL) - 1))) + +#define SHORTALIGN(LEN) TYPEALIGN(sizeof(int16), (LEN)) +#define INTALIGN(LEN) TYPEALIGN(sizeof(int32), (LEN)) +#define MAXALIGN(LEN) TYPEALIGN(sizeof(int64), (LEN)) +#define PTRALIGN(LEN) TYPEALIGN(sizeof(void*), (LEN)) +#endif + #endif -- 2.46.1