From 27ebeff879126434740af3db63004cb4eb4b0021 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 12 Feb 2016 16:03:05 +0100 Subject: Move code to util.c and filebuffer.c --- c_src/permdb.c | 464 ++++++++++----------------------------------------------- 1 file changed, 81 insertions(+), 383 deletions(-) (limited to 'c_src/permdb.c') diff --git a/c_src/permdb.c b/c_src/permdb.c index c5187be..7550db4 100644 --- a/c_src/permdb.c +++ b/c_src/permdb.c @@ -18,28 +18,19 @@ #include "erlport.h" #include "permdb.h" #include "hash.h" +#include "filebuffer.h" +#include "util.h" #define INDEX_COMMIT_TRAILER_SIZE (sizeof(uint64_t) + SHA256_DIGEST_SIZE + sizeof(index_commit_cookie)) static const int bitsperlevel = 2; static const int keylen = 32; -typedef struct { - int fd; - char *name; - node_offset datasize; - node_offset lastcommit; - node_offset filesize; - char *writebuffer; - uint64_t writebufferalloc; - struct sha256_ctx commit_checksum_context; -} buffered_file; - struct permdb_object { Hashtab *nodecache; Hashtab *dirtynodes; - buffered_file datafile; - buffered_file indexfile; + buffered_file *datafile; + buffered_file *indexfile; char *error; }; @@ -50,34 +41,6 @@ static const node_object errornode = {{NODE_ENTRY_ERROR_NODE, NODE_ENTRY_ERROR_NODE, NODE_ENTRY_ERROR_NODE}}; -int -calc_padding(int offset, int alignment) -{ - int misalign = offset % alignment; - if (misalign == 0) { - return 0; - } - return alignment - misalign; -} - - -#if 0 -static void -print_entry(node_object node) -{ - for (int i = 0; i < entriespernode; i++) { - fprintf(stderr, " %016llx", (long long unsigned int)node.data[i]); - } - fprintf(stderr, "\n"); -} -#endif - -static void -writebuffer_add(buffered_file *file, const void *data, uint64_t length); -static int -writebuffer_flush(buffered_file *file); -static uint64_t -writebuffer_length(buffered_file *file); struct nodecache { node_object value; @@ -113,23 +76,6 @@ hashnode(void *node_v) return hash; } -#if DEBUG_CACHE || 1 -static void -print_hex(const void *data, int length) -{ - int i; - for (i = 0; i < length; i++) { - fprintf(stderr, "%02X ", ((unsigned char *)data)[i]); - } - fprintf(stderr, "\n"); -} -#endif - -#define DEBUG_CACHE 0 -#define DEBUG_WRITE 0 -#define DEBUG_READ 0 -#define DEBUG_PORT 0 - static node_object get_node_from_cache(permdb_object *state, const char *key) { @@ -247,43 +193,22 @@ static const uint8_t data_commit_end_cookie[] = {0x2b, 0x05, 0xee, 0xd6, 0x1b, 0 int committree(permdb_object *state); -node_offset +void indexfile_add_header(buffered_file *file) { - writebuffer_add(file, &index_file_cookie, sizeof(index_file_cookie)); - uint64_t length = writebuffer_length(file); - writebuffer_flush(file); - return length; -} - -static void -writebuffer_add_host64(buffered_file *file, uint64_t value) { - writebuffer_add(file, &value, sizeof(uint64_t)); -} - -static void -writebuffer_add_be32(buffered_file *file, uint32_t value) { - uint32_t value_be = htonl(value); - writebuffer_add(file, &value_be, sizeof(uint32_t)); + bf_add(file, &index_file_cookie, sizeof(index_file_cookie)); + bf_flush(file); } -static void -writebuffer_add_be16(buffered_file *file, uint16_t value) { - uint16_t value_be = htons(value); - writebuffer_add(file, &value_be, sizeof(uint16_t)); -} - -node_offset +void datafile_add_header(buffered_file *file) { - fprintf(stderr, "adding header to %s\n", file->name); - writebuffer_add(file, &data_file_cookie, sizeof(data_file_cookie)); - writebuffer_add_be32(file, 4096); - writebuffer_add_be32(file, 2); - writebuffer_add_be32(file, 32); - uint64_t length = writebuffer_length(file); - writebuffer_flush(file); - return length; + //fprintf(stderr, "adding header to %s\n", bf_name(file)); + bf_add(file, &data_file_cookie, sizeof(data_file_cookie)); + bf_add_be32(file, 4096); + bf_add_be32(file, 2); + bf_add_be32(file, 32); + bf_flush(file); } void @@ -303,25 +228,6 @@ initial_commit(permdb_object *state) return committree(state); } -static void -set_error(permdb_object *state, const char * __restrict, ...) __attribute__ ((__format__ (__printf__, 2, 3))); - -unsigned char * -read_from_file(buffered_file *file, size_t length, off_t offset) -{ - unsigned char *buffer = malloc(length); - if (buffer == NULL) { - return NULL; - } - ssize_t ret = pread(file->fd, buffer, length, (off_t) offset); - if (ret != length) { - free(buffer); - return NULL; - } - return buffer; -} - - struct commit_info { node_offset start; node_offset length; @@ -332,7 +238,7 @@ int validate_checksum(struct commit_info *commit, buffered_file *file) { //fprintf(stderr, "validate_checksum: read from file: length %llu start %llu\n", commit->length, commit->start); - unsigned char *checksumdata = read_from_file(file, commit->length, commit->start); + unsigned char *checksumdata = bf_read(file, commit->start, commit->length, NULL); if (checksumdata == NULL) { return -1; @@ -355,22 +261,12 @@ validate_checksum(struct commit_info *commit, buffered_file *file) return -1; } -static uint64_t -read_host64(void *ptr) -{ - uint64_t data; - - memcpy(&data, ptr, sizeof(data)); - - return data; -} - int verify_index_commit(buffered_file *file, node_offset offset) { //fprintf(stderr, "verifying index file: commit verification\n"); offset -= INDEX_COMMIT_TRAILER_SIZE; - unsigned char *data = read_from_file(file, INDEX_COMMIT_TRAILER_SIZE, offset); + unsigned char *data = bf_read(file, offset, INDEX_COMMIT_TRAILER_SIZE, NULL); if (memcmp(data + sizeof(uint64_t) + SHA256_DIGEST_SIZE, &index_commit_cookie, sizeof(index_commit_cookie)) != 0) { fprintf(stderr, "verifying index file: incorrect commit cookie\n"); @@ -390,14 +286,14 @@ int indexfile_verify_file(buffered_file *file) { //fprintf(stderr, "verifying index file\n"); - unsigned char *header = read_from_file(file, sizeof(index_file_cookie), 0); + unsigned char *header = bf_read(file, 0, sizeof(index_file_cookie), NULL); if (memcmp(header, &index_file_cookie, sizeof(index_file_cookie)) != 0) { free(header); fprintf(stderr, "verifying index file: incorrect file cookie\n"); return -1; } free(header); - if (verify_index_commit(file, file->filesize) < 0) { + if (verify_index_commit(file, bf_total_length(file)) < 0) { fprintf(stderr, "verifying index file: commit verification failed\n"); return -1; } @@ -410,14 +306,14 @@ read_data_commit_backward(buffered_file *file, node_offset *offset); int datafile_verify_file(buffered_file *file) { - unsigned char *header = read_from_file(file, sizeof(data_file_cookie), 0); + unsigned char *header = bf_read(file, 0, sizeof(data_file_cookie), NULL); if (header == NULL || memcmp(header, &data_file_cookie, sizeof(data_file_cookie)) != 0) { free(header); return -1; } free(header); - node_offset offset = file->lastcommit; + node_offset offset = bf_lastcommit(file); //fprintf(stderr, "verifying commit: %llu\n", offset); struct commit_info *data_commit = read_data_commit_backward(file, &offset); @@ -433,16 +329,13 @@ datafile_verify_file(buffered_file *file) } -static uint32_t -read_be32(void *ptr); - static unsigned char * readdatakeyandlen(permdb_object *state, node_offset offset, size_t *datalen); struct commit_info * read_data_commit(buffered_file *file, node_offset *offset) { - unsigned char *data = read_from_file(file, sizeof(uint32_t) + SHA256_DIGEST_SIZE + sizeof(data_commit_end_cookie), *offset); + unsigned char *data = bf_read(file, *offset, sizeof(uint32_t) + SHA256_DIGEST_SIZE + sizeof(data_commit_end_cookie), NULL); if (data == NULL || memcmp(data + sizeof(uint32_t) + SHA256_DIGEST_SIZE, data_commit_end_cookie, sizeof(data_commit_end_cookie)) != 0) { free(data); return NULL; @@ -481,19 +374,14 @@ addindex(permdb_object *state, const unsigned char *key, unsigned int keylength, int rebuild_index_file(permdb_object *state) { - state->indexfile.filesize = 0; - state->indexfile.datasize = 0; - state->indexfile.lastcommit = state->indexfile.datasize; - sha256_init(&state->indexfile.commit_checksum_context); - ftruncate(state->indexfile.fd, 0); - - state->indexfile.lastcommit = indexfile_add_header(&state->indexfile); + bf_truncate(state->indexfile); + indexfile_add_header(state->indexfile); initial_node(state); node_offset offset = sizeof(data_file_cookie) + sizeof(uint32_t) * 3; while (1) { - unsigned char *cookie = read_from_file(&state->datafile, sizeof(data_entry_cookie), offset); + unsigned char *cookie = bf_read(state->datafile, offset, sizeof(data_entry_cookie), NULL); if (cookie == NULL) { break; } @@ -511,10 +399,10 @@ rebuild_index_file(permdb_object *state) return -1; } } else if (memcmp(&data_commit_start_cookie, cookie, sizeof(data_commit_start_cookie)) == 0) { - struct commit_info *data_commit = read_data_commit_forward(&state->datafile, &offset); + struct commit_info *data_commit = read_data_commit_forward(state->datafile, &offset); //fprintf(stderr, "verifying commit: %llu %p\n", offset, data_commit); - if (data_commit == NULL || validate_checksum(data_commit, &state->datafile) < 0) { + if (data_commit == NULL || validate_checksum(data_commit, state->datafile) < 0) { free(data_commit); fprintf(stderr, "commit broken: %llu\n", (unsigned long long) offset); free(cookie); @@ -542,75 +430,35 @@ permdb_alloc(const char *dbpath) return NULL; } - int fd; - int idxfd; - - int flags = O_RDWR|O_CREAT; - - fd = open(dbpath, flags, 0666); - if (fd == -1) { - warn("open %s", dbpath); - return NULL; - } - - idxfd = open(idxpath, flags, 0666); - if (idxfd == -1) { - warn("open %s", idxpath); - return NULL; - } + permdb_object *state = malloc(sizeof(permdb_object)); + state->datafile = bf_open(dbpath, O_RDWR|O_CREAT, "datafile"); + state->indexfile = bf_open(idxpath, O_RDWR|O_CREAT, "indexfile"); free(idxpath); - permdb_object *state = malloc(sizeof(permdb_object)); - state->datafile.fd = fd; - state->datafile.name = "datafile"; - state->indexfile.fd = idxfd; - state->indexfile.name = "indexfile"; state->nodecache = hashtabnewf(1000000, comparenodes, hashnode, HASHTAB_GROW); state->dirtynodes = hashtabnewf(1000000, comparenodes, hashnode, HASHTAB_GROW); - off_t datafile_filesize = lseek(fd, 0, SEEK_END); - if (datafile_filesize < 0) { - warn("lseek %s", dbpath); - return NULL; - } - state->datafile.filesize = (node_offset) datafile_filesize; - state->datafile.datasize = state->datafile.filesize; - state->datafile.lastcommit = state->datafile.datasize; - state->datafile.writebufferalloc = 1024*1024; - state->datafile.writebuffer = calloc(state->datafile.writebufferalloc, 1); - sha256_init(&state->datafile.commit_checksum_context); - off_t indexfile_filesize = lseek(idxfd, 0, SEEK_END); - if (indexfile_filesize < 0) { - warn("lseek %s", idxpath); - return NULL; - } - state->indexfile.filesize = (node_offset) indexfile_filesize; - state->indexfile.datasize = state->indexfile.filesize; - state->indexfile.lastcommit = state->indexfile.datasize; - state->indexfile.writebufferalloc = 1024*1024; - state->indexfile.writebuffer = calloc(state->indexfile.writebufferalloc, 1); - sha256_init(&state->indexfile.commit_checksum_context); state->error = NULL; - if (state->datafile.filesize == 0 && state->indexfile.filesize == 0) { + if (bf_total_length(state->datafile) == 0 && bf_total_length(state->indexfile) == 0) { #if DEBUG_WRITE fprintf(stderr, "writing header\n"); #endif - state->indexfile.lastcommit = indexfile_add_header(&state->indexfile); - state->datafile.lastcommit = datafile_add_header(&state->datafile); + indexfile_add_header(state->indexfile); + datafile_add_header(state->datafile); initial_commit(state); - } else if (state->datafile.filesize > 0 && state->indexfile.filesize == 0) { + } else if (bf_total_length(state->datafile) > 0 && bf_total_length(state->indexfile) == 0) { if (rebuild_index_file(state) < 0) { warnx("index file rebuilding failed"); permdb_free(state); return NULL; } } - if (datafile_verify_file(&state->datafile) < 0) { + if (datafile_verify_file(state->datafile) < 0) { warnx("data file verification failed"); permdb_free(state); return NULL; } - if (indexfile_verify_file(&state->indexfile) < 0) { + if (indexfile_verify_file(state->indexfile) < 0) { warnx("index file verification failed, rebuilding"); if (rebuild_index_file(state) < 0) { @@ -625,8 +473,8 @@ permdb_alloc(const char *dbpath) void permdb_free(permdb_object *state) { - free(state->datafile.writebuffer); - free(state->indexfile.writebuffer); + bf_close(state->datafile); + bf_close(state->indexfile); delete_all_nodes_in_cache(state); delete_all_dirty_nodes(state); hashtabrelease(state->dirtynodes); @@ -634,80 +482,6 @@ permdb_free(permdb_object *state) free(state); } -static uint64_t -writebuffer_length(buffered_file *file) -{ - return (uint64_t) file->datasize - (uint64_t) file->filesize; -} - -static int -writebuffer_flush_nosync(buffered_file *file); - -static void -writebuffer_add(buffered_file *file, const void *data, uint64_t length) -{ - sha256_update(&file->commit_checksum_context, length, data); -#if DEBUG_WRITE - fprintf(stderr, "adding data to %s: ", file->name); - print_hex(data, length); -#endif - uint64_t needspace = length + writebuffer_length(file); - - if (needspace > file->writebufferalloc) { - int ret = writebuffer_flush_nosync(file); - if (ret < 0) { - err(1, "writebuffer_flush_nosync failed"); - } - - needspace = length + writebuffer_length(file); - - if (needspace > file->writebufferalloc) { - - uint64_t newsize = file->writebufferalloc * 2; - if (needspace > newsize) { - newsize = needspace; - } - file->writebuffer = realloc(file->writebuffer, newsize); - memset(file->writebuffer + file->writebufferalloc, 0, newsize - file->writebufferalloc); - file->writebufferalloc = newsize; - } - } - - memcpy(file->writebuffer + writebuffer_length(file), data, length); - file->datasize += length; -} - -static int -writebuffer_flush_nosync(buffered_file *file) -{ - ssize_t ret; - - uint64_t length = writebuffer_length(file); - ret = write(file->fd, file->writebuffer, length); - if (ret != length) { - return -1; - } - file->filesize += (node_offset) ret; - return 0; -} - -static int -writebuffer_flush(buffered_file *file) -{ - int ret; - - ret = writebuffer_flush_nosync(file); - if (ret) { - return ret; - } - - ret = fsync(file->fd); - sha256_init(&file->commit_checksum_context); -#if DEBUG_WRITE - fprintf(stderr, "clearing data for %s\n", file->name); -#endif - return ret; -} static unsigned char keybits(const unsigned char *key, unsigned int level) @@ -755,34 +529,13 @@ get_entry_in_node(node_object node, unsigned char n) return node.data[n]; } -static void -set_error(permdb_object *state, const char *format, ...) -{ - va_list args; - int ret; - - va_start(args, format); - char *formatted = NULL; - ret = vasprintf (&formatted, format, args); - if (ret == -1) { - fprintf(stderr, "vasprintf error\n"); - return; - } - if (state->error) { - free(state->error); - } - fprintf(stderr, "error: %s\n", formatted); - state->error = formatted; - va_end(args); -} - static node_object -unpacknode(permdb_object *state, const char *data, size_t datalen) +unpacknode(permdb_object *state, const unsigned char *data, size_t datalen) { if (memcmp(&index_node_cookie, data, sizeof(index_node_cookie)) != 0) { print_hex(data, sizeof(index_node_cookie)); print_hex(&index_node_cookie, sizeof(index_node_cookie)); - set_error(state, "incorrect magic (node) %02x%02x\n", (unsigned char)data[0], (unsigned char)data[1]); + set_error(&state->error, "incorrect magic (node) %02x%02x\n", (unsigned char)data[0], (unsigned char)data[1]); return errornode; } @@ -800,34 +553,12 @@ unpacknode(permdb_object *state, const char *data, size_t datalen) unsigned char * read_internal_data(permdb_object *state, node_offset offset, size_t length) { - unsigned char *result = malloc(length); - buffered_file *file = &state->datafile; + buffered_file *file = state->datafile; #if DEBUG_READ fprintf(stderr, "reading data: offset %llu\n", offset); #endif - if (offset >= file->filesize) { - node_offset writebufferoffset = offset - file->filesize; - if (offset + length > file->datasize) { - set_error(state, "pread: not enough data for offset %llu and length %zu\n", (long long unsigned int) offset, length); - return NULL; - } - memcpy(result, file->writebuffer + writebufferoffset, length); - } else { - if (offset + length > file->filesize) { - set_error(state, "pread: trying to read over file/writebuffer boundary for offset %llu and length %zu\n", (long long unsigned int) offset, length); - return NULL; - } - - ssize_t ret = pread(file->fd, result, length, (off_t) offset); - if (ret != length) { - free(result); - set_error(state, "short pread: %zd (wanted %zu) at offset %llu\n", ret, length, (long long unsigned int) offset); - return NULL; - } - } - - return result; + return bf_read(file, offset, length, &state->error); } static int @@ -855,7 +586,7 @@ readnode(permdb_object *state, node_offset offset, const char *cachekey) } if (offset == NODE_ENTRY_DIRTY_NODE) { - set_error(state, "referring to dirty node at key %s, but node not in dirtynodes\n", cachekey); + set_error(&state->error, "referring to dirty node at key %s, but node not in dirtynodes\n", cachekey); #if DEBUG_READ fprintf(stderr, "reading node: referring to dirty node at key %s, but node not in dirtynodes\n", cachekey); #endif @@ -873,16 +604,10 @@ readnode(permdb_object *state, node_offset offset, const char *cachekey) size_t length = sizeof(index_node_cookie) + sizeof(node_object); - char *buffer = malloc(length); + unsigned char *buffer = bf_read(state->indexfile, offset, length, &state->error); if (buffer == NULL) { return errornode; } - ssize_t ret = pread(state->indexfile.fd, buffer, length, (off_t) offset); - if (ret != length) { - free(buffer); - set_error(state, "node not present at %llu: length %zd\n", (long long unsigned int) offset, ret); - return errornode; - } node_object result = unpacknode(state, buffer, length); @@ -945,18 +670,18 @@ getpath(permdb_object *state, const unsigned char *key, struct nodelist *nodes) unsigned int level = 0; #if 0 - if (state->indexfile.lastcommit < (sizeof(index_node_cookie) + sizeof(node_object) + INDEX_COMMIT_TRAILER_SIZE)) { - fprintf(stderr, "No commit exists (lastcommit %llu)\n", (long long unsigned int) state->indexfile.lastcommit); + if (bf_lastcommit(state->indexfile) < (sizeof(index_node_cookie) + sizeof(node_object) + INDEX_COMMIT_TRAILER_SIZE)) { + fprintf(stderr, "No commit exists (lastcommit %llu)\n", (long long unsigned int) bf_lastcommit(state->indexfile)); return -1; } #endif - node_offset rootoffset = state->indexfile.lastcommit - (sizeof(index_node_cookie) + sizeof(node_object) + INDEX_COMMIT_TRAILER_SIZE); + node_offset rootoffset = bf_lastcommit(state->indexfile) - (sizeof(index_node_cookie) + sizeof(node_object) + INDEX_COMMIT_TRAILER_SIZE); node_object node = readnode(state, rootoffset, ""); if (iserrornode(node)) { - fprintf(stderr, "cannot find root node at offset %llu (lastcommit %llu)\n", (long long unsigned int) rootoffset, (long long unsigned int) state->indexfile.lastcommit); + fprintf(stderr, "cannot find root node at offset %llu (lastcommit %llu)\n", (long long unsigned int) rootoffset, (long long unsigned int) bf_lastcommit(state->indexfile)); return -1; } @@ -998,7 +723,7 @@ getpathlastnode(permdb_object *state, const unsigned char *key) { unsigned int level = 0; - node_offset rootoffset = state->indexfile.lastcommit - (sizeof(index_node_cookie) + sizeof(node_object) + INDEX_COMMIT_TRAILER_SIZE); + node_offset rootoffset = bf_lastcommit(state->indexfile) - (sizeof(index_node_cookie) + sizeof(node_object) + INDEX_COMMIT_TRAILER_SIZE); node_object node = readnode(state, rootoffset, ""); if (iserrornode(node)) { @@ -1032,15 +757,15 @@ getpathlastnode(permdb_object *state, const unsigned char *key) static node_offset writenode(permdb_object *state, node_object node, const char *cachekey) { - node_offset offset = state->indexfile.datasize; + node_offset offset = bf_total_length(state->indexfile); put_node_in_cache(state, cachekey, node); #if DEBUG_WRITE fprintf(stderr, "writing node: offset %llu\n", offset); #endif - writebuffer_add(&state->indexfile, &index_node_cookie, sizeof(index_node_cookie)); - writebuffer_add(&state->indexfile, &node, sizeof(node_object)); + bf_add(state->indexfile, &index_node_cookie, sizeof(index_node_cookie)); + bf_add(state->indexfile, &node, sizeof(node_object)); return offset; } @@ -1048,7 +773,7 @@ writenode(permdb_object *state, node_object node, const char *cachekey) node_offset datasize(permdb_object *state) { - return state->indexfile.datasize; + return bf_total_length(state->indexfile); } @@ -1079,7 +804,7 @@ readdatakey(permdb_object *state, node_offset offset) } if (memcmp(&data_entry_cookie, data, sizeof(data_entry_cookie)) != 0) { free(data); - set_error(state, "incorrect magic (entry) %02x%02x\n", (unsigned char)data[0], (unsigned char)data[1]); + set_error(&state->error, "incorrect magic (entry) %02x%02x\n", (unsigned char)data[0], (unsigned char)data[1]); return NULL; } unsigned char *result = memsub(data, sizeof(data_entry_cookie), keylen); @@ -1087,26 +812,6 @@ readdatakey(permdb_object *state, node_offset offset) return result; } -static uint32_t -read_be32(void *ptr) -{ - uint32_t data; - - memcpy(&data, ptr, sizeof(data)); - - return ntohl(data); -} - -static uint16_t -read_be16(void *ptr) -{ - uint16_t data; - - memcpy(&data, ptr, sizeof(data)); - - return ntohs(data); -} - static unsigned char * readdatakeyandlen(permdb_object *state, node_offset offset, size_t *datalen) { @@ -1116,7 +821,7 @@ readdatakeyandlen(permdb_object *state, node_offset offset, size_t *datalen) } if (memcmp(&data_entry_cookie, data, sizeof(data_entry_cookie)) != 0) { free(data); - set_error(state, "incorrect magic (entry) %02x%02x\n", (unsigned char)data[0], (unsigned char)data[1]); + set_error(&state->error, "incorrect magic (entry) %02x%02x\n", (unsigned char)data[0], (unsigned char)data[1]); return NULL; } unsigned char *result = memsub(data, sizeof(data_entry_cookie), keylen); @@ -1142,16 +847,16 @@ writedata(permdb_object *state, const unsigned char *key, const unsigned char *d if (datalength > 65535) { errx(1, "data length is %zu, but only < 64K lengths are supported right now", datalength); } - node_offset offset = state->datafile.datasize; + node_offset offset = bf_total_length(state->datafile); #if DEBUG_WRITE fprintf(stderr, "writing data: offset %llu\n", offset); #endif - writebuffer_add(&state->datafile, &data_entry_cookie, sizeof(data_entry_cookie)); - writebuffer_add(&state->datafile, key, keylen); - writebuffer_add_be16(&state->datafile, 1); - writebuffer_add_be16(&state->datafile, datalength); - writebuffer_add(&state->datafile, data, datalength); + bf_add(state->datafile, &data_entry_cookie, sizeof(data_entry_cookie)); + bf_add(state->datafile, key, keylen); + bf_add_be16(state->datafile, 1); + bf_add_be16(state->datafile, datalength); + bf_add(state->datafile, data, datalength); return offset; } @@ -1450,7 +1155,7 @@ committree(permdb_object *state) unsigned int ncommits = ndirtynodes; unsigned int i; #if DEBUG_WRITE - fprintf(stderr, "committing %d dirty nodes at offset %llu\n", ncommits, state->indexfile.datasize); + fprintf(stderr, "committing %d dirty nodes at offset %llu\n", ncommits, bf_total_length(state->indexfile)); #endif for (i = 0; i < ncommits; i++) { get_node_from_dirtynodes(state, ""); @@ -1478,55 +1183,48 @@ committree(permdb_object *state) free(commitlist); #if DEBUG_WRITE - fprintf(stderr, "writing data commit trailer at offset %llu\n", state->datafile.datasize); + fprintf(stderr, "writing data commit trailer at offset %llu\n", bf_total_length(state->datafile)); #endif - int data_commit_padding_size = calc_padding(state->datafile.datasize, 4); + int data_commit_padding_size = calc_padding(bf_total_length(state->datafile), 4); uint8_t padding[4] = {0, 0, 0, 0}; unsigned char data_commit_checksum[SHA256_DIGEST_SIZE]; - writebuffer_add(&state->datafile, data_commit_start_cookie, 8); - writebuffer_add(&state->datafile, padding, data_commit_padding_size); - writebuffer_add_be32(&state->datafile, state->datafile.datasize - state->datafile.lastcommit + sizeof(uint32_t)); - sha256_digest(&state->datafile.commit_checksum_context, SHA256_DIGEST_SIZE, data_commit_checksum); - writebuffer_add(&state->datafile, data_commit_checksum, SHA256_DIGEST_SIZE); - writebuffer_add(&state->datafile, &data_commit_end_cookie, sizeof(data_commit_end_cookie)); + bf_add(state->datafile, data_commit_start_cookie, 8); + bf_add(state->datafile, padding, data_commit_padding_size); + bf_add_be32(state->datafile, bf_total_length(state->datafile) - bf_lastcommit(state->datafile) + sizeof(uint32_t)); + bf_sha256(state->datafile, data_commit_checksum); + bf_add(state->datafile, data_commit_checksum, SHA256_DIGEST_SIZE); + bf_add(state->datafile, &data_commit_end_cookie, sizeof(data_commit_end_cookie)); #if DEBUG_WRITE - fprintf(stderr, "finished writing data commit trailer at offset %llu\n", state->datafile.datasize); + fprintf(stderr, "finished writing data commit trailer at offset %llu\n", bf_total_length(state->datafile)); #endif - if (writebuffer_flush(&state->datafile) == -1) { - set_error(state, "data file flushing failed\n"); + if (bf_flush(state->datafile) == -1) { + set_error(&state->error, "data file flushing failed\n"); return -1; } - state->datafile.lastcommit = state->datafile.datasize; - #if DEBUG_WRITE - fprintf(stderr, "writing index commit trailer at offset %llu\n", state->indexfile.datasize); + fprintf(stderr, "writing index commit trailer at offset %llu\n", bf_total_length(state->indexfile)); #endif - uint64_t index_commit_length = state->indexfile.datasize - state->indexfile.lastcommit + sizeof(uint64_t); + uint64_t index_commit_length = bf_total_length(state->indexfile) - bf_lastcommit(state->indexfile) + sizeof(uint64_t); unsigned char index_commit_checksum[SHA256_DIGEST_SIZE]; - writebuffer_add_host64(&state->indexfile, index_commit_length); - sha256_digest(&state->indexfile.commit_checksum_context, SHA256_DIGEST_SIZE, index_commit_checksum); - writebuffer_add(&state->indexfile, index_commit_checksum, SHA256_DIGEST_SIZE); - writebuffer_add(&state->indexfile, &index_commit_cookie, sizeof(index_commit_cookie)); + bf_add_host64(state->indexfile, index_commit_length); + bf_sha256(state->indexfile, index_commit_checksum); + bf_add(state->indexfile, index_commit_checksum, SHA256_DIGEST_SIZE); + bf_add(state->indexfile, &index_commit_cookie, sizeof(index_commit_cookie)); #if DEBUG_WRITE - fprintf(stderr, "finished writing index commit trailer at offset %llu\n", state->indexfile.datasize); + fprintf(stderr, "finished writing index commit trailer at offset %llu\n", bf_total_length(state->indexfile)); #endif - if (writebuffer_flush(&state->indexfile) == -1) { - set_error(state, "index file flushing failed\n"); + if (bf_flush(state->indexfile) == -1) { + set_error(&state->error, "index file flushing failed\n"); return -1; } - state->indexfile.lastcommit = state->indexfile.datasize; -#if DEBUG_WRITE - fprintf(stderr, "setting lastcommit to %llu\n", state->indexfile.lastcommit); -#endif - delete_all_dirty_nodes(state); return 0; -- cgit v1.1