From d3d526123beb8083bb05dd9a784081a413331448 Mon Sep 17 00:00:00 2001 From: Linus Nordberg Date: Wed, 27 Apr 2016 14:56:45 +0200 Subject: Break long lines and adjust other whitespace. --- c_src/filebuffer.c | 33 ++++-- c_src/permdb.c | 330 +++++++++++++++++++++++++++++++++++++---------------- c_src/permdbport.c | 11 +- c_src/permdbpy.c | 24 +++- c_src/permdbtest.c | 35 ++++-- c_src/util.c | 10 +- 6 files changed, 317 insertions(+), 126 deletions(-) (limited to 'c_src') diff --git a/c_src/filebuffer.c b/c_src/filebuffer.c index 70d7827..82fd4d2 100644 --- a/c_src/filebuffer.c +++ b/c_src/filebuffer.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, NORDUnet A/S. + * Copyright (c) 2016, NORDUnet A/S. * See LICENSE for licensing information. */ @@ -87,15 +87,14 @@ bf_add(buffered_file *file, const void *data, uint64_t length) } needspace = length + bf_unwritten_length(file); - if (needspace > file->writebufferalloc) { - uint64_t newsize = file->writebufferalloc * 2; if (needspace > newsize) { newsize = needspace; } file->writebuffer = realloc(file->writebuffer, newsize); - memset(file->writebuffer + file->writebufferalloc, 0, newsize - file->writebufferalloc); + memset(file->writebuffer + file->writebufferalloc, 0, + newsize - file->writebufferalloc); file->writebufferalloc = newsize; } } @@ -139,27 +138,36 @@ unsigned char * bf_read(buffered_file *file, uint64_t offset, size_t length, char **error) { unsigned char *result = malloc(length); - dprintf(READ, (stderr, "reading data: offset %llu\n", (unsigned long long) offset)); + dprintf(READ, (stderr, "reading data: offset %llu\n", + (unsigned long long) offset)); if (offset >= file->filesize) { uint64_t writebufferoffset = offset - file->filesize; if (offset + length > file->datasize) { free(result); - set_error(error, "pread: not enough data for offset %llu and length %zu\n", (long long unsigned int) offset, length); + set_error(error, + "pread: not enough data for offset %llu and " + "length %zu\n", + (long long unsigned int) offset, length); return NULL; } memcpy(result, file->writebuffer + writebufferoffset, length); } else { if (offset + length > file->filesize) { free(result); - set_error(error, "pread: trying to read over file/writebuffer boundary for offset %llu and length %zu\n", (long long unsigned int) offset, length); + set_error(error, + "pread: trying to read over file/writebuffer " + "boundary for offset %llu and length %zu\n", + (long long unsigned int) offset, length); return NULL; } - + ssize_t ret = pread(file->fd, result, length, (off_t) offset); if (ret != length) { free(result); - set_error(error, "short pread: %zd (wanted %zu) at offset %llu\n", ret, length, (long long unsigned int) offset); + set_error(error, + "short pread: %zd (wanted %zu) at offset %llu\n", + ret, length, (long long unsigned int) offset); return NULL; } } @@ -217,5 +225,10 @@ bf_truncate(buffered_file *file) void bf_sha256(buffered_file *file, unsigned char *checksum) { - sha256_digest(&file->commit_checksum_context, SHA256_DIGEST_SIZE, checksum); + sha256_digest(&file->commit_checksum_context, SHA256_DIGEST_SIZE, + checksum); } + +/* Local Variables: */ +/* c-file-style: "linux" */ +/* End: */ diff --git a/c_src/permdb.c b/c_src/permdb.c index dec0bbb..0cf04ce 100644 --- a/c_src/permdb.c +++ b/c_src/permdb.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, NORDUnet A/S. + * Copyright (c) 2015,2016, NORDUnet A/S. * See LICENSE for licensing information. */ @@ -24,7 +24,9 @@ #include "utarray.h" #include "pstring.h" -#define INDEX_COMMIT_TRAILER_SIZE (sizeof(uint64_t) + SHA256_DIGEST_SIZE + sizeof(index_commit_cookie)) +#define INDEX_COMMIT_TRAILER_SIZE (sizeof(uint64_t) \ + + SHA256_DIGEST_SIZE \ + + sizeof(index_commit_cookie)) static const int keylen = 32; @@ -54,14 +56,17 @@ static node_object get_node_from_cache(permdb_object *state, const ps_string *key) { dprintf(CACHE, (stderr, "getting cache key %*s, ", PS_PRINTF(key))); + struct nodecache *node; HASH_FIND(hh, state->nodecache, key->value, key->length, node); if (node == NULL) { dprintf(CACHE, (stderr, "found nothing in cache\n")); return errornode; } + dprintf(CACHE, (stderr, "got cache key %*s: ", PS_PRINTF(node->key))); dprinthex(CACHE, &node->value, sizeof(struct node_object)); + return node->value; } @@ -69,15 +74,17 @@ static node_object get_node_from_dirtynodes(permdb_object *state, const ps_string *key) { dprintf(CACHE, (stderr, "getting key %*s, ", PS_PRINTF(key))); - + struct nodecache *node; HASH_FIND(hh, state->dirtynodes, key->value, key->length, node); if (node == NULL) { dprintf(CACHE, (stderr, "found nothing\n")); return errornode; } + dprintf(CACHE, (stderr, "got key %*s: ", PS_PRINTF(node->key))); dprinthex(CACHE, &node->value, sizeof(struct node_object)); + return node->value; } @@ -101,10 +108,13 @@ put_node_in_cache(permdb_object *state, const ps_string *key, node_object value) } static void -put_node_in_dirtynodes(permdb_object *state, const ps_string *key, node_object value) +put_node_in_dirtynodes(permdb_object *state, + const ps_string *key, + node_object value) { dprintf(CACHE, (stderr, "putting key %*s: ", PS_PRINTF(key))); dprinthex(CACHE, &value, sizeof(node_object)); + struct nodecache *node; HASH_FIND(hh, state->dirtynodes, key->value, key->length, node); if (node) { @@ -144,12 +154,14 @@ static const uint64_t index_file_cookie = 0xb7e16b02ba8a6d1b; static const uint64_t index_commit_cookie = 0x2fb1778c74a402e4; static const uint64_t index_node_cookie = 0x2e0f555ad73210d1; -static const uint8_t data_file_cookie[] = {0xd5, 0x35, 0x51, 0xba, 0x53, 0x9a, 0x42, 0x52}; - -static const uint8_t data_entry_cookie[] = {0xe7, 0xc1, 0xcd, 0xc2, 0xba, 0x3d, 0xc7, 0x7c}; - -static const uint8_t data_commit_start_cookie[] = {0x75, 0xc2, 0xe4, 0xb3, 0xd5, 0xf6, 0x43, 0xa1}; -static const uint8_t data_commit_end_cookie[] = {0x2b, 0x05, 0xee, 0xd6, 0x1b, 0x5a, 0xf5, 0x50}; +static const uint8_t data_file_cookie[] = + {0xd5, 0x35, 0x51, 0xba, 0x53, 0x9a, 0x42, 0x52}; +static const uint8_t data_entry_cookie[] = + {0xe7, 0xc1, 0xcd, 0xc2, 0xba, 0x3d, 0xc7, 0x7c}; +static const uint8_t data_commit_start_cookie[] = + {0x75, 0xc2, 0xe4, 0xb3, 0xd5, 0xf6, 0x43, 0xa1}; +static const uint8_t data_commit_end_cookie[] = + {0x2b, 0x05, 0xee, 0xd6, 0x1b, 0x5a, 0xf5, 0x50}; int committree(permdb_object *state); @@ -194,8 +206,11 @@ struct commit_info { int validate_checksum(struct commit_info *commit, buffered_file *file) { - //fprintf(stderr, "validate_checksum: read from file: length %llu start %llu\n", commit->length, commit->start); - unsigned char *checksumdata = bf_read(file, commit->start, commit->length, NULL); + /*fprintf(stderr, + "validate_checksum: read from file: length %llu start %llu\n", + commit->length, commit->start);*/ + unsigned char *checksumdata = bf_read(file, commit->start, + commit->length, NULL); if (checksumdata == NULL) { return -1; @@ -223,10 +238,13 @@ verify_index_commit(buffered_file *file, node_offset offset) { //fprintf(stderr, "verifying index file: commit verification\n"); offset -= INDEX_COMMIT_TRAILER_SIZE; - unsigned char *data = bf_read(file, offset, INDEX_COMMIT_TRAILER_SIZE, NULL); + unsigned char *data = + bf_read(file, offset, INDEX_COMMIT_TRAILER_SIZE, NULL); - if (memcmp(data + sizeof(uint64_t) + SHA256_DIGEST_SIZE, &index_commit_cookie, sizeof(index_commit_cookie)) != 0) { - fprintf(stderr, "verifying index file: incorrect commit cookie\n"); + if (memcmp(data + sizeof(uint64_t) + SHA256_DIGEST_SIZE, + &index_commit_cookie, sizeof(index_commit_cookie)) != 0) { + fprintf(stderr, + "verifying index file: incorrect commit cookie\n"); free(data); return -1; } @@ -251,7 +269,8 @@ indexfile_verify_file(buffered_file *file) } free(header); if (verify_index_commit(file, bf_total_length(file)) < 0) { - fprintf(stderr, "verifying index file: commit verification failed\n"); + fprintf(stderr, + "verifying index file: commit verification failed\n"); return -1; } return 0; @@ -264,7 +283,8 @@ int datafile_verify_file(buffered_file *file) { unsigned char *header = bf_read(file, 0, sizeof(data_file_cookie), NULL); - if (header == NULL || memcmp(header, &data_file_cookie, sizeof(data_file_cookie)) != 0) { + if (header == NULL || memcmp(header, &data_file_cookie, + sizeof(data_file_cookie)) != 0) { free(header); return -1; } @@ -273,7 +293,8 @@ datafile_verify_file(buffered_file *file) node_offset offset = bf_lastcommit(file); //fprintf(stderr, "verifying commit: %llu\n", offset); - struct commit_info *data_commit = read_data_commit_backward(file, &offset); + struct commit_info *data_commit = + read_data_commit_backward(file, &offset); if (data_commit == NULL || validate_checksum(data_commit, file) < 0) { //fprintf(stderr, "commit broken: %llu\n", offset); @@ -292,8 +313,15 @@ readdatakeyandlen(permdb_object *state, node_offset offset, size_t *datalen); struct commit_info * read_data_commit(buffered_file *file, node_offset *offset) { - unsigned char *data = bf_read(file, *offset, sizeof(uint32_t) + SHA256_DIGEST_SIZE + sizeof(data_commit_end_cookie), NULL); - if (data == NULL || memcmp(data + sizeof(uint32_t) + SHA256_DIGEST_SIZE, data_commit_end_cookie, sizeof(data_commit_end_cookie)) != 0) { + const size_t length = + sizeof(uint32_t) + + SHA256_DIGEST_SIZE + + sizeof(data_commit_end_cookie); + unsigned char *data = bf_read(file, *offset, length, NULL); + if (data == NULL + || memcmp(data + sizeof(uint32_t) + SHA256_DIGEST_SIZE, + data_commit_end_cookie, + sizeof(data_commit_end_cookie)) != 0) { free(data); return NULL; } @@ -309,7 +337,6 @@ read_data_commit(buffered_file *file, node_offset *offset) return commit; } - struct commit_info * read_data_commit_forward(buffered_file *file, node_offset *offset) { @@ -321,7 +348,9 @@ read_data_commit_forward(buffered_file *file, node_offset *offset) struct commit_info * read_data_commit_backward(buffered_file *file, node_offset *offset) { - *offset -= sizeof(uint32_t) + SHA256_DIGEST_SIZE + sizeof(data_commit_end_cookie); + *offset -= sizeof(uint32_t) + + SHA256_DIGEST_SIZE + + sizeof(data_commit_end_cookie); return read_data_commit(file, offset); } @@ -335,16 +364,23 @@ rebuild_index_file(permdb_object *state) node_offset offset = sizeof(data_file_cookie) + sizeof(uint32_t) * 3; while (1) { - unsigned char *cookie = bf_read(state->datafile, offset, sizeof(data_entry_cookie), NULL); + unsigned char *cookie = bf_read(state->datafile, offset, + sizeof(data_entry_cookie), NULL); if (cookie == NULL) { break; } - if (memcmp(&data_entry_cookie, cookie, sizeof(data_entry_cookie)) == 0) { + if (memcmp(&data_entry_cookie, cookie, sizeof(data_entry_cookie)) + == 0) { size_t datalen; - unsigned char *datakey = readdatakeyandlen(state, offset, &datalen); + unsigned char *datakey = + readdatakeyandlen(state, offset, &datalen); //fprintf(stderr, "entry %llu: %zu\n", offset, datalen); - int result = addvalue(state, datakey, keylen, NULL, 0, offset); - offset += sizeof(data_entry_cookie) + keylen + sizeof(uint32_t) + datalen; + int result = + addvalue(state, datakey, keylen, NULL, 0, offset); + offset += sizeof(data_entry_cookie) + + keylen + + sizeof(uint32_t) + + datalen; free(datakey); @@ -352,13 +388,20 @@ rebuild_index_file(permdb_object *state) free(cookie); return -1; } - } else if (memcmp(&data_commit_start_cookie, cookie, sizeof(data_commit_start_cookie)) == 0) { - struct commit_info *data_commit = read_data_commit_forward(state->datafile, &offset); - //fprintf(stderr, "verifying commit: %llu %p\n", offset, data_commit); - - if (data_commit == NULL || validate_checksum(data_commit, state->datafile) < 0) { + } else if (memcmp(&data_commit_start_cookie, cookie, + sizeof(data_commit_start_cookie)) == 0) { + struct commit_info *data_commit = + read_data_commit_forward(state->datafile, + &offset); + /*fprintf(stderr, "verifying commit: %llu %p\n", offset, + data_commit);*/ + + if (data_commit == NULL + || validate_checksum(data_commit, + state->datafile) < 0) { free(data_commit); - fprintf(stderr, "commit broken: %llu\n", (unsigned long long) offset); + fprintf(stderr, "commit broken: %llu\n", + (unsigned long long) offset); free(cookie); return -1; } @@ -393,12 +436,14 @@ permdb_alloc(const char *dbpath) state->nodecache = NULL; state->dirtynodes = NULL; state->error = NULL; - if (bf_total_length(state->datafile) == 0 && bf_total_length(state->indexfile) == 0) { + if (bf_total_length(state->datafile) == 0 + && bf_total_length(state->indexfile) == 0) { dprintf(WRITE, (stderr, "writing header\n")); indexfile_add_header(state->indexfile); datafile_add_header(state->datafile); initial_commit(state); - } else if (bf_total_length(state->datafile) > 0 && bf_total_length(state->indexfile) == 0) { + } else if (bf_total_length(state->datafile) > 0 + && bf_total_length(state->indexfile) == 0) { if (rebuild_index_file(state) < 0) { warnx("index file rebuilding failed"); permdb_free(state); @@ -437,7 +482,7 @@ static unsigned char keybits(const unsigned char *key, unsigned int level) { unsigned char b = key[level/4]; - + return (b >> (6-(level*2) % 8)) & 0x3; } @@ -483,7 +528,8 @@ unpacknode(permdb_object *state, const unsigned char *data, size_t datalen) if (memcmp(&index_node_cookie, data, sizeof(index_node_cookie)) != 0) { print_hex(data, sizeof(index_node_cookie)); print_hex(&index_node_cookie, sizeof(index_node_cookie)); - set_error(&state->error, "incorrect magic (node) %02x%02x\n", (unsigned char)data[0], (unsigned char)data[1]); + set_error(&state->error, "incorrect magic (node) %02x%02x\n", + (unsigned char)data[0], (unsigned char)data[1]); return errornode; } @@ -502,7 +548,8 @@ unsigned char * read_internal_data(permdb_object *state, node_offset offset, size_t length) { buffered_file *file = state->datafile; - dprintf(READ, (stderr, "reading data: offset %llu\n", (unsigned long long) offset)); + dprintf(READ, (stderr, "reading data: offset %llu\n", + (unsigned long long) offset)); return bf_read(file, offset, length, &state->error); } @@ -510,46 +557,58 @@ read_internal_data(permdb_object *state, node_offset offset, size_t length) static int iserrornode(node_object node) { - return node.data[0] == NODE_ENTRY_ERROR_NODE && - node.data[1] == NODE_ENTRY_ERROR_NODE && - node.data[2] == NODE_ENTRY_ERROR_NODE && - node.data[3] == NODE_ENTRY_ERROR_NODE; + return node.data[0] == NODE_ENTRY_ERROR_NODE + && node.data[1] == NODE_ENTRY_ERROR_NODE + && node.data[2] == NODE_ENTRY_ERROR_NODE + && node.data[3] == NODE_ENTRY_ERROR_NODE; } static node_object readnode(permdb_object *state, node_offset offset, const ps_string *cachekey) { if (cachekey) { - dprintf(READ, (stderr, "reading node: offset %llu cachekey '%*s'\n", (unsigned long long) offset, PS_PRINTF(cachekey))); + dprintf(READ, (stderr, + "reading node: offset %llu cachekey '%*s'\n", + (unsigned long long) offset, PS_PRINTF(cachekey))); node_object dirtynode = get_node_from_dirtynodes(state, cachekey); if (!iserrornode(dirtynode)) { - dprintf(READ, (stderr, "reading node: found node in dirty nodes\n")); + dprintf(READ, (stderr, + "reading node: found node in dirty " + "nodes\n")); return dirtynode; } if (offset == NODE_ENTRY_DIRTY_NODE) { - set_error(&state->error, "referring to dirty node at key %*s, but node not in dirtynodes\n", PS_PRINTF(cachekey)); - dprintf(READ, (stderr, "reading node: referring to dirty node at key %*s, but node not in dirtynodes\n", PS_PRINTF(cachekey))); + set_error(&state->error, + "referring to dirty node at key %*s, but " + "node not in dirtynodes\n", + PS_PRINTF(cachekey)); + dprintf(READ, (stderr, + "reading node: referring to dirty " + "node at key %*s, but node not in " + "dirtynodes\n", PS_PRINTF(cachekey))); return errornode; } node_object cachednode = get_node_from_cache(state, cachekey); if (!iserrornode(cachednode)) { - dprintf(READ, (stderr, "reading node: found node in cache\n")); + dprintf(READ, (stderr, + "reading node: found node in cache\n")); return cachednode; } } else { - dprintf(READ, (stderr, "reading node: offset %llu no cachekey\n", (unsigned long long) offset)); + dprintf(READ, (stderr, "reading node: offset %llu no cachekey\n", + (unsigned long long) offset)); } size_t length = sizeof(index_node_cookie) + sizeof(node_object); - unsigned char *buffer = bf_read(state->indexfile, offset, length, &state->error); + unsigned char *buffer = + bf_read(state->indexfile, offset, length, &state->error); if (buffer == NULL) { return errornode; } - node_object result = unpacknode(state, buffer, length); free(buffer); @@ -583,12 +642,20 @@ getpath(permdb_object *state, const unsigned char *key, UT_array *nodes) { unsigned int level = 0; - node_offset rootoffset = bf_lastcommit(state->indexfile) - (sizeof(index_node_cookie) + sizeof(node_object) + INDEX_COMMIT_TRAILER_SIZE); + node_offset rootoffset = + bf_lastcommit(state->indexfile) + - (sizeof(index_node_cookie) + + sizeof(node_object) + + INDEX_COMMIT_TRAILER_SIZE); node_object node = readnode(state, rootoffset, PS_STRING("")); if (iserrornode(node)) { - fprintf(stderr, "cannot find root node at offset %llu (lastcommit %llu)\n", (long long unsigned int) rootoffset, (long long unsigned int) bf_lastcommit(state->indexfile)); + fprintf(stderr, + "cannot find root node at offset %llu " + "(lastcommit %llu)\n", + (long long unsigned int) rootoffset, + (long long unsigned int) bf_lastcommit(state->indexfile)); return -1; } @@ -619,7 +686,11 @@ getpathlastnode(permdb_object *state, const unsigned char *key) { unsigned int level = 0; - node_offset rootoffset = bf_lastcommit(state->indexfile) - (sizeof(index_node_cookie) + sizeof(node_object) + INDEX_COMMIT_TRAILER_SIZE); + node_offset rootoffset = + bf_lastcommit(state->indexfile) + - (sizeof(index_node_cookie) + + sizeof(node_object) + + INDEX_COMMIT_TRAILER_SIZE); node_object node = readnode(state, rootoffset, PS_STRING("")); if (iserrornode(node)) { @@ -653,7 +724,8 @@ writenode(permdb_object *state, node_object node, ps_string *cachekey) put_node_in_cache(state, cachekey, node); - dprintf(WRITE, (stderr, "writing node: offset %llu\n", (unsigned long long) offset)); + dprintf(WRITE, (stderr, "writing node: offset %llu\n", + (unsigned long long) offset)); bf_add(state->indexfile, &index_node_cookie, sizeof(index_node_cookie)); bf_add(state->indexfile, &node, sizeof(node_object)); @@ -688,13 +760,16 @@ memsub(void *src, size_t offset, size_t length) static unsigned char * readdatakey(permdb_object *state, node_offset offset) { - unsigned char *data = read_internal_data(state, offset, sizeof(data_entry_cookie) + keylen); + unsigned char *data = + read_internal_data(state, offset, + sizeof(data_entry_cookie) + keylen); if (data == NULL) { return NULL; } if (memcmp(&data_entry_cookie, data, sizeof(data_entry_cookie)) != 0) { free(data); - set_error(&state->error, "incorrect magic (entry) %02x%02x\n", (unsigned char)data[0], (unsigned char)data[1]); + set_error(&state->error, "incorrect magic (entry) %02x%02x\n", + (unsigned char)data[0], (unsigned char)data[1]); return NULL; } unsigned char *result = memsub(data, sizeof(data_entry_cookie), keylen); @@ -705,20 +780,26 @@ readdatakey(permdb_object *state, node_offset offset) static unsigned char * readdatakeyandlen(permdb_object *state, node_offset offset, size_t *datalen) { - unsigned char *data = read_internal_data(state, offset, sizeof(data_entry_cookie) + keylen + 4); + unsigned char *data = + read_internal_data(state, offset, + sizeof(data_entry_cookie) + keylen + 4); if (data == NULL) { return NULL; } if (memcmp(&data_entry_cookie, data, sizeof(data_entry_cookie)) != 0) { free(data); - set_error(&state->error, "incorrect magic (entry) %02x%02x\n", (unsigned char)data[0], (unsigned char)data[1]); + set_error(&state->error, "incorrect magic (entry) %02x%02x\n", + (unsigned char)data[0], (unsigned char)data[1]); return NULL; } unsigned char *result = memsub(data, sizeof(data_entry_cookie), keylen); uint16_t nchunks = read_be16(data+sizeof(data_entry_cookie)+keylen); - *datalen = read_be16(data+sizeof(data_entry_cookie)+keylen+sizeof(uint16_t)); + *datalen = read_be16(data + + sizeof(data_entry_cookie) + + keylen+sizeof(uint16_t)); if (nchunks != 1) { - errx(1, "number of chunks is %d, but only one chunk is supported right now", nchunks); + errx(1, "number of chunks is %d, but only one chunk is supported " + "right now", nchunks); } free(data); return result; @@ -727,19 +808,24 @@ readdatakeyandlen(permdb_object *state, node_offset offset, size_t *datalen) static unsigned char * readdata(permdb_object *state, node_offset offset, size_t datalen) { - return read_internal_data(state, offset + sizeof(data_entry_cookie) + keylen + 4, datalen); + const node_offset internal_offset = + offset + sizeof(data_entry_cookie) + keylen + 4; + return read_internal_data(state, internal_offset, datalen); } static node_offset -writedata(permdb_object *state, const unsigned char *key, const unsigned char *data, size_t datalength) +writedata(permdb_object *state, const unsigned char *key, + const unsigned char *data, size_t datalength) { if (datalength > 65535) { - errx(1, "data length is %zu, but only < 64K lengths are supported right now", datalength); + errx(1, "data length is %zu, but only < 64K lengths are " + "supported right now", datalength); } node_offset offset = bf_total_length(state->datafile); - dprintf(WRITE, (stderr, "writing data: offset %llu\n", (unsigned long long) offset)); + dprintf(WRITE, (stderr, "writing data: offset %llu\n", + (unsigned long long) offset)); bf_add(state->datafile, &data_entry_cookie, sizeof(data_entry_cookie)); bf_add(state->datafile, key, keylen); bf_add_be16(state->datafile, 1); @@ -751,7 +837,8 @@ writedata(permdb_object *state, const unsigned char *key, const unsigned char *d int -addvalue(permdb_object *state, const unsigned char *key, unsigned int keylength, const unsigned char *data, size_t datalength, node_offset dataoffset) +addvalue(permdb_object *state, const unsigned char *key, unsigned int keylength, + const unsigned char *data, size_t datalength, node_offset dataoffset) { UT_array *nodes; utarray_new(nodes, &node_object_icd); @@ -770,9 +857,12 @@ addvalue(permdb_object *state, const unsigned char *key, unsigned int keylength, if (data != NULL) { dataoffset = writedata(state, key, data, datalength); } - addentry(&lastnode, keybits(key, foundlevel), buildentry(1, dataoffset)); + addentry(&lastnode, keybits(key, foundlevel), + buildentry(1, dataoffset)); } else { - node_offset olddataoffset = entryoffset(get_entry_in_node(lastnode, (unsigned char) kb)); + node_offset olddataoffset = + entryoffset(get_entry_in_node(lastnode, + (unsigned char) kb)); unsigned char *olddatakey = readdatakey(state, olddataoffset); if (olddatakey == NULL) { utarray_free(nodes); @@ -791,8 +881,10 @@ addvalue(permdb_object *state, const unsigned char *key, unsigned int keylength, level++; } node_object leafnode = nullnode; - addentry(&leafnode, keybits(key, level), buildentry(1, dataoffset)); - addentry(&leafnode, keybits(olddatakey, level), buildentry(1, olddataoffset)); + addentry(&leafnode, keybits(key, level), + buildentry(1, dataoffset)); + addentry(&leafnode, keybits(olddatakey, level), + buildentry(1, olddataoffset)); free(olddatakey); { ps_string cachekey; @@ -802,13 +894,15 @@ addvalue(permdb_object *state, const unsigned char *key, unsigned int keylength, level--; while (level > foundlevel) { node_object node = nullnode; - addentry(&node, keybits(key, level), NODE_ENTRY_DIRTY_NODE); + addentry(&node, keybits(key, level), + NODE_ENTRY_DIRTY_NODE); ps_string cachekey; keypart(key, level, &cachekey); put_node_in_dirtynodes(state, &cachekey, node); level--; } - overwriteentry(&lastnode, keybits(key, foundlevel), NODE_ENTRY_DIRTY_NODE); + overwriteentry(&lastnode, keybits(key, foundlevel), + NODE_ENTRY_DIRTY_NODE); } int level = (int) foundlevel; @@ -822,7 +916,8 @@ addvalue(permdb_object *state, const unsigned char *key, unsigned int keylength, level--; while (level >= 0) { node_object node = *(node_object *)utarray_eltptr(nodes, level); - overwriteentry(&node, keybits(key, (unsigned int) level), NODE_ENTRY_DIRTY_NODE); + overwriteentry(&node, keybits(key, (unsigned int) level), + NODE_ENTRY_DIRTY_NODE); ps_string cachekey; keypart(key, (unsigned int) level, &cachekey); put_node_in_dirtynodes(state, &cachekey, node); @@ -835,7 +930,8 @@ addvalue(permdb_object *state, const unsigned char *key, unsigned int keylength, } unsigned char * -getvalue(permdb_object *state, const unsigned char *key, size_t keylength, size_t *datalen) +getvalue(permdb_object *state, const unsigned char *key, size_t keylength, + size_t *datalen) { node_entry entry = getpathlastnode(state, key); if (entry == 0) { @@ -859,7 +955,7 @@ getvalue(permdb_object *state, const unsigned char *key, size_t keylength, size_ return readdata(state, olddataoffset, *datalen); } -static int +static int string_length_comparison(struct nodecache *a, struct nodecache *b) { size_t a_len = a->key->length; size_t b_len = b->key->length; @@ -882,60 +978,87 @@ committree(permdb_object *state) HASH_SORT(state->dirtynodes, string_length_comparison); - struct nodecache *lastobject = ELMT_FROM_HH(state->dirtynodes->hh.tbl, state->dirtynodes->hh.tbl->tail); + struct nodecache *lastobject = + ELMT_FROM_HH(state->dirtynodes->hh.tbl, + state->dirtynodes->hh.tbl->tail); if (lastobject->key->length != 0) { fprintf(stderr, "sorted list doesn't end with root node\n"); return -1; } - - dprintf(WRITE, (stderr, "committing %d dirty nodes at offset %llu\n", HASH_COUNT(state->dirtynodes), (unsigned long long) bf_total_length(state->indexfile))); + + dprintf(WRITE, (stderr, "committing %d dirty nodes at offset %llu\n", + HASH_COUNT(state->dirtynodes), + (unsigned long long) bf_total_length(state->indexfile))); struct nodecache *node, *tmp; HASH_ITER(hh, state->dirtynodes, node, tmp) { - assert(get_entry_in_node(node->value, 0) != NODE_ENTRY_DIRTY_NODE); - assert(get_entry_in_node(node->value, 1) != NODE_ENTRY_DIRTY_NODE); - assert(get_entry_in_node(node->value, 2) != NODE_ENTRY_DIRTY_NODE); - assert(get_entry_in_node(node->value, 3) != NODE_ENTRY_DIRTY_NODE); + assert(get_entry_in_node(node->value, 0) + != NODE_ENTRY_DIRTY_NODE); + assert(get_entry_in_node(node->value, 1) + != NODE_ENTRY_DIRTY_NODE); + assert(get_entry_in_node(node->value, 2) + != NODE_ENTRY_DIRTY_NODE); + assert(get_entry_in_node(node->value, 3) + != NODE_ENTRY_DIRTY_NODE); node_offset offset = writenode(state, node->value, node->key); size_t keylength = node->key->length; if (keylength != 0) { - ps_string *parent = ps_resize(node->key, node->key->length - 1); - unsigned int entrynumber = (unsigned int) (node->key->value[keylength - 1] - '0'); - node_object parentnode = get_node_from_dirtynodes(state, parent); + ps_string *parent = ps_resize(node->key, + node->key->length - 1); + unsigned int entrynumber = (unsigned int) + (node->key->value[keylength - 1] - '0'); + node_object parentnode = + get_node_from_dirtynodes(state, parent); overwriteentry(&parentnode, entrynumber, offset); put_node_in_dirtynodes(state, parent, parentnode); free(parent); } } - dprintf(WRITE, (stderr, "writing data commit trailer at offset %llu\n", (unsigned long long) bf_total_length(state->datafile))); + dprintf(WRITE, (stderr, "writing data commit trailer at offset %llu\n", + (unsigned long long) bf_total_length(state->datafile))); - int data_commit_padding_size = calc_padding(bf_total_length(state->datafile), 4); + int data_commit_padding_size = + calc_padding(bf_total_length(state->datafile), 4); uint8_t padding[4] = {0, 0, 0, 0}; unsigned char data_commit_checksum[SHA256_DIGEST_SIZE]; bf_add(state->datafile, data_commit_start_cookie, 8); bf_add(state->datafile, padding, data_commit_padding_size); - bf_add_be32(state->datafile, bf_total_length(state->datafile) - bf_lastcommit(state->datafile) + sizeof(uint32_t)); + bf_add_be32(state->datafile, + bf_total_length(state->datafile) + - bf_lastcommit(state->datafile) + + sizeof(uint32_t)); bf_sha256(state->datafile, data_commit_checksum); bf_add(state->datafile, data_commit_checksum, SHA256_DIGEST_SIZE); - bf_add(state->datafile, &data_commit_end_cookie, sizeof(data_commit_end_cookie)); + bf_add(state->datafile, &data_commit_end_cookie, + sizeof(data_commit_end_cookie)); - dprintf(WRITE, (stderr, "finished writing data commit trailer at offset %llu\n", (unsigned long long) bf_total_length(state->datafile))); + dprintf(WRITE, (stderr, + "finished writing data commit trailer at offset %llu\n", + (unsigned long long) bf_total_length(state->datafile))); if (bf_flush(state->datafile) == -1) { set_error(&state->error, "data file flushing failed\n"); return -1; } - dprintf(WRITE, (stderr, "writing index commit trailer at offset %llu\n", (unsigned long long) bf_total_length(state->indexfile))); + dprintf(WRITE, (stderr, "writing index commit trailer at offset %llu\n", + (unsigned long long) bf_total_length(state->indexfile))); - uint64_t index_commit_length = bf_total_length(state->indexfile) - bf_lastcommit(state->indexfile) + sizeof(uint64_t); + uint64_t index_commit_length = + bf_total_length(state->indexfile) + - bf_lastcommit(state->indexfile) + + sizeof(uint64_t); unsigned char index_commit_checksum[SHA256_DIGEST_SIZE]; bf_add_host64(state->indexfile, index_commit_length); bf_sha256(state->indexfile, index_commit_checksum); bf_add(state->indexfile, index_commit_checksum, SHA256_DIGEST_SIZE); - bf_add(state->indexfile, &index_commit_cookie, sizeof(index_commit_cookie)); + bf_add(state->indexfile, + &index_commit_cookie, + sizeof(index_commit_cookie)); - dprintf(WRITE, (stderr, "finished writing index commit trailer at offset %llu\n", (unsigned long long) bf_total_length(state->indexfile))); + dprintf(WRITE, (stderr, + "finished writing index commit trailer at offset %llu\n", + (unsigned long long) bf_total_length(state->indexfile))); if (bf_flush(state->indexfile) == -1) { set_error(&state->error, "index file flushing failed\n"); @@ -963,7 +1086,8 @@ portloop(permdb_object *state) assert(len == keylen+1); size_t datalen; - unsigned char *result = getvalue(state, (buf+1), keylen, &datalen); + unsigned char *result = getvalue(state, (buf+1), keylen, + &datalen); if (result == NULL) { write_reply(NULL, 0, 4); @@ -976,7 +1100,8 @@ portloop(permdb_object *state) dprintf(PORT, (stderr, "add\n")); if (len < (keylen + 1)) { write_reply(NULL, 0, 4); - fprintf(stderr, "invalid addvalue command, length was %zd\n", len); + fprintf(stderr, "invalid addvalue command, " + "length was %zd\n", len); continue; } @@ -984,12 +1109,14 @@ portloop(permdb_object *state) unsigned char *key = buf + 1; unsigned char *data = key + keylen; - int result = addvalue(state, key, keylen, data, datalen, 0); + int result = + addvalue(state, key, keylen, data, datalen, 0); if (result < 0) { write_reply(NULL, 0, 4); } else { - unsigned char result_byte = (unsigned char) result; + unsigned char result_byte = + (unsigned char) result; write_reply(&result_byte, 1, 4); } dprintf(PORT, (stderr, "add reply\n")); @@ -997,7 +1124,8 @@ portloop(permdb_object *state) dprintf(PORT, (stderr, "commit\n")); if (len != 1) { write_reply(NULL, 0, 4); - fprintf(stderr, "invalid commit command, length was %zd\n", len); + fprintf(stderr, "invalid commit command, " + "length was %zd\n", len); continue; } @@ -1007,7 +1135,8 @@ portloop(permdb_object *state) write_reply(NULL, 0, 4); continue; } else { - unsigned char result_byte = (unsigned char) result; + unsigned char result_byte = + (unsigned char) result; write_reply(&result_byte, 1, 4); } dprintf(PORT, (stderr, "commit reply\n")); @@ -1022,3 +1151,6 @@ portloop(permdb_object *state) permdb_free(state); } +/* Local Variables: */ +/* c-file-style: "linux" */ +/* End: */ diff --git a/c_src/permdbport.c b/c_src/permdbport.c index 8f7dd7c..a0e730b 100644 --- a/c_src/permdbport.c +++ b/c_src/permdbport.c @@ -39,7 +39,16 @@ main(int argc, char *argv[]) struct rusage rusage; getrusage(RUSAGE_SELF, &rusage); - fprintf(stderr, "permdbport user %ld.%d sys %ld.%d maxrss %ld M\n", rusage.ru_utime.tv_sec, (int)rusage.ru_utime.tv_usec, rusage.ru_stime.tv_sec, (int)rusage.ru_utime.tv_usec, rusage.ru_maxrss/1024); + fprintf(stderr, "permdbport user %ld.%d sys %ld.%d maxrss %ld M\n", + rusage.ru_utime.tv_sec, + (int)rusage.ru_utime.tv_usec, + rusage.ru_stime.tv_sec, + (int)rusage.ru_utime.tv_usec, + rusage.ru_maxrss/1024); return 0; } + +/* Local Variables: */ +/* c-file-style: "linux" */ +/* End: */ diff --git a/c_src/permdbpy.c b/c_src/permdbpy.c index ff3dc2a..cc1e36f 100644 --- a/c_src/permdbpy.c +++ b/c_src/permdbpy.c @@ -1,3 +1,8 @@ +/* + * Copyright (c) 2016, NORDUnet A/S. + * See LICENSE for licensing information. + */ + #include #include "permdb.h" @@ -99,11 +104,13 @@ addvalue_wrapper(PyObject *self, PyObject *args) const char *data; unsigned int datalength; - if (!PyArg_ParseTuple(args, "O!s#s#", &permdb_type, &state, &key, &keylength, &data, &datalength)) { + if (!PyArg_ParseTuple(args, "O!s#s#", &permdb_type, &state, &key, + &keylength, &data, &datalength)) { return NULL; } - int result = addvalue(state->permdb, (unsigned char *) key, keylength, (unsigned char *) data, datalength, 0); + int result = addvalue(state->permdb, (unsigned char *) key, keylength, + (unsigned char *) data, datalength, 0); if (result < 0) { return NULL; @@ -124,19 +131,22 @@ getvalue_wrapper(PyObject *self, PyObject *args) const char *key; int keylen; - if (!PyArg_ParseTuple(args, "O!s#", &permdb_type, &state, &key, &keylen)) { + if (!PyArg_ParseTuple(args, "O!s#", &permdb_type, &state, &key, + &keylen)) { return NULL; } size_t datalen; - unsigned char *result = getvalue(state->permdb, (unsigned char *) key, keylen, &datalen); + unsigned char *result = + getvalue(state->permdb, (unsigned char *) key, keylen, &datalen); if (result == NULL) { Py_INCREF(Py_None); return Py_None; } - PyObject* resultObj = PyString_FromStringAndSize((char *) result, datalen); + PyObject* resultObj = + PyString_FromStringAndSize((char *) result, datalen); free(result); return resultObj; } @@ -193,3 +203,7 @@ initpermdb() { (void) Py_InitModule("permdb", UtilMethods); } + +/* Local Variables: */ +/* c-file-style: "linux" */ +/* End: */ diff --git a/c_src/permdbtest.c b/c_src/permdbtest.c index 5e2ea1d..5b1e21b 100644 --- a/c_src/permdbtest.c +++ b/c_src/permdbtest.c @@ -1,3 +1,8 @@ +/* + * Copyright (c) 2016, NORDUnet A/S. + * See LICENSE for licensing information. + */ + #define _GNU_SOURCE #include #include @@ -40,8 +45,10 @@ gentestdata(long long nentries) CC_SHA256(key, sizeof(key), testdata + i * 32 * 2); CC_SHA256(value, sizeof(value), testdata + i * 32 * 2 + 32); #else - rhash_msg(RHASH_SHA256, key, sizeof(key), testdata + i * 32 * 2); - rhash_msg(RHASH_SHA256, value, sizeof(value), testdata + i * 32 * 2 + 32); + rhash_msg(RHASH_SHA256, key, sizeof(key), + testdata + i * 32 * 2); + rhash_msg(RHASH_SHA256, value, sizeof(value), + testdata + i * 32 * 2 + 32); #endif } return testdata; @@ -75,9 +82,10 @@ main(int argc, char *argv[]) memcpy(value + j * 32, testdata + i * 32 * 2 + 32, 32); } if (datasize % 32 > 0) { - memcpy(value + (datasize / 32) * 32, testdata + i * 32 * 2 + 32, datasize % 32); + memcpy(value + (datasize / 32) * 32, + testdata + i * 32 * 2 + 32, datasize % 32); } - + int result = addvalue(state, key, 32, value, datasize, 0); if (result < 0) { free(testdata); @@ -103,7 +111,7 @@ main(int argc, char *argv[]) errx(1, "committree: %d\n", result); } written_since_fsync = 0; - + printf("reading test data\n"); for (long long i = 0; i < nentries; i++) { unsigned char *key = testdata + i * 32 * 2; @@ -116,7 +124,8 @@ main(int argc, char *argv[]) errx(1, "getvalue returned datalen %zd\n", datalen); } for (int j = 0; j < datasize / 32; j++) { - if (memcmp(result + j * 32, testdata + i * 32 * 2 + 32, 32)) { + if (memcmp(result + j * 32, + testdata + i * 32 * 2 + 32, 32)) { free(testdata); permdb_free(state); free(result); @@ -124,7 +133,8 @@ main(int argc, char *argv[]) } } if (datasize % 32 > 0) { - if (memcmp(result + (datasize / 32) * 32, testdata + i * 32 * 2 + 32, datasize % 32)) { + if (memcmp(result + (datasize / 32) * 32, + testdata + i * 32 * 2 + 32, datasize % 32)) { free(testdata); permdb_free(state); free(result); @@ -136,10 +146,19 @@ main(int argc, char *argv[]) struct rusage rusage; getrusage(RUSAGE_SELF, &rusage); - fprintf(stderr, "user %ld.%d sys %ld.%d maxrss %ld M\n", rusage.ru_utime.tv_sec, (int)rusage.ru_utime.tv_usec, rusage.ru_stime.tv_sec, (int)rusage.ru_utime.tv_usec, rusage.ru_maxrss/1024); + fprintf(stderr, "user %ld.%d sys %ld.%d maxrss %ld M\n", + rusage.ru_utime.tv_sec, + (int)rusage.ru_utime.tv_usec, + rusage.ru_stime.tv_sec, + (int)rusage.ru_utime.tv_usec, + rusage.ru_maxrss/1024); free(testdata); permdb_free(state); return 0; } + +/* Local Variables: */ +/* c-file-style: "linux" */ +/* End: */ diff --git a/c_src/util.c b/c_src/util.c index 76294eb..9a743a8 100644 --- a/c_src/util.c +++ b/c_src/util.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, NORDUnet A/S. + * Copyright (c) 2016, NORDUnet A/S. * See LICENSE for licensing information. */ @@ -48,9 +48,11 @@ int calc_padding(int offset, int alignment) { int misalign = offset % alignment; + if (misalign == 0) { return 0; } + return alignment - misalign; } @@ -66,8 +68,7 @@ print_entry(node_object node) void print_hex(const void *data, int length) { - int i; - for (i = 0; i < length; i++) { + for (int i = 0; i < length; i++) { fprintf(stderr, "%02X ", ((unsigned char *)data)[i]); } fprintf(stderr, "\n"); @@ -103,3 +104,6 @@ read_be16(void *ptr) return ntohs(data); } +/* Local Variables: */ +/* c-file-style: "linux" */ +/* End: */ -- cgit v1.1