From aa6e0022ddc71459e6fbfaa16bd851082d06edd5 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 11 Feb 2016 21:58:02 +0100 Subject: Refactor marshalling Add chunking changes to listpermdb.erl --- c_src/permdb.c | 104 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 54 insertions(+), 50 deletions(-) (limited to 'c_src') diff --git a/c_src/permdb.c b/c_src/permdb.c index 624e1da..c5187be 100644 --- a/c_src/permdb.c +++ b/c_src/permdb.c @@ -19,7 +19,7 @@ #include "permdb.h" #include "hash.h" -#define INDEX_COMMIT_TRAILER_SIZE (8 + SHA256_DIGEST_SIZE + 8) +#define INDEX_COMMIT_TRAILER_SIZE (sizeof(uint64_t) + SHA256_DIGEST_SIZE + sizeof(index_commit_cookie)) static const int bitsperlevel = 2; static const int keylen = 32; @@ -256,16 +256,31 @@ indexfile_add_header(buffered_file *file) return length; } +static void +writebuffer_add_host64(buffered_file *file, uint64_t value) { + writebuffer_add(file, &value, sizeof(uint64_t)); +} + +static void +writebuffer_add_be32(buffered_file *file, uint32_t value) { + uint32_t value_be = htonl(value); + writebuffer_add(file, &value_be, sizeof(uint32_t)); +} + +static void +writebuffer_add_be16(buffered_file *file, uint16_t value) { + uint16_t value_be = htons(value); + writebuffer_add(file, &value_be, sizeof(uint16_t)); +} + node_offset datafile_add_header(buffered_file *file) { fprintf(stderr, "adding header to %s\n", file->name); - uint32_t parameters[3]; writebuffer_add(file, &data_file_cookie, sizeof(data_file_cookie)); - parameters[0] = htonl(4096); - parameters[1] = htonl(2); - parameters[2] = htonl(32); - writebuffer_add(file, parameters, sizeof(parameters)); + writebuffer_add_be32(file, 4096); + writebuffer_add_be32(file, 2); + writebuffer_add_be32(file, 32); uint64_t length = writebuffer_length(file); writebuffer_flush(file); return length; @@ -340,6 +355,16 @@ validate_checksum(struct commit_info *commit, buffered_file *file) return -1; } +static uint64_t +read_host64(void *ptr) +{ + uint64_t data; + + memcpy(&data, ptr, sizeof(data)); + + return data; +} + int verify_index_commit(buffered_file *file, node_offset offset) { @@ -353,10 +378,8 @@ verify_index_commit(buffered_file *file, node_offset offset) return -1; } struct commit_info commit; - uint64_t length; - memcpy(&length, data, sizeof(uint64_t)); - commit.length = length; - commit.start = offset - commit.length; + commit.length = read_host64(data); + commit.start = offset + sizeof(uint64_t) - commit.length; memcpy(commit.checksum, data + sizeof(uint64_t), SHA256_DIGEST_SIZE); free(data); @@ -411,7 +434,7 @@ datafile_verify_file(buffered_file *file) static uint32_t -readnet32(void *ptr); +read_be32(void *ptr); static unsigned char * readdatakeyandlen(permdb_object *state, node_offset offset, size_t *datalen); @@ -428,7 +451,7 @@ read_data_commit(buffered_file *file, node_offset *offset) struct commit_info *commit = malloc(sizeof(struct commit_info)); //fprintf(stderr, "read commit: %llu\n", *offset); //print_hex(data, sizeof(uint32_t) + SHA256_DIGEST_SIZE); - commit->length = readnet32(data); + commit->length = read_be32(data); commit->start = *offset - commit->length; memcpy(&commit->checksum, data + sizeof(uint32_t), SHA256_DIGEST_SIZE); *offset += SHA256_DIGEST_SIZE + sizeof(data_commit_end_cookie); @@ -710,16 +733,6 @@ keypart(const unsigned char *key, unsigned int level) return (char *)result; } -static char * -packnode(node_object node) -{ - char *data = malloc(sizeof(index_node_cookie) + sizeof(node_object)); - memcpy(data, &index_node_cookie, sizeof(index_node_cookie)); - memcpy(data+sizeof(index_node_cookie), &node, sizeof(node_object)); - - return data; -} - static void addentry(node_object *node, unsigned int n, node_entry entry) { @@ -1023,14 +1036,11 @@ writenode(permdb_object *state, node_object node, const char *cachekey) put_node_in_cache(state, cachekey, node); - char *data = packnode(node); - #if DEBUG_WRITE fprintf(stderr, "writing node: offset %llu\n", offset); #endif - writebuffer_add(&state->indexfile, data, sizeof(index_node_cookie) + sizeof(node_object)); - - free(data); + writebuffer_add(&state->indexfile, &index_node_cookie, sizeof(index_node_cookie)); + writebuffer_add(&state->indexfile, &node, sizeof(node_object)); return offset; } @@ -1078,7 +1088,7 @@ readdatakey(permdb_object *state, node_offset offset) } static uint32_t -readnet32(void *ptr) +read_be32(void *ptr) { uint32_t data; @@ -1088,7 +1098,7 @@ readnet32(void *ptr) } static uint16_t -readnet16(void *ptr) +read_be16(void *ptr) { uint16_t data; @@ -1110,8 +1120,8 @@ readdatakeyandlen(permdb_object *state, node_offset offset, size_t *datalen) return NULL; } unsigned char *result = memsub(data, sizeof(data_entry_cookie), keylen); - uint16_t nchunks = readnet16(data+sizeof(data_entry_cookie)+keylen); - *datalen = readnet16(data+sizeof(data_entry_cookie)+keylen+sizeof(uint16_t)); + uint16_t nchunks = read_be16(data+sizeof(data_entry_cookie)+keylen); + *datalen = read_be16(data+sizeof(data_entry_cookie)+keylen+sizeof(uint16_t)); if (nchunks != 1) { errx(1, "number of chunks is %d, but only one chunk is supported right now", nchunks); } @@ -1132,8 +1142,6 @@ writedata(permdb_object *state, const unsigned char *key, const unsigned char *d if (datalength > 65535) { errx(1, "data length is %zu, but only < 64K lengths are supported right now", datalength); } - uint16_t chunk_length = htons(datalength); - uint16_t nchunks = htons(1); node_offset offset = state->datafile.datasize; #if DEBUG_WRITE @@ -1141,8 +1149,8 @@ writedata(permdb_object *state, const unsigned char *key, const unsigned char *d #endif writebuffer_add(&state->datafile, &data_entry_cookie, sizeof(data_entry_cookie)); writebuffer_add(&state->datafile, key, keylen); - writebuffer_add(&state->datafile, &nchunks, sizeof(uint16_t)); - writebuffer_add(&state->datafile, &chunk_length, sizeof(uint16_t)); + writebuffer_add_be16(&state->datafile, 1); + writebuffer_add_be16(&state->datafile, datalength); writebuffer_add(&state->datafile, data, datalength); return offset; @@ -1474,21 +1482,18 @@ committree(permdb_object *state) #endif int data_commit_padding_size = calc_padding(state->datafile.datasize, 4); - int data_commit_trailer_size = SHA256_DIGEST_SIZE + 8; - unsigned char *data_commit_trailer = malloc(data_commit_trailer_size); uint8_t padding[4] = {0, 0, 0, 0}; + unsigned char data_commit_checksum[SHA256_DIGEST_SIZE]; writebuffer_add(&state->datafile, data_commit_start_cookie, 8); writebuffer_add(&state->datafile, padding, data_commit_padding_size); - uint32_t data_commit_length = htonl(state->datafile.datasize - state->datafile.lastcommit + sizeof(uint32_t)); - writebuffer_add(&state->datafile, &data_commit_length, sizeof(uint32_t)); - sha256_digest(&state->datafile.commit_checksum_context, SHA256_DIGEST_SIZE, data_commit_trailer); - memcpy(data_commit_trailer + SHA256_DIGEST_SIZE, &data_commit_end_cookie, sizeof(data_commit_end_cookie)); - writebuffer_add(&state->datafile, data_commit_trailer, data_commit_trailer_size); + writebuffer_add_be32(&state->datafile, state->datafile.datasize - state->datafile.lastcommit + sizeof(uint32_t)); + sha256_digest(&state->datafile.commit_checksum_context, SHA256_DIGEST_SIZE, data_commit_checksum); + writebuffer_add(&state->datafile, data_commit_checksum, SHA256_DIGEST_SIZE); + writebuffer_add(&state->datafile, &data_commit_end_cookie, sizeof(data_commit_end_cookie)); #if DEBUG_WRITE fprintf(stderr, "finished writing data commit trailer at offset %llu\n", state->datafile.datasize); #endif - free(data_commit_trailer); if (writebuffer_flush(&state->datafile) == -1) { set_error(state, "data file flushing failed\n"); @@ -1501,17 +1506,16 @@ committree(permdb_object *state) fprintf(stderr, "writing index commit trailer at offset %llu\n", state->indexfile.datasize); #endif - uint64_t index_commit_length = state->indexfile.datasize - state->indexfile.lastcommit; - unsigned char *index_commit_trailer = malloc(INDEX_COMMIT_TRAILER_SIZE); - memcpy(index_commit_trailer, &index_commit_length, 8); - sha256_digest(&state->indexfile.commit_checksum_context, SHA256_DIGEST_SIZE, index_commit_trailer + 8); - memcpy(index_commit_trailer + 8 + SHA256_DIGEST_SIZE, &index_commit_cookie, 8); - writebuffer_add(&state->indexfile, index_commit_trailer, INDEX_COMMIT_TRAILER_SIZE); + uint64_t index_commit_length = state->indexfile.datasize - state->indexfile.lastcommit + sizeof(uint64_t); + unsigned char index_commit_checksum[SHA256_DIGEST_SIZE]; + writebuffer_add_host64(&state->indexfile, index_commit_length); + sha256_digest(&state->indexfile.commit_checksum_context, SHA256_DIGEST_SIZE, index_commit_checksum); + writebuffer_add(&state->indexfile, index_commit_checksum, SHA256_DIGEST_SIZE); + writebuffer_add(&state->indexfile, &index_commit_cookie, sizeof(index_commit_cookie)); #if DEBUG_WRITE fprintf(stderr, "finished writing index commit trailer at offset %llu\n", state->indexfile.datasize); #endif - free(index_commit_trailer); if (writebuffer_flush(&state->indexfile) == -1) { set_error(state, "index file flushing failed\n"); -- cgit v1.1