Move code to util.c and filebuffer.c

1 files changed, 225 insertions, 0 deletions

diff --git a/c_src/filebuffer.c b/c_src/filebuffer.c
new file mode 100644
index 0000000..c6068e8
--- /dev/null
+++ b/c_src/filebuffer.c
@@ -0,0 +1,225 @@
+/*
+ * Copyright (c) 2015, NORDUnet A/S.
+ * See LICENSE for licensing information.
+ */
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <assert.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <arpa/inet.h>
+#include <fcntl.h>
+#include <err.h>
+#include <nettle/sha2.h>
+
+#include "permdb.h"
+#include "filebuffer.h"
+#include "util.h"
+
+struct buffered_file {
+        int fd;
+        const char *name;
+        uint64_t datasize;
+        uint64_t lastcommit;
+        uint64_t filesize;
+        char *writebuffer;
+        uint64_t writebufferalloc;
+        struct sha256_ctx commit_checksum_context;
+};
+
+void
+bf_add_host64(buffered_file *file, uint64_t value) {
+        bf_add(file, &value, sizeof(uint64_t));
+}
+
+void
+bf_add_be32(buffered_file *file, uint32_t value) {
+        uint32_t value_be = htonl(value);
+        bf_add(file, &value_be, sizeof(uint32_t));
+}
+
+void
+bf_add_be16(buffered_file *file, uint16_t value) {
+        uint16_t value_be = htons(value);
+        bf_add(file, &value_be, sizeof(uint16_t));
+}
+
+uint64_t
+bf_total_length(buffered_file *file)
+{
+        return file->datasize;
+}
+
+uint64_t
+bf_lastcommit(buffered_file *file)
+{
+        return file->lastcommit;
+}
+
+const char *
+bf_name(buffered_file *file)
+{
+        return file->name;
+}
+
+static uint64_t
+bf_unwritten_length(buffered_file *file)
+{
+        return file->datasize - file->filesize;
+}
+
+void
+bf_add(buffered_file *file, const void *data, uint64_t length)
+{
+        sha256_update(&file->commit_checksum_context, length, data);
+#if DEBUG_WRITE
+        fprintf(stderr, "adding data to %s: ", file->name);
+        print_hex(data, length);
+#endif
+        uint64_t needspace = length + bf_unwritten_length(file);
+
+        if (needspace > file->writebufferalloc) {
+                int ret = bf_flush_nosync(file);
+                if (ret < 0) {
+                        err(1, "bf_flush_nosync failed");
+                }
+
+                needspace = length + bf_unwritten_length(file);
+
+                if (needspace > file->writebufferalloc) {
+
+                        uint64_t newsize = file->writebufferalloc * 2;
+                        if (needspace > newsize) {
+                                newsize = needspace;
+                        }
+                        file->writebuffer = realloc(file->writebuffer, newsize);
+                        memset(file->writebuffer + file->writebufferalloc, 0, newsize - file->writebufferalloc);
+                        file->writebufferalloc = newsize;
+                }
+        }
+
+        memcpy(file->writebuffer + bf_unwritten_length(file), data, length);
+        file->datasize += length;
+}
+
+int
+bf_flush_nosync(buffered_file *file)
+{
+        ssize_t ret;
+
+        uint64_t length = bf_unwritten_length(file);
+        ret = write(file->fd, file->writebuffer, length);
+        if (ret != length) {
+                return -1;
+        }
+        file->filesize += (uint64_t) ret;
+        return 0;
+}
+
+int
+bf_flush(buffered_file *file)
+{
+        int ret;
+
+        ret = bf_flush_nosync(file);
+        if (ret) {
+                return ret;
+        }
+
+        ret = fsync(file->fd);
+        sha256_init(&file->commit_checksum_context);
+#if DEBUG_WRITE
+        fprintf(stderr, "clearing data for %s\n", file->name);
+#endif
+        file->lastcommit = bf_total_length(file);
+        return ret;
+}
+
+unsigned char *
+bf_read(buffered_file *file, uint64_t offset, size_t length, char **error)
+{
+    unsigned char *result = malloc(length);
+#if DEBUG_READ
+    fprintf(stderr, "reading data: offset %llu\n", offset);
+#endif
+
+    if (offset >= file->filesize) {
+            uint64_t writebufferoffset = offset - file->filesize;
+            if (offset + length > file->datasize) {
+                    set_error(error, "pread: not enough data for offset %llu and length %zu\n", (long long unsigned int) offset, length);
+                    return NULL;
+            }
+            memcpy(result, file->writebuffer + writebufferoffset, length);
+    } else {
+            if (offset + length > file->filesize) {
+                    set_error(error, "pread: trying to read over file/writebuffer boundary for offset %llu and length %zu\n", (long long unsigned int) offset, length);
+                    return NULL;
+            }
+            
+            ssize_t ret = pread(file->fd, result, length, (off_t) offset);
+            if (ret != length) {
+                    free(result);
+                    set_error(error, "short pread: %zd (wanted %zu) at offset %llu\n", ret, length, (long long unsigned int) offset);
+                    return NULL;
+            }
+    }
+
+    return result;
+}
+
+buffered_file *
+bf_open(const char *path, int flags, const char *name)
+{
+        buffered_file *file = malloc(sizeof(buffered_file));
+
+        file->fd = open(path, flags, 0666);
+        if (file->fd == -1) {
+                warn("open %s", path);
+                return NULL;
+        }
+
+        file->name = name;
+
+        off_t datafile_filesize = lseek(file->fd, 0, SEEK_END);
+        if (datafile_filesize < 0) {
+                warn("lseek %s", path);
+                return NULL;
+        }
+        file->filesize = (uint64_t) datafile_filesize;
+        file->datasize = file->filesize;
+        file->lastcommit = file->datasize;
+        file->writebufferalloc = 1024*1024;
+        file->writebuffer = calloc(file->writebufferalloc, 1);
+        sha256_init(&file->commit_checksum_context);
+
+        return file;
+}
+
+void
+bf_close(buffered_file *file)
+{
+        bf_flush(file);
+        close(file->fd);
+        free(file->writebuffer);
+        free(file);
+}
+
+void
+bf_truncate(buffered_file *file)
+{
+        file->filesize = 0;
+        file->datasize = 0;
+        file->lastcommit = 0;
+        sha256_init(&file->commit_checksum_context);
+        ftruncate(file->fd, 0);
+}
+
+void
+bf_sha256(buffered_file *file, unsigned char *checksum)
+{
+        sha256_digest(&file->commit_checksum_context, SHA256_DIGEST_SIZE, checksum);
+}