diff options
| -rw-r--r-- | Makefile | 1 | ||||
| -rwxr-xr-x | tools/merge_backup.py | 8 | ||||
| -rwxr-xr-x | tools/merge_dist.py | 6 | ||||
| -rwxr-xr-x | tools/merge_fetch.py | 9 | ||||
| -rw-r--r-- | tools/mergetools.py | 31 |
5 files changed, 43 insertions, 12 deletions
@@ -26,7 +26,6 @@ tests-prepare: (cd $(INSTDIR)/tests/keys ; ../../../tools/create-key.sh logkey) openssl pkcs8 -topk8 -nocrypt -in $(INSTDIR)/tests/keys/logkey-private.pem -out $(INSTDIR)/tests/keys/logkey-private.pkcs8 mkdir $(INSTDIR)/tests/mergedb - mkdir $(INSTDIR)/tests/mergedb/chains touch $(INSTDIR)/tests/mergedb/logorder mkdir $(INSTDIR)/tests/mergedb-secondary touch $(INSTDIR)/tests/mergedb-secondary/logorder diff --git a/tools/merge_backup.py b/tools/merge_backup.py index 48ac133..ab2625d 100755 --- a/tools/merge_backup.py +++ b/tools/merge_backup.py @@ -12,9 +12,9 @@ from time import sleep from certtools import timing_point, build_merkle_tree, write_file, \ create_ssl_context from mergetools import chunks, backup_sendlog, get_logorder, \ - get_verifiedsize, get_missingentriesforbackup, read_chain, \ + get_verifiedsize, get_missingentriesforbackup, \ hexencode, setverifiedsize, sendentries_merge, verifyroot, \ - get_nfetched, parse_args + get_nfetched, parse_args, perm def backup_loop(nodename, nodeaddress, own_key, paths, verifiedsize, chunk): for trynumber in range(5, 0, -1): @@ -37,7 +37,7 @@ def merge_backup(args, config, localconfig, secondaries): "%s/%s-private.pem" % (paths["privatekeys"], localconfig["nodename"])) mergedb = paths["mergedb"] - chainsdir = mergedb + "/chains" + chainsdb = perm(localconfig.get("dbbackend", "filedb"), mergedb + "/chains") logorderfile = mergedb + "/logorder" currentsizefile = mergedb + "/fetched" timing = timing_point() @@ -119,7 +119,7 @@ def merge_backup(args, config, localconfig, secondaries): with requests.sessions.Session() as session: for missingentry_chunk in chunks(missingentries, 100): missingentry_hashes = [base64.b64decode(missingentry) for missingentry in missingentry_chunk] - hashes_and_entries = [(hash, read_chain(chainsdir, hash)) for hash in missingentry_hashes] + hashes_and_entries = [(hash, chainsdb.get(hash)) for hash in missingentry_hashes] sendentryresult = sendentries_merge(nodename, nodeaddress, own_key, paths, hashes_and_entries, session) diff --git a/tools/merge_dist.py b/tools/merge_dist.py index 9831f97..2af1d6c 100755 --- a/tools/merge_dist.py +++ b/tools/merge_dist.py @@ -11,7 +11,7 @@ from base64 import b64encode, b64decode from certtools import timing_point, \ create_ssl_context from mergetools import get_curpos, get_logorder, chunks, get_missingentries, \ - sendsth, sendlog, sendentry, read_chain, parse_args + sendsth, sendlog, sendentry, parse_args, perm def merge_dist(args, localconfig, frontendnodes, timestamp): paths = localconfig["paths"] @@ -19,7 +19,7 @@ def merge_dist(args, localconfig, frontendnodes, timestamp): "%s/%s-private.pem" % (paths["privatekeys"], localconfig["nodename"])) mergedb = paths["mergedb"] - chainsdir = mergedb + "/chains" + chainsdb = perm(localconfig.get("dbbackend", "filedb"), mergedb + "/chains") logorderfile = mergedb + "/logorder" sthfile = mergedb + "/sth" create_ssl_context(cafile=paths["https_cacertfile"]) @@ -92,7 +92,7 @@ def merge_dist(args, localconfig, frontendnodes, timestamp): for missingentry in missingentries: ehash = b64decode(missingentry) sendentryresult = sendentry(nodename, nodeaddress, own_key, paths, - read_chain(chainsdir, ehash), ehash) + chainsdb.get(ehash), ehash) if sendentryresult["result"] != "ok": print >>sys.stderr, "sendentry:", sendentryresult sys.exit(1) diff --git a/tools/merge_fetch.py b/tools/merge_fetch.py index bb1e17e..b77dea6 100755 --- a/tools/merge_fetch.py +++ b/tools/merge_fetch.py @@ -9,8 +9,8 @@ import struct import subprocess from time import sleep from mergetools import get_logorder, verify_entry, get_new_entries, \ - chunks, fsync_logorder, get_entries, write_chain, add_to_logorder, \ - hexencode, parse_args + chunks, fsync_logorder, get_entries, add_to_logorder, \ + hexencode, parse_args, perm from certtools import timing_point, write_file, create_ssl_context def merge_fetch(args, config, localconfig): @@ -18,7 +18,7 @@ def merge_fetch(args, config, localconfig): storagenodes = config["storagenodes"] mergedb = paths["mergedb"] logorderfile = mergedb + "/logorder" - chainsdir = mergedb + "/chains" + chainsdb = perm(localconfig.get("dbbackend", "filedb"), mergedb + "/chains") own_key = (localconfig["nodename"], "%s/%s-private.pem" % (paths["privatekeys"], localconfig["nodename"])) @@ -70,7 +70,7 @@ def merge_fetch(args, config, localconfig): for ehash in chunk: entry = entries[ehash] verify_entry(verifycert, entry, ehash) - write_chain(ehash, entry, chainsdir) + chainsdb.add(ehash, entry) add_to_logorder(logorderfile, ehash) logorder.append(ehash) certsinlog.add(ehash) @@ -79,6 +79,7 @@ def merge_fetch(args, config, localconfig): sys.stderr.flush() print >>sys.stderr sys.stderr.flush() + chainsdb.commit() fsync_logorder(logorderfile) timing_point(timing, "add entries") print >>sys.stderr, "added", added_entries, "entries" diff --git a/tools/mergetools.py b/tools/mergetools.py index ec4fd2a..80fbf0b 100644 --- a/tools/mergetools.py +++ b/tools/mergetools.py @@ -10,6 +10,10 @@ import json import yaml import argparse import requests +try: + import permdb +except ImportError: + pass from certtools import get_leaf_hash, http_request, get_leaf_hash def parselogrow(row): @@ -409,3 +413,30 @@ def parse_args(): localconfig = yaml.load(open(args.localconfig)) return (args, config, localconfig) + +def perm(dbtype, path): + if dbtype == "filedb": + return FileDB(path) + elif dbtype == "permdb": + return PermDB(path) + assert False + +class FileDB: + def __init__(self, path): + self.path = path + def get(self, key): + return read_chain(self.path, key) + def add(self, key, value): + return write_chain(key, value, self.path) + def commit(self): + pass + +class PermDB: + def __init__(self, path): + self.permdbobj = permdb.alloc(path) + def get(self, key): + return permdb.getvalue(self.permdbobj, key) + def add(self, key, value): + return permdb.addvalue(self.permdbobj, key, value) + def commit(self): + permdb.committree(self.permdbobj) |