Added permdb support to merge

4 files changed, 43 insertions, 11 deletions

diff --git a/tools/merge_backup.py b/tools/merge_backup.py
index 48ac133..ab2625d 100755
--- a/tools/merge_backup.py
+++ b/tools/merge_backup.py
@@ -12,9 +12,9 @@ from time import sleep
 from certtools import timing_point, build_merkle_tree, write_file, \
      create_ssl_context
 from mergetools import chunks, backup_sendlog, get_logorder, \
-     get_verifiedsize, get_missingentriesforbackup, read_chain, \
+     get_verifiedsize, get_missingentriesforbackup, \
      hexencode, setverifiedsize, sendentries_merge, verifyroot, \
-     get_nfetched, parse_args
+     get_nfetched, parse_args, perm
 
 def backup_loop(nodename, nodeaddress, own_key, paths, verifiedsize, chunk):
     for trynumber in range(5, 0, -1):
@@ -37,7 +37,7 @@ def merge_backup(args, config, localconfig, secondaries):
                "%s/%s-private.pem" % (paths["privatekeys"],
                                       localconfig["nodename"]))
     mergedb = paths["mergedb"]
-    chainsdir = mergedb + "/chains"
+    chainsdb = perm(localconfig.get("dbbackend", "filedb"), mergedb + "/chains")
     logorderfile = mergedb + "/logorder"
     currentsizefile = mergedb + "/fetched"
     timing = timing_point()
@@ -119,7 +119,7 @@ def merge_backup(args, config, localconfig, secondaries):
             with requests.sessions.Session() as session:
                 for missingentry_chunk in chunks(missingentries, 100):
                     missingentry_hashes = [base64.b64decode(missingentry) for missingentry in missingentry_chunk]
-                    hashes_and_entries = [(hash, read_chain(chainsdir, hash)) for hash in missingentry_hashes]
+                    hashes_and_entries = [(hash, chainsdb.get(hash)) for hash in missingentry_hashes]
                     sendentryresult = sendentries_merge(nodename, nodeaddress,
                                                         own_key, paths,
                                                         hashes_and_entries, session)
diff --git a/tools/merge_dist.py b/tools/merge_dist.py
index 9831f97..2af1d6c 100755
--- a/tools/merge_dist.py
+++ b/tools/merge_dist.py
@@ -11,7 +11,7 @@ from base64 import b64encode, b64decode
 from certtools import timing_point, \
      create_ssl_context
 from mergetools import get_curpos, get_logorder, chunks, get_missingentries, \
-     sendsth, sendlog, sendentry, read_chain, parse_args
+     sendsth, sendlog, sendentry, parse_args, perm
 
 def merge_dist(args, localconfig, frontendnodes, timestamp):
     paths = localconfig["paths"]
@@ -19,7 +19,7 @@ def merge_dist(args, localconfig, frontendnodes, timestamp):
                "%s/%s-private.pem" % (paths["privatekeys"],
                                       localconfig["nodename"]))
     mergedb = paths["mergedb"]
-    chainsdir = mergedb + "/chains"
+    chainsdb = perm(localconfig.get("dbbackend", "filedb"), mergedb + "/chains")
     logorderfile = mergedb + "/logorder"
     sthfile = mergedb + "/sth"
     create_ssl_context(cafile=paths["https_cacertfile"])
@@ -92,7 +92,7 @@ def merge_dist(args, localconfig, frontendnodes, timestamp):
         for missingentry in missingentries:
             ehash = b64decode(missingentry)
             sendentryresult = sendentry(nodename, nodeaddress, own_key, paths,
-                                        read_chain(chainsdir, ehash), ehash)
+                                        chainsdb.get(ehash), ehash)
             if sendentryresult["result"] != "ok":
                 print >>sys.stderr, "sendentry:", sendentryresult
                 sys.exit(1)
diff --git a/tools/merge_fetch.py b/tools/merge_fetch.py
index bb1e17e..b77dea6 100755
--- a/tools/merge_fetch.py
+++ b/tools/merge_fetch.py
@@ -9,8 +9,8 @@ import struct
 import subprocess
 from time import sleep
 from mergetools import get_logorder, verify_entry, get_new_entries, \
-     chunks, fsync_logorder, get_entries, write_chain, add_to_logorder, \
-     hexencode, parse_args
+     chunks, fsync_logorder, get_entries, add_to_logorder, \
+     hexencode, parse_args, perm
 from certtools import timing_point, write_file, create_ssl_context
 
 def merge_fetch(args, config, localconfig):
@@ -18,7 +18,7 @@ def merge_fetch(args, config, localconfig):
     storagenodes = config["storagenodes"]
     mergedb = paths["mergedb"]
     logorderfile = mergedb + "/logorder"
-    chainsdir = mergedb + "/chains"
+    chainsdb = perm(localconfig.get("dbbackend", "filedb"), mergedb + "/chains")
     own_key = (localconfig["nodename"],
                "%s/%s-private.pem" % (paths["privatekeys"],
                                       localconfig["nodename"]))
@@ -70,7 +70,7 @@ def merge_fetch(args, config, localconfig):
             for ehash in chunk:
                 entry = entries[ehash]
                 verify_entry(verifycert, entry, ehash)
-                write_chain(ehash, entry, chainsdir)
+                chainsdb.add(ehash, entry)
                 add_to_logorder(logorderfile, ehash)
                 logorder.append(ehash)
                 certsinlog.add(ehash)
@@ -79,6 +79,7 @@ def merge_fetch(args, config, localconfig):
             sys.stderr.flush()
         print >>sys.stderr
         sys.stderr.flush()
+    chainsdb.commit()
     fsync_logorder(logorderfile)
     timing_point(timing, "add entries")
     print >>sys.stderr, "added", added_entries, "entries"
diff --git a/tools/mergetools.py b/tools/mergetools.py
index ec4fd2a..80fbf0b 100644
--- a/tools/mergetools.py
+++ b/tools/mergetools.py
@@ -10,6 +10,10 @@ import json
 import yaml
 import argparse
 import requests
+try:
+    import permdb
+except ImportError:
+    pass
 from certtools import get_leaf_hash, http_request, get_leaf_hash
 
 def parselogrow(row):
@@ -409,3 +413,30 @@ def parse_args():
     localconfig = yaml.load(open(args.localconfig))
 
     return (args, config, localconfig)
+
+def perm(dbtype, path):
+    if dbtype == "filedb":
+        return FileDB(path)
+    elif dbtype == "permdb":
+        return PermDB(path)
+    assert False
+
+class FileDB:
+    def __init__(self, path):
+        self.path = path
+    def get(self, key):
+        return read_chain(self.path, key)
+    def add(self, key, value):
+        return write_chain(key, value, self.path)
+    def commit(self):
+        pass
+
+class PermDB:
+    def __init__(self, path):
+        self.permdbobj = permdb.alloc(path)
+    def get(self, key):
+        return permdb.getvalue(self.permdbobj, key)
+    def add(self, key, value):
+        return permdb.addvalue(self.permdbobj, key, value)
+    def commit(self):
+        permdb.committree(self.permdbobj)