From 7893d7dcc660194b66fad9220ebb54f592ccb420 Mon Sep 17 00:00:00 2001
From: Magnus Ahltorp <map@kth.se>
Date: Sun, 23 Aug 2015 13:13:44 +0200
Subject: storagegc: start from lastverifiednewentry and write file
 periodically

---
 tools/storagegc.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/tools/storagegc.py b/tools/storagegc.py
index c13dcb5..e5d6d9d 100755
--- a/tools/storagegc.py
+++ b/tools/storagegc.py
@@ -50,15 +50,34 @@ def verifyleafhash(leaf_hash):
 
 starttime = datetime.datetime.now()
 
-lastverified = (-1, None)
 
 try:
-    for i, line in enumerate(open(db_path + "newentries")):
+    lastverifiedstring = open(db_path + "lastverifiednewentry").read()
+    lastverified = json.loads(lastverifiedstring)
+except IOError:
+    lastverified = {"index": -1, "hash": None}
+print "starting at", lastverified
+
+newentriesfile = open(db_path + "newentries")
+if lastverified["index"] >= 0:
+    newentriesfile.seek(lastverified["index"]*65)
+    assert(newentriesfile.read(64).lower() == lastverified["hash"])
+newentriesfile.seek((lastverified["index"]+1)*65)
+
+try:
+    i = lastverified["index"] + 1
+    sincewritten = 0
+    for line in newentriesfile:
         leaf_hash = base64.b16decode(line.strip(), casefold=True)
         result = verifyleafhash(leaf_hash)
         if not result:
             break
         lastverified = {"index": i, "hash": base64.b16encode(leaf_hash).lower()}
+        i += 1
+        sincewritten += 1
+        if sincewritten > 1000:
+            write_file(db_path + "lastverifiednewentry", lastverified)
+            sincewritten = 0
     if lastverified["index"] >= 0:
         write_file(db_path + "lastverifiednewentry", lastverified)
     print "lastverified", lastverified
-- 
cgit v1.1


From 00ddbf7b7cb158141fd5e641782ef7c2e59d5997 Mon Sep 17 00:00:00 2001
From: Magnus Ahltorp <map@kth.se>
Date: Mon, 24 Aug 2015 00:39:44 +0200
Subject: Use paths/public_cacertfile config variable for public https cert

---
 test/catlfish-test-local-1.cfg | 1 +
 tools/storagegc.py             | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/catlfish-test-local-1.cfg b/test/catlfish-test-local-1.cfg
index 88eda59..cbe04d8 100644
--- a/test/catlfish-test-local-1.cfg
+++ b/test/catlfish-test-local-1.cfg
@@ -18,6 +18,7 @@ paths:
         https_certfile: tests/httpscert/httpscert-1.pem
         https_keyfile: tests/httpscert/httpskey-1.pem
         https_cacertfile: tests/httpsca/demoCA/cacert.pem
+        public_cacertfile: tests/httpsca/demoCA/cacert.pem
         db: tests/machine/machine-1/db/
         publickeys: tests/publickeys
         logpublickey: tests/keys/logkey.pem
diff --git a/tools/storagegc.py b/tools/storagegc.py
index e5d6d9d..2ec0720 100755
--- a/tools/storagegc.py
+++ b/tools/storagegc.py
@@ -22,7 +22,7 @@ localconfig = yaml.load(open(args.localconfig))
 
 paths = localconfig["paths"]
 db_path = paths["db"]
-create_ssl_context(cafile=paths["https_cacertfile"])
+create_ssl_context(cafile=paths.get("public_cacertfile", None))
 
 baseurl = config["baseurl"]
 
-- 
cgit v1.1