summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLinus Nordberg <linus@nordu.net>2015-08-24 10:03:40 +0200
committerLinus Nordberg <linus@nordu.net>2015-08-24 10:03:40 +0200
commit1a0506b2368c3eb0e4383d78db3b4577d67ff1df (patch)
tree21bff9fba34b98eb008caaa845f301bbe35b1f42
parent0e779011e0663ecfd2c088c38cbc78d3457af871 (diff)
parent00ddbf7b7cb158141fd5e641782ef7c2e59d5997 (diff)
Merge remote-tracking branch 'refs/remotes/map/storagegc2'
-rw-r--r--test/catlfish-test-local-1.cfg1
-rwxr-xr-xtools/storagegc.py25
2 files changed, 23 insertions, 3 deletions
diff --git a/test/catlfish-test-local-1.cfg b/test/catlfish-test-local-1.cfg
index 88eda59..cbe04d8 100644
--- a/test/catlfish-test-local-1.cfg
+++ b/test/catlfish-test-local-1.cfg
@@ -18,6 +18,7 @@ paths:
https_certfile: tests/httpscert/httpscert-1.pem
https_keyfile: tests/httpscert/httpskey-1.pem
https_cacertfile: tests/httpsca/demoCA/cacert.pem
+ public_cacertfile: tests/httpsca/demoCA/cacert.pem
db: tests/machine/machine-1/db/
publickeys: tests/publickeys
logpublickey: tests/keys/logkey.pem
diff --git a/tools/storagegc.py b/tools/storagegc.py
index f18074a..a4f15b7 100755
--- a/tools/storagegc.py
+++ b/tools/storagegc.py
@@ -22,7 +22,7 @@ localconfig = yaml.load(open(args.localconfig))
paths = localconfig["paths"]
db_path = paths["db"]
-create_ssl_context(cafile=paths["https_cacertfile"])
+create_ssl_context(cafile=paths.get("public_cacertfile", None))
baseurl = config["baseurl"]
@@ -50,15 +50,34 @@ def verifyleafhash(leaf_hash):
starttime = datetime.datetime.now()
-lastverified = (-1, None)
try:
- for i, line in enumerate(open(db_path + "newentries")):
+ lastverifiedstring = open(db_path + "lastverifiednewentry").read()
+ lastverified = json.loads(lastverifiedstring)
+except IOError:
+ lastverified = {"index": -1, "hash": None}
+print "starting at", lastverified
+
+newentriesfile = open(db_path + "newentries")
+if lastverified["index"] >= 0:
+ newentriesfile.seek(lastverified["index"]*65)
+ assert(newentriesfile.read(64).lower() == lastverified["hash"])
+newentriesfile.seek((lastverified["index"]+1)*65)
+
+try:
+ i = lastverified["index"] + 1
+ sincewritten = 0
+ for line in newentriesfile:
leaf_hash = base64.b16decode(line.strip(), casefold=True)
result = verifyleafhash(leaf_hash)
if not result:
break
lastverified = {"index": i, "hash": base64.b16encode(leaf_hash).lower()}
+ i += 1
+ sincewritten += 1
+ if sincewritten > 1000:
+ write_file(db_path + "lastverifiednewentry", lastverified)
+ sincewritten = 0
if lastverified["index"] >= 0:
write_file(db_path + "lastverifiednewentry", lastverified)
print "lastverified", lastverified