diff options
| -rw-r--r-- | test/catlfish-test-local-1.cfg | 1 | ||||
| -rwxr-xr-x | tools/storagegc.py | 25 |
2 files changed, 23 insertions, 3 deletions
diff --git a/test/catlfish-test-local-1.cfg b/test/catlfish-test-local-1.cfg index 88eda59..cbe04d8 100644 --- a/test/catlfish-test-local-1.cfg +++ b/test/catlfish-test-local-1.cfg @@ -18,6 +18,7 @@ paths: https_certfile: tests/httpscert/httpscert-1.pem https_keyfile: tests/httpscert/httpskey-1.pem https_cacertfile: tests/httpsca/demoCA/cacert.pem + public_cacertfile: tests/httpsca/demoCA/cacert.pem db: tests/machine/machine-1/db/ publickeys: tests/publickeys logpublickey: tests/keys/logkey.pem diff --git a/tools/storagegc.py b/tools/storagegc.py index f18074a..a4f15b7 100755 --- a/tools/storagegc.py +++ b/tools/storagegc.py @@ -22,7 +22,7 @@ localconfig = yaml.load(open(args.localconfig)) paths = localconfig["paths"] db_path = paths["db"] -create_ssl_context(cafile=paths["https_cacertfile"]) +create_ssl_context(cafile=paths.get("public_cacertfile", None)) baseurl = config["baseurl"] @@ -50,15 +50,34 @@ def verifyleafhash(leaf_hash): starttime = datetime.datetime.now() -lastverified = (-1, None) try: - for i, line in enumerate(open(db_path + "newentries")): + lastverifiedstring = open(db_path + "lastverifiednewentry").read() + lastverified = json.loads(lastverifiedstring) +except IOError: + lastverified = {"index": -1, "hash": None} +print "starting at", lastverified + +newentriesfile = open(db_path + "newentries") +if lastverified["index"] >= 0: + newentriesfile.seek(lastverified["index"]*65) + assert(newentriesfile.read(64).lower() == lastverified["hash"]) +newentriesfile.seek((lastverified["index"]+1)*65) + +try: + i = lastverified["index"] + 1 + sincewritten = 0 + for line in newentriesfile: leaf_hash = base64.b16decode(line.strip(), casefold=True) result = verifyleafhash(leaf_hash) if not result: break lastverified = {"index": i, "hash": base64.b16encode(leaf_hash).lower()} + i += 1 + sincewritten += 1 + if sincewritten > 1000: + write_file(db_path + "lastverifiednewentry", lastverified) + sincewritten = 0 if lastverified["index"] >= 0: write_file(db_path + "lastverifiednewentry", lastverified) print "lastverified", lastverified |