diff options
| -rwxr-xr-x | tools/storagegc.py | 23 |
1 files changed, 21 insertions, 2 deletions
diff --git a/tools/storagegc.py b/tools/storagegc.py index c13dcb5..e5d6d9d 100755 --- a/tools/storagegc.py +++ b/tools/storagegc.py @@ -50,15 +50,34 @@ def verifyleafhash(leaf_hash): starttime = datetime.datetime.now() -lastverified = (-1, None) try: - for i, line in enumerate(open(db_path + "newentries")): + lastverifiedstring = open(db_path + "lastverifiednewentry").read() + lastverified = json.loads(lastverifiedstring) +except IOError: + lastverified = {"index": -1, "hash": None} +print "starting at", lastverified + +newentriesfile = open(db_path + "newentries") +if lastverified["index"] >= 0: + newentriesfile.seek(lastverified["index"]*65) + assert(newentriesfile.read(64).lower() == lastverified["hash"]) +newentriesfile.seek((lastverified["index"]+1)*65) + +try: + i = lastverified["index"] + 1 + sincewritten = 0 + for line in newentriesfile: leaf_hash = base64.b16decode(line.strip(), casefold=True) result = verifyleafhash(leaf_hash) if not result: break lastverified = {"index": i, "hash": base64.b16encode(leaf_hash).lower()} + i += 1 + sincewritten += 1 + if sincewritten > 1000: + write_file(db_path + "lastverifiednewentry", lastverified) + sincewritten = 0 if lastverified["index"] >= 0: write_file(db_path + "lastverifiednewentry", lastverified) print "lastverified", lastverified |