summaryrefslogtreecommitdiff
path: root/tools/storagegc.py
diff options
context:
space:
mode:
authorMagnus Ahltorp <map@kth.se>2015-08-23 13:13:44 +0200
committerMagnus Ahltorp <map@kth.se>2015-08-23 13:13:44 +0200
commit7893d7dcc660194b66fad9220ebb54f592ccb420 (patch)
tree8b9e9f081e64760cb98fc925ba140714e8ba1a47 /tools/storagegc.py
parent03b95596ecc425cfbb0d29d40ba9accbc9e67729 (diff)
storagegc: start from lastverifiednewentry and write file periodically
Diffstat (limited to 'tools/storagegc.py')
-rwxr-xr-xtools/storagegc.py23
1 files changed, 21 insertions, 2 deletions
diff --git a/tools/storagegc.py b/tools/storagegc.py
index c13dcb5..e5d6d9d 100755
--- a/tools/storagegc.py
+++ b/tools/storagegc.py
@@ -50,15 +50,34 @@ def verifyleafhash(leaf_hash):
starttime = datetime.datetime.now()
-lastverified = (-1, None)
try:
- for i, line in enumerate(open(db_path + "newentries")):
+ lastverifiedstring = open(db_path + "lastverifiednewentry").read()
+ lastverified = json.loads(lastverifiedstring)
+except IOError:
+ lastverified = {"index": -1, "hash": None}
+print "starting at", lastverified
+
+newentriesfile = open(db_path + "newentries")
+if lastverified["index"] >= 0:
+ newentriesfile.seek(lastverified["index"]*65)
+ assert(newentriesfile.read(64).lower() == lastverified["hash"])
+newentriesfile.seek((lastverified["index"]+1)*65)
+
+try:
+ i = lastverified["index"] + 1
+ sincewritten = 0
+ for line in newentriesfile:
leaf_hash = base64.b16decode(line.strip(), casefold=True)
result = verifyleafhash(leaf_hash)
if not result:
break
lastverified = {"index": i, "hash": base64.b16encode(leaf_hash).lower()}
+ i += 1
+ sincewritten += 1
+ if sincewritten > 1000:
+ write_file(db_path + "lastverifiednewentry", lastverified)
+ sincewritten = 0
if lastverified["index"] >= 0:
write_file(db_path + "lastverifiednewentry", lastverified)
print "lastverified", lastverified