summaryrefslogtreecommitdiff
path: root/tools/storagegc.py
blob: a4f15b798a5c0ec41fe16aa3500417b7219af012 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
#!/usr/bin/env python

# Copyright (c) 2015, NORDUnet A/S.
# See LICENSE for licensing information.

import argparse
import urllib2
import urllib
import json
import base64
import sys
import yaml
from certtools import *

parser = argparse.ArgumentParser(description='')
parser.add_argument('--config', help="System configuration", required=True)
parser.add_argument('--localconfig', help="Local configuration", required=True)
args = parser.parse_args()

config = yaml.load(open(args.config))
localconfig = yaml.load(open(args.localconfig))

paths = localconfig["paths"]
db_path = paths["db"]
create_ssl_context(cafile=paths.get("public_cacertfile", None))

baseurl = config["baseurl"]

sth = get_sth(baseurl)

def verifyleafhash(leaf_hash):
    try:
        proof = get_proof_by_hash(baseurl, leaf_hash, sth["tree_size"])
    except SystemExit:
        return False

    leaf_index = proof["leaf_index"]
    inclusion_proof = [base64.b64decode(e) for e in proof["audit_path"]]

    calc_root_hash = verify_inclusion_proof(inclusion_proof, leaf_index, sth["tree_size"], leaf_hash)

    root_hash = base64.b64decode(sth["sha256_root_hash"])
    if root_hash != calc_root_hash:
        print "sth calculation incorrect:"
        print base64.b16encode(root_hash)
        print base64.b16encode(calc_root_hash)
        sys.exit(1)

    return True

starttime = datetime.datetime.now()


try:
    lastverifiedstring = open(db_path + "lastverifiednewentry").read()
    lastverified = json.loads(lastverifiedstring)
except IOError:
    lastverified = {"index": -1, "hash": None}
print "starting at", lastverified

newentriesfile = open(db_path + "newentries")
if lastverified["index"] >= 0:
    newentriesfile.seek(lastverified["index"]*65)
    assert(newentriesfile.read(64).lower() == lastverified["hash"])
newentriesfile.seek((lastverified["index"]+1)*65)

try:
    i = lastverified["index"] + 1
    sincewritten = 0
    for line in newentriesfile:
        leaf_hash = base64.b16decode(line.strip(), casefold=True)
        result = verifyleafhash(leaf_hash)
        if not result:
            break
        lastverified = {"index": i, "hash": base64.b16encode(leaf_hash).lower()}
        i += 1
        sincewritten += 1
        if sincewritten > 1000:
            write_file(db_path + "lastverifiednewentry", lastverified)
            sincewritten = 0
    if lastverified["index"] >= 0:
        write_file(db_path + "lastverifiednewentry", lastverified)
    print "lastverified", lastverified
except KeyboardInterrupt:
    pass