1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
|
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (c) 2014, NORDUnet A/S.
# See LICENSE for licensing information.
import argparse
import urllib2
import urllib
import json
import base64
import sys
import struct
import hashlib
import itertools
from certtools import *
import zipfile
import os
import time
parser = argparse.ArgumentParser(description='')
parser.add_argument('baseurl', help="Base URL for CT server")
parser.add_argument('--store', default=None, metavar="dir", help='Store certificates in directory dir')
parser.add_argument('--write-sth', action='store_true', help='Write STH')
args = parser.parse_args()
def extract_original_entry(entry):
leaf_input = base64.decodestring(entry["leaf_input"])
(leaf_cert, timestamp, issuer_key_hash) = unpack_mtl(leaf_input)
extra_data = base64.decodestring(entry["extra_data"])
if issuer_key_hash != None:
(precert, extra_data) = extract_precertificate(extra_data)
leaf_cert = precert
certchain = decode_certificate_chain(extra_data)
return ([leaf_cert] + certchain, timestamp, issuer_key_hash)
def get_entries_wrapper(baseurl, start, end):
fetched_entries = 0
while start + fetched_entries < (end + 1):
print "fetching from", start + fetched_entries
entries = get_entries(baseurl, start + fetched_entries, end)["entries"]
if len(entries) == 0:
break
for entry in entries:
fetched_entries += 1
yield entry
def print_layer(layer):
for entry in layer:
print base64.b16encode(entry)
sth = get_sth(args.baseurl)
check_sth_signature(args.baseurl, sth)
tree_size = sth["tree_size"]
root_hash = base64.decodestring(sth["sha256_root_hash"])
try:
if args.store:
oldsth = json.load(open(args.store + "/currentsth"))
else:
oldsth = None
except IOError:
oldsth = None
sth_timestamp = datetime.datetime.fromtimestamp(sth["timestamp"]/1000)
since_timestamp = time.time() - sth["timestamp"]/1000
print "Log last updated %s, %d seconds ago" % (sth_timestamp.ctime(), since_timestamp)
print "tree size", tree_size
print "root hash", base64.b16encode(root_hash)
if oldsth:
if oldsth["tree_size"] == tree_size:
print "Tree size has not changed"
if oldsth["sha256_root_hash"] != sth["sha256_root_hash"]:
print "Root hash is different even though tree size is the same."
print "Log has violated the append-only property."
print "Old hash:", oldsth["sha256_root_hash"]
print "New hash:", sth["sha256_root_hash"]
sys.exit(1)
if oldsth["timestamp"] == sth["timestamp"]:
print "Timestamp has not changed"
else:
print "Tree size changed, old tree size was", oldsth["tree_size"]
merkle_64klayer = []
if args.store:
ncerts = None
for blocknumber in range(0, (tree_size / 65536) + 1):
(resulttype, result) = get_merkle_hash_64k(args.store, blocknumber, write_to_cache=True)
if resulttype == "incomplete":
(incompletelength, hash) = result
ncerts = blocknumber * 65536 + incompletelength
break
assert resulttype == "hash"
hash = result
merkle_64klayer.append(hash)
print blocknumber * 65536,
sys.stdout.flush()
print
print "ncerts", ncerts
else:
ncerts = 0
entries = get_entries_wrapper(args.baseurl, ncerts, tree_size - 1)
if not args.store:
layer0 = [get_leaf_hash(base64.decodestring(entry["leaf_input"])) for entry in entries]
tree = build_merkle_tree(layer0)
calculated_root_hash = tree[-1][0]
else:
currentfilename = None
zf = None
for entry, i in itertools.izip(entries, itertools.count(ncerts)):
try:
(chain, timestamp, issuer_key_hash) = extract_original_entry(entry)
zipfilename = args.store + "/" + ("%04d.zip" % (i / 10000))
if zipfilename != currentfilename:
if zf:
zf.close()
zf = zipfile.ZipFile(zipfilename, "a",
compression=zipfile.ZIP_DEFLATED)
currentfilename = zipfilename
s = ""
s += "Timestamp: %s\n" % timestamp
leaf_input = base64.decodestring(entry["leaf_input"])
leaf_hash = get_leaf_hash(leaf_input)
s += "Leafhash: %s\n" % base64.b16encode(leaf_hash)
if issuer_key_hash:
s += "-----BEGIN PRECERTIFICATE-----\n"
s += base64.encodestring(chain[0]).rstrip() + "\n"
s += "-----END PRECERTIFICATE-----\n"
s += "\n"
chain = chain[1:]
for cert in chain:
s += "-----BEGIN CERTIFICATE-----\n"
s += base64.encodestring(cert).rstrip() + "\n"
s += "-----END CERTIFICATE-----\n"
s += "\n"
zf.writestr("%08d" % i, s)
except AssertionError, e:
print "error for cert", i, e
if zf:
zf.close()
for blocknumber in range(ncerts / 65536, (tree_size / 65536) + 1):
(resulttype, result) = get_merkle_hash_64k(args.store, blocknumber, write_to_cache=True)
if resulttype == "incomplete":
(incompletelength, hash) = result
ncerts = blocknumber * 65536 + incompletelength
merkle_64klayer.append(hash)
break
assert resulttype == "hash"
hash = result
merkle_64klayer.append(hash)
print blocknumber * 65536, base64.b16encode(hash)
tree = build_merkle_tree(merkle_64klayer)
calculated_root_hash = tree[-1][0]
assert ncerts == tree_size
print "calculated root hash", base64.b16encode(calculated_root_hash)
if oldsth and oldsth["tree_size"] > 0 and oldsth["tree_size"] != tree_size:
consistency_proof = [base64.decodestring(entry) for entry in get_consistency_proof(args.baseurl, oldsth["tree_size"], tree_size)]
(old_treehead, new_treehead) = verify_consistency_proof(consistency_proof, oldsth["tree_size"], tree_size)
assert old_treehead == base64.b64decode(oldsth["sha256_root_hash"])
assert new_treehead == base64.b64decode(sth["sha256_root_hash"])
if calculated_root_hash != root_hash:
print "fetched root hash and calculated root hash different"
sys.exit(1)
if args.store and args.write_sth:
f = open(args.store + "/currentsth", "w")
f.write(json.dumps(sth))
f.close()
|
