1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
|
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (c) 2014, NORDUnet A/S.
# See LICENSE for licensing information.
import argparse
import urllib2
import urllib
import json
import base64
import sys
import struct
import hashlib
import itertools
from certtools import *
import zipfile
import os
import time
parser = argparse.ArgumentParser(description='')
parser.add_argument('baseurl', help="Base URL for CT server")
parser.add_argument('--store', default=None, metavar="dir", help='Store certificates in directory dir')
parser.add_argument('--write-sth', action='store_true', help='Write STH')
args = parser.parse_args()
def extract_original_entry(entry):
leaf_input = base64.decodestring(entry["leaf_input"])
(leaf_cert, timestamp, issuer_key_hash) = unpack_mtl(leaf_input)
extra_data = base64.decodestring(entry["extra_data"])
if issuer_key_hash != None:
(precert, extra_data) = extract_precertificate(extra_data)
leaf_cert = precert
certchain = decode_certificate_chain(extra_data)
return ([leaf_cert] + certchain, timestamp, issuer_key_hash)
def get_entries_wrapper(baseurl, start, end):
fetched_entries = 0
while start + fetched_entries < (end + 1):
print "fetching from", start + fetched_entries
entries = get_entries(baseurl, start + fetched_entries, end)["entries"]
if len(entries) == 0:
break
for entry in entries:
fetched_entries += 1
yield entry
def print_layer(layer):
for entry in layer:
print base64.b16encode(entry)
def get_hash_from_certfile(cert):
for line in cert.split("\n"):
if line.startswith("-----"):
return None
if line.startswith("Leafhash: "):
return base64.b16decode(line[len("Leafhash: "):])
return None
def get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf):
for i in range(firstleaf, lastleaf + 1):
try:
yield zipfiles[i / 10000].read("%08d" % i)
except KeyError:
return
def get_merkle_hash_64k(store, blocknumber):
hashfilename = "%s/%04x.64khash" % (store, blocknumber)
try:
hash = base64.b16decode(open(hashfilename).read())
assert len(hash) == 32
return ("hash", hash)
except IOError:
pass
firstleaf = blocknumber * 65536
lastleaf = firstleaf + 65535
firstfile = firstleaf / 10000
lastfile = lastleaf / 10000
zipfiles = {}
for i in range(firstfile, lastfile + 1):
try:
zipfiles[i] = zipfile.ZipFile("%s/%04d.zip" % (store, i))
except IOError:
break
certs = get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf)
layer0 = [get_hash_from_certfile(cert) for cert in certs]
tree = build_merkle_tree(layer0)
calculated_hash = tree[-1][0]
for zf in zipfiles.values():
zf.close()
if len(layer0) != 65536:
return ("incomplete", (len(layer0), calculated_hash))
f = open(hashfilename, "w")
f.write(base64.b16encode(calculated_hash))
f.close()
return ("hash", calculated_hash)
sth = get_sth(args.baseurl)
check_sth_signature(args.baseurl, sth)
tree_size = sth["tree_size"]
root_hash = base64.decodestring(sth["sha256_root_hash"])
try:
if args.store:
oldsth = json.load(open(args.store + "/currentsth"))
else:
oldsth = None
except IOError:
oldsth = None
sth_timestamp = datetime.datetime.fromtimestamp(sth["timestamp"]/1000)
since_timestamp = time.time() - sth["timestamp"]/1000
print "Log last updated %s, %d seconds ago" % (sth_timestamp.ctime(), since_timestamp)
print "tree size", tree_size
print "root hash", base64.b16encode(root_hash)
if oldsth:
if oldsth["tree_size"] == tree_size:
print "Tree size has not changed"
if oldsth["sha256_root_hash"] != sth["sha256_root_hash"]:
print "Root hash is different even though tree size is the same."
print "Log has violated the append-only property."
print "Old hash:", oldsth["sha256_root_hash"]
print "New hash:", sth["sha256_root_hash"]
sys.exit(1)
if oldsth["timestamp"] == sth["timestamp"]:
print "Timestamp has not changed"
else:
print "Tree size changed, old tree size was", oldsth["tree_size"]
merkle_64klayer = []
if args.store:
ncerts = None
for blocknumber in range(0, (tree_size / 65536) + 1):
(resulttype, result) = get_merkle_hash_64k(args.store, blocknumber)
if resulttype == "incomplete":
(incompletelength, hash) = result
ncerts = blocknumber * 65536 + incompletelength
break
assert resulttype == "hash"
hash = result
merkle_64klayer.append(hash)
print blocknumber * 65536,
sys.stdout.flush()
print
print "ncerts", ncerts
else:
ncerts = 0
entries = get_entries_wrapper(args.baseurl, ncerts, tree_size - 1)
if not args.store:
layer0 = [get_leaf_hash(base64.decodestring(entry["leaf_input"])) for entry in entries]
tree = build_merkle_tree(layer0)
calculated_root_hash = tree[-1][0]
else:
currentfilename = None
zf = None
for entry, i in itertools.izip(entries, itertools.count(ncerts)):
try:
(chain, timestamp, issuer_key_hash) = extract_original_entry(entry)
zipfilename = args.store + "/" + ("%04d.zip" % (i / 10000))
if zipfilename != currentfilename:
if zf:
zf.close()
zf = zipfile.ZipFile(zipfilename, "a",
compression=zipfile.ZIP_DEFLATED)
currentfilename = zipfilename
s = ""
s += "Timestamp: %s\n" % timestamp
leaf_input = base64.decodestring(entry["leaf_input"])
leaf_hash = get_leaf_hash(leaf_input)
s += "Leafhash: %s\n" % base64.b16encode(leaf_hash)
if issuer_key_hash:
s += "-----BEGIN PRECERTIFICATE-----\n"
s += base64.encodestring(chain[0]).rstrip() + "\n"
s += "-----END PRECERTIFICATE-----\n"
s += "\n"
chain = chain[1:]
for cert in chain:
s += "-----BEGIN CERTIFICATE-----\n"
s += base64.encodestring(cert).rstrip() + "\n"
s += "-----END CERTIFICATE-----\n"
s += "\n"
zf.writestr("%08d" % i, s)
except AssertionError, e:
print "error for cert", i, e
if zf:
zf.close()
for blocknumber in range(ncerts / 65536, (tree_size / 65536) + 1):
(resulttype, result) = get_merkle_hash_64k(args.store, blocknumber)
if resulttype == "incomplete":
(incompletelength, hash) = result
ncerts = blocknumber * 65536 + incompletelength
merkle_64klayer.append(hash)
break
assert resulttype == "hash"
hash = result
merkle_64klayer.append(hash)
print blocknumber * 65536, base64.b16encode(hash)
tree = build_merkle_tree(merkle_64klayer)
calculated_root_hash = tree[-1][0]
assert ncerts == tree_size
print "calculated root hash", base64.b16encode(calculated_root_hash)
if calculated_root_hash != root_hash:
print "fetched root hash and calculated root hash different"
sys.exit(1)
if args.store and args.write_sth:
f = open(args.store + "/currentsth", "w")
f.write(json.dumps(sth))
f.close()
|
