Implement function to fetch consistency proof

Implement function to calculate tree head from disk
Implement function to calculate an intermediate node from disk

1 files changed, 99 insertions, 9 deletions

diff --git a/tools/certtools.py b/tools/certtools.py
index 6a144c9..31045b9 100644
--- a/tools/certtools.py
+++ b/tools/certtools.py
@@ -105,6 +105,17 @@ def get_proof_by_hash(baseurl, hash, tree_size):
         print "ERROR:", e.read()
         sys.exit(1)
 
+def get_consistency_proof(baseurl, tree_size1, tree_size2):
+    try:
+        params = urllib.urlencode({"first":tree_size1,
+                                   "second":tree_size2})
+        result = \
+          urllib2.urlopen(baseurl + "ct/v1/get-sth-consistency?" + params).read()
+        return json.loads(result)["consistency"]
+    except urllib2.HTTPError, e:
+        print "ERROR:", e.read()
+        sys.exit(1)
+
 def tls_array(data, length_len):
     length_bytes = struct.pack(">Q", len(data))[-length_len:]
     return length_bytes + data
@@ -340,16 +351,24 @@ def get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf):
         except KeyError:
             return
 
-def get_merkle_hash_64k(store, blocknumber, write_to_cache=False):
-    hashfilename = "%s/%04x.64khash" % (store, blocknumber)
-    try:
-        hash = base64.b16decode(open(hashfilename).read())
-        assert len(hash) == 32
-        return ("hash", hash)
-    except IOError:
-        pass
+def get_merkle_hash_64k(store, blocknumber, write_to_cache=False, treesize=None):
     firstleaf = blocknumber * 65536
     lastleaf = firstleaf + 65535
+    if treesize != None:
+        assert firstleaf < treesize
+        usecache = lastleaf < treesize
+        lastleaf = min(lastleaf, treesize - 1)
+    else:
+        usecache = True
+
+    hashfilename = "%s/%04x.64khash" % (store, blocknumber)
+    if usecache:
+        try:
+            hash = base64.b16decode(open(hashfilename).read())
+            assert len(hash) == 32
+            return ("hash", hash)
+        except IOError:
+            pass
     firstfile = firstleaf / 10000
     lastfile = lastleaf / 10000
     zipfiles = {}
@@ -364,10 +383,81 @@ def get_merkle_hash_64k(store, blocknumber, write_to_cache=False):
     calculated_hash = tree[-1][0]
     for zf in zipfiles.values():
         zf.close()
-    if len(layer0) != 65536:
+    if len(layer0) != lastleaf - firstleaf + 1:
         return ("incomplete", (len(layer0), calculated_hash))
     if write_to_cache:
         f = open(hashfilename, "w")
         f.write(base64.b16encode(calculated_hash))
         f.close()
     return ("hash", calculated_hash)
+
+def get_tree_head(store, treesize):
+    merkle_64klayer = []
+
+    for blocknumber in range(0, (treesize / 65536) + 1):
+        (resulttype, result) = get_merkle_hash_64k(store, blocknumber, treesize=treesize)
+        if resulttype == "incomplete":
+            print >>sys.stderr, "Couldn't read until tree size", treesize
+            (incompletelength, hash) = result
+            print >>sys.stderr, "Stopped at", blocknumber * 65536 + incompletelength
+            sys.exit(1)
+        assert resulttype == "hash"
+        hash = result
+        merkle_64klayer.append(hash)
+        #print >>sys.stderr, print blocknumber * 65536,
+        sys.stdout.flush()
+    tree = build_merkle_tree(merkle_64klayer)
+    calculated_root_hash = tree[-1][0]
+    return calculated_root_hash
+
+def get_intermediate_hash(store, treesize, level, index):
+    if level >= 16:
+        merkle_64klayer = []
+
+        levelsize = (2**(level-16))
+
+        for blocknumber in range(index * levelsize, (index + 1) * levelsize):
+            if blocknumber * (2 ** 16) >= treesize:
+                break
+            #print "looking at block", blocknumber
+            (resulttype, result) = get_merkle_hash_64k(store, blocknumber, treesize=treesize)
+            if resulttype == "incomplete":
+                print >>sys.stderr, "Couldn't read until tree size", treesize
+                (incompletelength, hash) = result
+                print >>sys.stderr, "Stopped at", blocknumber * 65536 + incompletelength
+                sys.exit(1)
+            assert resulttype == "hash"
+            hash = result
+            #print "block hash", base64.b16encode(hash)
+            merkle_64klayer.append(hash)
+            #print >>sys.stderr, print blocknumber * 65536,
+            sys.stdout.flush()
+        tree = build_merkle_tree(merkle_64klayer)
+        return tree[-1][0]
+    else:
+        levelsize = 2 ** level
+        firstleaf = index * levelsize
+        lastleaf = firstleaf + levelsize - 1
+        #print "firstleaf", firstleaf
+        #print "lastleaf", lastleaf
+        assert firstleaf < treesize
+        lastleaf = min(lastleaf, treesize - 1)
+        #print "modified lastleaf", lastleaf
+        firstfile = firstleaf / 10000
+        lastfile = lastleaf / 10000
+        #print "files", firstfile, lastfile
+        zipfiles = {}
+        for i in range(firstfile, lastfile + 1):
+            try:
+                zipfiles[i] = zipfile.ZipFile("%s/%04d.zip" % (store, i))
+            except IOError:
+                break
+        certs = get_certs_from_zipfiles(zipfiles, firstleaf, lastleaf)
+        layer0 = [get_hash_from_certfile(cert) for cert in certs]
+        #print "layer0", repr(layer0)
+        tree = build_merkle_tree(layer0)
+        calculated_hash = tree[-1][0]
+        for zf in zipfiles.values():
+            zf.close()
+        assert len(layer0) == lastleaf - firstleaf + 1
+        return calculated_hash