implementing continous incremental build from entries

1 files changed, 55 insertions, 15 deletions

diff --git a/tools/josef_experimental_auditor.py b/tools/josef_experimental_auditor.py
index 9268d03..a27da17 100755
--- a/tools/josef_experimental_auditor.py
+++ b/tools/josef_experimental_auditor.py
@@ -186,17 +186,40 @@ def verify_inclusion_all(old, new):
             print time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url
             errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url)
 
-def fetch_and_build_tree(old_sth, base_url):
+def fetch_and_increment_subtree(old_sth, new_sth_in, subtree, base_url):
+    try:
+        sth = old_sth[base_url]
+        new_sth = new_sth_in[base_url]
+        idx = sth["tree_size"]
+
+        # print time.strftime('%H:%M:%S') + " Getting all entries from " + base_url
+        while idx < new_sth["tree_size"]:
+            pre_size = idx
+            entries = get_entries(base_url, idx, new_sth["tree_size"]-1)["entries"]
+            
+            new_leafs = []
+            for item in entries:
+                new_leafs.append(get_leaf_hash(base64.b64decode(item["leaf_input"])))
+            idx += len(new_leafs)
+            print time.strftime('%H:%M:%S') + " Got entries " + str(pre_size) + " to " + str(idx) + " from " + base_url
+            subtree = reduce_tree(new_leafs, subtree)
+
+    except:
+        print "Failed to build subtree :("
+
+    return subtree 
+
+def fetch_and_build_subtree(old_sth, base_url):
     try:
         sth = old_sth[base_url]
         subtree = [[]]
         idx = 0
 
-        res_strings = [""]
-
-        print time.strftime('%H:%M:%S') + " Getting all entries from " + base_url
+        # print time.strftime('%H:%M:%S') + " Getting all entries from " + base_url
         while idx < sth["tree_size"]:
             pre_size = idx
+            ### DEBUG!!
+            # entries = get_entries(base_url, idx, sth["tree_size"]-100)["entries"]
             entries = get_entries(base_url, idx, sth["tree_size"]-1)["entries"]
             
             new_leafs = []
@@ -206,19 +229,22 @@ def fetch_and_build_tree(old_sth, base_url):
             print time.strftime('%H:%M:%S') + " Got entries " + str(pre_size) + " to " + str(idx) + " from " + base_url
             subtree = reduce_tree(new_leafs, subtree)
 
+    except:
+        print "Failed to build subtree :("
+
+    return subtree
+
+
+def verify_subtree(old_sth, subtree, base_url):
+    try:
+        sth = old_sth[base_url]
         root = base64.b64encode(reduce_subtree_to_root(subtree)[0])
 
         if root == sth["sha256_root_hash"]:
             print time.strftime('%H:%M:%S') + " Verifying root hashes for " + base_url + "...OK."
-            res_strings.append("STH for " + base_url + " built successfully.")
         else:
             print time.strftime('%H:%M:%S') + " ERROR: Failed to verify root hashes! STH root: " + sth["sha256_root_hash"] + ", Tree root: " + root
-            res_strings.append(time.strftime('%H:%M:%S') + " " + base_url + " Failed! STH root: " + sth["sha256_root_hash"] + " Calculated root: " + root)
             errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to verify root hash for " + base_url + ", tre size " + sth["tree_size"])
-
-        for item in res_strings:
-            print item + "\n"
-
     except:
         print time.strftime('%H:%M:%S') + " ERROR: Failed to build STH for " + base_url
         errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to build STH for " + base_url)
@@ -300,7 +326,6 @@ def print_timings(timings):
         print item + " last seen " + datetime.datetime.fromtimestamp(int(timings[item]["last"])/1000).strftime('%Y-%m-%d %H:%M:%S') \
             + " longest between two STH: " + str(int(h)) + "h " + str(int(m)) + "m "# + str(int(s)) + "s."
 
-
 def read_sth(fn):
     try:
         f = open(fn)
@@ -310,18 +335,16 @@ def read_sth(fn):
         raise e
     return json.loads(f.read())
 
-
 def write_file(fn, sth):
     tempname = fn + ".new"
     open(tempname, 'w').write(json.dumps(sth))
     mv_file(tempname, fn)
 
-
 def main(args):
     
     # print time.strftime("%H:%M:%S") + " Starting..."
     if args.verify_index is None and not args.build_sth and not args.audit and not args.audit2 \
-        and not args.audit3 and not args.audit4 and not args.roots:
+        and not args.audit3 and not args.audit4 and not args.roots and not args.monitor:
         
         print time.strftime('%H:%M:%S') + " Nothing to do."
         return
@@ -343,7 +366,8 @@ def main(args):
     if args.build_sth:
         print time.strftime('%H:%M:%S') + " Building trees from entries. This may take a while, go get coffee or something..."
         for base_url in base_urls:
-            fetch_and_build_tree(sth, base_url)
+            subtree = fetch_and_build_subtree(sth, base_url)
+            verify_subtree(sth, subtree, base_url)
         # fetch_and_build_tree(sth, base_urls[2])
 
     if args.audit:
@@ -401,6 +425,22 @@ def main(args):
             sys.exit(NAGIOS_CRIT)
         sys.exit(NAGIOS_OK)
 
+    if args.monitor:
+        # Run for one log only
+        url = base_urls[0]
+        print time.strftime('%H:%M:%S') + " Building trees from entries. This may take a while, go get coffee or something..."
+        # sth[url]["tree_size"] -= 100
+        subtree = fetch_and_build_subtree(sth, url)
+        verify_subtree(sth, subtree, url)
+        
+        while True:
+            time.sleep(30)
+            new_sth = fetch_all_sth()
+            if sth[url]["tree_size"] != new_sth[url]["tree_size"]:
+                subtree = fetch_and_increment_subtree(sth, new_sth, subtree, url)
+                verify_subtree(sth, subtree, url)
+                sth = new_sth
+
 
 if __name__ == '__main__':
     # try: