adding experimental support for leveldb

1 files changed, 53 insertions, 41 deletions

diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py
index 4fb99ee..2812c37 100755
--- a/monitor/josef_monitor.py
+++ b/monitor/josef_monitor.py
@@ -8,23 +8,19 @@ import argparse
 import errno
 from copy import deepcopy
 from josef_lib import *
-from logs import ctlogs
+from josef_leveldb import db_add_certs, db_open
 import os.path
 
-# NAGIOS_OK = 0
-# NAGIOS_WARN = 1
-# NAGIOS_CRIT = 2
-# NAGIOS_UNKNOWN = 3
+# Import from config file
+if os.path.isfile("monitor_conf.py"):
+    from monitor_conf import ctlogs, OUTPUT_DIR, INTERVAL, DEFAULT_CERT_FILE, DB_PATH
+else:
+    print "Config file not found!"
+    sys.exit()
 
-INTERVAL = 30 # interval (in seconds) between updates
-
-OUTPUT_DIR = "output/"
-DEFAULT_CERT_FILE = OUTPUT_DIR + "cert_data.json"
+DB = None
 
 parser = argparse.ArgumentParser(description="")
-# TODO implement silent mode
-# parser.add_argument('--silent', action='store_true', help="Dont output to stdout. logging only")
-
 
 class ctlog:
     def __init__(self, name, url, key):
@@ -43,10 +39,14 @@ class ctlog:
     def incremental_build(self):
         # Keeps state current during build, partial builds are possible.
         self.sth = get_sth(self.url)
-        self.log("Building....")
+        # self.log("Building....")
         start_size = self.entries
         while self.entries < self.sth["tree_size"]:
+            tmp_size = self.entries
             self.subtree, self.entries = fetch_and_increment_subtree(self.entries, self.sth["tree_size"] -1, self.url, self.subtree)
+            if tmp_size != self.entries:
+                self.log("Got entries " + str(tmp_size) + " to " \
+                + str(self.entries -1 ) + " of " + str(self.sth["tree_size"]-1))
 
         if self.entries != start_size:
             if verify_subtree(self.sth, self.subtree, self.url):
@@ -54,8 +54,8 @@ class ctlog:
                     " new entries. Size: " + str(self.entries))
             else:
                 self.log("ERROR Failed to build tree from entries.")
-        else:
-            self.log("No new entries.")
+        # else:
+        #     self.log("No new entries.")
 
 
 
@@ -85,7 +85,8 @@ class ctlog:
             raise e
 
     def log(self, string):
-        s = time.strftime('%H:%M:%S') + " " + string
+        # TODO change to UTC?
+        s = time.strftime('%Y-%m-%d, %H:%M:%S') + " " + string
         with open(self.logfile, 'a') as f:
             f.write(s + "\n")
             f.close()
@@ -140,28 +141,35 @@ class ctlog:
             print s
 
 
-def verify_consistency(old, new):
-    for url in old:
+    def verify_consistency(self, old):
+        new = self.sth
+        # for url in old:
         try:
-            if old[url] and new[url] and old[url]["tree_size"]!= new[url]["tree_size"]:
-                consistency_proof = get_consistency_proof(url, old[url]["tree_size"], new[url]["tree_size"])
+            if old["tree_size"]!= new["tree_size"]:
+                consistency_proof = get_consistency_proof(self.url, old["tree_size"], new["tree_size"])
                 decoded_consistency_proof = []
                 for item in consistency_proof:
                     decoded_consistency_proof.append(base64.b64decode(item))
-                res = verify_consistency_proof(decoded_consistency_proof, old[url]["tree_size"], new[url]["tree_size"], old[url]["sha256_root_hash"])
+                res = verify_consistency_proof(decoded_consistency_proof, old["tree_size"], new["tree_size"], old["sha256_root_hash"])
                 
-                if old[url]["sha256_root_hash"] != str(base64.b64encode(res[0])):
-                    print time.strftime('%H:%M:%S') + " Verification of old hash failed! " + old[url]["sha256_root_hash"], str(base64.b64encode(res[0]))
-                    errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to verify consistency for " + url + ", tree size " + old[url]["tree_size"])
-                elif new[url]["sha256_root_hash"] != str(base64.b64encode(res[1])):
-                    print time.strftime('%H:%M:%S') + " Verification of new hash failed! " + new[url]["sha256_root_hash"], str(base64.b64encode(res[1]))
-                    errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to verify consistency for " + url + ", tree size " + new[url]["tree_size"])
-                else:
-                    print time.strftime("%H:%M:%S") + " New STH from " + url + ", timestamp: " + \
-                        str(new[url]["timestamp"]) + ", size: " + str(new[url]["tree_size"]) + "...OK."
+                if old["sha256_root_hash"] != str(base64.b64encode(res[0])):
+                    s = " Verification of old hash failed! " + \
+                        old["sha256_root_hash"], str(base64.b64encode(res[0]))
+                    self.log(s)
+                    print s
+                elif new["sha256_root_hash"] != str(base64.b64encode(res[1])):
+                    s = " Verification of new hash failed! " + \
+                        new["sha256_root_hash"], str(base64.b64encode(res[1]))
+                    self.log(s)
+                    print s
+                # else:
+                #     s = "New STH, timestamp: " + str(new["timestamp"]) + \
+                #         ", size: " + str(new["tree_size"]) + "...OK."
+                #     self.log(s)
 
         except:
-          print "ERROR: Could not verify consistency for " + url
+            self.log("ERROR: Could not verify consistency!")
+            print "ERROR: Could not verify consistency for " + self.url
 
 def verify_inclusion_all(old, new):
     for url in old:
@@ -197,6 +205,7 @@ def check_domain(raw_entry, log=None):
     return cert_info
 
 def fetch_and_increment_subtree(first, last, url, subtree =[[]]):
+        global DB
     # try:
         new_leafs = []
         if first <= last:
@@ -205,10 +214,11 @@ def fetch_and_increment_subtree(first, last, url, subtree =[[]]):
             for item in entries:
                 tmp_cert_data.append(check_domain(item, url))
                 new_leafs.append(get_leaf_hash(base64.b64decode(item["leaf_input"])))
-            append_file(DEFAULT_CERT_FILE, tmp_cert_data)
-            print time.strftime('%H:%M:%S') + " Got entries " + str(first) + " to " \
-                + str(first + len(new_leafs) -1 ) + " of " + str(last) +" entries from " + url
-            
+            if DEFAULT_CERT_FILE:
+                if DB is None:
+                    append_file(DEFAULT_CERT_FILE, tmp_cert_data)
+                else:
+                    db_add_certs(DB, tmp_cert_data)
             subtree = reduce_tree(new_leafs, subtree)
     # except:
     #     print "Failed to build subtree :("
@@ -220,7 +230,7 @@ def verify_subtree(sth, subtree, base_url):
         root = base64.b64encode(reduce_subtree_to_root(tmp)[0])
 
         if root == sth["sha256_root_hash"]:
-            print time.strftime('%H:%M:%S') + " Verifying root hashes for " + base_url + "...OK."
+            # print time.strftime('%H:%M:%S') + " Verifying root hashes for " + base_url + "...OK."
             return True
         else:
             print time.strftime('%H:%M:%S') + " ERROR: Failed to verify root hashes! STH root: " \
@@ -297,14 +307,16 @@ def get_all_roots(base_url):
 
 
 def main(args):
-    # TODO cleanup files
-
+    global DB
     # Create logs
     logs = []
+    if DB_PATH:
+        DB = db_open(DB_PATH)
     try:
         for item in ctlogs:
             logs.append(ctlog(item, ctlogs[item][0], ctlogs[item][1]))
 
+        print time.strftime('%H:%M:%S') + " Setting up monitor for " + str(len(logs)) + " logs..."
         # Set up state 
         for log in logs:
             if os.path.isfile(log.savefile):
@@ -314,6 +326,7 @@ def main(args):
             log.incremental_build()
 
         # Main loop: Monitor
+        print time.strftime('%H:%M:%S') + " Running... (see logfiles for output)"
         while True:
             time.sleep(INTERVAL)
             for log in logs:
@@ -321,9 +334,8 @@ def main(args):
                 log.update_sth() # Should this be done is later checks fail? (reorder?)
                 if old_sth["timestamp"] != log.sth["timestamp"]:
                     log.verify_progress(old_sth)
+                    log.verify_consistency(old_sth)
                     log.incremental_build()
-                    # TODO check consistency proof
-                    pass
 
         # Unreachable... usually.
         for log in logs:
@@ -331,7 +343,7 @@ def main(args):
 
 
     except KeyboardInterrupt:
-        print 'Received interrupt from user. Saving and exiting....'
+        print time.strftime('%H:%M:%S') + ' Received interrupt from user. Saving and exiting....'
         for log in logs:
             log.save()