bugfixes. Building on updates

2 files changed, 58 insertions, 107 deletions

diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py
index f3a38bf..4fb99ee 100755
--- a/monitor/josef_monitor.py
+++ b/monitor/josef_monitor.py
@@ -26,16 +26,6 @@ parser = argparse.ArgumentParser(description="")
 # parser.add_argument('--silent', action='store_true', help="Dont output to stdout. logging only")
 
 
-# monitored_domains = [
-#     # "google.com",
-#     # "preishelden.de",
-#     # "liu.se",
-#     # "nordu.net",
-#     # "symantec.com",
-#     # "sunet.se",
-#     # ".se",
-# ]
-
 class ctlog:
     def __init__(self, name, url, key):
         self.name = name
@@ -50,40 +40,32 @@ class ctlog:
         self.log("Starting monitor")
 
 
-    def build(self):
-        self.sth = get_sth(self.url)
-        tmp_subtree = fetch_and_build_subtree(self.sth, self.url)
-        if verify_subtree(self.sth, self.subtree, self.url):
-            self.log("Successfully build tree from entries.")
-            self.subtree = tmp_subtree
-            self.entries = self.sth["tree_size"]
-        else:
-            self.log("ERROR Failed to build tree from entries.")
-
     def incremental_build(self):
         # Keeps state current during build, partial builds are possible.
         self.sth = get_sth(self.url)
+        self.log("Building....")
+        start_size = self.entries
         while self.entries < self.sth["tree_size"]:
             self.subtree, self.entries = fetch_and_increment_subtree(self.entries, self.sth["tree_size"] -1, self.url, self.subtree)
 
-        if verify_subtree(self.sth, self.subtree, self.url):
-            self.log("Successfully build tree from entries.")
-            # self.subtree = tmp_subtree
-            # self.entries = self.sth["tree_size"]
+        if self.entries != start_size:
+            if verify_subtree(self.sth, self.subtree, self.url):
+                self.log("Successfully build tree with " + str(self.entries - start_size) + \
+                    " new entries. Size: " + str(self.entries))
+            else:
+                self.log("ERROR Failed to build tree from entries.")
         else:
-            self.log("ERROR Failed to build tree from entries.")
+            self.log("No new entries.")
+
 
 
     def to_dict(self):
         d = {}
-        # d["name"] = self.name
-        # d["url"] = self.url
         d["entries"] = self.entries
         d["subtree"] = encode_tree(self.subtree)
         d["sth"] = self.sth
         return d
 
-
     def save(self):
         self.log("Saving state to file")
         open(self.savefile, 'w').write(json.dumps(self.to_dict()))
@@ -110,62 +92,53 @@ class ctlog:
 
     def update_sth(self):
         new_sth = get_sth(self.url)
+
+        try:
+            check_sth_signature(self.url, new_sth, None)
+        except:
+            self.log("ERROR: Could not verify STH signature")
+            print "ERROR: Could not verify STH signature from " + url
+
         sth_time = datetime.datetime.fromtimestamp(new_sth['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S")
         if new_sth["timestamp"] != self.sth["timestamp"]:
             self.log("STH updated. Size: " + str(new_sth["tree_size"]) + ", Time: " + sth_time)
             self.sth = new_sth
 
 
-# def fetch_all_sth():
-#     sths = {}
-#     for base_url in base_urls:
-#         # Fetch STH
-#         try:
-#             sths[base_url] =  get_sth(base_url)
-#         except:
-#             sths[base_url] = None
-#             error_str = time.strftime('%H:%M:%S') + " ERROR: Failed to retrieve STH from " + base_url
-#             print error_str
-#             errors.append(error_str)
-#             continue
-
-#         # Check signature on the STH
-#         try:
-#             # check_sth_signature(base_url, sths[base_url], logkeys[base_url])
-#             check_sth_signature(base_url, sths[base_url], None)
-#         except:
-#             error_str = time.strftime('%H:%M:%S') + " ERROR: Could not verify signature from " + base_url
-#             print error_str
-#             errors.append(error_str)
-#             continue
-
-#     return sths
-
-def verify_progress(old, new):
-    print "Verifying progress"
-    try:
-        for url in new:
-            if new and old and new[url] and old[url]:
-                if new[url]["tree_size"] == old[url]["tree_size"]:
-                    if old[url]["sha256_root_hash"] != new[url]["sha256_root_hash"]:
-                        errors.append(time.strftime('%H:%M:%S') + " CRITICAL: root hash is different for same tree size in " + url)
-                elif new[url]["tree_size"] < old[url]["tree_size"]:
-                        errors.append(time.strftime('%H:%M:%S') + " CRITICAL: new tree smaller than previous tree (%d < %d)" % \
-                          (new[url]["tree_size"], old[url]["tree_size"]))
-            if new[url]:
-                age = time.time() - new[url]["timestamp"]/1000
-                sth_time = datetime.datetime.fromtimestamp(new[url]['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S")
-                roothash = new[url]['sha256_root_hash']
+    def verify_progress(self, old):
+        new = self.sth
+        try:
+            if new["tree_size"] == old["tree_size"]:
+                if old["sha256_root_hash"] != new["sha256_root_hash"]:
+                    s = time.strftime('%H:%M:%S') + " CRITICAL: root hash is different for same tree size"
+                    self.log(s)
+                    print s
+            elif new["tree_size"] < old["tree_size"]:
+                s = time.strftime('%H:%M:%S') + " CRITICAL: new tree smaller than previous tree (%d < %d)" % \
+                  (new["tree_size"], old["tree_size"])
+                self.log(s)
+                print s
+            else:
+                age = time.time() - new["timestamp"]/1000
+                sth_time = datetime.datetime.fromtimestamp(new['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S")
+                roothash = new['sha256_root_hash']
                 if age > 24 * 3600:
-                    errors.append(time.strftime('%H:%M:%S') + " CRITICAL: %s is older than 24h: %s UTC" % (url, sth_time))
+                    s = "CRITICAL: STH is older than 24h: %s UTC" % (sth_time)
+                    self.log(s)
+                    print s
                 elif age > 12 * 3600:
-                    errors.append(time.strftime('%H:%M:%S') + " WARNING: %s is older than 12h: %s UTC" % (url, sth_time))
+                    s = "WARNING: STH is older than 12h: %s UTC" % (sth_time)
+                    self.log(s)
+                    # print s
                 elif age > 6 * 3600:
-                    errors.append(time.strftime('%H:%M:%S') + " WARNING: %s is older than 6h: %s UTC" % (url, sth_time))
-                # elif age > 2 * 3600:
-                #     errors.append(time.strftime('%H:%M:%S') + " WARNING: %s is older than 2h: %s UTC" % (url, sth_time))
-    except:
-        print time.strftime('%H:%M:%S') + " ERROR: Failed to verify progress for " + url
+                    s = "WARNING: STH is older than 6h: %s UTC" % (sth_time)
+                    self.log(s)
+                    # print s
+        except:
+            s = " ERROR: Failed to verify progress"
+            self.log(s)
+            print s
+
 
 def verify_consistency(old, new):
     for url in old:
@@ -234,35 +207,13 @@ def fetch_and_increment_subtree(first, last, url, subtree =[[]]):
                 new_leafs.append(get_leaf_hash(base64.b64decode(item["leaf_input"])))
             append_file(DEFAULT_CERT_FILE, tmp_cert_data)
             print time.strftime('%H:%M:%S') + " Got entries " + str(first) + " to " \
-                + str(first + len(new_leafs)) + " (" + str(len(new_leafs)) +" entries) from " + url
+                + str(first + len(new_leafs) -1 ) + " of " + str(last) +" entries from " + url
             
             subtree = reduce_tree(new_leafs, subtree)
     # except:
     #     print "Failed to build subtree :("
         return subtree, len(new_leafs) + first
 
-def fetch_and_build_subtree(sth, base_url):
-    try:
-        subtree = [[]]
-        idx = 0
-
-        while idx < sth["tree_size"]:
-            pre_size = idx
-            entries = get_entries(base_url, idx, sth["tree_size"]-1)["entries"]
-            new_leafs = []
-            tmp_cert_data = []
-            for item in entries:
-                tmp_cert_data.append(check_domain(item, base_url))
-                new_leafs.append(get_leaf_hash(base64.b64decode(item["leaf_input"])))
-            idx += len(new_leafs)
-            append_file(DEFAULT_CERT_FILE, tmp_cert_data)
-            print time.strftime('%H:%M:%S') + " Got entries " + str(pre_size) + " to " + str(idx -1) + " from " + base_url
-            subtree = reduce_tree(new_leafs, subtree)
-    except:
-        print "Failed to build subtree :("
-    return subtree
-
-
 def verify_subtree(sth, subtree, base_url):
     try:
         tmp = deepcopy(subtree)
@@ -358,6 +309,8 @@ def main(args):
         for log in logs:
             if os.path.isfile(log.savefile):
                 log.load()
+        # Build what was not loaded
+        for log in logs:
             log.incremental_build()
 
         # Main loop: Monitor
@@ -365,13 +318,11 @@ def main(args):
             time.sleep(INTERVAL)
             for log in logs:
                 old_sth = log.sth
-                log.update_sth()
+                log.update_sth() # Should this be done is later checks fail? (reorder?)
                 if old_sth["timestamp"] != log.sth["timestamp"]:
-                    # TODO verify signature
-                    # TODO fetch updates
-                    # TODO verify progress
-                    # TODO verify tree
-                    # TODO verify consistency proof?
+                    log.verify_progress(old_sth)
+                    log.incremental_build()
+                    # TODO check consistency proof
                     pass
 
         # Unreachable... usually.
diff --git a/monitor/logs.py b/monitor/logs.py
index 12c58a5..2925017 100644
--- a/monitor/logs.py
+++ b/monitor/logs.py
@@ -8,9 +8,9 @@ ctlogs = {
     # ["https://plausible.ct.nordu.net/",
     # "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9UV9+jO2MCTzkabodO2F7LM03MUBc8MrdAtkcW6v6GA9taTTw9QJqofm0BbdAsbtJL/unyEf0zIkRgXjjzaYqQ=="],
 
-    # "digicert":
-    # ["https://ct1.digicert-ct.com/log/",
-    # "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEAkbFvhu7gkAW6MHSrBlpE1n4+HCFRkC5OLAjgqhkTH+/uzSfSl8ois8ZxAD2NgaTZe1M9akhYlrYkes4JECs6A=="],
+    "digicert":
+    ["https://ct1.digicert-ct.com/log/",
+    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEAkbFvhu7gkAW6MHSrBlpE1n4+HCFRkC5OLAjgqhkTH+/uzSfSl8ois8ZxAD2NgaTZe1M9akhYlrYkes4JECs6A=="],
 
     "izenpe":
     ["https://ct.izenpe.com/",