extracting monitor from auditor

3 files changed, 48 insertions, 41 deletions

diff --git a/tools/certkeys.py b/tools/certkeys.py
index dd0570f..f7c83b1 100644
--- a/tools/certkeys.py
+++ b/tools/certkeys.py
@@ -4,10 +4,6 @@ publickeys = {
     "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTD"
     "M0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA==",
 
-    "https://flimsy.ct.nordu.net/":
-    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4qWq6afhBUi0OdcWUYhyJLNXTkGqQ9"
-    "PMS5lqoCgkV2h1ZvpNjBH2u8UbgcOQwqDo66z6BWQJGolozZYmNHE2kQ==",
-
     "https://plausible.ct.nordu.net/":
     "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9UV9+jO2MCTzkabodO2F7LM03MUB"
 	"c8MrdAtkcW6v6GA9taTTw9QJqofm0BbdAsbtJL/unyEf0zIkRgXjjzaYqQ==",
diff --git a/tools/josef_experimental.py b/tools/josef_experimental.py
index 383e385..7f27945 100755
--- a/tools/josef_experimental.py
+++ b/tools/josef_experimental.py
@@ -136,26 +136,30 @@ monitored_domains = [
     "symantec.com",
 ]
 
-# data = read_sth("plausible_cert_data.json")
-# ss = []
-# for item in data:
-#     try:
-#         s = item["subject"].split("CN=")[1]
-#         print s
-#     except:
-#         # if not item["subject"] in ss:
-#         #     print item["subject"]
-#         #     ss.append(item["subject"])
-#         pass
-
-# print "\nTotal entries: " + str(len(data))
-
-base_url = base_urls[0]
-
-entries = get_entries(base_url, 11, 11)["entries"]
-for item in entries:
-    orig_entry = extract_original_entry(item)
-    cert_info = my_get_cert_info(orig_entry[0][0])
+data = []
+# data = read_sth("cert_data.json")
+f = open("cert_data.json")
+for line in f:
+    data.append(json.loads(line))
+ss = []
+for item in data:
+    try:
+        s = item["subject"].split("CN=")[1]
+        print s
+    except:
+        # if not item["subject"] in ss:
+        #     print item["subject"]
+        #     ss.append(item["subject"])
+        pass
+
+print "\nTotal entries: " + str(len(data))
+
+# base_url = base_urls[0]
+
+# entries = get_entries(base_url, 11, 11)["entries"]
+# for item in entries:
+#     orig_entry = extract_original_entry(item)
+#     cert_info = my_get_cert_info(orig_entry[0][0])
     # prev = ""
     # res = {}
     # for line in cert_info:
@@ -172,4 +176,4 @@ for item in entries:
     #         res["not_after"] = line.split(": ")[1]
 
     #     prev = line
-    print cert_info
+    # print cert_info
diff --git a/tools/josef_experimental_auditor.py b/tools/josef_experimental_auditor.py
index 78a3fe2..7efd2dc 100755
--- a/tools/josef_experimental_auditor.py
+++ b/tools/josef_experimental_auditor.py
@@ -14,16 +14,17 @@ NAGIOS_CRIT = 2
 NAGIOS_UNKNOWN = 3
 
 DEFAULT_CUR_FILE = 'all-sth.json'
+DEFAULT_CERT_FILE = "cert_data.json"
 
 base_urls = [
     # "https://plausible.ct.nordu.net/",
     # "https://ct1.digicert-ct.com/log/", 
-    # "https://ct.izenpe.com/",
+    "https://ct.izenpe.com/",
     # "https://log.certly.io/", 
     # "https://ct.googleapis.com/aviator/", 
     # "https://ct.googleapis.com/pilot/",
     # "https://ct.googleapis.com/rocketeer/",
-    # "https://ctlog.api.venafi.com/",
+    "https://ctlog.api.venafi.com/",
     "https://ct.ws.symantec.com/",
 ]
 
@@ -55,14 +56,6 @@ monitored_domains = [
     # ".se",
 ]
 
-# cert_data = []
-
-# class cert(subject, issuer, log):
-#     def __init__(self):
-#         self.subject = subject
-#         self.issuer = issuer
-        # self.logs = [log]
-
 class UTC(datetime.tzinfo):
     def utcoffset(self, dt):
       return datetime.timedelta(hours=0)
@@ -215,9 +208,12 @@ def check_domain(raw_entry, log=None):
 
 def fetch_and_increment_subtree(old_sth, new_sth_in, subtree, base_url):
     try:
+        print "Initial hash:", hash(str(subtree))
         sth = old_sth[base_url]
         new_sth = new_sth_in[base_url]
         idx = sth["tree_size"]
+        tmp_tree = list(subtree)
+        print "tmp hash:", hash(str(tmp_tree))
 
         while idx < new_sth["tree_size"]:
             pre_size = idx
@@ -227,16 +223,22 @@ def fetch_and_increment_subtree(old_sth, new_sth_in, subtree, base_url):
             for item in entries:
                 tmp_cert_data.append(check_domain(item, base_url))
                 new_leafs.append(get_leaf_hash(base64.b64decode(item["leaf_input"])))
-            append_file("cert_data.json", tmp_cert_data)
+            append_file(DEFAULT_CERT_FILE, tmp_cert_data)
             idx += len(new_leafs)
             print time.strftime('%H:%M:%S') + " Got entries " + str(pre_size) + " to " \
                 + str(idx -1) + " (" + str(len(new_leafs)) +" entries) from " + base_url
-            subtree = reduce_tree(new_leafs, subtree)
+            
+            print "Before reduction:", hash(str(tmp_tree))
+            res_tree = reduce_tree(new_leafs, tmp_tree)
+            print "After reduction:", hash(str(res_tree))
 
     except:
         print "Failed to build subtree :("
 
-    return subtree 
+    if subtree == res_tree:
+        print "Final subtree hash", hash(str(subtree))
+        print "Final restree hash", hash(str(res_tree))
+    return res_tree
 
 def fetch_and_build_subtree(old_sth, base_url):
     try:
@@ -253,8 +255,8 @@ def fetch_and_build_subtree(old_sth, base_url):
                 tmp_cert_data.append(check_domain(item, base_url))
                 new_leafs.append(get_leaf_hash(base64.b64decode(item["leaf_input"])))
             idx += len(new_leafs)
-            append_file("cert_data.json", tmp_cert_data)
-            print time.strftime('%H:%M:%S') + " Got entries " + str(pre_size) + " to " + str(idx) + " from " + base_url
+            append_file(DEFAULT_CERT_FILE, tmp_cert_data)
+            print time.strftime('%H:%M:%S') + " Got entries " + str(pre_size) + " to " + str(idx -1) + " from " + base_url
             subtree = reduce_tree(new_leafs, subtree)
 
     except:
@@ -376,7 +378,7 @@ def append_file(fn, content):
     with open(fn, 'a') as f:
         for item in content:
             try: 
-                f.write(json.dumps(item))
+                f.write(json.dumps(item) + "\n")
             except:
                 # print "failed to write " + str(item)
                 pass
@@ -454,8 +456,13 @@ def main(args):
             new_sth = fetch_all_sth()
             for url in base_urls:
                 if url in sth and url in new_sth and sth[url]["tree_size"] != new_sth[url]["tree_size"]:
+                    # print "Len before:", len(all_subtrees[url])
                     all_subtrees[url] = fetch_and_increment_subtree(sth, new_sth, all_subtrees[url], url)
+                    # print "Len between:", len(all_subtrees[url])
                     verify_subtree(new_sth, all_subtrees[url], url)
+                    # print "Len after:", len(all_subtrees[url])
+                    print "Old sth:", sth[url]["sha256_root_hash"]
+                    print "New sth:", new_sth[url]["sha256_root_hash"]
             sth = new_sth