implementing root monitoring

6 files changed, 206 insertions, 84 deletions

diff --git a/monitor/josef_experimental.py b/monitor/josef_experimental.py
index 3e34584..97ea876 100755
--- a/monitor/josef_experimental.py
+++ b/monitor/josef_experimental.py
@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-     
 
 import sys
+import os
 from josef_lib import *
 import leveldb
 import argparse
@@ -9,51 +10,103 @@ import json
 from josef_leveldb import *
 from datetime import datetime as dt
 from josef_monitor import verify_inclusion_by_hash
-
-
-def verify_sct(baseurl, sctentry, key, sth_in=None):
-    if sth_in is None:
-        if baseurl:
-            sth = get_sth(baseurl)
-        else:
-            print "No sth provided!"
-    else:
-        sth = sth_in
-
-    # Verify signature
-    leafcert = base64.b64decode(sctentry["leafcert"])
-    if "issuer_key_hash" in sctentry:
-        issuer_key_hash = base64.b64decode(sctentry["issuer_key_hash"])
-    else:
-        issuer_key_hash = None
-    try:
-        if issuer_key_hash:
-            signed_entry = pack_precert(leafcert, issuer_key_hash)
-        else:
-            signed_entry = pack_cert(leafcert)
-        check_sct_signature(baseurl, signed_entry, sctentry["sct"], precert=issuer_key_hash, publickey=key)
-        print "Signature OK"
-    except AssertionError, e:
-        print "ERROR:", e
-    except urllib2.HTTPError, e:
-        print "ERROR:", e
-    except ecdsa.keys.BadSignatureError, e:
-        print "ERROR: bad signature"
+from monitor_conf import *
+
+
+# def verify_sct(baseurl, sctentry, key, sth_in=None):
+#     if sth_in is None:
+#         if baseurl:
+#             sth = get_sth(baseurl)
+#         else:
+#             print "No sth provided!"
+#     else:
+#         sth = sth_in
+
+#     # Verify signature
+#     leafcert = base64.b64decode(sctentry["leafcert"])
+#     if "issuer_key_hash" in sctentry:
+#         issuer_key_hash = base64.b64decode(sctentry["issuer_key_hash"])
+#     else:
+#         issuer_key_hash = None
+#     try:
+#         if issuer_key_hash:
+#             signed_entry = pack_precert(leafcert, issuer_key_hash)
+#         else:
+#             signed_entry = pack_cert(leafcert)
+#         check_sct_signature(baseurl, signed_entry, sctentry["sct"], precert=issuer_key_hash, publickey=key)
+#         print "Signature OK"
+#     except AssertionError, e:
+#         print "ERROR:", e
+#     except urllib2.HTTPError, e:
+#         print "ERROR:", e
+#     except ecdsa.keys.BadSignatureError, e:
+#         print "ERROR: bad signature"
     
-    # Verify inclusion
-    h = get_leaf_hash(base64.b64decode(sctentry["leafcert"]))
+#     # Verify inclusion
+#     h = get_leaf_hash(base64.b64decode(sctentry["leafcert"]))
     
-    verify_inclusion_by_hash("https://localhost:8080/", h)
+#     verify_inclusion_by_hash("https://localhost:8080/", h)
 
 
 
+def update_roots(log):
+    roots_hash = None
 
+    roots = get_all_roots(log["url"])
+    new_roots_hash = str(hash(str(roots)))
 
-if __name__ == '__main__':
+    if new_roots_hash != roots_hash:
+        cert_dir = OUTPUT_DIR + log["name"] + "-roots"
+        if not os.path.exists(cert_dir):
+            os.makedirs(cert_dir)
+
+        hash_list = []
+        for cert in roots:
+            h = str(hash(str(cert)))
+            hash_list.append(h)
+
+        loaded_list = os.listdir(cert_dir)
+
+        added, removed = compare_lists(hash_list[:-1], loaded_list)
 
+        # TODO log changes
+        if len(added) != 0:
+            print str(len(added)) + " new roots found!"
+        if len(removed) != 0:
+            print str(len(removed)) + " roots removed!"
 
+        for item in removed:
+            data = open(cert_dir + "/" + item).read()
 
+            root_cert = base64.decodestring(data)
+            subject = get_cert_info(root_cert)["subject"]
+            issuer = get_cert_info(root_cert)["issuer"]
+            if subject == issuer:
+                print "Removed Root: " + item + ", " + subject
+            else: 
+                print "WTF? Not a root..."
+
+
+        for item in added:
+            root_cert = base64.decodestring(roots[hash_list.index(item)])
+            subject = get_cert_info(root_cert)["subject"]
+            issuer = get_cert_info(root_cert)["issuer"]
+            if subject == issuer:
+                print "New Root: " + item + ", " + subject
+            else: 
+                print "WTF? Not a root..."
+
+            fn = cert_dir + "/" + item
+            tempname = fn + ".new"
+            data = roots[hash_list.index(item)]
+            open(tempname, 'w').write(data)
+            mv_file(tempname, fn)
+
+
+if __name__ == '__main__':
 
+    for log in ctlogs:
+        update_roots(log)
 
 
 
diff --git a/monitor/josef_lib.py b/monitor/josef_lib.py
index b2643c1..2d24bce 100644
--- a/monitor/josef_lib.py
+++ b/monitor/josef_lib.py
@@ -22,6 +22,31 @@ from Crypto.Hash import SHA256
 import Crypto.PublicKey.RSA as RSA
 from Crypto.Signature import PKCS1_v1_5
 
+def compare_lists(new, old):
+    added_items  = []
+    removed_items  = []
+
+    for item in new:
+        if not item in old:
+            added_items.append(item)
+
+    for item in old:
+        if not item in new:
+            removed_items.append(item)
+    return added_items, removed_items
+
+def get_all_roots(base_url):
+    result = urlopen(base_url + "ct/v1/get-roots").read()
+    certs = json.loads(result)["certificates"]
+    # print time.strftime('%H:%M:%S') + " Received " + str(len(certs)) + " certs from " + base_url
+
+    for accepted_cert in certs:
+        subject = get_cert_info(base64.decodestring(accepted_cert))["subject"]
+        issuer = get_cert_info(base64.decodestring(accepted_cert))["issuer"]
+        if subject == issuer:
+            root_cert = base64.decodestring(accepted_cert)
+    return certs
+
 def check_domain_all(raw_entry, log=None):
     orig_entry = extract_original_entry(raw_entry)
     try:
@@ -277,9 +302,9 @@ def add_chain(baseurl, submission):
         return json.loads(result)
     except urllib2.HTTPError, e:
         print "ERROR", e.code,":", e.read()
-        if e.code == 400:
-            return None
-        sys.exit(1)
+        # if e.code == 400:
+        return None
+        # sys.exit(1)
     except ValueError, e:
         print "==== FAILED REQUEST ===="
         print submission
@@ -295,9 +320,9 @@ def add_prechain(baseurl, submission):
         return json.loads(result)
     except urllib2.HTTPError, e:
         print "ERROR", e.code,":", e.read()
-        if e.code == 400:
-            return None
-        sys.exit(1)
+        # if e.code == 400:
+        return None
+        # sys.exit(1)
     except ValueError, e:
         print "==== FAILED REQUEST ===="
         print submission
diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py
index dfdeacf..0e02a3c 100755
--- a/monitor/josef_monitor.py
+++ b/monitor/josef_monitor.py
@@ -44,6 +44,7 @@ class ctlog:
         self.subtree = [[]]
         self.sth = None
         self.entries = 0
+        self.root_hash = None
 
         self.log("Starting monitor")
 
@@ -133,6 +134,59 @@ class ctlog:
             self.sth = new_sth
 
 
+    def update_roots(self):
+        roots = get_all_roots(self.url)
+        new_root_hash = str(hash(str(roots)))
+        
+        if new_root_hash != self.root_hash:
+            self.root_hash = new_root_hash
+            cert_dir = OUTPUT_DIR + self.name + "-roots"
+            if not os.path.exists(cert_dir):
+                os.makedirs(cert_dir)
+
+            hash_list = []
+            for cert in roots:
+                h = str(hash(str(cert)))
+                hash_list.append(h)
+
+            loaded_list = os.listdir(cert_dir)
+
+            added, removed = compare_lists(hash_list, loaded_list)
+
+            if len(added) != 0:
+                print str(len(added)) + " new roots found for " + self.name
+            if len(removed) != 0:
+                print str(len(removed)) + " roots removed for " + self.name
+            
+            for item in removed:
+                data = open(cert_dir + "/" + item).read()
+
+                root_cert = base64.decodestring(data)
+                subject = get_cert_info(root_cert)["subject"]
+                issuer = get_cert_info(root_cert)["issuer"]
+                if subject == issuer:
+                    print "Removed Root: " + item + ", " + subject
+                    self.log("Removed Root: " + item + ", " + subject)
+                else: 
+                    print "WTF? Not a root..."
+
+            for item in added:
+                root_cert = base64.decodestring(roots[hash_list.index(item)])
+                subject = get_cert_info(root_cert)["subject"]
+                issuer = get_cert_info(root_cert)["issuer"]
+                if subject == issuer:
+                    print "New Root: " + item + ", " + subject
+                    self.log("New Root: " + item + ", " + subject)
+                else: 
+                    print "WTF? Not a root..."
+
+                fn = cert_dir + "/" + item
+                tempname = fn + ".new"
+                data = roots[hash_list.index(item)]
+                open(tempname, 'w').write(data)
+                mv_file(tempname, fn)
+
+
     def verify_progress(self, old):
         new = self.sth
         try:
@@ -306,13 +360,8 @@ def get_all_roots(base_url):
     result = urlopen(base_url + "ct/v1/get-roots").read()
     certs = json.loads(result)["certificates"]
     print time.strftime('%H:%M:%S') + " Received " + str(len(certs)) + " certs from " + base_url
+    return certs
 
-    for accepted_cert in certs:
-        subject = get_cert_info(base64.decodestring(accepted_cert))["subject"]
-        issuer = get_cert_info(base64.decodestring(accepted_cert))["issuer"]
-        if subject == issuer:
-            root_cert = base64.decodestring(accepted_cert)
-            print get_cert_info(root_cert)["subject"]
 
 def setup_domain_monitoring():
     monitored_domains = []
diff --git a/monitor/josef_mover.py b/monitor/josef_mover.py
index 511a156..5fb22da 100755
--- a/monitor/josef_mover.py
+++ b/monitor/josef_mover.py
@@ -36,8 +36,8 @@ def print_reply(rep, entry, precert):
 source = ctlogs[0]
 dests = [ctlogs[1]]
 
-first = 21001
-last = 22000
+first = 153357
+last = 154357
 
 entries = get_entries(source["url"], first, last)["entries"]
 
@@ -45,23 +45,28 @@ entries = get_entries(source["url"], first, last)["entries"]
 
 for log in dests:
     for item in entries:
-        entry = extract_original_entry(item)
-        if entry[2]:
-            precert = True
-        else:
-            precert = False
-        submission = []
-
-        for e in entry[0]:
-            submission.append(base64.b64encode(e))
-
-        if precert:
-            res = add_prechain(log["url"], {"chain" : submission})
-        else:
-            res = add_chain(log["url"], {"chain" : submission})
-
-        print_reply(res, entry, precert)
-        time.sleep(5)
+        try:
+            entry = extract_original_entry(item)
+            if entry[2]:
+                precert = True
+            else:
+                precert = False
+            submission = []
+
+            for e in entry[0]:
+                submission.append(base64.b64encode(e))
+
+            if precert:
+                res = add_prechain(log["url"], {"chain" : submission})
+            else:
+                res = add_chain(log["url"], {"chain" : submission})
+
+            print_reply(res, entry, precert)
+            time.sleep(5)
+        except KeyboardInterrupt:
+            break
+        except:
+            print "FAILED!"
 
 
 
diff --git a/monitor/josef_reader.py b/monitor/josef_reader.py
index f17545d..b6fe55e 100755
--- a/monitor/josef_reader.py
+++ b/monitor/josef_reader.py
@@ -56,16 +56,7 @@ class monitored_domain:
         self.entries = entries
 
     def compare_and_log_entries(self, new, old):
-        added_items  = []
-        removed_items  = []
-
-        for item in new:
-            if not item in old:
-                added_items.append(item)
-
-        for item in old:
-            if not item in new:
-                removed_items.append(item)
+        added_items, removed_items = compare_lists(new,old)
 
         if len(added_items) != 0:
             self.log(str(len(added_items)) + " new item(s):")
@@ -109,7 +100,6 @@ class monitored_entry:
         return d
 
 
-
 def db_monitor_domain(domain, log=None, exclude_invalid=None, get_cert=None, issuer=None):
     # print domain
     raw = db_lookup_domain(DB_PATH, domain)
diff --git a/monitor/monitor_conf.py b/monitor/monitor_conf.py
index f838360..913c466 100644
--- a/monitor/monitor_conf.py
+++ b/monitor/monitor_conf.py
@@ -29,10 +29,10 @@ MONITORED_DOMAINS = [
 
 # CT logs and associated keys
 ctlogs = [
-    # {"name" : "pilot",
-    # "url" : "https://ct.googleapis.com/pilot/",
-    # "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTDM0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA==",
-    # "id" : "pLkJkLQYWBSHuxOizGdwCjw1mAT5G9+443fNDsgN3BA="},
+    {"name" : "pilot",
+    "url" : "https://ct.googleapis.com/pilot/",
+    "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTDM0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA==",
+    "id" : "pLkJkLQYWBSHuxOizGdwCjw1mAT5G9+443fNDsgN3BA="},
 
     {"name" : "plausible",
     "url" : "https://plausible.ct.nordu.net/",
@@ -59,10 +59,10 @@ ctlogs = [
     # "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1/TMabLkDpCjiupacAlP7xNi0I1JYP8bQFAHDG1xhtolSY1l4QgNRzRrvSe8liE+NPWHdjGxfx3JhTsN9x8/6Q==",
     # "id" : "aPaY+B9kgr46jO65KB1M/HFRXWeT1ETRCmesu09P+8Q="},
 
-    {"name" : "rocketeer",
-    "url" : "https://ct.googleapis.com/rocketeer/",
-   	"key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIFsYyDzBi7MxCAC/oJBXK7dHjG+1aLCOkHjpoHPqTyghLpzA9BYbqvnV16mAw04vUjyYASVGJCUoI3ctBcJAeg==",
-    "id": "7ku9t3XOYLrhQmkfq+GeZqMPfl+wctiDAMR7iXqo/cs="},
+    # {"name" : "rocketeer",
+    # "url" : "https://ct.googleapis.com/rocketeer/",
+   	# "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIFsYyDzBi7MxCAC/oJBXK7dHjG+1aLCOkHjpoHPqTyghLpzA9BYbqvnV16mAw04vUjyYASVGJCUoI3ctBcJAeg==",
+    # "id": "7ku9t3XOYLrhQmkfq+GeZqMPfl+wctiDAMR7iXqo/cs="},
 
     {"name" : "symantec",
     "url" : "https://ct.ws.symantec.com/",