monitor experiments

2 files changed, 49 insertions, 54 deletions

diff --git a/tools/josef_experimental.py b/tools/josef_experimental.py
index e44dcd4..6d95894 100755
--- a/tools/josef_experimental.py
+++ b/tools/josef_experimental.py
@@ -81,6 +81,15 @@ def my_get_cert_info(s):
             result[key] = value
     return result
 
+def read_sth(fn):
+    try:
+        f = open(fn)
+    except IOError, e:
+        if e.errno == errno.ENOENT:
+            return None
+        raise e
+    return json.loads(f.read())
+
 base_urls = [
             "https://plausible.ct.nordu.net/",
             # "https://ct1.digicert-ct.com/log/", 
@@ -115,21 +124,19 @@ monitored_domains = [
     "symantec.com",
 ]
 
-raw_entry = get_entries(base_urls[0], 1000, 1005)["entries"]
-orig_entries = []
-
-for item in raw_entry:
-    # print  item
-    orig_entry = extract_original_entry(item)
-    cert_info = my_get_cert_info(orig_entry[0][0])
-
-    print cert_info
-    # for md in monitored_domains:
-    #     if md in cert_info["subject"]:
-    #         print md + " certifed by " + cert_info["issuer"]    
-
-    # print "\n\n"
-    # print item
+data = read_sth("plausible_cert_data.json")
+ss = []
+for item in data:
+    try:
+        s = item["subject"].split("CN=")[1]
+        print s
+    except:
+        # if not item["subject"] in ss:
+        #     print item["subject"]
+        #     ss.append(item["subject"])
+        pass
+
+print "\nTotal entries: " + str(len(data))
 
 
 
diff --git a/tools/josef_experimental_auditor.py b/tools/josef_experimental_auditor.py
index e8e7b52..05f7a9a 100755
--- a/tools/josef_experimental_auditor.py
+++ b/tools/josef_experimental_auditor.py
@@ -18,7 +18,7 @@ DEFAULT_CUR_FILE = 'all-sth.json'
 base_urls = [
     # "https://plausible.ct.nordu.net/",
     # "https://ct1.digicert-ct.com/log/", 
-    # "https://ct.izenpe.com/",
+    "https://ct.izenpe.com/",
     # "https://log.certly.io/", 
     # "https://ct.googleapis.com/aviator/", 
     # "https://ct.googleapis.com/pilot/",
@@ -55,6 +55,14 @@ monitored_domains = [
     # ".se",
 ]
 
+cert_data = []
+
+# class cert(subject, issuer, log):
+#     def __init__(self):
+#         self.subject = subject
+#         self.issuer = issuer
+        # self.logs = [log]
+
 class UTC(datetime.tzinfo):
     def utcoffset(self, dt):
       return datetime.timedelta(hours=0)
@@ -200,9 +208,14 @@ def verify_inclusion_all(old, new):
 def check_domain(raw_entry):
     orig_entry = extract_original_entry(raw_entry)
     cert_info = get_cert_info(orig_entry[0][0])
-    for md in monitored_domains:
-        if md in cert_info["subject"]:
-            print md + " (" + cert_info["subject"].split("CN=")[1] + ") certifed by " + cert_info["issuer"]
+    # for md in monitored_domains:
+        # if md in cert_info["subject"]:
+            # print md + " (" + cert_info["subject"].split("CN=")[1] + ") certifed by " + cert_info["issuer"]
+    try:
+        # print cert_info["subject"] + " certifed by " + cert_info["issuer"]
+        cert_data.append(cert_info)
+    except:
+        pass
 
 
 def fetch_and_increment_subtree(old_sth, new_sth_in, subtree, base_url):
@@ -211,15 +224,14 @@ def fetch_and_increment_subtree(old_sth, new_sth_in, subtree, base_url):
         new_sth = new_sth_in[base_url]
         idx = sth["tree_size"]
 
-        # print time.strftime('%H:%M:%S') + " Getting all entries from " + base_url
         while idx < new_sth["tree_size"]:
             pre_size = idx
             entries = get_entries(base_url, idx, new_sth["tree_size"]-1)["entries"]
-            
             new_leafs = []
             for item in entries:
                 check_domain(item)
                 new_leafs.append(get_leaf_hash(base64.b64decode(item["leaf_input"])))
+            # write_file("cert_data.json", cert_data)
             idx += len(new_leafs)
             print time.strftime('%H:%M:%S') + " Got entries " + str(pre_size) + " to " \
                 + str(idx -1) + " (" + str(len(new_leafs)) +" entries) from " + base_url
@@ -244,6 +256,7 @@ def fetch_and_build_subtree(old_sth, base_url):
                 check_domain(item)
                 new_leafs.append(get_leaf_hash(base64.b64decode(item["leaf_input"])))
             idx += len(new_leafs)
+            # write_file("cert_data.json", cert_data)
             print time.strftime('%H:%M:%S') + " Got entries " + str(pre_size) + " to " + str(idx) + " from " + base_url
             subtree = reduce_tree(new_leafs, subtree)
 
@@ -263,11 +276,11 @@ def verify_subtree(old_sth, subtree, base_url):
         else:
             print time.strftime('%H:%M:%S') + " ERROR: Failed to verify root hashes! STH root: " \
                 + sth["sha256_root_hash"] + ", Tree root: " + root
-            errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to verify root hash for " 
-                + base_url + ", tre size " + sth["tree_size"])
+            # errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to verify root hash for " 
+            #     + base_url + ", tre size " + sth["tree_size"])
     except:
         print time.strftime('%H:%M:%S') + " ERROR: Failed to build STH for " + base_url
-        errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to build STH for " + base_url)
+        # errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to build STH for " + base_url)
 
 def verify_inclusion_by_hash(base_url, leaf_hash):
     try: 
@@ -355,10 +368,11 @@ def read_sth(fn):
         raise e
     return json.loads(f.read())
 
-def write_file(fn, sth):
+def write_file(fn, content):
     tempname = fn + ".new"
-    open(tempname, 'w').write(json.dumps(sth))
+    open(tempname, 'w').write(json.dumps(content))
     mv_file(tempname, fn)
+    print "wrote " + fn
 
 def main(args):
     
@@ -421,41 +435,15 @@ def main(args):
             verify_inclusion_all(old_sth, sth)
         write_file(args.cur_sth, sth)
     
-    # Experimental for plausible + nagios
-    if args.audit4:
-        base_url = base_urls[0]
-        old_sth = read_sth("plausible-sth.json")
-        print "Running auditor4 for " + base_url
-        try:
-            tmp_sth =  get_sth(base_url)
-        except:
-            error_str = time.strftime('%H:%M:%S') + " ERROR: Failed to retrieve STH from " + base_url
-            print error_str
-            errors.append(error_str)
-            sys.exit(NAGIOS_WARN)
-
-        # Check signature on the STH
-        try:
-            check_sth_signature(base_url, tmp_sth, None)
-            write_file("plausible-sth.json", tmp_sth)
-        except:
-            error_str = time.strftime('%H:%M:%S') + " ERROR: Could not verify signature from " + base_url
-            print error_str
-            errors.append(error_str)
-            sys.exit(NAGIOS_CRIT)
-        sys.exit(NAGIOS_OK)
-
     if args.monitor:
         # Run for one log only
         all_subtrees = {}
         print time.strftime('%H:%M:%S') + " Building trees from entries. This may take a while, go get coffee or something..."
         for url in base_urls:
-            sth[url]["tree_size"] -= 10
-            sth[url]["sha256_root_hash"] = "na"
             all_subtrees[url] = fetch_and_build_subtree(sth, url)
-            # verify_subtree(sth, all_subtrees[url], url)
+            verify_subtree(sth, all_subtrees[url], url)
+
 
-        
         while True:
             time.sleep(30)
             new_sth = fetch_all_sth()