refactoring and moving things around

5 files changed, 142 insertions, 128 deletions

diff --git a/monitor/josef_leveldb.py b/monitor/josef_leveldb.py
index 769f6f3..fa948ed 100755
--- a/monitor/josef_leveldb.py
+++ b/monitor/josef_leveldb.py
@@ -12,6 +12,9 @@ SEP = ";"
 dbs = {}
 
 
+
+
+
 def match_domain(d1, d2):
     # Exact match
     if d1 == d2:
@@ -34,6 +37,7 @@ def match_domain(d1, d2):
 
 
 def db_open(fn='./cert_db'):
+    print "Opening " + fn
     global dbs
     if fn in dbs:
         return dbs[fn]
@@ -43,6 +47,11 @@ def db_open(fn='./cert_db'):
         dbs[fn] = db
         return db
 
+def db_close():
+    print "Closing databases."
+    for db in dbs:
+        del db        
+
 def db_append(db, key, val):
     if db is None:
         print "ERROR: NO DATABASE SET!"
@@ -112,6 +121,7 @@ def db_add_certs(db_dir, data):
             pass
         except IndexError:
             pass
+    db_close()
 
 
 def db_lookup_domain(db_dir, domain):
diff --git a/monitor/josef_lib.py b/monitor/josef_lib.py
index 45ca80f..61b315e 100644
--- a/monitor/josef_lib.py
+++ b/monitor/josef_lib.py
@@ -31,6 +31,37 @@ from Crypto.Signature import PKCS1_v1_5
 #         raise e
 #     return json.loads(f.read())
 
+
+def check_domain_all(raw_entry, log=None):
+    orig_entry = extract_original_entry(raw_entry)
+    try:
+        cert_info = my_get_all_cert_info(orig_entry[0][0])
+        if log:
+            cert_info["log"] = log[8:-1] # strip generic URL stuff
+        return cert_info
+    except IndexError:
+        return None
+
+
+def get_full_cert(entry):
+    try:
+        log = "https://" + entry["log"] + "/"
+        leaf_hash = entry["leaf_hash"]
+    except:
+        print "Could not get stats from entry."
+        return
+    # Get tree size in sth
+    sth = get_sth(log)
+    # Get index (rest of proof discarded)
+    proof = get_proof_by_hash(log, base64.b64decode(leaf_hash), sth["tree_size"])
+    leaf_index = proof["leaf_index"]
+    # Get full entry
+    raw_entry = get_entries(log, leaf_index, leaf_index)["entries"][0]
+    cert = check_domain_all(raw_entry)
+    return cert
+
+
+
 def encode_tree(tree):
     res = []
     for layer in tree:
@@ -107,10 +138,7 @@ def my_get_all_cert_info(s):
     if parsed[1]:
         print "ERROR:", parsed[1]
         sys.exit(1)
-    result = []
-    for line in parsed[0].split("\n"):
-        result.append(line)
-    return result
+    return parsed[0]
 
 
 def get_pemlike(filename, marker):
diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py
index 23ba147..d11d38f 100755
--- a/monitor/josef_monitor.py
+++ b/monitor/josef_monitor.py
@@ -70,29 +70,23 @@ class ctlog:
 
 
     def fetch_and_increment_subtree(self, first, last, url, subtree =[[]]):
-        # global DB
-        # try:
-            new_leafs = []
-            if first <= last:
-                entries = get_entries(url, first, last)["entries"]
-                tmp_cert_data = []
-                for item in entries:
-                    tmp_data = check_domain(item, url)
-                    entry_hash = get_leaf_hash(base64.b64decode(item["leaf_input"]))
-                    if tmp_data:
-                        tmp_data["leaf_hash"] = base64.b64encode(entry_hash)
-                        tmp_cert_data.append(tmp_data)
-                    new_leafs.append(entry_hash)
-                if DB_PATH:
-                    self.log("Adding to database...")
-                    db_add_certs(DB_PATH, tmp_cert_data)
-                    self.log("done adding to DB.")
-                if DEFAULT_CERT_FILE:
-                    append_file(DEFAULT_CERT_FILE, tmp_cert_data)
-                subtree = reduce_tree(new_leafs, subtree)
-        # except:
-        #     print "Failed to build subtree :("
-            return subtree, len(new_leafs) + first
+        new_leafs = []
+        if first <= last:
+            entries = get_entries(url, first, last)["entries"]
+            tmp_cert_data = []
+            for item in entries:
+                tmp_data = check_domain(item, url)
+                entry_hash = get_leaf_hash(base64.b64decode(item["leaf_input"]))
+                if tmp_data:
+                    tmp_data["leaf_hash"] = base64.b64encode(entry_hash)
+                    tmp_cert_data.append(tmp_data)
+                new_leafs.append(entry_hash)
+            if DB_PATH:
+                db_add_certs(DB_PATH, tmp_cert_data)
+            if DEFAULT_CERT_FILE:
+                append_file(DEFAULT_CERT_FILE, tmp_cert_data)
+            subtree = reduce_tree(new_leafs, subtree)
+        return subtree, len(new_leafs) + first
 
 
     def to_dict(self):
@@ -243,6 +237,8 @@ def check_domain(raw_entry, log=None):
     except IndexError:
         return None
 
+
+
 def verify_subtree(sth, subtree, base_url):
     try:
         tmp = deepcopy(subtree)
diff --git a/monitor/josef_reader.py b/monitor/josef_reader.py
index a100b0a..92bd510 100755
--- a/monitor/josef_reader.py
+++ b/monitor/josef_reader.py
@@ -31,111 +31,91 @@ monitored_domains = [
     "symantec.com",
 ]
 
-
-def check_domain(raw_entry, log=None):
-    orig_entry = extract_original_entry(raw_entry)
-    try:
-        cert_info = my_get_all_cert_info(orig_entry[0][0])
-        if log:
-            cert_info["log"] = log[8:-1] # strip generic URL stuff
-        return cert_info
-    except IndexError:
-        return None
-
-
-def get_full_cert(entry):
-    try:
-        log = "https://" + entry["log"] + "/"
-        leaf_hash = entry["leaf_hash"]
-    except:
-        print "Could not get stats from entry."
-        return
-    # print log, leaf_hash
-    tree_size = 5000000
-    proof = get_proof_by_hash(log, base64.b64decode(leaf_hash), tree_size)
-    leaf_index = proof["leaf_index"]
-    raw_entry = get_entries(log, leaf_index, leaf_index)["entries"][0]
-    cert = check_domain(raw_entry)
-    for line in cert:
-        print line
-
-
-# db = "./tmpdb/"
 db = DB_PATH
 
-if args.domain:
-    raw = db_lookup_domain(db, args.domain)
-else:
-    print "No domain selected!"
-    sys.exit()
 
-cur_time = dt.now()
-count_valid = 0
-count_expired = 0
-count_not_yet_valid = 0
-count_all = 0
-for item in raw:
-    try:
-        entry = ast.literal_eval(item)
-    except:
-        print (item + '}').replace("'", '"')
-    success = True
-    not_after_time = dt.strptime(entry["not_after"], "%b %d %H:%M:%S %Y GMT")
-    not_before_time = dt.strptime(entry["not_before"], "%b %d %H:%M:%S %Y GMT")
-
-
-    if args.log:
-        if args.log in entry["log"]:
-            pass
-        else:
-            success = False
 
-    if cur_time > not_after_time:
-        valid = False
-        expired = True
-    elif cur_time < not_before_time:
-        valid = False
-        expired = False
-    else:
-        expired = False
-        valid = True
-
-    # Exclude expired
-    if args.exclude_invalid and not valid:
-        success = False
-        
-    
-    # Set count matches
-    if success:
-        count_all += 1
-        if valid:
-            count_valid += 1
-        elif expired:
-            count_expired += 1
-        else:
-            count_not_yet_valid += 1
-
-    # Print matching
-    if success:
-        s = entry["subject"].split("CN=")[1] + \
-        " certified by " + entry["issuer"].split("CN=")[1] + \
-        " (" + entry["log"] + ") "
-        if valid:
-            print "(VALID) " + s
-        else:
-            print "(NOT VALID) " + s
+def db_monitor_domain(domain, log=None, exclude_invalid=None, get_cert=None):
+    print domain
+    raw = db_lookup_domain(db, domain)
 
-        if args.get_cert:
-            get_full_cert(entry)
+    cur_time = dt.now()
+    count_valid = 0
+    count_expired = 0
+    count_not_yet_valid = 0
+    count_all = 0
+    for item in raw:
+        try:
+            entry = ast.literal_eval(item)
+        except:
+            print (item + '}').replace("'", '"')
+        success = True
+        not_after_time = dt.strptime(entry["not_after"], "%b %d %H:%M:%S %Y GMT")
+        not_before_time = dt.strptime(entry["not_before"], "%b %d %H:%M:%S %Y GMT")
 
 
-print str(count_all) + " matches found. " \
-+ str(count_valid) + " valid, " \
-+ str(count_expired) + " expired and " \
-+ str(count_not_yet_valid) + " not yet valid."
+        if log:
+            if log in entry["log"]:
+                pass
+            else:
+                success = False
+
+        if cur_time > not_after_time:
+            valid = False
+            expired = True
+        elif cur_time < not_before_time:
+            valid = False
+            expired = False
+        else:
+            expired = False
+            valid = True
 
+        # Exclude expired
+        if exclude_invalid and not valid:
+            success = False
+            
+        
+        # Set count matches
+        if success:
+            count_all += 1
+            if valid:
+                count_valid += 1
+            elif expired:
+                count_expired += 1
+            else:
+                count_not_yet_valid += 1
+
+        # Print matching
+        if success:
+            s = entry["subject"].split("CN=")[1] + \
+            " certified by " + entry["issuer"].split("CN=")[1] + \
+            " (" + entry["log"] + ") "
+            if valid:
+                print "(VALID) " + s
+            else:
+                print "(NOT VALID) " + s
+
+            if get_cert:
+                print get_full_cert(entry)
+
+
+    print str(count_all) + " matches found. " \
+    + str(count_valid) + " valid, " \
+    + str(count_expired) + " expired and " \
+    + str(count_not_yet_valid) + " not yet valid."
 
 
+if args.domain:
+    # if args.log:
+    #     log = args.log
+    # else:
+    #     log = None
+    # d = args.domain
+    db_monitor_domain(args.domain)
+    # db_monitor_domain(args.domain, args.log, args.exclude_invalid, args.get_cert)
+else:
+    print "No domain selected!"
+    sys.exit()
 
 
 
diff --git a/monitor/monitor_conf.py b/monitor/monitor_conf.py
index 1f51a10..4a472a4 100644
--- a/monitor/monitor_conf.py
+++ b/monitor/monitor_conf.py
@@ -20,13 +20,13 @@ ctlogs = {
     # ["https://ct.googleapis.com/pilot/",
     # "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTDM0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA=="],
 
-    "plausible":
-    ["https://plausible.ct.nordu.net/",
-    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9UV9+jO2MCTzkabodO2F7LM03MUBc8MrdAtkcW6v6GA9taTTw9QJqofm0BbdAsbtJL/unyEf0zIkRgXjjzaYqQ=="],
+    # "plausible":
+    # ["https://plausible.ct.nordu.net/",
+    # "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9UV9+jO2MCTzkabodO2F7LM03MUBc8MrdAtkcW6v6GA9taTTw9QJqofm0BbdAsbtJL/unyEf0zIkRgXjjzaYqQ=="],
 
-    "digicert":
-    ["https://ct1.digicert-ct.com/log/",
-    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEAkbFvhu7gkAW6MHSrBlpE1n4+HCFRkC5OLAjgqhkTH+/uzSfSl8ois8ZxAD2NgaTZe1M9akhYlrYkes4JECs6A=="],
+    # "digicert":
+    # ["https://ct1.digicert-ct.com/log/",
+    # "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEAkbFvhu7gkAW6MHSrBlpE1n4+HCFRkC5OLAjgqhkTH+/uzSfSl8ois8ZxAD2NgaTZe1M9akhYlrYkes4JECs6A=="],
 
     "izenpe":
     ["https://ct.izenpe.com/",