saving entry hashes as well

4 files changed, 35 insertions, 21 deletions

diff --git a/monitor/josef_experimental.py b/monitor/josef_experimental.py
index c9605bd..a306d11 100755
--- a/monitor/josef_experimental.py
+++ b/monitor/josef_experimental.py
@@ -34,9 +34,9 @@ monitored_domains = [
 # db_add_domain_2(db, "www.cox.b.com", "{dummydata}")
 # print db_lookup_domain(db, "www.cox.b.com")
 # print db.Get("www.cox.b.com")
-db_add_domain_2("./tmpdatabase/", "www.cox.a.com", "{dummydata3}")
-db_add_domain_2("./tmpdatabase/", "www.ct.nordu.net", "{dummydata1}")
-print db_lookup_domain("./tmpdatabase/", "www.cox.a.com")
+db_add_domain_2("./tmpdb/", "www.cox.a.com", "{'leaf_hash':'123456'}")
+# db_add_domain_2("./tmpdatabase/", "www.ct.nordu.net", "{dummydata1}")
+print db_lookup_domain("./tmpdb/", "www.cox.a.com")
 # print db.Get("com")
 # print db.Get("a.com")
 # print db.Get("cox.a.com")
diff --git a/monitor/josef_leveldb.py b/monitor/josef_leveldb.py
index 7ad43f3..769f6f3 100755
--- a/monitor/josef_leveldb.py
+++ b/monitor/josef_leveldb.py
@@ -4,6 +4,7 @@
 import sys
 from josef_lib import *
 import leveldb
+import ast
 
 SEP = ";"
 # db = None
@@ -37,7 +38,7 @@ def db_open(fn='./cert_db'):
     if fn in dbs:
         return dbs[fn]
     else:
-        print "Opening " + fn
+        # print "Opening " + fn
         db = leveldb.LevelDB(fn)
         dbs[fn] = db
         return db
@@ -60,7 +61,9 @@ def db_append(db, key, val):
 
 def db_add_domain_2(db_dir, key_in, val_in, db_in=None):
     if db_in == None:
-        val = str(hash(val_in))
+        # print val_in
+        val = "###" + ast.literal_eval(val_in)["leaf_hash"]
+        # val = str(hash(val_in))
         cert_db = db_open(db_dir + "certificates")
         cert_db.Put(val, val_in)
 
@@ -116,15 +119,20 @@ def db_lookup_domain(db_dir, domain):
     res = []
 
     if domain_list[-1] == "com":
-        key  = '.'.join(domain_list[:-1])
         db = db_open(db_dir + domain_list[-1])
+        domain_list = domain_list[:-1]
+        key  = '.'.join(domain_list)
     else:
         key  = '.'.join(domain)
         db = db_open(db_dir + "other")
 
-    domain_list = domain_list[:-1]
     cur_domain = domain_list.pop()
-    intermediate = db.Get(cur_domain).split(SEP)
+    # print cur_domain
+    try:
+        intermediate = db.Get(cur_domain).split(SEP)
+    except KeyError:
+        return []
+
     last = False
     one_more = False
     while True:
@@ -140,13 +148,16 @@ def db_lookup_domain(db_dir, domain):
         # Prune
         next_level = []
         for item in intermediate:
+            # print item
             if match_domain(cur_domain, item):
+                # print item
                 try:
                     tmp = db.Get(item).split(SEP)
+                    # print tmp
                     for tmp_item in tmp:
-                        try:
-                            res.append(int(tmp_item))
-                        except:
+                        if tmp_item[0:3] == "###":
+                            res.append(tmp_item[3:])
+                        else:
                             next_level.append(tmp_item)
                 except KeyError:
                     pass
@@ -154,8 +165,9 @@ def db_lookup_domain(db_dir, domain):
 
     certs = []
     cert_db = db_open(db_dir + "certificates")
+    # print res
     for item in res:
-        tmp = cert_db.Get(str(item))
+        tmp = cert_db.Get("###" + str(item))
         certs.append(tmp)
 
     return certs
diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py
index 4b8b721..23ba147 100755
--- a/monitor/josef_monitor.py
+++ b/monitor/josef_monitor.py
@@ -78,9 +78,11 @@ class ctlog:
                 tmp_cert_data = []
                 for item in entries:
                     tmp_data = check_domain(item, url)
+                    entry_hash = get_leaf_hash(base64.b64decode(item["leaf_input"]))
                     if tmp_data:
+                        tmp_data["leaf_hash"] = base64.b64encode(entry_hash)
                         tmp_cert_data.append(tmp_data)
-                    new_leafs.append(get_leaf_hash(base64.b64decode(item["leaf_input"])))
+                    new_leafs.append(entry_hash)
                 if DB_PATH:
                     self.log("Adding to database...")
                     db_add_certs(DB_PATH, tmp_cert_data)
@@ -360,11 +362,10 @@ def main(args):
         for log in logs:
             log.save()
 
-    except:
-    # Unreachable... usually.
-        print "FATAL ERROR! Saving state and terminating!"
-        for log in logs:
-            log.save()
+    # except Exception, err:
+    #     print Exception, err
+    #     for log in logs:
+    #         log.save()
 
 
 
diff --git a/monitor/josef_reader.py b/monitor/josef_reader.py
index 736fb3d..c5755f1 100755
--- a/monitor/josef_reader.py
+++ b/monitor/josef_reader.py
@@ -8,6 +8,7 @@ import argparse
 import json
 from josef_leveldb import *
 from datetime import datetime as dt
+import ast
 
 
 
@@ -26,10 +27,10 @@ monitored_domains = [
     "symantec.com",
 ]
 
-
+db = "./tmpdb/"
 
 if args.domain:
-    db = db_open()
+    # db = db_open()
     raw = db_lookup_domain(db, args.domain)
 else:
     print "No domain selected!"
@@ -41,7 +42,7 @@ count_all = 0
 for item in raw:
     # print item + '}', type(item)
     try:
-        entry = json.loads((item + '}').replace("'", '"'))
+        entry = ast.literal_eval(item)
     except:
         print (item + '}').replace("'", '"')
     # print entry, type(entry)