moving leveldb-reader to reader, bugfix: dropping last character in data

3 files changed, 165 insertions, 128 deletions

diff --git a/monitor/josef_experimental.py b/monitor/josef_experimental.py
index 14b8e99..7f1f7d6 100755
--- a/monitor/josef_experimental.py
+++ b/monitor/josef_experimental.py
@@ -11,12 +11,12 @@ from datetime import datetime as dt
 
 
 
-parser = argparse.ArgumentParser(description="")
-parser.add_argument('--domain', default=None, help="RTFM")
-parser.add_argument('--log', default=None, help="RTFM")
-parser.add_argument('--exclude-expired', action='store_true', help="RTFM")
+# parser = argparse.ArgumentParser(description="")
+# parser.add_argument('--domain', default=None, help="RTFM")
+# parser.add_argument('--log', default=None, help="RTFM")
+# parser.add_argument('--exclude-expired', action='store_true', help="RTFM")
 
-args = parser.parse_args()
+# args = parser.parse_args()
 
 monitored_domains = [
     "google.com",
@@ -27,62 +27,74 @@ monitored_domains = [
 ]
 
 
-
-if args.domain:
-    db = db_open()
-    raw = db_lookup_domain(db, args.domain)
-else:
-    print "No domain selected!"
-    sys.exit()
-
-cur_time = dt.now()
-count_valid = 0
-count_all = 0
-for item in raw:
-    # print item + '}', type(item)
-    entry = json.loads((item + '}').replace("'", '"'))
-    # print entry, type(entry)
-    success = True
-    not_after_time = dt.strptime(entry["not_after"], "%b %d %H:%M:%S %Y GMT")
-    not_before_time = dt.strptime(entry["not_before"], "%b %d %H:%M:%S %Y GMT")
-
-
-    if args.log:
-        if args.log in entry["log"]:
-            pass
-        else:
-            success = False
-
-    if cur_time > not_after_time:
-        expired = True
-    elif cur_time < not_before_time:
-        expired = True
-    else:
-        expired = False
-
-    # Exclude expired
-    if args.exclude_expired and expired:
-        success = False
+db = db_open("./tmpdatabase")
+
+db_add_domain(db, "www.cox.a.com", "{dummydata}")
+# print db.Get("com")
+# print db.Get("a.com")
+# print db.Get("cox.a.com")
+# print db.Get("www.cox.a.com")
+print db_lookup_domain(db, "www.cox.a.com")
+
+
+# if args.domain:
+#     db = db_open()
+#     raw = db_lookup_domain(db, args.domain)
+# else:
+#     print "No domain selected!"
+#     sys.exit()
+
+# cur_time = dt.now()
+# count_valid = 0
+# count_all = 0
+# for item in raw:
+#     # print item + '}', type(item)
+#     try:
+#         entry = json.loads((item + '}').replace("'", '"'))
+#     except:
+#         print (item + '}').replace("'", '"')
+#     # print entry, type(entry)
+#     success = True
+#     not_after_time = dt.strptime(entry["not_after"], "%b %d %H:%M:%S %Y GMT")
+#     not_before_time = dt.strptime(entry["not_before"], "%b %d %H:%M:%S %Y GMT")
+
+
+#     if args.log:
+#         if args.log in entry["log"]:
+#             pass
+#         else:
+#             success = False
+
+#     if cur_time > not_after_time:
+#         expired = True
+#     elif cur_time < not_before_time:
+#         expired = True
+#     else:
+#         expired = False
+
+#     # Exclude expired
+#     if args.exclude_expired and expired:
+#         success = False
         
     
-    # Set count matches
-    if success:
-        count_all += 1
-        if not expired:
-            count_valid += 1
-
-    # Print matching
-    if success:
-        s = entry["subject"].split("CN=")[1] + \
-        " certified by " + entry["issuer"].split("CN=")[1] + \
-        " (" + entry["log"] + ") "
-        if expired:
-            print "(NOT VALID) " + s
-        else:
-            print "(VALID) " + s
-
-
-print str(count_all) + " matches found."
+#     # Set count matches
+#     if success:
+#         count_all += 1
+#         if not expired:
+#             count_valid += 1
+
+#     # Print matching
+#     if success:
+#         s = entry["subject"].split("CN=")[1] + \
+#         " certified by " + entry["issuer"].split("CN=")[1] + \
+#         " (" + entry["log"] + ") "
+#         if expired:
+#             print "(NOT VALID) " + s
+#         else:
+#             print "(VALID) " + s
+
+
+# print str(count_all) + " matches found."
 
 
 # print res
diff --git a/monitor/josef_leveldb.py b/monitor/josef_leveldb.py
index cee0035..9557bb2 100755
--- a/monitor/josef_leveldb.py
+++ b/monitor/josef_leveldb.py
@@ -10,6 +10,7 @@ SEP = ";"
 
 
 def match_domain(d1, d2):
+    # print d1, d2
     # Exact match
     if d1 == d2:
         return True
@@ -60,12 +61,14 @@ def db_add_domain(db, domain, data):
     tmpl = domain.split('.')
     k = ""
     for item in reversed(tmpl):
-        next_k = item + '.' + k
-        if k != "":
-            db_append(db, k[:-1], next_k[:-1])
+        if k == "":
+            next_k = item
+        else:
+            next_k = item + '.' + k
+            db_append(db, k, next_k)
         k = next_k
-    db.Delete(k[:-1])
-    db_append(db, k[:-1], data)
+    db.Delete(k)
+    db_append(db, k, data)
 
 
 def db_add_certs(db, data):
@@ -95,9 +98,13 @@ def db_lookup_domain(db, domain):
 
     cur_domain = domain_list.pop()
     intermediate = db.Get(cur_domain).split(SEP)
-
     while True:
         try:
+            intermediate.remove("")
+        except ValueError:
+            pass
+
+        try:
             cur_domain = domain_list.pop() + "." + cur_domain
         except IndexError:
             return res
@@ -105,11 +112,11 @@ def db_lookup_domain(db, domain):
         next_level = []
         for item in intermediate:
             if match_domain(cur_domain, item):
-                # print item
+                print item
                 try:
                     tmp = db.Get(item)
                     if tmp[1] == '{':
-                        res.append(tmp[1:-1])
+                        res.append(tmp[1:])
                     next_level += tmp.split(SEP)
                 except KeyError:
                     # print "Could not find " + item
@@ -118,10 +125,6 @@ def db_lookup_domain(db, domain):
             else:
                 intermediate.remove(item)
         intermediate = next_level
-        try:
-            intermediate.remove("")
-        except ValueError:
-            pass
 
     return res
 
diff --git a/monitor/josef_reader.py b/monitor/josef_reader.py
index c2653c1..736fb3d 100755
--- a/monitor/josef_reader.py
+++ b/monitor/josef_reader.py
@@ -3,7 +3,10 @@
 
 import sys
 from josef_lib import *
+import leveldb
 import argparse
+import json
+from josef_leveldb import *
 from datetime import datetime as dt
 
 
@@ -25,69 +28,88 @@ monitored_domains = [
 
 
 
+if args.domain:
+    db = db_open()
+    raw = db_lookup_domain(db, args.domain)
+else:
+    print "No domain selected!"
+    sys.exit()
+
 cur_time = dt.now()
 count_valid = 0
 count_all = 0
-f = open("output/cert_data.json")
-for line in f:
-    tmp = json.loads(line)
-    try: 
-        success = True
-        not_after_time = dt.strptime(tmp["not_after"], "%b %d %H:%M:%S %Y GMT")
-        not_before_time = dt.strptime(tmp["not_before"], "%b %d %H:%M:%S %Y GMT")
-
-        if args.domain:
-            if args.domain in tmp["subject"].split("CN=")[1] or \
-            args.domain in tmp["SAN"]:
-                pass
-            else:
-                success = False
-        else:
-            print "No domain selected!"
-            sys.exit()
-
-        if args.log:
-            if args.log in tmp["log"]:
-                pass
-            else:
-                success = False
-
-        if cur_time > not_after_time:
-            expired = True
-        elif cur_time < not_before_time:
-            expired = True
-        else:
-            expired = False
+for item in raw:
+    # print item + '}', type(item)
+    try:
+        entry = json.loads((item + '}').replace("'", '"'))
+    except:
+        print (item + '}').replace("'", '"')
+    # print entry, type(entry)
+    success = True
+    not_after_time = dt.strptime(entry["not_after"], "%b %d %H:%M:%S %Y GMT")
+    not_before_time = dt.strptime(entry["not_before"], "%b %d %H:%M:%S %Y GMT")
+
 
-        # Exclude expired
-        if args.exclude_expired and expired:
+    if args.log:
+        if args.log in entry["log"]:
+            pass
+        else:
             success = False
-            
-        
-        # Set count matches
-        if success:
-            count_all += 1
-            if not expired:
-                count_valid += 1
-
-        # Print matching
-        if success:
-            s = tmp["subject"].split("CN=")[1] + \
-            " certified by " + tmp["issuer"].split("CN=")[1] + \
-            " (" + tmp["log"] + ") "
-            if expired:
-                print "(NOT VALID) " + s
-            else:
-                print "(VALID) " + s
-                
 
+    if cur_time > not_after_time:
+        expired = True
+    elif cur_time < not_before_time:
+        expired = True
+    else:
+        expired = False
+
+    # Exclude expired
+    if args.exclude_expired and expired:
+        success = False
+        
+    
+    # Set count matches
+    if success:
+        count_all += 1
+        if not expired:
+            count_valid += 1
+
+    # Print matching
+    if success:
+        s = entry["subject"].split("CN=")[1] + \
+        " certified by " + entry["issuer"].split("CN=")[1] + \
+        " (" + entry["log"] + ") "
+        if expired:
+            print "(NOT VALID) " + s
+        else:
+            print "(VALID) " + s
 
-    except:
-        pass
 
-f.close()
 print str(count_all) + " matches found."
-# if count_valid == 0:
-#     print "No matching certificates found."
-# else:
-#     print str(count_valid) + " of " + str(count_all) + " certs valid. (" + str(int(float(count_valid)/float(count_all)*100)) + "%)"
+
+
+# print res
+# print "Found " + str(len(res)) + " results"
+# print db.Get("wush.net")
+# print db.Get("wush.net")
+
+# f = open("output/cert_data.json")
+# max_count  = 1
+# for line in f:
+#     # print max_count
+#     # try:
+#     tmp = json.loads(line)
+#     # print tmp
+#     # d  = tmp["subject"].split("CN=")[1]
+#     db_add_cert(tmp)
+#     # print d
+
+#     max_count -= 1
+#     if max_count == 0:
+#         break
+
+
+
+
+
+