adding domain monitoring to monitor

3 files changed, 113 insertions, 52 deletions

diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py
index d11d38f..0569a81 100755
--- a/monitor/josef_monitor.py
+++ b/monitor/josef_monitor.py
@@ -13,12 +13,13 @@ import argparse
 import errno
 from copy import deepcopy
 from josef_lib import *
+from josef_reader import monitored_domain
 from josef_leveldb import db_add_certs, db_open
 import os.path
 
 # Import from config file
 if os.path.isfile("monitor_conf.py"):
-    from monitor_conf import ctlogs, OUTPUT_DIR, INTERVAL, DEFAULT_CERT_FILE, DB_PATH
+    from monitor_conf import ctlogs, OUTPUT_DIR, INTERVAL, DEFAULT_CERT_FILE, DB_PATH, MONITORED_DOMAINS, DOMAINS_FILE
 else:
     print "Config file not found!"
     sys.exit()
@@ -115,7 +116,6 @@ class ctlog:
             raise e
 
     def log(self, string):
-        # TODO change to UTC?
         s = time.strftime('%Y-%m-%d, %H:%M:%S') + " " + string
         with open(self.logfile, 'a') as f:
             f.write(s + "\n")
@@ -324,8 +324,21 @@ def main(args):
     global DB
     # Create logs
     logs = []
-    # if DB_PATH:
-    #     DB = db_open(DB_PATH)
+
+    monitored_domains = []
+    with open(DOMAINS_FILE) as fp:
+        for line in fp:
+            tmp = json.loads(line)
+            for domain in tmp:
+                md = monitored_domain(domain["url"])
+                md.load_entries(domain["entries"])
+
+    for md in MONITORED_DOMAINS: 
+        tmp = monitored_domain(md)
+        if not tmp in monitored_domains:
+            tmp.set()   
+            monitored_domains.append(tmp)
+
     try:
         for item in ctlogs:
             logs.append(ctlog(item, ctlogs[item][0], ctlogs[item][1]))
@@ -352,17 +365,31 @@ def main(args):
                     log.verify_consistency(old_sth)
                     log.incremental_build()
 
+            for md in monitored_domains:
+                md.update()
+
 
     except KeyboardInterrupt:
         print time.strftime('%H:%M:%S') + ' Received interrupt from user. Saving and exiting....'
         for log in logs:
             log.save()
 
+        domain_dict = []
+        for md in monitored_domains:
+            domain_dict.append(md.to_dict())
+        open(DOMAINS_FILE, 'w').write(json.dumps(domain_dict))
+
     # except Exception, err:
     #     print Exception, err
     #     for log in logs:
     #         log.save()
 
+    #     domain_dict = []
+    #     for md in monitored_domains:
+    #         domain_dict.append(md.to_dict())
+
+    #     open(DOMAINS_FILE, 'w').write(json.dumps(domain_dict))
+
 
 
 if __name__ == '__main__':
diff --git a/monitor/josef_reader.py b/monitor/josef_reader.py
index 3abadb2..199fe29 100755
--- a/monitor/josef_reader.py
+++ b/monitor/josef_reader.py
@@ -2,6 +2,8 @@
 # -*- coding: utf-8 -*-     
 
 import sys
+import time
+import datetime
 from josef_lib import *
 import leveldb
 import argparse
@@ -11,7 +13,7 @@ import subprocess
 from josef_leveldb import *
 from datetime import datetime as dt
 import ast
-from monitor_conf import DB_PATH, MONITORED_DOMAINS
+from monitor_conf import DB_PATH, MONITORED_DOMAINS, OUTPUT_DIR
 
 
 
@@ -20,26 +22,67 @@ class monitored_domain:
         self.url = url
         self.entries = []
 
-    def add(self, item):
-        self.entries.appent(item)
+    def __eq__(self, other):
+        return self.url == other.url
+
+    def log(self, string):
+        s = time.strftime('%Y-%m-%d, %H:%M:%S') + " " + string
+        with open(OUTPUT_DIR + "monitor.log", 'a') as f:
+            f.write(s + "\n")
+            f.close()
+
+    # def add(self, item):
+    #     self.entries.appent(item)
 
     def set(self):
         self.entries = db_monitor_domain(self.url, None, True, None)
-        print "Got " + str(len(self.entries)) + " certs for " + self.url
+        self.log("Got " + str(len(self.entries)) + " certs for " + self.url)
 
     def update(self):
         new = db_monitor_domain(self.url, None, True, None)
-        if len(new) != len(self.entries):
+        # if len(new) != len(self.entries):
             # print self.entries
             # print new
-            compare_entry_lists(new, self.entries)
-            self.entries = new
-        else:
-            for i in range(len(new)):
-                if new[i] == self.entries[i]:
-                    pass
-                else:
-                    print "ITEM CHANGED!"
+        compare_entry_lists(new, self.entries)
+        self.entries = new
+ 
+
+    def to_dict(self):
+        d = {}
+        d["url"] = self.url
+        entries = []
+        for e in self.entries:
+            entries.append(e.to_dict())
+        d["entries"] = entries
+        return d
+
+    def load_entries(self, l):
+        entries = []
+        for item in l:
+            entries.append(monitored_entry(item["subject"],item["issuer"],item["log"],item["status"],item["leaf_hash"]))
+        self.entries = entries
+
+    def compare_entry_lists(self, new, old):
+        added_items  = []
+        removed_items  = []
+
+        for item in new:
+            if not item in old:
+                added_items.append(item)
+
+        for item in old:
+            if not item in new:
+                removed_items.append(item)
+
+        if len(added_items) != 0:
+            self.log(str(len(added_items)) + " new item(s):")
+            for item in added_items:
+                self.log(item)
+        
+        if len(removed_items) != 0:
+            self.log(str(len(removed_items)) + " removed item(s):")
+            for item in removed_items:
+                self.log(item)
 
 
 class monitored_entry:
@@ -52,6 +95,7 @@ class monitored_entry:
 
     def __eq__(self, other):
         return self.leaf_hash == other.leaf_hash
+        # TODO check status (or expired wont expire)
 
     def __str__(self):
         s = self.subject + \
@@ -62,6 +106,14 @@ class monitored_entry:
         else:
             return "(NOT VALID) " + s
 
+    def to_dict(self):
+        d = {}
+        d["issuer"] = self.issuer
+        d["subject"] = self.subject
+        d["log"] = self.log
+        d["status"] = self.status
+        d["leaf_hash"] = self.leaf_hash
+        return d
 
 
 
@@ -79,28 +131,6 @@ for md in MONITORED_DOMAINS:
 
 db = DB_PATH
 
-def compare_entry_lists(new, old):
-    added_items  = []
-    removed_items  = []
-
-    for item in new:
-        if not item in old:
-            added_items.append(item)
-
-    for item in old:
-        if not item in new:
-            removed_items.append(item)
-
-    if len(added_items) != 0:
-        print str(len(added_items)) + " new item(s):"
-        for item in added_items:
-            print item
-    
-    if len(removed_items) != 0:
-        print str(len(removed_items)) + " removed item(s):"
-        for item in removed_items:
-            print item
-
 
 
 def db_monitor_domain(domain, log=None, exclude_invalid=None, get_cert=None):
@@ -178,15 +208,15 @@ def db_monitor_domain(domain, log=None, exclude_invalid=None, get_cert=None):
     # + domain
     return res
 
-
-if args.domain:
-    db_monitor_domain(args.domain, args.log, args.exclude_invalid, args.get_cert)
-else:
-    print "Running on " + str(len(monitored_domains)) + " monitored domains."
-    for d in monitored_domains:
-        d.set()
-    for d in monitored_domains:
-        d.update()
+if __name__ == "__main__":
+    if args.domain:
+        db_monitor_domain(args.domain, args.log, args.exclude_invalid, args.get_cert)
+    else:
+        print "Running on " + str(len(monitored_domains)) + " monitored domains."
+        for d in monitored_domains:
+            d.set()
+        for d in monitored_domains:
+            print d.to_dict()
 
 
 
diff --git a/monitor/monitor_conf.py b/monitor/monitor_conf.py
index d1a21fb..1106183 100644
--- a/monitor/monitor_conf.py
+++ b/monitor/monitor_conf.py
@@ -11,6 +11,8 @@ OUTPUT_DIR = "output/"
 DEFAULT_CERT_FILE = None
 # DEFAULT_CERT_FILE = OUTPUT_DIR + "cert_data.json"
 
+DOMAINS_FILE = OUTPUT_DIR + "domains.json"
+
 # Set to None to disable database output
 DB_PATH = './tmpdb/'
 
@@ -19,7 +21,9 @@ MONITORED_DOMAINS = [
     "*.liu.se",
     "*.kth.se",
     "*.nordu.net",
-    "mail.google.com",
+    # "mail.google.com",
+    # "*.symantec.com",
+    # "*.se",
 ]
 
 # CT logs and associated keys
@@ -28,9 +32,9 @@ ctlogs = {
     # ["https://ct.googleapis.com/pilot/",
     # "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTDM0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA=="],
 
-    # "plausible":
-    # ["https://plausible.ct.nordu.net/",
-    # "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9UV9+jO2MCTzkabodO2F7LM03MUBc8MrdAtkcW6v6GA9taTTw9QJqofm0BbdAsbtJL/unyEf0zIkRgXjjzaYqQ=="],
+    "plausible":
+    ["https://plausible.ct.nordu.net/",
+    "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9UV9+jO2MCTzkabodO2F7LM03MUBc8MrdAtkcW6v6GA9taTTw9QJqofm0BbdAsbtJL/unyEf0zIkRgXjjzaYqQ=="],
 
     # "digicert":
     # ["https://ct1.digicert-ct.com/log/",