enabling mixed auditing/monitoring. changing timespamp format

5 files changed, 60 insertions, 48 deletions

diff --git a/monitor/josef_logreader.py b/monitor/josef_logreader.py
index 4be2a55..622674d 100755
--- a/monitor/josef_logreader.py
+++ b/monitor/josef_logreader.py
@@ -8,7 +8,7 @@ import os
 
 from monitor_conf import *
 
-TIME_LEN = 21
+TIME_LEN = 20
 NEW_STH_STR = "STH updated"
 START_STR = "Starting monitor"
 
@@ -20,7 +20,7 @@ def get_logs():
     return logs
 
 def get_age_from_line(line):
-    past = datetime.datetime.strptime(line[:20], '%Y-%m-%d, %H:%M:%S')
+    past = datetime.datetime.strptime(line[:20], '%Y-%m-%d %H:%M:%S')
     present = datetime.datetime.now()
     return present - past
 
diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py
index 035ef72..f33039a 100755
--- a/monitor/josef_monitor.py
+++ b/monitor/josef_monitor.py
@@ -36,7 +36,7 @@ else:
 
 
 class ctlog:
-    def __init__(self, name, url, key, log_id=None):
+    def __init__(self, name, url, key, log_id=None, build=True):
         self.name = name
         self.url = url
         self.key = key
@@ -47,6 +47,7 @@ class ctlog:
         self.sth = None
         self.entries = 0
         self.root_hash = None
+        self.build = build
 
         self.log("Starting monitor")
 
@@ -58,20 +59,22 @@ class ctlog:
         except Exception, e:
             self.log("Failed to fetch STH. " + str(e))
             return
-        start_size = self.entries
-        while self.entries < self.sth["tree_size"]:
-            tmp_size = self.entries
-            self.subtree, self.entries = self.fetch_and_increment_subtree(self.entries, self.sth["tree_size"] -1, self.url, self.subtree)
-            if tmp_size != self.entries:
-                self.log("Got entries " + str(tmp_size) + " to " \
-                + str(self.entries -1 ) + " of " + str(self.sth["tree_size"]-1))
-
-        if self.entries != start_size:
-            if verify_subtree(self.sth, self.subtree, self.url):
-                self.log("Successfully build tree with " + str(self.entries - start_size) + \
-                    " new entries. Size: " + str(self.entries))
-            else:
-                self.log("ERROR Failed to build tree from entries.")
+
+        if self.build:
+            start_size = self.entries
+            while self.entries < self.sth["tree_size"]:
+                tmp_size = self.entries
+                self.subtree, self.entries = self.fetch_and_increment_subtree(self.entries, self.sth["tree_size"] -1, self.url, self.subtree)
+                if tmp_size != self.entries:
+                    self.log("Got entries " + str(tmp_size) + " to " \
+                    + str(self.entries -1 ) + " of " + str(self.sth["tree_size"]-1))
+
+            if self.entries != start_size:
+                if verify_subtree(self.sth, self.subtree, self.url):
+                    self.log("Successfully build tree with " + str(self.entries - start_size) + \
+                        " new entries. Size: " + str(self.entries))
+                else:
+                    self.log("ERROR Failed to build tree from entries.")
 
     def fetch_and_increment_subtree(self, first, last, url, subtree =[[]]):
         new_leafs = []
@@ -118,7 +121,7 @@ class ctlog:
             raise e
 
     def log(self, string):
-        s = time.strftime('%Y-%m-%d, %H:%M:%S') + " " + string
+        s = time.strftime('%Y-%m-%d %H:%M:%S') + " " + string
         with open(self.logfile, 'a') as f:
             f.write(s + "\n")
             f.close()
@@ -131,7 +134,7 @@ class ctlog:
             return
 
         try:
-            check_sth_signature(self.url, new_sth, base64.b64deode(self.key))
+            check_sth_signature(self.url, new_sth, base64.b64decode(self.key))
         except:
             self.log("ERROR: Could not verify STH signature")
             print "ERROR: Could not verify STH signature from " + self.url
@@ -397,7 +400,7 @@ def main(args):
     logs = []
     try:
         for item in CONFIG.CTLOGS:
-            logs.append(ctlog(item["name"], item["url"], item["key"], item["id"]))
+            logs.append(ctlog(item["name"], item["url"], item["key"], item["id"], item["build"]))
 
         print time.strftime('%H:%M:%S') + " Setting up monitor for " + str(len(logs)) + " logs..."
         # Set up state 
@@ -406,6 +409,7 @@ def main(args):
                 log.load()
         # Build new entries 
         for log in logs:
+            # if log.build:
             log.incremental_build()
 
         # Main loop: Monitor
@@ -420,6 +424,7 @@ def main(args):
                 if old_sth["timestamp"] != log.sth["timestamp"]:
                     log.verify_progress(old_sth)
                     log.verify_consistency(old_sth)
+                    # if log.build:
                     log.incremental_build()
 
             for md in monitored_domains:
@@ -453,7 +458,7 @@ def main(args):
 
 
 if __name__ == '__main__':
-    if not os.path.exists(CONFIG.OUTPUT_DIR):
+    if CONFIG.OUTPUT_DIR and not os.path.exists(CONFIG.OUTPUT_DIR):
         os.makedirs(CONFIG.OUTPUT_DIR)
 
     if CONFIG.DB_PATH and not os.path.exists(CONFIG.DB_PATH):
diff --git a/monitor/josef_reader.py b/monitor/josef_reader.py
index 6b14b70..7974221 100755
--- a/monitor/josef_reader.py
+++ b/monitor/josef_reader.py
@@ -29,7 +29,7 @@ class monitored_domain:
         return self.url == other.url
 
     def log(self, string):
-        s = time.strftime('%Y-%m-%d, %H:%M:%S') + " " + string
+        s = time.strftime('%Y-%m-%d %H:%M:%S') + " " + string
         with open(OUTPUT_DIR + "monitor.log", 'a') as f:
             f.write(s + "\n")
             f.close()
diff --git a/monitor/monitor_conf.py b/monitor/monitor_conf.py
index b921e9f..f5f7941 100644
--- a/monitor/monitor_conf.py
+++ b/monitor/monitor_conf.py
@@ -18,62 +18,71 @@ DOMAINS_FILE = OUTPUT_DIR + "domains.json"
 DB_PATH = './tmpdb/'
 
 MONITORED_DOMAINS = [
-    "*.preishelden.de",
+    # "*.preishelden.de",
     "*.liu.se",
     "*.kth.se",
     "*.nordu.net",
     "*.sunet.se",
-    "mail.google.com",
-    "*.symantec.com",
+    # "mail.google.com",
+    # "*.symantec.com",
     # "*.se",
 ]
 
 # CT logs and associated keys
 CTLOGS = [
-    # {"name" : "pilot",
-    # "url" : "https://ct.googleapis.com/pilot/",
-    # "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTDM0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA==",
-    # "id" : "pLkJkLQYWBSHuxOizGdwCjw1mAT5G9+443fNDsgN3BA="},
+    {"name" : "pilot",
+    "url" : "https://ct.googleapis.com/pilot/",
+    "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTDM0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA==",
+    "id" : "pLkJkLQYWBSHuxOizGdwCjw1mAT5G9+443fNDsgN3BA=",
+    "build" : False},
 
     {"name" : "plausible",
     "url" : "https://plausible.ct.nordu.net/",
     "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9UV9+jO2MCTzkabodO2F7LM03MUBc8MrdAtkcW6v6GA9taTTw9QJqofm0BbdAsbtJL/unyEf0zIkRgXjjzaYqQ==",
-    "id" : "qucLfzy41WbIbC8Wl5yfRF9pqw60U1WJsvd6AwEE880="},
+    "id" : "qucLfzy41WbIbC8Wl5yfRF9pqw60U1WJsvd6AwEE880=",
+    "build" : False},
 
     {"name" : "digicert",
     "url" : "https://ct1.digicert-ct.com/log/",
     "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEAkbFvhu7gkAW6MHSrBlpE1n4+HCFRkC5OLAjgqhkTH+/uzSfSl8ois8ZxAD2NgaTZe1M9akhYlrYkes4JECs6A==",
-    "id" : "VhQGmi/XwuzT9eG9RLI+x0Z2ubyZEVzA75SYVdaJ0N0="},
+    "id" : "VhQGmi/XwuzT9eG9RLI+x0Z2ubyZEVzA75SYVdaJ0N0=",
+    "build" : False},
 
     {"name" : "izenpe",
     "url" : "https://ct.izenpe.com/",
     "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJ2Q5DC3cUBj4IQCiDu0s6j51up+TZAkAEcQRF6tczw90rLWXkJMAW7jr9yc92bIKgV8vDXU4lDeZHvYHduDuvg==",
-    "id" : "dGG0oJz7PUHXUVlXWy52SaRFqNJ3CbDMVkpkgrfrQaM="},
+    "id" : "dGG0oJz7PUHXUVlXWy52SaRFqNJ3CbDMVkpkgrfrQaM=",
+    "build" : True},
     
     {"name" : "certly",
     "url" : "https://log.certly.io/",
     "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAECyPLhWKYYUgEc+tUXfPQB4wtGS2MNvXrjwFCCnyYJifBtd2Sk7Cu+Js9DNhMTh35FftHaHu6ZrclnNBKwmbbSA==",
-    "id" : "zbUXm3/BwEb+6jETaj+PAC5hgvr4iW/syLL1tatgSQA="}, 
+    "id" : "zbUXm3/BwEb+6jETaj+PAC5hgvr4iW/syLL1tatgSQA=",
+    "build" : True}, 
 
-    # {"name" : "aviator",
-    # "url" : "https://ct.googleapis.com/aviator/",
-    # "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1/TMabLkDpCjiupacAlP7xNi0I1JYP8bQFAHDG1xhtolSY1l4QgNRzRrvSe8liE+NPWHdjGxfx3JhTsN9x8/6Q==",
-    # "id" : "aPaY+B9kgr46jO65KB1M/HFRXWeT1ETRCmesu09P+8Q="},
+    {"name" : "aviator",
+    "url" : "https://ct.googleapis.com/aviator/",
+    "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1/TMabLkDpCjiupacAlP7xNi0I1JYP8bQFAHDG1xhtolSY1l4QgNRzRrvSe8liE+NPWHdjGxfx3JhTsN9x8/6Q==",
+    "id" : "aPaY+B9kgr46jO65KB1M/HFRXWeT1ETRCmesu09P+8Q=",
+    "build" : False},
 
     {"name" : "rocketeer",
     "url" : "https://ct.googleapis.com/rocketeer/",
     "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIFsYyDzBi7MxCAC/oJBXK7dHjG+1aLCOkHjpoHPqTyghLpzA9BYbqvnV16mAw04vUjyYASVGJCUoI3ctBcJAeg==",
-    "id": "7ku9t3XOYLrhQmkfq+GeZqMPfl+wctiDAMR7iXqo/cs="},
+    "id": "7ku9t3XOYLrhQmkfq+GeZqMPfl+wctiDAMR7iXqo/cs=",
+    "build" : False},
 
     {"name" : "symantec",
     "url" : "https://ct.ws.symantec.com/",
     "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEluqsHEYMG1XcDfy1lCdGV0JwOmkY4r87xNuroPS2bMBTP01CEDPwWJePa75y9CrsHEKqAy8afig1dpkIPSEUhg==",
-    "id" : "3esdK3oNT6Ygi4GtgWhwfi6OnQHVXIiNPRHEzbbsvsw="},
+    "id" : "3esdK3oNT6Ygi4GtgWhwfi6OnQHVXIiNPRHEzbbsvsw=",
+    "build" : True},
     
     {"name" : "venafi",
     "url" : "https://ctlog.api.venafi.com/",
     "key" : "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAolpIHxdSlTXLo1s6H1OCdpSj/4DyHDc8wLG9wVmLqy1lk9fz4ATVmm+/1iN2Nk8jmctUKK2MFUtlWXZBSpym97M7frGlSaQXUWyA3CqQUEuIJOmlEjKTBEiQAvpfDjCHjlV2Be4qTM6jamkJbiWtgnYPhJL6ONaGTiSPm7Byy57iaz/hbckldSOIoRhYBiMzeNoA0DiRZ9KmfSeXZ1rB8y8X5urSW+iBzf2SaOfzBvDpcoTuAaWx2DPazoOl28fP1hZ+kHUYvxbcMjttjauCFx+JII0dmuZNIwjfeG/GBb9frpSX219k1O4Wi6OEbHEr8at/XQ0y7gTikOxBn/s5wQIDAQAB",
-    "id" : "rDua7X+pZ0dXFZ5tfVdWcvnZgQCUHpve/+yhMTt1eC0="},
+    "id" : "rDua7X+pZ0dXFZ5tfVdWcvnZgQCUHpve/+yhMTt1eC0=",
+    "build" : True},
 
     # {"name" : "devp",
     # "url" : "https://localhost:8080/",
diff --git a/tools/josef_auditor.py b/tools/josef_auditor.py
index 44eb5f8..d612a83 100755
--- a/tools/josef_auditor.py
+++ b/tools/josef_auditor.py
@@ -110,7 +110,7 @@ def fetch_all_sth():
     return sths
 
 def verify_progress(old, new):
-    print "Verifying progress"
+    # print "Verifying progress"
     try:
         for url in new:
             if new and old and new[url] and old[url]:
@@ -361,7 +361,8 @@ def main(args):
             time.sleep(30)
             new_sth = fetch_all_sth()
             verify_consistency(sth, new_sth)
-            verify_inclusion_all(sth, new_sth)
+            # verify_inclusion_all(sth, new_sth)
+            verify_progress(sth, new_sth)
             sth = new_sth
 
     if args.audit2:
@@ -402,7 +403,7 @@ def main(args):
 
 
 if __name__ == '__main__':
-    # try:
+    try:
         main(parser.parse_args())
         if len(errors) == 0:
             print time.strftime('%H:%M:%S') + " Everything OK."
@@ -411,11 +412,8 @@ if __name__ == '__main__':
             # print "errors found!"
             print_errors(errors)
             sys.exit(NAGIOS_WARN)
-    # except:
-    #     pass
-    # finally:
-    #     # print_timings(timings)
-    #     print_errors(errors)
+    except KeyboardInterrupt:
+        print_errors(errors)