adding issuer monitoring for lets encrypt

6 files changed, 95 insertions, 36 deletions

diff --git a/monitor/josef_experimental.py b/monitor/josef_experimental.py
index 6c74db9..2ccfceb 100755
--- a/monitor/josef_experimental.py
+++ b/monitor/josef_experimental.py
@@ -4,9 +4,9 @@
 import sys
 import os
 from josef_lib import *
-from josef_lib2 import *
+# from josef_lib2 import *
 # import leveldb
-import argparse
+# import argparse
 import json
 import time
 # from josef_leveldb import *
@@ -134,7 +134,10 @@ def parse_entry(e, idx, log):
     s = log["name"]
     s += sep + str(idx) # index
     s += sep + e["subject"] # Subject
-    s += sep + e["SAN"] # SAN
+    if "SAN" in e:
+        s += sep + e["SAN"] # SAN
+    else:
+        s += sep
     s += sep + e["issuer"] # issuer
     s += sep + e["chain_length"] # path length
     s += sep + e["sig_algorithm"] # Signature algothithm
@@ -146,16 +149,18 @@ def parse_entry(e, idx, log):
 
     return s
 
-def check_api2(url):
-    print "\nTesting " + url
-    try:
-        print get_sth_v2(url)
-    except:
-        print "GET STH Failed..."
+# def check_api2(url):
+#     print "\nTesting " + url
+#     try:
+#         print get_sth_v2(url)
+#     except:
+#         print "GET STH Failed..."
+
+
 
 if __name__ == '__main__':
 
-    
+    # prompt_confirm("you are about to remove file")
 
     # Find let's encrypt certs
     if False:
@@ -177,21 +182,24 @@ if __name__ == '__main__':
 
 
     # Data gathering for Niklas
-    if False:
+    if True:
         log = CTLOGS[0]
         sth = get_sth(log["url"])
         # size = sth["tree_size"]
         # for i in range(15,200):
-        start = 5757748
-        end = 5757847
+        start = 0
+        end = int(sth["tree_size"]) - 1
         print "Getting " + str(start) + " to " + str(end)
         entries = get_entries(log["url"],start ,end)["entries"]
 
         # TODO set filename
-        filename = "ct_log_content.txt"
+        filename = log["name"] + "_content.txt"
         # TODO remove file if exists
         if os.path.exists(filename):
-            os.remove(filename)
+            if prompt_confirm("You are about to overwrite " + filename):
+                os.remove(filename)
+            else:
+                sys.exit()
         # TODO open file
         with open(filename, 'a') as f:
         # TODO write lines
diff --git a/monitor/josef_lib.py b/monitor/josef_lib.py
index f401b2c..46a0eee 100644
--- a/monitor/josef_lib.py
+++ b/monitor/josef_lib.py
@@ -23,6 +23,28 @@ from Crypto.Hash import SHA256
 import Crypto.PublicKey.RSA as RSA
 from Crypto.Signature import PKCS1_v1_5
 
+
+def prompt_confirm(msg = "", default = True):
+    print msg
+
+    while True:
+        if default:
+            print "Are you sure? (Y/n)"
+        else:
+            print "Are you sure? (y/N)"
+        
+        import sys
+        data = sys.stdin.readline()
+        
+        if data == "y\n":
+            return True
+        elif data == "n\n":
+            return False
+        elif data == "\n":
+            return default
+        else:
+            print "Answer either y or n"
+
 def time_str(ts = None):
     if ts is None:
         return datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py
index a05a898..2a33ef0 100755
--- a/monitor/josef_monitor.py
+++ b/monitor/josef_monitor.py
@@ -129,6 +129,7 @@ class ctlog:
                     tmp_data["leaf_hash"] = base64.b64encode(entry_hash)
                     tmp_cert_data.append(tmp_data)
                 new_leafs.append(entry_hash)
+                monitor_issuer(tmp_data)
             if self.dbdir:
                 db_add_certs(self.dbdir, tmp_cert_data)
             if CONFIG.DEFAULT_CERT_FILE:
@@ -344,10 +345,24 @@ class ctlog:
 #         print "ERROR:", e.read()
 #         sys.exit(0)
 
+def monitor_issuer(data):
+    # print data["issuer"]
+    if CONFIG.ISSUERS_FILE:
+        filename = CONFIG.ISSUERS_FILE
+
+        for issuer in CONFIG.MONITORED_ISSUERS:
+            if issuer in data["issuer"]:
+                with open(filename, 'a') as f:
+                    f.write(time_str() + str(data) + "\n")
+                    f.close()
+
+
 
 
 def setup_domain_monitoring():
     monitored_domains = []
+    if not CONFIG.MONITORED_DOMAINS:
+        return []
 
     try:
         with open(CONFIG.DOMAINS_FILE) as fp:
diff --git a/monitor/josef_reader.py b/monitor/josef_reader.py
index bd069bb..e402da1 100755
--- a/monitor/josef_reader.py
+++ b/monitor/josef_reader.py
@@ -10,8 +10,8 @@ import json
 import base64
 import subprocess
 try:
-    import leveldb
     from josef_leveldb import *
+    import leveldb
 except:
     print "No database support found."
 from datetime import datetime as dt
diff --git a/monitor/monitor_conf.py b/monitor/monitor_conf.py
index 674c565..9bc197d 100644
--- a/monitor/monitor_conf.py
+++ b/monitor/monitor_conf.py
@@ -12,10 +12,13 @@ DEFAULT_CERT_FILE = None
 # DEFAULT_CERT_FILE = OUTPUT_DIR + "cert_data.json"
 
 # Set to None to disable database writing
-DOMAINS_FILE = OUTPUT_DIR + "domains.json"
+# DOMAINS_FILE = OUTPUT_DIR + "domains.json"
+DOMAINS_FILE = None
+ISSUERS_FILE = DOMAINS_FILE = OUTPUT_DIR + "issuers.log"
 
 # Set to None to disable database output
-DB_PATH = './tmpdb/'
+# DB_PATH = './tmpdb/'
+DB_PATH = None
 
 MONITORED_DOMAINS = [
     "*.liu.se",
@@ -26,6 +29,10 @@ MONITORED_DOMAINS = [
     "*.iis.se",
 ]
 
+MONITORED_ISSUERS = [
+    "Let's Encrypt",
+]
+
 # Some strings
 ERROR_STR = "ERROR: "
 
@@ -47,19 +54,19 @@ CTLOGS = [
     "url" : "https://ct1.digicert-ct.com/log/",
     "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEAkbFvhu7gkAW6MHSrBlpE1n4+HCFRkC5OLAjgqhkTH+/uzSfSl8ois8ZxAD2NgaTZe1M9akhYlrYkes4JECs6A==",
     "id" : "VhQGmi/XwuzT9eG9RLI+x0Z2ubyZEVzA75SYVdaJ0N0=",
-    "build" : True},
+    "build" : False},
 
     {"name" : "izenpe",
     "url" : "https://ct.izenpe.com/",
     "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJ2Q5DC3cUBj4IQCiDu0s6j51up+TZAkAEcQRF6tczw90rLWXkJMAW7jr9yc92bIKgV8vDXU4lDeZHvYHduDuvg==",
     "id" : "dGG0oJz7PUHXUVlXWy52SaRFqNJ3CbDMVkpkgrfrQaM=",
-    "build" : True},
+    "build" : False},
     
     {"name" : "certly",
     "url" : "https://log.certly.io/",
     "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAECyPLhWKYYUgEc+tUXfPQB4wtGS2MNvXrjwFCCnyYJifBtd2Sk7Cu+Js9DNhMTh35FftHaHu6ZrclnNBKwmbbSA==",
     "id" : "zbUXm3/BwEb+6jETaj+PAC5hgvr4iW/syLL1tatgSQA=",
-    "build" : True}, 
+    "build" : False}, 
 
     {"name" : "aviator",
     "url" : "https://ct.googleapis.com/aviator/",
@@ -71,25 +78,25 @@ CTLOGS = [
     "url" : "https://ct.googleapis.com/rocketeer/",
     "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIFsYyDzBi7MxCAC/oJBXK7dHjG+1aLCOkHjpoHPqTyghLpzA9BYbqvnV16mAw04vUjyYASVGJCUoI3ctBcJAeg==",
     "id": "7ku9t3XOYLrhQmkfq+GeZqMPfl+wctiDAMR7iXqo/cs=",
-    "build" : True},
+    "build" : False},
 
     {"name" : "symantec",
     "url" : "https://ct.ws.symantec.com/",
     "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEluqsHEYMG1XcDfy1lCdGV0JwOmkY4r87xNuroPS2bMBTP01CEDPwWJePa75y9CrsHEKqAy8afig1dpkIPSEUhg==",
     "id" : "3esdK3oNT6Ygi4GtgWhwfi6OnQHVXIiNPRHEzbbsvsw=",
-    "build" : True},
+    "build" : False},
     
     {"name" : "venafi",
     "url" : "https://ctlog.api.venafi.com/",
     "key" : "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAolpIHxdSlTXLo1s6H1OCdpSj/4DyHDc8wLG9wVmLqy1lk9fz4ATVmm+/1iN2Nk8jmctUKK2MFUtlWXZBSpym97M7frGlSaQXUWyA3CqQUEuIJOmlEjKTBEiQAvpfDjCHjlV2Be4qTM6jamkJbiWtgnYPhJL6ONaGTiSPm7Byy57iaz/hbckldSOIoRhYBiMzeNoA0DiRZ9KmfSeXZ1rB8y8X5urSW+iBzf2SaOfzBvDpcoTuAaWx2DPazoOl28fP1hZ+kHUYvxbcMjttjauCFx+JII0dmuZNIwjfeG/GBb9frpSX219k1O4Wi6OEbHEr8at/XQ0y7gTikOxBn/s5wQIDAQAB",
     "id" : "rDua7X+pZ0dXFZ5tfVdWcvnZgQCUHpve/+yhMTt1eC0=",
-    "build" : True},
+    "build" : False},
 
     {"name" : "wosign",
     "url" : "https://ct.wosign.com/",
     "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1+wvK3VPN7yjQ7qLZWY8fWrlDCqmwuUm/gx9TnzwOrzi0yLcAdAfbkOcXG6DrZwV9sSNYLUdu6NiaX7rp6oBmw==",
     "id" : "nk/3PcPOIgtpIXyJnkaAdqv414Y21cz8haMadWKLqIs=",
-    "build" : True},    
+    "build" : False},    
 
 ]
 
diff --git a/monitor/monitor_conf_local.py b/monitor/monitor_conf_local.py
index b52d132..b7dfc20 100644
--- a/monitor/monitor_conf_local.py
+++ b/monitor/monitor_conf_local.py
@@ -1,7 +1,7 @@
 # All configuration for the CT monitor is done from this file!
 
 # interval (in seconds) between updates
-INTERVAL = 60 
+INTERVAL = 20 
 
 # Directories for various output files
 OUTPUT_DIR = "output/"
@@ -14,17 +14,24 @@ DEFAULT_CERT_FILE = None
 # Set to None to disable database writing
 # DOMAINS_FILE = OUTPUT_DIR + "domains.json"
 DOMAINS_FILE = None
+ISSUERS_FILE = DOMAINS_FILE = OUTPUT_DIR + "issuers.log"
 
 # Set to None to disable database output
-DB_PATH = './tmpdb/'
-
-MONITORED_DOMAINS = [
-    "*.liu.se",
-    "*.kth.se",
-    "*.nordu.net",
-    "*.sunet.se",
-    "*.dfri.se",
-    "*.iis.se",
+# DB_PATH = './tmpdb/'
+DB_PATH = None
+
+# MONITORED_DOMAINS = [
+#     "*.liu.se",
+#     "*.kth.se",
+#     "*.nordu.net",
+#     "*.sunet.se",
+#     "*.dfri.se",
+#     "*.iis.se",
+# ]
+MONITORED_DOMAINS = None
+
+MONITORED_ISSUERS = [
+    "Let's Encrypt",
 ]
 
 # Some strings
@@ -42,7 +49,7 @@ CTLOGS = [
     "url" : "https://plausible.ct.nordu.net/",
     "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9UV9+jO2MCTzkabodO2F7LM03MUBc8MrdAtkcW6v6GA9taTTw9QJqofm0BbdAsbtJL/unyEf0zIkRgXjjzaYqQ==",
     "id" : "qucLfzy41WbIbC8Wl5yfRF9pqw60U1WJsvd6AwEE880=",
-    "build" : False},
+    "build" : True},
 
     {"name" : "digicert",
     "url" : "https://ct1.digicert-ct.com/log/",