data gathering for niklas

4 files changed, 228 insertions, 16 deletions

diff --git a/monitor/josef_experimental.py b/monitor/josef_experimental.py
index 979bae8..5df124b 100755
--- a/monitor/josef_experimental.py
+++ b/monitor/josef_experimental.py
@@ -4,6 +4,7 @@
 import sys
 import os
 from josef_lib import *
+from josef_lib2 import *
 # import leveldb
 import argparse
 import json
@@ -126,17 +127,67 @@ def update_roots(log):
             open(tempname, 'w').write(data)
             mv_file(tempname, fn)
 
+def parse_entry(e, idx, log):
+    # print the following fields, separated by sep
+    sep = ";"
+
+    s = log["name"]
+    s += sep + str(idx) # index
+    s += sep + e["subject"] # Subject
+    s += sep + e["SAN"] # SAN
+    s += sep + e["issuer"] # issuer
+    s += sep + e["chain_length"] # path length
+    s += sep + e["sig_algorithm"] # Signature algothithm
+    s += sep + e["pubkey_algorithm"] # pubkey algorithm
+    s += sep + e["not_before"] # valid from
+    s += sep + e["not_after"] # valid to
+    s += sep + e["validation"] # EV?
+    s += sep + e["in_mozilla"] # chains to mozilla root?
+
+    return s
+
+def check_api2(url):
+    print "\nTesting " + url
+    try:
+        print get_sth_v2(url)
+    except:
+        print "GET STH Failed..."
 
 if __name__ == '__main__':
-    # for log in [CTLOGS[4]]:
-    #     url = log["url"]
-    #     try:
-    #         get_entries(url,8,8)
-    #     except Exception, e:
-    #         print "Failed to get entry from " + log["name"], e
-    log = CTLOGS[9]
-    entries = get_entries(log["url"],0,5000)["entries"]
-    print log["name"], len(entries)
-    # check_inclusion_by_submission(1,1,[CTLOGS[3]],[CTLOGS[3]])  
+
+    
+
+
+
+
+    # Data gathering for Niklas
+    if False:
+        log = CTLOGS[0]
+        sth = get_sth(log["url"])
+        # size = sth["tree_size"]
+        # for i in range(15,200):
+        start = 5757748
+        end = 5757847
+        print "Getting " + str(start) + " to " + str(end)
+        entries = get_entries(log["url"],start ,end)["entries"]
+
+        # TODO set filename
+        filename = "ct_log_content.txt"
+        # TODO remove file if exists
+        if os.path.exists(filename):
+            os.remove(filename)
+        # TODO open file
+        with open(filename, 'a') as f:
+        # TODO write lines
+            for i in range(len(entries)):
+                entry = entries[i]
+                res = check_domain_extended(entry)
+                string = parse_entry(res, i + start, log)
+                f.write(string + "\n")            
+
+
+
+
+
 
 
diff --git a/monitor/josef_lib.py b/monitor/josef_lib.py
index 922636c..f401b2c 100644
--- a/monitor/josef_lib.py
+++ b/monitor/josef_lib.py
@@ -84,6 +84,21 @@ def check_domain(raw_entry, log=None):
     except IndexError:
         return None
 
+def check_domain_extended(raw_entry, log=None):
+    orig_entry = extract_original_entry(raw_entry)
+    try:
+        cert_info = my_get_more_cert_info(orig_entry[0][0])
+        # print len(orig_entry[0])
+        cert_info["chain_length"] = str(len(orig_entry[0]))
+        cert_info["validation"] = get_validation_type(cert_info["policy"])
+        cert_info["in_mozilla"] = validate_cert(orig_entry[0][-1])
+        # print my_get_all_cert_info(orig_entry[0][-1])
+        if log:
+            cert_info["log"] = log[8:-1] # strip generic URL stuff
+        return cert_info
+    except IndexError:
+        return None
+
 def check_domain_all(raw_entry, log=None):
     orig_entry = extract_original_entry(raw_entry)
     try:
@@ -138,6 +153,26 @@ def append_file(fn, content):
             except:
                 pass
 
+def validate_cert(s):
+    p = subprocess.Popen(
+        ["openssl", "x509", "-inform", "DER"],
+        stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE)
+    converted = p.communicate(s)
+    # print converted
+    p = subprocess.Popen(
+        ["openssl", "verify", "-x509_strict", "-CAfile", "certdata.txt"],
+        stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE)
+    parsed = p.communicate(converted[0])
+    # print parsed
+    res = parsed[0][7:-1]
+    if res == "OK":
+        return "OK"
+    else:
+        return res.split("\n")[-2]
+
+
 def get_cert_info(s):
     p = subprocess.Popen(
         ["openssl", "x509", "-noout", "-subject", "-issuer", "-inform", "der"],
@@ -181,6 +216,95 @@ def my_get_cert_info(s):
         prev = line
     return result
 
+def my_get_more_cert_info(s):
+    p = subprocess.Popen(
+        ["openssl", "x509", "-fingerprint", "-text", "-noout", "-inform", "der"],
+        stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE)
+    parsed = p.communicate(s)
+    if parsed[1]:
+        print "ERROR:", parsed[1]
+        # sys.exit(1)
+        raise Exception
+    result = {}
+    result["policy"] = []
+    prev = ""
+    for line in parsed[0].split("\n"):
+        if "Subject:" in line:
+            result["subject"] = line.split("Subject: ")[1]
+        if "Issuer:" in line:
+            result["issuer"] = line.split("Issuer: ")[1]
+        if "Signature Algorithm:" in line:
+            result["sig_algorithm"] = line.split("Signature Algorithm: ")[1]
+        if "Public Key Algorithm:" in line:
+            result["pubkey_algorithm"] = line.split("Public Key Algorithm: ")[1]
+        if "Subject Alternative Name" in prev:
+            result["SAN"] = line.lstrip()
+        if "Not After" in line:
+            result["not_after"] = line.split(": ")[1]
+        if "Not Before" in line:
+            result["not_before"] = line.split(": ")[1]
+        if "Policy:" in line:
+            result["policy"].append(line.split("Policy: ")[1])
+        prev = line
+    return result
+
+def get_validation_type(policy_list):
+    DV_list = ["2.23.140.1.2.1"]
+    OV_list = ["2.23.140.1.2.2"]
+    EV_list = [
+    "1.3.159.1.17.1",
+    "1.3.6.1.4.1.34697.2.1",
+    "1.3.6.1.4.1.34697.2.2",
+    "1.3.6.1.4.1.34697.2.3",
+    "1.3.6.1.4.1.34697.2.4",
+    "1.2.40.0.17.1.22",
+    "2.16.578.1.26.1.3.3",
+    "1.3.6.1.4.1.17326.10.14.2.1.2",
+    "1.3.6.1.4.1.17326.10.8.12.1.2",
+    "1.3.6.1.4.1.6449.1.2.1.5.1",
+    "2.16.840.1.114412.2.1",
+    "2.16.840.1.114412.1.3.0.2",
+    "2.16.528.1.1001.1.1.1.12.6.1.1.1",
+    "2.16.792.3.0.4.1.1.4",
+    "2.16.840.1.114028.10.1.2",
+    "0.4.0.2042.1.4",
+    "0.4.0.2042.1.5",
+    "1.3.6.1.4.1.13177.10.1.3.10",
+    "1.3.6.1.4.1.14370.1.6",
+    "1.3.6.1.4.1.4146.1.1",
+    "2.16.840.1.114413.1.7.23.3",
+    "1.3.6.1.4.1.14777.6.1.1",
+    "2.16.792.1.2.1.1.5.7.1.9",
+    "1.3.6.1.4.1.22234.2.5.2.3.1",
+    "1.3.6.1.4.1.782.1.2.1.8.1",
+    "1.3.6.1.4.1.8024.0.2.100.1.2",
+    "1.2.392.200091.100.721.1",
+    "2.16.840.1.114414.1.7.23.3",
+    "1.3.6.1.4.1.23223.2",
+    "1.3.6.1.4.1.23223.1.1.1",
+    "2.16.756.1.83.21.0",
+    "2.16.756.1.89.1.2.1.1",
+    "2.16.840.1.113733.1.7.48.1",
+    "2.16.840.1.114404.1.1.2.4.1",
+    "2.16.840.1.113733.1.7.23.6",
+    "1.3.6.1.4.1.6334.1.100.1",
+    "2.16.840.1.114171.500.9",
+    "1.3.6.1.4.1.36305.2"
+]
+
+    status = "DV"
+
+    for policy in policy_list:
+        if policy in EV_list:
+            status = "EV"
+        if policy in OV_list:
+            status = "OV"
+        if policy in DV_list:
+            status = "DV"
+    return status
+
+
 def my_get_all_cert_info(s):
     p = subprocess.Popen(
         ["openssl", "x509", "-fingerprint", "-text", "-noout", "-inform", "der"],
diff --git a/monitor/josef_lib2.py b/monitor/josef_lib2.py
new file mode 100644
index 0000000..5af6490
--- /dev/null
+++ b/monitor/josef_lib2.py
@@ -0,0 +1,34 @@
+# The intent is that this lib will eventually contain code for using the CT v2 API
+
+import urllib2
+import json
+
+
+def get_sth_v1(baseurl):
+    result = urllib2.urlopen(baseurl + "ct/v1/get-sth").read()
+    return json.loads(result)
+
+def get_sth_v2(baseurl):
+    result = urllib2.urlopen(baseurl + "ct/v2/get-sth").read()
+    return json.loads(result)
+
+def add_chain_v2(baseurl):
+    pass
+
+def add_prechain_v2(baseurl):
+    pass
+
+def get_consistency_v2(baseurl):
+    pass
+
+def get_proof_by_hash_v2(baseurl):
+    pass
+
+def get_all_by_hash_v2(baseurl):
+    pass
+
+def get_entries_v2(baseurl):
+    pass
+
+def get_roots_v2(baseurl):
+    pass
diff --git a/monitor/josef_mover.py b/monitor/josef_mover.py
index 2267d00..3a5ae42 100755
--- a/monitor/josef_mover.py
+++ b/monitor/josef_mover.py
@@ -51,7 +51,7 @@ def print_reply(rep, entry):
 def is_new_timestamp(ts):
     MAX_TIMEDIFF = 300 # 5 min, allows for some clock skew
     ts_time = datetime.datetime.fromtimestamp(ts / 1000, UTC()).strftime('%Y-%m-%d %H:%M:%S')
-    start_time = datetime.datetime.utcnow().strftime('2015-10-19 00:00:00')
+    start_time = datetime.datetime.utcnow().strftime('2015-11-02 00:00:00')
     # delta_time = datetime.datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S') - datetime.datetime.strptime(ts_time, '%Y-%m-%d %H:%M:%S')
     # print delta_time.seconds
     if ts_time < start_time:
@@ -70,7 +70,7 @@ def check_inclusion_all(first, last, source, dest):
 
         for e in entries:
             inclusions = []
-            print base64.b64decode(e["leaf_input"])
+            # print base64.b64decode(e["leaf_input"])
             h = get_leaf_hash(base64.b64decode(e["leaf_input"]))
             for log in dest:
                 url = log["url"]
@@ -190,14 +190,15 @@ def check_submission_inner(d_log, item, inclusions):
         res = add_prechain(d_log["url"], {"chain" : submission})
     else:
         res = add_chain(d_log["url"], {"chain" : submission})
+    # print res
     if not is_new_timestamp(res["timestamp"]):
         inclusions.append(d_log["name"])
 
 def check_submission_outer(first, last, s_log, dest, logfile=None):
     MAX_CHUNK_SIZE = 65
-    MAX_RETRIES = 3
+    MAX_RETRIES = 1
     idx = 0
-    while first + idx < last:
+    while first + idx <= last:
         if first + idx + MAX_CHUNK_SIZE < last:
             tmp_last = first + idx + MAX_CHUNK_SIZE - 1
         else:
@@ -207,6 +208,7 @@ def check_submission_outer(first, last, s_log, dest, logfile=None):
         while retries <= MAX_RETRIES:
             try:
                 print "Getting " + str(first + idx) + " to " + str(tmp_last)
+                # print s_log
                 entries = get_entries(s_log["url"], first + idx, tmp_last)["entries"]
                 break
             except Exception, e:
@@ -296,11 +298,12 @@ def log(fn, string):
 
 
 if __name__ == "__main__":
-    source = [CTLOGS[4]]
+    source = CTLOGS[0]
     dests = CTLOGS
 
     # stress_test([CTLOGS[0]])    
-    check_overlap(source, dests)
+    # check_overlap(source, dests)
+    check_submission_outer(10243750,10243750,source, dests)