diff options
| -rwxr-xr-x | monitor/josef_experimental.py | 71 | ||||
| -rw-r--r-- | monitor/josef_lib.py | 124 | ||||
| -rw-r--r-- | monitor/josef_lib2.py | 34 | ||||
| -rwxr-xr-x | monitor/josef_mover.py | 15 |
4 files changed, 228 insertions, 16 deletions
diff --git a/monitor/josef_experimental.py b/monitor/josef_experimental.py index 979bae8..5df124b 100755 --- a/monitor/josef_experimental.py +++ b/monitor/josef_experimental.py @@ -4,6 +4,7 @@ import sys import os from josef_lib import * +from josef_lib2 import * # import leveldb import argparse import json @@ -126,17 +127,67 @@ def update_roots(log): open(tempname, 'w').write(data) mv_file(tempname, fn) +def parse_entry(e, idx, log): + # print the following fields, separated by sep + sep = ";" + + s = log["name"] + s += sep + str(idx) # index + s += sep + e["subject"] # Subject + s += sep + e["SAN"] # SAN + s += sep + e["issuer"] # issuer + s += sep + e["chain_length"] # path length + s += sep + e["sig_algorithm"] # Signature algothithm + s += sep + e["pubkey_algorithm"] # pubkey algorithm + s += sep + e["not_before"] # valid from + s += sep + e["not_after"] # valid to + s += sep + e["validation"] # EV? + s += sep + e["in_mozilla"] # chains to mozilla root? + + return s + +def check_api2(url): + print "\nTesting " + url + try: + print get_sth_v2(url) + except: + print "GET STH Failed..." if __name__ == '__main__': - # for log in [CTLOGS[4]]: - # url = log["url"] - # try: - # get_entries(url,8,8) - # except Exception, e: - # print "Failed to get entry from " + log["name"], e - log = CTLOGS[9] - entries = get_entries(log["url"],0,5000)["entries"] - print log["name"], len(entries) - # check_inclusion_by_submission(1,1,[CTLOGS[3]],[CTLOGS[3]]) + + + + + + + # Data gathering for Niklas + if False: + log = CTLOGS[0] + sth = get_sth(log["url"]) + # size = sth["tree_size"] + # for i in range(15,200): + start = 5757748 + end = 5757847 + print "Getting " + str(start) + " to " + str(end) + entries = get_entries(log["url"],start ,end)["entries"] + + # TODO set filename + filename = "ct_log_content.txt" + # TODO remove file if exists + if os.path.exists(filename): + os.remove(filename) + # TODO open file + with open(filename, 'a') as f: + # TODO write lines + for i in range(len(entries)): + entry = entries[i] + res = check_domain_extended(entry) + string = parse_entry(res, i + start, log) + f.write(string + "\n") + + + + + diff --git a/monitor/josef_lib.py b/monitor/josef_lib.py index 922636c..f401b2c 100644 --- a/monitor/josef_lib.py +++ b/monitor/josef_lib.py @@ -84,6 +84,21 @@ def check_domain(raw_entry, log=None): except IndexError: return None +def check_domain_extended(raw_entry, log=None): + orig_entry = extract_original_entry(raw_entry) + try: + cert_info = my_get_more_cert_info(orig_entry[0][0]) + # print len(orig_entry[0]) + cert_info["chain_length"] = str(len(orig_entry[0])) + cert_info["validation"] = get_validation_type(cert_info["policy"]) + cert_info["in_mozilla"] = validate_cert(orig_entry[0][-1]) + # print my_get_all_cert_info(orig_entry[0][-1]) + if log: + cert_info["log"] = log[8:-1] # strip generic URL stuff + return cert_info + except IndexError: + return None + def check_domain_all(raw_entry, log=None): orig_entry = extract_original_entry(raw_entry) try: @@ -138,6 +153,26 @@ def append_file(fn, content): except: pass +def validate_cert(s): + p = subprocess.Popen( + ["openssl", "x509", "-inform", "DER"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + converted = p.communicate(s) + # print converted + p = subprocess.Popen( + ["openssl", "verify", "-x509_strict", "-CAfile", "certdata.txt"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + parsed = p.communicate(converted[0]) + # print parsed + res = parsed[0][7:-1] + if res == "OK": + return "OK" + else: + return res.split("\n")[-2] + + def get_cert_info(s): p = subprocess.Popen( ["openssl", "x509", "-noout", "-subject", "-issuer", "-inform", "der"], @@ -181,6 +216,95 @@ def my_get_cert_info(s): prev = line return result +def my_get_more_cert_info(s): + p = subprocess.Popen( + ["openssl", "x509", "-fingerprint", "-text", "-noout", "-inform", "der"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + parsed = p.communicate(s) + if parsed[1]: + print "ERROR:", parsed[1] + # sys.exit(1) + raise Exception + result = {} + result["policy"] = [] + prev = "" + for line in parsed[0].split("\n"): + if "Subject:" in line: + result["subject"] = line.split("Subject: ")[1] + if "Issuer:" in line: + result["issuer"] = line.split("Issuer: ")[1] + if "Signature Algorithm:" in line: + result["sig_algorithm"] = line.split("Signature Algorithm: ")[1] + if "Public Key Algorithm:" in line: + result["pubkey_algorithm"] = line.split("Public Key Algorithm: ")[1] + if "Subject Alternative Name" in prev: + result["SAN"] = line.lstrip() + if "Not After" in line: + result["not_after"] = line.split(": ")[1] + if "Not Before" in line: + result["not_before"] = line.split(": ")[1] + if "Policy:" in line: + result["policy"].append(line.split("Policy: ")[1]) + prev = line + return result + +def get_validation_type(policy_list): + DV_list = ["2.23.140.1.2.1"] + OV_list = ["2.23.140.1.2.2"] + EV_list = [ + "1.3.159.1.17.1", + "1.3.6.1.4.1.34697.2.1", + "1.3.6.1.4.1.34697.2.2", + "1.3.6.1.4.1.34697.2.3", + "1.3.6.1.4.1.34697.2.4", + "1.2.40.0.17.1.22", + "2.16.578.1.26.1.3.3", + "1.3.6.1.4.1.17326.10.14.2.1.2", + "1.3.6.1.4.1.17326.10.8.12.1.2", + "1.3.6.1.4.1.6449.1.2.1.5.1", + "2.16.840.1.114412.2.1", + "2.16.840.1.114412.1.3.0.2", + "2.16.528.1.1001.1.1.1.12.6.1.1.1", + "2.16.792.3.0.4.1.1.4", + "2.16.840.1.114028.10.1.2", + "0.4.0.2042.1.4", + "0.4.0.2042.1.5", + "1.3.6.1.4.1.13177.10.1.3.10", + "1.3.6.1.4.1.14370.1.6", + "1.3.6.1.4.1.4146.1.1", + "2.16.840.1.114413.1.7.23.3", + "1.3.6.1.4.1.14777.6.1.1", + "2.16.792.1.2.1.1.5.7.1.9", + "1.3.6.1.4.1.22234.2.5.2.3.1", + "1.3.6.1.4.1.782.1.2.1.8.1", + "1.3.6.1.4.1.8024.0.2.100.1.2", + "1.2.392.200091.100.721.1", + "2.16.840.1.114414.1.7.23.3", + "1.3.6.1.4.1.23223.2", + "1.3.6.1.4.1.23223.1.1.1", + "2.16.756.1.83.21.0", + "2.16.756.1.89.1.2.1.1", + "2.16.840.1.113733.1.7.48.1", + "2.16.840.1.114404.1.1.2.4.1", + "2.16.840.1.113733.1.7.23.6", + "1.3.6.1.4.1.6334.1.100.1", + "2.16.840.1.114171.500.9", + "1.3.6.1.4.1.36305.2" +] + + status = "DV" + + for policy in policy_list: + if policy in EV_list: + status = "EV" + if policy in OV_list: + status = "OV" + if policy in DV_list: + status = "DV" + return status + + def my_get_all_cert_info(s): p = subprocess.Popen( ["openssl", "x509", "-fingerprint", "-text", "-noout", "-inform", "der"], diff --git a/monitor/josef_lib2.py b/monitor/josef_lib2.py new file mode 100644 index 0000000..5af6490 --- /dev/null +++ b/monitor/josef_lib2.py @@ -0,0 +1,34 @@ +# The intent is that this lib will eventually contain code for using the CT v2 API + +import urllib2 +import json + + +def get_sth_v1(baseurl): + result = urllib2.urlopen(baseurl + "ct/v1/get-sth").read() + return json.loads(result) + +def get_sth_v2(baseurl): + result = urllib2.urlopen(baseurl + "ct/v2/get-sth").read() + return json.loads(result) + +def add_chain_v2(baseurl): + pass + +def add_prechain_v2(baseurl): + pass + +def get_consistency_v2(baseurl): + pass + +def get_proof_by_hash_v2(baseurl): + pass + +def get_all_by_hash_v2(baseurl): + pass + +def get_entries_v2(baseurl): + pass + +def get_roots_v2(baseurl): + pass diff --git a/monitor/josef_mover.py b/monitor/josef_mover.py index 2267d00..3a5ae42 100755 --- a/monitor/josef_mover.py +++ b/monitor/josef_mover.py @@ -51,7 +51,7 @@ def print_reply(rep, entry): def is_new_timestamp(ts): MAX_TIMEDIFF = 300 # 5 min, allows for some clock skew ts_time = datetime.datetime.fromtimestamp(ts / 1000, UTC()).strftime('%Y-%m-%d %H:%M:%S') - start_time = datetime.datetime.utcnow().strftime('2015-10-19 00:00:00') + start_time = datetime.datetime.utcnow().strftime('2015-11-02 00:00:00') # delta_time = datetime.datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S') - datetime.datetime.strptime(ts_time, '%Y-%m-%d %H:%M:%S') # print delta_time.seconds if ts_time < start_time: @@ -70,7 +70,7 @@ def check_inclusion_all(first, last, source, dest): for e in entries: inclusions = [] - print base64.b64decode(e["leaf_input"]) + # print base64.b64decode(e["leaf_input"]) h = get_leaf_hash(base64.b64decode(e["leaf_input"])) for log in dest: url = log["url"] @@ -190,14 +190,15 @@ def check_submission_inner(d_log, item, inclusions): res = add_prechain(d_log["url"], {"chain" : submission}) else: res = add_chain(d_log["url"], {"chain" : submission}) + # print res if not is_new_timestamp(res["timestamp"]): inclusions.append(d_log["name"]) def check_submission_outer(first, last, s_log, dest, logfile=None): MAX_CHUNK_SIZE = 65 - MAX_RETRIES = 3 + MAX_RETRIES = 1 idx = 0 - while first + idx < last: + while first + idx <= last: if first + idx + MAX_CHUNK_SIZE < last: tmp_last = first + idx + MAX_CHUNK_SIZE - 1 else: @@ -207,6 +208,7 @@ def check_submission_outer(first, last, s_log, dest, logfile=None): while retries <= MAX_RETRIES: try: print "Getting " + str(first + idx) + " to " + str(tmp_last) + # print s_log entries = get_entries(s_log["url"], first + idx, tmp_last)["entries"] break except Exception, e: @@ -296,11 +298,12 @@ def log(fn, string): if __name__ == "__main__": - source = [CTLOGS[4]] + source = CTLOGS[0] dests = CTLOGS # stress_test([CTLOGS[0]]) - check_overlap(source, dests) + # check_overlap(source, dests) + check_submission_outer(10243750,10243750,source, dests) |