From 42760f059aff98fe64d304e2d50639ab02f79886 Mon Sep 17 00:00:00 2001 From: Josef Gustafsson Date: Wed, 7 Oct 2015 10:40:07 +0200 Subject: adding error handling in reader --- monitor/josef_reader.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) (limited to 'monitor') diff --git a/monitor/josef_reader.py b/monitor/josef_reader.py index 7974221..54f586a 100755 --- a/monitor/josef_reader.py +++ b/monitor/josef_reader.py @@ -16,7 +16,7 @@ except: print "No database support found." from datetime import datetime as dt import ast -from monitor_conf import DB_PATH, MONITORED_DOMAINS, OUTPUT_DIR +from monitor_conf import DB_PATH, MONITORED_DOMAINS, OUTPUT_DIR, ERROR_STR @@ -164,19 +164,22 @@ def db_monitor_domain(domain, log=None, exclude_invalid=None, get_cert=None, iss # Print matching if success: - me = monitored_entry(entry["subject"].split("CN=")[1], \ - entry["issuer"].split("CN=")[1], \ - entry["log"], \ - valid, \ - entry["leaf_hash"]) - # print str(me) - - if get_cert: - print get_full_cert(entry) - if "index" in entry: - print "INDEX:", entry["index"] - if me not in res: - res.append(me) + try: + me = monitored_entry(entry["subject"].split("CN=")[1], \ + entry["issuer"].split("CN=")[1], \ + entry["log"], \ + valid, \ + entry["leaf_hash"]) + # print str(me) + + if get_cert: + print get_full_cert(entry) + if "index" in entry: + print "INDEX:", entry["index"] + if me not in res: + res.append(me) + except: + print ERROR_STR + "READER failed to create monitored_entry" # print str(count_all) + " matches found. " \ -- cgit v1.1 From 77c7e3f4d9a444092768ab3622e2ef829ea13efb Mon Sep 17 00:00:00 2001 From: Josef Gustafsson Date: Wed, 7 Oct 2015 11:02:47 +0200 Subject: proper handling of issuers and subjects in reader --- monitor/josef_reader.py | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) (limited to 'monitor') diff --git a/monitor/josef_reader.py b/monitor/josef_reader.py index 54f586a..5fb565a 100755 --- a/monitor/josef_reader.py +++ b/monitor/josef_reader.py @@ -165,21 +165,33 @@ def db_monitor_domain(domain, log=None, exclude_invalid=None, get_cert=None, iss # Print matching if success: try: - me = monitored_entry(entry["subject"].split("CN=")[1], \ - entry["issuer"].split("CN=")[1], \ - entry["log"], \ - valid, \ - entry["leaf_hash"]) - # print str(me) - - if get_cert: - print get_full_cert(entry) - if "index" in entry: - print "INDEX:", entry["index"] - if me not in res: - res.append(me) + tmp_issuer = entry["issuer"].split("CN=")[1] except: - print ERROR_STR + "READER failed to create monitored_entry" + if "issuer" in entry: + tmp_issuer = entry["issuer"] + else: + tmp_issuer = "No Issuer Found" + + try: + tmp_subject = entry["subject"].split("CN=")[1] + except: + if "subject" in entry: + tmp_subject = entry["subject"] + else: + tmp_subject = "No Subject Found" + + me = monitored_entry(tmp_subject, \ + tmp_issuer, \ + entry["log"], \ + valid, \ + entry["leaf_hash"]) + + if get_cert: + print get_full_cert(entry) + if "index" in entry: + print "INDEX:", entry["index"] + if me not in res: + res.append(me) # print str(count_all) + " matches found. " \ -- cgit v1.1 From 784919377eeb841fb330d95df940262eea7bae8b Mon Sep 17 00:00:00 2001 From: Josef Gustafsson Date: Thu, 8 Oct 2015 17:24:03 +0200 Subject: cleaning --- monitor/josef_experimental.py | 17 +++++++++++------ monitor/josef_monitor.py | 11 ----------- 2 files changed, 11 insertions(+), 17 deletions(-) (limited to 'monitor') diff --git a/monitor/josef_experimental.py b/monitor/josef_experimental.py index 579331b..9c141bc 100755 --- a/monitor/josef_experimental.py +++ b/monitor/josef_experimental.py @@ -72,17 +72,22 @@ if __name__ == '__main__': # entry = get_entries(ctlogs[0]["url"], 1,1)["entries"] # print extract_original_entry(entry[0]) - for url in [CTLOGS[0]["url"],CTLOGS[5]["url"],CTLOGS[6]["url"]]: + for url in [CTLOGS[6]["url"]]: + # for url in [CTLOGS[0]["url"],CTLOGS[5]["url"],CTLOGS[6]["url"]]: # for log in CTLOGS: # url = log["url"] # url = CTLOGS[1]["url"] + entry = get_entries(url, 3638637,3638637)["entries"][0] + # raw_entry = get_entries(log, leaf_index, leaf_index)["entries"][0] + cert = check_domain_all(entry) + print cert # result = urlopen(url + "ct/v1/get-sth").info() # print get_sth(url) - import urllib2, socket, urlparse - # set up your request as before, then: - data = urllib2.urlopen(url+ "ct/v1/get-sth") - addr = socket.gethostbyname(urlparse.urlparse(data.geturl()).hostname) - print addr, data.fp._sock.fp._sock.getpeername()[0] + # import urllib2, socket, urlparse + # # set up your request as before, then: + # data = urllib2.urlopen(url+ "ct/v1/get-sth") + # addr = socket.gethostbyname(urlparse.urlparse(data.geturl()).hostname) + # print addr, data.fp._sock.fp._sock.getpeername()[0] # data = urllib2.urlopen(request) diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py index 86b6852..3896493 100755 --- a/monitor/josef_monitor.py +++ b/monitor/josef_monitor.py @@ -84,7 +84,6 @@ class ctlog: try: self.subtree, self.entries = self.fetch_and_increment_subtree(self.entries, self.sth["tree_size"] -1, self.url, self.subtree) except Exception, e: - # print ERROR_STR + "Failed fetch and increment for " + self.name self.log(ERROR_STR + "Failed fetch and increment tree. Current Size: " + str(self.entries) + " Sth: " + str(self.sth) + " Error: " + str(e)) self.rollback() return @@ -192,7 +191,6 @@ class ctlog: self.log(ERROR_STR + "Could not verify STH signature " + str(new_sth)) self.rollback() - # sth_time = datetime.datetime.fromtimestamp(new_sth['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S") sth_time = time_str(new_sth["timestamp"]) if new_sth["timestamp"] != self.sth["timestamp"]: self.log("STH updated. Size: " + str(new_sth["tree_size"]) + ", Time: " + sth_time) @@ -260,22 +258,17 @@ class ctlog: try: if new["tree_size"] == old["tree_size"]: if old["sha256_root_hash"] != new["sha256_root_hash"]: - # print ERROR_STR + "Root hash is different for same tree size in " + self.name self.log(ERROR_STR + "New root hash for same tree size! Old:" + str(old) + " New:" + str(new)) self.rollback() elif new["tree_size"] < old["tree_size"]: - # print ERROR_STR + "New tree smaller than previous tree (%d < %d) in %s" % \ - # (new["tree_size"], old["tree_size"], self.name) self.log(ERROR_STR + "New tree is smaller than old tree! Old:" + str(old) + " New:" + str(new)) self.rollback() if new["timestamp"] < old["timestamp"]: self.log(ERROR_STR + "Regression in timestamps! Old:" + str(old) + " New:" + str(new)) self.rollback() - # print ERROR_STR + " Regression in timestamps in " + self.name else: age = time.time() - new["timestamp"]/1000 - # sth_time = datetime.datetime.fromtimestamp(new['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S") sth_time = time_str(new["timestamp"]) roothash = new['sha256_root_hash'] if age > 24 * 3600: @@ -291,7 +284,6 @@ class ctlog: except Exception, e: self.log(ERROR_STR + "Failed to verify progress! Old:" + str(old) + " New:" + str(new) + " Exception: " + str(e)) self.rollback() - # print "Failed to verify progress in " + self.name def verify_consistency(self, old): new = self.sth @@ -307,17 +299,14 @@ class ctlog: self.log(ERROR_STR + "Verification of consistency for old hash failed! Old:" \ + str(old) + " New:" + str(new) + " Proof:" + str(consistency_proof)) self.rollback() - # print ERROR_STR + "Failed to verify consistency for " + self.name elif new["sha256_root_hash"] != str(base64.b64encode(res[1])): self.log(ERROR_STR + "Verification of consistency for new hash failed! Old:" \ + str(old) + " New:" + str(new) + " Proof:" + str(consistency_proof)) self.rollback() - # print ERROR_STR + "Failed to verify consistency for " + self.name except Exception, e: self.log(ERROR_STR + "Could not verify consistency! " + " Old:" + str(old) + " New:" + str(new) + " Error:" + str(e)) self.rollback() - # print ERROR_STR + "Could not verify consistency for " + self.url # def verify_inclusion_all(old, new): -- cgit v1.1 From e8762a413c95a2cbab15f4d776cfdb9fed5b255d Mon Sep 17 00:00:00 2001 From: Josef Gustafsson Date: Fri, 9 Oct 2015 09:47:07 +0200 Subject: testing... --- monitor/josef_experimental.py | 34 +++++++++++++++++++--------------- monitor/josef_leveldb.py | 4 ++++ monitor/josef_reader.py | 1 + 3 files changed, 24 insertions(+), 15 deletions(-) (limited to 'monitor') diff --git a/monitor/josef_experimental.py b/monitor/josef_experimental.py index 9c141bc..d64012d 100755 --- a/monitor/josef_experimental.py +++ b/monitor/josef_experimental.py @@ -70,6 +70,7 @@ def update_roots(log): if __name__ == '__main__': + dbdir = "tmpdb/" # entry = get_entries(ctlogs[0]["url"], 1,1)["entries"] # print extract_original_entry(entry[0]) for url in [CTLOGS[6]["url"]]: @@ -77,21 +78,24 @@ if __name__ == '__main__': # for log in CTLOGS: # url = log["url"] # url = CTLOGS[1]["url"] - entry = get_entries(url, 3638637,3638637)["entries"][0] - # raw_entry = get_entries(log, leaf_index, leaf_index)["entries"][0] - cert = check_domain_all(entry) - print cert - # result = urlopen(url + "ct/v1/get-sth").info() - # print get_sth(url) - # import urllib2, socket, urlparse - # # set up your request as before, then: - # data = urllib2.urlopen(url+ "ct/v1/get-sth") - # addr = socket.gethostbyname(urlparse.urlparse(data.geturl()).hostname) - # print addr, data.fp._sock.fp._sock.getpeername()[0] - - # data = urllib2.urlopen(request) - - # print addr + entries = get_entries(url, 3638637,3638637)["entries"] + # entries = get_entries(url, first, last)["entries"] + tmp_cert_data = [] + for item in entries: + tmp_data = check_domain(item, url) + entry_hash = get_leaf_hash(base64.b64decode(item["leaf_input"])) + if tmp_data: + tmp_data["leaf_hash"] = base64.b64encode(entry_hash) + tmp_cert_data.append(tmp_data) + print tmp_data + # new_leafs.append(entry_hash) + # if self.dbdir:/ + + # db_add_certs(dbdir, tmp_cert_data) + + # if CONFIG.DEFAULT_CERT_FILE: + # append_file(CONFIG.DEFAULT_CERT_FILE, tmp_cert_data) + # subtree = reduce_tree(new_leafs, subtree) diff --git a/monitor/josef_leveldb.py b/monitor/josef_leveldb.py index 49c5594..9e57e20 100755 --- a/monitor/josef_leveldb.py +++ b/monitor/josef_leveldb.py @@ -61,12 +61,14 @@ def db_append(db, key, val): pass else: tmpl.append(val) + print "Storing:",key,SEP.join(tmpl) db.Put(key,SEP.join(tmpl)) def db_add_domain_2(db_dir, key_in, val_in, db_in=None): if db_in == None: val = "###" + ast.literal_eval(val_in)["leaf_hash"] cert_db = db_open(db_dir + "certificates") + # print "Storing: ",val,val_in cert_db.Put(val, val_in) key_list = key_in.split('/')[0].split(',')[0].split('.') @@ -88,9 +90,11 @@ def db_add_domain_2(db_dir, key_in, val_in, db_in=None): return else: tmpl.append(val) + # print "Storing: ",key,SEP.join(tmpl) db.Put(key,SEP.join(tmpl)) except KeyError: + # print "Storing: ",key,SEP.join([val]) db.Put(key,SEP.join([val])) new_key = key.split('.',1)[-1] if new_key != key: diff --git a/monitor/josef_reader.py b/monitor/josef_reader.py index 5fb565a..82bdc61 100755 --- a/monitor/josef_reader.py +++ b/monitor/josef_reader.py @@ -106,6 +106,7 @@ class monitored_entry: def db_monitor_domain(domain, log=None, exclude_invalid=None, get_cert=None, issuer=None): # print domain raw = db_lookup_domain(DB_PATH, domain) + print raw cur_time = dt.now() count_valid = 0 -- cgit v1.1 From 3e3e73f189c73dc2c0ebaca7f1dcb5b06d98c6c9 Mon Sep 17 00:00:00 2001 From: Josef Gustafsson Date: Fri, 9 Oct 2015 09:51:00 +0200 Subject: testing... --- monitor/josef_leveldb.py | 2 +- monitor/josef_reader.py | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) (limited to 'monitor') diff --git a/monitor/josef_leveldb.py b/monitor/josef_leveldb.py index 9e57e20..ef6437e 100755 --- a/monitor/josef_leveldb.py +++ b/monitor/josef_leveldb.py @@ -61,7 +61,7 @@ def db_append(db, key, val): pass else: tmpl.append(val) - print "Storing:",key,SEP.join(tmpl) + # print "Storing:",key,SEP.join(tmpl) db.Put(key,SEP.join(tmpl)) def db_add_domain_2(db_dir, key_in, val_in, db_in=None): diff --git a/monitor/josef_reader.py b/monitor/josef_reader.py index 82bdc61..698aa6c 100755 --- a/monitor/josef_reader.py +++ b/monitor/josef_reader.py @@ -106,7 +106,6 @@ class monitored_entry: def db_monitor_domain(domain, log=None, exclude_invalid=None, get_cert=None, issuer=None): # print domain raw = db_lookup_domain(DB_PATH, domain) - print raw cur_time = dt.now() count_valid = 0 @@ -195,11 +194,11 @@ def db_monitor_domain(domain, log=None, exclude_invalid=None, get_cert=None, iss res.append(me) - # print str(count_all) + " matches found. " \ - # + str(count_valid) + " valid, " \ - # + str(count_expired) + " expired and " \ - # + str(count_not_yet_valid) + " not yet valid for " \ - # + domain + print str(count_all) + " matches found. " \ + + str(count_valid) + " valid, " \ + + str(count_expired) + " expired and " \ + + str(count_not_yet_valid) + " not yet valid for " \ + + domain return res if __name__ == "__main__": -- cgit v1.1 From 5ff68a6f8de8cd1500d82f17f11f8e9c03f399df Mon Sep 17 00:00:00 2001 From: Josef Gustafsson Date: Fri, 9 Oct 2015 09:51:40 +0200 Subject: done testing --- monitor/josef_reader.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'monitor') diff --git a/monitor/josef_reader.py b/monitor/josef_reader.py index 698aa6c..5fb565a 100755 --- a/monitor/josef_reader.py +++ b/monitor/josef_reader.py @@ -194,11 +194,11 @@ def db_monitor_domain(domain, log=None, exclude_invalid=None, get_cert=None, iss res.append(me) - print str(count_all) + " matches found. " \ - + str(count_valid) + " valid, " \ - + str(count_expired) + " expired and " \ - + str(count_not_yet_valid) + " not yet valid for " \ - + domain + # print str(count_all) + " matches found. " \ + # + str(count_valid) + " valid, " \ + # + str(count_expired) + " expired and " \ + # + str(count_not_yet_valid) + " not yet valid for " \ + # + domain return res if __name__ == "__main__": -- cgit v1.1 From 1b1fd3523c5f99981e4e89aa05fa3c9a16a45754 Mon Sep 17 00:00:00 2001 From: Josef Gustafsson Date: Fri, 9 Oct 2015 10:28:25 +0200 Subject: adding wosign log --- monitor/monitor_conf.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'monitor') diff --git a/monitor/monitor_conf.py b/monitor/monitor_conf.py index 5a3737c..674c565 100644 --- a/monitor/monitor_conf.py +++ b/monitor/monitor_conf.py @@ -85,6 +85,12 @@ CTLOGS = [ "id" : "rDua7X+pZ0dXFZ5tfVdWcvnZgQCUHpve/+yhMTt1eC0=", "build" : True}, + {"name" : "wosign", + "url" : "https://ct.wosign.com/", + "key" : "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1+wvK3VPN7yjQ7qLZWY8fWrlDCqmwuUm/gx9TnzwOrzi0yLcAdAfbkOcXG6DrZwV9sSNYLUdu6NiaX7rp6oBmw==", + "id" : "nk/3PcPOIgtpIXyJnkaAdqv414Y21cz8haMadWKLqIs=", + "build" : True}, + ] -- cgit v1.1