diff options
| -rwxr-xr-x | monitor/josef_leveldb.py | 13 | ||||
| -rwxr-xr-x | monitor/josef_monitor.py | 44 | ||||
| -rwxr-xr-x | monitor/josef_reader.py | 46 | ||||
| -rw-r--r-- | monitor/monitor_conf.py | 7 |
4 files changed, 47 insertions, 63 deletions
diff --git a/monitor/josef_leveldb.py b/monitor/josef_leveldb.py index 9994609..9c6f77d 100755 --- a/monitor/josef_leveldb.py +++ b/monitor/josef_leveldb.py @@ -7,8 +7,6 @@ import leveldb import ast SEP = ";" -# db = None - dbs = {} @@ -41,7 +39,6 @@ def db_open(fn='./cert_db'): if fn in dbs: return dbs[fn] else: - # print "Opening " + fn db = leveldb.LevelDB(fn) dbs[fn] = db return db @@ -68,9 +65,7 @@ def db_append(db, key, val): def db_add_domain_2(db_dir, key_in, val_in, db_in=None): if db_in == None: - # print val_in val = "###" + ast.literal_eval(val_in)["leaf_hash"] - # val = str(hash(val_in)) cert_db = db_open(db_dir + "certificates") cert_db.Put(val, val_in) @@ -119,7 +114,7 @@ def db_add_certs(db_dir, data): pass except IndexError: pass - db_close() + # db_close() def db_lookup_domain(db_dir, domain): @@ -135,7 +130,6 @@ def db_lookup_domain(db_dir, domain): db = db_open(db_dir + "other") cur_domain = domain_list.pop() - # print cur_domain try: intermediate = db.Get(cur_domain).split(SEP) except KeyError: @@ -156,13 +150,11 @@ def db_lookup_domain(db_dir, domain): # Prune next_level = [] for item in intermediate: - # print item if match_domain(cur_domain, item): - # print item try: tmp = db.Get(item).split(SEP) - # print tmp for tmp_item in tmp: + # This is really not a good solution... if tmp_item[0:3] == "###": res.append(tmp_item[3:]) else: @@ -173,7 +165,6 @@ def db_lookup_domain(db_dir, domain): certs = [] cert_db = db_open(db_dir + "certificates") - # print res for item in res: tmp = cert_db.Get("###" + str(item)) certs.append(tmp) diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py index 0569a81..83e3663 100755 --- a/monitor/josef_monitor.py +++ b/monitor/josef_monitor.py @@ -30,7 +30,6 @@ if not os.path.exists(OUTPUT_DIR): if not os.path.exists(DB_PATH): os.makedirs(DB_PATH) -# DB = None parser = argparse.ArgumentParser(description="") @@ -51,7 +50,6 @@ class ctlog: def incremental_build(self): # Keeps state current during build, partial builds are possible. self.sth = get_sth(self.url) - # self.log("Building....") start_size = self.entries while self.entries < self.sth["tree_size"]: tmp_size = self.entries @@ -66,8 +64,6 @@ class ctlog: " new entries. Size: " + str(self.entries)) else: self.log("ERROR Failed to build tree from entries.") - # else: - # self.log("No new entries.") def fetch_and_increment_subtree(self, first, last, url, subtree =[[]]): @@ -237,8 +233,6 @@ def check_domain(raw_entry, log=None): except IndexError: return None - - def verify_subtree(sth, subtree, base_url): try: tmp = deepcopy(subtree) @@ -320,11 +314,7 @@ def get_all_roots(base_url): root_cert = base64.decodestring(accepted_cert) print get_cert_info(root_cert)["subject"] -def main(args): - global DB - # Create logs - logs = [] - +def setup_domain_monitoring(): monitored_domains = [] with open(DOMAINS_FILE) as fp: for line in fp: @@ -332,13 +322,23 @@ def main(args): for domain in tmp: md = monitored_domain(domain["url"]) md.load_entries(domain["entries"]) + monitored_domains.append(md) + for md in MONITORED_DOMAINS: tmp = monitored_domain(md) if not tmp in monitored_domains: + print "New domain (not in file) " + md tmp.set() monitored_domains.append(tmp) + return monitored_domains + +def main(args): + global DB + monitored_domains = setup_domain_monitoring() + # Create logs + logs = [] try: for item in ctlogs: logs.append(ctlog(item, ctlogs[item][0], ctlogs[item][1])) @@ -368,27 +368,29 @@ def main(args): for md in monitored_domains: md.update() - + # Normal exit oof the program except KeyboardInterrupt: print time.strftime('%H:%M:%S') + ' Received interrupt from user. Saving and exiting....' for log in logs: log.save() + # Save info about monitored domains domain_dict = [] for md in monitored_domains: domain_dict.append(md.to_dict()) open(DOMAINS_FILE, 'w').write(json.dumps(domain_dict)) - # except Exception, err: - # print Exception, err - # for log in logs: - # log.save() - - # domain_dict = [] - # for md in monitored_domains: - # domain_dict.append(md.to_dict()) + # Something went horribly wrong! + except Exception, err: + print Exception, err + for log in logs: + log.save() - # open(DOMAINS_FILE, 'w').write(json.dumps(domain_dict)) + # Save info about monitored domains + domain_dict = [] + for md in monitored_domains: + domain_dict.append(md.to_dict()) + open(DOMAINS_FILE, 'w').write(json.dumps(domain_dict)) diff --git a/monitor/josef_reader.py b/monitor/josef_reader.py index 199fe29..8ffbd36 100755 --- a/monitor/josef_reader.py +++ b/monitor/josef_reader.py @@ -31,21 +31,14 @@ class monitored_domain: f.write(s + "\n") f.close() - # def add(self, item): - # self.entries.appent(item) - def set(self): self.entries = db_monitor_domain(self.url, None, True, None) self.log("Got " + str(len(self.entries)) + " certs for " + self.url) def update(self): new = db_monitor_domain(self.url, None, True, None) - # if len(new) != len(self.entries): - # print self.entries - # print new - compare_entry_lists(new, self.entries) + self.compare_and_log_entries(new, self.entries) self.entries = new - def to_dict(self): d = {} @@ -62,7 +55,7 @@ class monitored_domain: entries.append(monitored_entry(item["subject"],item["issuer"],item["log"],item["status"],item["leaf_hash"])) self.entries = entries - def compare_entry_lists(self, new, old): + def compare_and_log_entries(self, new, old): added_items = [] removed_items = [] @@ -94,8 +87,8 @@ class monitored_entry: self.leaf_hash = leaf_hash def __eq__(self, other): - return self.leaf_hash == other.leaf_hash - # TODO check status (or expired wont expire) + return self.leaf_hash == other.leaf_hash + # return self.leaf_hash == other.leaf_hash and self.status == other.status def __str__(self): s = self.subject + \ @@ -117,25 +110,9 @@ class monitored_entry: -parser = argparse.ArgumentParser(description="") -parser.add_argument('--domain', default=None) -parser.add_argument('--log', default=None) -parser.add_argument('--exclude-invalid', action='store_true') -parser.add_argument('--get-cert', action='store_true') - -args = parser.parse_args() - -monitored_domains = [] -for md in MONITORED_DOMAINS: - monitored_domains.append(monitored_domain(md)) - -db = DB_PATH - - - def db_monitor_domain(domain, log=None, exclude_invalid=None, get_cert=None): # print domain - raw = db_lookup_domain(db, domain) + raw = db_lookup_domain(DB_PATH, domain) cur_time = dt.now() count_valid = 0 @@ -209,6 +186,19 @@ def db_monitor_domain(domain, log=None, exclude_invalid=None, get_cert=None): return res if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="") + parser.add_argument('--domain', default=None) + parser.add_argument('--log', default=None) + parser.add_argument('--exclude-invalid', action='store_true') + parser.add_argument('--get-cert', action='store_true') + + args = parser.parse_args() + + monitored_domains = [] + for md in MONITORED_DOMAINS: + monitored_domains.append(monitored_domain(md)) + if args.domain: db_monitor_domain(args.domain, args.log, args.exclude_invalid, args.get_cert) else: diff --git a/monitor/monitor_conf.py b/monitor/monitor_conf.py index 1106183..2628185 100644 --- a/monitor/monitor_conf.py +++ b/monitor/monitor_conf.py @@ -21,6 +21,7 @@ MONITORED_DOMAINS = [ "*.liu.se", "*.kth.se", "*.nordu.net", + "*.sunet.se", # "mail.google.com", # "*.symantec.com", # "*.se", @@ -32,9 +33,9 @@ ctlogs = { # ["https://ct.googleapis.com/pilot/", # "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTDM0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA=="], - "plausible": - ["https://plausible.ct.nordu.net/", - "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9UV9+jO2MCTzkabodO2F7LM03MUBc8MrdAtkcW6v6GA9taTTw9QJqofm0BbdAsbtJL/unyEf0zIkRgXjjzaYqQ=="], + # "plausible": + # ["https://plausible.ct.nordu.net/", + # "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9UV9+jO2MCTzkabodO2F7LM03MUBc8MrdAtkcW6v6GA9taTTw9QJqofm0BbdAsbtJL/unyEf0zIkRgXjjzaYqQ=="], # "digicert": # ["https://ct1.digicert-ct.com/log/", |