diff options
Diffstat (limited to 'monitor')
| -rwxr-xr-x | monitor/josef_monitor.py | 159 | ||||
| -rw-r--r-- | monitor/logs.py | 6 |
2 files changed, 58 insertions, 107 deletions
diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py index f3a38bf..4fb99ee 100755 --- a/monitor/josef_monitor.py +++ b/monitor/josef_monitor.py @@ -26,16 +26,6 @@ parser = argparse.ArgumentParser(description="") # parser.add_argument('--silent', action='store_true', help="Dont output to stdout. logging only") -# monitored_domains = [ -# # "google.com", -# # "preishelden.de", -# # "liu.se", -# # "nordu.net", -# # "symantec.com", -# # "sunet.se", -# # ".se", -# ] - class ctlog: def __init__(self, name, url, key): self.name = name @@ -50,40 +40,32 @@ class ctlog: self.log("Starting monitor") - def build(self): - self.sth = get_sth(self.url) - tmp_subtree = fetch_and_build_subtree(self.sth, self.url) - if verify_subtree(self.sth, self.subtree, self.url): - self.log("Successfully build tree from entries.") - self.subtree = tmp_subtree - self.entries = self.sth["tree_size"] - else: - self.log("ERROR Failed to build tree from entries.") - def incremental_build(self): # Keeps state current during build, partial builds are possible. self.sth = get_sth(self.url) + self.log("Building....") + start_size = self.entries while self.entries < self.sth["tree_size"]: self.subtree, self.entries = fetch_and_increment_subtree(self.entries, self.sth["tree_size"] -1, self.url, self.subtree) - if verify_subtree(self.sth, self.subtree, self.url): - self.log("Successfully build tree from entries.") - # self.subtree = tmp_subtree - # self.entries = self.sth["tree_size"] + if self.entries != start_size: + if verify_subtree(self.sth, self.subtree, self.url): + self.log("Successfully build tree with " + str(self.entries - start_size) + \ + " new entries. Size: " + str(self.entries)) + else: + self.log("ERROR Failed to build tree from entries.") else: - self.log("ERROR Failed to build tree from entries.") + self.log("No new entries.") + def to_dict(self): d = {} - # d["name"] = self.name - # d["url"] = self.url d["entries"] = self.entries d["subtree"] = encode_tree(self.subtree) d["sth"] = self.sth return d - def save(self): self.log("Saving state to file") open(self.savefile, 'w').write(json.dumps(self.to_dict())) @@ -110,62 +92,53 @@ class ctlog: def update_sth(self): new_sth = get_sth(self.url) + + try: + check_sth_signature(self.url, new_sth, None) + except: + self.log("ERROR: Could not verify STH signature") + print "ERROR: Could not verify STH signature from " + url + sth_time = datetime.datetime.fromtimestamp(new_sth['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S") if new_sth["timestamp"] != self.sth["timestamp"]: self.log("STH updated. Size: " + str(new_sth["tree_size"]) + ", Time: " + sth_time) self.sth = new_sth -# def fetch_all_sth(): -# sths = {} -# for base_url in base_urls: -# # Fetch STH -# try: -# sths[base_url] = get_sth(base_url) -# except: -# sths[base_url] = None -# error_str = time.strftime('%H:%M:%S') + " ERROR: Failed to retrieve STH from " + base_url -# print error_str -# errors.append(error_str) -# continue - -# # Check signature on the STH -# try: -# # check_sth_signature(base_url, sths[base_url], logkeys[base_url]) -# check_sth_signature(base_url, sths[base_url], None) -# except: -# error_str = time.strftime('%H:%M:%S') + " ERROR: Could not verify signature from " + base_url -# print error_str -# errors.append(error_str) -# continue - -# return sths - -def verify_progress(old, new): - print "Verifying progress" - try: - for url in new: - if new and old and new[url] and old[url]: - if new[url]["tree_size"] == old[url]["tree_size"]: - if old[url]["sha256_root_hash"] != new[url]["sha256_root_hash"]: - errors.append(time.strftime('%H:%M:%S') + " CRITICAL: root hash is different for same tree size in " + url) - elif new[url]["tree_size"] < old[url]["tree_size"]: - errors.append(time.strftime('%H:%M:%S') + " CRITICAL: new tree smaller than previous tree (%d < %d)" % \ - (new[url]["tree_size"], old[url]["tree_size"])) - if new[url]: - age = time.time() - new[url]["timestamp"]/1000 - sth_time = datetime.datetime.fromtimestamp(new[url]['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S") - roothash = new[url]['sha256_root_hash'] + def verify_progress(self, old): + new = self.sth + try: + if new["tree_size"] == old["tree_size"]: + if old["sha256_root_hash"] != new["sha256_root_hash"]: + s = time.strftime('%H:%M:%S') + " CRITICAL: root hash is different for same tree size" + self.log(s) + print s + elif new["tree_size"] < old["tree_size"]: + s = time.strftime('%H:%M:%S') + " CRITICAL: new tree smaller than previous tree (%d < %d)" % \ + (new["tree_size"], old["tree_size"]) + self.log(s) + print s + else: + age = time.time() - new["timestamp"]/1000 + sth_time = datetime.datetime.fromtimestamp(new['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S") + roothash = new['sha256_root_hash'] if age > 24 * 3600: - errors.append(time.strftime('%H:%M:%S') + " CRITICAL: %s is older than 24h: %s UTC" % (url, sth_time)) + s = "CRITICAL: STH is older than 24h: %s UTC" % (sth_time) + self.log(s) + print s elif age > 12 * 3600: - errors.append(time.strftime('%H:%M:%S') + " WARNING: %s is older than 12h: %s UTC" % (url, sth_time)) + s = "WARNING: STH is older than 12h: %s UTC" % (sth_time) + self.log(s) + # print s elif age > 6 * 3600: - errors.append(time.strftime('%H:%M:%S') + " WARNING: %s is older than 6h: %s UTC" % (url, sth_time)) - # elif age > 2 * 3600: - # errors.append(time.strftime('%H:%M:%S') + " WARNING: %s is older than 2h: %s UTC" % (url, sth_time)) - except: - print time.strftime('%H:%M:%S') + " ERROR: Failed to verify progress for " + url + s = "WARNING: STH is older than 6h: %s UTC" % (sth_time) + self.log(s) + # print s + except: + s = " ERROR: Failed to verify progress" + self.log(s) + print s + def verify_consistency(old, new): for url in old: @@ -234,35 +207,13 @@ def fetch_and_increment_subtree(first, last, url, subtree =[[]]): new_leafs.append(get_leaf_hash(base64.b64decode(item["leaf_input"]))) append_file(DEFAULT_CERT_FILE, tmp_cert_data) print time.strftime('%H:%M:%S') + " Got entries " + str(first) + " to " \ - + str(first + len(new_leafs)) + " (" + str(len(new_leafs)) +" entries) from " + url + + str(first + len(new_leafs) -1 ) + " of " + str(last) +" entries from " + url subtree = reduce_tree(new_leafs, subtree) # except: # print "Failed to build subtree :(" return subtree, len(new_leafs) + first -def fetch_and_build_subtree(sth, base_url): - try: - subtree = [[]] - idx = 0 - - while idx < sth["tree_size"]: - pre_size = idx - entries = get_entries(base_url, idx, sth["tree_size"]-1)["entries"] - new_leafs = [] - tmp_cert_data = [] - for item in entries: - tmp_cert_data.append(check_domain(item, base_url)) - new_leafs.append(get_leaf_hash(base64.b64decode(item["leaf_input"]))) - idx += len(new_leafs) - append_file(DEFAULT_CERT_FILE, tmp_cert_data) - print time.strftime('%H:%M:%S') + " Got entries " + str(pre_size) + " to " + str(idx -1) + " from " + base_url - subtree = reduce_tree(new_leafs, subtree) - except: - print "Failed to build subtree :(" - return subtree - - def verify_subtree(sth, subtree, base_url): try: tmp = deepcopy(subtree) @@ -358,6 +309,8 @@ def main(args): for log in logs: if os.path.isfile(log.savefile): log.load() + # Build what was not loaded + for log in logs: log.incremental_build() # Main loop: Monitor @@ -365,13 +318,11 @@ def main(args): time.sleep(INTERVAL) for log in logs: old_sth = log.sth - log.update_sth() + log.update_sth() # Should this be done is later checks fail? (reorder?) if old_sth["timestamp"] != log.sth["timestamp"]: - # TODO verify signature - # TODO fetch updates - # TODO verify progress - # TODO verify tree - # TODO verify consistency proof? + log.verify_progress(old_sth) + log.incremental_build() + # TODO check consistency proof pass # Unreachable... usually. diff --git a/monitor/logs.py b/monitor/logs.py index 12c58a5..2925017 100644 --- a/monitor/logs.py +++ b/monitor/logs.py @@ -8,9 +8,9 @@ ctlogs = { # ["https://plausible.ct.nordu.net/", # "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9UV9+jO2MCTzkabodO2F7LM03MUBc8MrdAtkcW6v6GA9taTTw9QJqofm0BbdAsbtJL/unyEf0zIkRgXjjzaYqQ=="], - # "digicert": - # ["https://ct1.digicert-ct.com/log/", - # "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEAkbFvhu7gkAW6MHSrBlpE1n4+HCFRkC5OLAjgqhkTH+/uzSfSl8ois8ZxAD2NgaTZe1M9akhYlrYkes4JECs6A=="], + "digicert": + ["https://ct1.digicert-ct.com/log/", + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEAkbFvhu7gkAW6MHSrBlpE1n4+HCFRkC5OLAjgqhkTH+/uzSfSl8ois8ZxAD2NgaTZe1M9akhYlrYkes4JECs6A=="], "izenpe": ["https://ct.izenpe.com/", |