diff options
Diffstat (limited to 'monitor')
| -rwxr-xr-x | monitor/josef_logreader.py | 3 | ||||
| -rwxr-xr-x | monitor/josef_monitor.py | 236 | ||||
| -rw-r--r-- | monitor/monitor_conf.py | 4 |
3 files changed, 114 insertions, 129 deletions
diff --git a/monitor/josef_logreader.py b/monitor/josef_logreader.py index 703ae98..42e1b61 100755 --- a/monitor/josef_logreader.py +++ b/monitor/josef_logreader.py @@ -11,6 +11,7 @@ from monitor_conf import * TIME_LEN = 20 NEW_STH_STR = "STH updated" START_STR = "Starting monitor" +ERROR_STR = "ERROR" def get_logs(): logs = [] @@ -89,7 +90,7 @@ def print_errors(l): ages = [] for item in rev_log: line = item[TIME_LEN:] - if "ERROR" in line: + if ERROR_STR in line: print item[:-1] if line[:len(START_STR)] == START_STR: break # comment this line to print all errors ever diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py index d9b65a1..d23f292 100755 --- a/monitor/josef_monitor.py +++ b/monitor/josef_monitor.py @@ -79,12 +79,8 @@ class ctlog: + str(self.entries -1 ) + " of " + str(self.sth["tree_size"]-1)) if self.entries != start_size: - if verify_subtree(self.sth, self.subtree, self.url): - pass - # self.log("Successfully build tree with " + str(self.entries - start_size) + \ - # " new entries. Size: " + str(self.entries)) - else: - self.log("ERROR Failed to build tree from entries.") + if not verify_subtree(self.sth, self.subtree, self.url): + self.log(ERROR_STR + "Failed to build tree from entries!") def fetch_and_increment_subtree(self, first, last, url, subtree =[[]]): new_leafs = [] @@ -141,14 +137,14 @@ class ctlog: try: new_sth = get_sth(self.url) except Exception, e: - self.log("Failed to fetch STH. " +str(e)) + self.log(ERROR_STR + "Failed to fetch STH. " +str(e)) return try: check_sth_signature(self.url, new_sth, base64.b64decode(self.key)) except: - self.log("ERROR: Could not verify STH signature") - print "ERROR: Could not verify STH signature from " + self.url + self.log(ERROR_STR + "Could not verify STH signature " + str(new_sth)) + print ERROR_STR + "Could not verify STH signature from " + self.url sth_time = datetime.datetime.fromtimestamp(new_sth['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S") if new_sth["timestamp"] != self.sth["timestamp"]: @@ -159,7 +155,7 @@ class ctlog: try: roots = get_all_roots(self.url) except Exception, e: - self.log("Failed to fetch roots. " + str(e)) + self.log(ERROR_STR + "Failed to fetch roots. " + str(e)) return new_root_hash = str(hash(str(roots))) @@ -180,15 +176,15 @@ class ctlog: added, removed = compare_lists(hash_list, loaded_list) if len(added) != 0: - self.log(str(len(added)) + " new roots found") + # self.log(str(len(added)) + " new roots added") for item in added: root_cert = base64.decodestring(roots[hash_list.index(item)]) subject = get_cert_info(root_cert)["subject"] issuer = get_cert_info(root_cert)["issuer"] if subject == issuer: - self.log("New Root: " + item + ", " + subject) + self.log("Added Root: " + item + ", " + subject) else: - self.log("WTF? Not a root...") + self.log(ERROR_STR + "Non self-signed root cert added! Subject:" + subject + " Issuer:" + issuer) fn = cert_dir + "/" + item tempname = fn + ".new" @@ -198,7 +194,7 @@ class ctlog: if len(removed) != 0: - self.log(str(len(removed)) + " roots removed") + # self.log(str(len(removed)) + " roots removed") for item in removed: data = open(cert_dir + "/" + item).read() root_cert = base64.decodestring(data) @@ -207,28 +203,26 @@ class ctlog: if subject == issuer: self.log("Removed Root: " + item + ", " + subject) else: - self.log("WTF? Not a root...") + self.log(ERROR_STR + "Non self-signed root cert removed! Subject:" + subject + " Issuer:" + issuer) def verify_progress(self, old): new = self.sth try: if new["tree_size"] == old["tree_size"]: if old["sha256_root_hash"] != new["sha256_root_hash"]: - s = time.strftime('%H:%M:%S') + " CRITICAL: root hash is different for same tree size" - self.log(s) - print s + print ERROR_STR + "Root hash is different for same tree size in " + self.name + self.log(ERROR_STR + "New root hash for same tree size! Old:" + str(old) + " New:" + str(new)) elif new["tree_size"] < old["tree_size"]: - s = time.strftime('%H:%M:%S') + " CRITICAL: new tree smaller than previous tree (%d < %d)" % \ - (new["tree_size"], old["tree_size"]) - self.log(s) - print s + print ERROR_STR + "New tree smaller than previous tree (%d < %d) in %s" % \ + (new["tree_size"], old["tree_size"], self.name) + self.log(ERROR_STR + "New tree is smaller than old tree! Old:" + str(old) + " New:" + str(new)) else: age = time.time() - new["timestamp"]/1000 sth_time = datetime.datetime.fromtimestamp(new['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S") roothash = new['sha256_root_hash'] if age > 24 * 3600: - s = "CRITICAL: STH is older than 24h: %s UTC" % (sth_time) - self.log(s) + s = ERROR_STR + "STH is older than 24h: %s UTC" % (sth_time) + self.log(s + str(new)) print s elif age > 12 * 3600: s = "WARNING: STH is older than 12h: %s UTC" % (sth_time) @@ -238,14 +232,12 @@ class ctlog: s = "WARNING: STH is older than 6h: %s UTC" % (sth_time) self.log(s) # print s - except: - s = " ERROR: Failed to verify progress" - self.log(s) - print s + except Exception, e: + self.log(ERROR_STR + "Failed to verify progress! Old:" + str(old) + " New:" + str(new) + " Exception: " + str(e)) + print "Failed to verify progress in " + self.name def verify_consistency(self, old): new = self.sth - # for url in old: try: if old["tree_size"]!= new["tree_size"]: consistency_proof = get_consistency_proof(self.url, old["tree_size"], new["tree_size"]) @@ -255,50 +247,44 @@ class ctlog: res = verify_consistency_proof(decoded_consistency_proof, old["tree_size"], new["tree_size"], old["sha256_root_hash"]) if old["sha256_root_hash"] != str(base64.b64encode(res[0])): - s = " Verification of old hash failed! " + \ - old["sha256_root_hash"], str(base64.b64encode(res[0])) - self.log(s) - print s + self.log(ERROR_STR + "Verification of consistency for old hash failed! Old:" \ + + str(old) + " New:" + str(new) + " Proof:" + str(consistency_proof)) + print ERROR_STR + "Failed to verify consistency for " + self.name elif new["sha256_root_hash"] != str(base64.b64encode(res[1])): - s = " Verification of new hash failed! " + \ - new["sha256_root_hash"], str(base64.b64encode(res[1])) - self.log(s) - print s - # else: - # s = "New STH, timestamp: " + str(new["timestamp"]) + \ - # ", size: " + str(new["tree_size"]) + "...OK." - # self.log(s) + self.log(ERROR_STR + "Verification of consistency for new hash failed! Old:" \ + + str(old) + " New:" + str(new) + " Proof:" + str(consistency_proof)) + print ERROR_STR + "Failed to verify consistency for " + self.name - except: - self.log("ERROR: Could not verify consistency!") - print "ERROR: Could not verify consistency for " + self.url - - -def verify_inclusion_all(old, new): - for url in old: - try: - if old[url] and new[url]: - if old[url]["tree_size"]!= new[url]["tree_size"]: - entries = [] - - while len(entries) + old[url]["tree_size"]!= new[url]["tree_size"]: - entries += get_entries(url, str(int(old[url]["tree_size"]) + len(entries)), new[url]["tree_size"] -1)["entries"] - print "Got " + str(len(entries)) + " entries..." - - success = True - for i in entries: - h = get_leaf_hash(base64.b64decode(i["leaf_input"])) - if not verify_inclusion_by_hash(url, h): - success = False - - if success: - print time.strftime("%H:%M:%S") + " Verifying inclusion for " + str(len(entries)) + " new entries in " + url + " ...OK" - else: - print time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url - errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url) - except: - print time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url - errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url) + except Exception, e: + self.log(ERROR_STR + "Could not verify consistency! " + " Old:" + str(old) + " New:" + str(new) + " Error:" + str(e)) + print ERROR_STR + "Could not verify consistency for " + self.url + + +# def verify_inclusion_all(old, new): +# for url in old: +# try: +# if old[url] and new[url]: +# if old[url]["tree_size"]!= new[url]["tree_size"]: +# entries = [] + +# while len(entries) + old[url]["tree_size"]!= new[url]["tree_size"]: +# entries += get_entries(url, str(int(old[url]["tree_size"]) + len(entries)), new[url]["tree_size"] -1)["entries"] +# print "Got " + str(len(entries)) + " entries..." + +# success = True +# for i in entries: +# h = get_leaf_hash(base64.b64decode(i["leaf_input"])) +# if not verify_inclusion_by_hash(url, h): +# success = False + +# if success: +# print time.strftime("%H:%M:%S") + " Verifying inclusion for " + str(len(entries)) + " new entries in " + url + " ...OK" +# else: +# print time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url +# errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url) +# except: +# print time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url +# errors.append(time.strftime('%H:%M:%S') + " ERROR: Failed to prove inclusion of all new entries in " + url) def check_domain(raw_entry, log=None): orig_entry = extract_original_entry(raw_entry) @@ -316,66 +302,62 @@ def verify_subtree(sth, subtree, base_url): root = base64.b64encode(reduce_subtree_to_root(tmp)[0]) if root == sth["sha256_root_hash"]: - # print time.strftime('%H:%M:%S') + " Verifying root hashes for " + base_url + "...OK." return True else: - print time.strftime('%H:%M:%S') + " ERROR: Failed to verify root hashes! STH root: " \ - + sth["sha256_root_hash"] + ", Tree root: " + root return False except: - print time.strftime('%H:%M:%S') + " ERROR: Failed to build STH for " + base_url return False -def verify_inclusion_by_hash(base_url, leaf_hash): - try: - tmp_sth = get_sth(base_url) - proof = get_proof_by_hash(base_url, leaf_hash, tmp_sth["tree_size"]) +# def verify_inclusion_by_hash(base_url, leaf_hash): +# try: +# tmp_sth = get_sth(base_url) +# proof = get_proof_by_hash(base_url, leaf_hash, tmp_sth["tree_size"]) - decoded_inclusion_proof = [] - for item in proof["audit_path"]: - decoded_inclusion_proof.append(base64.b64decode(item)) +# decoded_inclusion_proof = [] +# for item in proof["audit_path"]: +# decoded_inclusion_proof.append(base64.b64decode(item)) - root = base64.b64encode(verify_inclusion_proof(decoded_inclusion_proof, proof["leaf_index"], tmp_sth["tree_size"], leaf_hash)) - - if tmp_sth["sha256_root_hash"] == root: - return True - else: - print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(proof["leaf_index"]) + " in " + base_url - return False - except: - print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for hashed entry in " + base_url - return False - -def verify_inclusion_by_index(base_url, index): - try: - tmp_sth = get_sth(base_url) - proof = get_proof_by_index(base_url, index, tmp_sth["tree_size"]) - - decoded_inclusion_proof = [] - for item in proof["audit_path"]: - decoded_inclusion_proof.append(base64.b64decode(item)) - - root = base64.b64encode(verify_inclusion_proof(decoded_inclusion_proof, index, tmp_sth["tree_size"], get_leaf_hash(base64.b64decode(proof["leaf_input"])))) - - if tmp_sth["sha256_root_hash"] == root: - print time.strftime('%H:%M:%S') + " Verifying inclusion for entry " + str(index) + " in " + base_url + "...OK." - else: - print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url - errors.append(time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url) - except: - print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url - errors.append(time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url) - -def get_proof_by_index(baseurl, index, tree_size): - try: - params = urllib.urlencode({"leaf_index":index, - "tree_size":tree_size}) - result = \ - urlopen(baseurl + "ct/v1/get-entry-and-proof?" + params).read() - return json.loads(result) - except urllib2.HTTPError, e: - print "ERROR:", e.read() - sys.exit(0) +# root = base64.b64encode(verify_inclusion_proof(decoded_inclusion_proof, proof["leaf_index"], tmp_sth["tree_size"], leaf_hash)) + +# if tmp_sth["sha256_root_hash"] == root: +# return True +# else: +# print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(proof["leaf_index"]) + " in " + base_url +# return False +# except: +# print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for hashed entry in " + base_url +# return False + +# def verify_inclusion_by_index(base_url, index): +# try: +# tmp_sth = get_sth(base_url) +# proof = get_proof_by_index(base_url, index, tmp_sth["tree_size"]) + +# decoded_inclusion_proof = [] +# for item in proof["audit_path"]: +# decoded_inclusion_proof.append(base64.b64decode(item)) + +# root = base64.b64encode(verify_inclusion_proof(decoded_inclusion_proof, index, tmp_sth["tree_size"], get_leaf_hash(base64.b64decode(proof["leaf_input"])))) + +# if tmp_sth["sha256_root_hash"] == root: +# print time.strftime('%H:%M:%S') + " Verifying inclusion for entry " + str(index) + " in " + base_url + "...OK." +# else: +# print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url +# errors.append(time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url) +# except: +# print time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url +# errors.append(time.strftime('%H:%M:%S') + " ERROR: Could not prove inclusion for entry " + str(index) + " in " + base_url) + +# def get_proof_by_index(baseurl, index, tree_size): +# try: +# params = urllib.urlencode({"leaf_index":index, +# "tree_size":tree_size}) +# result = \ +# urlopen(baseurl + "ct/v1/get-entry-and-proof?" + params).read() +# return json.loads(result) +# except urllib2.HTTPError, e: +# print "ERROR:", e.read() +# sys.exit(0) def get_all_roots(base_url): result = urlopen(base_url + "ct/v1/get-roots").read() @@ -410,17 +392,18 @@ def main(args): # Create logs logs = [] try: + # Create log objects for item in CONFIG.CTLOGS: logs.append(ctlog(item["name"], item["url"], item["key"], item["id"], item["build"])) - print time.strftime('%H:%M:%S') + " Setting up monitor for " + str(len(logs)) + " logs..." + # Set up state for log in logs: if os.path.isfile(log.savefile): log.load() + # Build new entries for log in logs: - # if log.build: log.incremental_build() # Main loop: Monitor @@ -429,13 +412,11 @@ def main(args): time.sleep(CONFIG.INTERVAL) for log in logs: log.update_roots() - old_sth = log.sth log.update_sth() # Should this be done if later checks fail? (reorder?) if old_sth["timestamp"] != log.sth["timestamp"]: log.verify_progress(old_sth) log.verify_consistency(old_sth) - # if log.build: log.incremental_build() for md in monitored_domains: @@ -467,7 +448,6 @@ def main(args): open(CONFIG.DOMAINS_FILE, 'w').write(json.dumps(domain_dict)) - if __name__ == '__main__': if CONFIG.OUTPUT_DIR and not os.path.exists(CONFIG.OUTPUT_DIR): os.makedirs(CONFIG.OUTPUT_DIR) diff --git a/monitor/monitor_conf.py b/monitor/monitor_conf.py index 61df927..ca7b254 100644 --- a/monitor/monitor_conf.py +++ b/monitor/monitor_conf.py @@ -28,6 +28,9 @@ MONITORED_DOMAINS = [ # "*.se", ] +# Some strings +ERROR_STR = "ERROR: " + # CT logs and associated keys CTLOGS = [ {"name" : "pilot", @@ -88,3 +91,4 @@ CTLOGS = [ + |