diff options
| -rwxr-xr-x | monitor/josef_monitor.py | 93 |
1 files changed, 62 insertions, 31 deletions
diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py index 7160145..d97ba02 100755 --- a/monitor/josef_monitor.py +++ b/monitor/josef_monitor.py @@ -51,6 +51,10 @@ class ctlog: self.root_hash = None self.build = build + self.saved_sth = None + self.saved_entries = None + self.saved_subtree = None + if CONFIG.DB_PATH: self.dbdir = CONFIG.DB_PATH # self.dbdir = CONFIG.DB_PATH + name + "/" @@ -73,28 +77,45 @@ class ctlog: if self.build: start_size = self.entries - # try: - while self.entries < self.sth["tree_size"]: - tmp_size = self.entries - try: - self.subtree, self.entries = self.fetch_and_increment_subtree(self.entries, self.sth["tree_size"] -1, self.url, self.subtree) - except Exception, e: - print ERROR_STR + "Failed fetch and increment for " + self.name - self.log(ERROR_STR + "Failed fetch and increment tree. Current Size: " + str(self.entries) + " Sth: " + str(self.sth) + " Error: " + str(e)) - return - - if tmp_size != self.entries: - self.log("Got entries " + str(tmp_size) + " to " \ - + str(self.entries -1 ) + " of " + str(self.sth["tree_size"]-1)) - - if self.entries != start_size: - if verify_subtree(self.sth, self.subtree, self.url): - pass - else: - self.log(ERROR_STR + "Failed to verify newly built subtree!") - # except Exception, e: - # print "Failed incremental build for " + self.name - # self.log(ERROR_STR + "Failed incremental build. Error: " + str(e)) + try: + while self.entries < self.sth["tree_size"]: + tmp_size = self.entries + try: + self.subtree, self.entries = self.fetch_and_increment_subtree(self.entries, self.sth["tree_size"] -1, self.url, self.subtree) + except Exception, e: + # print ERROR_STR + "Failed fetch and increment for " + self.name + self.log(ERROR_STR + "Failed fetch and increment tree. Current Size: " + str(self.entries) + " Sth: " + str(self.sth) + " Error: " + str(e)) + self.rollback() + return + + if tmp_size != self.entries: + self.log("Got entries " + str(tmp_size) + " to " \ + + str(self.entries -1 ) + " of " + str(self.sth["tree_size"]-1)) + + if self.entries != start_size: + if verify_subtree(self.sth, self.subtree, self.url): + pass + else: + self.log(ERROR_STR + "Failed to verify newly built subtree!") + self.rollback() + except Exception, e: + # print "Failed incremental build for " + self.name + self.log(ERROR_STR + "Failed incremental build. Error: " + str(e)) + self.rollback() + + def save_state(self): + self.saved_sth = self.sth + self.saved_subtree = self.subtree + self.saved_entries = self.entries + + def rollback(self): + if self.saved_entries and self.saved_subtree and self.saved_sth: + self.log("Rolling back to last saved state") + self.sth = self.saved_sth + self.subtree = self.saved_subtree + self.entries = self.saved_entries + else: + self.log(ERROR_STR + "Could not roll back, no saved state found!") def fetch_and_increment_subtree(self, first, last, url, subtree =[[]]): new_leafs = [] @@ -158,7 +179,8 @@ class ctlog: check_sth_signature(self.url, new_sth, base64.b64decode(self.key)) except: self.log(ERROR_STR + "Could not verify STH signature " + str(new_sth)) - print ERROR_STR + "Could not verify STH signature from " + self.url + self.rollback() + # print ERROR_STR + "Could not verify STH signature from " + self.url sth_time = datetime.datetime.fromtimestamp(new_sth['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S") if new_sth["timestamp"] != self.sth["timestamp"]: @@ -227,16 +249,19 @@ class ctlog: try: if new["tree_size"] == old["tree_size"]: if old["sha256_root_hash"] != new["sha256_root_hash"]: - print ERROR_STR + "Root hash is different for same tree size in " + self.name + # print ERROR_STR + "Root hash is different for same tree size in " + self.name self.log(ERROR_STR + "New root hash for same tree size! Old:" + str(old) + " New:" + str(new)) + self.rollback() elif new["tree_size"] < old["tree_size"]: - print ERROR_STR + "New tree smaller than previous tree (%d < %d) in %s" % \ - (new["tree_size"], old["tree_size"], self.name) + # print ERROR_STR + "New tree smaller than previous tree (%d < %d) in %s" % \ + # (new["tree_size"], old["tree_size"], self.name) self.log(ERROR_STR + "New tree is smaller than old tree! Old:" + str(old) + " New:" + str(new)) + self.rollback() if new["timestamp"] < old["timestamp"]: self.log(ERROR_STR + "Regression in timestamps! Old:" + str(old) + " New:" + str(new)) - print ERROR_STR + " Regression in timestamps in " + self.name + self.rollback() + # print ERROR_STR + " Regression in timestamps in " + self.name else: age = time.time() - new["timestamp"]/1000 sth_time = datetime.datetime.fromtimestamp(new['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S") @@ -253,7 +278,8 @@ class ctlog: self.log(s) except Exception, e: self.log(ERROR_STR + "Failed to verify progress! Old:" + str(old) + " New:" + str(new) + " Exception: " + str(e)) - print "Failed to verify progress in " + self.name + self.rollback() + # print "Failed to verify progress in " + self.name def verify_consistency(self, old): new = self.sth @@ -268,15 +294,18 @@ class ctlog: if old["sha256_root_hash"] != str(base64.b64encode(res[0])): self.log(ERROR_STR + "Verification of consistency for old hash failed! Old:" \ + str(old) + " New:" + str(new) + " Proof:" + str(consistency_proof)) - print ERROR_STR + "Failed to verify consistency for " + self.name + self.rollback() + # print ERROR_STR + "Failed to verify consistency for " + self.name elif new["sha256_root_hash"] != str(base64.b64encode(res[1])): self.log(ERROR_STR + "Verification of consistency for new hash failed! Old:" \ + str(old) + " New:" + str(new) + " Proof:" + str(consistency_proof)) - print ERROR_STR + "Failed to verify consistency for " + self.name + self.rollback() + # print ERROR_STR + "Failed to verify consistency for " + self.name except Exception, e: self.log(ERROR_STR + "Could not verify consistency! " + " Old:" + str(old) + " New:" + str(new) + " Error:" + str(e)) - print ERROR_STR + "Could not verify consistency for " + self.url + self.rollback() + # print ERROR_STR + "Could not verify consistency for " + self.url # def verify_inclusion_all(old, new): @@ -409,6 +438,8 @@ def main(args): for log in logs: log.update_roots() old_sth = log.sth + + log.save_state() # Create rollback point in case of failure log.update_sth() # Should this be done if later checks fail? (reorder?) if old_sth["timestamp"] != log.sth["timestamp"]: log.verify_progress(old_sth) |