adding rollback on failure

1 files changed, 62 insertions, 31 deletions

diff --git a/monitor/josef_monitor.py b/monitor/josef_monitor.py
index 7160145..d97ba02 100755
--- a/monitor/josef_monitor.py
+++ b/monitor/josef_monitor.py
@@ -51,6 +51,10 @@ class ctlog:
         self.root_hash = None
         self.build = build
 
+        self.saved_sth = None
+        self.saved_entries = None
+        self.saved_subtree = None
+
         if CONFIG.DB_PATH:
             self.dbdir = CONFIG.DB_PATH
             # self.dbdir = CONFIG.DB_PATH + name + "/"
@@ -73,28 +77,45 @@ class ctlog:
 
         if self.build:
             start_size = self.entries
-            # try:
-            while self.entries < self.sth["tree_size"]:
-                tmp_size = self.entries
-                try:    
-                    self.subtree, self.entries = self.fetch_and_increment_subtree(self.entries, self.sth["tree_size"] -1, self.url, self.subtree)
-                except Exception, e:
-                    print ERROR_STR  + "Failed fetch and increment for " + self.name
-                    self.log(ERROR_STR + "Failed fetch and increment tree. Current Size: " + str(self.entries) + " Sth: " + str(self.sth) + " Error: " + str(e))
-                    return
-
-                if tmp_size != self.entries:
-                    self.log("Got entries " + str(tmp_size) + " to " \
-                    + str(self.entries -1 ) + " of " + str(self.sth["tree_size"]-1))
-
-            if self.entries != start_size:
-                if verify_subtree(self.sth, self.subtree, self.url):
-                    pass
-                else:
-                    self.log(ERROR_STR + "Failed to verify newly built subtree!")
-            # except Exception, e:
-            #     print "Failed incremental build for " + self.name
-            #     self.log(ERROR_STR + "Failed incremental build. Error: " + str(e))
+            try:
+                while self.entries < self.sth["tree_size"]:
+                    tmp_size = self.entries
+                    try:    
+                        self.subtree, self.entries = self.fetch_and_increment_subtree(self.entries, self.sth["tree_size"] -1, self.url, self.subtree)
+                    except Exception, e:
+                        # print ERROR_STR  + "Failed fetch and increment for " + self.name
+                        self.log(ERROR_STR + "Failed fetch and increment tree. Current Size: " + str(self.entries) + " Sth: " + str(self.sth) + " Error: " + str(e))
+                        self.rollback()
+                        return
+
+                    if tmp_size != self.entries:
+                        self.log("Got entries " + str(tmp_size) + " to " \
+                        + str(self.entries -1 ) + " of " + str(self.sth["tree_size"]-1))
+
+                if self.entries != start_size:
+                    if verify_subtree(self.sth, self.subtree, self.url):
+                        pass
+                    else:
+                        self.log(ERROR_STR + "Failed to verify newly built subtree!")
+                        self.rollback()
+            except Exception, e:
+                # print "Failed incremental build for " + self.name
+                self.log(ERROR_STR + "Failed incremental build. Error: " + str(e))
+                self.rollback()
+
+    def save_state(self):
+        self.saved_sth = self.sth
+        self.saved_subtree = self.subtree
+        self.saved_entries = self.entries
+
+    def rollback(self):
+        if self.saved_entries and self.saved_subtree and self.saved_sth:
+            self.log("Rolling back to last saved state")
+            self.sth = self.saved_sth
+            self.subtree = self.saved_subtree
+            self.entries = self.saved_entries
+        else:
+            self.log(ERROR_STR + "Could not roll back, no saved state found!")
 
     def fetch_and_increment_subtree(self, first, last, url, subtree =[[]]):
         new_leafs = []
@@ -158,7 +179,8 @@ class ctlog:
             check_sth_signature(self.url, new_sth, base64.b64decode(self.key))
         except:
             self.log(ERROR_STR + "Could not verify STH signature " + str(new_sth))
-            print ERROR_STR + "Could not verify STH signature from " + self.url
+            self.rollback()
+            # print ERROR_STR + "Could not verify STH signature from " + self.url
 
         sth_time = datetime.datetime.fromtimestamp(new_sth['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S")
         if new_sth["timestamp"] != self.sth["timestamp"]:
@@ -227,16 +249,19 @@ class ctlog:
         try:
             if new["tree_size"] == old["tree_size"]:
                 if old["sha256_root_hash"] != new["sha256_root_hash"]:
-                    print ERROR_STR + "Root hash is different for same tree size in " + self.name
+                    # print ERROR_STR + "Root hash is different for same tree size in " + self.name
                     self.log(ERROR_STR + "New root hash for same tree size! Old:" + str(old) + " New:" + str(new))
+                    self.rollback()
             elif new["tree_size"] < old["tree_size"]:
-                print ERROR_STR + "New tree smaller than previous tree (%d < %d) in %s" % \
-                  (new["tree_size"], old["tree_size"], self.name)
+                # print ERROR_STR + "New tree smaller than previous tree (%d < %d) in %s" % \
+                #   (new["tree_size"], old["tree_size"], self.name)
                 self.log(ERROR_STR + "New tree is smaller than old tree! Old:" + str(old) + " New:" + str(new))
+                self.rollback()
 
             if new["timestamp"] < old["timestamp"]:
                 self.log(ERROR_STR + "Regression in timestamps! Old:" + str(old) + " New:" + str(new))
-                print ERROR_STR + " Regression in timestamps in " + self.name
+                self.rollback()
+                # print ERROR_STR + " Regression in timestamps in " + self.name
             else:
                 age = time.time() - new["timestamp"]/1000
                 sth_time = datetime.datetime.fromtimestamp(new['timestamp'] / 1000, UTC()).strftime("%Y-%m-%d %H:%M:%S")
@@ -253,7 +278,8 @@ class ctlog:
                     self.log(s)
         except Exception, e:
             self.log(ERROR_STR + "Failed to verify progress! Old:" + str(old) + " New:" + str(new) + " Exception: " + str(e))
-            print "Failed to verify progress in " + self.name
+            self.rollback()
+            # print "Failed to verify progress in " + self.name
 
     def verify_consistency(self, old):
         new = self.sth
@@ -268,15 +294,18 @@ class ctlog:
                 if old["sha256_root_hash"] != str(base64.b64encode(res[0])):
                     self.log(ERROR_STR + "Verification of consistency for old hash failed! Old:" \
                         + str(old) + " New:" + str(new) + " Proof:" + str(consistency_proof))
-                    print ERROR_STR + "Failed to verify consistency for " + self.name
+                    self.rollback()
+                    # print ERROR_STR + "Failed to verify consistency for " + self.name
                 elif new["sha256_root_hash"] != str(base64.b64encode(res[1])):
                     self.log(ERROR_STR + "Verification of consistency for new hash failed! Old:" \
                         + str(old) + " New:" + str(new) + " Proof:" + str(consistency_proof))
-                    print ERROR_STR + "Failed to verify consistency for " + self.name
+                    self.rollback()
+                    # print ERROR_STR + "Failed to verify consistency for " + self.name
 
         except Exception, e:
             self.log(ERROR_STR + "Could not verify consistency! " + " Old:" + str(old) + " New:" + str(new) + " Error:"  + str(e))
-            print ERROR_STR + "Could not verify consistency for " + self.url
+            self.rollback()
+            # print ERROR_STR + "Could not verify consistency for " + self.url
 
 
 # def verify_inclusion_all(old, new):
@@ -409,6 +438,8 @@ def main(args):
             for log in logs:
                 log.update_roots()
                 old_sth = log.sth
+
+                log.save_state() # Create rollback point in case of failure
                 log.update_sth() # Should this be done if later checks fail? (reorder?)
                 if old_sth["timestamp"] != log.sth["timestamp"]:
                     log.verify_progress(old_sth)