Parallelised merge, backup phase.

3 files changed, 19 insertions, 3 deletions

diff --git a/test/catlfish-test-local-merge.cfg b/test/catlfish-test-local-merge.cfg
index bf1d3c1..e0841e9 100644
--- a/test/catlfish-test-local-merge.cfg
+++ b/test/catlfish-test-local-merge.cfg
@@ -22,6 +22,9 @@ logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ
 
 merge:
         min-delay: 1
+        backup-window-size: 2
+        backup-sendlog-chunksize: 1
+        backup-sendentries-chunksize: 1
         dist-window-size: 2
         dist-sendlog-chunksize: 1
         dist-sendentries-chunksize: 1
diff --git a/tools/compileconfig.py b/tools/compileconfig.py
index aea7ace..47fd19a 100755
--- a/tools/compileconfig.py
+++ b/tools/compileconfig.py
@@ -186,12 +186,17 @@ def allowed_servers_frontend(signingnodenames, storagenodenames):
         ("/plop/v1/signing/sct", signingnodenames),
     ]
 
-def allowed_servers_primarymerge(frontendnodenames):
+def allowed_servers_primarymerge(frontendnodenames, secondarymergenames):
     return [
         ("/plop/v1/frontend/verify-entries", frontendnodenames),
         ("/plop/v1/frontend/sendlog", frontendnodenames),
         ("/plop/v1/frontend/sendentry", frontendnodenames),
         ("/plop/v1/frontend/publish-sth", frontendnodenames),
+        ("/plop/v1/merge/verifiedsize", secondarymergenames),
+        ("/plop/v1/merge/verifyroot", secondarymergenames),
+        ("/plop/v1/merge/setverifiedsize", secondarymergenames),
+        ("/plop/v1/merge/sendlog", secondarymergenames),
+        ("/plop/v1/merge/sendentry", secondarymergenames),
     ]
 
 def parse_ratelimit_expression(expression):
@@ -352,15 +357,23 @@ def gen_config(nodename, config, localconfig):
         reloadableplopconfig.append((Symbol("storage_nodes_quorum"), config["storage-quorum-size"]))
         services.add(Symbol("ht"))
         if nodename == primarymergenodename:
+            mergesecondarynames = [node["name"] for node in config["mergenodes"] if node["name"] != primarymergenodename]
+            mergesecondaryaddresses = ["https://%s/plop/v1/merge/" % node["address"] for node in config["mergenodes"] if node["name"] != primarymergenodename]
             merge = localconfig["merge"]
             plopconfig.append((Symbol("db_backend_opt"), [(Symbol("write_flag"), Symbol("read"))]))
             plopconfig.append((Symbol("merge_delay"), merge["min-delay"]))
             plopconfig.append((Symbol("merge_dist_winsize"), merge["dist-window-size"]))
             plopconfig.append((Symbol("merge_dist_sendlog_chunksize"), merge["dist-sendlog-chunksize"]))
             plopconfig.append((Symbol("merge_dist_sendentries_chunksize"), merge["dist-sendentries-chunksize"]))
+            reloadableplopconfig.append((Symbol("merge_backup_winsize"), merge["backup-window-size"]))
+            reloadableplopconfig.append((Symbol("merge_backup_sendlog_chunksize"), merge["backup-sendlog-chunksize"]))
+            reloadableplopconfig.append((Symbol("merge_backup_sendentries_chunksize"), merge["backup-sendentries-chunksize"]))
             plopconfig.append((Symbol("frontend_nodes"), frontendnodeaddresses))
+            reloadableplopconfig.append((Symbol("merge_secondaries"), zip(mergesecondarynames, mergesecondaryaddresses)))
             plopconfig.append((Symbol("sth_path"), paths["mergedb"] + "/sth"))
-            allowed_servers += allowed_servers_primarymerge(frontendnodenames)
+            plopconfig.append((Symbol("fetched_path"), paths["mergedb"] + "/fetched"))
+            plopconfig.append((Symbol("verified_path"), paths["mergedb"] + "/verified"))
+            allowed_servers += allowed_servers_primarymerge(frontendnodenames, mergesecondarynames)
         else:
             allowed_clients += allowed_clients_mergesecondary(primarymergenodename)
 
diff --git a/tools/merge b/tools/merge
index 0d3f36c..d0198ec 100755
--- a/tools/merge
+++ b/tools/merge
@@ -5,6 +5,6 @@ set -o errexit
 BINDIR=$(dirname $0)
 
 $BINDIR/merge_fetch.py "$@"
-$BINDIR/merge_backup.py "$@"
+#$BINDIR/merge_backup.py "$@"
 $BINDIR/merge_sth.py "$@"
 #$BINDIR/merge_dist.py "$@"