From 6610afbf6a5e802fc15733d6a499281cd981af13 Mon Sep 17 00:00:00 2001 From: Linus Nordberg Date: Thu, 2 Feb 2017 02:47:49 +0100 Subject: Parallelised merge, backup phase. --- test/catlfish-test-local-merge.cfg | 3 +++ tools/compileconfig.py | 17 +++++++++++++++-- tools/merge | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/test/catlfish-test-local-merge.cfg b/test/catlfish-test-local-merge.cfg index bf1d3c1..e0841e9 100644 --- a/test/catlfish-test-local-merge.cfg +++ b/test/catlfish-test-local-merge.cfg @@ -22,6 +22,9 @@ logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ merge: min-delay: 1 + backup-window-size: 2 + backup-sendlog-chunksize: 1 + backup-sendentries-chunksize: 1 dist-window-size: 2 dist-sendlog-chunksize: 1 dist-sendentries-chunksize: 1 diff --git a/tools/compileconfig.py b/tools/compileconfig.py index aea7ace..47fd19a 100755 --- a/tools/compileconfig.py +++ b/tools/compileconfig.py @@ -186,12 +186,17 @@ def allowed_servers_frontend(signingnodenames, storagenodenames): ("/plop/v1/signing/sct", signingnodenames), ] -def allowed_servers_primarymerge(frontendnodenames): +def allowed_servers_primarymerge(frontendnodenames, secondarymergenames): return [ ("/plop/v1/frontend/verify-entries", frontendnodenames), ("/plop/v1/frontend/sendlog", frontendnodenames), ("/plop/v1/frontend/sendentry", frontendnodenames), ("/plop/v1/frontend/publish-sth", frontendnodenames), + ("/plop/v1/merge/verifiedsize", secondarymergenames), + ("/plop/v1/merge/verifyroot", secondarymergenames), + ("/plop/v1/merge/setverifiedsize", secondarymergenames), + ("/plop/v1/merge/sendlog", secondarymergenames), + ("/plop/v1/merge/sendentry", secondarymergenames), ] def parse_ratelimit_expression(expression): @@ -352,15 +357,23 @@ def gen_config(nodename, config, localconfig): reloadableplopconfig.append((Symbol("storage_nodes_quorum"), config["storage-quorum-size"])) services.add(Symbol("ht")) if nodename == primarymergenodename: + mergesecondarynames = [node["name"] for node in config["mergenodes"] if node["name"] != primarymergenodename] + mergesecondaryaddresses = ["https://%s/plop/v1/merge/" % node["address"] for node in config["mergenodes"] if node["name"] != primarymergenodename] merge = localconfig["merge"] plopconfig.append((Symbol("db_backend_opt"), [(Symbol("write_flag"), Symbol("read"))])) plopconfig.append((Symbol("merge_delay"), merge["min-delay"])) plopconfig.append((Symbol("merge_dist_winsize"), merge["dist-window-size"])) plopconfig.append((Symbol("merge_dist_sendlog_chunksize"), merge["dist-sendlog-chunksize"])) plopconfig.append((Symbol("merge_dist_sendentries_chunksize"), merge["dist-sendentries-chunksize"])) + reloadableplopconfig.append((Symbol("merge_backup_winsize"), merge["backup-window-size"])) + reloadableplopconfig.append((Symbol("merge_backup_sendlog_chunksize"), merge["backup-sendlog-chunksize"])) + reloadableplopconfig.append((Symbol("merge_backup_sendentries_chunksize"), merge["backup-sendentries-chunksize"])) plopconfig.append((Symbol("frontend_nodes"), frontendnodeaddresses)) + reloadableplopconfig.append((Symbol("merge_secondaries"), zip(mergesecondarynames, mergesecondaryaddresses))) plopconfig.append((Symbol("sth_path"), paths["mergedb"] + "/sth")) - allowed_servers += allowed_servers_primarymerge(frontendnodenames) + plopconfig.append((Symbol("fetched_path"), paths["mergedb"] + "/fetched")) + plopconfig.append((Symbol("verified_path"), paths["mergedb"] + "/verified")) + allowed_servers += allowed_servers_primarymerge(frontendnodenames, mergesecondarynames) else: allowed_clients += allowed_clients_mergesecondary(primarymergenodename) diff --git a/tools/merge b/tools/merge index 0d3f36c..d0198ec 100755 --- a/tools/merge +++ b/tools/merge @@ -5,6 +5,6 @@ set -o errexit BINDIR=$(dirname $0) $BINDIR/merge_fetch.py "$@" -$BINDIR/merge_backup.py "$@" +#$BINDIR/merge_backup.py "$@" $BINDIR/merge_sth.py "$@" #$BINDIR/merge_dist.py "$@" -- cgit v1.1