diff options
| -rw-r--r-- | reltool.config | 6 | ||||
| -rw-r--r-- | test/catlfish-test-local-statusserver.cfg | 29 | ||||
| -rw-r--r-- | test/catlfish-test-mergefailover.cfg.in | 4 | ||||
| -rw-r--r-- | test/catlfish-test-redistribute-frontend.cfg.in | 4 | ||||
| -rw-r--r-- | test/catlfish-test.cfg.in | 4 | ||||
| -rwxr-xr-x | test/scripts/light-system-test.sh | 4 | ||||
| -rwxr-xr-x | test/scripts/perf-test.sh | 4 | ||||
| -rwxr-xr-x | tools/compileconfig.py | 48 | ||||
| -rwxr-xr-x | tools/merge_fetch.py | 14 |
9 files changed, 73 insertions, 44 deletions
diff --git a/reltool.config b/reltool.config index 2de8936..7797104 100644 --- a/reltool.config +++ b/reltool.config @@ -4,16 +4,16 @@ {app_file, strip}, {rel, "catlfish", "1.0.1-alpha-dev", [sasl, catlfish]}, {rel, "merge", "1.0.1-alpha-dev", [sasl, merge]}, - {rel, "statsserver", "1.0.1-alpha-dev", [sasl, statsserver]}, + {rel, "statusserver", "1.0.1-alpha-dev", [sasl, statusserver]}, {boot_rel, "catlfish"}, {profile, standalone}, {incl_sys_filters, ["^bin/", "^erts-.*/", "^lib"]}, - {incl_app_filters, ["^ebin/", "^priv/", "^src/", "^merge/ebin/", "^merge/src/", "^statsserver/ebin/", "^statsserver/src/"]}, + {incl_app_filters, ["^ebin/", "^priv/", "^src/", "^merge/ebin/", "^merge/src/", "^statusserver/ebin/", "^statusserver/src/"]}, {escript, "./verifycert.erl", [{incl_cond, include}]}, {app, catlfish, [{app_file, all}, {lib_dir, "."}]}, {app, plop, [{app_file, all}, {lib_dir, "../plop"}]}, {app, merge, [{app_file, all}, {lib_dir, "../plop/merge"}]}, - {app, statsserver, [{app_file, all}, {lib_dir, "../plop/statsserver"}]}, + {app, statusserver, [{app_file, all}, {lib_dir, "../plop/statusserver"}]}, {app, mochiweb, [{app_file, all}, {lib_dir, "../mochiweb"}]}, {app, idna, [{app_file, all}, {lib_dir, "../hackney/deps/idna"}]}, {app, hackney, [{app_file, all}, {lib_dir, "../hackney"}]}, diff --git a/test/catlfish-test-local-statusserver.cfg b/test/catlfish-test-local-statusserver.cfg new file mode 100644 index 0000000..190c611 --- /dev/null +++ b/test/catlfish-test-local-statusserver.cfg @@ -0,0 +1,29 @@ +localnodes: + - statusserver + +publichttpaddresses: + statusserver: 127.0.0.1:9080 + +paths: + configdir: . + https_certfile: httpscert/httpscert-1.pem + https_keyfile: httpscert/httpskey-1.pem + https_cacertfile: httpsca/demoCA/cacert.pem + public_cacertfile: httpsca/demoCA/cacert.pem + publickeys: publickeys + logpublickey: keys/logkey.pem + privatekeys: privatekeys + +ratelimits: + add_chain: 1000 per second + +dbbackend: permdb + +logadminkey: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQ + Ah9sZ2CD+JeLbprS6AFcZbo0TGCH0rtEnr2Q3JW0ylhfA+ + 0/WLu755b3soVX/wI23vqCVGC7N9fOB2WUltveQ== + +configurl: file:///CURRENTWORKINGDIRECTORY/catlfish-test-generated.cfg + +#options: +# - sctcaching diff --git a/test/catlfish-test-mergefailover.cfg.in b/test/catlfish-test-mergefailover.cfg.in index 7f0fbcb..81cbfc7 100644 --- a/test/catlfish-test-mergefailover.cfg.in +++ b/test/catlfish-test-mergefailover.cfg.in @@ -39,7 +39,7 @@ storage-quorum-size: 1 mmd: 86400 -statsservers: - - name: statsserver +statusservers: + - name: statusserver address: localhost:9081 publicaddress: localhost:9082 diff --git a/test/catlfish-test-redistribute-frontend.cfg.in b/test/catlfish-test-redistribute-frontend.cfg.in index 725c7c5..ef59a2d 100644 --- a/test/catlfish-test-redistribute-frontend.cfg.in +++ b/test/catlfish-test-redistribute-frontend.cfg.in @@ -39,7 +39,7 @@ storage-quorum-size: 1 mmd: 86400 -statsservers: - - name: statsserver +statusservers: + - name: statusserver address: localhost:9081 publicaddress: localhost:9082 diff --git a/test/catlfish-test.cfg.in b/test/catlfish-test.cfg.in index 18724e8..d16e5d3 100644 --- a/test/catlfish-test.cfg.in +++ b/test/catlfish-test.cfg.in @@ -39,7 +39,7 @@ storage-quorum-size: 1 mmd: 86400 -statsservers: - - name: statsserver +statusservers: + - name: statusserver address: localhost:9081 publicaddress: localhost:9082 diff --git a/test/scripts/light-system-test.sh b/test/scripts/light-system-test.sh index da91929..f8b63f8 100755 --- a/test/scripts/light-system-test.sh +++ b/test/scripts/light-system-test.sh @@ -18,11 +18,11 @@ tests_stop() { } tests_stop_all() { - ${SCRIPTS}/light-system-test-stop.sh mergeprimary mergesecondary frontend signing statsserver + ${SCRIPTS}/light-system-test-stop.sh mergeprimary mergesecondary frontend signing statusserver } ${SCRIPTS}/light-system-test-prepare.sh -tests_start statsserver signing +tests_start statusserver signing tests_start merge_2only frontendexceptlast ${top_srcdir}/tools/initlog.py --config machine/merge-1/catlfish-test.cfg --localconfig machine/merge-1/catlfish-test-local-merge-1.cfg tests_start mergeprimary diff --git a/test/scripts/perf-test.sh b/test/scripts/perf-test.sh index 0d6daa3..9b77ddb 100755 --- a/test/scripts/perf-test.sh +++ b/test/scripts/perf-test.sh @@ -18,7 +18,7 @@ tests_stop() { } tests_stop_all() { - ${SCRIPTS}/light-system-test-stop.sh mergeprimary mergesecondary frontend signing statsserver + ${SCRIPTS}/light-system-test-stop.sh mergeprimary mergesecondary frontend signing statusserver } ${SCRIPTS}/light-system-test-prepare.sh @@ -26,7 +26,7 @@ ${top_srcdir}/tools/initlog.py --config machine/merge-1/catlfish-test.cfg --loca cp ${top_srcdir}/test/known_roots/* known_roots -tests_start signing statsserver +tests_start signing statusserver tests_start merge_2only frontendexceptlast mergeprimary do_merge 0 diff --git a/tools/compileconfig.py b/tools/compileconfig.py index c5ad00c..44db767 100755 --- a/tools/compileconfig.py +++ b/tools/compileconfig.py @@ -74,7 +74,7 @@ def parse_address(address): def get_node_config(nodename, config): nodetype = [] nodeconfig = {} - for t in ["frontendnodes", "storagenodes", "signingnodes", "mergenodes", "statsservers"]: + for t in ["frontendnodes", "storagenodes", "signingnodes", "mergenodes", "statusservers"]: for node in config[t]: if node["name"] == nodename: nodetype.append(t) @@ -125,22 +125,22 @@ def gen_http_servers(nodetype, nodeconfig, bind_addresses, bind_publicaddress, b if "mergenodes" in nodetype: (host, port) = get_address(bind_addresses["merge"], nodeconfig["mergenodes"]) https_servers.append((Symbol("frontend_https_api"), host, port, Symbol("frontend"))) - if "statsservers" in nodetype: - (host, port) = get_address(None, nodeconfig["statsservers"]) - https_servers.append((Symbol("statsserver_https_api"), host, port, Symbol("statsserver"))) + if "statusservers" in nodetype: + (host, port) = get_address(None, nodeconfig["statusservers"]) + https_servers.append((Symbol("statusserver_https_api"), host, port, Symbol("statusserver"))) if bind_publicaddress: (publichost, publicport) = parse_address(bind_publicaddress) else: - (_, publicport) = parse_address(nodeconfig["statsservers"]["publicaddress"]) + (_, publicport) = parse_address(nodeconfig["statusservers"]["publicaddress"]) publichost = "0.0.0.0" if bind_publichttpaddress: (publichttphost, publichttpport) = parse_address(bind_publichttpaddress) - http_servers.append((Symbol("external_http_api"), publichttphost, publichttpport, Symbol("statsserver"))) - https_servers.append((Symbol("external_https_api"), publichost, publicport, Symbol("statsserver"))) + http_servers.append((Symbol("external_http_api"), publichttphost, publichttpport, Symbol("statusserver"))) + https_servers.append((Symbol("external_https_api"), publichost, publicport, Symbol("statusserver"))) - if nodetype - set(["frontendnodes", "storagenodes", "signingnodes", "mergenodes", "statsservers"]): + if nodetype - set(["frontendnodes", "storagenodes", "signingnodes", "mergenodes", "statusservers"]): print >>sys.stderr, "unknown nodetype", nodetype sys.exit(1) @@ -284,7 +284,7 @@ def gen_config(nodename, config, localconfig): (Symbol("https_keyfile"), paths["https_keyfile"]), ] - if "statsservers" in nodetype: + if "statusservers" in nodetype: plopconfig += [ (Symbol("https_servers"), https_servers), (Symbol("http_servers"), http_servers), @@ -363,8 +363,8 @@ def gen_config(nodename, config, localconfig): storagenodeaddresses = ["https://%s/plop/v1/storage/" % node["address"] for node in config["storagenodes"]] frontendnodenames = [node["name"] for node in config["frontendnodes"]] frontendnodeaddresses = ["https://%s/plop/v1/frontend/" % node["address"] for node in config["frontendnodes"]] - statsservernames = [node["name"] for node in config["statsservers"]] - statsserveraddresses = ["https://%s/plop/v1/status/" % node["address"] for node in config["statsservers"]] + statusservernames = [node["name"] for node in config["statusservers"]] + statusserveraddresses = ["https://%s/plop/v1/status/" % node["address"] for node in config["statusservers"]] allowed_clients = [] allowed_servers = [] @@ -437,17 +437,17 @@ def gen_config(nodename, config, localconfig): (Symbol("plopcontrol"), plopcontrolfilename), ] - reloadableplopconfig.append((Symbol("statsservers"), statsserveraddresses)) + reloadableplopconfig.append((Symbol("statusservers"), statusserveraddresses)) allowed_servers += [ - ("/plop/v1/status/merge_dist", statsservernames), - ("/plop/v1/status/merge_backup", statsservernames), - ("/plop/v1/status/merge_sth", statsservernames), - ("/plop/v1/status/merge_fetch", statsservernames), - ("/plop/v1/status/storage", statsservernames), - ("/plop/v1/status/merge_errors", statsservernames), + ("/plop/v1/status/merge_dist", statusservernames), + ("/plop/v1/status/merge_backup", statusservernames), + ("/plop/v1/status/merge_sth", statusservernames), + ("/plop/v1/status/merge_fetch", statusservernames), + ("/plop/v1/status/storage", statusservernames), + ("/plop/v1/status/merge_errors", statusservernames), ] - if "statsservers" in nodetype: + if "statusservers" in nodetype: allowed_clients += [ ("/plop/v1/status/merge_dist", mergenodenames), ("/plop/v1/status/merge_backup", mergenodenames), @@ -509,12 +509,12 @@ def gen_testmakefile(config, testmakefile, shellvars=False): signingnodenames = set([node["name"] for node in config["signingnodes"]]) mergenodenames = set([node["name"] for node in config["mergenodes"]]) mergesecondarynodenames = set([node["name"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]]) - statsservernodenames = set([node["name"] for node in config.get("statsservers")]) + statusservernodenames = set([node["name"] for node in config.get("statusservers")]) frontendnodenames_except_last = sorted(frontendnodenames)[:-1] frontendnodenames_except_first = sorted(frontendnodenames)[1:] - allnodes = config["frontendnodes"] + config["storagenodes"] + config["signingnodes"] + config["mergenodes"] + config["statsservers"] + allnodes = config["frontendnodes"] + config["storagenodes"] + config["signingnodes"] + config["mergenodes"] + config["statusservers"] testaddresses = multivaldict([(node["name"], node["address"]) for node in allnodes]) @@ -527,7 +527,7 @@ def gen_testmakefile(config, testmakefile, shellvars=False): print_nodevar(configfile, delimiter, "SIGNING", "catlfish", signingnodenames, testaddresses) print_nodevar(configfile, delimiter, "MERGESECONDARY", "catlfish", mergesecondarynodenames, testaddresses) print_nodevar(configfile, delimiter, "MERGEPRIMARY", "merge", [config["primarymergenode"]], testaddresses, gentesturl=False) - print_nodevar(configfile, delimiter, "STATSSERVER", "statsserver", statsservernodenames, testaddresses) + print_nodevar(configfile, delimiter, "STATUSSERVER", "statusserver", statusservernodenames, testaddresses) print >>configfile, "NODES=" + delimiter + " ".join(set([node["name"] for node in allnodes])) + delimiter @@ -541,9 +541,9 @@ def printnodenames(config): storagenodenames = set([node["name"] for node in config["storagenodes"]]) signingnodenames = set([node["name"] for node in config["signingnodes"]]) mergenodenames = set([node["name"] for node in config["mergenodes"]]) - statsservernodenames = set([node["name"] for node in config.get("statsservers")]) + statusservernodenames = set([node["name"] for node in config.get("statusservers")]) - print " ".join(frontendnodenames|storagenodenames|signingnodenames|mergenodenames|statsservernodenames) + print " ".join(frontendnodenames|storagenodenames|signingnodenames|mergenodenames|statusservernodenames) def main(): parser = argparse.ArgumentParser(description="") diff --git a/tools/merge_fetch.py b/tools/merge_fetch.py index 165dee8..2a2491d 100755 --- a/tools/merge_fetch.py +++ b/tools/merge_fetch.py @@ -18,9 +18,9 @@ from mergetools import get_logorder, verify_entry, get_new_entries, \ from certtools import timing_point, write_file, create_ssl_context, http_request import json -def reportstatus(statsservers, own_key, target, variable, status): - for statsserver in statsservers: - do_reportstatus(statsserver["name"], "https://%s/" % statsserver["address"], own_key, target, variable, status) +def reportstatus(statusservers, own_key, target, variable, status): + for statusserver in statusservers: + do_reportstatus(statusserver["name"], "https://%s/" % statusserver["address"], own_key, target, variable, status) def do_reportstatus(node, baseurl, own_key, target, variable, status): try: @@ -50,7 +50,7 @@ def merge_fetch(args, config, localconfig, currentsizefile): "%s/%s-private.pem" % (paths["privatekeys"], localconfig["nodename"])) assert(localconfig["nodename"] == config["primarymergenode"]) - statsservers = config.get("statsservers") + statusservers = config.get("statusservers") timing = timing_point() logorder = get_logorder(logorderfile) @@ -79,7 +79,7 @@ def merge_fetch(args, config, localconfig, currentsizefile): new_entries -= certsinlog print >>sys.stderr, "adding", len(new_entries), "entries" sys.stderr.flush() - reportstatus(statsservers, own_key, "fetch", "total", len(certsinlog) + len(new_entries)) + reportstatus(statusservers, own_key, "fetch", "total", len(certsinlog) + len(new_entries)) for ehash in new_entries: for storagenode in storagenodes: @@ -123,7 +123,7 @@ def merge_fetch(args, config, localconfig, currentsizefile): tree_size = len(logorder) currentsize = {"index": tree_size - 1, "hash": hexencode(logorder[tree_size-1])} write_file(currentsizefile, currentsize) - reportstatus(statsservers, own_key, "fetch", "fetched", tree_size) + reportstatus(statusservers, own_key, "fetch", "fetched", tree_size) print >>sys.stderr, added_entries, sys.stderr.flush() print >>sys.stderr @@ -141,7 +141,7 @@ def merge_fetch(args, config, localconfig, currentsizefile): sys.stderr.flush() tree_size = len(logorder) - reportstatus(statsservers, own_key, "fetch", "fetched", tree_size) + reportstatus(statusservers, own_key, "fetch", "fetched", tree_size) if tree_size == 0: return (0, '') else: |