From d645194006bf3c81372073af9784f7d993096444 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 23 Mar 2015 16:12:13 +0100 Subject: Generate config from master config. Verify responses in merge.py. --- tools/compileconfig.py | 283 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100755 tools/compileconfig.py (limited to 'tools/compileconfig.py') diff --git a/tools/compileconfig.py b/tools/compileconfig.py new file mode 100755 index 0000000..30424c5 --- /dev/null +++ b/tools/compileconfig.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python + +# Copyright (c) 2014, NORDUnet A/S. +# See LICENSE for licensing information. + +import argparse +import sys +import yaml +import re + +class Symbol(str): + pass + +clean_string = re.compile(r'^[-.:_/A-Za-z0-9 ]*$') +clean_symbol = re.compile(r'^[_A-Za-z0-9]*$') + +def quote_erlang_string(s): + if clean_string.match(s): + return '"' + s + '"' + else: + return "[" + ",".join([str(ord(c)) for c in s]) + "]" + +def quote_erlang_symbol(s): + if clean_symbol.match(s): + return s + elif clean_string.match(s): + return "'" + s + "'" + else: + print >>sys.stderr, "Cannot generate symbol", s + sys.exit(1) + +def gen_erlang(term, level=1): + indent = " " * level + separator = ",\n" + indent + if isinstance(term, Symbol): + return quote_erlang_symbol(term) + elif isinstance(term, basestring): + return quote_erlang_string(term) + elif isinstance(term, int): + return str(term) + elif isinstance(term, tuple): + tuplecontents = [gen_erlang(e, level=level+1) for e in term] + if "\n" not in "".join(tuplecontents): + separator = ", " + return "{" + separator.join(tuplecontents) + "}" + elif isinstance(term, list): + listcontents = [gen_erlang(e, level=level+1) for e in term] + return "[" + separator.join(listcontents) + "]" + else: + print "unknown type", type(term) + sys.exit(1) + +saslconfig = [(Symbol("sasl_error_logger"), Symbol("false")), + (Symbol("errlog_type"), Symbol("error")), + (Symbol("error_logger_mf_dir"), "log"), + (Symbol("error_logger_mf_maxbytes"), 10485760), + (Symbol("error_logger_mf_maxfiles"), 10), + ] + +def parse_address(address): + parsed_address = address.split(":") + if len(parsed_address) != 2: + print >>sys.stderr, "Invalid address format", address + sys.exit(1) + return (parsed_address[0], int(parsed_address[1])) + +def get_node_config(nodename, config): + nodetype = None + nodeconfig = None + for t in ["frontendnodes", "storagenodes", "signingnodes"]: + for node in config[t]: + if node["name"] == nodename: + nodetype = t + nodeconfig = node + if nodeconfig == None: + print >>sys.stderr, "Cannot find config for node", nodename + sys.exit(1) + return (nodetype, nodeconfig) + +def gen_https_servers(nodetype, nodeconfig): + if nodetype == "frontendnodes": + (publichost, publicport) = parse_address(nodeconfig["publicaddress"]) + (host, port) = parse_address(nodeconfig["address"]) + return [(Symbol("external_https_api"), publichost, publicport, Symbol("v1")), + (Symbol("frontend_https_api"), host, port, Symbol("frontend"))] + elif nodetype == "storagenodes": + (host, port) = parse_address(nodeconfig["address"]) + return [(Symbol("storage_https_api"), host, port, Symbol("storage"))] + elif nodetype == "signingnodes": + (host, port) = parse_address(nodeconfig["address"]) + return [(Symbol("signing_https_api"), host, port, Symbol("signing"))] + +def allowed_clients_frontend(mergenodenames): + return [ + ("/ct/frontend/sendentry", mergenodenames), + ("/ct/frontend/sendlog", mergenodenames), + ("/ct/frontend/sendsth", mergenodenames), + ("/ct/frontend/currentposition", mergenodenames), + ("/ct/frontend/missingentries", mergenodenames), + ] + +def allowed_clients_public(): + noauth = Symbol("noauth") + return [ + ("/ct/v1/add-chain", noauth), + ("/ct/v1/add-pre-chain", noauth), + ("/ct/v1/get-sth", noauth), + ("/ct/v1/get-sth-consistency", noauth), + ("/ct/v1/get-proof-by-hash", noauth), + ("/ct/v1/get-entries", noauth), + ("/ct/v1/get-entry-and-proof", noauth), + ("/ct/v1/get-roots", noauth), + ] + +def allowed_clients_signing(frontendnodenames, mergenodenames): + return [ + ("/ct/signing/sct", frontendnodenames), + ("/ct/signing/sth", mergenodenames), + ] + +def allowed_clients_storage(frontendnodenames, mergenodenames): + return [ + ("/ct/storage/sendentry", frontendnodenames), + ("/ct/storage/entrycommitted", frontendnodenames), + ("/ct/storage/fetchnewentries", mergenodenames), + ("/ct/storage/getentry", mergenodenames), + ] + +def allowed_servers_frontend(signingnodenames, storagenodenames): + return [ + ("/ct/storage/sendentry", storagenodenames), + ("/ct/storage/entrycommitted", storagenodenames), + ("/ct/signing/sct", signingnodenames), + ] + +def gen_config(nodename, config, localconfig): + print "generating config for", nodename + paths = localconfig["paths"] + options = localconfig.get("options", []) + + configfile = open(paths["configdir"] + nodename + ".config", "w") + print >>configfile, "%% catlfish configuration file (-*- erlang -*-)" + + (nodetype, nodeconfig) = get_node_config(nodename, config) + https_servers = gen_https_servers(nodetype, nodeconfig) + + catlfishconfig = [] + plopconfig = [] + + if nodetype == "frontendnodes": + catlfishconfig.append((Symbol("known_roots_path"), localconfig["paths"]["knownroots"])) + if "sctcaching" in options: + catlfishconfig.append((Symbol("sctcache_root_path"), paths["db"] + "sctcache/")) + + catlfishconfig += [ + (Symbol("https_servers"), https_servers), + (Symbol("https_certfile"), paths["https_certfile"]), + (Symbol("https_keyfile"), paths["https_keyfile"]), + (Symbol("https_cacertfile"), paths["https_cacertfile"]), + ] + + lagerconfig = [ + (Symbol("handlers"), [ + (Symbol("lager_console_backend"), Symbol("info")), + (Symbol("lager_file_backend"), [(Symbol("file"), nodename + "-error.log"), (Symbol("level"), Symbol("error"))]), + (Symbol("lager_file_backend"), [(Symbol("file"), nodename + "-debug.log"), (Symbol("level"), Symbol("debug"))]), + (Symbol("lager_file_backend"), [(Symbol("file"), nodename + "-console.log"), (Symbol("level"), Symbol("info"))]), + ]) + ] + + if nodetype in ("frontendnodes", "storagenodes"): + plopconfig += [ + (Symbol("entry_root_path"), paths["db"] + "certentries/"), + ] + if nodetype == "frontendnodes": + plopconfig += [ + (Symbol("index_path"), paths["db"] + "index"), + ] + elif nodetype == "storagenodes": + plopconfig += [ + (Symbol("newentries_path"), paths["db"] + "newentries"), + ] + if nodetype in ("frontendnodes", "storagenodes"): + plopconfig += [ + (Symbol("entryhash_root_path"), paths["db"] + "entryhash/"), + (Symbol("indexforhash_root_path"), paths["db"] + "certindex/"), + ] + if nodetype == "frontendnodes": + plopconfig += [ + (Symbol("sth_path"), paths["db"] + "sth"), + ] + + signingnode = config["signingnodes"][0] + mergenodenames = [node["name"] for node in config["mergenodes"]] + storagenodeaddresses = ["https://%s/ct/storage/" % node["address"] for node in config["storagenodes"]] + frontendnodenames = [node["name"] for node in config["frontendnodes"]] + + allowed_clients = [] + allowed_servers = [] + + if nodetype == "frontendnodes": + storagenodenames = [node["name"] for node in config["storagenodes"]] + plopconfig.append((Symbol("storage_nodes"), storagenodeaddresses)) + plopconfig.append((Symbol("storage_nodes_quorum"), config["storage-quorum-size"])) + services = [Symbol("ht")] + allowed_clients += allowed_clients_frontend(mergenodenames) + allowed_clients += allowed_clients_public() + allowed_servers += allowed_servers_frontend([signingnode["name"]], storagenodenames) + elif nodetype == "storagenodes": + allowed_clients += allowed_clients_storage(frontendnodenames, mergenodenames) + services = [] + elif nodetype == "signingnodes": + allowed_clients += allowed_clients_signing(frontendnodenames, mergenodenames) + services = [Symbol("sign")] + + plopconfig += [ + (Symbol("publickey_path"), paths["publickeys"]), + (Symbol("services"), services), + ] + if nodetype == "signingnodes": + plopconfig.append((Symbol("log_private_key"), paths["logprivatekey"])) + plopconfig += [ + (Symbol("log_public_key"), paths["logpublickey"]), + (Symbol("own_key"), (nodename, "%s/%s-private.pem" % (paths["privatekeys"], nodename))), + ] + if nodetype == "frontendnodes": + plopconfig.append((Symbol("signing_node"), "https://%s/ct/signing/" % signingnode["address"])) + plopconfig += [ + (Symbol("allowed_clients"), allowed_clients), + (Symbol("allowed_servers"), allowed_servers), + ] + + erlangconfig = [ + (Symbol("sasl"), saslconfig), + (Symbol("catlfish"), catlfishconfig), + (Symbol("lager"), lagerconfig), + (Symbol("plop"), plopconfig), + ] + + print >>configfile, gen_erlang(erlangconfig) + ".\n" + + configfile.close() + + +def gen_testmakefile(config, testmakefile, machines): + configfile = open(testmakefile, "w") + frontendnodenames = [node["name"] for node in config["frontendnodes"]] + storagenodenames = [node["name"] for node in config["storagenodes"]] + signingnodename = [node["name"] for node in config["signingnodes"]] + + frontendnodeaddresses = [node["publicaddress"] for node in config["frontendnodes"]] + storagenodeaddresses = [node["address"] for node in config["storagenodes"]] + signingnodeaddresses = [node["address"] for node in config["signingnodes"]] + + print >>configfile, "NODES=" + " ".join(frontendnodenames+storagenodenames+signingnodename) + print >>configfile, "MACHINES=" + " ".join([str(e) for e in range(1, machines+1)]) + print >>configfile, "TESTURLS=" + " ".join(frontendnodeaddresses+storagenodeaddresses+signingnodeaddresses) + print >>configfile, "BASEURL=" + config["baseurl"] + + configfile.close() + + +def main(): + parser = argparse.ArgumentParser(description="") + parser.add_argument('--config', help="System configuration", required=True) + parser.add_argument('--localconfig', help="Local configuration") + parser.add_argument("--testmakefile", metavar="file", help="Generate makefile variables for test") + parser.add_argument("--machines", type=int, metavar="n", help="Number of machines") + args = parser.parse_args() + + config = yaml.load(open(args.config)) + if args.testmakefile and args.machines: + gen_testmakefile(config, args.testmakefile, args.machines) + elif args.localconfig: + localconfig = yaml.load(open(args.localconfig)) + localnodes = localconfig["localnodes"] + for localnode in localnodes: + gen_config(localnode, config, localconfig) + else: + print >>sys.stderr, "Nothing to do" + sys.exit(1) + +main() -- cgit v1.1 From db1c5cf0eeba9b47a3f89bebc1abd9985ee9fef7 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 26 Mar 2015 23:59:30 +0100 Subject: Provide function for calculating entryhash from entry --- tools/compileconfig.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'tools/compileconfig.py') diff --git a/tools/compileconfig.py b/tools/compileconfig.py index 30424c5..e3d9ee0 100755 --- a/tools/compileconfig.py +++ b/tools/compileconfig.py @@ -188,6 +188,8 @@ def gen_config(nodename, config, localconfig): if nodetype == "frontendnodes": plopconfig += [ (Symbol("sth_path"), paths["db"] + "sth"), + (Symbol("entryhash_from_entry"), + (Symbol("catlfish"), Symbol("entryhash_from_entry"))), ] signingnode = config["signingnodes"][0] -- cgit v1.1 From 6cd3c80c61711cabf9e308da1963b56f6dd49cfd Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 27 Mar 2015 03:08:06 +0100 Subject: Handle multiple signing nodes --- tools/compileconfig.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'tools/compileconfig.py') diff --git a/tools/compileconfig.py b/tools/compileconfig.py index e3d9ee0..52f10e8 100755 --- a/tools/compileconfig.py +++ b/tools/compileconfig.py @@ -192,7 +192,8 @@ def gen_config(nodename, config, localconfig): (Symbol("catlfish"), Symbol("entryhash_from_entry"))), ] - signingnode = config["signingnodes"][0] + signingnodes = config["signingnodes"] + signingnodeaddresses = ["https://%s/ct/signing/" % node["address"] for node in config["signingnodes"]] mergenodenames = [node["name"] for node in config["mergenodes"]] storagenodeaddresses = ["https://%s/ct/storage/" % node["address"] for node in config["storagenodes"]] frontendnodenames = [node["name"] for node in config["frontendnodes"]] @@ -207,7 +208,7 @@ def gen_config(nodename, config, localconfig): services = [Symbol("ht")] allowed_clients += allowed_clients_frontend(mergenodenames) allowed_clients += allowed_clients_public() - allowed_servers += allowed_servers_frontend([signingnode["name"]], storagenodenames) + allowed_servers += allowed_servers_frontend([node["name"] for node in signingnodes], storagenodenames) elif nodetype == "storagenodes": allowed_clients += allowed_clients_storage(frontendnodenames, mergenodenames) services = [] @@ -226,7 +227,7 @@ def gen_config(nodename, config, localconfig): (Symbol("own_key"), (nodename, "%s/%s-private.pem" % (paths["privatekeys"], nodename))), ] if nodetype == "frontendnodes": - plopconfig.append((Symbol("signing_node"), "https://%s/ct/signing/" % signingnode["address"])) + plopconfig.append((Symbol("signing_nodes"), signingnodeaddresses)) plopconfig += [ (Symbol("allowed_clients"), allowed_clients), (Symbol("allowed_servers"), allowed_servers), -- cgit v1.1 From 2d8d55bb9b6672ebe829b185beb05d4a399167f5 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Fri, 27 Mar 2015 17:33:35 +0100 Subject: Allow local administrator to override IP address and port for web server Closes CATLFISH-30 --- tools/compileconfig.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'tools/compileconfig.py') diff --git a/tools/compileconfig.py b/tools/compileconfig.py index 52f10e8..88d6b51 100755 --- a/tools/compileconfig.py +++ b/tools/compileconfig.py @@ -77,17 +77,24 @@ def get_node_config(nodename, config): sys.exit(1) return (nodetype, nodeconfig) -def gen_https_servers(nodetype, nodeconfig): +def gen_https_servers(nodetype, nodeconfig, bind_address, bind_publicaddress): + if bind_address: + (host, port) = parse_address(bind_address) + else: + (_, port) = parse_address(nodeconfig["address"]) + host = "0.0.0.0" if nodetype == "frontendnodes": - (publichost, publicport) = parse_address(nodeconfig["publicaddress"]) - (host, port) = parse_address(nodeconfig["address"]) + if bind_publicaddress: + (publichost, publicport) = parse_address(bind_publicaddress) + else: + (_, publicport) = parse_address(nodeconfig["publicaddress"]) + publichost = "0.0.0.0" + return [(Symbol("external_https_api"), publichost, publicport, Symbol("v1")), (Symbol("frontend_https_api"), host, port, Symbol("frontend"))] elif nodetype == "storagenodes": - (host, port) = parse_address(nodeconfig["address"]) return [(Symbol("storage_https_api"), host, port, Symbol("storage"))] elif nodetype == "signingnodes": - (host, port) = parse_address(nodeconfig["address"]) return [(Symbol("signing_https_api"), host, port, Symbol("signing"))] def allowed_clients_frontend(mergenodenames): @@ -136,13 +143,15 @@ def allowed_servers_frontend(signingnodenames, storagenodenames): def gen_config(nodename, config, localconfig): print "generating config for", nodename paths = localconfig["paths"] + bind_address = localconfig.get("addresses", {}).get(nodename) + bind_publicaddress = localconfig.get("publicaddresses", {}).get(nodename) options = localconfig.get("options", []) configfile = open(paths["configdir"] + nodename + ".config", "w") print >>configfile, "%% catlfish configuration file (-*- erlang -*-)" (nodetype, nodeconfig) = get_node_config(nodename, config) - https_servers = gen_https_servers(nodetype, nodeconfig) + https_servers = gen_https_servers(nodetype, nodeconfig, bind_address, bind_publicaddress) catlfishconfig = [] plopconfig = [] -- cgit v1.1 From 22cefc84254cae1f57195da819eba69dbacb5a6e Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 30 Mar 2015 15:41:13 +0200 Subject: Allow non-TLS http Closes CATLFISH-31 --- tools/compileconfig.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) (limited to 'tools/compileconfig.py') diff --git a/tools/compileconfig.py b/tools/compileconfig.py index 88d6b51..4996994 100755 --- a/tools/compileconfig.py +++ b/tools/compileconfig.py @@ -77,7 +77,7 @@ def get_node_config(nodename, config): sys.exit(1) return (nodetype, nodeconfig) -def gen_https_servers(nodetype, nodeconfig, bind_address, bind_publicaddress): +def gen_http_servers(nodetype, nodeconfig, bind_address, bind_publicaddress, bind_publichttpaddress): if bind_address: (host, port) = parse_address(bind_address) else: @@ -90,12 +90,22 @@ def gen_https_servers(nodetype, nodeconfig, bind_address, bind_publicaddress): (_, publicport) = parse_address(nodeconfig["publicaddress"]) publichost = "0.0.0.0" - return [(Symbol("external_https_api"), publichost, publicport, Symbol("v1")), - (Symbol("frontend_https_api"), host, port, Symbol("frontend"))] + http_servers = [] + https_servers = [] + if bind_publichttpaddress: + (publichttphost, publichttpport) = parse_address(bind_publichttpaddress) + http_servers.append((Symbol("external_http_api"), publichttphost, publichttpport, Symbol("v1"))) + https_servers.append((Symbol("external_https_api"), publichost, publicport, Symbol("v1"))) + https_servers.append((Symbol("frontend_https_api"), host, port, Symbol("frontend"))) + return (http_servers, + https_servers) + elif nodetype == "storagenodes": - return [(Symbol("storage_https_api"), host, port, Symbol("storage"))] + return ([], + [(Symbol("storage_https_api"), host, port, Symbol("storage"))]) elif nodetype == "signingnodes": - return [(Symbol("signing_https_api"), host, port, Symbol("signing"))] + return ([], + [(Symbol("signing_https_api"), host, port, Symbol("signing"))]) def allowed_clients_frontend(mergenodenames): return [ @@ -145,13 +155,14 @@ def gen_config(nodename, config, localconfig): paths = localconfig["paths"] bind_address = localconfig.get("addresses", {}).get(nodename) bind_publicaddress = localconfig.get("publicaddresses", {}).get(nodename) + bind_publichttpaddress = localconfig.get("publichttpaddresses", {}).get(nodename) options = localconfig.get("options", []) configfile = open(paths["configdir"] + nodename + ".config", "w") print >>configfile, "%% catlfish configuration file (-*- erlang -*-)" (nodetype, nodeconfig) = get_node_config(nodename, config) - https_servers = gen_https_servers(nodetype, nodeconfig, bind_address, bind_publicaddress) + (http_servers, https_servers) = gen_http_servers(nodetype, nodeconfig, bind_address, bind_publicaddress, bind_publichttpaddress=bind_publichttpaddress) catlfishconfig = [] plopconfig = [] @@ -163,6 +174,7 @@ def gen_config(nodename, config, localconfig): catlfishconfig += [ (Symbol("https_servers"), https_servers), + (Symbol("http_servers"), http_servers), (Symbol("https_certfile"), paths["https_certfile"]), (Symbol("https_keyfile"), paths["https_keyfile"]), (Symbol("https_cacertfile"), paths["https_cacertfile"]), -- cgit v1.1 From 6b62ebbf1de5b9e55b04e9cfafd0620f1374c2d4 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Tue, 31 Mar 2015 14:27:23 +0200 Subject: Cleanup tests and use urllib2.build_opener Remove unused files Generate test config files directly in release directory Move test database files to "tests" directory Generate log key when preparing tests Report error when STH not found in v1.erl Make merge, fetchallcerts, submitcert, verifysct, and testcase1 take log key as argument --- tools/compileconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/compileconfig.py') diff --git a/tools/compileconfig.py b/tools/compileconfig.py index 4996994..c239bd0 100755 --- a/tools/compileconfig.py +++ b/tools/compileconfig.py @@ -158,7 +158,7 @@ def gen_config(nodename, config, localconfig): bind_publichttpaddress = localconfig.get("publichttpaddresses", {}).get(nodename) options = localconfig.get("options", []) - configfile = open(paths["configdir"] + nodename + ".config", "w") + configfile = open(paths["configdir"] + "/" + nodename + ".config", "w") print >>configfile, "%% catlfish configuration file (-*- erlang -*-)" (nodetype, nodeconfig) = get_node_config(nodename, config) -- cgit v1.1