From d645194006bf3c81372073af9784f7d993096444 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Mon, 23 Mar 2015 16:12:13 +0100 Subject: Generate config from master config. Verify responses in merge.py. --- tools/compileconfig.py | 283 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100755 tools/compileconfig.py (limited to 'tools/compileconfig.py') diff --git a/tools/compileconfig.py b/tools/compileconfig.py new file mode 100755 index 0000000..30424c5 --- /dev/null +++ b/tools/compileconfig.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python + +# Copyright (c) 2014, NORDUnet A/S. +# See LICENSE for licensing information. + +import argparse +import sys +import yaml +import re + +class Symbol(str): + pass + +clean_string = re.compile(r'^[-.:_/A-Za-z0-9 ]*$') +clean_symbol = re.compile(r'^[_A-Za-z0-9]*$') + +def quote_erlang_string(s): + if clean_string.match(s): + return '"' + s + '"' + else: + return "[" + ",".join([str(ord(c)) for c in s]) + "]" + +def quote_erlang_symbol(s): + if clean_symbol.match(s): + return s + elif clean_string.match(s): + return "'" + s + "'" + else: + print >>sys.stderr, "Cannot generate symbol", s + sys.exit(1) + +def gen_erlang(term, level=1): + indent = " " * level + separator = ",\n" + indent + if isinstance(term, Symbol): + return quote_erlang_symbol(term) + elif isinstance(term, basestring): + return quote_erlang_string(term) + elif isinstance(term, int): + return str(term) + elif isinstance(term, tuple): + tuplecontents = [gen_erlang(e, level=level+1) for e in term] + if "\n" not in "".join(tuplecontents): + separator = ", " + return "{" + separator.join(tuplecontents) + "}" + elif isinstance(term, list): + listcontents = [gen_erlang(e, level=level+1) for e in term] + return "[" + separator.join(listcontents) + "]" + else: + print "unknown type", type(term) + sys.exit(1) + +saslconfig = [(Symbol("sasl_error_logger"), Symbol("false")), + (Symbol("errlog_type"), Symbol("error")), + (Symbol("error_logger_mf_dir"), "log"), + (Symbol("error_logger_mf_maxbytes"), 10485760), + (Symbol("error_logger_mf_maxfiles"), 10), + ] + +def parse_address(address): + parsed_address = address.split(":") + if len(parsed_address) != 2: + print >>sys.stderr, "Invalid address format", address + sys.exit(1) + return (parsed_address[0], int(parsed_address[1])) + +def get_node_config(nodename, config): + nodetype = None + nodeconfig = None + for t in ["frontendnodes", "storagenodes", "signingnodes"]: + for node in config[t]: + if node["name"] == nodename: + nodetype = t + nodeconfig = node + if nodeconfig == None: + print >>sys.stderr, "Cannot find config for node", nodename + sys.exit(1) + return (nodetype, nodeconfig) + +def gen_https_servers(nodetype, nodeconfig): + if nodetype == "frontendnodes": + (publichost, publicport) = parse_address(nodeconfig["publicaddress"]) + (host, port) = parse_address(nodeconfig["address"]) + return [(Symbol("external_https_api"), publichost, publicport, Symbol("v1")), + (Symbol("frontend_https_api"), host, port, Symbol("frontend"))] + elif nodetype == "storagenodes": + (host, port) = parse_address(nodeconfig["address"]) + return [(Symbol("storage_https_api"), host, port, Symbol("storage"))] + elif nodetype == "signingnodes": + (host, port) = parse_address(nodeconfig["address"]) + return [(Symbol("signing_https_api"), host, port, Symbol("signing"))] + +def allowed_clients_frontend(mergenodenames): + return [ + ("/ct/frontend/sendentry", mergenodenames), + ("/ct/frontend/sendlog", mergenodenames), + ("/ct/frontend/sendsth", mergenodenames), + ("/ct/frontend/currentposition", mergenodenames), + ("/ct/frontend/missingentries", mergenodenames), + ] + +def allowed_clients_public(): + noauth = Symbol("noauth") + return [ + ("/ct/v1/add-chain", noauth), + ("/ct/v1/add-pre-chain", noauth), + ("/ct/v1/get-sth", noauth), + ("/ct/v1/get-sth-consistency", noauth), + ("/ct/v1/get-proof-by-hash", noauth), + ("/ct/v1/get-entries", noauth), + ("/ct/v1/get-entry-and-proof", noauth), + ("/ct/v1/get-roots", noauth), + ] + +def allowed_clients_signing(frontendnodenames, mergenodenames): + return [ + ("/ct/signing/sct", frontendnodenames), + ("/ct/signing/sth", mergenodenames), + ] + +def allowed_clients_storage(frontendnodenames, mergenodenames): + return [ + ("/ct/storage/sendentry", frontendnodenames), + ("/ct/storage/entrycommitted", frontendnodenames), + ("/ct/storage/fetchnewentries", mergenodenames), + ("/ct/storage/getentry", mergenodenames), + ] + +def allowed_servers_frontend(signingnodenames, storagenodenames): + return [ + ("/ct/storage/sendentry", storagenodenames), + ("/ct/storage/entrycommitted", storagenodenames), + ("/ct/signing/sct", signingnodenames), + ] + +def gen_config(nodename, config, localconfig): + print "generating config for", nodename + paths = localconfig["paths"] + options = localconfig.get("options", []) + + configfile = open(paths["configdir"] + nodename + ".config", "w") + print >>configfile, "%% catlfish configuration file (-*- erlang -*-)" + + (nodetype, nodeconfig) = get_node_config(nodename, config) + https_servers = gen_https_servers(nodetype, nodeconfig) + + catlfishconfig = [] + plopconfig = [] + + if nodetype == "frontendnodes": + catlfishconfig.append((Symbol("known_roots_path"), localconfig["paths"]["knownroots"])) + if "sctcaching" in options: + catlfishconfig.append((Symbol("sctcache_root_path"), paths["db"] + "sctcache/")) + + catlfishconfig += [ + (Symbol("https_servers"), https_servers), + (Symbol("https_certfile"), paths["https_certfile"]), + (Symbol("https_keyfile"), paths["https_keyfile"]), + (Symbol("https_cacertfile"), paths["https_cacertfile"]), + ] + + lagerconfig = [ + (Symbol("handlers"), [ + (Symbol("lager_console_backend"), Symbol("info")), + (Symbol("lager_file_backend"), [(Symbol("file"), nodename + "-error.log"), (Symbol("level"), Symbol("error"))]), + (Symbol("lager_file_backend"), [(Symbol("file"), nodename + "-debug.log"), (Symbol("level"), Symbol("debug"))]), + (Symbol("lager_file_backend"), [(Symbol("file"), nodename + "-console.log"), (Symbol("level"), Symbol("info"))]), + ]) + ] + + if nodetype in ("frontendnodes", "storagenodes"): + plopconfig += [ + (Symbol("entry_root_path"), paths["db"] + "certentries/"), + ] + if nodetype == "frontendnodes": + plopconfig += [ + (Symbol("index_path"), paths["db"] + "index"), + ] + elif nodetype == "storagenodes": + plopconfig += [ + (Symbol("newentries_path"), paths["db"] + "newentries"), + ] + if nodetype in ("frontendnodes", "storagenodes"): + plopconfig += [ + (Symbol("entryhash_root_path"), paths["db"] + "entryhash/"), + (Symbol("indexforhash_root_path"), paths["db"] + "certindex/"), + ] + if nodetype == "frontendnodes": + plopconfig += [ + (Symbol("sth_path"), paths["db"] + "sth"), + ] + + signingnode = config["signingnodes"][0] + mergenodenames = [node["name"] for node in config["mergenodes"]] + storagenodeaddresses = ["https://%s/ct/storage/" % node["address"] for node in config["storagenodes"]] + frontendnodenames = [node["name"] for node in config["frontendnodes"]] + + allowed_clients = [] + allowed_servers = [] + + if nodetype == "frontendnodes": + storagenodenames = [node["name"] for node in config["storagenodes"]] + plopconfig.append((Symbol("storage_nodes"), storagenodeaddresses)) + plopconfig.append((Symbol("storage_nodes_quorum"), config["storage-quorum-size"])) + services = [Symbol("ht")] + allowed_clients += allowed_clients_frontend(mergenodenames) + allowed_clients += allowed_clients_public() + allowed_servers += allowed_servers_frontend([signingnode["name"]], storagenodenames) + elif nodetype == "storagenodes": + allowed_clients += allowed_clients_storage(frontendnodenames, mergenodenames) + services = [] + elif nodetype == "signingnodes": + allowed_clients += allowed_clients_signing(frontendnodenames, mergenodenames) + services = [Symbol("sign")] + + plopconfig += [ + (Symbol("publickey_path"), paths["publickeys"]), + (Symbol("services"), services), + ] + if nodetype == "signingnodes": + plopconfig.append((Symbol("log_private_key"), paths["logprivatekey"])) + plopconfig += [ + (Symbol("log_public_key"), paths["logpublickey"]), + (Symbol("own_key"), (nodename, "%s/%s-private.pem" % (paths["privatekeys"], nodename))), + ] + if nodetype == "frontendnodes": + plopconfig.append((Symbol("signing_node"), "https://%s/ct/signing/" % signingnode["address"])) + plopconfig += [ + (Symbol("allowed_clients"), allowed_clients), + (Symbol("allowed_servers"), allowed_servers), + ] + + erlangconfig = [ + (Symbol("sasl"), saslconfig), + (Symbol("catlfish"), catlfishconfig), + (Symbol("lager"), lagerconfig), + (Symbol("plop"), plopconfig), + ] + + print >>configfile, gen_erlang(erlangconfig) + ".\n" + + configfile.close() + + +def gen_testmakefile(config, testmakefile, machines): + configfile = open(testmakefile, "w") + frontendnodenames = [node["name"] for node in config["frontendnodes"]] + storagenodenames = [node["name"] for node in config["storagenodes"]] + signingnodename = [node["name"] for node in config["signingnodes"]] + + frontendnodeaddresses = [node["publicaddress"] for node in config["frontendnodes"]] + storagenodeaddresses = [node["address"] for node in config["storagenodes"]] + signingnodeaddresses = [node["address"] for node in config["signingnodes"]] + + print >>configfile, "NODES=" + " ".join(frontendnodenames+storagenodenames+signingnodename) + print >>configfile, "MACHINES=" + " ".join([str(e) for e in range(1, machines+1)]) + print >>configfile, "TESTURLS=" + " ".join(frontendnodeaddresses+storagenodeaddresses+signingnodeaddresses) + print >>configfile, "BASEURL=" + config["baseurl"] + + configfile.close() + + +def main(): + parser = argparse.ArgumentParser(description="") + parser.add_argument('--config', help="System configuration", required=True) + parser.add_argument('--localconfig', help="Local configuration") + parser.add_argument("--testmakefile", metavar="file", help="Generate makefile variables for test") + parser.add_argument("--machines", type=int, metavar="n", help="Number of machines") + args = parser.parse_args() + + config = yaml.load(open(args.config)) + if args.testmakefile and args.machines: + gen_testmakefile(config, args.testmakefile, args.machines) + elif args.localconfig: + localconfig = yaml.load(open(args.localconfig)) + localnodes = localconfig["localnodes"] + for localnode in localnodes: + gen_config(localnode, config, localconfig) + else: + print >>sys.stderr, "Nothing to do" + sys.exit(1) + +main() -- cgit v1.1