summaryrefslogtreecommitdiff
path: root/tools/compileconfig.py
diff options
context:
space:
mode:
authorLinus Nordberg <linus@nordberg.se>2015-03-25 11:00:21 +0100
committerLinus Nordberg <linus@nordberg.se>2015-03-25 11:00:21 +0100
commit54e0457f1e4c4376bdecc891a2d5ae423eb2266d (patch)
tree8a756b221ded2625f186ff15b150a99365abe067 /tools/compileconfig.py
parentf5ee5ed3e1b445c52efd7978357adf5552643fd4 (diff)
parent842b07ef461483fcc297cc56e128918ddd273932 (diff)
Merge remote-tracking branch 'refs/remotes/map/compileconfig2'
Conflicts: .gitignore
Diffstat (limited to 'tools/compileconfig.py')
-rwxr-xr-xtools/compileconfig.py283
1 files changed, 283 insertions, 0 deletions
diff --git a/tools/compileconfig.py b/tools/compileconfig.py
new file mode 100755
index 0000000..30424c5
--- /dev/null
+++ b/tools/compileconfig.py
@@ -0,0 +1,283 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2014, NORDUnet A/S.
+# See LICENSE for licensing information.
+
+import argparse
+import sys
+import yaml
+import re
+
+class Symbol(str):
+ pass
+
+clean_string = re.compile(r'^[-.:_/A-Za-z0-9 ]*$')
+clean_symbol = re.compile(r'^[_A-Za-z0-9]*$')
+
+def quote_erlang_string(s):
+ if clean_string.match(s):
+ return '"' + s + '"'
+ else:
+ return "[" + ",".join([str(ord(c)) for c in s]) + "]"
+
+def quote_erlang_symbol(s):
+ if clean_symbol.match(s):
+ return s
+ elif clean_string.match(s):
+ return "'" + s + "'"
+ else:
+ print >>sys.stderr, "Cannot generate symbol", s
+ sys.exit(1)
+
+def gen_erlang(term, level=1):
+ indent = " " * level
+ separator = ",\n" + indent
+ if isinstance(term, Symbol):
+ return quote_erlang_symbol(term)
+ elif isinstance(term, basestring):
+ return quote_erlang_string(term)
+ elif isinstance(term, int):
+ return str(term)
+ elif isinstance(term, tuple):
+ tuplecontents = [gen_erlang(e, level=level+1) for e in term]
+ if "\n" not in "".join(tuplecontents):
+ separator = ", "
+ return "{" + separator.join(tuplecontents) + "}"
+ elif isinstance(term, list):
+ listcontents = [gen_erlang(e, level=level+1) for e in term]
+ return "[" + separator.join(listcontents) + "]"
+ else:
+ print "unknown type", type(term)
+ sys.exit(1)
+
+saslconfig = [(Symbol("sasl_error_logger"), Symbol("false")),
+ (Symbol("errlog_type"), Symbol("error")),
+ (Symbol("error_logger_mf_dir"), "log"),
+ (Symbol("error_logger_mf_maxbytes"), 10485760),
+ (Symbol("error_logger_mf_maxfiles"), 10),
+ ]
+
+def parse_address(address):
+ parsed_address = address.split(":")
+ if len(parsed_address) != 2:
+ print >>sys.stderr, "Invalid address format", address
+ sys.exit(1)
+ return (parsed_address[0], int(parsed_address[1]))
+
+def get_node_config(nodename, config):
+ nodetype = None
+ nodeconfig = None
+ for t in ["frontendnodes", "storagenodes", "signingnodes"]:
+ for node in config[t]:
+ if node["name"] == nodename:
+ nodetype = t
+ nodeconfig = node
+ if nodeconfig == None:
+ print >>sys.stderr, "Cannot find config for node", nodename
+ sys.exit(1)
+ return (nodetype, nodeconfig)
+
+def gen_https_servers(nodetype, nodeconfig):
+ if nodetype == "frontendnodes":
+ (publichost, publicport) = parse_address(nodeconfig["publicaddress"])
+ (host, port) = parse_address(nodeconfig["address"])
+ return [(Symbol("external_https_api"), publichost, publicport, Symbol("v1")),
+ (Symbol("frontend_https_api"), host, port, Symbol("frontend"))]
+ elif nodetype == "storagenodes":
+ (host, port) = parse_address(nodeconfig["address"])
+ return [(Symbol("storage_https_api"), host, port, Symbol("storage"))]
+ elif nodetype == "signingnodes":
+ (host, port) = parse_address(nodeconfig["address"])
+ return [(Symbol("signing_https_api"), host, port, Symbol("signing"))]
+
+def allowed_clients_frontend(mergenodenames):
+ return [
+ ("/ct/frontend/sendentry", mergenodenames),
+ ("/ct/frontend/sendlog", mergenodenames),
+ ("/ct/frontend/sendsth", mergenodenames),
+ ("/ct/frontend/currentposition", mergenodenames),
+ ("/ct/frontend/missingentries", mergenodenames),
+ ]
+
+def allowed_clients_public():
+ noauth = Symbol("noauth")
+ return [
+ ("/ct/v1/add-chain", noauth),
+ ("/ct/v1/add-pre-chain", noauth),
+ ("/ct/v1/get-sth", noauth),
+ ("/ct/v1/get-sth-consistency", noauth),
+ ("/ct/v1/get-proof-by-hash", noauth),
+ ("/ct/v1/get-entries", noauth),
+ ("/ct/v1/get-entry-and-proof", noauth),
+ ("/ct/v1/get-roots", noauth),
+ ]
+
+def allowed_clients_signing(frontendnodenames, mergenodenames):
+ return [
+ ("/ct/signing/sct", frontendnodenames),
+ ("/ct/signing/sth", mergenodenames),
+ ]
+
+def allowed_clients_storage(frontendnodenames, mergenodenames):
+ return [
+ ("/ct/storage/sendentry", frontendnodenames),
+ ("/ct/storage/entrycommitted", frontendnodenames),
+ ("/ct/storage/fetchnewentries", mergenodenames),
+ ("/ct/storage/getentry", mergenodenames),
+ ]
+
+def allowed_servers_frontend(signingnodenames, storagenodenames):
+ return [
+ ("/ct/storage/sendentry", storagenodenames),
+ ("/ct/storage/entrycommitted", storagenodenames),
+ ("/ct/signing/sct", signingnodenames),
+ ]
+
+def gen_config(nodename, config, localconfig):
+ print "generating config for", nodename
+ paths = localconfig["paths"]
+ options = localconfig.get("options", [])
+
+ configfile = open(paths["configdir"] + nodename + ".config", "w")
+ print >>configfile, "%% catlfish configuration file (-*- erlang -*-)"
+
+ (nodetype, nodeconfig) = get_node_config(nodename, config)
+ https_servers = gen_https_servers(nodetype, nodeconfig)
+
+ catlfishconfig = []
+ plopconfig = []
+
+ if nodetype == "frontendnodes":
+ catlfishconfig.append((Symbol("known_roots_path"), localconfig["paths"]["knownroots"]))
+ if "sctcaching" in options:
+ catlfishconfig.append((Symbol("sctcache_root_path"), paths["db"] + "sctcache/"))
+
+ catlfishconfig += [
+ (Symbol("https_servers"), https_servers),
+ (Symbol("https_certfile"), paths["https_certfile"]),
+ (Symbol("https_keyfile"), paths["https_keyfile"]),
+ (Symbol("https_cacertfile"), paths["https_cacertfile"]),
+ ]
+
+ lagerconfig = [
+ (Symbol("handlers"), [
+ (Symbol("lager_console_backend"), Symbol("info")),
+ (Symbol("lager_file_backend"), [(Symbol("file"), nodename + "-error.log"), (Symbol("level"), Symbol("error"))]),
+ (Symbol("lager_file_backend"), [(Symbol("file"), nodename + "-debug.log"), (Symbol("level"), Symbol("debug"))]),
+ (Symbol("lager_file_backend"), [(Symbol("file"), nodename + "-console.log"), (Symbol("level"), Symbol("info"))]),
+ ])
+ ]
+
+ if nodetype in ("frontendnodes", "storagenodes"):
+ plopconfig += [
+ (Symbol("entry_root_path"), paths["db"] + "certentries/"),
+ ]
+ if nodetype == "frontendnodes":
+ plopconfig += [
+ (Symbol("index_path"), paths["db"] + "index"),
+ ]
+ elif nodetype == "storagenodes":
+ plopconfig += [
+ (Symbol("newentries_path"), paths["db"] + "newentries"),
+ ]
+ if nodetype in ("frontendnodes", "storagenodes"):
+ plopconfig += [
+ (Symbol("entryhash_root_path"), paths["db"] + "entryhash/"),
+ (Symbol("indexforhash_root_path"), paths["db"] + "certindex/"),
+ ]
+ if nodetype == "frontendnodes":
+ plopconfig += [
+ (Symbol("sth_path"), paths["db"] + "sth"),
+ ]
+
+ signingnode = config["signingnodes"][0]
+ mergenodenames = [node["name"] for node in config["mergenodes"]]
+ storagenodeaddresses = ["https://%s/ct/storage/" % node["address"] for node in config["storagenodes"]]
+ frontendnodenames = [node["name"] for node in config["frontendnodes"]]
+
+ allowed_clients = []
+ allowed_servers = []
+
+ if nodetype == "frontendnodes":
+ storagenodenames = [node["name"] for node in config["storagenodes"]]
+ plopconfig.append((Symbol("storage_nodes"), storagenodeaddresses))
+ plopconfig.append((Symbol("storage_nodes_quorum"), config["storage-quorum-size"]))
+ services = [Symbol("ht")]
+ allowed_clients += allowed_clients_frontend(mergenodenames)
+ allowed_clients += allowed_clients_public()
+ allowed_servers += allowed_servers_frontend([signingnode["name"]], storagenodenames)
+ elif nodetype == "storagenodes":
+ allowed_clients += allowed_clients_storage(frontendnodenames, mergenodenames)
+ services = []
+ elif nodetype == "signingnodes":
+ allowed_clients += allowed_clients_signing(frontendnodenames, mergenodenames)
+ services = [Symbol("sign")]
+
+ plopconfig += [
+ (Symbol("publickey_path"), paths["publickeys"]),
+ (Symbol("services"), services),
+ ]
+ if nodetype == "signingnodes":
+ plopconfig.append((Symbol("log_private_key"), paths["logprivatekey"]))
+ plopconfig += [
+ (Symbol("log_public_key"), paths["logpublickey"]),
+ (Symbol("own_key"), (nodename, "%s/%s-private.pem" % (paths["privatekeys"], nodename))),
+ ]
+ if nodetype == "frontendnodes":
+ plopconfig.append((Symbol("signing_node"), "https://%s/ct/signing/" % signingnode["address"]))
+ plopconfig += [
+ (Symbol("allowed_clients"), allowed_clients),
+ (Symbol("allowed_servers"), allowed_servers),
+ ]
+
+ erlangconfig = [
+ (Symbol("sasl"), saslconfig),
+ (Symbol("catlfish"), catlfishconfig),
+ (Symbol("lager"), lagerconfig),
+ (Symbol("plop"), plopconfig),
+ ]
+
+ print >>configfile, gen_erlang(erlangconfig) + ".\n"
+
+ configfile.close()
+
+
+def gen_testmakefile(config, testmakefile, machines):
+ configfile = open(testmakefile, "w")
+ frontendnodenames = [node["name"] for node in config["frontendnodes"]]
+ storagenodenames = [node["name"] for node in config["storagenodes"]]
+ signingnodename = [node["name"] for node in config["signingnodes"]]
+
+ frontendnodeaddresses = [node["publicaddress"] for node in config["frontendnodes"]]
+ storagenodeaddresses = [node["address"] for node in config["storagenodes"]]
+ signingnodeaddresses = [node["address"] for node in config["signingnodes"]]
+
+ print >>configfile, "NODES=" + " ".join(frontendnodenames+storagenodenames+signingnodename)
+ print >>configfile, "MACHINES=" + " ".join([str(e) for e in range(1, machines+1)])
+ print >>configfile, "TESTURLS=" + " ".join(frontendnodeaddresses+storagenodeaddresses+signingnodeaddresses)
+ print >>configfile, "BASEURL=" + config["baseurl"]
+
+ configfile.close()
+
+
+def main():
+ parser = argparse.ArgumentParser(description="")
+ parser.add_argument('--config', help="System configuration", required=True)
+ parser.add_argument('--localconfig', help="Local configuration")
+ parser.add_argument("--testmakefile", metavar="file", help="Generate makefile variables for test")
+ parser.add_argument("--machines", type=int, metavar="n", help="Number of machines")
+ args = parser.parse_args()
+
+ config = yaml.load(open(args.config))
+ if args.testmakefile and args.machines:
+ gen_testmakefile(config, args.testmakefile, args.machines)
+ elif args.localconfig:
+ localconfig = yaml.load(open(args.localconfig))
+ localnodes = localconfig["localnodes"]
+ for localnode in localnodes:
+ gen_config(localnode, config, localconfig)
+ else:
+ print >>sys.stderr, "Nothing to do"
+ sys.exit(1)
+
+main()