From e0be874a28c9ee9c9b07e3cff89301cd58cfd31f Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Wed, 26 Jul 2017 00:40:13 +0200 Subject: Allow optional reading only for optional configuration keys Add storage-sign-quorum-size configuration key. Add default for dbbackend configuration key. --- tools/compileconfig.py | 7 +++---- tools/configschema.py | 9 +++++++++ tools/merge_fetch.py | 2 +- tools/mergestatus.py | 2 +- tools/readconfig.py | 26 +++++++++++++++++++------- tools/storagegc.py | 2 +- tools/testcase1.py | 2 +- 7 files changed, 35 insertions(+), 15 deletions(-) (limited to 'tools') diff --git a/tools/compileconfig.py b/tools/compileconfig.py index 264cb42..199e446 100755 --- a/tools/compileconfig.py +++ b/tools/compileconfig.py @@ -248,7 +248,6 @@ def gen_config(nodename, config, localconfig): } bind_publicaddress = localconfig.get("ctapiaddress") bind_publichttpaddress = localconfig.get("publichttpaddress") - options = localconfig.get("options", []) configfile = open(paths["configdir"] + "/" + nodename + ".config", "w") print >>configfile, "%% catlfish configuration file (-*- erlang -*-)" @@ -373,7 +372,7 @@ def gen_config(nodename, config, localconfig): allowed_servers = [] storagenodenames = [node["name"] for node in config["storagenodes"]] services = set() - storage_sign_quorum = config.get("storage-sign-quorum-size", 0) + storage_sign_quorum = config["storage-sign-quorum-size"] allnodenames = set(signingnodenames + mergenodenames + frontendnodenames + statusservernames + storagenodenames) @@ -526,7 +525,7 @@ def gen_testmakefile(config, testmakefile, shellvars=False): signingnodenames = set([node["name"] for node in config["signingnodes"]]) mergenodenames = set([node["name"] for node in config["mergenodes"]]) mergesecondarynodenames = set([node["name"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]]) - statusservernodenames = set([node["name"] for node in config.get("statusservers")]) + statusservernodenames = set([node["name"] for node in config["statusservers"]]) frontendnodenames_except_last = sorted(frontendnodenames)[:-1] frontendnodenames_except_first = sorted(frontendnodenames)[1:] @@ -558,7 +557,7 @@ def printnodenames(config): storagenodenames = set([node["name"] for node in config["storagenodes"]]) signingnodenames = set([node["name"] for node in config["signingnodes"]]) mergenodenames = set([node["name"] for node in config["mergenodes"]]) - statusservernodenames = set([node["name"] for node in config.get("statusservers")]) + statusservernodenames = set([node["name"] for node in config["statusservers"]]) print " ".join(frontendnodenames|storagenodenames|signingnodenames|mergenodenames|statusservernodenames) diff --git a/tools/configschema.py b/tools/configschema.py index 9f4a468..e78c8c7 100644 --- a/tools/configschema.py +++ b/tools/configschema.py @@ -52,6 +52,7 @@ globalconfigschema = [ ("primarymergenode", "string", "nodename"), ("storage-quorum-size", "integer", "number of nodes"), ("backup-quorum-size", "integer", "number of nodes"), + ("storage-sign-quorum-size", "integer", "number of nodes"), ("logpublickey", "string", "key"), ("cafingerprint", "string", "fingerprint"), ("version", "integer", "version"), @@ -60,4 +61,12 @@ globalconfigschema = [ globalconfigdefaults = [ ("statusservers", []), + ("storage-sign-quorum-size", 0), ] + +localconfigdefaults = [ + ("dbbackend", "fsdb"), +] + +localconfigoptionals = set(["storageaddress", "frontendaddress", "signingaddress", "mergeaddress", "ctapiaddress", "publichttpaddress", "hsm"]) +globalconfigoptionals = set() diff --git a/tools/merge_fetch.py b/tools/merge_fetch.py index 6b0a110..d0fbaa7 100755 --- a/tools/merge_fetch.py +++ b/tools/merge_fetch.py @@ -88,7 +88,7 @@ def merge_fetch(args, config, localconfig, currentsizefile): "%s/%s-private.pem" % (paths["privatekeys"], localconfig["nodename"])) assert(localconfig["nodename"] == config["primarymergenode"]) - statusservers = config.get("statusservers") + statusservers = config["statusservers"] timing = timing_point() report_timing_point(timing, statusservers, own_key, None) diff --git a/tools/mergestatus.py b/tools/mergestatus.py index 9dbbb7b..863f6a1 100755 --- a/tools/mergestatus.py +++ b/tools/mergestatus.py @@ -21,7 +21,7 @@ from certtools import create_ssl_context, get_public_key_from_file, \ def main(): args, config, localconfig = parse_args() paths = localconfig["paths"] - mergenodes = config.get("mergenodes", []) + mergenodes = config["mergenodes"] mergedb = paths["db"] sthfile = mergedb + "/sth" currentsizefile = mergedb + "/fetched" diff --git a/tools/readconfig.py b/tools/readconfig.py index d883815..b9a0b07 100644 --- a/tools/readconfig.py +++ b/tools/readconfig.py @@ -12,28 +12,38 @@ def render_path(path): else: return "the top level" +import traceback + class ErrorHandlingDict(dict): - def __init__(self, filename, path): + def __init__(self, filename, path, optionals): self._filename = filename self._path = path + self._optionals = optionals dict.__init__({}) + def get(self, key, default=None): + path = "/".join(self._path + [key]) + if path in self._optionals: + return dict.get(self, key, default) + print >>sys.stderr, "error: read key", path, "optionally with default", default + traceback.print_stack() + sys.exit(1) def __missing__(self, key): path = render_path(self._path) print >>sys.stderr, "error: could not find configuration key '%s' at %s in %s" % (key, path, self._filename) sys.exit(1) -def errorhandlify(term, filename, path=[]): +def errorhandlify(term, filename, optionals, path=[]): if isinstance(term, basestring): return term elif isinstance(term, int): return term elif isinstance(term, dict): - result = ErrorHandlingDict(filename, path) + result = ErrorHandlingDict(filename, path, optionals) for k, v in term.items(): - result[k] = errorhandlify(v, filename, path + [k]) + result[k] = errorhandlify(v, filename, optionals, path + [k]) return result elif isinstance(term, list): - return [errorhandlify(e, filename, path + ["item %d" % i]) for i, e in enumerate(term, start=1)] + return [errorhandlify(e, filename, optionals, path + ["item %d" % i]) for i, e in enumerate(term, start=1)] else: print "unknown type", type(term) sys.exit(1) @@ -134,14 +144,16 @@ def insert_defaults(config, configdefaults): def common_read_config(f, filename, localconfig=True): if localconfig: schema = configschema.localconfigschema - configdefaults = [] + configdefaults = configschema.localconfigdefaults + optionals = configschema.localconfigoptionals else: schema = configschema.globalconfigschema configdefaults = configschema.globalconfigdefaults + optionals = configschema.globalconfigoptionals config = yaml.load(f, yaml.SafeLoader) insert_defaults(config, configdefaults) check_config_schema(config, schema) - return errorhandlify(config, filename) + return errorhandlify(config, filename, optionals) def read_config(filename, localconfig=True): return common_read_config(open(filename), filename, localconfig=localconfig) diff --git a/tools/storagegc.py b/tools/storagegc.py index 6360495..7680b99 100755 --- a/tools/storagegc.py +++ b/tools/storagegc.py @@ -22,7 +22,7 @@ config = readconfig.verify_and_read_config(args.config, localconfig["logadminkey paths = localconfig["paths"] db_path = paths["db"] -create_ssl_context(cafile=paths.get("public_cacertfile", None)) +create_ssl_context(cafile=paths["public_cacertfile"]) baseurl = config["baseurl"] diff --git a/tools/testcase1.py b/tools/testcase1.py index 7ec853a..164740f 100755 --- a/tools/testcase1.py +++ b/tools/testcase1.py @@ -163,7 +163,7 @@ def mergestatus(configfile, localconfigfile): config = readconfig.verify_and_read_config(configfile, localconfig["logadminkey"]) paths = localconfig["paths"] - mergenodes = config.get("mergenodes", []) + mergenodes = config["mergenodes"] mergedb = paths["db"] sthfile = mergedb + "/sth" currentsizefile = mergedb + "/fetched" -- cgit v1.1