Added formalized config defaults

Refactor reading of config
Move config schemas to separate file

3 files changed, 97 insertions, 77 deletions

diff --git a/tools/compileconfig.py b/tools/compileconfig.py
index 35ecb91..e6c5ec6 100755
--- a/tools/compileconfig.py
+++ b/tools/compileconfig.py
@@ -566,65 +566,6 @@ def gen_manpage(manpagedir):
     manpage.rewrite_manpage(manpagedir + "/catlfish-log.cfg.in.5.adoc", globalconfigschema, "Catlfish", "Catlfish Manual", "CATLFISH-LOG.CFG.IN(5)", "catlfish-log.cfg.in - catlfish log configuration")
     manpage.rewrite_manpage(manpagedir + "/catlfish-node.cfg.5.adoc", localconfigschema, "Catlfish", "Catlfish Manual", "CATLFISH-NODE.CFG(5)", "catlfish-node.cfg - catlfish node configuration")
 
-localconfigschema = [
-    ("nodename", "string", "nodename"),
-    ("frontendaddress", "string", "ip address"),
-    ("ctapiaddress", "string", "ip address"),
-    ("storageaddress", "string", "ip address"),
-    ("signingaddress", "string", "ip address"),
-    ("mergeaddress", "string", "ip address"),
-    ("publichttpaddress", "string", "ip address"),
-    ("configurl", "string", "url"),
-    ("logadminkey", "string", "key"),
-    ("dbbackend", "string", ["permdb", "fsdb"]),
-    ("paths/configdir", "string", "path"),
-    ("paths/db", "string", "path"),
-    ("paths/https_cacertfile", "string", "path"),
-    ("paths/https_certfile", "string", "path"),
-    ("paths/https_keyfile", "string", "path"),
-    ("paths/knownroots", "string", "path"),
-    ("paths/logpublickey", "string", "path"),
-    ("paths/privatekeys", "string", "path"),
-    ("paths/public_cacertfile", "string", "path"),
-    ("paths/publickeys", "string", "path"),
-    ("paths/verifycert_bin", "string", "path"),
-    ("paths/mergedb", "string", "path"),
-    ("paths/logprivatekey", "string", "path"),
-    ("ratelimits/add_chain", "string", "rate"),
-    ("merge/min-delay", "integer", "seconds"),
-    ("merge/backup-window-size", "integer", "number of entries"),
-    ("merge/backup-sendlog-chunksize", "integer", "number of entries"),
-    ("merge/backup-sendentries-chunksize", "integer", "number of entries"),
-    ("merge/dist-window-size", "integer", "number of entries"),
-    ("merge/dist-sendlog-chunksize", "integer", "number of entries"),
-    ("merge/dist-sendentries-chunksize", "integer", "number of entries"),
-]
-
-globalconfigschema = [
-    ("frontendnodes/[]/name", "string", "nodename"),
-    ("frontendnodes/[]/address", "string", "ip address"),
-    ("frontendnodes/[]/publicaddress", "string", "ip address"),
-    ("mergenodes/[]/name", "string", "nodename"),
-    ("mergenodes/[]/address", "string", "ip address"),
-    ("signingnodes/[]/name", "string", "nodename"),
-    ("signingnodes/[]/address", "string", "ip address"),
-    ("storagenodes/[]/name", "string", "nodename"),
-    ("storagenodes/[]/address", "string", "ip address"),
-    ("statusservers/[]/name", "string", "nodename"),
-    ("statusservers/[]/address", "string", "ip address"),
-    ("statusservers/[]/publicaddress", "string", "ip address"),
-    ("apikeys/[]/nodename", "string", "nodename"),
-    ("apikeys/[]/publickey", "string", "key"),
-    ("baseurl", "string", "url"),
-    ("primarymergenode", "string", "nodename"),
-    ("storage-quorum-size", "integer", "number of nodes"),
-    ("backup-quorum-size", "integer", "number of nodes"),
-    ("logpublickey", "string", "key"),
-    ("cafingerprint", "string", "fingerprint"),
-    ("version", "integer", "version"),
-    ("mmd", "integer", "seconds"),
-]
-
 def main():
     parser = argparse.ArgumentParser(description="")
     parser.add_argument('--config', help="System configuration")
@@ -644,17 +585,17 @@ def main():
         sys.exit(1)
 
     if args.testmakefile:
-        config = readconfig.read_config(args.config, schema=globalconfigschema)
+        config = readconfig.read_config(args.config, localconfig=False)
         gen_testmakefile(config, args.testmakefile)
     elif args.testshellvars:
-        config = readconfig.read_config(args.config, schema=globalconfigschema)
+        config = readconfig.read_config(args.config, localconfig=False)
         gen_testmakefile(config, args.testshellvars, shellvars=True)
     elif args.getnodenames:
-        config = readconfig.read_config(args.config, schema=globalconfigschema)
+        config = readconfig.read_config(args.config, localconfig=False)
         printnodenames(config)
     elif args.localconfig:
-        localconfig = readconfig.read_config(args.localconfig, schema=localconfigschema)
-        config = readconfig.verify_and_read_config(args.config, localconfig["logadminkey"], schema=globalconfigschema)
+        localconfig = readconfig.read_config(args.localconfig)
+        config = readconfig.verify_and_read_config(args.config, localconfig["logadminkey"])
 
         gen_config(localconfig["nodename"], config, localconfig)
     else:
diff --git a/tools/configschema.py b/tools/configschema.py
new file mode 100644
index 0000000..9bb4b15
--- /dev/null
+++ b/tools/configschema.py
@@ -0,0 +1,65 @@
+# Copyright (c) 2017, NORDUnet A/S.
+# See LICENSE for licensing information.
+
+localconfigschema = [
+    ("nodename", "string", "nodename"),
+    ("frontendaddress", "string", "ip address"),
+    ("ctapiaddress", "string", "ip address"),
+    ("storageaddress", "string", "ip address"),
+    ("signingaddress", "string", "ip address"),
+    ("mergeaddress", "string", "ip address"),
+    ("publichttpaddress", "string", "ip address"),
+    ("configurl", "string", "url"),
+    ("logadminkey", "string", "key"),
+    ("dbbackend", "string", ["permdb", "fsdb"]),
+    ("paths/configdir", "string", "path"),
+    ("paths/db", "string", "path"),
+    ("paths/https_cacertfile", "string", "path"),
+    ("paths/https_certfile", "string", "path"),
+    ("paths/https_keyfile", "string", "path"),
+    ("paths/knownroots", "string", "path"),
+    ("paths/logpublickey", "string", "path"),
+    ("paths/privatekeys", "string", "path"),
+    ("paths/public_cacertfile", "string", "path"),
+    ("paths/publickeys", "string", "path"),
+    ("paths/verifycert_bin", "string", "path"),
+    ("paths/mergedb", "string", "path"),
+    ("paths/logprivatekey", "string", "path"),
+    ("ratelimits/add_chain", "string", "rate"),
+    ("merge/min-delay", "integer", "seconds"),
+    ("merge/backup-window-size", "integer", "number of entries"),
+    ("merge/backup-sendlog-chunksize", "integer", "number of entries"),
+    ("merge/backup-sendentries-chunksize", "integer", "number of entries"),
+    ("merge/dist-window-size", "integer", "number of entries"),
+    ("merge/dist-sendlog-chunksize", "integer", "number of entries"),
+    ("merge/dist-sendentries-chunksize", "integer", "number of entries"),
+]
+
+globalconfigschema = [
+    ("frontendnodes/[]/name", "string", "nodename"),
+    ("frontendnodes/[]/address", "string", "ip address"),
+    ("frontendnodes/[]/publicaddress", "string", "ip address"),
+    ("mergenodes/[]/name", "string", "nodename"),
+    ("mergenodes/[]/address", "string", "ip address"),
+    ("signingnodes/[]/name", "string", "nodename"),
+    ("signingnodes/[]/address", "string", "ip address"),
+    ("storagenodes/[]/name", "string", "nodename"),
+    ("storagenodes/[]/address", "string", "ip address"),
+    ("statusservers/[]/name", "string", "nodename"),
+    ("statusservers/[]/address", "string", "ip address"),
+    ("statusservers/[]/publicaddress", "string", "ip address"),
+    ("apikeys/[]/nodename", "string", "nodename"),
+    ("apikeys/[]/publickey", "string", "key"),
+    ("baseurl", "string", "url"),
+    ("primarymergenode", "string", "nodename"),
+    ("storage-quorum-size", "integer", "number of nodes"),
+    ("backup-quorum-size", "integer", "number of nodes"),
+    ("logpublickey", "string", "key"),
+    ("cafingerprint", "string", "fingerprint"),
+    ("version", "integer", "version"),
+    ("mmd", "integer", "seconds"),
+]
+
+globalconfigdefaults = [
+    ("statusservers", []),
+]
diff --git a/tools/readconfig.py b/tools/readconfig.py
index 15b9c61..9c023b1 100644
--- a/tools/readconfig.py
+++ b/tools/readconfig.py
@@ -4,6 +4,7 @@ import hashlib
 import yaml
 import base64
 import sys
+import configschema
 
 def render_path(path):
     if path:
@@ -47,18 +48,12 @@ def verify_config(rawconfig, signature, publickey_base64, filename):
     except ecdsa.keys.BadSignatureError:
         print >>sys.stderr, "error: configuration file %s did not have a correct signature" % (filename,)
         sys.exit(1)
+    return common_read_config(io.BytesIO(rawconfig), filename, localconfig=False)
 
-    return errorhandlify(yaml.load(io.BytesIO(rawconfig), yaml.SafeLoader), filename)
-
-def verify_and_read_config(filename, publickey_base64, schema=None):
+def verify_and_read_config(filename, publickey_base64):
     rawconfig = open(filename).read()
     signature = open(filename + ".sig").read()
-
-    verify_config(rawconfig, signature, publickey_base64, filename)
-    config = yaml.load(io.BytesIO(rawconfig), yaml.SafeLoader)
-    if schema:
-        check_config_schema(config, schema)
-    return errorhandlify(config, filename)
+    return verify_config(rawconfig, signature, publickey_base64, filename)
 
 def insert_schema_path(schema, path, datatype, highleveldatatype):
     if len(path) == 1:
@@ -122,9 +117,28 @@ def check_config_schema_part(term, schema, path=[]):
         print >>sys.stderr, "unknown type", type(term)
         sys.exit(1)
 
+def insert_defaults(config, configdefaults):
+    for (rawpath, value) in configdefaults:
+        path = rawpath.split("/")
+        node = config
+        for e in path[:-1]:
+            assert(e != "[]")
+            node = node[e]
+        lastelem = path[-1]
+        if lastelem not in node:
+            node[lastelem] = value
 
-def read_config(filename, schema=None):
-    config = yaml.load(open(filename), yaml.SafeLoader)
-    if schema:
-        check_config_schema(config, schema)
+def common_read_config(f, filename, localconfig=True):
+    if localconfig:
+        schema = configschema.localconfigschema
+        configdefaults = []
+    else:
+        schema = configschema.globalconfigschema
+        configdefaults = configschema.globalconfigdefaults
+    config = yaml.load(f, yaml.SafeLoader)
+    insert_defaults(config, configdefaults)
+    check_config_schema(config, schema)
     return errorhandlify(config, filename)
+
+def read_config(filename, localconfig=True):
+    return common_read_config(open(filename), filename, localconfig=localconfig)