diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/compileconfig.py | 66 | ||||
| -rwxr-xr-x | tools/merge_fetch.py | 23 | ||||
| -rwxr-xr-x | tools/testcase1.py | 8 |
3 files changed, 62 insertions, 35 deletions
diff --git a/tools/compileconfig.py b/tools/compileconfig.py index 6ecfcad..d5a22df 100755 --- a/tools/compileconfig.py +++ b/tools/compileconfig.py @@ -434,7 +434,27 @@ def gen_config(nodename, config, localconfig): plopconfigfile.close() -def gen_testmakefile(config, testmakefile, machines, shellvars=False): +def print_nodevar(configfile, delimiter, nodetype, bootname, names, testaddresses, gennodes=True, gentesturl=True): + if gennodes: + print >>configfile, nodetype + "NODES=" + delimiter + " ".join(['%s:%s' % (name, bootname) for name in names]) + delimiter + for name in names: + print >>configfile, name.upper().replace("-", "_") + "ONLY" + "NODES=" + delimiter + '%s:%s' % (name, bootname) + delimiter + if gentesturl: + print >>configfile, name.upper().replace("-", "_") + "ONLY" + "TESTURLS=" + delimiter + " ".join(testaddresses[name]) + delimiter + else: + print >>configfile, nodetype + "NODES=" + delimiter + delimiter + if gentesturl: + print >>configfile, nodetype + "TESTURLS=" + delimiter + " ".join([address for name in names for address in testaddresses[name]]) + delimiter + else: + print >>configfile, nodetype + "TESTURLS=" + delimiter + delimiter + +def multivaldict(dictlist): + d = {} + for (key, value) in dictlist: + d.setdefault(key, []).append(value) + return d + +def gen_testmakefile(config, testmakefile, shellvars=False): configfile = open(testmakefile, "w") print >>configfile, "#", testmakefile, "generated by", sys.argv[0], datetime.now() @@ -444,27 +464,26 @@ def gen_testmakefile(config, testmakefile, machines, shellvars=False): mergenodenames = set([node["name"] for node in config["mergenodes"]]) mergesecondarynodenames = set([node["name"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]]) - frontendnodeaddresses = [node["publicaddress"] for node in config["frontendnodes"]] - storagenodeaddresses = [node["address"] for node in config["storagenodes"]] - signingnodeaddresses = [node["address"] for node in config["signingnodes"]] - mergesecondarynodeaddresses = [node["address"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]] + frontendnodenames_except_last = sorted(frontendnodenames)[:-1] + frontendnodenames_except_first = sorted(frontendnodenames)[1:] + + allnodes = config["frontendnodes"] + config["storagenodes"] + config["signingnodes"] + config["mergenodes"] + testaddresses = multivaldict([(node["name"], node["address"]) for node in allnodes]) + delimiter = '"' if shellvars else '' + + print_nodevar(configfile, delimiter, "FRONTEND", "catlfish", frontendnodenames, testaddresses) + print_nodevar(configfile, delimiter, "FRONTENDEXCEPTLAST", "catlfish", frontendnodenames_except_last, testaddresses) + print_nodevar(configfile, delimiter, "FRONTENDEXCEPTFIRST", "catlfish", frontendnodenames_except_first, testaddresses) + print_nodevar(configfile, delimiter, "STORAGE", "catlfish", storagenodenames, testaddresses, gennodes=False) + print_nodevar(configfile, delimiter, "SIGNING", "catlfish", signingnodenames, testaddresses) + print_nodevar(configfile, delimiter, "MERGESECONDARY", "catlfish", mergesecondarynodenames, testaddresses) + print_nodevar(configfile, delimiter, "MERGEPRIMARY", "merge", [config["primarymergenode"]], testaddresses, gentesturl=False) - print >>configfile, "NODES=" + delimiter + " ".join(frontendnodenames|storagenodenames|signingnodenames|mergenodenames) + delimiter - - print >>configfile, "FRONTENDNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in frontendnodenames]) + delimiter - print >>configfile, "STORAGENODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in storagenodenames if name not in frontendnodenames]) + delimiter - print >>configfile, "SIGNINGNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in signingnodenames]) + delimiter - print >>configfile, "MERGESECONDARYNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in mergesecondarynodenames]) + delimiter - print >>configfile, "MERGEPRIMARYNODES=" + delimiter + '%s:%s' % (config["primarymergenode"], 'merge') + delimiter - - print >>configfile, "MACHINES=" + delimiter + " ".join([str(e) for e in range(1, machines+1)]) + delimiter - print >>configfile, "FRONTENDTESTURLS=" + delimiter + " ".join(frontendnodeaddresses) + delimiter - print >>configfile, "STORAGETESTURLS=" + delimiter + " ".join(storagenodeaddresses) + delimiter - print >>configfile, "SIGNINGTESTURLS=" + delimiter + " ".join(signingnodeaddresses) + delimiter - print >>configfile, "MERGESECONDARYTESTURLS=" + delimiter + " ".join(mergesecondarynodeaddresses) + delimiter - print >>configfile, "MERGEPRIMARYTESTURLS=" + delimiter + " ".join([]) + delimiter + print >>configfile, "NODES=" + delimiter + " ".join(set([node["name"] for node in allnodes])) + delimiter + + print >>configfile, "MACHINES=" + delimiter + " ".join([str(e) for e in frontendnodenames]) + delimiter print >>configfile, "BASEURL=" + delimiter + config["baseurl"] + delimiter configfile.close() @@ -484,15 +503,14 @@ def main(): parser.add_argument("--testmakefile", metavar="file", help="Generate makefile variables for test") parser.add_argument("--testshellvars", metavar="file", help="Generate shell variable file for test") parser.add_argument("--getnodenames", action='store_true', help="Get list of node names") - parser.add_argument("--machines", type=int, metavar="n", help="Number of machines") args = parser.parse_args() - if args.testmakefile and args.machines: + if args.testmakefile: config = readconfig.read_config(args.config) - gen_testmakefile(config, args.testmakefile, args.machines) - elif args.testshellvars and args.machines: + gen_testmakefile(config, args.testmakefile) + elif args.testshellvars: config = readconfig.read_config(args.config) - gen_testmakefile(config, args.testshellvars, args.machines, shellvars=True) + gen_testmakefile(config, args.testshellvars, shellvars=True) elif args.getnodenames: config = readconfig.read_config(args.config) printnodenames(config) diff --git a/tools/merge_fetch.py b/tools/merge_fetch.py index 42a3089..e71d3f1 100755 --- a/tools/merge_fetch.py +++ b/tools/merge_fetch.py @@ -38,14 +38,17 @@ def merge_fetch(args, config, localconfig): entries_to_fetch = {} for storagenode in storagenodes: - print >>sys.stderr, "getting new entries from", storagenode["name"] - sys.stderr.flush() - new_entries_per_node[storagenode["name"]] = \ - set(get_new_entries(storagenode["name"], - "https://%s/" % storagenode["address"], - own_key, paths)) - new_entries.update(new_entries_per_node[storagenode["name"]]) - entries_to_fetch[storagenode["name"]] = [] + try: + print >>sys.stderr, "getting new entries from", storagenode["name"] + sys.stderr.flush() + new_entries_per_node[storagenode["name"]] = \ + set(get_new_entries(storagenode["name"], + "https://%s/" % storagenode["address"], + own_key, paths)) + new_entries.update(new_entries_per_node[storagenode["name"]]) + entries_to_fetch[storagenode["name"]] = [] + except requests.exceptions.ConnectionError: + pass timing_point(timing, "get new entries") new_entries -= certsinlog @@ -54,6 +57,8 @@ def merge_fetch(args, config, localconfig): for ehash in new_entries: for storagenode in storagenodes: + if storagenode["name"] not in new_entries_per_node: + continue if ehash in new_entries_per_node[storagenode["name"]]: entries_to_fetch[storagenode["name"]].append(ehash) break @@ -64,6 +69,8 @@ def merge_fetch(args, config, localconfig): added_entries = 0 for storagenode in storagenodes: + if storagenode["name"] not in entries_to_fetch: + continue print >>sys.stderr, "getting %d entries from %s:" % \ (len(entries_to_fetch[storagenode["name"]]), storagenode["name"]), sys.stderr.flush() diff --git a/tools/testcase1.py b/tools/testcase1.py index fb081e5..a616841 100755 --- a/tools/testcase1.py +++ b/tools/testcase1.py @@ -192,11 +192,11 @@ def mergestatus(configfile, localconfigfile): def merge(expected=None, wait=0): rv = subprocess.call([toolsdir + "/merge", "--config", testdir + "/catlfish-test.cfg", - "--localconfig", testdir + "/catlfish-test-local-merge.cfg"]) + "--localconfig", testdir + "/catlfish-test-local-merge-1.cfg"]) if rv: return rv for i in range(10): - mergestatus(testdir + "/catlfish-test.cfg", testdir + "/catlfish-test-local-merge.cfg") + mergestatus(testdir + "/catlfish-test.cfg", testdir + "/catlfish-test-local-merge-1.cfg") if i < wait: sleep(1) continue @@ -214,6 +214,7 @@ for baseurl in baseurls: testgroup("cert1") result1 = do_add_chain(cc1, baseurls[0]) +result2 = do_add_chain(cc1, baseurls[0]) mergeresult = merge(expected=1) assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True) @@ -224,9 +225,10 @@ for baseurl in baseurls: print_and_check_tree_size(1, baseurl) size_sth[1] = base64.b64decode(get_sth(baseurls[0])["sha256_root_hash"]) -result2 = do_add_chain(cc1, baseurls[0]) +result3 = do_add_chain(cc1, baseurls[0]) assert_equal(result2["timestamp"], result1["timestamp"], "timestamp") +assert_equal(result3["timestamp"], result1["timestamp"], "timestamp") mergeresult = merge(expected=1, wait=3) assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True) |