summaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rwxr-xr-xtools/compileconfig.py66
-rwxr-xr-xtools/merge_fetch.py23
-rwxr-xr-xtools/testcase1.py8
3 files changed, 62 insertions, 35 deletions
diff --git a/tools/compileconfig.py b/tools/compileconfig.py
index 6ecfcad..d5a22df 100755
--- a/tools/compileconfig.py
+++ b/tools/compileconfig.py
@@ -434,7 +434,27 @@ def gen_config(nodename, config, localconfig):
plopconfigfile.close()
-def gen_testmakefile(config, testmakefile, machines, shellvars=False):
+def print_nodevar(configfile, delimiter, nodetype, bootname, names, testaddresses, gennodes=True, gentesturl=True):
+ if gennodes:
+ print >>configfile, nodetype + "NODES=" + delimiter + " ".join(['%s:%s' % (name, bootname) for name in names]) + delimiter
+ for name in names:
+ print >>configfile, name.upper().replace("-", "_") + "ONLY" + "NODES=" + delimiter + '%s:%s' % (name, bootname) + delimiter
+ if gentesturl:
+ print >>configfile, name.upper().replace("-", "_") + "ONLY" + "TESTURLS=" + delimiter + " ".join(testaddresses[name]) + delimiter
+ else:
+ print >>configfile, nodetype + "NODES=" + delimiter + delimiter
+ if gentesturl:
+ print >>configfile, nodetype + "TESTURLS=" + delimiter + " ".join([address for name in names for address in testaddresses[name]]) + delimiter
+ else:
+ print >>configfile, nodetype + "TESTURLS=" + delimiter + delimiter
+
+def multivaldict(dictlist):
+ d = {}
+ for (key, value) in dictlist:
+ d.setdefault(key, []).append(value)
+ return d
+
+def gen_testmakefile(config, testmakefile, shellvars=False):
configfile = open(testmakefile, "w")
print >>configfile, "#", testmakefile, "generated by", sys.argv[0], datetime.now()
@@ -444,27 +464,26 @@ def gen_testmakefile(config, testmakefile, machines, shellvars=False):
mergenodenames = set([node["name"] for node in config["mergenodes"]])
mergesecondarynodenames = set([node["name"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]])
- frontendnodeaddresses = [node["publicaddress"] for node in config["frontendnodes"]]
- storagenodeaddresses = [node["address"] for node in config["storagenodes"]]
- signingnodeaddresses = [node["address"] for node in config["signingnodes"]]
- mergesecondarynodeaddresses = [node["address"] for node in config["mergenodes"] if node["name"] != config["primarymergenode"]]
+ frontendnodenames_except_last = sorted(frontendnodenames)[:-1]
+ frontendnodenames_except_first = sorted(frontendnodenames)[1:]
+
+ allnodes = config["frontendnodes"] + config["storagenodes"] + config["signingnodes"] + config["mergenodes"]
+ testaddresses = multivaldict([(node["name"], node["address"]) for node in allnodes])
+
delimiter = '"' if shellvars else ''
+
+ print_nodevar(configfile, delimiter, "FRONTEND", "catlfish", frontendnodenames, testaddresses)
+ print_nodevar(configfile, delimiter, "FRONTENDEXCEPTLAST", "catlfish", frontendnodenames_except_last, testaddresses)
+ print_nodevar(configfile, delimiter, "FRONTENDEXCEPTFIRST", "catlfish", frontendnodenames_except_first, testaddresses)
+ print_nodevar(configfile, delimiter, "STORAGE", "catlfish", storagenodenames, testaddresses, gennodes=False)
+ print_nodevar(configfile, delimiter, "SIGNING", "catlfish", signingnodenames, testaddresses)
+ print_nodevar(configfile, delimiter, "MERGESECONDARY", "catlfish", mergesecondarynodenames, testaddresses)
+ print_nodevar(configfile, delimiter, "MERGEPRIMARY", "merge", [config["primarymergenode"]], testaddresses, gentesturl=False)
- print >>configfile, "NODES=" + delimiter + " ".join(frontendnodenames|storagenodenames|signingnodenames|mergenodenames) + delimiter
-
- print >>configfile, "FRONTENDNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in frontendnodenames]) + delimiter
- print >>configfile, "STORAGENODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in storagenodenames if name not in frontendnodenames]) + delimiter
- print >>configfile, "SIGNINGNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in signingnodenames]) + delimiter
- print >>configfile, "MERGESECONDARYNODES=" + delimiter + " ".join(['%s:%s' % (name, 'catlfish') for name in mergesecondarynodenames]) + delimiter
- print >>configfile, "MERGEPRIMARYNODES=" + delimiter + '%s:%s' % (config["primarymergenode"], 'merge') + delimiter
-
- print >>configfile, "MACHINES=" + delimiter + " ".join([str(e) for e in range(1, machines+1)]) + delimiter
- print >>configfile, "FRONTENDTESTURLS=" + delimiter + " ".join(frontendnodeaddresses) + delimiter
- print >>configfile, "STORAGETESTURLS=" + delimiter + " ".join(storagenodeaddresses) + delimiter
- print >>configfile, "SIGNINGTESTURLS=" + delimiter + " ".join(signingnodeaddresses) + delimiter
- print >>configfile, "MERGESECONDARYTESTURLS=" + delimiter + " ".join(mergesecondarynodeaddresses) + delimiter
- print >>configfile, "MERGEPRIMARYTESTURLS=" + delimiter + " ".join([]) + delimiter
+ print >>configfile, "NODES=" + delimiter + " ".join(set([node["name"] for node in allnodes])) + delimiter
+
+ print >>configfile, "MACHINES=" + delimiter + " ".join([str(e) for e in frontendnodenames]) + delimiter
print >>configfile, "BASEURL=" + delimiter + config["baseurl"] + delimiter
configfile.close()
@@ -484,15 +503,14 @@ def main():
parser.add_argument("--testmakefile", metavar="file", help="Generate makefile variables for test")
parser.add_argument("--testshellvars", metavar="file", help="Generate shell variable file for test")
parser.add_argument("--getnodenames", action='store_true', help="Get list of node names")
- parser.add_argument("--machines", type=int, metavar="n", help="Number of machines")
args = parser.parse_args()
- if args.testmakefile and args.machines:
+ if args.testmakefile:
config = readconfig.read_config(args.config)
- gen_testmakefile(config, args.testmakefile, args.machines)
- elif args.testshellvars and args.machines:
+ gen_testmakefile(config, args.testmakefile)
+ elif args.testshellvars:
config = readconfig.read_config(args.config)
- gen_testmakefile(config, args.testshellvars, args.machines, shellvars=True)
+ gen_testmakefile(config, args.testshellvars, shellvars=True)
elif args.getnodenames:
config = readconfig.read_config(args.config)
printnodenames(config)
diff --git a/tools/merge_fetch.py b/tools/merge_fetch.py
index 42a3089..e71d3f1 100755
--- a/tools/merge_fetch.py
+++ b/tools/merge_fetch.py
@@ -38,14 +38,17 @@ def merge_fetch(args, config, localconfig):
entries_to_fetch = {}
for storagenode in storagenodes:
- print >>sys.stderr, "getting new entries from", storagenode["name"]
- sys.stderr.flush()
- new_entries_per_node[storagenode["name"]] = \
- set(get_new_entries(storagenode["name"],
- "https://%s/" % storagenode["address"],
- own_key, paths))
- new_entries.update(new_entries_per_node[storagenode["name"]])
- entries_to_fetch[storagenode["name"]] = []
+ try:
+ print >>sys.stderr, "getting new entries from", storagenode["name"]
+ sys.stderr.flush()
+ new_entries_per_node[storagenode["name"]] = \
+ set(get_new_entries(storagenode["name"],
+ "https://%s/" % storagenode["address"],
+ own_key, paths))
+ new_entries.update(new_entries_per_node[storagenode["name"]])
+ entries_to_fetch[storagenode["name"]] = []
+ except requests.exceptions.ConnectionError:
+ pass
timing_point(timing, "get new entries")
new_entries -= certsinlog
@@ -54,6 +57,8 @@ def merge_fetch(args, config, localconfig):
for ehash in new_entries:
for storagenode in storagenodes:
+ if storagenode["name"] not in new_entries_per_node:
+ continue
if ehash in new_entries_per_node[storagenode["name"]]:
entries_to_fetch[storagenode["name"]].append(ehash)
break
@@ -64,6 +69,8 @@ def merge_fetch(args, config, localconfig):
added_entries = 0
for storagenode in storagenodes:
+ if storagenode["name"] not in entries_to_fetch:
+ continue
print >>sys.stderr, "getting %d entries from %s:" % \
(len(entries_to_fetch[storagenode["name"]]), storagenode["name"]),
sys.stderr.flush()
diff --git a/tools/testcase1.py b/tools/testcase1.py
index fb081e5..a616841 100755
--- a/tools/testcase1.py
+++ b/tools/testcase1.py
@@ -192,11 +192,11 @@ def mergestatus(configfile, localconfigfile):
def merge(expected=None, wait=0):
rv = subprocess.call([toolsdir + "/merge", "--config", testdir + "/catlfish-test.cfg",
- "--localconfig", testdir + "/catlfish-test-local-merge.cfg"])
+ "--localconfig", testdir + "/catlfish-test-local-merge-1.cfg"])
if rv:
return rv
for i in range(10):
- mergestatus(testdir + "/catlfish-test.cfg", testdir + "/catlfish-test-local-merge.cfg")
+ mergestatus(testdir + "/catlfish-test.cfg", testdir + "/catlfish-test-local-merge-1.cfg")
if i < wait:
sleep(1)
continue
@@ -214,6 +214,7 @@ for baseurl in baseurls:
testgroup("cert1")
result1 = do_add_chain(cc1, baseurls[0])
+result2 = do_add_chain(cc1, baseurls[0])
mergeresult = merge(expected=1)
assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True)
@@ -224,9 +225,10 @@ for baseurl in baseurls:
print_and_check_tree_size(1, baseurl)
size_sth[1] = base64.b64decode(get_sth(baseurls[0])["sha256_root_hash"])
-result2 = do_add_chain(cc1, baseurls[0])
+result3 = do_add_chain(cc1, baseurls[0])
assert_equal(result2["timestamp"], result1["timestamp"], "timestamp")
+assert_equal(result3["timestamp"], result1["timestamp"], "timestamp")
mergeresult = merge(expected=1, wait=3)
assert_equal(mergeresult, 0, "merge", quiet=True, fatal=True)