Bug 890015 - Run test-masters.sh in parallel production
authorSteve Fink <sfink@mozilla.com>
Thu, 18 Jul 2013 11:13:33 -0700
branchproduction
changeset 9146 601d9ffd21c8cdc3609e0c5d680da152ed0e42ce
parent 9143 2472e7d36b62a927413dd00260a0f1a73d84f212
child 9150 244757ddbbfac5b14b529d7aa9eaefc2d5d0635e
push id1
push userroot
push dateWed, 17 Dec 2014 00:18:48 +0000
bugs890015
Bug 890015 - Run test-masters.sh in parallel
setup-master.py
test-masters.sh
--- a/setup-master.py
+++ b/setup-master.py
@@ -1,10 +1,13 @@
 #!/usr/bin/env python
-"""setup-master.py master_dir master_name
+"""
+  setup-master.py master_dir master_name
+  setup-master.py -t [masters...]
+  setup-master.py -l [--tested-only] [masters...]
 
 Sets up mozilla buildbot master in master_dir."""
 
 import os
 import glob
 import shutil
 import subprocess
 import urllib
@@ -307,23 +310,27 @@ if __name__ == "__main__":
     parser.set_defaults(action=None, masters_json=None)
     parser.add_option("-l", "--list", action="store_true", dest="list")
     parser.add_option("-t", "--test", action="store_true", dest="test")
     parser.add_option(
         "-8", action="store_true", dest="buildbot08", default=False)
     parser.add_option("-b", "--buildbot", dest="buildbot", default="buildbot")
     parser.add_option("-j", "--masters-json", dest="masters_json",
                       default="http://hg.mozilla.org/build/tools/raw-file/tip/buildfarm/maintenance/production-masters.json")
-    parser.add_option("-R", "--role", dest="role", default=None)
+    parser.add_option("-R", "--role", dest="role", default=None,
+                      help="Filter by given comma-separated role(s), eg try, build, tests, scheduler")
     parser.add_option(
-        "-u", "--universal", dest="universal", action="store_true")
+        "-u", "--universal", dest="universal", action="store_true",
+        help="Set up a universal master")
     parser.add_option("-q", "--quiet", dest="quiet", action="store_true")
     parser.add_option(
         "-e", "--error-logs", dest="error_logs", action="store_true")
     parser.add_option("-d", "--debug", dest="debug", action="store_true")
+    parser.add_option("--tested-only", dest="tested_only", action="store_true",
+                      help="Restrict to the set of masters that would be used with -t")
     parser.add_option("--ignore-role", dest="ignored_roles", action="append", default=[],
                       help="Ignore masters with this role. May be passed multiple times.")
 
     options, args = parser.parse_args()
 
     if options.debug:
         loglvl = logging.DEBUG
     elif options.quiet:
@@ -340,40 +347,52 @@ if __name__ == "__main__":
     cf = logging.Formatter('%(levelname)-5s - %(message)s')
     ch.setFormatter(cf)
     log.addHandler(ch)
 
     log.debug('using master json file from "%s"' % options.masters_json)
     if options.role:
         log.info('filtering by "%s" roles' % options.role)
 
-    dedupe = options.test
+    if options.test:
+        options.tested_only = True
+
     master_list = load_masters_json(options.masters_json, role=options.role,
                                     log=log, universal=options.universal,
-                                    dedupe=dedupe, ignored_roles=ignored_roles)
-    if options.test:
+                                    dedupe=options.tested_only,
+                                    ignored_roles=ignored_roles)
+    if options.tested_only:
         log.debug('adding universal builders because we are testing')
         # a universal scheduler master doesn't make any sense
         ignored_roles += ['scheduler']
         uni_masters = load_masters_json(options.masters_json, role=options.role, universal=not options.universal, log=log, ignored_roles=ignored_roles)
         master_list.extend(uni_masters)
 
     # Make sure we don't have duplicate names
     master_map = dict((m.name, m) for m in master_list)
     assert len(
         master_map.values()) == len(master_list), "Duplicate master names"
     assert len(master_list) > 0, "No masters specified. Bad role?"
 
+    if options.list or options.test:
+        masters = filter_masters(master_list)
+        if len(args) > 0:
+            wanted = set(args)
+            available = set([ m.name for m in masters ])
+            unknown = wanted - available
+            assert len(unknown) == 0, "%d unknown masters requested: %s" % (len(unknown), " ".join(unknown))
+            masters = [ m for m in masters if m.name in wanted ]
+
     if options.list:
-        for m in filter_masters(master_list):
+        for m in masters:
             print m.name
     elif options.test:
         failing_masters = []
         # Test the masters, once normally and onces as a universal master
-        for m in filter_masters(master_list):
+        for m in masters:
             rc, logfile, dir = m.testMaster(
                 options.buildbot, error_logs=options.error_logs)
             if rc != 0:
                 failing_masters.append((m.name, logfile, dir))
         # Print a summary including a list of useful output
         log.info("TEST-SUMMARY: %s tested, %s failed" % (len(
             master_list), len(failing_masters)))
         for rc, logfile, dir in failing_masters:
--- a/test-masters.sh
+++ b/test-masters.sh
@@ -1,16 +1,56 @@
 #!/bin/bash
 # This script has been rewritten in setup_master.py using
 # the -t option.  We use that now
 exit=0
 
-./setup-master.py -t "$@" || exit=1
+mkdir test-output 2>/dev/null
+
+actioning="Checking"
+MASTERS_JSON_URL="${MASTERS_JSON_URL:-http://hg.mozilla.org/build/tools/raw-file/tip/buildfarm/maintenance/production-masters.json}"
+
+atexit=()
+trap 'for cmd in "${atexit[@]}"; do eval $cmd; done' EXIT
+
+# I have had problems where a whole bunch of parallel HTTP requests caused
+# errors (?), so fetch it once here and pass it in.
+MASTERS_JSON=$(mktemp)
+wget -q -O$MASTERS_JSON "$MASTERS_JSON_URL" || exit 1
+atexit+=("rm $MASTERS_JSON")
+
+FAILFILE=$(mktemp)
+
+# Construct the set of masters that we will test.
+MASTERS=($(./setup-master.py -l -j "$MASTERS_JSON" --tested-only "$@"))
+
+# Fire off all the tests in parallel.
+for MASTER in ${MASTERS[*]}; do (
+    OUTFILE=$(mktemp)
+
+    ./setup-master.py -t -j "$MASTERS_JSON" "$@" $MASTER > $OUTFILE 2>&1 || echo "$MASTER" >> $FAILFILE
+    cat $OUTFILE # Make the output a little less interleaved
+    rm $OUTFILE
+) &
+atexit+=("[ -e /proc/$! ] && kill $!")
+done
+
+echo "$actioning ${#MASTERS[*]} masters..."
+echo "${MASTERS[*]}"
+wait
+
+if [ -s $FAILFILE ]; then
+    echo "*** $(wc -l < $FAILFILE) master tests failed ***" >&2
+    echo "Failed masters:" >&2
+    sed -e 's/^/  /' "$FAILFILE" >&2
+    exit 1
+fi
 
 for dir in mozilla mozilla-tests; do
   cd $dir
   for f in test/*.py; do
     trial $f || exit=1
   done
   rm -rf _trial_temp
   cd ..
 done
+
 exit $exit