Bug 548768 - buildbotcustom changes to run packaged unittests on talos p=lsblakk r=catlee,alice
authorLukas Blakk <lsblakk@mozilla.com>
Fri, 02 Apr 2010 08:24:07 -0700
changeset 683 d5ba95f1dcb8714576bee2bebf376fbce91aa0d7
parent 678 5caaf42de95b75e22aa468f8483c991c43ddac49
child 684 7af99e00e1f13ed23df057ffa2f9ef03fcb71d21
push id411
push userlsblakk@mozilla.com
push dateFri, 02 Apr 2010 15:25:16 +0000
reviewerscatlee, alice
bugs548768
Bug 548768 - buildbotcustom changes to run packaged unittests on talos p=lsblakk r=catlee,alice
misc.py
--- a/misc.py
+++ b/misc.py
@@ -177,19 +177,23 @@ def _nextFastSlave(builder, available_sl
             log.msg("No fast or slow slaves found for builder '%s', choosing randomly instead" % builder.name)
             return random.choice(available_slaves)
     except:
         log.msg("Error choosing next fast slave for builder '%s', choosing randomly instead" % builder.name)
         log.err()
         return random.choice(available_slaves)
 
 def generateTestBuilder(config, branch_name, platform, name_prefix, build_dir_prefix,
-        suites_name, suites, mochitestLeakThreshold, crashtestLeakThreshold):
+        suites_name, suites, mochitestLeakThreshold, crashtestLeakThreshold, slaves=None):
     builders = []
     pf = config['platforms'].get(platform, {})
+    if slaves == None:
+        slavenames = config['platforms'][platform]['slaves']
+    else:
+        slavenames = slaves
     if isinstance(suites, dict) and "totalChunks" in suites:
         totalChunks = suites['totalChunks']
         for i in range(totalChunks):
             factory = UnittestPackagedBuildFactory(
                 platform=platform,
                 test_suites=[suites['suite']],
                 mochitest_leak_threshold=mochitestLeakThreshold,
                 crashtest_leak_threshold=crashtestLeakThreshold,
@@ -201,17 +205,17 @@ def generateTestBuilder(config, branch_n
                 totalChunks=totalChunks,
                 thisChunk=i+1,
                 chunkByDir=suites.get('chunkByDir'),
                 env=pf.get('unittest-env', {}),
                 downloadSymbols=pf.get('download_symbols', True),
             )
             builder = {
                 'name': '%s %s-%i/%i' % (name_prefix, suites_name, i+1, totalChunks),
-                'slavenames': config['platforms'][platform]['slaves'],
+                'slavenames': slavenames,
                 'builddir': '%s-%s-%i' % (build_dir_prefix, suites_name, i+1),
                 'factory': factory,
                 'category': branch_name,
                 'nextSlave': _nextSlowSlave,
             }
             builders.append(builder)
     else:
         factory = UnittestPackagedBuildFactory(
@@ -220,20 +224,21 @@ def generateTestBuilder(config, branch_n
             mochitest_leak_threshold=mochitestLeakThreshold,
             crashtest_leak_threshold=crashtestLeakThreshold,
             hgHost=config['hghost'],
             repoPath=config['repo_path'],
             buildToolsRepoPath=config['build_tools_repo_path'],
             buildSpace=1.0,
             buildsBeforeReboot=config['platforms'][platform]['builds_before_reboot'],
             downloadSymbols=pf.get('download_symbols', True),
+            env=pf.get('unittest-env', {}),
         )
         builder = {
             'name': '%s %s' % (name_prefix, suites_name),
-            'slavenames': config['platforms'][platform]['slaves'],
+            'slavenames': slavenames,
             'builddir': '%s-%s' % (build_dir_prefix, suites_name),
             'factory': factory,
             'category': branch_name,
         }
         builders.append(builder)
     return builders
 
 def generateCCTestBuilder(config, branch_name, platform, name_prefix, build_dir_prefix,
@@ -1685,20 +1690,21 @@ def generateCCBranchObjects(config, name
                  'factory': mozilla2_shark_factory,
                  'category': name,
              }
              branchObjects['builders'].append(mozilla2_shark_builder)
 
     return branchObjects
 
 
-def generateTalosBranchObjects(branch, branch_config, PLATFORMS, SUITES,
-        factory_class=TalosFactory):
+def generateTalosBranchObjects(branch, branch_config, PLATFORMS, SUITES, 
+        ACTIVE_UNITTEST_PLATFORMS, factory_class=TalosFactory):
     branchObjects = {'schedulers': [], 'builders': [], 'status': []}
     branch_builders = []
+    test_builders = []
 
     branchName = branch_config['branch_name']
     buildBranch = branch_config['build_branch']
     talosCmd = branch_config['talos_command']
 
     for platform, platform_config in PLATFORMS.items():
         for slave_platform in platform_config['slave_platforms']:
             for suite, talosConfig in SUITES.items():
@@ -1742,34 +1748,80 @@ def generateTalosBranchObjects(branch, b
                               treeStableTimer=0,
                               builderNames=[builder['name']],
                               numberOfBuildsToTrigger=tests,
                               )
                 branchObjects['schedulers'].append(s)
                 branchObjects['builders'].append(builder)
                 branch_builders.append(builder['name'])
 
+            if platform in ACTIVE_UNITTEST_PLATFORMS.keys():
+                testTypes = []
+
+                if branch_config['platforms'][platform].get('enable_opt_unittests'):
+                    testTypes.append('opt')
+                if branch_config['platforms'][platform].get('enable_debug_unittests'):
+                    testTypes.append('debug')
+
+                for test_type in testTypes:
+                    test_builders = []
+                    triggeredUnittestBuilders = []
+                    unittest_suites = "%s_unittest_suites" % test_type
+
+                    # create builder names for TinderboxMailNotifier
+                    for suites_name, suites in branch_config['platforms'][platform][unittest_suites]:
+                        test_builders.extend(generateTestBuilderNames(
+                            '%s %s %s test' % (platform_name, branch, test_type), suites_name, suites))
+
+                    triggeredUnittestBuilders.append(('%s-%s-%s-unittest' % (branch, slave_platform, test_type), test_builders, True))
+
+                    for suites_name, suites in branch_config['platforms'][platform][unittest_suites]:
+                        # create the builders
+                        branchObjects['builders'].extend(generateTestBuilder(
+                                branch_config, branch, platform, "%s %s %s test" % (platform_name, branch, test_type),
+                                "%s-%s-%s-u" % (branch, slave_platform, test_type),
+                                suites_name, suites, branch_config.get('mochitest_leak_threshold', None),
+                                branch_config.get('crashtest_leak_threshold', None),
+                                platform_config[slave_platform]['slaves']))
+
+                    for scheduler_name, test_builders, merge in triggeredUnittestBuilders:
+                        scheduler_branch = ('%s-%s-%s-unittest' % (branch, platform, test_type))
+                        if not merge:
+                            branchObjects['schedulers'].append(NoMergeScheduler(name=scheduler_name, branch=scheduler_branch, builderNames=test_builders, treeStableTimer=0))
+                        else:
+                            branchObjects['schedulers'].append(Scheduler(name=scheduler_name, branch=scheduler_branch, builderNames=test_builders, treeStableTimer=0))
+
     branchObjects['status'].append(TinderboxMailNotifier(
                            fromaddr="talos.buildbot@build.mozilla.org",
                            tree=branch_config['tinderbox_tree'],
                            extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org",],
                            relayhost="smtp.mozilla.org",
                            builders=branch_builders,
                            useChangeTime=False,
                            logCompression="bzip2"))
+    ###  Unittests need specific errorparser
+    branchObjects['status'].append(TinderboxMailNotifier(
+                           fromaddr="talos.buildbot@build.mozilla.org",
+                           tree=branch_config['tinderbox_tree'],
+                           extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org",],
+                           relayhost="smtp.mozilla.org",
+                           builders=test_builders,
+                           useChangeTime=False,
+                           errorparser="unittest",
+                           logCompression="bzip2"))
 
     if branch_config.get('release_tests'):
         releaseObjects = generateTalosReleaseBranchObjects(branch,
-                branch_config, PLATFORMS, SUITES, factory_class)
+                branch_config, PLATFORMS, SUITES, ACTIVE_UNITTEST_PLATFORMS, factory_class)
         for k,v in releaseObjects.items():
             branchObjects[k].extend(v)
     return branchObjects
 
 def generateTalosReleaseBranchObjects(branch, branch_config, PLATFORMS, SUITES,
-        factory_class=TalosFactory):
+        ACTIVE_UNITTEST_PLATFORMS, factory_class=TalosFactory):
     branch_config = branch_config.copy()
     release_tests = branch_config['release_tests']
 
     # Update the # of tests to run with our release_tests number
     # Force no merging
     for suite, talosConfig in SUITES.items():
         tests, merge, extra, platforms = branch_config['%s_tests' % suite]
         if tests > 0:
@@ -1781,9 +1833,10 @@ def generateTalosReleaseBranchObjects(br
     branch_config['branch_name'] += '-Release'
     branch += "-release"
 
     # Remove the release_tests key so we don't call ourselves again
     del branch_config['release_tests']
 
     # Don't fetch symbols
     branch_config['fetch_symbols'] = branch_config['fetch_release_symbols']
-    return generateTalosBranchObjects(branch, branch_config, PLATFORMS, SUITES, factory_class)
+    return generateTalosBranchObjects(branch, branch_config, PLATFORMS, SUITES, 
+        ACTIVE_UNITTEST_PLATFORMS, factory_class)