Bug 673834 - Obsolete ReleaseRepackFactory, fold logic into CCReleaseRepackFactory. r=rail
authorJustin Wood <Callek@gmail.com>
Thu, 26 Jan 2012 12:23:29 -0500
changeset 2037 29b8548e5348b3cf12996249d552d276ee01adee
parent 2036 83f17929c032820cf8ecef762365cae9931af7f3
child 2038 083d22a0cbdf7ebbeb0c719a7f7a7baa96a08f10
child 2039 89644e1c553d83981ed8fe507ffc6b66f7686bf6
push id1500
push userCallek@gmail.com
push dateThu, 26 Jan 2012 17:25:33 +0000
reviewersrail
bugs673834
Bug 673834 - Obsolete ReleaseRepackFactory, fold logic into CCReleaseRepackFactory. r=rail
process/factory.py
--- a/process/factory.py
+++ b/process/factory.py
@@ -3059,20 +3059,16 @@ class BaseRepackFactory(MozillaBuildFact
             # L10NBASEDIR is relative to MOZ_OBJDIR
             self.env.update({'MOZ_OBJDIR': objdir,
                              'L10NBASEDIR':  '../../%s' % self.l10nRepoPath})            
 
         if platform == 'macosx64':
             # use "mac" instead of "mac64" for macosx64
             self.env.update({'MOZ_PKG_PLATFORM': 'mac'})
 
-        # Configure step gets executed before the downloadBuilds and we can't
-        # render at that point of execution the environment variable 'srcdir'
-        self.configure_env = self.env.copy()
-
         self.uploadEnv = self.env.copy() # pick up any env variables in our subclass
         self.uploadEnv.update({
             'AB_CD': WithProperties('%(locale)s'),
             'UPLOAD_HOST': stageServer,
             'UPLOAD_USER': stageUsername,
             'UPLOAD_TO_TEMP': '1',
             'POST_UPLOAD_CMD': self.postUploadCmd # defined in subclasses
         })
@@ -3214,17 +3210,17 @@ class BaseRepackFactory(MozillaBuildFact
 
     def tinderboxPrintBuildInfo(self):
         '''Display some build properties for scraping in Tinderbox.
         '''
         self.tinderboxPrint('locale',WithProperties('%(locale)s'))
         self.tinderboxPrint('tree',self.tree)
         self.tinderboxPrint('buildnumber',WithProperties('%(buildnumber)s'))
 
-    def doUpload(self):
+    def doUpload(self, postUploadBuildDir=None, uploadMulti=False):
         self.addStep(RetryingShellCommand(
          name='make_upload',
          command=['make', 'upload', WithProperties('AB_CD=%(locale)s')],
          env=self.uploadEnv,
          workdir='%s/%s/%s/locales' % (self.baseWorkDir, self.objdir,
                                        self.appName),
          haltOnFailure=True,
          flunkOnFailure=True,
@@ -3460,17 +3456,17 @@ class CCBaseRepackFactory(BaseRepackFact
             co_command.append('--chatzilla-rev=%s' % self.buildRevision)
         # execute the checkout
         self.addStep(ShellCommand(
          command=co_command,
          description=['running', 'client.py', 'checkout'],
          descriptionDone=['client.py', 'checkout'],
          haltOnFailure=True,
          workdir='%s/%s' % (self.baseWorkDir, self.origSrcDir),
-         timeout=60*60 # 1 hour
+         timeout=60*60*3 # 3 hours (crazy, but necessary for now)
         ))
 
 class NightlyRepackFactory(BaseRepackFactory, NightlyBuildFactory):
     extraConfigureArgs = []
 
     def __init__(self, enUSBinaryURL, nightly=False, env={},
                  ausBaseUploadDir=None, updatePlatform=None,
                  downloadBaseURL=None, ausUser=None, ausSshKey=None,
@@ -3805,20 +3801,27 @@ class ReleaseFactory(MozillaBuildFactory
         return hgSshKey
 
     def makeLongVersion(self, version):
         version = re.sub('a([0-9]+)$', ' Alpha \\1', version)
         version = re.sub('b([0-9]+)$', ' Beta \\1', version)
         version = re.sub('rc([0-9]+)$', ' RC \\1', version)
         return version
 
-
-class ReleaseRepackFactory(BaseRepackFactory, ReleaseFactory):
+class CCReleaseRepackFactory(CCBaseRepackFactory, ReleaseFactory):
     def __init__(self, platform, buildRevision, version, buildNumber,
-                 env={}, brandName=None, mergeLocales=False, **kwargs):
+                 env={}, brandName=None, mergeLocales=False,
+                 mozRepoPath='', inspectorRepoPath='', venkmanRepoPath='',
+                 chatzillaRepoPath='', cvsroot='', **kwargs):
+        self.skipBlankRepos = True
+        self.mozRepoPath = mozRepoPath
+        self.inspectorRepoPath = inspectorRepoPath
+        self.venkmanRepoPath = venkmanRepoPath
+        self.chatzillaRepoPath = chatzillaRepoPath
+        self.cvsroot = cvsroot
         self.buildRevision = buildRevision
         self.version = version
         self.buildNumber = buildNumber
         if brandName:
             self.brandName = brandName
         else:
             self.brandName = kwargs['project'].capitalize()
         # more vars are added in downloadBuilds
@@ -3831,17 +3834,89 @@ class ReleaseRepackFactory(BaseRepackFac
         assert 'project' in kwargs
         # TODO: better place to put this/call this
         self.postUploadCmd = 'post_upload.py ' + \
                              '-p %s ' % kwargs['project'] + \
                              '-v %s ' % self.version + \
                              '-n %s ' % self.buildNumber + \
                              '--release-to-candidates-dir'
         BaseRepackFactory.__init__(self, env=env, platform=platform,
-                                   mergeLocales=mergeLocales, **kwargs)
+                                   mergeLocales=mergeLocales, mozillaDir='mozilla',
+                                   mozconfigBranch='default', **kwargs)
+
+    # Repeated here since the Parent classes fail hgtool/checkouts due to
+    # relbranch issues, and not actually having the tag after clone
+    def getSources(self):
+        self.addStep(MercurialCloneCommand(
+         name='get_enUS_src',
+         command=['sh', '-c',
+          WithProperties('if [ -d '+self.origSrcDir+'/.hg ]; then ' +
+                         'hg -R '+self.origSrcDir+' pull && '+
+                         'hg -R '+self.origSrcDir+' up -C ;'+
+                         'else ' +
+                         'hg clone ' +
+                         'http://'+self.hgHost+'/'+self.repoPath+' ' +
+                         self.origSrcDir+' ; ' +
+                         'fi ' +
+                         '&& hg -R '+self.origSrcDir+' update -C -r %(en_revision)s')],
+         descriptionDone="en-US source",
+         workdir=self.baseWorkDir,
+         haltOnFailure=True,
+         timeout=30*60 # 30 minutes
+        ))
+        self.addStep(MercurialCloneCommand(
+         name='get_locale_src',
+         command=['sh', '-c',
+          WithProperties('if [ -d %(locale)s/.hg ]; then ' +
+                         'hg -R %(locale)s pull -r default ; ' +
+                         'else ' +
+                         'hg clone ' +
+                         'http://'+self.hgHost+'/'+self.l10nRepoPath+ 
+                           '/%(locale)s/ ; ' +
+                         'fi ' +
+                         '&& hg -R %(locale)s update -C -r %(l10n_revision)s')],
+         descriptionDone="locale source",
+         timeout=10*60, # 10 minutes
+         haltOnFailure=True,
+         workdir='%s/%s' % (self.baseWorkDir, self.l10nRepoPath)
+        ))
+        # build up the checkout command with all options
+        co_command = ['python', 'client.py', 'checkout',
+                      WithProperties('--comm-rev=%(en_revision)s')]
+        if self.mozRepoPath:
+            co_command.append('--mozilla-repo=%s' % self.getRepository(self.mozRepoPath))
+        if self.inspectorRepoPath:
+            co_command.append('--inspector-repo=%s' % self.getRepository(self.inspectorRepoPath))
+        elif self.skipBlankRepos:
+            co_command.append('--skip-inspector')
+        if self.venkmanRepoPath:
+            co_command.append('--venkman-repo=%s' % self.getRepository(self.venkmanRepoPath))
+        elif self.skipBlankRepos:
+            co_command.append('--skip-venkman')
+        if self.chatzillaRepoPath:
+            co_command.append('--chatzilla-repo=%s' % self.getRepository(self.chatzillaRepoPath))
+        elif self.skipBlankRepos:
+            co_command.append('--skip-chatzilla')
+        if self.cvsroot:
+            co_command.append('--cvsroot=%s' % self.cvsroot)
+        if self.buildRevision:
+            co_command.append('--comm-rev=%s' % self.buildRevision)
+            co_command.append('--mozilla-rev=%s' % self.buildRevision)
+            co_command.append('--inspector-rev=%s' % self.buildRevision)
+            co_command.append('--venkman-rev=%s' % self.buildRevision)
+            co_command.append('--chatzilla-rev=%s' % self.buildRevision)
+        # execute the checkout
+        self.addStep(ShellCommand(
+         command=co_command,
+         description=['running', 'client.py', 'checkout'],
+         descriptionDone=['client.py', 'checkout'],
+         haltOnFailure=True,
+         workdir='%s/%s' % (self.baseWorkDir, self.origSrcDir),
+         timeout=60*60*3 # 3 hours (crazy, but necessary for now)
+        ))
 
     def updateSources(self):
         self.addStep(ShellCommand(
          name='update_sources',
          command=['hg', 'up', '-C', '-r', self.buildRevision],
          workdir='build/'+self.origSrcDir,
          description=['update %s' % self.branchName,
                       'to %s' % self.buildRevision],
@@ -3855,26 +3930,58 @@ class ReleaseRepackFactory(BaseRepackFac
         ))
         self.addStep(SetProperty(
                      command=['hg', 'ident', '-i'],
                      haltOnFailure=True,
                      property='l10n_revision',
                      workdir=WithProperties('build/' + self.l10nRepoPath + 
                                             '/%(locale)s')
         ))
-
-    def downloadBuilds(self):
+        self.addStep(ShellCommand(
+         command=['hg', 'up', '-C', '-r', self.buildRevision],
+         workdir='build/'+self.mozillaSrcDir,
+         description=['update mozilla',
+                      'to %s' % self.buildRevision],
+         haltOnFailure=True
+        ))
+        if self.venkmanRepoPath:
+            self.addStep(ShellCommand(
+             command=['hg', 'up', '-C', '-r', self.buildRevision],
+             workdir='build/'+self.mozillaSrcDir+'/extensions/venkman',
+             description=['update venkman',
+                          'to %s' % self.buildRevision],
+             haltOnFailure=True
+            ))
+        if self.inspectorRepoPath:
+            self.addStep(ShellCommand(
+             command=['hg', 'up', '-C', '-r', self.buildRevision],
+             workdir='build/'+self.mozillaSrcDir+'/extensions/inspector',
+             description=['update inspector',
+                          'to %s' % self.buildRevision],
+             haltOnFailure=True
+            ))
+        if self.chatzillaRepoPath:
+            self.addStep(ShellCommand(
+             command=['hg', 'up', '-C', '-r', self.buildRevision],
+             workdir='build/'+self.mozillaSrcDir+'/extensions/irc',
+             description=['update chatzilla',
+                          'to %s' % self.buildRevision],
+             haltOnFailure=True
+            ))
+
+    def preClean(self):
         # We need to know the absolute path to the input builds when we repack,
         # so we need retrieve at run-time as a build property
         self.addStep(SetProperty(
          command=['bash', '-c', 'pwd'],
          property='srcdir',
          workdir='build/'+self.origSrcDir
         ))
 
+    def downloadBuilds(self):
         candidatesDir = 'http://%s' % self.stageServer + \
                         '/pub/mozilla.org/%s/nightly' % self.project + \
                         '/%s-candidates/build%s' % (self.version,
                                                     self.buildNumber)
         longVersion = self.makeLongVersion(self.version)
 
         # This block sets platform specific data that our wget command needs.
         #  build is mapping between the local and remote filenames
@@ -3912,25 +4019,16 @@ class ReleaseRepackFactory(BaseRepackFac
              command=['wget', '-O', name, '--no-check-certificate',
                       '%s/%s/en-US/%s' % (candidatesDir, platformDir,
                                           builds[name])],
              workdir='build/'+self.origSrcDir,
              haltOnFailure=True
             ))
 
     def doRepack(self):
-        # For releases we have to make memory/jemalloc
-        if self.platform.startswith('win32'):
-            self.addStep(ShellCommand(
-             name='make_memory_jemalloc',
-             command=['make'],
-             workdir='build/'+self.mozillaObjdir+'/memory/jemalloc',
-             description=['make memory/jemalloc'],
-             haltOnFailure=True
-            ))
         # Because we're generating updates we need to build the libmar tools
         self.addStep(ShellCommand(
             name='make_tier_nspr',
             command=['make','tier_nspr'],
             workdir='build/'+self.mozillaObjdir,
             description=['make tier_nspr'],
             haltOnFailure=True
             ))
@@ -3947,72 +4045,16 @@ class ReleaseRepackFactory(BaseRepackFac
          description=['repack', 'installers'],
          command=['sh','-c',
                   WithProperties('make installers-%(locale)s LOCALE_MERGEDIR=$PWD/merged')],
          env=self.env,
          haltOnFailure=True,
          workdir='build/'+self.objdir+'/'+self.appName+'/locales'
         ))
 
-class CCReleaseRepackFactory(CCBaseRepackFactory, ReleaseRepackFactory):
-    def __init__(self, mozRepoPath='', inspectorRepoPath='',
-                 venkmanRepoPath='', chatzillaRepoPath='', cvsroot='',
-                 **kwargs):
-        self.skipBlankRepos = True
-        self.mozRepoPath = mozRepoPath
-        self.inspectorRepoPath = inspectorRepoPath
-        self.venkmanRepoPath = venkmanRepoPath
-        self.chatzillaRepoPath = chatzillaRepoPath
-        self.cvsroot = cvsroot
-        ReleaseRepackFactory.__init__(self, mozillaDir='mozilla',
-            mozconfigBranch='default', **kwargs)
-
-    def updateSources(self):
-        ReleaseRepackFactory.updateSources(self)
-        self.addStep(ShellCommand(
-         command=['hg', 'up', '-C', '-r', self.buildRevision],
-         workdir='build/'+self.mozillaSrcDir,
-         description=['update mozilla',
-                      'to %s' % self.buildRevision],
-         haltOnFailure=True
-        ))
-        if self.venkmanRepoPath:
-            self.addStep(ShellCommand(
-             command=['hg', 'up', '-C', '-r', self.buildRevision],
-             workdir='build/'+self.mozillaSrcDir+'/extensions/venkman',
-             description=['update venkman',
-                          'to %s' % self.buildRevision],
-             haltOnFailure=True
-            ))
-        if self.inspectorRepoPath:
-            self.addStep(ShellCommand(
-             command=['hg', 'up', '-C', '-r', self.buildRevision],
-             workdir='build/'+self.mozillaSrcDir+'/extensions/inspector',
-             description=['update inspector',
-                          'to %s' % self.buildRevision],
-             haltOnFailure=True
-            ))
-        if self.chatzillaRepoPath:
-            self.addStep(ShellCommand(
-             command=['hg', 'up', '-C', '-r', self.buildRevision],
-             workdir='build/'+self.mozillaSrcDir+'/extensions/irc',
-             description=['update chatzilla',
-                          'to %s' % self.buildRevision],
-             haltOnFailure=True
-            ))
-
-    def downloadBuilds(self):
-        ReleaseRepackFactory.downloadBuilds(self)
-
-    # unsure why we need to explicitely do this but after bug 478436 we stopped
-    # executing the actual repackaging without this def here
-    def doRepack(self):
-        ReleaseRepackFactory.doRepack(self)
-
-
 class StagingRepositorySetupFactory(ReleaseFactory):
     """This Factory should be run at the start of a staging release run. It
        deletes and reclones all of the repositories in 'repositories'. Note that
        the staging buildTools repository should _not_ be recloned, as it is
        used by many other builders, too.
     """
     def __init__(self, username, sshKey, repositories, userRepoRoot,
                  **kwargs):
@@ -5606,17 +5648,17 @@ class UnittestBuildFactory(MozillaBuildF
 
         self.addTestSteps()
 
         self.addPostTestSteps()
 
         if self.buildsBeforeReboot and self.buildsBeforeReboot > 0:
             self.addPeriodicRebootSteps()
 
-    def doUpload(self):
+    def doUpload(self, postUploadBuildDir=None, uploadMulti=False):
         if self.uploadPackages:
             self.addStep(ShellCommand(
              name='make_pkg',
              command=['make', 'package'],
              env=self.env,
              workdir='build/%s' % self.objdir,
              haltOnFailure=True
             ))
@@ -5720,17 +5762,17 @@ class UnittestBuildFactory(MozillaBuildF
     def addPostTestSteps(self):
         pass
 
 class TryUnittestBuildFactory(UnittestBuildFactory):
     def __init__(self, **kwargs):
 
         UnittestBuildFactory.__init__(self, **kwargs)
 
-    def doUpload(self):
+    def doUpload(self, postUploadBuildDir=None, uploadMulti=False):
         if self.uploadPackages:
             self.addStep(ShellCommand(
              name='make_pkg',
              command=['make', 'package'],
              env=self.env,
              workdir='build/%s' % self.objdir,
              haltOnFailure=True
             ))
@@ -5821,16 +5863,17 @@ class CCUnittestBuildFactory(MozillaBuil
             assert self.unittestBranch
         if brandName:
             self.brandName = brandName
         else:
             self.brandName = productName.capitalize()
         self.mochitest_leak_threshold = mochitest_leak_threshold
         self.mochichrome_leak_threshold = mochichrome_leak_threshold
         self.mochibrowser_leak_threshold = mochibrowser_leak_threshold
+        self.crashtest_leak_threshold = crashtest_leak_threshold
         self.exec_xpcshell_suites = exec_xpcshell_suites
         self.exec_reftest_suites = exec_reftest_suites
         self.exec_mochi_suites = exec_mochi_suites
         self.exec_mozmill_suites = exec_mozmill_suites
 
         self.config_repo_url = self.getRepository(self.config_repo_path)
 
         env_map = {
@@ -5965,17 +6008,17 @@ class CCUnittestBuildFactory(MozillaBuil
 
         self.addTestSteps()
 
         self.addPostTestSteps()
 
         if self.buildsBeforeReboot and self.buildsBeforeReboot > 0:
             self.addPeriodicRebootSteps()
 
-    def doUpload(self):
+    def doUpload(self, postUploadBuildDir=None, uploadMulti=False):
         if self.uploadPackages:
             self.addStep(ShellCommand(
              name='make_pkg',
              command=['make', 'package'],
              env=self.env,
              workdir='build/%s' % self.objdir,
              haltOnFailure=True
             ))
@@ -6071,17 +6114,17 @@ class CCUnittestBuildFactory(MozillaBuil
         if self.exec_reftest_suites:
             self.addStep(unittest_steps.MozillaReftest, warnOnWarnings=True,
              test_name="reftest",
              workdir="build/%s" % self.objdir,
              timeout=5*60,
             )
             self.addStep(unittest_steps.MozillaReftest, warnOnWarnings=True,
              test_name="crashtest",
-             leakThreshold=crashtest_leak_threshold,
+             leakThreshold=self.crashtest_leak_threshold,
              workdir="build/%s" % self.objdir,
             )
 
         if self.exec_mochi_suites:
             self.addStep(unittest_steps.MozillaMochitest, warnOnWarnings=True,
              test_name="mochitest-plain",
              workdir="build/%s" % self.objdir,
              leakThreshold=self.mochitest_leak_threshold,