Bug 627182 - Automate signing and publishing of XULRunner builds. r=bhearsum
authorRail Aliiev <rail@mozilla.com>
Fri, 23 Mar 2012 13:21:57 -0400
changeset 2390 a540057c0257ae592c48f32989222f3867e542b0
parent 2389 24cf416d19534cf3c1f88f31620d37e4e4b03d16
child 2391 b6ba25b1e85535cc3efc58670322534b334174fc
push id1695
push userraliiev@mozilla.com
push dateFri, 23 Mar 2012 17:23:47 +0000
reviewersbhearsum
bugs627182
Bug 627182 - Automate signing and publishing of XULRunner builds. r=bhearsum
scripts/release/generate-sums.py
scripts/release/generate-sums.sh
scripts/release/push-to-mirrors.py
scripts/release/push-to-mirrors.sh
--- a/scripts/release/generate-sums.py
+++ b/scripts/release/generate-sums.py
@@ -54,30 +54,35 @@ if __name__ == '__main__':
                                        DEFAULT_BUILDBOT_CONFIGS_REPO),
         sourceRepoKey="mozilla",
     )
     parser.add_option("-c", "--configfile", dest="configfile")
     parser.add_option("-r", "--release-config", dest="releaseConfig")
     parser.add_option("-b", "--buildbot-configs", dest="buildbotConfigs")
     parser.add_option("-t", "--release-tag", dest="releaseTag")
     parser.add_option("--source-repo-key", dest="sourceRepoKey")
+    parser.add_option("--product", dest="product")
+    parser.add_option("--ssh-user", dest="ssh_username")
+    parser.add_option("--ssh-key", dest="ssh_key")
+    parser.add_option("--create-contrib-dirs", dest="create_contrib_dirs",
+                      action="store_true")
 
     options, args = parser.parse_args()
     mercurial(options.buildbotConfigs, "buildbot-configs")
     update("buildbot-configs", revision=options.releaseTag)
 
     branchConfig, releaseConfig = validate(options, args)
 
-    productName = releaseConfig['productName']
+    productName = options.product or releaseConfig['productName']
     version = releaseConfig['version']
     buildNumber = releaseConfig['buildNumber']
     stageServer = branchConfig['stage_server']
-    stageUsername = branchConfig['stage_username']
-    stageSshKey = path.join(os.path.expanduser("~"), ".ssh",
-                            branchConfig["stage_ssh_key"])
+    stageUsername = options.ssh_username or branchConfig['stage_username']
+    stageSshKey = options.ssh_key or branchConfig["stage_ssh_key"]
+    stageSshKey = path.join(os.path.expanduser("~"), ".ssh", stageSshKey)
 
     candidatesDir = makeCandidatesDir(productName, version, buildNumber)
     rsyncFilesByPattern(server=stageServer, userName=stageUsername,
                         sshKey=stageSshKey, source_dir=candidatesDir,
                         target_dir='temp/', pattern='*.checksums')
     types = {'sha1': 'SHA1SUMS', 'md5': 'MD5SUMS', 'sha512': 'SHA512SUMS'}
     generateChecksums('temp', types)
     files = types.values()
@@ -85,12 +90,13 @@ if __name__ == '__main__':
     upload_files = files + ['%s.asc' % x for x in files] + \
         [path.join(path.dirname(__file__), 'KEY')]
     log.info("Fixing permissions...")
     for f in upload_files:
         log.info("chmod 644 %s" % f)
         os.chmod(f, 0644)
     rsyncFiles(files=upload_files, server=stageServer, userName=stageUsername,
                sshKey=stageSshKey, target_dir=candidatesDir)
-    cmd = 'mkdir -v -m 2775 %s/contrib %s/contrib-localized' % \
-        (candidatesDir, candidatesDir)
-    run_remote_cmd(cmd, server=stageServer, username=stageUsername,
-                   sshKey=stageSshKey)
+    if options.create_contrib_dirs:
+        cmd = 'mkdir -v -m 2775 %s/contrib %s/contrib-localized' % \
+            (candidatesDir, candidatesDir)
+        run_remote_cmd(cmd, server=stageServer, username=stageUsername,
+                       sshKey=stageSshKey)
--- a/scripts/release/generate-sums.sh
+++ b/scripts/release/generate-sums.sh
@@ -10,16 +10,17 @@ SCRIPTS_DIR="$MY_DIR/../../"
 PYTHON="/tools/python/bin/python"
 if [ ! -x $PYTHON ]; then
     PYTHON=python
 fi
 JSONTOOL="$PYTHON $SCRIPTS_DIR/buildfarm/utils/jsontool.py"
 workdir=`pwd`
 
 branchConfig=$1
+shift
 
 branch=$(basename $($JSONTOOL -k properties.branch $PROPERTIES_FILE))
 builder=$($JSONTOOL -k properties.buildername $PROPERTIES_FILE)
 slavebuilddir=$($JSONTOOL -k properties.slavebuilddir $PROPERTIES_FILE)
 slavename=$($JSONTOOL -k properties.slavename $PROPERTIES_FILE)
 master=$($JSONTOOL -k properties.master $PROPERTIES_FILE)
 releaseConfig=$($JSONTOOL -k properties.release_config $PROPERTIES_FILE)
 releaseTag=$($JSONTOOL -k properties.script_repo_revision $PROPERTIES_FILE)
@@ -37,9 +38,9 @@ cd $SCRIPTS_DIR/../..
 $PYTHON $SCRIPTS_DIR/clobberer/clobberer.py -s scripts -s buildprops.json \
   $CLOBBERER_URL $branch $builder $slavebuilddir $slavename $master
 cd $SCRIPTS_DIR/..
 $PYTHON $SCRIPTS_DIR/buildfarm/maintenance/purge_builds.py \
   -s 0.5 -n info -n 'rel-*' -n $slavebuilddir
 cd $workdir
 
 $PYTHON $MY_DIR/generate-sums.py -c $branchConfig -r $releaseConfig \
-  -b $BUILDBOT_CONFIGS -t $releaseTag
+  -b $BUILDBOT_CONFIGS -t $releaseTag $@
\ No newline at end of file
--- a/scripts/release/push-to-mirrors.py
+++ b/scripts/release/push-to-mirrors.py
@@ -25,17 +25,16 @@ DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_
 
 REQUIRED_BRANCH_CONFIG = ("stage_server", "stage_username", "stage_ssh_key")
 REQUIRED_RELEASE_CONFIG = ("productName", "version", "buildNumber")
 
 DEFAULT_RSYNC_EXCLUDES = ['--exclude=*tests*',
                           '--exclude=*crashreporter*',
                           '--exclude=*.log',
                           '--exclude=*.txt',
-                          '--exclude=*.zip',
                           '--exclude=*unsigned*',
                           '--exclude=*update-backup*',
                           '--exclude=*partner-repacks*',
                           '--exclude=*.checksums',
                           '--exclude=*.checksums.asc',
                           '--exclude=logs',
                           '--exclude=jsshell*',
                           ]
@@ -105,17 +104,17 @@ def pushToMirrors(productName, version, 
     appended to DEFAULT_RSYNC_EXCLUDES. """
 
     source_dir = makeCandidatesDir(productName, version, buildNumber)
     target_dir = makeReleasesDir(productName, version)
 
     if not excludes:
         excludes = DEFAULT_RSYNC_EXCLUDES
     if extra_excludes:
-        excludes += extra_excludes
+        excludes += ['--exclude=%s' % ex for ex in extra_excludes]
 
     # fail/warn if target directory exists depending on dry run mode
     try:
         run_remote_cmd(['test', '!', '-d', target_dir], server=stageServer,
                        username=stageUsername, sshKey=stageSshKey)
     except CalledProcessError:
         if not dryRun:
             raise
@@ -143,30 +142,35 @@ if __name__ == '__main__':
                                        DEFAULT_BUILDBOT_CONFIGS_REPO),
         sourceRepoKey="mozilla",
     )
     parser.add_option("-c", "--configfile", dest="configfile")
     parser.add_option("-r", "--release-config", dest="releaseConfig")
     parser.add_option("-b", "--buildbot-configs", dest="buildbotConfigs")
     parser.add_option("-t", "--release-tag", dest="releaseTag")
     parser.add_option("--source-repo-key", dest="sourceRepoKey")
+    parser.add_option("--product", dest="product")
+    parser.add_option("--ssh-user", dest="ssh_username")
+    parser.add_option("--ssh-key", dest="ssh_key")
+    parser.add_option("--extra-excludes", dest="extra_excludes",
+                      action="append")
 
     options, args = parser.parse_args()
     mercurial(options.buildbotConfigs, "buildbot-configs")
     update("buildbot-configs", revision=options.releaseTag)
 
     branchConfig, releaseConfig = validate(options, args)
 
-    productName = releaseConfig['productName']
+    productName = options.product or releaseConfig['productName']
     version = releaseConfig['version']
     buildNumber = releaseConfig['buildNumber']
     stageServer = branchConfig['stage_server']
-    stageUsername = branchConfig['stage_username']
-    stageSshKey = path.join(os.path.expanduser("~"), ".ssh",
-                            branchConfig["stage_ssh_key"])
+    stageUsername = options.ssh_username or branchConfig['stage_username']
+    stageSshKey = options.ssh_key or branchConfig["stage_ssh_key"]
+    stageSshKey = path.join(os.path.expanduser("~"), ".ssh", stageSshKey)
 
     if 'permissions' in args:
         checkStagePermissions(stageServer=stageServer,
                               stageUsername=stageUsername,
                               stageSshKey=stageSshKey,
                               productName=productName,
                               version=version,
                               buildNumber=buildNumber)
@@ -181,17 +185,19 @@ if __name__ == '__main__':
 
     if 'permissions' in args or 'antivirus' in args:
         pushToMirrors(stageServer=stageServer,
                       stageUsername=stageUsername,
                       stageSshKey=stageSshKey,
                       productName=productName,
                       version=version,
                       buildNumber=buildNumber,
+                      extra_excludes=options.extra_excludes,
                       dryRun=True)
 
     if 'push' in args:
         pushToMirrors(stageServer=stageServer,
                       stageUsername=stageUsername,
                       stageSshKey=stageSshKey,
                       productName=productName,
                       version=version,
+                      extra_excludes=options.extra_excludes,
                       buildNumber=buildNumber)
--- a/scripts/release/push-to-mirrors.sh
+++ b/scripts/release/push-to-mirrors.sh
@@ -11,16 +11,18 @@ PYTHON="/tools/python/bin/python"
 if [ ! -x $PYTHON ]; then
     PYTHON=python
 fi
 JSONTOOL="$PYTHON $SCRIPTS_DIR/buildfarm/utils/jsontool.py"
 workdir=`pwd`
 
 branchConfig=$1
 action=$2 # permissions, antivirus, or push
+shift
+shift
 
 branch=$(basename $($JSONTOOL -k properties.branch $PROPERTIES_FILE))
 builder=$($JSONTOOL -k properties.buildername $PROPERTIES_FILE)
 slavebuilddir=$($JSONTOOL -k properties.slavebuilddir $PROPERTIES_FILE)
 slavename=$($JSONTOOL -k properties.slavename $PROPERTIES_FILE)
 master=$($JSONTOOL -k properties.master $PROPERTIES_FILE)
 releaseConfig=$($JSONTOOL -k properties.release_config $PROPERTIES_FILE)
 releaseTag=$($JSONTOOL -k properties.script_repo_revision $PROPERTIES_FILE)
@@ -36,9 +38,9 @@ cd $SCRIPTS_DIR/../..
 $PYTHON $SCRIPTS_DIR/clobberer/clobberer.py -s scripts -s buildprops.json \
   $CLOBBERER_URL $branch $builder $slavebuilddir $slavename $master
 cd $SCRIPTS_DIR/..
 $PYTHON $SCRIPTS_DIR/buildfarm/maintenance/purge_builds.py \
   -s 0.3 -n info -n 'rel-*' -n $slavebuilddir
 cd $workdir
 
 $PYTHON $MY_DIR/push-to-mirrors.py -c $branchConfig -r $releaseConfig \
-  -b $BUILDBOT_CONFIGS -t $releaseTag $action
+  -b $BUILDBOT_CONFIGS -t $releaseTag $action $@