automatically sync with stage p=joduinn r=rhelmer
authorrhelmer@mozilla.com
Mon, 01 Oct 2007 20:00:18 -0700
changeset 6517 dd33b0bbc7e71d000c479945435249c9206e0fb6
parent 6516 86a6108a22bfbc2fe8ae5f659b7a15627c7a8a3f
child 6518 3116c83b57f6948afe3848c29a0a23b9db612f0e
push idunknown
push userunknown
push dateunknown
reviewersrhelmer
milestone1.9a9pre
automatically sync with stage p=joduinn r=rhelmer
tools/release/Bootstrap/Config.pm
tools/release/Bootstrap/Step/Build.pm
tools/release/Bootstrap/Step/PatcherConfig.pm
tools/release/Bootstrap/Step/Repack.pm
tools/release/Bootstrap/Step/Source.pm
tools/release/Bootstrap/Step/Updates.pm
tools/release/Bootstrap/Util.pm
tools/release/bootstrap.cfg.example
--- a/tools/release/Bootstrap/Config.pm
+++ b/tools/release/Bootstrap/Config.pm
@@ -5,16 +5,17 @@
 package Bootstrap::Config;
 
 use strict;
 
 use POSIX "uname";
 use File::Copy qw(move);
 
 use Bootstrap::Util qw(GetLocaleManifest CvsCatfile);
+use Bootstrap::Util qw(GetFtpNightlyDir);
 
 # shared static config
 my %config;
 
 # single shared instance
 my $singleton = undef;
 
 sub new {
@@ -145,22 +146,20 @@ sub GetFtpCandidateDir {
     my $this = shift;
     my %args = @_;
 
     if (! defined($args{'bitsUnsigned'})) {
       die "ASSERT: Bootstep::Config::GetFtpCandidateDir(): bitsUnsigned is a required argument";
     }
     my $bitsUnsigned = $args{'bitsUnsigned'};
 
-    my $product = $this->Get(var => 'product');
     my $version = $this->Get(var => 'version');
     my $rc = $this->Get(var => 'rc');
 
-    my $candidateDir = CvsCatfile('/home', 'ftp', 'pub', $product, 'nightly',
-                            $version . '-candidates', 'rc' . $rc ) . '/';
+    my $candidateDir = CvsCatfile(GetFtpNightlyDir(), $version . '-candidates', 'rc' . $rc ) . '/';
 
     my $osFileMatch = $this->SystemInfo(var => 'osname');
 
     if ($bitsUnsigned && ($osFileMatch eq 'win32')  && ($version ge '2.0')) {
         $candidateDir .= 'unsigned/';
     }
 
     return $candidateDir;  
--- a/tools/release/Bootstrap/Step/Build.pm
+++ b/tools/release/Bootstrap/Step/Build.pm
@@ -1,17 +1,17 @@
 #
 # Build step. Calls tinderbox to produce en-US Firefox build.
 #
 package Bootstrap::Step::Build;
 
 use File::Temp qw(tempfile);
 
 use Bootstrap::Step;
-use Bootstrap::Util qw(CvsCatfile);
+use Bootstrap::Util qw(CvsCatfile SyncNightlyDirToStaging);
 
 @ISA = ("Bootstrap::Step");
 
 sub Execute {
     my $this = shift;
 
     my $config = new Bootstrap::Config();
     my $buildDir = $config->Get(sysvar => 'buildDir');
@@ -150,16 +150,18 @@ sub Push {
       cmdArgs => ['-2', '-l', $stagingUser, $stagingServer,
                   'rsync', '-av', 
                   '--include=*' . $osFileMatch . '*',
                   '--exclude=*', 
                   $pushDir, 
                   $candidateDir],
       logFile => $pushLog,
     );
+
+    SyncNightlyDirToStaging(); 
 }
 
 sub Announce {
     my $this = shift;
 
     my $config = new Bootstrap::Config();
     my $product = $config->Get(var => 'product');
     my $productTag = $config->Get(var => 'productTag');
@@ -236,9 +238,8 @@ sub StoreBuildID() {
       cmdArgs => ['-p', $buildIDTempFile, 
                   $stagingUser . '@' . $stagingServer . ':' .
                   $pushDir . '/' . $buildIDFile],
       logFile => $pushLog,
     );
 }
 
 1;
-
--- a/tools/release/Bootstrap/Step/PatcherConfig.pm
+++ b/tools/release/Bootstrap/Step/PatcherConfig.pm
@@ -25,17 +25,16 @@ sub Execute {
 
     my $config = new Bootstrap::Config();
     my $logDir = $config->Get(sysvar => 'logDir');
     my $configBumpDir = $config->Get(var => 'configBumpDir');
     my $version = $config->Get(var => 'version');
     my $oldVersion = $config->Get(var => 'oldVersion');
     my $mofoCvsroot = $config->Get(var => 'mofoCvsroot');
     my $patcherConfig = $config->Get(var => 'patcherConfig');
-    my $stagingServer = $config->Get(var => 'stagingServer');
     my $ftpServer = $config->Get(var => 'ftpServer');
     my $bouncerServer = $config->Get(var => 'bouncerServer');
 
     # Create patcher config area in the config bump area.
     if (not -d $configBumpDir) {
         MkdirWithPath(dir => $configBumpDir) 
           or die("Cannot mkdir $configBumpDir: $!");
     }
@@ -84,16 +83,17 @@ sub BumpPatcherConfig {
     my $version = $config->Get(var => 'version');
     my $oldVersion = $config->Get(var => 'oldVersion');
     my $rc = $config->Get(var => 'rc');
     my $oldRc = $config->Get(var => 'oldRc');
     my $localeInfo = $config->GetLocaleInfo();
     my $patcherConfig = $config->Get(var => 'patcherConfig');
     my $stagingUser = $config->Get(var => 'stagingUser');
     my $stagingServer = $config->Get(var => 'stagingServer');
+    my $externalStagingServer = $config->Get(var => 'externalStagingServer');
     my $ftpServer = $config->Get(var => 'ftpServer');
     my $bouncerServer = $config->Get(var => 'bouncerServer');
     my $logDir = $config->Get(sysvar => 'logDir');
 
     # First, parse the file.
     my $checkedOutPatcherConfig = catfile($configBumpDir, 'patcher', 
      $patcherConfig);
 
@@ -165,19 +165,19 @@ sub BumpPatcherConfig {
                              '&os=%bouncer-platform%&lang=%locale%';
 
     $partialUpdate->{'path'} = catfile($product, 'nightly', $version . 
                                '-candidates', $rcStr, $product. '-' . 
                                $oldVersion . '-' . $version .
                                '.%locale%.%platform%.partial.mar');
 
     $partialUpdate->{'betatest-url'} =
-     'http://' . $stagingServer. '/pub/mozilla.org/' . $product. '/nightly/' . 
-     $version . '-candidates/' . $rcStr . '/' . $product . '-' . $oldVersion . 
-     '-' . $version . '.%locale%.%platform%.partial.mar';
+     'http://' . $externalStagingServer. '/pub/mozilla.org/' . $product . 
+     '/nightly/' .  $version . '-candidates/' . $rcStr . '/' . $product . 
+     '-' . $oldVersion .  '-' . $version . '.%locale%.%platform%.partial.mar';
 
     $partialUpdate->{'beta-url'} =
      'http://' . $ftpServer . '/pub/mozilla.org/' . $product. '/nightly/' . 
      $version . '-candidates/' . $rcStr . '/' . $product . '-' . $oldVersion . 
      '-' . $version . '.%locale%.%platform%.partial.mar';
 
     $currentUpdateObj->{'partial'} = $partialUpdate;
 
@@ -187,19 +187,19 @@ sub BumpPatcherConfig {
      $product . '-' . $version . 
      '-complete&os=%bouncer-platform%&lang=%locale%';
 
     $completeUpdate->{'path'} = catfile($product, 'nightly', $version . 
      '-candidates', $rcStr, $product . '-' . $version .
      '.%locale%.%platform%.complete.mar');
 
     $completeUpdate->{'betatest-url'} = 
-     'http://' . $stagingServer . '/pub/mozilla.org/' . $product. '/nightly/' .
-     $version . '-candidates/' . $rcStr .  '/' . $product . '-' . $version .
-     '.%locale%.%platform%.complete.mar';
+     'http://' . $externalStagingServer . '/pub/mozilla.org/' . $product . 
+     '/nightly/' .  $version . '-candidates/' . $rcStr .  '/' . $product . 
+     '-' . $version .  '.%locale%.%platform%.complete.mar';
 
     $completeUpdate->{'beta-url'} = 
      'http://' . $ftpServer . '/pub/mozilla.org/' . $product. '/nightly/' .
      $version . '-candidates/' . $rcStr .  '/' . $product . '-' . $version .
      '.%locale%.%platform%.complete.mar';
 
     $currentUpdateObj->{'complete'} = $completeUpdate;
 
@@ -257,19 +257,19 @@ sub BumpPatcherConfig {
 
     $releaseObj->{'platforms'} = { 'linux-i686' => $linBuildId,
                                    'win32' => $winBuildId,
                                    'mac' => $macBuildId };
 
     $releaseObj->{'locales'} = join(' ', sort (keys(%{$localeInfo})));
 
     $releaseObj->{'completemarurl'} = 
-     'http://' . $stagingServer . '/pub/mozilla.org/' . $product. '/nightly/' .
-     $version . '-candidates/' . $rcStr . '/' . $product . '-'. $version .
-     '.%locale%.%platform%.complete.mar',
+     'http://' . $externalStagingServer . '/pub/mozilla.org/' . $product. 
+     '/nightly/' .  $version . '-candidates/' . $rcStr . '/' . $product . '-'. 
+     $version . '.%locale%.%platform%.complete.mar',
 
     # Compute locale exceptions; 
     # $localeInfo is hash ref of locales -> array of platforms the locale
     # is for.
     #
     # To calculate the exceptions to this rule, create a hash with 
     # all known platforms in it. It's a hash so we can easily delete() keys
     # out of it. Then, we iterate through all the platforms we should build
--- a/tools/release/Bootstrap/Step/Repack.pm
+++ b/tools/release/Bootstrap/Step/Repack.pm
@@ -1,16 +1,16 @@
 #
 # Repack step. Unpacks, modifies, repacks a Firefox en-US build.
 # Primary use is for l10n (localization) builds.
 #
 package Bootstrap::Step::Repack;
 use Bootstrap::Step;
 use Bootstrap::Config;
-use Bootstrap::Util qw(CvsCatfile);
+use Bootstrap::Util qw(CvsCatfile SyncNightlyDirToStaging);
 use MozBuild::Util qw(MkdirWithPath);
 @ISA = ("Bootstrap::Step");
 
 sub Execute {
     my $this = shift;
 
     my $config = new Bootstrap::Config();
     my $l10n_buildDir = $config->Get(sysvar => 'l10n_buildDir');
@@ -227,16 +227,18 @@ sub Push {
       cmdArgs => ['-2', '-l', $stagingUser, $stagingServer,
                   'rsync', '-av',
                   '--include=*' . $osFileMatch . '*',
                   '--include=*.xpi',
                   '--exclude=*', 
                   $pushDir, $candidateDir],
       logFile => $pushLog,
     );
+
+    SyncNightlyDirToStaging(); 
 }
 
 sub Announce {
     my $this = shift;
 
     my $config = new Bootstrap::Config();
     my $product = $config->Get(var => 'product');
     my $productTag = $config->Get(var => 'productTag');
--- a/tools/release/Bootstrap/Step/Source.pm
+++ b/tools/release/Bootstrap/Step/Source.pm
@@ -2,16 +2,17 @@
 # Source step. Creates a source tarball equivalent to what was used to 
 # build the binary release, in the Build step.
 # 
 package Bootstrap::Step::Source;
 use Bootstrap::Step;
 use Bootstrap::Config;
 use File::Copy qw(move);
 use MozBuild::Util qw(MkdirWithPath);
+use Bootstrap::Util qw(SyncNightlyDirToStaging);
 @ISA = ("Bootstrap::Step");
 
 sub Execute {
     my $this = shift;
 
     my $config = new Bootstrap::Config();
     my $product = $config->Get(var => 'product');
     my $productTag = $config->Get(var => 'productTag');
@@ -83,16 +84,18 @@ sub Push {
     $this->Shell(
       cmd => 'rsync',
       cmdArgs => ['-av', catfile('batch-source', 'rc' . $rc, 
                             $product . '-' . $version . '-source.tar.bz2'),
                   $candidateDir],
       logFile => catfile($logDir, 'source.log'),
       dir => catfile($stageDir),
     );
+
+    SyncNightlyDirToStaging();
 }
 
 sub Announce {
     my $this = shift;
 
     my $config = new Bootstrap::Config();
     my $product = $config->Get(var => 'product');
     my $version = $config->Get(var => 'version');
--- a/tools/release/Bootstrap/Step/Updates.pm
+++ b/tools/release/Bootstrap/Step/Updates.pm
@@ -1,17 +1,17 @@
 #
 # Updates step. Generates binary update (MAR) files as well as AUS config
 # snippets.
 # 
 package Bootstrap::Step::Updates;
 
 use Bootstrap::Step;
 use Bootstrap::Config;
-use Bootstrap::Util qw(CvsCatfile);
+use Bootstrap::Util qw(CvsCatfile SyncNightlyDirToStaging);
 
 use File::Find qw(find);
 use POSIX qw(strftime);
 
 use MozBuild::Util qw(MkdirWithPath);
 
 @ISA = ("Bootstrap::Step");
 
@@ -255,16 +255,18 @@ sub Push {
         cmd => 'rsync',
         cmdArgs => ['-av', 
                     '-e', 'ssh -i ' . catfile($ENV{'HOME'},'.ssh','aus'),
                     $dir . '/', 
                     $ausUser . '@' . $ausServer . ':' . $targetDir],
         logFile => $pushLog,
       );
     }
+
+    SyncNightlyDirToStaging();
 }
 
 sub Announce {
     my $this = shift;
 
     my $config = new Bootstrap::Config();
     my $product = $config->Get(var => 'product');
     my $version = $config->Get(var => 'version');
--- a/tools/release/Bootstrap/Util.pm
+++ b/tools/release/Bootstrap/Util.pm
@@ -1,25 +1,26 @@
 #
 # Bootstrap utility functions.
 # 
 
 package Bootstrap::Util;
 
 use File::Temp qw(tempfile tempdir);
-
+use File::Spec::Functions;
 use MozBuild::Util qw(RunShellCommand);
-
 use base qw(Exporter);
 
 our @EXPORT_OK = qw(CvsCatfile CvsTag
                     GetDiffFileList
+                    GetFtpNightlyDir
                     GetLocaleManifest
                     GetBouncerPlatforms GetPatcherPlatforms
-                    GetBouncerToPatcherPlatformMap);
+                    GetBouncerToPatcherPlatformMap
+                    SyncNightlyDirToStaging);
 
 our($DEFAULT_SHELL_TIMEOUT);
 
 use strict;
 
 # This maps Bouncer platforms, used in bouncer and the shipped-locales file
 # to patcher2 platforms used in... patcher2. They're different for some
 # historical reason, which should be considered a bug.
@@ -56,16 +57,31 @@ sub CvsCatfile {
 sub GetBouncerToPatcherPlatformMap {
    return %PLATFORM_MAP;
 }
 
 sub GetBouncerPlatforms {
    return keys(%PLATFORM_MAP);
 }
 
+##
+# GetFtpNightlyDir - construct the FTP path for pushing builds & updates to
+# returns scalar
+#
+# no mandatory arguments
+##
+
+sub GetFtpNightlyDir {
+    my $config = new Bootstrap::Config();
+    my $product = $config->Get(var => 'product');
+
+    my $nightlyDir = CvsCatfile('/home', 'ftp', 'pub', $product, 'nightly') . '/';
+    return $nightlyDir;  
+}
+
 sub GetPatcherPlatforms {
    return values(%PLATFORM_MAP);
 }
 
 # Loads and parses the shipped-locales manifest file, so get hash of
 # locale -> [bouncer] platform mappings; returns success/failure in 
 # reading/parsing the locales file.
 
@@ -265,9 +281,34 @@ sub GetDiffFileList {
             push(@differentFiles, $1);
         }
     }
 
 
     return \@differentFiles;
 }
 
+sub SyncNightlyDirToStaging {
+    my $config = new Bootstrap::Config();
+    my $productTag = $config->Get(var => 'productTag');
+    my $rc = $config->Get(var => 'rc');
+    my $logDir = $config->Get(sysvar => 'logDir');
+    my $externalStagingUser = $config->Get(var => 'externalStagingUser');
+    my $externalStagingServer = $config->Get(var => 'externalStagingServer');
+    
+    my $rcTag = $productTag . '_RC' . $rc;
+    my $pushLog  = catfile($logDir, 'build_' . $rcTag . '-push.log');
+    my $nightlyDir = $config->GetFtpNightlyDir();
+
+
+    my $command = 'rsync';
+    my @cmdArgs = ('-av', $nightlyDir, $externalStagingUser.'@'.$externalStagingServer.':'.$nightlyDir);
+    print 'Bootstrap::Util::SyncNightlyDirToStaging() Running shell command: '.$command.' '.join(' ', @cmdArgs)."\n";
+    my $rv = RunShellCommand(command => $command,
+                             args => \@cmdArgs, 
+                             redirectStderr => 0,
+                             logfile => $pushLog);
+    if ($rv->{'exitValue'} != 0) {
+        die "ASSERT: SyncNightlyDirToStaging(): rsync failed\n";
+    }
+
+}
 1;
--- a/tools/release/bootstrap.cfg.example
+++ b/tools/release/bootstrap.cfg.example
@@ -51,12 +51,14 @@ dumpLogs        = 1
 # username and server to push update snippets to
 ausUser         = cltbld
 ausServer       = localhost
 # Tinderbox server tree that clients should report to
 buildTree       = MozillaTest
 # where QA updates/builds go
 stagingUser     = cltbld
 stagingServer   = stage.mozilla.org
+externalStagingUser     = cltbld
+externalStagingServer   = staging-build-console.build.mozilla.org
 # where beta updates/builds go
 ftpServer       = ftp.mozilla.org
 # where release updates/builds go
 bouncerServer   = download.mozilla.org