Bug 873640 - Fix and cleanup profiledbuild profile script invocation. r=ted
authorMike Hommey <mh+mozilla@glandium.org>
Tue, 21 May 2013 08:28:49 +0200
changeset 144849 d7fa776152731442dd63beef490d437825e8c2fc
parent 144848 668d1525ef292bf2e3208d0938ea9e134e0175ab
child 144850 bf78a5b36a96bb7aba1c673ae5a774458cebad75
push id368
push userbbajaj@mozilla.com
push dateMon, 09 Sep 2013 22:57:58 +0000
treeherdermozilla-release@5a4f47ae1217 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs873640
milestone24.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 873640 - Fix and cleanup profiledbuild profile script invocation. r=ted
b2g/config/mozconfigs/linux32_gecko/nightly
b2g/config/mozconfigs/linux64_gecko/nightly
b2g/config/mozconfigs/win32_gecko/nightly
browser/config/mozconfigs/linux32/beta
browser/config/mozconfigs/linux32/nightly
browser/config/mozconfigs/linux32/qt
browser/config/mozconfigs/linux32/rpm
browser/config/mozconfigs/linux64/beta
browser/config/mozconfigs/linux64/nightly
browser/config/mozconfigs/linux64/rpm
browser/config/mozconfigs/win32/beta
browser/config/mozconfigs/win32/nightly
browser/config/mozconfigs/win64/nightly
build/profile_pageloader.html
build/profile_pageloader.pl
client.mk
--- a/b2g/config/mozconfigs/linux32_gecko/nightly
+++ b/b2g/config/mozconfigs/linux32_gecko/nightly
@@ -11,19 +11,16 @@ ac_add_options --enable-signmar
 # Avoid dependency on libstdc++ 4.5
 ac_add_options --enable-stdcxx-compat
 
 # This will overwrite the default of stripping everything and keep the symbol table.
 # This is useful for profiling and debugging and only increases the package size
 # by 2 MBs.
 STRIP_FLAGS="--strip-debug"
 
-# PGO
-mk_add_options PROFILE_GEN_SCRIPT='$(PYTHON) @MOZ_OBJDIR@/_profile/pgo/profileserver.py 10'
-
 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
 
 export MOZ_TELEMETRY_REPORTING=1
 
 # Treat warnings as errors in directories with FAIL_ON_WARNINGS.
 # DISABLED WHILE NOT ON TRY ac_add_options --enable-warnings-as-errors
 
--- a/b2g/config/mozconfigs/linux64_gecko/nightly
+++ b/b2g/config/mozconfigs/linux64_gecko/nightly
@@ -11,19 +11,16 @@ ac_add_options --enable-signmar
 # Avoid dependency on libstdc++ 4.5
 ac_add_options --enable-stdcxx-compat
 
 # This will overwrite the default of stripping everything and keep the symbol table.
 # This is useful for profiling and debugging and only increases the package size
 # by 2 MBs.
 STRIP_FLAGS="--strip-debug"
 
-# PGO
-mk_add_options PROFILE_GEN_SCRIPT='$(PYTHON) @MOZ_OBJDIR@/_profile/pgo/profileserver.py 10'
-
 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
 
 export MOZ_TELEMETRY_REPORTING=1
 
 # Treat warnings as errors in directories with FAIL_ON_WARNINGS.
 # DISABLED WHILE NOT ON TRY ac_add_options --enable-warnings-as-errors
 
--- a/b2g/config/mozconfigs/win32_gecko/nightly
+++ b/b2g/config/mozconfigs/win32_gecko/nightly
@@ -1,13 +1,10 @@
 . "$topsrcdir/b2g/config/mozconfigs/common"
 
-# for pgo
-mk_add_options PROFILE_GEN_SCRIPT='$(PYTHON) $(MOZ_OBJDIR)/_profile/pgo/profileserver.py'
-
 ac_add_options --enable-update-channel=${MOZ_UPDATE_CHANNEL}
 ac_add_options --enable-update-packaging
 ac_add_options --enable-jemalloc
 ac_add_options --enable-signmar
 
 # Nightlies only since this has a cost in performance
 ac_add_options --enable-js-diagnostics
 
--- a/browser/config/mozconfigs/linux32/beta
+++ b/browser/config/mozconfigs/linux32/beta
@@ -2,19 +2,17 @@ ac_add_options --enable-update-channel=$
 ac_add_options --enable-update-packaging
 ac_add_options --enable-official-branding
 
 . $topsrcdir/build/unix/mozconfig.linux32
 
 # Avoid dependency on libstdc++ 4.5
 ac_add_options --enable-stdcxx-compat
 
-# PGO
 mk_add_options MOZ_PGO=1
-mk_add_options PROFILE_GEN_SCRIPT='EXTRA_TEST_ARGS=10 $(MAKE) -C $(MOZ_OBJDIR) pgo-profile-run'
 
 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
 
 export MOZ_TELEMETRY_REPORTING=1
 
 # Treat warnings as errors in directories with FAIL_ON_WARNINGS.
 ac_add_options --enable-warnings-as-errors
--- a/browser/config/mozconfigs/linux32/nightly
+++ b/browser/config/mozconfigs/linux32/nightly
@@ -12,19 +12,16 @@ ac_add_options --enable-js-diagnostics
 # Avoid dependency on libstdc++ 4.5
 ac_add_options --enable-stdcxx-compat
 
 # This will overwrite the default of stripping everything and keep the symbol table.
 # This is useful for profiling and debugging and only increases the package size
 # by 2 MBs.
 STRIP_FLAGS="--strip-debug"
 
-# PGO
-mk_add_options PROFILE_GEN_SCRIPT='EXTRA_TEST_ARGS=10 $(MAKE) -C $(MOZ_OBJDIR) pgo-profile-run'
-
 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
 
 export MOZ_TELEMETRY_REPORTING=1
 
 # Treat warnings as errors in directories with FAIL_ON_WARNINGS.
 ac_add_options --enable-warnings-as-errors
 
--- a/browser/config/mozconfigs/linux32/qt
+++ b/browser/config/mozconfigs/linux32/qt
@@ -4,19 +4,16 @@ ac_add_options --enable-codesighs
 . $topsrcdir/build/unix/mozconfig.linux32
 
 # Avoid dependency on libstdc++ 4.5
 ac_add_options --enable-stdcxx-compat
 
 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
 
-# PGO
-mk_add_options PROFILE_GEN_SCRIPT='EXTRA_TEST_ARGS=10 $(MAKE) -C $(MOZ_OBJDIR) pgo-profile-run'
-
 #Use ccache
 ac_add_options --with-ccache=/usr/bin/ccache
 
 # QT Options
 export PKG_CONFIG_PATH=/tools/qt-4.6.3/qt/lib/pkgconfig
 ac_add_options --with-qtdir=/tools/qt-4.6.3/qt
 ac_add_options --enable-default-toolkit=cairo-qt
 ac_add_options --disable-crashreporter
--- a/browser/config/mozconfigs/linux32/rpm
+++ b/browser/config/mozconfigs/linux32/rpm
@@ -14,17 +14,15 @@ ac_add_options --libdir=$LIBDIR
 # Avoid dependency on libstdc++ 4.5
 ac_add_options --enable-stdcxx-compat
 
 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
 
 export MOZ_TELEMETRY_REPORTING=1
 
-mk_add_options PROFILE_GEN_SCRIPT='$(MAKE) -C $(MOZ_OBJDIR) pgo-profile-run'
-
 #Use ccache
 ac_add_options --with-ccache=/usr/bin/ccache
 
 # Treat warnings as errors in directories with FAIL_ON_WARNINGS.
 ac_add_options --enable-warnings-as-errors
 
 . "$topsrcdir/build/mozconfig.common.override"
--- a/browser/config/mozconfigs/linux64/beta
+++ b/browser/config/mozconfigs/linux64/beta
@@ -2,19 +2,17 @@ ac_add_options --enable-update-channel=$
 ac_add_options --enable-update-packaging
 ac_add_options --enable-official-branding
 
 . $topsrcdir/build/unix/mozconfig.linux
 
 # Avoid dependency on libstdc++ 4.5
 ac_add_options --enable-stdcxx-compat
 
-# PGO
 mk_add_options MOZ_PGO=1
-mk_add_options PROFILE_GEN_SCRIPT='EXTRA_TEST_ARGS=10 $(MAKE) -C $(MOZ_OBJDIR) pgo-profile-run'
 
 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
 
 export MOZ_TELEMETRY_REPORTING=1
 
 # Treat warnings as errors in directories with FAIL_ON_WARNINGS.
 ac_add_options --enable-warnings-as-errors
--- a/browser/config/mozconfigs/linux64/nightly
+++ b/browser/config/mozconfigs/linux64/nightly
@@ -12,19 +12,16 @@ ac_add_options --enable-js-diagnostics
 # Avoid dependency on libstdc++ 4.5
 ac_add_options --enable-stdcxx-compat
 
 # This will overwrite the default of stripping everything and keep the symbol table.
 # This is useful for profiling and debugging and only increases the package size
 # by 2 MBs.
 STRIP_FLAGS="--strip-debug"
 
-# PGO
-mk_add_options PROFILE_GEN_SCRIPT='EXTRA_TEST_ARGS=10 $(MAKE) -C $(MOZ_OBJDIR) pgo-profile-run'
-
 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
 
 export MOZ_TELEMETRY_REPORTING=1
 
 # Treat warnings as errors in directories with FAIL_ON_WARNINGS.
 ac_add_options --enable-warnings-as-errors
 
--- a/browser/config/mozconfigs/linux64/rpm
+++ b/browser/config/mozconfigs/linux64/rpm
@@ -14,17 +14,15 @@ ac_add_options --libdir=$LIBDIR
 # Avoid dependency on libstdc++ 4.5
 ac_add_options --enable-stdcxx-compat
 
 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
 
 export MOZ_TELEMETRY_REPORTING=1
 
-mk_add_options PROFILE_GEN_SCRIPT='$(MAKE) -C $(MOZ_OBJDIR) pgo-profile-run'
-
 #Use ccache
 ac_add_options --with-ccache=/usr/bin/ccache
 
 # Treat warnings as errors in directories with FAIL_ON_WARNINGS.
 ac_add_options --enable-warnings-as-errors
 
 . "$topsrcdir/build/mozconfig.common.override"
--- a/browser/config/mozconfigs/win32/beta
+++ b/browser/config/mozconfigs/win32/beta
@@ -1,13 +1,11 @@
 . "$topsrcdir/browser/config/mozconfigs/common"
 
-# for pgo
 mk_add_options MOZ_PGO=1
-mk_add_options PROFILE_GEN_SCRIPT='$(MAKE) -C $(MOZ_OBJDIR) pgo-profile-run'
 
 ac_add_options --enable-update-channel=${MOZ_UPDATE_CHANNEL}
 ac_add_options --enable-update-packaging
 ac_add_options --enable-jemalloc
 ac_add_options --enable-official-branding
 
 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
--- a/browser/config/mozconfigs/win32/nightly
+++ b/browser/config/mozconfigs/win32/nightly
@@ -1,13 +1,10 @@
 . "$topsrcdir/browser/config/mozconfigs/common"
 
-# for pgo
-mk_add_options PROFILE_GEN_SCRIPT='$(MAKE) -C $(MOZ_OBJDIR) pgo-profile-run'
-
 ac_add_options --enable-update-channel=${MOZ_UPDATE_CHANNEL}
 ac_add_options --enable-update-packaging
 ac_add_options --enable-jemalloc
 ac_add_options --enable-signmar
 ac_add_options --enable-profiling
 ac_add_options --enable-metro
 
 # Nightlies only since this has a cost in performance
--- a/browser/config/mozconfigs/win64/nightly
+++ b/browser/config/mozconfigs/win64/nightly
@@ -1,16 +1,13 @@
 . "$topsrcdir/browser/config/mozconfigs/common"
 
 ac_add_options --target=x86_64-pc-mingw32
 ac_add_options --host=x86_64-pc-mingw32
 
-# for pgo
-mk_add_options PROFILE_GEN_SCRIPT='$(MAKE) -C $(MOZ_OBJDIR) pgo-profile-run'
-
 ac_add_options --enable-update-channel=${MOZ_UPDATE_CHANNEL}
 ac_add_options --enable-update-packaging
 ac_add_options --enable-jemalloc
 ac_add_options --enable-signmar
 ac_add_options --enable-profiling
 ac_add_options --enable-metro
 
 # Nightlies only since this has a cost in performance
deleted file mode 100644
--- a/build/profile_pageloader.html
+++ /dev/null
@@ -1,11 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-
-<html>
-<script>
-setTimeout("window.close();", 1000);
-</script>
-<body>
-</body>
-</html>
deleted file mode 100755
--- a/build/profile_pageloader.pl
+++ /dev/null
@@ -1,177 +0,0 @@
-#!/usr/bin/perl
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-use Cwd;
-use File::Find ();
-
-use POSIX qw(sys_wait_h);
-
-sub kill_process {
-    my ($target_pid) = @_;
-    my $start_time = time;
-
-    # Try to kill and wait 10 seconds, then try a kill -9
-    my $sig;
-    for $sig ('TERM', 'KILL') {
-        print "kill $sig $target_pid\n";
-        kill $sig => $target_pid;
-        my $interval_start = time;
-        while (time - $interval_start < 10) {
-            # the following will work with 'cygwin' perl on win32, but not
-            # with 'MSWin32' (ActiveState) perl
-            my $pid = waitpid($target_pid, POSIX::WNOHANG());
-            if (($pid == $target_pid and POSIX::WIFEXITED($?)) or $pid == -1) {
-                my $secs = time - $start_time;
-                $secs = $secs == 1 ? '1 second' : "$secs seconds";
-                print "Process killed. Took $secs to die.\n";
-                return;
-            }
-            sleep 1;
-        }
-    }
-    die "Unable to kill process: $target_pid";
-}
-
-# Stripped down version of fork_and_log().
-sub system_fork_and_log {
-    # Fork a sub process and log the output.
-    my ($cmd) = @_;
-
-    my $pid = fork; # Fork off a child process.
-
-    unless ($pid) { # child
-        exec { $cmd->[0] } @$cmd;
-        die "Could not exec()";
-    }
-    return $pid;
-}
-
-
-sub wait_for_pid {
-    # Wait for a process to exit or kill it if it takes too long.
-    my ($pid, $timeout_secs) = @_;
-    my ($exit_value, $signal_num, $dumped_core, $timed_out) = (0,0,0,0);
-    my $sig_name;
-    my $loop_count;
-
-    die ("Invalid timeout value passed to wait_for_pid()\n")
-        if ($timeout_secs <= 0);
-
-    eval {
-        $loop_count = 0;
-        while (++$loop_count < $timeout_secs) {
-            my $wait_pid = waitpid($pid, POSIX::WNOHANG());
-            # the following will work with 'cygwin' perl on win32, but not 
-            # with 'MSWin32' (ActiveState) perl
-            last if ($wait_pid == $pid and POSIX::WIFEXITED($?)) or $wait_pid == -1;
-            sleep 1;
-        }
-
-        $exit_value = $? >> 8;
-        $signal_num = $? >> 127;
-        $dumped_core = $? & 128;
-        if ($loop_count >= $timeout_secs) {
-            die "timeout";
-        }
-        return "done";
-    };
-
-    if ($@) {
-        if ($@ =~ /timeout/) {
-            kill_process($pid);
-            $timed_out = 1;
-        } else { # Died for some other reason.
-            die; # Propagate the error up.
-        }
-    }
-#    $sig_name = $signal_num ? signal_name($signal_num) : '';
-#
-#    return { timed_out=>$timed_out,
-#             exit_value=>$exit_value,
-#             sig_name=>$sig_name,
-#             dumped_core=>$dumped_core };
-}
-
-# System version of run_cmd().
-sub run_system_cmd {
-    my ($cmd, $timeout_secs) = @_;
-
-#    print_log "cmd = $cmd\n";
-    my $pid = system_fork_and_log($cmd);
-    my $result = wait_for_pid($pid, $timeout_secs);
-
-    return $result;
-}
-
-#
-# Given profile directory, find pref file hidden in salt directory.
-# profile $Settings::MozProfileName must exist before calling this sub.
-#
-sub find_pref_file {
-    my $profile_dir = shift;
-
-    # default to *nix
-    my $pref_file = "prefs.js";
-
-    unless (-e $profile_dir) {
-        return; # empty list
-    }
-
-    my $found = undef;
-    my $sub = sub {$pref_file = $File::Find::name, $found++ if $pref_file eq $_};
-    File::Find::find($sub, $profile_dir);
-    unless ($found) {
-        return; # empty list
-    }
-
-    return $pref_file;
-}
-
-my $topdir = cwd();
-
-chdir $ENV{OBJDIR};
-my $app_name = `grep "MOZ_APP_NAME	" config/autoconf.mk | sed "s/.*= //"`;
-chomp($app_name);
-
-# On mac, the app directory is the product name with the first
-# letter capitalized
-
-my $toolkit = `grep "MOZ_WIDGET_TOOLKIT        " config/autoconf.mk  |sed "s/.*= //"`;
-chomp($toolkit);
-
-if ($toolkit =~ /(mac|cocoa)/) {
-    my $app_dir = uc(substr($app_name, 0, 1)).substr($app_name, 1);
-    chdir "dist/$app_dir.app/Contents/MacOS";
-} else {
-    chdir "dist/bin";
-}
-
-my $bin_suffix = "";
-if ($toolkit =~ /(windows|os2)/) {
-    $bin_suffix = ".exe";
-}
-
-my $old_home = $ENV{HOME};
-$ENV{HOME} = cwd();
-
-# Create a profile to test with.
-run_system_cmd(["./".$app_name.$bin_suffix, "-createProfile", "testprofile"], 45);
-
-my $pref_file = find_pref_file(".mozilla/".$app_name);
-open PREFS, ">>$pref_file";
-# Add allow_scripts_to_close_windows; this lets us cleanly exit.
-print PREFS "user_pref(\"dom.allow_scripts_to_close_windows\", true);\n";
-# Suppress the default browser dialog since it keeps the test from starting.
-print PREFS "user_pref(\"browser.shell.checkDefaultBrowser\", false);\n";
-close PREFS;
-
-# Run the pageload test.
-run_system_cmd(["./".$app_name.$bin_suffix, $ENV{PAGELOAD_URL}."/loader.pl?maxcyc=2&delay=500&nocache=0&timeout=30000&auto=1"], 240);
-
-# Start up again; this will gather data for reading global history and
-# reading the fastload file.
-run_system_cmd(["./".$app_name.$bin_suffix, "file://$topdir/build/profile_pageloader.html"], 45);
-
-chdir $topdir;
--- a/client.mk
+++ b/client.mk
@@ -192,36 +192,30 @@ clobber clobber_all: clean
 # helper target for mobile
 build_and_deploy: build fast-package install
 
 # Do everything from scratch
 everything: clean build
 
 ####################################
 # Profile-Guided Optimization
-#  To use this, you should set the following variables in your mozconfig
-#    mk_add_options PROFILE_GEN_SCRIPT=/path/to/profile-script
-#
-#  The profile script should exercise the functionality to be included
-#  in the profile feedback.
-#
 #  This is up here, outside of the MOZ_CURRENT_PROJECT logic so that this
 #  is usable in multi-pass builds, where you might not have a runnable
 #  application until all the build passes and postflight scripts have run.
 ifdef MOZ_OBJDIR
   PGO_OBJDIR = $(MOZ_OBJDIR)
 else
   PGO_OBJDIR := $(TOPSRCDIR)
 endif
 
 profiledbuild::
 	$(MAKE) -f $(TOPSRCDIR)/client.mk realbuild MOZ_PROFILE_GENERATE=1 MOZ_PGO_INSTRUMENTED=1
 	$(MAKE) -C $(PGO_OBJDIR) package MOZ_PGO_INSTRUMENTED=1 MOZ_INTERNAL_SIGNING_FORMAT= MOZ_EXTERNAL_SIGNING_FORMAT=
 	rm -f ${PGO_OBJDIR}/jarlog/en-US.log
-	MOZ_PGO_INSTRUMENTED=1 OBJDIR=${PGO_OBJDIR} JARLOG_FILE=${PGO_OBJDIR}/jarlog/en-US.log $(PROFILE_GEN_SCRIPT)
+	MOZ_PGO_INSTRUMENTED=1 JARLOG_FILE=jarlog/en-US.log EXTRA_TEST_ARGS=10 $(MAKE) -C $(PGO_OBJDIR) pgo-profile-run
 	$(MAKE) -f $(TOPSRCDIR)/client.mk maybe_clobber_profiledbuild
 	$(MAKE) -f $(TOPSRCDIR)/client.mk realbuild MOZ_PROFILE_USE=1
 
 #####################################################
 # Build date unification
 
 ifdef MOZ_UNIFY_BDATE
 ifndef MOZ_BUILD_DATE