Bug 1144944 -Add support for emulator kk x86 r=kli
authorjlal@mozilla.com
Fri, 27 Mar 2015 00:48:15 -0700
changeset 235925 6140e2b27712e096c5bd70679098b72b35e5af5a
parent 235924 83c1b78abd10c60598b5e17a40a003c48ae381d5
child 235926 a9d42e5c0c4426dfa4e37d79ab495337f83dcb5d
push id14770
push userjlal@mozilla.com
push dateFri, 27 Mar 2015 09:30:07 +0000
treeherderb2g-inbound@6140e2b27712 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerskli
bugs1144944
milestone39.0a1
Bug 1144944 -Add support for emulator kk x86 r=kli
b2g/config/emulator-x86-kk/config.json
b2g/config/emulator-x86-kk/releng-emulator-kk.tt
b2g/config/emulator-x86-kk/sources.xml
b2g/config/emulator-x86-l/config.json
b2g/config/emulator-x86-l/releng-emulator-l.tt
b2g/config/emulator-x86-l/sources.xml
testing/taskcluster/scripts/builder/build-emulator-x86.sh
testing/taskcluster/scripts/builder/build-emulator.sh
testing/taskcluster/tasks/branches/base_job_flags.yml
testing/taskcluster/tasks/branches/try/job_flags.yml
testing/taskcluster/tasks/builds/b2g_emulator_x86_base.yml
testing/taskcluster/tasks/builds/b2g_emulator_x86_kk_opt.yml
testing/taskcluster/tasks/builds/b2g_emulator_x86_l_opt.yml
testing/taskcluster/tasks/job_flags.yml
new file mode 100644
--- /dev/null
+++ b/b2g/config/emulator-x86-kk/config.json
@@ -0,0 +1,32 @@
+{
+    "config_version": 2,
+    "tooltool_manifest": "releng-emulator-kk.tt",
+    "mock_target": "mozilla-centos6-x86_64",
+    "mock_packages": ["ccache", "make", "bison", "flex", "gcc", "g++", "mpfr", "zlib-devel", "ncurses-devel", "zip", "autoconf213", "glibc-static", "perl-Digest-SHA", "wget", "alsa-lib", "atk", "cairo", "dbus-glib", "fontconfig", "freetype", "glib2", "gtk2", "libXRender", "libXt", "pango", "mozilla-python27-mercurial", "openssh-clients", "nss-devel", "glibc-devel.i686", "libstdc++.i686", "zlib-devel.i686", "ncurses-devel.i686", "libX11-devel.i686", "mesa-libGL-devel.i686", "mesa-libGL-devel", "libX11-devel", "git", "libxml2"],
+    "mock_files": [["/home/cltbld/.ssh", "/home/mock_mozilla/.ssh"]],
+    "build_targets": ["droid", "package-emulator", "package-tests"],
+    "upload_files": [
+        "{workdir}/out/target/product/generic/*.tar.bz2",
+        "{workdir}/out/target/product/generic/tests/*.zip",
+        "{workdir}/out/emulator.tar.gz",
+        "{objdir}/dist/b2g-*.crashreporter-symbols.zip",
+        "{workdir}/sources.xml"
+    ],
+    "public_upload_files": [
+        "{workdir}/out/target/product/generic/*.tar.bz2",
+        "{workdir}/out/target/product/generic/tests/*.zip",
+        "{objdir}/dist/b2g-*.crashreporter-symbols.zip",
+        "{objdir}/dist/b2g-*.tar.gz",
+        "{workdir}/sources.xml"
+    ],
+    "upload_platform": "emulator-kk",
+    "gecko_l10n_root": "https://hg.mozilla.org/l10n-central",
+    "gaia": {
+        "l10n": {
+            "vcs": "hgtool",
+            "root": "https://hg.mozilla.org/gaia-l10n"
+        }
+    },
+    "b2g_manifest": "emulator-kk.xml",
+    "b2g_manifest_intree": true
+}
new file mode 100644
--- /dev/null
+++ b/b2g/config/emulator-x86-kk/releng-emulator-kk.tt
@@ -0,0 +1,9 @@
+[
+{
+"size": 80458572,
+"digest": "e5101f9dee1e462f6cbd3897ea57eede41d23981825c7b20d91d23ab461875d54d3dfc24999aa58a31e8b01f49fb3140e05ffe5af2957ef1d1afb89fd0dfe1ad",
+"algorithm": "sha512",
+"filename": "gcc.tar.xz",
+"unpack": "True"
+}
+]
new file mode 120000
--- /dev/null
+++ b/b2g/config/emulator-x86-kk/sources.xml
@@ -0,0 +1,1 @@
+../emulator-kk/sources.xml
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/b2g/config/emulator-x86-l/config.json
@@ -0,0 +1,32 @@
+{
+    "config_version": 2,
+    "tooltool_manifest": "releng-emulator-l.tt",
+    "mock_target": "mozilla-centos6-x86_64",
+    "mock_packages": ["ccache", "make", "bison", "flex", "gcc", "g++", "mpfr", "zlib-devel", "ncurses-devel", "zip", "autoconf213", "glibc-static", "perl-Digest-SHA", "wget", "alsa-lib", "atk", "cairo", "dbus-glib", "fontconfig", "freetype", "glib2", "gtk2", "libXRender", "libXt", "pango", "mozilla-python27-mercurial", "openssh-clients", "nss-devel", "glibc-devel.i686", "libstdc++.i686", "zlib-devel.i686", "ncurses-devel.i686", "libX11-devel.i686", "mesa-libGL-devel.i686", "mesa-libGL-devel", "libX11-devel", "git", "libxml2"],
+    "mock_files": [["/home/cltbld/.ssh", "/home/mock_mozilla/.ssh"]],
+    "build_targets": ["droid", "package-emulator", "package-tests"],
+    "upload_files": [
+        "{workdir}/out/target/product/generic/*.tar.bz2",
+        "{workdir}/out/target/product/generic/tests/*.zip",
+        "{workdir}/out/emulator.tar.gz",
+        "{objdir}/dist/b2g-*.crashreporter-symbols.zip",
+        "{workdir}/sources.xml"
+    ],
+    "public_upload_files": [
+        "{workdir}/out/target/product/generic/*.tar.bz2",
+        "{workdir}/out/target/product/generic/tests/*.zip",
+        "{objdir}/dist/b2g-*.crashreporter-symbols.zip",
+        "{objdir}/dist/b2g-*.tar.gz",
+        "{workdir}/sources.xml"
+    ],
+    "upload_platform": "emulator-l",
+    "gecko_l10n_root": "https://hg.mozilla.org/l10n-central",
+    "gaia": {
+        "l10n": {
+            "vcs": "hgtool",
+            "root": "https://hg.mozilla.org/gaia-l10n"
+        }
+    },
+    "b2g_manifest": "emulator-l.xml",
+    "b2g_manifest_intree": true
+}
new file mode 100644
--- /dev/null
+++ b/b2g/config/emulator-x86-l/releng-emulator-l.tt
@@ -0,0 +1,9 @@
+[
+{
+"size": 80458572,
+"digest": "e5101f9dee1e462f6cbd3897ea57eede41d23981825c7b20d91d23ab461875d54d3dfc24999aa58a31e8b01f49fb3140e05ffe5af2957ef1d1afb89fd0dfe1ad",
+"algorithm": "sha512",
+"filename": "gcc.tar.xz",
+"unpack": "True"
+}
+]
new file mode 120000
--- /dev/null
+++ b/b2g/config/emulator-x86-l/sources.xml
@@ -0,0 +1,1 @@
+../emulator-l/sources.xml
\ No newline at end of file
new file mode 100755
--- /dev/null
+++ b/testing/taskcluster/scripts/builder/build-emulator-x86.sh
@@ -0,0 +1,71 @@
+#! /bin/bash -vex
+
+# Ensure all the scripts in this dir are on the path....
+DIRNAME=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+PATH=$DIRNAME:$PATH
+
+WORKSPACE=$1
+
+### Check that require variables are defined
+test -d $WORKSPACE
+test $GECKO_HEAD_REPOSITORY # Should be an hg repository url to pull from
+test $GECKO_BASE_REPOSITORY # Should be an hg repository url to clone from
+test $GECKO_HEAD_REV # Should be an hg revision to pull down
+test $MOZHARNESS_REPOSITORY # mozharness repository
+test $MOZHARNESS_REV # mozharness revision
+test $TARGET
+
+. setup-ccache.sh
+
+# First check if the mozharness directory is available. This is intended to be
+# used locally in development to test mozharness changes:
+#
+#   $ docker -v your_mozharness:/home/worker/mozharness ...
+#
+if [ ! -d mozharness ]; then
+  tc-vcs checkout mozharness $MOZHARNESS_REPOSITORY $MOZHARNESS_REPOSITORY $MOZHARNESS_REV
+fi
+
+# Figure out where the remote manifest is so we can use caches for it.
+
+if [ -z "$MANIFEST" ]; then
+  MANIFEST="$WORKSPACE/gecko/b2g/config/$TARGET/sources.xml"
+fi
+
+tc-vcs repo-checkout $WORKSPACE/B2G https://git.mozilla.org/b2g/B2G.git $MANIFEST
+
+# Ensure symlink has been created to gecko...
+rm -f $WORKSPACE/B2G/gecko
+ln -s $WORKSPACE/gecko $WORKSPACE/B2G/gecko
+
+debug_flag=""
+if [ 0$B2G_DEBUG -ne 0 ]; then
+  debug_flag='--debug'
+fi
+
+./mozharness/scripts/b2g_build.py \
+  --config b2g/taskcluster-emulator.py \
+  "$debug_flag" \
+  --disable-mock \
+  --work-dir=$WORKSPACE/B2G \
+  --log-level=debug \
+  --target=$TARGET \
+  --b2g-config-dir=$TARGET \
+  --checkout-revision=$GECKO_HEAD_REV \
+  --base-repo=$GECKO_BASE_REPOSITORY \
+  --repo=$GECKO_HEAD_REPOSITORY
+
+# Move files into artifact locations!
+mkdir -p $HOME/artifacts
+
+ls -lah $WORKSPACE/B2G/out
+ls -lah $WORKSPACE/B2G/objdir-gecko/dist/
+
+mv $WORKSPACE/B2G/sources.xml $HOME/artifacts/sources.xml
+mv $WORKSPACE/B2G/out/target/product/generic_x86/tests/gaia-tests.zip $HOME/artifacts/gaia-tests.zip
+mv $WORKSPACE/B2G/out/target/product/generic_x86/tests/b2g-*.zip $HOME/artifacts/b2g-tests.zip
+mv $WORKSPACE/B2G/out/emulator.tar.gz $HOME/artifacts/emulator.tar.gz
+mv $WORKSPACE/B2G/objdir-gecko/dist/b2g-*.crashreporter-symbols.zip $HOME/artifacts/b2g-crashreporter-symbols.zip
+
+ccache -s
+
--- a/testing/taskcluster/scripts/builder/build-emulator.sh
+++ b/testing/taskcluster/scripts/builder/build-emulator.sh
@@ -23,17 +23,21 @@ test $TARGET
 #
 #   $ docker -v your_mozharness:/home/worker/mozharness ...
 #
 if [ ! -d mozharness ]; then
   tc-vcs checkout mozharness $MOZHARNESS_REPOSITORY $MOZHARNESS_REPOSITORY $MOZHARNESS_REV $MOZHARNESS_REF
 fi
 
 # Figure out where the remote manifest is so we can use caches for it.
-MANIFEST=$(repository-url.py $GECKO_HEAD_REPOSITORY $GECKO_HEAD_REV b2g/config/$TARGET/sources.xml)
+
+if [ -z "$MANIFEST" ]; then
+  MANIFEST="$WORKSPACE/gecko/b2g/config/$TARGET/sources.xml"
+fi
+
 tc-vcs repo-checkout $WORKSPACE/B2G https://git.mozilla.org/b2g/B2G.git $MANIFEST
 
 # Ensure symlink has been created to gecko...
 rm -f $WORKSPACE/B2G/gecko
 ln -s $WORKSPACE/gecko $WORKSPACE/B2G/gecko
 
 debug_flag=""
 if [ 0$B2G_DEBUG -ne 0 ]; then
--- a/testing/taskcluster/tasks/branches/base_job_flags.yml
+++ b/testing/taskcluster/tasks/branches/base_job_flags.yml
@@ -4,16 +4,17 @@
 flags:
   aliases:
     mochitests: mochitest
 
   builds:
     - emulator
     - emulator-jb
     - emulator-kk
+    - emulator-x86-kk
     - emulator-l
     - linux32_gecko  # b2g desktop linux 32 bit
     - linux64_gecko  # b2g desktop linux 64 bit
     - linux64-mulet  # Firefox desktop - b2g gecko linux 64 bit
     - macosx64_gecko # b2g desktop osx 64 bit
     - win32_gecko    # b2g desktop win 32 bit
     - flame-kk       # b2g flame kitkat
     - flame-kk-eng   # b2g flame eng build
--- a/testing/taskcluster/tasks/branches/try/job_flags.yml
+++ b/testing/taskcluster/tasks/branches/try/job_flags.yml
@@ -31,16 +31,28 @@ builds:
   emulator-kk:
     platfoms:
       - b2g
     types:
       opt:
         task: tasks/builds/b2g_emulator_kk_opt.yml
       debug:
         task: tasks/builds/b2g_emulator_kk_debug.yml
+  emulator-x86-l:
+    platfoms:
+      - b2g
+    types:
+      opt:
+        task: tasks/builds/b2g_emulator_x86_l_opt.yml
+  emulator-x86-kk:
+    platfoms:
+      - b2g
+    types:
+      opt:
+        task: tasks/builds/b2g_emulator_x86_kk_opt.yml
   emulator-jb:
     platfoms:
       - b2g
     types:
       opt:
         task: tasks/builds/b2g_emulator_jb_opt.yml
       debug:
         task: tasks/builds/b2g_emulator_jb_debug.yml
@@ -129,38 +141,48 @@ tests:
       tasks/builds/b2g_emulator_ics_opt.yml:
         task: tasks/tests/b2g_emulator_marionette_webapi.yml
   mochitest:
     allowed_build_tasks:
       tasks/builds/b2g_emulator_kk_debug.yml:
         task: tasks/tests/b2g_emulator_mochitest.yml
       tasks/builds/b2g_emulator_kk_opt.yml:
         task: tasks/tests/b2g_emulator_mochitest.yml
+      tasks/builds/b2g_emulator_x86_l_opt.yml:
+        task: tasks/tests/b2g_emulator_mochitest.yml
+      tasks/builds/b2g_emulator_x86_kk_opt.yml:
+        task: tasks/tests/b2g_emulator_mochitest.yml
       tasks/builds/b2g_emulator_ics_opt.yml:
         task: tasks/tests/b2g_emulator_mochitest.yml
       tasks/builds/b2g_emulator_ics_debug.yml:
         task: tasks/tests/b2g_emulator_mochitest.yml
       tasks/builds/mulet_linux.yml:
         task: tasks/tests/mulet_mochitests.yml
   mochitest-oop:
     allowed_build_tasks:
       tasks/builds/b2g_desktop_opt.yml:
         task: tasks/tests/b2g_mochitest_oop.yml
   reftest:
     allowed_build_tasks:
       tasks/builds/mulet_linux.yml:
         task: tasks/tests/mulet_reftests.yml
       tasks/builds/b2g_emulator_kk_opt.yml:
         task: tasks/tests/b2g_emulator_reftest.yml
+      tasks/builds/b2g_emulator_x86_l_opt.yml:
+        task: tasks/tests/b2g_emulator_reftest.yml
+      tasks/builds/b2g_emulator_x86_kk_opt.yml:
+        task: tasks/tests/b2g_emulator_reftest.yml
       tasks/builds/b2g_emulator_ics_opt.yml:
         task: tasks/tests/b2g_emulator_reftest.yml
   reftest-sanity-oop:
     allowed_build_tasks:
       tasks/builds/b2g_desktop_opt.yml:
         task: tasks/tests/b2g_reftests_sanity_oop.yml
   xpcshell:
     allowed_build_tasks:
+      tasks/builds/b2g_emulator_x86_kk_opt.yml:
+        task: tasks/tests/b2g_emulator_xpcshell_chunked.yml
       tasks/builds/b2g_emulator_kk_opt.yml:
         task: tasks/tests/b2g_emulator_xpcshell_chunked.yml
       tasks/builds/b2g_emulator_ics_opt.yml:
         task: tasks/tests/b2g_emulator_xpcshell_chunked.yml
       tasks/builds/b2g_emulator_ics_debug.yml:
         task: tasks/tests/b2g_emulator_xpcshell_chunked.yml
new file mode 100644
--- /dev/null
+++ b/testing/taskcluster/tasks/builds/b2g_emulator_x86_base.yml
@@ -0,0 +1,34 @@
+$inherits:
+  from: 'tasks/build.yml'
+task:
+  metadata:
+    description: |
+      Android emulators + b2g environment used in full stack testing.
+  payload:
+    env:
+      TARGET: 'emulator'
+      B2G_DEBUG: 0
+
+    # Emulators can take a very long time to build!
+    maxRunTime: 14400
+
+    command:
+      - /bin/bash
+      - -c
+      - >
+        checkout-gecko workspace &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/builder &&
+        buildbot_step 'Build' ./build-emulator-x86.sh $HOME/workspace
+
+  extra:
+    treeherder:
+      groupSymbol: x86
+    # Rather then enforcing particular conventions we require that all build
+    # tasks provide the "build" extra field to specify where the build and tests
+    # files are located.
+    locations:
+      build: 'public/build/emulator.tar.gz'
+      tests: 'public/build/b2g-tests.zip'
+      symbols: 'public/build/b2g-crashreporter-symbols.zip'
+      sources: 'public/build/sources.xml'
+
new file mode 100644
--- /dev/null
+++ b/testing/taskcluster/tasks/builds/b2g_emulator_x86_kk_opt.yml
@@ -0,0 +1,22 @@
+$inherits:
+  from: 'tasks/builds/b2g_emulator_x86_base.yml'
+task:
+  workerType: emualtor-x86-kk
+  scopes:
+    - 'docker-worker:cache:workspace-emulator-kk-x86-opt'
+  metadata:
+    name: '[TC] B2G KK X86 Emulator (Opt)'
+
+  extra:
+    treeherderEnv:
+      - staging
+    treeherder:
+      # Disable "TC" prefix...
+      machine:
+        platform: b2g-emu-kk
+
+  payload:
+    cache:
+      workspace-emulator-kk-x86-opt: /home/worker/workspace
+    env:
+      TARGET: 'emulator-x86-kk'
new file mode 100644
--- /dev/null
+++ b/testing/taskcluster/tasks/builds/b2g_emulator_x86_l_opt.yml
@@ -0,0 +1,34 @@
+$inherits:
+  from: 'tasks/builds/b2g_emulator_base.yml'
+task:
+  workerType: emulator-l
+  scopes:
+    - 'docker-worker:cache:workspace-emulator-l-x86-opt'
+  metadata:
+    name: '[TC] B2G X86 L Emulator (Opt)'
+
+  extra:
+    treeherderEnv:
+      - staging
+    treeherder:
+      # Disable "TC" prefix...
+      groupSymbol: "X86"
+      machine:
+        platform: b2g-emu-l
+
+  payload:
+    command:
+      - /bin/bash
+      - -c
+      - >
+        checkout-gecko workspace &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/builder &&
+        buildbot_step 'Build' ./build-emulator-x86.sh $HOME/workspace
+
+    cache:
+      workspace-emulator-l-x86-opt: /home/worker/workspace
+    env:
+      TARGET: 'emulator-x86-l'
+
+
+
--- a/testing/taskcluster/tasks/job_flags.yml
+++ b/testing/taskcluster/tasks/job_flags.yml
@@ -3,16 +3,17 @@
 
 # List of all possible flags for each category of tests used in the case where
 # "all" is specified.
 flags:
   builds:
     - emulator
     - emulator-jb
     - emulator-kk
+    - emulator-x86-kk
     - linux32_gecko  # b2g desktop linux 32 bit
     - linux64_gecko  # b2g desktop linux 64 bit
     - linux64-mulet  # Firefox desktop - b2g gecko linux 64 bit
     - macosx64_gecko # b2g desktop osx 64 bit
     - win32_gecko    # b2g desktop win 32 bit
 
   tests:
     - cppunit