Bug 1252928 - Part 3: Replace android-gradle-build-dependencies with android-api-15-gradle-dependencies. r?dustin draft
authorNick Alexander <nalexander@mozilla.com>
Tue, 08 Mar 2016 13:06:59 -0800
changeset 338376 5fff48094b8d4789235203fb27691f6709acf0cb
parent 338375 d333ea281ab70695c5c2e66d65a37edd9082f714
child 515788 6e04af6d5d59e84412738c6fe01e07f7afa475a5
push id12496
push usernalexander@mozilla.com
push dateTue, 08 Mar 2016 23:31:04 +0000
reviewersdustin
bugs1252928
milestone47.0a1
Bug 1252928 - Part 3: Replace android-gradle-build-dependencies with android-api-15-gradle-dependencies. r?dustin The existing task's outcome is best achieved with a special purpose build task, so here it is. MozReview-Commit-ID: 3gYnAb69TdK
mobile/android/config/mozconfigs/android-api-15-gradle-dependencies/nightly
mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest
testing/mozharness/configs/builds/releng_sub_android_configs/64_api_15_gradle_dependencies.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
testing/taskcluster/scripts/misc/repackage-jdk-centos.sh
testing/taskcluster/tasks/android-gradle-build-dependencies.yml
testing/taskcluster/tasks/branches/base_jobs.yml
testing/taskcluster/tasks/builds/android_api_15_gradle_dependencies.yml
copy from mobile/android/config/mozconfigs/android-api-15-frontend/nightly
copy to mobile/android/config/mozconfigs/android-api-15-gradle-dependencies/nightly
--- a/mobile/android/config/mozconfigs/android-api-15-frontend/nightly
+++ b/mobile/android/config/mozconfigs/android-api-15-gradle-dependencies/nightly
@@ -6,18 +6,20 @@ MOZ_AUTOMATION_PACKAGE=0
 MOZ_AUTOMATION_PACKAGE_TESTS=0
 MOZ_AUTOMATION_SDK=0
 MOZ_AUTOMATION_UPDATE_PACKAGING=0
 MOZ_AUTOMATION_UPLOAD=0
 MOZ_AUTOMATION_UPLOAD_SYMBOLS=0
 
 . "$topsrcdir/mobile/android/config/mozconfigs/common"
 
-ac_add_options --with-gradle="$topsrcdir/gradle/bin/gradle"
-export GRADLE_MAVEN_REPOSITORY="file://$topsrcdir/jcentral"
+# We want to download Gradle.
+ac_add_options --with-gradle
+# We want to use (and populate!) the local Nexus repository.
+export GRADLE_MAVEN_REPOSITORY="http://localhost:8081/nexus/content/repositories/central/"
 
 ac_add_options --disable-compile-environment
 ac_add_options --disable-tests
 
 # From here on, like ../android-api-15/nightly.
 
 ac_add_options --enable-profiling
 
copy from mobile/android/config/tooltool-manifests/android-frontend/releng.manifest
copy to mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest
--- a/mobile/android/config/tooltool-manifests/android-frontend/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest
@@ -11,40 +11,24 @@
 "size": 167175,
 "visibility": "public",
 "digest": "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
 "algorithm": "sha512",
 "filename": "sccache.tar.bz2",
 "unpack": true
 },
 {
-"size": 31013068,
+"size": 30899096,
 "visibility": "public",
-"digest": "e30a26f98a3448064857491aee1a7a26f98494f86a89113de9be17c37c8181ed60250706fed881ec1f035002fcdaf8b9b4a7d9ae70ce40acff2f1acfbb40f8d9",
+"digest": "ac9f5f95d11580d3dbeff87e80a585fe4d324b270dabb91b1165686acab47d99fa6651074ab0be09420239a5d6af38bb2c539506962a7b44e0ed4d080bba2953",
 "algorithm": "sha512",
-"filename": "java_home-1.7.0-openjdk-1.7.0.85.x86_64.tar.xz",
-"unpack": true
+"unpack": true,
+"filename": "java_home-1.7.0-openjdk-1.7.0.85.x86_64.tar.xz"
 },
 {
 "algorithm": "sha512",
 "visibility": "public",
-"filename": "jcentral.tar.xz",
-"unpack": true,
-"digest": "b5d85a917785e1c034318f7495fef27a6274b04d8640245726b0cf1331b7ac374f5757868901c3fadd930bf10603173a706be653d769dde8ddfdb8673b143363",
-"size": 38596168
-},
-{
-"algorithm": "sha512",
-"visibility": "public",
-"filename": "gradle.tar.xz",
+"filename": "dotgradle-online.tar.xz",
 "unpack": true,
-"digest": "ef1d0038da879cc6840fced87671f8f6a18c51375498804f64d21fa48d7089ded4da2be36bd06a1457083e9110e59c0884f1e074dc609d29617c131caea8f234",
-"size": 50542140
-},
-{
-"algorithm": "sha512",
-"visibility": "public",
-"filename": "dotgradle.tar.xz",
-"unpack": true,
-"digest": "9f082ccd71ad18991eb71fcad355c6990f50a72a09ab9b79696521485656083a72faf5a8d4714de9c4b901ee2319b6786a51964846bb7075061642a8505501c2",
+"digest": "0979eb1dcd9349a9c3f51f24747bb6e19e803226d7150fcf6846889ae24a8df925d03edfac038a5330822703c51130d4f5757d9f4caff7bcb2b6f71858c024d3",
 "size": 512
 }
 ]
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/builds/releng_sub_android_configs/64_api_15_gradle_dependencies.py
@@ -0,0 +1,8 @@
+config = {
+    'base_name': 'Android armv7 API 15+ Gradle dependencies %(branch)s',
+    'stage_platform': 'android-api-15-gradle-dependencies',
+    'build_type': 'api-15-opt',
+    'src_mozconfig': 'mobile/android/config/mozconfigs/android-api-15-gradle-dependencies/nightly',
+    'tooltool_manifest_src': 'mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest',
+    'multi_locale_config_platform': 'android',
+}
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -353,16 +353,17 @@ class BuildOptionParser(object):
         'mulet': 'builds/releng_sub_%s_configs/%s_mulet.py',
         'code-coverage': 'builds/releng_sub_%s_configs/%s_code_coverage.py',
         'graphene': 'builds/releng_sub_%s_configs/%s_graphene.py',
         'horizon': 'builds/releng_sub_%s_configs/%s_horizon.py',
         'source': 'builds/releng_sub_%s_configs/%s_source.py',
         'api-9': 'builds/releng_sub_%s_configs/%s_api_9.py',
         'api-11': 'builds/releng_sub_%s_configs/%s_api_11.py',
         'api-15-frontend': 'builds/releng_sub_%s_configs/%s_api_15_frontend.py',
+        'api-15-gradle-dependencies': 'builds/releng_sub_%s_configs/%s_api_15_gradle_dependencies.py',
         'api-15': 'builds/releng_sub_%s_configs/%s_api_15.py',
         'api-9-debug': 'builds/releng_sub_%s_configs/%s_api_9_debug.py',
         'api-11-debug': 'builds/releng_sub_%s_configs/%s_api_11_debug.py',
         'api-15-debug': 'builds/releng_sub_%s_configs/%s_api_15_debug.py',
         'x86': 'builds/releng_sub_%s_configs/%s_x86.py',
         'api-11-partner-sample1': 'builds/releng_sub_%s_configs/%s_api_11_partner_sample1.py',
         'api-15-partner-sample1': 'builds/releng_sub_%s_configs/%s_api_15_partner_sample1.py',
         'api-11-b2gdroid': 'builds/releng_sub_%s_configs/%s_api_11_b2gdroid.py',
--- a/testing/taskcluster/scripts/misc/repackage-jdk-centos.sh
+++ b/testing/taskcluster/scripts/misc/repackage-jdk-centos.sh
@@ -1,33 +1,40 @@
 #! /bin/bash
 
 set -e -x
 
-mkdir artifacts
-cd build
+mkdir -p artifacts
+pushd build
 
 rm -rf root && mkdir root && cd root
 
 # change these variables when updating java version
 mirror_url_base="http://mirror.centos.org/centos/6.7/updates/x86_64/Packages"
 openjdk=java-1.7.0-openjdk-1.7.0.85-2.6.1.3.el6_7.x86_64.rpm
-openjdk_devel=java-1.7.0-openjdk-1.7.0.85-2.6.1.3.el6_7.x86_64.rpm
+openjdk_devel=java-1.7.0-openjdk-devel-1.7.0.85-2.6.1.3.el6_7.x86_64.rpm
 jvm_openjdk_dir=java-1.7.0-openjdk-1.7.0.85.x86_64
 
 # grab the rpm and unpack it
-wget ${mirror_url_base}/${openjdk}
-wget ${mirror_url_base}/${openjdk_devel}
+wget --timestamping ${mirror_url_base}/${openjdk}
+wget --timestamping ${mirror_url_base}/${openjdk_devel}
 rpm2cpio $openjdk | cpio -ivd
 rpm2cpio $openjdk_devel | cpio -ivd
 
 cd usr/lib/jvm
 mv $jvm_openjdk_dir java_home
 
+# cacerts is a relative symlink, which doesn't work when we repackage.  Make it absolute.
+rm java_home/jre/lib/security/cacerts
+ln -s /etc/pki/java/cacerts java_home/jre/lib/security/cacerts
+
 # document version this is based on
 echo "Built from ${mirror_url_Base}
     ${openjdk}
     ${openjdk_devel}
 
 Run through rpm2cpio | cpio, and /usr/lib/jvm/${jvm_openjdk_dir} renamed to 'java_home'." > java_home/VERSION
 
 # tarball the unpacked rpm and put it in the taskcluster upload artifacts dir
-tar -Jvcf ~/artifacts/java_home-${jvm_openjdk_dir}.tar.xz java_home
+tar -Jvcf java_home-${jvm_openjdk_dir}.tar.xz java_home
+popd
+
+mv build/root/usr/lib/jvm/java_home-${jvm_openjdk_dir}.tar.xz artifacts
deleted file mode 100644
--- a/testing/taskcluster/tasks/android-gradle-build-dependencies.yml
+++ /dev/null
@@ -1,76 +0,0 @@
-# A task to fetch Android Gradle dependencies from jcentral and package them
-# for consumption by tooltool users.  Normally invoked manually.
----
-taskId: {{build_slugid}}
-
-task:
-  created: '{{now}}'
-  deadline: '{{#from_now}}24 hours{{/from_now}}'
-  metadata:
-    name: '[TC] - Android Gradle build dependencies'
-    description: 'Fetch and package Android Gradle build dependencies'
-    owner: nalexander@mozilla.com
-    source: http://todo.com/soon
-
-  tags:
-    createdForUser: {{owner}}
-
-  workerType: b2gtest
-  provisionerId: aws-provisioner-v1
-  schedulerId: task-graph-scheduler
-
-  routes:
-    - 'index.gecko.v1.{{project}}.revision.linux.{{head_rev}}.{{build_name}}.{{build_type}}'
-    - 'index.gecko.v1.{{project}}.latest.linux.{{build_name}}.{{build_type}}'
-
-  scopes:
-    - 'docker-worker:cache:level-{{level}}-{{project}}-tc-vcs'
-
-  payload:
-    image:
-      type: 'task-image'
-      path: 'public/image.tar'
-      taskId: '{{#task_id_for_image}}android-gradle-build{{/task_id_for_image}}'
-
-    command:
-      - bash
-      - /build/build.sh
-
-    maxRunTime: 1800
-
-    cache:
-      level-{{level}}-{{project}}-tc-vcs: '/home/worker/.tc-vcs'
-
-    artifacts:
-      'public/build':
-        type: directory
-        path: '/artifacts/'
-        expires: '{{#from_now}}1 year{{/from_now}}'
-
-    env:
-      # Common environment variables for checking out gecko
-      GECKO_BASE_REPOSITORY: '{{base_repository}}'
-      GECKO_HEAD_REPOSITORY: '{{head_repository}}'
-      GECKO_HEAD_REV: '{{head_rev}}'
-      GECKO_HEAD_REF: '{{head_ref}}'
-      TOOLTOOL_REPO: 'https://git.mozilla.org/build/tooltool.git'
-      TOOLTOOL_REV: 'master'
-      MOZ_BUILD_DATE: '{{pushdate}}'
-
-  extra:
-    build_product: '{{build_product}}'
-    build_name: '{{build_name}}'
-    build_type: '{{build_type}}'
-    index:
-      rank: {{pushlog_id}}
-    locations:
-        build: null
-        tests: null
-    treeherder:
-        machine:
-            platform: lint
-        groupSymbol: tc
-        symbol: AG
-    treeherderEnv:
-        - production
-        - staging
--- a/testing/taskcluster/tasks/branches/base_jobs.yml
+++ b/testing/taskcluster/tasks/branches/base_jobs.yml
@@ -346,8 +346,14 @@ tasks:
         - '**/*.xml'
         # Run when eslint policies change.
         - '**/.eslintignore'
         - '**/*eslintrc*'
         # The plugin implementing custom checks.
         - 'testing/eslint-plugin-mozilla/**'
         # Other misc lint related files.
         - 'tools/lint/**'
+  android-api-15-gradle-dependencies:
+    task: tasks/builds/android_api_15_gradle_dependencies.yml
+    root: true
+    when:
+      file_patterns:
+        - 'testing/docker/android-gradle-build/**'
new file mode 100644
--- /dev/null
+++ b/testing/taskcluster/tasks/builds/android_api_15_gradle_dependencies.yml
@@ -0,0 +1,80 @@
+# A build-like task to fetch Android Gradle dependencies from jcentral and
+# package them for consumption by tooltool users.  Normally invoked manually.
+
+$inherits:
+  from: 'tasks/builds/mobile_base.yml'
+  variables:
+    build_name: 'android-api-15-gradle-dependencies'
+    build_type: 'opt'
+task:
+  metadata:
+      name: '[TC] Android armv7 API 15+ gradle dependencies'
+      description: 'Android armv7 API 15+ gradle dependencies'
+
+  workerType: android-api-15
+
+  routes:
+    - 'index.buildbot.branches.{{project}}.android-api-15-gradle-dependencies'
+    - 'index.buildbot.revisions.{{head_rev}}.{{project}}.android-api-15-gradle-dependencies'
+
+  scopes:
+    - 'docker-worker:cache:level-{{level}}-{{project}}-build-android-api-15-gradle-dependencies-workspace'
+    - 'docker-worker:cache:tooltool-cache'
+    - 'docker-worker:relengapi-proxy:tooltool.download.internal'
+    - 'docker-worker:relengapi-proxy:tooltool.download.public'
+
+  payload:
+    cache:
+      level-{{level}}-{{project}}-build-android-api-15-gradle-dependencies-workspace: '/home/worker/workspace'
+      tooltool-cache: '/home/worker/tooltool-cache'
+
+    features:
+      relengAPIProxy: true
+
+    env:
+      # inputs to mozharness
+      MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
+      # TODO: make these additional configuration files go away
+      MOZHARNESS_CONFIG: >
+          builds/releng_base_android_64_builds.py
+          disable_signing.py
+          platform_supports_post_upload_to_latest.py
+      MH_CUSTOM_BUILD_VARIANT_CFG: api-15-gradle-dependencies
+      MH_BRANCH: {{project}}
+      MH_BUILD_POOL: taskcluster
+      GRADLE_USER_HOME: '/home/worker/workspace/build/src/dotgradle-online'
+
+    maxRunTime: 36000
+
+    image:
+      type: 'task-image'
+      path: 'public/image.tar'
+      taskId: '{{#task_id_for_image}}android-gradle-build{{/task_id_for_image}}'
+
+    command:
+      - /bin/bash
+      - -c
+      - >
+        /home/worker/bin/before.sh &&
+        /home/worker/bin/build.sh &&
+        /home/worker/bin/after.sh &&
+        true
+
+  extra:
+    treeherderEnv:
+      - production
+      - staging
+    treeherder:
+      machine:
+        # see https://github.com/mozilla/treeherder/blob/master/ui/js/values.js
+        platform: android-4-0-armv7-api15
+      groupSymbol: tc
+      groupName: Submitted by taskcluster
+      symbol: Deps
+      tier: 2
+    # Rather then enforcing particular conventions we require that all build
+    # tasks provide the "build" extra field to specify where the build and tests
+    # files are located.
+    locations:
+      build: 'public/build/target.linux-x86_64.tar.bz2'
+      tests: 'public/build/target.tests.zip'