Backed out changeset 1b269fb8a704 (bug 1302590) for Gecko Decision Task bustage
authorIris Hsiao <ihsiao@mozilla.com>
Tue, 20 Sep 2016 10:59:19 +0800
changeset 314469 d82fb090d6908f028207b663bb01a16460ecdce7
parent 314468 cee3a96d763b538ae6d985c6ee0e03bf1c58949d
child 314470 8192ae07b2ba5c67d170d9d5a4257aabd52dae2d
push id20574
push usercbook@mozilla.com
push dateTue, 20 Sep 2016 10:05:16 +0000
treeherderfx-team@14705f779a46 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1302590
milestone52.0a1
backs out1b269fb8a704a1a1fd2fe67543b9a083ce155563
Backed out changeset 1b269fb8a704 (bug 1302590) for Gecko Decision Task bustage
taskcluster/ci/build-signing/android-signing.yml
taskcluster/ci/build-signing/kind.yml
taskcluster/ci/build/android.yml
taskcluster/ci/nightly-fennec/android-api-15-nightly-build.yml
taskcluster/ci/nightly-fennec/build.yml
taskcluster/ci/nightly-fennec/docker_build.yml
taskcluster/ci/nightly-fennec/kind.yml
taskcluster/ci/nightly-fennec/mobile_base.yml
taskcluster/ci/nightly-fennec/routes.json
taskcluster/ci/signing/kind.yml
taskcluster/ci/signing/signing.yml
taskcluster/docs/attributes.rst
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/task/nightly_fennec.py
taskcluster/taskgraph/task/signing.py
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@@ -67,44 +67,16 @@ android-api-15/opt:
         config:
             - builds/releng_base_android_64_builds.py
             - disable_signing.py
             - platform_supports_post_upload_to_latest.py
         script: "mozharness/scripts/fx_desktop_build.py"
         custom-build-variant-cfg: api-15
         tooltool-downloads: internal
 
-android-api-15-nightly/opt:
-    description: "Android 4.0 API15+ Nightly"
-    attributes:
-        nightly: true
-    index:
-        product: mobile
-        job-name: android-api-15-nightly-opt
-    treeherder:
-        platform: android-4-0-armv7-api15/opt
-        symbol: tc(N)
-        tier: 2
-    worker-type: aws-provisioner-v1/android-api-15
-    worker:
-        implementation: docker-worker
-        max-run-time: 7200
-    run:
-        using: mozharness
-        actions: [get-secrets build multi-l10n update]
-        config:
-            - builds/releng_base_android_64_builds.py
-            - disable_signing.py
-            - platform_supports_post_upload_to_latest.py
-            - taskcluster_nightly.py
-        script: "mozharness/scripts/fx_desktop_build.py"
-        custom-build-variant-cfg: api-15
-        tooltool-downloads: internal
-    run-on-projects: []
-
 android-api-15-gradle/opt:
     description: "Android 4.0 API15+ (Gradle) Opt"
     index:
         product: mobile
         job-name: android-api-15-gradle-opt
     treeherder:
         platform: android-4-0-armv7-api15/opt
         symbol: tc(Bg)
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/nightly-fennec/android-api-15-nightly-build.yml
@@ -0,0 +1,66 @@
+$inherits:
+  from: 'mobile_base.yml'
+  variables:
+    build_name: 'android'
+    build_type: 'opt'
+task:
+  metadata:
+      name: '[TC] Android armv7 API 15+'
+      description: 'Android armv7 API 15+'
+
+  workerType: android-api-15
+
+  routes:
+    - 'index.buildbot.branches.{{project}}.android-api-15'
+    - 'index.buildbot.revisions.{{head_rev}}.{{project}}.android-api-15'
+
+  scopes:
+    - 'docker-worker:cache:level-{{level}}-{{project}}-build-android-api-15-workspace'
+    - 'docker-worker:cache:tooltool-cache'
+    - 'docker-worker:relengapi-proxy:tooltool.download.internal'
+    - 'docker-worker:relengapi-proxy:tooltool.download.public'
+
+  payload:
+    cache:
+      level-{{level}}-{{project}}-build-android-api-15-workspace: '/home/worker/workspace'
+      tooltool-cache: '/home/worker/tooltool-cache'
+
+    features:
+      relengAPIProxy: true
+
+    env:
+      # inputs to mozharness
+      MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
+      # TODO: make these additional configuration files go away
+      MOZHARNESS_CONFIG: >
+          builds/releng_base_android_64_builds.py
+          disable_signing.py
+          platform_supports_post_upload_to_latest.py
+          taskcluster_nightly.py
+      MOZHARNESS_ACTIONS: "get-secrets build multi-l10n update"
+      MH_CUSTOM_BUILD_VARIANT_CFG: api-15
+      MH_BRANCH: {{project}}
+      MH_BUILD_POOL: taskcluster
+      TOOLTOOL_CACHE: '/home/worker/tooltool-cache'
+
+    command: ["/bin/bash", "bin/build.sh"]
+
+  extra:
+    treeherderEnv:
+      - production
+      - staging
+    treeherder:
+      machine:
+        # see https://github.com/mozilla/treeherder/blob/master/ui/js/values.js
+        platform: android-4-0-armv7-api15
+      groupSymbol: tc
+      groupName: Submitted by taskcluster
+      symbol: B
+      tier: 2
+    # Rather then enforcing particular conventions we require that all build
+    # tasks provide the "build" extra field to specify where the build and tests
+    # files are located.
+    locations:
+      build: 'public/build/target.apk'
+      mozharness: 'public/build/mozharness.zip'
+      test_packages: 'public/build/target.test_packages.json'
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/nightly-fennec/build.yml
@@ -0,0 +1,46 @@
+# This is the "base" task which contains the common values all builds must
+# provide.
+---
+taskId: {{build_slugid}}
+
+task:
+  created:
+    relative-datestamp: "0 seconds"
+  deadline:
+    relative-datestamp: "24 hours"
+  metadata:
+    source: '{{source}}'
+    owner: mozilla-taskcluster-maintenance@mozilla.com
+
+  tags:
+    createdForUser: {{owner}}
+
+  provisionerId: aws-provisioner-v1
+  schedulerId: task-graph-scheduler
+
+  payload:
+    # Two hours is long but covers edge cases (and matches bb based infra)
+    maxRunTime: 7200
+
+    env:
+      # Common environment variables for checking out gecko
+      GECKO_BASE_REPOSITORY: '{{base_repository}}'
+      GECKO_HEAD_REPOSITORY: '{{head_repository}}'
+      GECKO_HEAD_REV: '{{head_rev}}'
+      GECKO_HEAD_REF: '{{head_ref}}'
+      TOOLTOOL_REPO: 'https://github.com/mozilla/build-tooltool'
+      TOOLTOOL_REV: 'master'
+      MOZ_BUILD_DATE: '{{pushdate}}'
+      MOZ_SCM_LEVEL: '{{level}}'
+
+  extra:
+    build_product: '{{build_product}}'
+    build_name: '{{build_name}}'
+    build_type: '{{build_type}}'
+    index:
+      rank: {{rank}}
+    treeherder:
+      jobKind: build
+      groupSymbol: tc
+      groupName: Submitted by taskcluster
+      symbol: B
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/nightly-fennec/docker_build.yml
@@ -0,0 +1,27 @@
+$inherits:
+  from: 'build.yml'
+
+
+task:
+  workerType: b2gbuild
+
+  routes:
+    - 'index.gecko.v1.{{project}}.revision.linux.{{head_rev}}.{{build_name}}.{{build_type}}'
+    - 'index.gecko.v1.{{project}}.latest.linux.{{build_name}}.{{build_type}}'
+
+  scopes:
+    # docker build tasks use tc-vcs so include the scope.
+    - 'docker-worker:cache:level-{{level}}-{{project}}-tc-vcs'
+
+  payload:
+
+    cache:
+      level-{{level}}-{{project}}-tc-vcs: '/home/worker/.tc-vcs'
+
+    # All docker builds share a common artifact directory for ease of uploading.
+    artifacts:
+      'public/build':
+        type: directory
+        path: '/home/worker/artifacts/'
+        expires:
+          relative-datestamp: '1 year'
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/nightly-fennec/kind.yml
@@ -0,0 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: 'taskgraph.task.nightly_fennec:NightlyFennecTask'
+nightly_fennec_path: '.'
+
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/nightly-fennec/mobile_base.yml
@@ -0,0 +1,13 @@
+$inherits:
+  from: 'docker_build.yml'
+  variables:
+    build_product: 'mobile'
+docker-image: desktop-build
+task:
+  payload:
+    image:
+      type: 'task-image'
+      path: 'public/image.tar'
+      taskId:
+        task-reference: "<docker-image>"
+
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/nightly-fennec/routes.json
@@ -0,0 +1,18 @@
+{
+    "routes": [
+        "{index}.gecko.v2.{project}.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
+        "{index}.gecko.v2.{project}.pushdate.{year}.{month}.{day}.{pushdate}.{build_product}.{build_name}-{build_type}",
+        "{index}.gecko.v2.{project}.latest.{build_product}.{build_name}-{build_type}"
+    ],
+    "nightly": [
+        "{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
+        "{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.latest.{build_product}.{build_name}-{build_type}",
+        "{index}.gecko.v2.{project}.nightly.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
+        "{index}.gecko.v2.{project}.nightly.latest.{build_product}.{build_name}-{build_type}"
+    ],
+    "l10n": [
+        "{index}.gecko.v2.{project}.revision.{head_rev}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
+        "{index}.gecko.v2.{project}.pushdate.{year}.{month}.{day}.{pushdate}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
+        "{index}.gecko.v2.{project}.latest.{build_product}-l10n.{build_name}-{build_type}.{locale}"
+    ]
+}
rename from taskcluster/ci/build-signing/kind.yml
rename to taskcluster/ci/signing/kind.yml
--- a/taskcluster/ci/build-signing/kind.yml
+++ b/taskcluster/ci/signing/kind.yml
@@ -1,11 +1,8 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 implementation: 'taskgraph.task.signing:SigningTask'
-
-jobs-from:
-  - android-signing.yml
-
+signing_path: '.'
 kind-dependencies:
-  - build
+  - nightly-fennec
rename from taskcluster/ci/build-signing/android-signing.yml
rename to taskcluster/ci/signing/signing.yml
--- a/taskcluster/ci/build-signing/android-signing.yml
+++ b/taskcluster/ci/signing/signing.yml
@@ -1,26 +1,18 @@
-signing-nightly-fennec:
-  task:
-    provisionerId: "scriptworker-prov-v1"
-    workerType: "signing-linux-v1"
-    scopes:
-      - "project:releng:signing:cert:dep-signing"
-      - "project:releng:signing:format:jar"
-    created:
-      relative-datestamp: "0 seconds"
-    deadline:
-      relative-datestamp: "24 hours"
-    payload:
-      unsignedArtifacts: []
-      maxRunTime: 600
-    metadata:
-      name: "Signing Scriptworker Task"
-      description: "Sign Android Build Tasks"
-      owner: "jlund@mozilla.com"
-      source: "https://tools.taskcluster.net/task-creator/"
-  attributes:
-    nightly: true
-  unsigned-task:
-    label: "build-android-api-15-nightly/opt"
-    artifacts:
-      - "public/build/target.apk"
-      - "public/build/en-US/target.apk"
+task:
+  provisionerId: "scriptworker-prov-v1"
+  workerType: "signing-linux-v1"
+  scopes:
+    - "project:releng:signing:cert:dep-signing"
+    - "project:releng:signing:format:jar"
+  created:
+    relative-datestamp: "0 seconds"
+  deadline:
+    relative-datestamp: "24 hours"
+  payload:
+    unsignedArtifacts: []
+    maxRunTime: 600
+  metadata:
+    name: "Signing Scriptworker Task"
+    description: "Testing the signing scriptworker"
+    owner: "amiyaguchi@mozilla.com"
+    source: "https://tools.taskcluster.net/task-creator/"
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -111,14 +111,8 @@ e10s
 
 For test suites which distinguish whether they run with or without e10s, this
 boolean value identifies this particular run.
 
 image_name
 ==========
 
 For the ``docker_image`` kind, this attribute contains the docker image name.
-
-nightly
-=======
-
-Signals whether the task is part of a nightly graph. Useful when filtering
-out nightly tasks from full task set at target stage.
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -101,11 +101,10 @@ def target_tasks_ash(full_task_graph, pa
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('nightly_fennec')
 def target_tasks_nightly(full_task_graph, parameters):
     """Select the set of tasks required for a nightly build of fennec. The
     nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
-    def filter(task):
-        return task.attributes.get('nightly', False)
-    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
+    return [t.label for t in full_task_graph.tasks.itervalues()
+            if t.attributes.get('kind') in ['nightly-fennec', 'signing']]
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/task/nightly_fennec.py
@@ -0,0 +1,201 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Temporary placeholder for nightly builds
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+import json
+import os
+import time
+from collections import namedtuple
+
+from . import base
+from slugid import nice as slugid
+from taskgraph.util.templates import Templates
+from taskgraph.util.docker import docker_image
+
+
+logger = logging.getLogger(__name__)
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
+INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
+TASKID_PLACEHOLDER = 'TaskLabel=={}'
+
+
+def mklabel():
+    return TASKID_PLACEHOLDER.format(slugid())
+
+
+def gaia_info():
+    '''Fetch details from in tree gaia.json (which links this version of
+    gecko->gaia) and construct the usual base/head/ref/rev pairing...'''
+    gaia = json.load(open(os.path.join(GECKO, 'b2g', 'config', 'gaia.json')))
+
+    if gaia['git'] is None or \
+       gaia['git']['remote'] == '' or \
+       gaia['git']['git_revision'] == '' or \
+       gaia['git']['branch'] == '':
+
+        # Just use the hg params...
+        return {
+            'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
+            'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
+            'gaia_ref': gaia['revision'],
+            'gaia_rev': gaia['revision']
+        }
+
+    else:
+        # Use git
+        return {
+            'gaia_base_repository': gaia['git']['remote'],
+            'gaia_head_repository': gaia['git']['remote'],
+            'gaia_rev': gaia['git']['git_revision'],
+            'gaia_ref': gaia['git']['branch'],
+        }
+
+
+def decorate_task_json_routes(task, json_routes, parameters):
+    """Decorate the given task with routes.json routes.
+
+    :param dict task: task definition.
+    :param json_routes: the list of routes to use from routes.json
+    :param parameters: dictionary of parameters to use in route templates
+    """
+    routes = task.get('routes', [])
+    for route in json_routes:
+        routes.append(route.format(**parameters))
+
+    task['routes'] = routes
+
+
+def query_vcs_info(repository, revision):
+    """Query the pushdate and pushid of a repository/revision.
+
+    This is intended to be used on hg.mozilla.org/mozilla-central and
+    similar. It may or may not work for other hg repositories.
+    """
+    if not repository or not revision:
+        logger.warning('cannot query vcs info because vcs info not provided')
+        return None
+
+    VCSInfo = namedtuple('VCSInfo', ['pushid', 'pushdate', 'changesets'])
+
+    try:
+        import requests
+        url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'),
+                                                  revision)
+        logger.debug("Querying version control for metadata: %s", url)
+        contents = requests.get(url).json()
+
+        changesets = []
+        for c in contents['changesets']:
+            changesets.append({k: c[k] for k in ('desc', 'files', 'node')})
+
+        pushid = contents['changesets'][-1]['pushid']
+        pushdate = contents['changesets'][-1]['pushdate'][0]
+
+        return VCSInfo(pushid, pushdate, changesets)
+
+    except Exception:
+        logger.exception("Error querying VCS info for '%s' revision '%s'",
+                         repository, revision)
+        return None
+
+
+class NightlyFennecTask(base.Task):
+
+    def __init__(self, *args, **kwargs):
+        try:
+            self.task_dict = kwargs.pop('task_dict')
+        except KeyError:
+            pass
+        super(NightlyFennecTask, self).__init__(*args, **kwargs)
+
+    @classmethod
+    def load_tasks(cls, kind, path, config, params, loaded_tasks):
+        root = os.path.abspath(os.path.join(path, config[
+            'nightly_fennec_path']))
+
+        project = params['project']
+
+        # Set up the parameters, including the time of the build
+        push_epoch = int(time.time())
+        vcs_info = query_vcs_info(params['head_repository'], params['head_rev'])
+        changed_files = set()
+        if vcs_info:
+            push_epoch = vcs_info.pushdate
+
+            logger.debug(
+                '{} commits influencing task scheduling:'.format(len(vcs_info.changesets)))
+            for c in vcs_info.changesets:
+                logger.debug("{cset} {desc}".format(
+                    cset=c['node'][0:12],
+                    desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
+                changed_files |= set(c['files'])
+
+        pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(push_epoch))
+
+        # Template parameters used when expanding the graph
+        parameters = dict(gaia_info().items() + {
+            'index': 'index',
+            'project': project,
+            'pushlog_id': params.get('pushlog_id', 0),
+            'base_repository': params['base_repository'] or params['head_repository'],
+            'docker_image': docker_image,
+            'head_repository': params['head_repository'],
+            'head_ref': params['head_ref'] or params['head_rev'],
+            'head_rev': params['head_rev'],
+            'pushdate': pushdate,
+            'pushtime': pushdate[8:],
+            'year': pushdate[0:4],
+            'month': pushdate[4:6],
+            'day': pushdate[6:8],
+            'rank': push_epoch,
+            'owner': params['owner'],
+            'level': params['level'],
+            'build_slugid': mklabel(),
+            'source': '{repo}file/{rev}/taskcluster/ci/nightly-fennec'.format(
+                repo=params['head_repository'], rev=params['head_rev']),
+            'build_name': 'android',
+            'build_type': 'opt',
+            'build_product': 'mobile'
+        }.items())
+
+        routes_file = os.path.join(root, 'routes.json')
+        with open(routes_file) as f:
+            contents = json.load(f)
+            json_routes = contents['routes']
+
+        tasks = []
+        templates = Templates(root)
+
+        task = templates.load('android-api-15-nightly-build.yml', parameters)
+        decorate_task_json_routes(task['task'], json_routes, parameters)
+
+        attributes = {'kind': 'nightly-fennec'}
+        tasks.append(cls(kind, 'build-nightly-fennec',
+                     task=task['task'], attributes=attributes,
+                         task_dict=task))
+
+        # Convert to a dictionary of tasks.  The process above has invented a
+        # taskId for each task, and we use those as the *labels* for the tasks;
+        # taskgraph will later assign them new taskIds.
+        return tasks
+
+    def get_dependencies(self, taskgraph):
+        deps = [(label, label) for label in self.task_dict.get('requires', [])]
+
+        # add a dependency on an image task, if needed
+        if 'docker-image' in self.task_dict:
+            deps.append(('build-docker-image-{docker-image}'.format(**self.task_dict),
+                         'docker-image'))
+
+        return deps
+
+    def optimize(self, params):
+        return False, None
--- a/taskcluster/taskgraph/task/signing.py
+++ b/taskcluster/taskgraph/task/signing.py
@@ -14,39 +14,42 @@ from taskgraph.util.templates import Tem
 logger = logging.getLogger(__name__)
 GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
 INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
 
 
 class SigningTask(base.Task):
 
-    def __init__(self, kind, name, task, attributes):
-        self.unsigned_artifact_label = task['unsigned-task']['label']
-        super(SigningTask, self).__init__(kind, name, task=task['task'],
-                                          attributes=attributes)
+    def __init__(self, *args, **kwargs):
+        super(SigningTask, self).__init__(*args, **kwargs)
 
     @classmethod
     def load_tasks(cls, kind, path, config, params, loaded_tasks):
-        root = os.path.abspath(path)
+        root = os.path.abspath(os.path.join(path, config['signing_path']))
+
+        # get each nightly-fennec and add its name to this task
+        fennec_tasks = [t for t in loaded_tasks if t.attributes.get('kind') == 'nightly-fennec']
 
         tasks = []
-        for filename in config.get('jobs-from', []):
+        for fennec_task in fennec_tasks:
             templates = Templates(root)
-            jobs = templates.load(filename, {})
+            task = templates.load('signing.yml', {})
 
-            for name, job in jobs.iteritems():
-                for artifact in job['unsigned-task']['artifacts']:
-                    url = ARTIFACT_URL.format('<{}>'.format('unsigned-artifact'), artifact)
-                    job['task']['payload']['unsignedArtifacts'].append({
-                        'task-reference': url
-                    })
-                attributes = job.setdefault('attributes', {})
-                attributes.update({'kind': 'signing'})
-                tasks.append(cls(kind, name, job, attributes=attributes))
+            artifacts = ['public/build/target.apk',
+                         'public/build/en-US/target.apk']
+            for artifact in artifacts:
+                url = ARTIFACT_URL.format('<build-nightly-fennec>', artifact)
+                task['task']['payload']['unsignedArtifacts'].append({
+                    'task-reference': url
+                })
+
+            attributes = {'kind': 'signing'}
+            tasks.append(cls(kind, 'signing-nightly-fennec', task=task['task'],
+                             attributes=attributes))
 
         return tasks
 
     def get_dependencies(self, taskgraph):
-        return [(self.unsigned_artifact_label, 'unsigned-artifact')]
+        return [('build-nightly-fennec', 'build-nightly-fennec')]
 
     def optimize(self, params):
         return False, None