Bug 1557269 - part 1: Re-expose all maven artifacts and beetmove them r=mtabara
authorJohan Lorenzo <jlorenzo@mozilla.com>
Fri, 05 Jul 2019 15:19:14 +0000
changeset 541165 178a2ab709eebee94aa64621823f6a65249c3d30
parent 541164 af32b84056af45f9f08c579432496ebc8e1ae295
child 541166 34e52d51840a433e51f1381b625bd233494bd7bb
push id11533
push userarchaeopteryx@coole-files.de
push dateMon, 08 Jul 2019 18:18:03 +0000
treeherdermozilla-beta@f4452e031aed [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmtabara
bugs1557269
milestone69.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1557269 - part 1: Re-expose all maven artifacts and beetmove them r=mtabara Differential Revision: https://phabricator.services.mozilla.com/D34865
mobile/android/mach_commands.py
taskcluster/ci/beetmover-geckoview/kind.yml
taskcluster/ci/build/android.yml
taskcluster/taskgraph/manifests/fennec_geckoview.yml
taskcluster/taskgraph/transforms/beetmover_geckoview.py
taskcluster/taskgraph/util/scriptworker.py
--- a/mobile/android/mach_commands.py
+++ b/mobile/android/mach_commands.py
@@ -414,16 +414,17 @@ class MachCommands(MachCommandBase):
     def android_archive_geckoview(self, args):
         ret = self.gradle(
             self.substs['GRADLE_ANDROID_ARCHIVE_GECKOVIEW_TASKS'] + args,
             verbose=True)
 
         if ret != 0:
             return ret
 
+        # TODO Bug 1563711 - Remove target.maven.zip
         # The zip archive is passed along in CI to ship geckoview onto a maven repo
         _craft_maven_zip_archive(self.topobjdir)
 
         return 0
 
     @SubCommand('android', 'build-geckoview_example',
                 """Build geckoview_example """)
     @CommandArgument('args', nargs=argparse.REMAINDER)
--- a/taskcluster/ci/beetmover-geckoview/kind.yml
+++ b/taskcluster/ci/beetmover-geckoview/kind.yml
@@ -35,16 +35,17 @@ not-for-build-platforms:
     - win64-devedition-nightly/opt
     - win64-aarch64-devedition-nightly/opt
     - linux64-asan-reporter-nightly/opt
     - win64-asan-reporter-nightly/opt
 
 job-template:
     attributes:
         artifact_map: taskcluster/taskgraph/manifests/fennec_geckoview.yml
+        artifact_prefix: public/build/maven
     run-on-projects: ['mozilla-central', 'mozilla-release']
     run-on-hg-branches:
         by-project:
             mozilla-release:
                 - '^GECKOVIEW_\d+_RELBRANCH$'
             default:
                 - '.*'
     shipping-phase:
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@@ -6,19 +6,23 @@ job-defaults:
     index:
         product: mobile
     worker:
         docker-image: {in-tree: android-build}
         max-run-time: 7200
         env:
             GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-offline"
         artifacts:
+            # TODO Bug 1563711 - Remove target.maven.zip
             - name: public/build/target.maven.zip
               path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/target.maven.zip
               type: file
+            - name: public/build/maven
+              path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/maven/
+              type: directory
             - name: public/build/geckoview-androidTest.apk
               path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk
               type: file
             - name: public/build/geckoview_example.apk
               path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk
               type: file
             - name: public/build
               path: /builds/worker/artifacts/
@@ -531,24 +535,16 @@ android-geckoview-fat-aar/opt:
             PERFHERDER_EXTRA_OPTIONS: android-geckoview-fat-aar-opt
             MOZ_ANDROID_FAT_AAR_ARCHITECTURES: 'armeabi-v7a,arm64-v8a,x86,x86_64'
             USE_ARTIFACT: '1'
             MOZ_ARTIFACT_TASK: {task-reference: '<android-api-16-opt>'}
             MOZ_ARTIFACT_TASK_ANDROID_API_16_OPT: {task-reference: '<android-api-16-opt>'}
             MOZ_ARTIFACT_TASK_ANDROID_AARCH64_OPT: {task-reference: '<android-aarch64-opt>'}
             MOZ_ARTIFACT_TASK_ANDROID_X86_OPT: {task-reference: '<android-x86-opt>'}
             MOZ_ARTIFACT_TASK_ANDROID_X86_64_OPT: {task-reference: '<android-x86_64-opt>'}
-        artifacts:
-            - name: public/android/maven
-              # TODO Bug 1433198. Remove the following entry once target.maven.zip is uploaded to a maven repository
-              path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/maven/
-              type: directory
-            - name: public/build/target.maven.zip
-              path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/target.maven.zip
-              type: file
     run:
         actions: [get-secrets, build]
         config: ["builds/releng_base_android_64_builds.py"]
         custom-build-variant-cfg: api-16
     toolchains:
         - linux64-sccache
 
 android-geckoview-fat-aar-nightly/opt:
--- a/taskcluster/taskgraph/manifests/fennec_geckoview.yml
+++ b/taskcluster/taskgraph/manifests/fennec_geckoview.yml
@@ -3,36 +3,30 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 s3_bucket_paths:
     - maven2
 default_locales:  # Ignored for geckoview
     - en-US
 tasktype_map:  # Map task reference to task type.
     build: build
-base_artifact_prefix: ''
+base_artifact_prefix: public/build/maven/
 
 # A default entry, which the mappings below extend and override.
 # Final 'destinations' will be the product of:
 # s3_bucket_paths + destinations + locale_prefix + pretty_name
 default: &default
     from:
         - build
     locale_prefix: ''
     source_path_modifier: org/mozilla/geckoview/${artifact_id}/${major_version}.${minor_version}.${build_date}
     description: "TO_BE_OVERRIDDEN"
     destinations:  # locale_prefix is appended
         - org/mozilla/geckoview/${artifact_id}/${major_version}.${minor_version}.${build_date}
 
-# Configuration for individual files. Extends 'default', above.
-upstream_mapping:
-    target.maven.zip:
-        from:
-            - build
-
 mapping:
     ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1:
         <<: *default
         pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1
         checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1
     ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.md5:
         <<: *default
         pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.md5
--- a/taskcluster/taskgraph/transforms/beetmover_geckoview.py
+++ b/taskcluster/taskgraph/transforms/beetmover_geckoview.py
@@ -11,17 +11,17 @@ import re
 
 from taskgraph.loader.single_dep import schema
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.beetmover import \
     craft_release_properties as beetmover_craft_release_properties
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
 from taskgraph.util.scriptworker import (generate_beetmover_artifact_map,
-                                         generate_beetmover_compressed_upstream_artifacts,
+                                         generate_beetmover_upstream_artifacts,
                                          get_worker_type_for_scope)
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Required, Optional
 
 
 _ARTIFACT_ID_PER_PLATFORM = {
     'android-aarch64': 'geckoview{update_channel}-arm64-v8a',
     'android-api-16': 'geckoview{update_channel}-armeabi-v7a',
@@ -117,57 +117,52 @@ def make_task_description(config, jobs):
             'run-on-projects': job['run-on-projects'],
             'treeherder': treeherder,
             'shipping-phase': job['shipping-phase'],
         }
 
         yield task
 
 
-def generate_upstream_artifacts(build_task_ref):
-    return [{
-        'taskId': {'task-reference': build_task_ref},
-        'taskType': 'build',
-        'paths': ['public/build/target.maven.zip'],
-        'zipExtract': True,
-    }]
-
-
 @transforms.add
 def make_task_worker(config, jobs):
     for job in jobs:
         valid_beetmover_job = len(job['dependencies']) == 1 and 'build' in job['dependencies']
         if not valid_beetmover_job:
             raise NotImplementedError(
                 'Beetmover-geckoview must have a single dependency. Got: {}'.format(
                     job['dependencies']
                 )
             )
 
         worker = {
             'implementation': 'beetmover-maven',
             'release-properties': craft_release_properties(config, job),
         }
 
-        upstream_artifacts = generate_beetmover_compressed_upstream_artifacts(job)
-
-        worker['upstream-artifacts'] = upstream_artifacts
-
         version_groups = re.match(r'(\d+).(\d+).*', config.params['version'])
         if version_groups:
             major_version, minor_version = version_groups.groups()
 
         template_vars = {
             'artifact_id': worker['release-properties']['artifact-id'],
             'build_date': config.params['moz_build_date'],
             'major_version': major_version,
             'minor_version': minor_version,
         }
         worker['artifact-map'] = generate_beetmover_artifact_map(
-            config, job, **template_vars)
+            config, job, **template_vars
+        )
+        upstream_artifacts = generate_beetmover_upstream_artifacts(
+            config, job, platform='', **template_vars
+        )
+        worker['upstream-artifacts'] = [{
+            key: value for key, value in upstream_artifact.items()
+            if key != 'locale'
+        } for upstream_artifact in upstream_artifacts]
 
         job["worker"] = worker
 
         yield job
 
 
 def craft_release_properties(config, job):
     props = beetmover_craft_release_properties(config, job)
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -404,17 +404,19 @@ def get_worker_type_for_scope(config, sc
                 for scopes in config.graph_config['scriptworker']['worker-types'].values()
                 for scope in scopes
             ),
         )
     )
 
 
 # generate_beetmover_upstream_artifacts {{{1
-def generate_beetmover_upstream_artifacts(config, job, platform, locale=None, dependencies=None):
+def generate_beetmover_upstream_artifacts(
+    config, job, platform, locale=None, dependencies=None, **kwargs
+):
     """Generate the upstream artifacts for beetmover, using the artifact map.
 
     Currently only applies to beetmover tasks.
 
     Args:
         job (dict): The current job being generated
         dependencies (list): A list of the job's dependency labels.
         platform (str): The current build platform
@@ -460,20 +462,23 @@ def generate_beetmover_upstream_artifact
                 platform in map_config['mapping'][filename]['not_for_platforms']):
                 continue
             if 'partials_only' in map_config['mapping'][filename]:
                 continue
             # The next time we look at this file it might be a different locale.
             file_config = deepcopy(map_config['mapping'][filename])
             resolve_keyed_by(file_config, "source_path_modifier",
                              'source path modifier', locale=locale)
+
+            kwargs['locale'] = locale
+
             paths.append(os.path.join(
                 base_artifact_prefix,
-                jsone.render(file_config['source_path_modifier'], {'locale': locale}),
-                filename,
+                jsone.render(file_config['source_path_modifier'], kwargs),
+                jsone.render(filename, kwargs),
             ))
 
         if getattr(job['dependencies'][dep], 'release_artifacts', None):
             paths = [
                 path for path in paths
                 if path in job['dependencies'][dep].release_artifacts]
 
         if not paths:
@@ -486,67 +491,16 @@ def generate_beetmover_upstream_artifact
             "taskType": map_config['tasktype_map'].get(dep),
             "paths": sorted(paths),
             "locale": locale,
         })
 
     return upstream_artifacts
 
 
-# generate_beetmover_compressed_upstream_artifacts {{{1
-def generate_beetmover_compressed_upstream_artifacts(job, dependencies=None):
-    """Generate compressed file upstream artifacts for beetmover.
-
-    These artifacts will not be beetmoved directly, but will be
-    decompressed from upstream_mapping and the contents beetmoved
-    using the `mapping` entry in the artifact map.
-
-    Currently only applies to beetmover tasks.
-
-    Args:
-        job (dict): The current job being generated
-        dependencies (list): A list of the job's dependency labels.
-
-    Returns:
-        list: A list of dictionaries conforming to the upstream_artifacts spec.
-    """
-    base_artifact_prefix = get_artifact_prefix(job)
-    map_config = deepcopy(cached_load_yaml(job['attributes']['artifact_map']))
-    upstream_artifacts = list()
-
-    if not dependencies:
-        dependencies = job['dependencies'].keys()
-
-    for dep in dependencies:
-        paths = list()
-
-        for filename in map_config['upstream_mapping']:
-            if dep not in map_config['upstream_mapping'][filename]['from']:
-                continue
-
-            paths.append(os.path.join(
-                base_artifact_prefix,
-                filename,
-            ))
-
-        if not paths:
-            continue
-
-        upstream_artifacts.append({
-            "taskId": {
-                "task-reference": "<{}>".format(dep)
-            },
-            "taskType": map_config['tasktype_map'].get(dep),
-            "paths": sorted(paths),
-            "zipExtract": True,
-        })
-
-    return upstream_artifacts
-
-
 # generate_beetmover_artifact_map {{{1
 def generate_beetmover_artifact_map(config, job, **kwargs):
     """Generate the beetmover artifact map.
 
     Currently only applies to beetmover tasks.
 
     Args:
         config (): Current taskgraph configuration.