Bug 1557269 - part 1: Remove target.maven.zip r=mtabara
☠☠ backed out by d2b5726bfce4 ☠ ☠
authorJohan Lorenzo <jlorenzo@mozilla.com>
Mon, 01 Jul 2019 14:01:13 +0000
changeset 540442 b469d332f316fc19f615a296ae4723330f382529
parent 540441 8f91e15f19be54724d5b1593500ca2b653c4b15f
child 540443 e3d495f69dfbacd48088c5136eac8dfb3272d218
push id11529
push userarchaeopteryx@coole-files.de
push dateThu, 04 Jul 2019 15:22:33 +0000
treeherdermozilla-beta@ebb510a784b8 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmtabara
bugs1557269
milestone69.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1557269 - part 1: Remove target.maven.zip r=mtabara Differential Revision: https://phabricator.services.mozilla.com/D34865
mobile/android/mach_commands.py
taskcluster/ci/beetmover-geckoview/kind.yml
taskcluster/ci/build/android.yml
taskcluster/taskgraph/manifests/fennec_geckoview.yml
taskcluster/taskgraph/transforms/beetmover_geckoview.py
taskcluster/taskgraph/util/scriptworker.py
--- a/mobile/android/mach_commands.py
+++ b/mobile/android/mach_commands.py
@@ -4,18 +4,16 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
 import logging
 import os
 import json
 
-from zipfile import ZipFile
-
 import mozpack.path as mozpath
 
 from mozbuild.base import (
     MachCommandBase,
     MachCommandConditions as conditions,
 )
 
 from mozbuild.shellutil import (
@@ -411,23 +409,17 @@ class MachCommands(MachCommandBase):
                 """Create GeckoView archives.
         See http://firefox-source-docs.mozilla.org/build/buildsystem/toolchains.html#firefox-for-android-with-gradle""")  # NOQA: E501
     @CommandArgument('args', nargs=argparse.REMAINDER)
     def android_archive_geckoview(self, args):
         ret = self.gradle(
             self.substs['GRADLE_ANDROID_ARCHIVE_GECKOVIEW_TASKS'] + args,
             verbose=True)
 
-        if ret != 0:
-            return ret
-
-        # The zip archive is passed along in CI to ship geckoview onto a maven repo
-        _craft_maven_zip_archive(self.topobjdir)
-
-        return 0
+        return ret
 
     @SubCommand('android', 'build-geckoview_example',
                 """Build geckoview_example """)
     @CommandArgument('args', nargs=argparse.REMAINDER)
     def android_build_geckoview_example(self, args):
         self.gradle(self.substs['GRADLE_ANDROID_BUILD_GECKOVIEW_EXAMPLE_TASKS'] + args,
                     verbose=True)
 
@@ -715,36 +707,16 @@ class MachCommands(MachCommandBase):
             extras=extras,
             url=url,
             wait=not no_wait,
             fail_if_running=fail_if_running)
 
         return 0
 
 
-def _get_maven_archive_abs_and_relative_paths(maven_folder):
-    for subdir, _, files in os.walk(maven_folder):
-        for file in files:
-            full_path = os.path.join(subdir, file)
-            relative_path = os.path.relpath(full_path, maven_folder)
-
-            # maven-metadata is intended to be generated on the real maven server
-            if 'maven-metadata.xml' not in relative_path:
-                yield full_path, relative_path
-
-
-def _craft_maven_zip_archive(topobjdir):
-    geckoview_folder = os.path.join(topobjdir, 'gradle/build/mobile/android/geckoview')
-    maven_folder = os.path.join(geckoview_folder, 'maven')
-
-    with ZipFile(os.path.join(geckoview_folder, 'target.maven.zip'), 'w') as target_zip:
-        for abs, rel in _get_maven_archive_abs_and_relative_paths(maven_folder):
-            target_zip.write(abs, arcname=rel)
-
-
 @CommandProvider
 class AndroidEmulatorCommands(MachCommandBase):
     """
        Run the Android emulator with one of the AVDs used in the Mozilla
        automated test environment. If necessary, the AVD is fetched from
        the tooltool server and installed.
     """
     @Command('android-emulator', category='devenv',
--- a/taskcluster/ci/beetmover-geckoview/kind.yml
+++ b/taskcluster/ci/beetmover-geckoview/kind.yml
@@ -35,16 +35,17 @@ not-for-build-platforms:
     - win64-devedition-nightly/opt
     - win64-aarch64-devedition-nightly/opt
     - linux64-asan-reporter-nightly/opt
     - win64-asan-reporter-nightly/opt
 
 job-template:
     attributes:
         artifact_map: taskcluster/taskgraph/manifests/fennec_geckoview.yml
+        artifact_prefix: public/build/maven
     run-on-projects: ['mozilla-central', 'mozilla-release']
     run-on-hg-branches:
         by-project:
             mozilla-release:
                 - '^GECKOVIEW_\d+_RELBRANCH$'
             default:
                 - '.*'
     shipping-phase:
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@@ -6,19 +6,19 @@ job-defaults:
     index:
         product: mobile
     worker:
         docker-image: {in-tree: android-build}
         max-run-time: 7200
         env:
             GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-offline"
         artifacts:
-            - name: public/build/target.maven.zip
-              path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/target.maven.zip
-              type: file
+            - name: public/build/maven
+              path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/maven/
+              type: directory
             - name: public/build/geckoview-androidTest.apk
               path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk
               type: file
             - name: public/build/geckoview_example.apk
               path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk
               type: file
             - name: public/build
               path: /builds/worker/artifacts/
@@ -531,24 +531,16 @@ android-geckoview-fat-aar/opt:
             PERFHERDER_EXTRA_OPTIONS: android-geckoview-fat-aar-opt
             MOZ_ANDROID_FAT_AAR_ARCHITECTURES: 'armeabi-v7a,arm64-v8a,x86,x86_64'
             USE_ARTIFACT: '1'
             MOZ_ARTIFACT_TASK: {task-reference: '<android-api-16-opt>'}
             MOZ_ARTIFACT_TASK_ANDROID_API_16_OPT: {task-reference: '<android-api-16-opt>'}
             MOZ_ARTIFACT_TASK_ANDROID_AARCH64_OPT: {task-reference: '<android-aarch64-opt>'}
             MOZ_ARTIFACT_TASK_ANDROID_X86_OPT: {task-reference: '<android-x86-opt>'}
             MOZ_ARTIFACT_TASK_ANDROID_X86_64_OPT: {task-reference: '<android-x86_64-opt>'}
-        artifacts:
-            - name: public/android/maven
-              # TODO Bug 1433198. Remove the following entry once target.maven.zip is uploaded to a maven repository
-              path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/maven/
-              type: directory
-            - name: public/build/target.maven.zip
-              path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/target.maven.zip
-              type: file
     run:
         actions: [get-secrets, build]
         config: ["builds/releng_base_android_64_builds.py"]
         custom-build-variant-cfg: api-16
     toolchains:
         - linux64-sccache
 
 android-geckoview-fat-aar-nightly/opt:
--- a/taskcluster/taskgraph/manifests/fennec_geckoview.yml
+++ b/taskcluster/taskgraph/manifests/fennec_geckoview.yml
@@ -3,36 +3,30 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 s3_bucket_paths:
     - maven2
 default_locales:  # Ignored for geckoview
     - en-US
 tasktype_map:  # Map task reference to task type.
     build: build
-base_artifact_prefix: ''
+base_artifact_prefix: public/build/maven/
 
 # A default entry, which the mappings below extend and override.
 # Final 'destinations' will be the product of:
 # s3_bucket_paths + destinations + locale_prefix + pretty_name
 default: &default
     from:
         - build
     locale_prefix: ''
     source_path_modifier: org/mozilla/geckoview/${artifact_id}/${major_version}.${minor_version}.${build_date}
     description: "TO_BE_OVERRIDDEN"
     destinations:  # locale_prefix is appended
         - org/mozilla/geckoview/${artifact_id}/${major_version}.${minor_version}.${build_date}
 
-# Configuration for individual files. Extends 'default', above.
-upstream_mapping:
-    target.maven.zip:
-        from:
-            - build
-
 mapping:
     ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1:
         <<: *default
         pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1
         checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1
     ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.md5:
         <<: *default
         pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.md5
--- a/taskcluster/taskgraph/transforms/beetmover_geckoview.py
+++ b/taskcluster/taskgraph/transforms/beetmover_geckoview.py
@@ -11,17 +11,17 @@ import re
 
 from taskgraph.loader.single_dep import schema
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.beetmover import \
     craft_release_properties as beetmover_craft_release_properties
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
 from taskgraph.util.scriptworker import (generate_beetmover_artifact_map,
-                                         generate_beetmover_compressed_upstream_artifacts,
+                                         generate_beetmover_upstream_artifacts,
                                          get_worker_type_for_scope)
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Required, Optional
 
 
 _ARTIFACT_ID_PER_PLATFORM = {
     'android-aarch64': 'geckoview{update_channel}-arm64-v8a',
     'android-api-16': 'geckoview{update_channel}-armeabi-v7a',
@@ -117,57 +117,52 @@ def make_task_description(config, jobs):
             'run-on-projects': job['run-on-projects'],
             'treeherder': treeherder,
             'shipping-phase': job['shipping-phase'],
         }
 
         yield task
 
 
-def generate_upstream_artifacts(build_task_ref):
-    return [{
-        'taskId': {'task-reference': build_task_ref},
-        'taskType': 'build',
-        'paths': ['public/build/target.maven.zip'],
-        'zipExtract': True,
-    }]
-
-
 @transforms.add
 def make_task_worker(config, jobs):
     for job in jobs:
         valid_beetmover_job = len(job['dependencies']) == 1 and 'build' in job['dependencies']
         if not valid_beetmover_job:
             raise NotImplementedError(
                 'Beetmover-geckoview must have a single dependency. Got: {}'.format(
                     job['dependencies']
                 )
             )
 
         worker = {
             'implementation': 'beetmover-maven',
             'release-properties': craft_release_properties(config, job),
         }
 
-        upstream_artifacts = generate_beetmover_compressed_upstream_artifacts(job)
-
-        worker['upstream-artifacts'] = upstream_artifacts
-
         version_groups = re.match(r'(\d+).(\d+).*', config.params['version'])
         if version_groups:
             major_version, minor_version = version_groups.groups()
 
         template_vars = {
             'artifact_id': worker['release-properties']['artifact-id'],
             'build_date': config.params['moz_build_date'],
             'major_version': major_version,
             'minor_version': minor_version,
         }
         worker['artifact-map'] = generate_beetmover_artifact_map(
-            config, job, **template_vars)
+            config, job, **template_vars
+        )
+        upstream_artifacts = generate_beetmover_upstream_artifacts(
+            config, job, platform='', **template_vars
+        )
+        worker['upstream-artifacts'] = [{
+            key: value for key, value in upstream_artifact.items()
+            if key != 'locale'
+        } for upstream_artifact in upstream_artifacts]
 
         job["worker"] = worker
 
         yield job
 
 
 def craft_release_properties(config, job):
     props = beetmover_craft_release_properties(config, job)
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -404,17 +404,19 @@ def get_worker_type_for_scope(config, sc
                 for scopes in config.graph_config['scriptworker']['worker-types'].values()
                 for scope in scopes
             ),
         )
     )
 
 
 # generate_beetmover_upstream_artifacts {{{1
-def generate_beetmover_upstream_artifacts(config, job, platform, locale=None, dependencies=None):
+def generate_beetmover_upstream_artifacts(
+    config, job, platform, locale=None, dependencies=None, **kwargs
+):
     """Generate the upstream artifacts for beetmover, using the artifact map.
 
     Currently only applies to beetmover tasks.
 
     Args:
         job (dict): The current job being generated
         dependencies (list): A list of the job's dependency labels.
         platform (str): The current build platform
@@ -460,20 +462,23 @@ def generate_beetmover_upstream_artifact
                 platform in map_config['mapping'][filename]['not_for_platforms']):
                 continue
             if 'partials_only' in map_config['mapping'][filename]:
                 continue
             # The next time we look at this file it might be a different locale.
             file_config = deepcopy(map_config['mapping'][filename])
             resolve_keyed_by(file_config, "source_path_modifier",
                              'source path modifier', locale=locale)
+
+            kwargs['locale'] = locale
+
             paths.append(os.path.join(
                 base_artifact_prefix,
-                jsone.render(file_config['source_path_modifier'], {'locale': locale}),
-                filename,
+                jsone.render(file_config['source_path_modifier'], kwargs),
+                jsone.render(filename, kwargs),
             ))
 
         if getattr(job['dependencies'][dep], 'release_artifacts', None):
             paths = [
                 path for path in paths
                 if path in job['dependencies'][dep].release_artifacts]
 
         if not paths:
@@ -486,67 +491,16 @@ def generate_beetmover_upstream_artifact
             "taskType": map_config['tasktype_map'].get(dep),
             "paths": sorted(paths),
             "locale": locale,
         })
 
     return upstream_artifacts
 
 
-# generate_beetmover_compressed_upstream_artifacts {{{1
-def generate_beetmover_compressed_upstream_artifacts(job, dependencies=None):
-    """Generate compressed file upstream artifacts for beetmover.
-
-    These artifacts will not be beetmoved directly, but will be
-    decompressed from upstream_mapping and the contents beetmoved
-    using the `mapping` entry in the artifact map.
-
-    Currently only applies to beetmover tasks.
-
-    Args:
-        job (dict): The current job being generated
-        dependencies (list): A list of the job's dependency labels.
-
-    Returns:
-        list: A list of dictionaries conforming to the upstream_artifacts spec.
-    """
-    base_artifact_prefix = get_artifact_prefix(job)
-    map_config = deepcopy(cached_load_yaml(job['attributes']['artifact_map']))
-    upstream_artifacts = list()
-
-    if not dependencies:
-        dependencies = job['dependencies'].keys()
-
-    for dep in dependencies:
-        paths = list()
-
-        for filename in map_config['upstream_mapping']:
-            if dep not in map_config['upstream_mapping'][filename]['from']:
-                continue
-
-            paths.append(os.path.join(
-                base_artifact_prefix,
-                filename,
-            ))
-
-        if not paths:
-            continue
-
-        upstream_artifacts.append({
-            "taskId": {
-                "task-reference": "<{}>".format(dep)
-            },
-            "taskType": map_config['tasktype_map'].get(dep),
-            "paths": sorted(paths),
-            "zipExtract": True,
-        })
-
-    return upstream_artifacts
-
-
 # generate_beetmover_artifact_map {{{1
 def generate_beetmover_artifact_map(config, job, **kwargs):
     """Generate the beetmover artifact map.
 
     Currently only applies to beetmover tasks.
 
     Args:
         config (): Current taskgraph configuration.