Bug 1225905 - create a mozharness script that manages each beet mover task logic, NPOTB DONTBUILD r=rail
authorJordan Lund <jlund@mozilla.com>
Wed, 30 Dec 2015 10:20:31 -0800
changeset 300616 a9ae29806909ed3a0fc92b171367352cd5934e4c
parent 300615 3f2b631cf18880f8a2b5a61b02b4cbd715304326
child 300617 9ba69c6976af4119453cba2ca7b3c86100f7430e
push id8978
push userraliiev@mozilla.com
push dateMon, 25 Jan 2016 14:05:32 +0000
treeherdermozilla-aurora@b9a803752a2c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrail
bugs1225905
milestone46.0a1
Bug 1225905 - create a mozharness script that manages each beet mover task logic, NPOTB DONTBUILD r=rail
testing/mozharness/configs/beetmover/en_us.yml.tmpl
testing/mozharness/scripts/release/beet_mover.py
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/beetmover/en_us.yml.tmpl
@@ -0,0 +1,84 @@
+---
+metadata:
+    name: "Beet Mover Manifest"
+    description: "Maps artifact locations to s3 key names for the en-US locale"
+    owner: "release@mozilla.com"
+
+mapping:
+{% for locale in locales %}
+  # common deliverables
+  {{ locale }}:
+    complete_mar:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.complete.mar
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/update/{{ platform }}/{{ locale }}/firefox-{{ version }}.complete.mar
+    checksum:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.checksums
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.checksums
+    checksum_sig:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.checksums.asc
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.checksums.asc
+    buildinfo:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.json
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.json
+    mozinfo:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.mozinfo.json
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.mozinfo.json
+    socorroinfo:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.txt
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.txt
+
+  {% if platform == "win32" %}
+    full_installer:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.installer.exe
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/Firefox Setup {{ version }}.exe
+    stub_installer:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.installer-stub.exe
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/Firefox Setup Stub {{ version }}.exe
+    package:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.zip
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.zip
+    symbols:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.crashreporter-symbols.zip
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.crashreporter-symbols.zip
+  {% endif %}
+
+  {% if platform == "win64" %}
+    full_installer:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.installer.exe
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/Firefox Setup {{ version }}.exe
+    package:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.zip
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.zip
+    symbols:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.crashreporter-symbols.zip
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.crashreporter-symbols.zip
+  {% endif %}
+
+  {% if platform == "linux-i686" %}
+    package:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.tar.bz2
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.tar.bz2
+    symbols:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.crashreporter-symbols.zip
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.crashreporter-symbols.zip
+  {% endif %}
+
+  {% if platform == "linux-x86_64" %}
+    package:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.tar.bz2
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.tar.bz2
+    symbols:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.crashreporter-symbols.zip
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/firefox-{{ version }}.crashreporter-symbols.zip
+  {% endif %}
+
+  {% if platform == "mac" %}
+    package:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.dmg
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/Firefox {{ version }}.dmg
+    symbols:
+      artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.crashreporter-symbols.zip
+      s3_key: {{ s3_prefix }}/{{ version }}-candidates/{{ build_num }}/{{ platform }}/{{ locale }}/Firefox {{ version }}.crashreporter-symbols.zip
+  {% endif %}
+
+{% endfor %}
new file mode 100755
--- /dev/null
+++ b/testing/mozharness/scripts/release/beet_mover.py
@@ -0,0 +1,240 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+"""beet_mover.py.
+
+downloads artifacts and uploads them to s3
+"""
+import hashlib
+import sys
+import os
+import pprint
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+from mozharness.base.log import FATAL
+from mozharness.base.python import VirtualenvMixin
+from mozharness.base.script import BaseScript
+
+
+def get_hash(content, hash_type="md5"):
+    h = hashlib.new(hash_type)
+    h.update(content)
+    return h.hexdigest()
+
+
+def get_aws_auth():
+    """
+    retrieves aws creds and deletes them from os.environ if present.
+    """
+    aws_key_id = os.environ.get("AWS_ACCESS_KEY_ID")
+    aws_secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY")
+
+    if aws_key_id and aws_secret_key:
+        del os.environ['AWS_ACCESS_KEY_ID']
+        del os.environ['AWS_SECRET_ACCESS_KEY']
+    else:
+        exit("could not determine aws credentials from os environment")
+
+    return aws_key_id, aws_secret_key
+
+
+CONFIG_OPTIONS = [
+    [["--template"], {
+        "dest": "template",
+        "help": "Specify jinja2 template file",
+    }],
+    [['--locale', ], {
+        "action": "extend",
+        "dest": "locales",
+        "type": "string",
+        "help": "Specify the locale(s) to upload."}],
+    [["--platform"], {
+        "dest": "platform",
+        "help": "Specify the platform of the build",
+    }],
+    [["--version"], {
+        "dest": "version",
+        "help": "full release version based on gecko and tag/stage identifier. e.g. '44.0b1'"
+    }],
+    [["--app-version"], {
+        "dest": "app_version",
+        "help": "numbered version based on gecko. e.g. '44.0'"
+    }],
+    [["--build-num"], {
+        "dest": "build_num",
+        "help": "the release build identifier"
+    }],
+    [["--taskid"], {
+        "dest": "taskid",
+        "help": "taskcluster task id to download artifacts from",
+    }],
+    [["--production"], {
+        "dest": "production",
+        "default": False,
+        "help": "taskcluster task id to download artifacts from",
+    }],
+]
+
+
+class BeetMover(BaseScript, VirtualenvMixin, object):
+    def __init__(self, aws_creds):
+        beetmover_kwargs = {
+            'config_options': CONFIG_OPTIONS,
+            'all_actions': [
+                # 'clobber',
+                'create-virtualenv',
+                'activate-virtualenv',
+                'generate-candidates-manifest',
+                'verify-bits',  # beets
+                'upload-bits',  # beets
+            ],
+            'require_config_file': False,
+            # Default configuration
+            'config': {
+                # base index url where to find taskcluster artifact based on taskid
+                # TODO - find out if we need to support taskcluster run number other than 0.
+                # e.g. maybe we could end up with artifacts in > 'run 0' in a re-trigger situation?
+                "artifact_base_url": 'https://queue.taskcluster.net/v1/task/{taskid}/runs/0/artifacts/public/build',
+                "virtualenv_modules": [
+                    "boto",
+                    "PyYAML",
+                    "Jinja2",
+                ],
+                "virtualenv_path": "venv",
+                'buckets': {
+                    'development': "mozilla-releng-beet-mover-dev",
+                    'production': "mozilla-releng-beet-mover",
+                },
+                'product': 'firefox',
+            },
+        }
+        super(BeetMover, self).__init__(**beetmover_kwargs)
+
+        c = self.config
+        self.manifest = {}
+        # assigned in _post_create_virtualenv
+        self.virtualenv_imports = None
+        self.bucket = c['buckets']['production'] if c['production'] else c['buckets']['development']
+        self.aws_key_id, self.aws_secret_key = aws_creds
+
+    def activate_virtualenv(self):
+        """
+        activates virtualenv and adds module imports to a instance wide namespace.
+
+        creating and activating a virtualenv onto the currently executing python interpreter is a
+        bit black magic. Rather than having import statements added in various places within the
+        script, we import them here immediately after we activate the newly created virtualenv
+        """
+        VirtualenvMixin.activate_virtualenv(self)
+
+        import boto
+        import yaml
+        import jinja2
+        self.virtualenv_imports = {
+            'boto': boto,
+            'yaml': yaml,
+            'jinja2': jinja2,
+        }
+        self.log("activated virtualenv with the modules: {}".format(str(self.virtualenv_imports)))
+
+    def generate_candidates_manifest(self):
+        """
+        generates and outputs a manifest that maps expected Taskcluster artifact names
+        to release deliverable names
+        """
+        self.log('generating manifest from {}...'.format(self.config['template']))
+        template_dir, template_file = os.path.split(os.path.abspath(self.config['template']))
+        jinja2 = self.virtualenv_imports['jinja2']
+        yaml = self.virtualenv_imports['yaml']
+
+        jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir),
+                                       undefined=jinja2.StrictUndefined)
+        template = jinja_env.get_template(template_file)
+        template_vars = {
+            "platform": self.config['platform'],
+            "locales": self.config['locales'],
+            "version": self.config['version'],
+            "app_version": self.config['app_version'],
+            "build_num": self.config['build_num'],
+            # mirror current release folder structure
+            "s3_prefix": 'pub/{}/candidates'.format(self.config['product']),
+            "artifact_base_url": self.config['artifact_base_url'].format(
+                    taskid=self.config['taskid']
+            )
+        }
+        self.manifest = yaml.safe_load(template.render(**template_vars))
+
+        self.log("manifest generated:")
+        self.log(pprint.pformat(self.manifest['mapping']))
+
+    def verify_bits(self):
+        """
+        inspects each artifact and verifies that they were created by trustworthy tasks
+        """
+        # TODO
+        self.log('skipping verification. unimplemented...')
+
+    def upload_bits(self):
+        """
+        downloads and uploads list of artifacts to s3 candidates dir based on a given manifest
+        """
+        self.log('downloading and uploading artifacts to s3...')
+
+        # connect to s3
+        boto = self.virtualenv_imports['boto']
+        conn = boto.connect_s3(self.aws_key_id, self.aws_secret_key)
+        bucket = conn.get_bucket(self.bucket)
+
+        for locale in self.manifest['mapping']:
+            for deliverable in self.manifest['mapping'][locale]:
+                self.log("uploading '{}' deliverable for '{}' locale".format(deliverable, locale))
+                self.upload_bit(
+                    source=self.manifest['mapping'][locale][deliverable]['artifact'],
+                    s3_key=self.manifest['mapping'][locale][deliverable]['s3_key'],
+                    bucket=bucket,
+                )
+        self.log('Success!')
+
+    def upload_bit(self, source, s3_key, bucket):
+        # TODO - do we want to mirror/upload to more than one region?
+        dirs = self.query_abs_dirs()
+        boto = self.virtualenv_imports['boto']
+
+        # download locally
+        file_name = self.retry(self.download_file,
+                               args=[source],
+                               kwargs={'parent_dir': dirs['abs_work_dir']},
+                               error_level=FATAL)
+
+        self.info('uploading to s3 with key: {}'.format(s3_key))
+        key = boto.s3.key.Key(bucket)  # create new key
+        key.key = s3_key  # set key name
+
+        self.info("Checking if `{}` already exists".format(s3_key))
+        key = bucket.get_key(s3_key)
+        if not key:
+            self.info("Uploading to `{}`".format(s3_key))
+            key = bucket.new_key(s3_key)
+
+            # set key value
+            self.retry(key.set_contents_from_filename, args=[file_name], error_level=FATAL),
+
+            # key.make_public() may lead to race conditions, because
+            # it doesn't pass version_id, so it may not set permissions
+            bucket.set_canned_acl(acl_str='public-read', key_name=s3_key,
+                                  version_id=key.version_id)
+        else:
+            if not get_hash(key.get_contents_as_string()) == get_hash(open(file_name).read()):
+                # for now, let's halt. If necessary, we can revisit this and allow for overwrites
+                #  to the same buildnum release with different bits
+                self.fatal("`{}` already exists with different checksum.".format(s3_key))
+            self.log("`{}` has the same MD5 checksum, not uploading".format(s3_key))
+
+
+
+if __name__ == '__main__':
+    beet_mover = BeetMover(get_aws_auth())
+    beet_mover.run_and_exit()