Bug 1491371 - Pin comm branches to mozilla revs without .taskcluster.yml. r=tomprince,dustin
authorRob Lemley <rob@thunderbird.net>
Mon, 18 Mar 2019 22:19:38 +0000
changeset 464941 58dec282b0bc6bc75db8870c139922e6801c0e74
parent 464940 115db59b20193a54d75520896656baf1e79e560d
child 464942 770808176c5c92e877b7f5fbf1ad4a4686bd2d6e
push id112486
push useropoprus@mozilla.com
push dateTue, 19 Mar 2019 16:41:04 +0000
treeherdermozilla-inbound@ee866fb50236 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstomprince, dustin
bugs1491371
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1491371 - Pin comm branches to mozilla revs without .taskcluster.yml. r=tomprince,dustin comm-task-env runs before run-task and updates the environment with GECKO_* variables that are defined in a file at the root of the comm repository, ".gecko_rev.yml". run-task needs these variables to be set to find the correct mozilla repository to check out for a particular TB build. The current pinning method of updating ".taskcluster.yml" with the mozilla repository and revision to pin tois no longer supported. Differential Revision: https://phabricator.services.mozilla.com/D16783
taskcluster/scripts/comm-task-env
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/comm-task-env
@@ -0,0 +1,199 @@
+#!/usr/bin/python3 -u
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Thunderbird build environment prep for run-task,
+for use with comm-central derived repositories.
+
+This script is meant to run prior to run-task on repositories like
+comm-central that need to check out a copy of a mozilla repository
+in order to build.
+See bug 1491371 for background on why this is necessary.
+
+A project will have a file named ".gecko_rev.yml" in it's root. See the
+constant "GECKO_REV_CONF" if you want to change that. To download it, the
+script uses the project repository URL and the revision number.
+Those are defined in the environment variables:
+COMM_HEAD_REPOSITORY
+COMM_HEAD_REV
+
+.gecko_rev.yml has a structure like (for comm-central):
+```
+GECKO_BASE_REPOSITORY: https://hg.mozilla.org/mozilla-unified
+GECKO_HEAD_REPOSITORY: https://hg.mozilla.org/mozilla-central
+GECKO_HEAD_REF: default
+```
+or for branches:
+```
+GECKO_BASE_REPOSITORY: https://hg.mozilla.org/mozilla-unified
+GECKO_HEAD_REPOSITORY: https://hg.mozilla.org/releases/mozilla-beta
+GECKO_HEAD_REF: THUNDERBIRD_60_VERBRANCH
+GECKO_HEAD_REV: 6a830d12f15493a70b1192022c9985eba2139910
+
+Note about GECKO_HEAD_REV and GECKO_HEAD_REF:
+GECKO_HEAD_REF is a branch name or "default".
+GECKO_HEAD_REV is a revision hash.
+```
+"""
+
+import sys
+
+import os
+import socket
+import time
+from datetime import datetime
+from pprint import pformat
+
+import urllib.error
+import urllib.request
+
+import yaml
+
+if sys.version_info[0:2] < (3, 5):
+    print('run-task-wrapper requires Python 3.5+')
+    sys.exit(1)
+
+GECKO_REV_CONF = ".gecko_rev.yml"
+DEBUG = bool(os.environ.get("RTW_DEBUG", False))
+
+
+def print_message(msg, prefix=__file__, level=""):
+    """
+    Print messages.
+    :param object msg: message to print, usually a string, but not always
+    :param str prefix: message prefix
+    :param str level: message level (DEBUG, ERROR, INFO)
+    """
+    if not isinstance(msg, str):
+        msg = pformat(msg)
+    now = datetime.utcnow().isoformat()
+    # slice microseconds to 3 decimals.
+    now = now[:-3] if now[-7:-6] == '.' else now
+    if level:
+        sys.stdout.write('[{prefix} {now}Z] {level}: {msg}\n'.format(
+            prefix=prefix, now=now, level=level, msg=msg))
+    else:
+        sys.stdout.write('[{prefix} {now}Z] {msg}\n'.format(
+            prefix=prefix, now=now, msg=msg))
+    sys.stdout.flush()
+
+
+def error_exit(msg):
+    """Print the error message and exit with error."""
+    print_message(msg, level="ERROR")
+    if DEBUG:
+        raise Exception(msg)
+
+    sys.exit(1)
+
+
+def print_debug(msg):
+    """Prints a message with DEBUG prefix if DEBUG is enabled
+    with the environment variable "RTW_DEBUG".
+    """
+    if DEBUG:
+        print_message(msg, level="DEBUG")
+
+
+def check_environ():
+    """Check that the necessary environment variables to find the
+    comm- repository are defined. (Set in .taskcluster.yml)
+    :return: tuple(str, str)
+    """
+    print_debug("Checking environment variables...")
+    project_head_repo = os.environ.get("COMM_HEAD_REPOSITORY", None)
+    project_head_rev = os.environ.get("COMM_HEAD_REV", None)
+
+    if project_head_repo is None or project_head_rev is None:
+        error_exit("Environment NOT Ok:\n\tHead: {}\n\tRev: {}\n").format(
+            project_head_repo, project_head_rev)
+
+    print_debug("Environment Ok:\n\tHead: {}\n\tRev: {}\n".format(
+        project_head_repo, project_head_rev))
+    return project_head_repo, project_head_rev
+
+
+def download_url(url, retry=1):
+    """Downloads the given URL. Naively retries (when asked) upon failure
+    :param url: str
+    :param retry: int
+    :return: str
+    """
+    # Use 1-based counting for display and calculation purposes.
+    for i in range(1, retry+1):
+        try:
+            print_message('Fetching {}. Attempt {} of {}.'.format(
+                url, i, retry))
+            with urllib.request.urlopen(url, timeout=10) as response:
+                data = response.read().decode("utf-8")
+            return data
+        except (urllib.error.URLError, socket.timeout) as exc:
+            print_message('Unable to retrieve {}'.format(url))
+            if isinstance(exc, urllib.error.URLError):
+                print_message(exc.reason)
+            else:  # socket.timeout
+                print_message('Connection timed out.')
+
+            if i < retry:  # No more retries
+                wait_time = i * 5  # fail #1: sleep 5s. #2, sleep 10s
+                print_message('Retrying in {} seconds.'.format(wait_time))
+                time.sleep(wait_time)
+
+    error_exit('No more retry attempts! Aborting.')
+
+
+def fetch_gecko_conf(project_head_repo, project_revision):
+    """Downloads .gecko_rev.yml from the project repository
+    :param project_head_repo: str
+    :param project_revision: str
+    :return: dict
+    """
+    gecko_conf_url = '/'.join(
+        [project_head_repo, 'raw-file', project_revision, GECKO_REV_CONF])
+
+    gecko_conf_yml = download_url(gecko_conf_url, retry=5)
+
+    try:
+        gecko_conf = yaml.safe_load(gecko_conf_yml)
+        return gecko_conf
+    except yaml.YAMLError as exc:
+        err_txt = ["Error processing Gecko YAML configuration."]
+        if hasattr(exc, "problem_mark"):
+            mark = exc.problem_mark  # pylint: disable=no-member
+            err_txt.append("Error position: line {}, column {}".format(
+                mark.line + 1, mark.column + 1))
+        error_exit('\n'.join(err_txt))
+
+
+def update_environment(gecko_conf):
+    """Adds the new variables defined in gecko_conf to the
+    running environment.
+    :param gecko_conf: dict
+    """
+    print_message("Updating environment with:")
+    print_message(gecko_conf)
+    os.environ.update(gecko_conf)
+
+    print_debug("New environment:")
+    print_debug(os.environ)
+
+
+def exec_run_task(args):
+    """Executes run-task with a modified environment."""
+    print_message("Executing: {}".format(pformat(args)))
+    os.execv(args[0], args[1:])
+
+
+def main():
+    """Main function."""
+    args = sys.argv[1:]  # Remaining args starting with run-task
+
+    project_head_repo, project_revision = check_environ()
+    gecko_conf = fetch_gecko_conf(project_head_repo, project_revision)
+    update_environment(gecko_conf)
+    exec_run_task(args)
+
+
+if __name__ == "__main__":
+    main()