Bug 1391427 - Port the repack_rust script to taskcluster. r?mshal draft
authorRalph Giles <giles@mozilla.com>
Tue, 12 Sep 2017 16:17:00 -0700
changeset 663406 973931045ab7184229d8f67d0a7e2478aec2677f
parent 663405 3fbcd1f4793cf1f1b4669c799f3a41e8a2c8c851
child 663407 146bb98c2bf5647c0efc65c5a9aeec5236064a6a
push id79430
push userbmo:giles@thaumas.net
push dateTue, 12 Sep 2017 23:54:08 +0000
reviewersmshal
bugs1391427
milestone57.0a1
Bug 1391427 - Port the repack_rust script to taskcluster. r?mshal Copy the repack_rust.py from the rust-build docker container so it can be used more generally by other taskcluster jobs. Add --host, --target, and --suffix switches, allowing control of the packaged toolchain and standard library builds from the command line. This drops the previous default behaviour or packaging for all supported platforms and targets. Add a hard-coded copy of the Rust release signing key to the script and add it to the running user's gpg config so we can validate downloaded artifacts from the project in automation. Remove the keybase artifact validation since it requires out-of-project network services and doesn't provide much advantage in automation. Add support for copying the packages to an output directory if the UPLOAD_DIR environment variable is set. This lets us hook up the script to taskcluster toolchain jobs without an external wrapper. MozReview-Commit-ID: 68LmY3QVU8V
taskcluster/scripts/misc/repack_rust.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/scripts/misc/repack_rust.py
@@ -0,0 +1,375 @@
+#!/bin/env python
+'''
+This script downloads and repacks official rust language builds
+with the necessary tool and target support for the Firefox
+build environment.
+'''
+
+from __future__ import absolute_import, print_function
+
+import argparse
+import errno
+import os
+import sys
+
+import requests
+import subprocess
+import pytoml as toml
+
+
+def log(msg):
+    print('repack: %s' % msg)
+
+
+def fetch_file(url):
+    '''Download a file from the given url if it's not already present.'''
+    filename = os.path.basename(url)
+    if os.path.exists(filename):
+        return
+    r = requests.get(url, stream=True)
+    r.raise_for_status()
+    with open(filename, 'wb') as fd:
+        for chunk in r.iter_content(4096):
+            fd.write(chunk)
+
+
+def sha256sum():
+    '''Return the command for verifying SHA-2 256-bit checksums.'''
+    if sys.platform.startswith('darwin'):
+        return 'shasum'
+    else:
+        return 'sha256sum'
+
+
+def setup_gpg():
+    '''Add the signing key to the current gpg config.
+
+    Import a hard-coded copy of the release signing public key
+    and mark it trusted in the gpg database so subsequent
+    signature checks can succeed or fail cleanly.'''
+    keyid = '0x85AB96E6FA1BE5FE'
+    log('Importing signing key %s...' % keyid)
+    key = '''
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFJEwMkBEADlPACa2K7reD4x5zd8afKx75QYKmxqZwywRbgeICeD4bKiQoJZ
+dUjmn1LgrGaXuBMKXJQhyA34e/1YZel/8et+HPE5XpljBfNYXWbVocE1UMUTnFU9
+CKXa4AhJ33f7we2/QmNRMUifw5adPwGMg4D8cDKXk02NdnqQlmFByv0vSaArR5kn
+gZKnLY6o0zZ9Buyy761Im/ShXqv4ATUgYiFc48z33G4j+BDmn0ryGr1aFdP58tHp
+gjWtLZs0iWeFNRDYDje6ODyu/MjOyuAWb2pYDH47Xu7XedMZzenH2TLM9yt/hyOV
+xReDPhvoGkaO8xqHioJMoPQi1gBjuBeewmFyTSPS4deASukhCFOcTsw/enzJagiS
+ZAq6Imehduke+peAL1z4PuRmzDPO2LPhVS7CDXtuKAYqUV2YakTq8MZUempVhw5n
+LqVaJ5/XiyOcv405PnkT25eIVVVghxAgyz6bOU/UMjGQYlkUxI7YZ9tdreLlFyPR
+OUL30E8q/aCd4PGJV24yJ1uit+yS8xjyUiMKm4J7oMP2XdBN98TUfLGw7SKeAxyU
+92BHlxg7yyPfI4TglsCzoSgEIV6xoGOVRRCYlGzSjUfz0bCMCclhTQRBkegKcjB3
+sMTyG3SPZbjTlCqrFHy13e6hGl37Nhs8/MvXUysq2cluEISn5bivTKEeeQARAQAB
+tERSdXN0IExhbmd1YWdlIChUYWcgYW5kIFJlbGVhc2UgU2lnbmluZyBLZXkpIDxy
+dXN0LWtleUBydXN0LWxhbmcub3JnPokCOAQTAQIAIgUCUkTAyQIbAwYLCQgHAwIG
+FQgCCQoLBBYCAwECHgECF4AACgkQhauW5vob5f5fYQ//b1DWK1NSGx5nZ3zYZeHJ
+9mwGCftIaA2IRghAGrNf4Y8DaPqR+w1OdIegWn8kCoGfPfGAVW5XXJg+Oxk6QIaD
+2hJojBUrq1DALeCZVewzTVw6BN4DGuUexsc53a8DcY2Yk5WE3ll6UKq/YPiWiPNX
+9r8FE2MJwMABB6mWZLqJeg4RCrriBiCG26NZxGE7RTtPHyppoVxWKAFDiWyNdJ+3
+UnjldWrT9xFqjqfXWw9Bhz8/EoaGeSSbMIAQDkQQpp1SWpljpgqvctZlc5fHhsG6
+lmzW5RM4NG8OKvq3UrBihvgzwrIfoEDKpXbk3DXqaSs1o81NH5ftVWWbJp/ywM9Q
+uMC6n0YWiMZMQ1cFBy7tukpMkd+VPbPkiSwBhPkfZIzUAWd74nanN5SKBtcnymgJ
++OJcxfZLiUkXRj0aUT1GLA9/7wnikhJI+RvwRfHBgrssXBKNPOfXGWajtIAmZc2t
+kR1E8zjBVLId7r5M8g52HKk+J+y5fVgJY91nxG0zf782JjtYuz9+knQd55JLFJCO
+hhbv3uRvhvkqgauHagR5X9vCMtcvqDseK7LXrRaOdOUDrK/Zg/abi5d+NIyZfEt/
+ObFsv3idAIe/zpU6xa1nYNe3+Ixlb6mlZm3WCWGxWe+GvNW/kq36jZ/v/8pYMyVO
+p/kJqnf9y4dbufuYBg+RLqC5Ag0EUkTAyQEQANxy2tTSeRspfrpBk9+ju+KZ3zc4
+umaIsEa5DxJ2zIKHywVAR67Um0K1YRG07/F5+tD9TIRkdx2pcmpjmSQzqdk3zqa9
+2Zzeijjz2RNyBY8qYmyE08IncjTsFFB8OnvdXcsAgjCFmI1BKnePxrABL/2k8X18
+aysPb0beWqQVsi5FsSpAHu6k1kaLKc+130x6Hf/YJAjeo+S7HeU5NeOz3zD+h5bA
+Q25qMiVHX3FwH7rFKZtFFog9Ogjzi0TkDKKxoeFKyADfIdteJWFjOlCI9KoIhfXq
+Et9JMnxApGqsJElJtfQjIdhMN4Lnep2WkudHAfwJ/412fe7wiW0rcBMvr/BlBGRY
+vM4sTgN058EwIuY9Qmc8RK4gbBf6GsfGNJjWozJ5XmXElmkQCAvbQFoAfi5TGfVb
+77QQrhrQlSpfIYrvfpvjYoqj618SbU6uBhzh758gLllmMB8LOhxWtq9eyn1rMWyR
+KL1fEkfvvMc78zP+Px6yDMa6UIez8jZXQ87Zou9EriLbzF4QfIYAqR9LUSMnLk6K
+o61tSFmFEDobC3tc1jkSg4zZe/wxskn96KOlmnxgMGO0vJ7ASrynoxEnQE8k3WwA
++/YJDwboIR7zDwTy3Jw3mn1FgnH+c7Rb9h9geOzxKYINBFz5Hd0MKx7kZ1U6WobW
+KiYYxcCmoEeguSPHABEBAAGJAh8EGAECAAkFAlJEwMkCGwwACgkQhauW5vob5f7f
+FA//Ra+itJF4NsEyyhx4xYDOPq4uj0VWVjLdabDvFjQtbBLwIyh2bm8uO3AY4r/r
+rM5WWQ8oIXQ2vvXpAQO9g8iNlFez6OLzbfdSG80AG74pQqVVVyCQxD7FanB/KGge
+tAoOstFxaCAg4nxFlarMctFqOOXCFkylWl504JVIOvgbbbyj6I7qCUmbmqazBSMU
+K8c/Nz+FNu2Uf/lYWOeGogRSBgS0CVBcbmPUpnDHLxZWNXDWQOCxbhA1Uf58hcyu
+036kkiWHh2OGgJqlo2WIraPXx1cGw1Ey+U6exbtrZfE5kM9pZzRG7ZY83CXpYWMp
+kyVXNWmf9JcIWWBrXvJmMi0FDvtgg3Pt1tnoxqdilk6yhieFc8LqBn6CZgFUBk0t
+NSaWk3PsN0N6Ut8VXY6sai7MJ0Gih1gE1xadWj2zfZ9sLGyt2jZ6wK++U881YeXA
+ryaGKJ8sIs182hwQb4qN7eiUHzLtIh8oVBHo8Q4BJSat88E5/gOD6IQIpxc42iRL
+T+oNZw1hdwNyPOT1GMkkn86l3o7klwmQUWCPm6vl1aHp3omo+GHC63PpNFO5RncJ
+Ilo3aBKKmoE5lDSMGE8KFso5awTo9z9QnVPkRsk6qeBYit9xE3x3S+iwjcSg0nie
+aAkc0N00nc9V9jfPvt4z/5A5vjHh+NhFwH5h2vBJVPdsz6m5Ag0EVI9keAEQAL3R
+oVsHncJTmjHfBOV4JJsvCum4DuJDZ/rDdxauGcjMUWZaG338ZehnDqG1Yn/ys7zE
+aKYUmqyT+XP+M2IAQRTyxwlU1RsDlemQfWrESfZQCCmbnFScL0E7cBzy4xvtInQe
+UaFgJZ1BmxbzQrx+eBBdOTDv7RLnNVygRmMzmkDhxO1IGEu1+3ETIg/DxFE7VQY0
+It/Ywz+nHu1o4Hemc/GdKxu9hcYvcRVc/Xhueq/zcIM96l0m+CFbs0HMKCj8dgMe
+Ng6pbbDjNM+cV+5BgpRdIpE2l9W7ImpbLihqcZt47J6oWt/RDRVoKOzRxjhULVyV
+2VP9ESr48HnbvxcpvUAEDCQUhsGpur4EKHFJ9AmQ4zf91gWLrDc6QmlACn9o9ARU
+fOV5aFsZI9ni1MJEInJTP37stz/uDECRie4LTL4O6P4Dkto8ROM2wzZq5CiRNfnT
+PP7ARfxlCkpg+gpLYRlxGUvRn6EeYwDtiMQJUQPfpGHSvThUlgDEsDrpp4SQSmdA
+CB+rvaRqCawWKoXs0In/9wylGorRUupeqGC0I0/rh+f5mayFvORzwy/4KK4QIEV9
+aYTXTvSRl35MevfXU1Cumlaqle6SDkLr3ZnFQgJBqap0Y+Nmmz2HfO/pohsbtHPX
+92SN3dKqaoSBvzNGY5WT3CsqxDtik37kR3f9/DHpABEBAAGJBD4EGAECAAkFAlSP
+ZHgCGwICKQkQhauW5vob5f7BXSAEGQECAAYFAlSPZHgACgkQXLSpNHs7CdwemA/+
+KFoGuFqU0uKT9qblN4ugRyil5itmTRVffl4tm5OoWkW8uDnu7Ue3vzdzy+9NV8X2
+wRG835qjXijWP++AGuxgW6LB9nV5OWiKMCHOWnUjJQ6pNQMAgSN69QzkFXVF/q5f
+bkma9TgSbwjrVMyPzLSRwq7HsT3V02Qfr4cyq39QeILGy/NHW5z6LZnBy3BaVSd0
+lGjCEc3yfH5OaB79na4W86WCV5n4IT7cojFM+LdL6P46RgmEtWSG3/CDjnJl6BLR
+WqatRNBWLIMKMpn+YvOOL9TwuP1xbqWr1vZ66wksm53NIDcWhptpp0KEuzbU0/Dt
+OltBhcX8tOmO36LrSadX9rwckSETCVYklmpAHNxPml011YNDThtBidvsicw1vZwR
+HsXn+txlL6RAIRN+J/Rw3uOiJAqN9Qgedpx2q+E15t8MiTg/FXtB9SysnskFT/BH
+z0USNKJUY0btZBw3eXWzUnZf59D8VW1M/9JwznCHAx0c9wy/gRDiwt9w4RoXryJD
+VAwZg8rwByjldoiThUJhkCYvJ0R3xH3kPnPlGXDW49E9R8C2umRC3cYOL4U9dOQ1
+5hSlYydF5urFGCLIvodtE9q80uhpyt8L/5jj9tbwZWv6JLnfBquZSnCGqFZRfXlb
+Jphk9+CBQWwiZSRLZRzqQ4ffl4xyLuolx01PMaatkQbRaw/+JpgRNlurKQ0PsTrO
+8tztO/tpBBj/huc2DGkSwEWvkfWElS5RLDKdoMVs/j5CLYUJzZVikUJRm7m7b+OA
+P3W1nbDhuID+XV1CSBmGifQwpoPTys21stTIGLgznJrIfE5moFviOLqD/LrcYlsq
+CQg0yleu7SjOs//8dM3mC2FyLaE/dCZ8l2DCLhHw0+ynyRAvSK6aGCmZz6jMjmYF
+MXgiy7zESksMnVFMulIJJhR3eB0wx2GitibjY/ZhQ7tD3i0yy9ILR07dFz4pgkVM
+afxpVR7fmrMZ0t+yENd+9qzyAZs0ksxORoc2ze90SCx2jwEX/3K+m4I0hP2H/w5W
+gqdvuRLiqf+4BGW4zqWkLLlNIe/okt0r82SwHtDN0Ui1asmZTGj6sm8SXtwx+5cE
+38MttWqjDiibQOSthRVcETByRYM8KcjYSUCi4PoBc3NpDONkFbZm6XofR/f5mTcl
+2jDw6fIeVc4Hd1jBGajNzEqtneqqbdAkPQaLsuD2TMkQfTDJfE/IljwjrhDa9Mi+
+odtnMWq8vlwOZZ24/8/BNK5qXuCYL67O7AJB4ZQ6BT+g4z96iRLbupzu/XJyXkQF
+rOY/Ghegvn7fDrnt2KC9MpgeFBXzUp+k5rzUdF8jbCx5apVjA1sWXB9Kh3L+DUwF
+Mve696B5tlHyc1KxjHR6w9GRsh4=
+=5FXw
+-----END PGP PUBLIC KEY BLOCK-----
+'''
+    gpg = subprocess.Popen(['gpg', '--import'], stdin=subprocess.PIPE)
+    gpg.communicate(key)
+    gpg = subprocess.Popen(['gpg', '--command-fd', '0',
+                           '--edit-key', keyid],
+                           stdin=subprocess.PIPE)
+    gpg.communicate('trust\n5\ny\n')
+
+
+def fetch(url):
+    '''Download and verify a package url.'''
+    base = os.path.basename(url)
+    log('Fetching %s...' % base)
+    fetch_file(url + '.asc')
+    fetch_file(url)
+    fetch_file(url + '.sha256')
+    log('Verifying %s...' % base)
+    shasum = sha256sum()
+    subprocess.check_call([shasum, '-c', base + '.sha256'])
+    subprocess.check_call(['gpg', '--keyid-format', '0xlong',
+                           '--verify', base + '.asc', base])
+
+
+def install(filename, target):
+    '''Run a package's installer script against the given target directory.'''
+    log('Unpacking %s...' % filename)
+    subprocess.check_call(['tar', 'xf', filename])
+    basename = filename.split('.tar')[0]
+    log('Installing %s...' % basename)
+    install_cmd = [os.path.join(basename, 'install.sh')]
+    install_cmd += ['--prefix=' + os.path.abspath(target)]
+    install_cmd += ['--disable-ldconfig']
+    subprocess.check_call(install_cmd)
+    log('Cleaning %s...' % basename)
+    subprocess.check_call(['rm', '-rf', basename])
+
+
+def package(manifest, pkg, target):
+    '''Pull out the package dict for a particular package and target
+    from the given manifest.'''
+    version = manifest['pkg'][pkg]['version']
+    info = manifest['pkg'][pkg]['target'][target]
+    return (version, info)
+
+
+def fetch_package(manifest, pkg, host):
+    version, info = package(manifest, pkg, host)
+    log('%s %s\n  %s\n  %s' % (pkg, version, info['url'], info['hash']))
+    if not info['available']:
+        log('%s marked unavailable for %s' % (pkg, host))
+        raise AssertionError
+    fetch(info['url'])
+    return info
+
+
+def fetch_std(manifest, targets):
+    stds = []
+    for target in targets:
+        info = fetch_package(manifest, 'rust-std', target)
+        stds.append(info)
+    return stds
+
+
+def tar_for_host(host):
+    if 'linux' in host:
+        tar_options = 'cJf'
+        tar_ext = '.tar.xz'
+    else:
+        tar_options = 'cjf'
+        tar_ext = '.tar.bz2'
+    return tar_options, tar_ext
+
+
+def fetch_manifest(channel='stable'):
+    url = 'https://static.rust-lang.org/dist/channel-rust-' + channel + '.toml'
+    req = requests.get(url)
+    req.raise_for_status()
+    manifest = toml.loads(req.content)
+    if manifest['manifest-version'] != '2':
+        raise NotImplementedError('Unrecognized manifest version %s.' %
+                                  manifest['manifest-version'])
+    return manifest
+
+
+def repack(host, targets, channel='stable', suffix='', cargo_channel=None):
+    log("Repacking rust for %s supporting %s..." % (host, targets))
+
+    manifest = fetch_manifest(channel)
+    log('Using manifest for rust %s as of %s.' % (channel, manifest['date']))
+    if cargo_channel == channel:
+        cargo_manifest = manifest
+    else:
+        cargo_manifest = fetch_manifest(cargo_channel)
+        log('Using manifest for cargo %s as of %s.' %
+            (cargo_channel, cargo_manifest['date']))
+
+    log('Fetching packages...')
+    rustc = fetch_package(manifest, 'rustc', host)
+    cargo = fetch_package(cargo_manifest, 'cargo', host)
+    stds = fetch_std(manifest, targets)
+
+    log('Installing packages...')
+    tar_basename = 'rustc-' + host
+    if suffix:
+        tar_basename += '-' + suffix
+    tar_basename += '-repack'
+    install_dir = 'rustc'
+    subprocess.check_call(['rm', '-rf', install_dir])
+    install(os.path.basename(rustc['url']), install_dir)
+    install(os.path.basename(cargo['url']), install_dir)
+    for std in stds:
+        install(os.path.basename(std['url']), install_dir)
+        pass
+
+    log('Tarring %s...' % tar_basename)
+    tar_options, tar_ext = tar_for_host(host)
+    subprocess.check_call(
+        ['tar', tar_options, tar_basename + tar_ext, install_dir])
+    subprocess.check_call(['rm', '-rf', install_dir])
+
+    upload_dir = os.environ.get('UPLOAD_DIR')
+    if upload_dir:
+        # Create the upload directory if it doesn't exist.
+        try:
+            log('Creating upload directory in %s...' % os.path.abspath(upload_dir))
+            os.mkdir(upload_dir)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+        # Move the tarball to the output directory for upload.
+        log('Moving %s to the upload directory...' % tar_basename)
+        subprocess.check_call(['mv', tar_basename + tar_ext, upload_dir])
+
+
+def repack_cargo(host, channel='nightly'):
+    log('Repacking cargo for %s...' % host)
+    # Cargo doesn't seem to have a .toml manifest.
+    base_url = 'https://static.rust-lang.org/cargo-dist/'
+    req = requests.get(os.path.join(base_url, 'channel-cargo-' + channel))
+    req.raise_for_status()
+    file = ''
+    for line in req.iter_lines():
+        if line.find(host) != -1:
+            file = line.strip()
+    if not file:
+        log('No manifest entry for %s!' % host)
+        return
+    manifest = {
+        'date': req.headers['Last-Modified'],
+        'pkg': {
+            'cargo': {
+                'version': channel,
+                'target': {
+                    host: {
+                        'url': os.path.join(base_url, file),
+                        'hash': None,
+                        'available': True,
+                    },
+                },
+            },
+        },
+    }
+    log('Using manifest for cargo %s.' % channel)
+    log('Fetching packages...')
+    cargo = fetch_package(manifest, 'cargo', host)
+    log('Installing packages...')
+    install_dir = 'cargo'
+    subprocess.check_call(['rm', '-rf', install_dir])
+    install(os.path.basename(cargo['url']), install_dir)
+    tar_basename = 'cargo-%s-repack' % host
+    log('Tarring %s...' % tar_basename)
+    tar_options, tar_ext = tar_for_host(host)
+    subprocess.check_call(
+        ['tar', tar_options, tar_basename + tar_ext, install_dir])
+    subprocess.check_call(['rm', '-rf', install_dir])
+
+
+def expand_platform(name):
+    '''Expand a shortcut name to a full Rust platform string.'''
+    platforms = {
+        'android': "armv7-linux-androideabi",
+        'android_x86': "i686-linux-android",
+        'android_aarch64': "aarch64-linux-android",
+        'linux64': "x86_64-unknown-linux-gnu",
+        'linux32': "i686-unknown-linux-gnu",
+        'mac': "x86_64-apple-darwin",
+        'macos': "x86_64-apple-darwin",
+        'mac64': "x86_64-apple-darwin",
+        'mac32': "i686-apple-darwin",
+        'win64': "x86_64-pc-windows-msvc",
+        'win32': "i686-pc-windows-msvc",
+        'mingw32': "i686-pc-windows-gnu",
+    }
+    if name in platforms.keys():
+        # Return the matching platform string.
+        return platforms[name]
+    else:
+        # No match; just pass through the unknown name.
+        return name
+
+
+def args():
+    '''Read command line arguments and return options.'''
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--channel',
+                        help='Release channel to use:'
+                             ' stable, beta, or nightly',
+                        default='stable')
+    parser.add_argument('--cargo-channel',
+                        help='Release channel to use for cargo:'
+                             ' stable, beta, or nightly.'
+                             ' Defaults to the same as --channel.')
+    parser.add_argument('--host',
+                        help='Host platform for the toolchain executable:'
+                             ' e.g. linux64 or aarch64-linux-android.'
+                             ' Defaults to linux64.')
+    parser.add_argument('--target', dest='targets', action='append', default=[],
+                        help='Additional target platform to support:'
+                             ' e.g. linux32 or i686-pc-windows-gnu.'
+                             ' can be given more than once.')
+    parser.add_argument('--suffix',
+                        help='suffix to append to the tarball filename.'
+                             ' Useful for distinguising different target combinations.')
+    args = parser.parse_args()
+    if not args.cargo_channel:
+        args.cargo_channel = args.channel
+    if not args.host:
+        args.host = 'linux64'
+    args.host = expand_platform(args.host)
+    args.targets = map(expand_platform, args.targets)
+
+    return args
+
+
+if __name__ == '__main__':
+    args = vars(args())
+    setup_gpg()
+    repack(**args)