Bug 1616925 - Support a taskcluster-based ssh key for fetch jobs r=tomprince
☠☠ backed out by e1cfb4c47f2c ☠ ☠
authorTom Ritter <tom@mozilla.com>
Mon, 03 Aug 2020 15:33:01 +0000
changeset 3102965 359f9a3acc75b37125b90e69e93c8318c33b4ee3
parent 3102964 f39ddb6e8d4c93783a7dcda28d198635f4573493
child 3102966 d7849eabe2b829d17b792d655e3868ddca7b2efc
push id578529
push userreviewbot
push dateTue, 04 Aug 2020 02:36:56 +0000
treeherdertry@538c88f9f9fe [default view] [failures only]
reviewerstomprince
bugs1616925
milestone81.0a1
Bug 1616925 - Support a taskcluster-based ssh key for fetch jobs r=tomprince Differential Revision: https://phabricator.services.mozilla.com/D81448
taskcluster/docker/fetch/Dockerfile
taskcluster/scripts/misc/fetch-content
taskcluster/taskgraph/transforms/fetch.py
--- a/taskcluster/docker/fetch/Dockerfile
+++ b/taskcluster/docker/fetch/Dockerfile
@@ -13,16 +13,17 @@ WORKDIR /builds/worker
 ARG TASKCLUSTER_ROOT_URL
 ARG DOCKER_IMAGE_PACKAGES
 RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES && \
     apt-get update && \
     apt-get install \
       gnupg \
       bzip2 \
       git \
+      openssh-client \
       python3-requests \
       python3-zstandard \
       unzip
 
 # %include taskcluster/scripts/run-task
 ADD topsrcdir/taskcluster/scripts/run-task /builds/worker/bin/run-task
 
 # %include taskcluster/scripts/misc/fetch-content
--- a/taskcluster/scripts/misc/fetch-content
+++ b/taskcluster/scripts/misc/fetch-content
@@ -478,17 +478,17 @@ def fetch_urls(downloads):
         for download in downloads:
             fs.append(e.submit(fetch_and_extract, *download))
 
         for f in fs:
             f.result()
 
 
 def git_checkout_archive(dest_path: pathlib.Path, repo: str, commit: str,
-                         prefix=None):
+                         prefix=None, ssh_key=None):
     """Produce an archive of the files comprising a Git checkout."""
     dest_path.parent.mkdir(parents=True, exist_ok=True)
 
     if dest_path.suffixes[-2:] != ['.tar', '.zst']:
         raise Exception('Only producing .tar.zst archives is supported.')
 
     with tempfile.TemporaryDirectory() as td:
         temp_dir = pathlib.Path(td)
@@ -496,25 +496,43 @@ def git_checkout_archive(dest_path: path
         if not prefix:
             prefix = repo.rstrip('/').rsplit('/', 1)[-1]
         git_dir = temp_dir / prefix
 
         # This could be faster with a shallow clone. However, Git requires a ref
         # to initiate a clone. Since the commit-ish may not refer to a ref, we
         # simply perform a full clone followed by a checkout.
         print('cloning %s to %s' % (repo, git_dir))
+
+        env = os.environ.copy()
+        keypath = ""
+        if ssh_key:
+            taskcluster_secret_url = api(os.environ.get('TASKCLUSTER_PROXY_URL'), "secrets", "v1", "secret/{keypath}".format(keypath=ssh_key))
+            taskcluster_secret = b''.join(stream_download(taskcluster_secret_url))
+            taskcluster_secret = json.loads(taskcluster_secret)
+            sshkey = taskcluster_secret['secret']['ssh_privkey']
+
+            keypath = temp_dir.joinpath("ssh-key")
+            keypath.write_text(sshkey)
+            keypath.chmod(0o600)
+
+            env = {"GIT_SSH_COMMAND": "ssh -o 'StrictHostKeyChecking no' -i {keypath}".format(keypath=keypath)}
+
         subprocess.run(['git', 'clone', '-n', repo, str(git_dir)],
-                       check=True)
+                       check=True, env=env)
 
         subprocess.run(['git', 'checkout', commit],
                        cwd=str(git_dir), check=True)
 
         subprocess.run(['git', 'submodule', 'update', '--init'],
                        cwd=str(git_dir), check=True)
 
+        if keypath:
+            os.remove(keypath)
+
         print('creating archive %s of commit %s' % (dest_path, commit))
         proc = subprocess.Popen([
             'tar', 'cf', '-', '--exclude=.git', '-C', str(temp_dir), prefix,
         ], stdout=subprocess.PIPE)
 
         with rename_after_close(dest_path, 'wb') as out:
             ctx = ZstdCompressor()
             ctx.copy_stream(proc.stdout, out)
@@ -522,17 +540,17 @@ def git_checkout_archive(dest_path: path
         proc.wait()
 
 
 def command_git_checkout_archive(args):
     dest = pathlib.Path(args.dest)
 
     try:
         git_checkout_archive(dest, args.repo, args.commit,
-                             prefix=args.path_prefix)
+                             prefix=args.path_prefix, ssh_key=args.ssh_key_secret)
     except Exception:
         try:
             dest.unlink()
         except FileNotFoundError:
             pass
 
         raise
 
@@ -635,16 +653,18 @@ def main():
     git_checkout.add_argument('--path-prefix',
                               help='Prefix for paths in produced archive')
     git_checkout.add_argument('repo',
                               help='URL to Git repository to be cloned')
     git_checkout.add_argument('commit',
                               help='Git commit to check out')
     git_checkout.add_argument('dest',
                               help='Destination path of archive')
+    git_checkout.add_argument('--ssh-key-secret',
+                              help='The scope path of the ssh key to used for checkout')
 
     url = subparsers.add_parser('static-url', help='Download a static URL')
     url.set_defaults(func=command_static_url)
     url.add_argument('--sha256', required=True,
                      help='SHA-256 of downloaded content')
     url.add_argument('--size', required=True, type=int,
                      help='Size of downloaded content, in bytes')
     url.add_argument('--gpg-sig-url',
--- a/taskcluster/taskgraph/transforms/fetch.py
+++ b/taskcluster/taskgraph/transforms/fetch.py
@@ -178,16 +178,20 @@ def make_task(config, jobs):
                 'artifacts': [{
                     'type': 'directory',
                     'name': artifact_prefix,
                     'path': '/builds/worker/artifacts',
                 }],
             },
         }
 
+        if job.get('secret', None):
+            task['scopes'] = ["secrets:get:" + job.get('secret')]
+            task['worker']['taskcluster-proxy'] = True
+
         if not taskgraph.fast:
             cache_name = task['label'].replace('{}-'.format(config.kind), '', 1)
 
             # This adds the level to the index path automatically.
             add_optimization(
                 config,
                 task,
                 cache_type=CACHE_TYPE,
@@ -290,16 +294,21 @@ def create_fetch_url_task(config, name, 
     }
 
 
 @fetch_builder('git', schema={
     Required('repo'): text_type,
     Required('revision'): text_type,
     Optional('artifact-name'): text_type,
     Optional('path-prefix'): text_type,
+    # ssh-key is a taskcluster secret path (e.g. project/civet/github-deploy-key)
+    # In the secret dictionary, the key should be specified as
+    #  "ssh_privkey": "-----BEGIN OPENSSH PRIVATE KEY-----\nkfksnb3jc..."
+    # n.b. The OpenSSH private key file format requires a newline at the end of the file.
+    Optional('ssh-key'): text_type,
 })
 def create_git_fetch_task(config, name, fetch):
     path_prefix = fetch.get('path-prefix')
     if not path_prefix:
         path_prefix = fetch['repo'].rstrip('/').rsplit('/', 1)[-1]
     artifact_name = fetch.get('artifact-name')
     if not artifact_name:
         artifact_name = '{}.tar.zst'.format(path_prefix)
@@ -313,20 +322,26 @@ def create_git_fetch_task(config, name, 
         'git-checkout-archive',
         '--path-prefix',
         path_prefix,
         fetch['repo'],
         fetch['revision'],
         '/builds/worker/artifacts/%s' % artifact_name,
     ]
 
+    ssh_key = fetch.get('ssh-key')
+    if ssh_key:
+        args.append('--ssh-key-secret')
+        args.append(ssh_key)
+
     return {
         'command': args,
         'artifact_name': artifact_name,
         'digest_data': [fetch['revision'], path_prefix, artifact_name],
+        'secret': ssh_key
     }
 
 
 @fetch_builder('chromium-fetch', schema={
     Required('script'): text_type,
 
     # Platform type for chromium build
     Required('platform'): text_type,