testing: create native Docker images for test environment (Bug 1598958) r=zeid
authorConnor Sheehan <sheehan@mozilla.com>
Fri, 15 May 2020 16:39:21 +0000
changeset 7550 d8c862917720bcc9183c8ada458d49a775dd0ab4
parent 7549 fc52de3babe0385d5f13d958c60c95a23970c4f5
child 7551 86e7f508187737d570b9e5465fcbbdd153be9fb2
push id3754
push usercosheehan@mozilla.com
push dateFri, 15 May 2020 16:39:56 +0000
treeherderversion-control-tools@d8c862917720 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerszeid
bugs1598958
testing: create native Docker images for test environment (Bug 1598958) r=zeid This commit converts the test runner Docker image into a Py2 only image and a Py3 only image. We introduce an `/app` directory and put the source code and virtualenv in separate directories (`/app/vct` and `/app/venv` respectively). We set `PYTHONUNBUFFERED=1` instead of manually modifying `sys.stdout` and remove the `NO_DOCKER=1` environment variable. We use separate cache volumes for each new image to avoid package problems in each environment. After moving the virtualenv out of the `/vct/venv` directory, a few hard-coded assumptions about the location of the test files need to be updated. We do so in this commit since it is assumed that running the tests outside of the Docker environment is no longer supported after this change. We also add several new commands to `run` that alias common operations. Differential Revision: https://phabricator.services.mozilla.com/D74767
d0cker
docker-compose.yml
hgext/hgmo/tests/test-cinnabarclone-pull.t
hgext/mozext/tests/test-mozext.t
pylib/mercurial-support/run-tests.py
run
run-tests
testing/docker/test-runner-py2/Dockerfile
testing/docker/test-runner/Dockerfile
testing/vcttesting/environment.py
--- a/d0cker
+++ b/d0cker
@@ -17,19 +17,16 @@ def main(args):
     m = Mach(os.getcwd())
     m.define_category('docker', 'Docker',
         'Common actions involving Docker')
     import vcttesting.docker_mach_commands
 
     return m.run(args)
 
 if __name__ == '__main__':
-    # Unbuffer stdout.
-    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
-
     if 'VIRTUAL_ENV' not in os.environ:
         activate = os.path.join(HERE, 'venv', 'bin', 'activate_this.py')
         with open(activate) as f:
             exec(f.read(), dict(__file__=activate))
         sys.executable = os.path.join(HERE, 'venv', 'bin', 'python')
         os.environ['VIRTUAL_ENV'] = os.path.join(HERE, 'venv')
 
     sys.exit(main(sys.argv[1:]))
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -27,32 +27,56 @@
 # The containers for docker tests can be built by running:
 # `$ docker-compose run --rm test-runner ./d0cker build-all`
 #
 # The full test-suite with docker tests can be run with:
 # `$ docker-compose run --rm test-runner /vct/run-tests`
 
 version: '3'
 services:
+  test-runner-py2:
+    build:
+      context: ./
+      dockerfile: ./testing/docker/test-runner-py2/Dockerfile
+      args:
+        USER_ID: ${UID}
+        GROUP_ID: ${GID}
+        DOCKER_GID: ${DOCKER_GID}
+    image: vct-test-runner-py2:built
+    command: ["/app/vct/run-tests", "--no-docker"]
+    network_mode: "host"
+    user: vct
+    environment:
+      - PYTHONUNBUFFERED=1
+      - USER=vct
+    volumes:
+      - ./:/app/vct
+      - test_runner_venv_py2:/app/venv
+      - test_runner_cache_py2:/app/vct/.cache
+      - /var/run/docker.sock:/var/run/docker.sock
+
   test-runner:
     build:
       context: ./
       dockerfile: ./testing/docker/test-runner/Dockerfile
       args:
         USER_ID: ${UID}
         GROUP_ID: ${GID}
         DOCKER_GID: ${DOCKER_GID}
     image: vct-test-runner:built
-    command: ["/vct/run-tests", "--no-docker"]
+    command: ["/app/vct/run-tests", "--no-docker"]
     network_mode: "host"
     user: vct
     environment:
-      - NO_DOCKER=1
+      - PYTHONUNBUFFERED=1
       - USER=vct
     volumes:
-      - ./:/vct
-      - test_runner_venv:/vct/venv
-      - test_runner_cache:/vct/.cache
+      - ./:/app/vct
+      - test_runner_venv:/app/venv
+      - test_runner_cache:/app/vct/.cache
       - /var/run/docker.sock:/var/run/docker.sock
 
 volumes:
+  test_runner_venv_py2:
   test_runner_venv:
+  test_runner_cache_py2:
   test_runner_cache:
+
--- a/hgext/hgmo/tests/test-cinnabarclone-pull.t
+++ b/hgext/hgmo/tests/test-cinnabarclone-pull.t
@@ -1,14 +1,14 @@
   $ . $TESTDIR/hgext/hgmo/tests/helpers.sh
 
   $ cat >> $HGRCPATH << EOF
   > [extensions]
   > hgmo = $TESTDIR/hgext/hgmo
-  > cinnabarclone = $TESTDIR/venv/git-cinnabar/mercurial/cinnabarclone.py
+  > cinnabarclone = /app/venv/git-cinnabar/mercurial/cinnabarclone.py
   > EOF
 
   $ startserver
 
   $ cd server
   $ touch foo
   $ hg -q commit -A -m initial
   $ cd ..
--- a/hgext/mozext/tests/test-mozext.t
+++ b/hgext/mozext/tests/test-mozext.t
@@ -6,17 +6,17 @@
   > mozext = $TESTDIR/hgext/mozext
   > localmozrepo = $TESTDIR/testing/local-mozilla-repos.py
   > 
   > [localmozrepo]
   > readuri = http://localhost:$HGPORT/
   > writeuri = ssh://user@dummy/$TESTTMP/
   > EOF
 
-  $ export PUSHLOGHG=$TESTDIR/venv/mercurials/5.3.2/bin/hg
+  $ export PUSHLOGHG=/app/venv/mercurials/5.3.2/bin/hg
 
   $ export USER=hguser
   $ hg init mozilla-central
   $ cd mozilla-central
   $ cat > .hg/hgrc << EOF
   > [extensions]
   > pushlog = $TESTDIR/hgext/pushlog/__init__.py
   > pushlog-feed = $TESTDIR/hgext/pushlog/feed.py
--- a/pylib/mercurial-support/run-tests.py
+++ b/pylib/mercurial-support/run-tests.py
@@ -1668,17 +1668,17 @@ class TTest(Test):
         # TRACKING MOZ
         # use custom hghave and use instead of tdir
         hghave_path = os.path.join(REPO_ROOT, 'testing', 'hghave.py')
 
         # TODO do something smarter when all other uses of hghave are gone.
         runtestdir = osenvironb[b'RUNTESTDIR']
         tdir = runtestdir.replace(b'\\', b'/')
         proc = Popen4(
-            b'%s -c "%s %s"' % (self._shell, hghave_path.encode('utf-8'), allreqs),
+            b'%s %s' % (hghave_path.encode('utf-8'), allreqs),
             self._testtmp,
             0,
             self._getenv(),
         )
         stdout, stderr = proc.communicate()
         ret = proc.wait()
         if wifexited(ret):
             ret = os.WEXITSTATUS(ret)
--- a/run
+++ b/run
@@ -55,19 +55,24 @@ def env_cmd():
     )
 
     print(env_content)
     return 0
 
 
 COMMANDS = {
     "build": ("docker-compose", "build"),
+    "d0cker": ("docker-compose", "run", "--rm", "test-runner", "/vct/d0cker"),
     "env": env_cmd,
-    "tests": ("docker-compose", "run", "--rm", "test-runner", "/vct/run-tests"),
     "shell": ("docker-compose", "run", "--rm", "test-runner", "/bin/bash"),
+    "tests": (
+        "docker-compose", "run", "--rm", "test-runner", "/app/vct/run-tests"),
+    "shell-py2": ("docker-compose", "run", "--rm", "test-runner-py2", "/bin/bash"),
+    "tests-py2": (
+        "docker-compose", "run", "--rm", "test-runner-py2", "/app/vct/run-tests"),
 }
 
 
 def main(args):
     if not args or args[0] not in COMMANDS:
         print("./run [command]\n")
         print("Possible Commands:")
         for cmd in COMMANDS:
--- a/run-tests
+++ b/run-tests
@@ -23,17 +23,17 @@ RUNTESTS = os.path.join(HERE, 'pylib', '
 
 def try_find_hg(with_hg):
     """Attempt to find `with_hg` in the known install directory for Mercurial
     versions.
 
     Args:
       with_hg (str): a Mercurial version string. Example: `5.3.2`
     """
-    hg = os.path.join(HERE, 'venv', 'mercurials', with_hg, 'bin', 'hg')
+    hg = os.path.join('/app', 'venv', 'mercurials', with_hg, 'bin', 'hg')
 
     if not os.path.exists(hg):
         raise ValueError("Couldn't find hg version {} at a known path".format(with_hg))
     
     return hg
 
 
 if __name__ == '__main__':
copy from testing/docker/test-runner/Dockerfile
copy to testing/docker/test-runner-py2/Dockerfile
--- a/testing/docker/test-runner/Dockerfile
+++ b/testing/docker/test-runner-py2/Dockerfile
@@ -8,18 +8,16 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -yq \
     docker \
     docker-compose \
     libldap2-dev \
     libsasl2-dev \
     libssl-dev \
     python \
     python-dev \
-    python3 \
-    python3-dev \
     rsync \
     sqlite3
 
 ARG USER_ID
 ARG GROUP_ID
 ARG DOCKER_GID
 RUN echo 'EXTRA_GROUPS="docker host-docker"' >> /etc/adduser.conf
 RUN addgroup --gid ${GROUP_ID} vct \
@@ -28,17 +26,63 @@ RUN addgroup --gid ${GROUP_ID} vct \
         --disabled-password \
         --uid ${USER_ID} \
         --gid ${GROUP_ID} \
         --add_extra_groups \
         --home /vct \
         --gecos "vct,,," \
         vct
 
-ADD ./ /vct/
-WORKDIR /vct/
+RUN mkdir /app
+RUN chown -R vct:vct /app
+
+# Add the scripts we need to create our venv
+ADD ./scripts/download-verify /app/vct/scripts/download-verify
+ADD ./testing/create-virtualenv /app/vct/testing/create-virtualenv
 
-RUN chown -R vct:vct /vct
+# Create Python 2 venv
+# Set VIRTUAL_ENV and PATH to replicate `source venv/bin/activate`
+ENV VIRTUAL_ENV=/app/venv
+RUN VENV=$VIRTUAL_ENV \
+    PYTHON_VERSION=python2.7 \
+    ROOT=/app/vct \
+    /app/vct/testing/create-virtualenv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# Install requirements for testing
+ADD test-requirements.txt /requirements.txt
+WORKDIR /app
+RUN pip install \
+    --upgrade \
+    --force-reinstall \
+    --require-hashes \
+    -r /requirements.txt
+
+ADD ./ /app/vct
+RUN chown -R vct:vct /app
 
 USER vct
-RUN NO_DOCKER=1 /vct/create-environment test
+
+# clone and install cinnabar into the venv
+RUN git clone \
+    --branch release \
+    https://github.com/glandium/git-cinnabar.git $VIRTUAL_ENV/git-cinnabar
+WORKDIR $VIRTUAL_ENV/git-cinnabar
+RUN make -j4 helper NO_OPENSSL=1 NO_GETTEXT=1
+
+WORKDIR /app
 
+# Install editable requirements py2/py3
+RUN pip install -e vct/pylib/Bugsy
+RUN pip install -e vct/pylib/mozansible
+RUN pip install -e vct/pylib/mozhg
+RUN pip install -e vct/pylib/mozhginfo
+RUN pip install -e vct/pylib/mozautomation
+RUN pip install -e vct/hgserver/hgmolib
+RUN pip install -e vct/pylib/vcsreplicator
+RUN pip install -e vct/hghooks
+RUN pip install -e vct/testing
+
+# Install Mercurials
+RUN python -m vcttesting.environment install-mercurials 2
+
+WORKDIR /app/vct
 CMD ["sh"]
--- a/testing/docker/test-runner/Dockerfile
+++ b/testing/docker/test-runner/Dockerfile
@@ -6,20 +6,21 @@ FROM buildpack-deps:latest
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -yq \
     docker \
     docker-compose \
     libldap2-dev \
     libsasl2-dev \
     libssl-dev \
-    python \
-    python-dev \
     python3 \
     python3-dev \
+    python3-pip \
+    python3-venv \
+    python3-wheel \
     rsync \
     sqlite3
 
 ARG USER_ID
 ARG GROUP_ID
 ARG DOCKER_GID
 RUN echo 'EXTRA_GROUPS="docker host-docker"' >> /etc/adduser.conf
 RUN addgroup --gid ${GROUP_ID} vct \
@@ -28,17 +29,68 @@ RUN addgroup --gid ${GROUP_ID} vct \
         --disabled-password \
         --uid ${USER_ID} \
         --gid ${GROUP_ID} \
         --add_extra_groups \
         --home /vct \
         --gecos "vct,,," \
         vct
 
-ADD ./ /vct/
-WORKDIR /vct/
+RUN mkdir /app
+RUN chown -R vct:vct /app
+
+
+# For testing older versions of Mercurial against Python 3
+# TODO remove once Mercurial 5.1 is not supported
+ENV HGPYTHON3=1
+
+# Create Python 3 venv
+# Set `VIRTUAL_ENV` and `PATH` to replicate `source venv/bin/activate`
+# Install `wheel` since it seems to be missing
+ENV VIRTUAL_ENV=/app/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN pip install wheel
 
-RUN chown -R vct:vct /vct
+
+# Install requirements for testing
+# Use /app as the workdir to the installed .egg-info
+# isn't mounted over by docker-compose
+ADD test-requirements-3.txt /requirements.txt
+WORKDIR /app
+RUN pip install \
+    --upgrade \
+    --force-reinstall \
+    --require-hashes \
+    -r /requirements.txt
+
+
+ADD ./ /app/vct/
+RUN chown -R vct:vct /app
 
 USER vct
-RUN NO_DOCKER=1 /vct/create-environment test
+
+# Install editable requirements py2/py3
+RUN pip install -e vct/pylib/Bugsy
+RUN pip install -e vct/pylib/mozansible
+RUN pip install -e vct/pylib/mozhg
+RUN pip install -e vct/pylib/mozhginfo
+RUN pip install -e vct/pylib/mozautomation
+RUN pip install -e vct/hgserver/hgmolib
+RUN pip install -e vct/pylib/vcsreplicator
+RUN pip install -e vct/hghooks
+RUN pip install -e vct/testing
+
+# clone and install cinnabar
+RUN git clone \
+    --branch release \
+    https://github.com/glandium/git-cinnabar.git /app/venv/git-cinnabar
+WORKDIR /app/venv/git-cinnabar
+RUN make -j4 helper NO_OPENSSL=1 NO_GETTEXT=1
+
+WORKDIR /app/vct
+
+# Install Mercurials
+RUN python -m vcttesting.environment install-mercurials 3
+
+VOLUME /app/vct
 
 CMD ["sh"]
--- a/testing/vcttesting/environment.py
+++ b/testing/vcttesting/environment.py
@@ -122,17 +122,17 @@ def install_mercurials(venv, hg='hg', py
     if not py3:
         VERSIONS.extend([
             '4.8.2',
             '4.9.1',
             '5.0.1',
             '5.1.2',
         ])
 
-    hg_dir = os.path.join(ROOT, 'venv', 'hg')
+    hg_dir = os.path.join("/app", "venv", "hg")
     mercurials = os.path.join(venv['path'], 'mercurials')
 
     # Setting HGRCPATH to an empty value stops the global and user hgrc from
     # being loaded. These could interfere with behavior we expect from
     # vanilla Mercurial.
     hg_env = dict(os.environ)
     hg_env['HGRCPATH'] = ''
     hg_env['HGPYTHON3'] = '1'
@@ -347,15 +347,15 @@ def create_global():
 
 if __name__ == '__main__':
     import sys
 
     if sys.argv[1] != 'install-mercurials':
         sys.exit(1)
 
     venv = {
-        'path': os.path.join(ROOT, 'venv'),
-        'python': os.path.join(ROOT, 'venv', 'bin', 'python'),
+        'path': os.path.join('/app', 'venv'),
+        'python': os.path.join('/app', 'venv', 'bin', 'python'),
     }
 
     py3 = sys.argv[2] == '3'
     install_mercurials(venv, hg='hg', py3=py3)
     sys.exit(0)