testing: add docker-compose env for running tests (Bug 1598958). r=sheehan
authorSteven MacLeod <steven@smacleod.ca>
Tue, 26 Nov 2019 18:31:18 +0000
changeset 7320 5bfcd45f5eaaa5c7410226cc0f4126adfae47564
parent 7319 f824da4501069359b208312172f071a0b5927942
child 7321 bff509d5446a40b799287191efa9a2e1f49dc787
push id3621
push usercosheehan@mozilla.com
push dateTue, 26 Nov 2019 18:31:42 +0000
treeherderversion-control-tools@5bfcd45f5eaa [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssheehan
bugs1598958
testing: add docker-compose env for running tests (Bug 1598958). r=sheehan This introduces a docker-compose.yml and a test-runner docker image for executing the tests. This image is built with everything needed to run the non-docker tests. when run through docker-compose the container makes use of the host docker daemon to build and run the full docker tests. Currently the container also requires using host networking so that it can access the containers through localhost. This means only a linux host is supported for running these tests, as docker for windows and docker for mac os do not support this feature. Eventually as we re-implement the docker test containers using docker files, and switch the docker-compose networking we'll be able to remove the requirement on host networking. Differential Revision: https://phabricator.services.mozilla.com/D54423
.dockerignore
.hgignore
ansible/roles/docker-hg-web/files/enable-fastannotate
ansible/roles/hg-web/files/make_user_wsgi_dirs.py
ansible/roles/hg-web/tasks/main.yml
create-test-environment
docker-compose.yml
pylib/mercurial-support/run-tests.py
scripts/update-repo-mtimes
scripts/upgrade-hg-repos.py
testing/docker/test-runner/Dockerfile
new file mode 100644
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,1 @@
+**/.testtimes
--- a/.hgignore
+++ b/.hgignore
@@ -1,16 +1,18 @@
 coverage/
 docs/_build/
 extra-files/
 testing/bmoserver/.vagrant
 testing/puppet/files/Mozilla-Bugzilla-Public*
 testing/unifiedserver/.vagrant
 venv/
+\.bash_history$
 \.egg-info/
+\.env$
 \.pyc$
 \.pyo$
 \.swp$
 \.t.err$
 \.idea/*
 ~$
 \.vagrant/*
 \.vagrantuser$
--- a/ansible/roles/docker-hg-web/files/enable-fastannotate
+++ b/ansible/roles/docker-hg-web/files/enable-fastannotate
@@ -1,9 +1,9 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
 import sys
 
 repo = sys.argv[1]
--- a/ansible/roles/hg-web/files/make_user_wsgi_dirs.py
+++ b/ansible/roles/hg-web/files/make_user_wsgi_dirs.py
@@ -1,9 +1,9 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
 import pathlib
 
 ROOT = '/repo/hg/mozilla/users'
--- a/ansible/roles/hg-web/tasks/main.yml
+++ b/ansible/roles/hg-web/tasks/main.yml
@@ -166,17 +166,17 @@
 
 - name: create virtualenv for tools
   include: ../../../tasks/virtualenv.yml
            venv=/var/hg/venv_tools
            requirements=../roles/hg-web/files/requirements-tools.txt
 
 - name: create virtualenv for Python 3 tools
   include: ../../../tasks/virtualenv3.yml
-           python=/usr/bin/python3.6
+           python=/usr/bin/python3
            venv=/var/hg/venv_tools_py3
            requirements=../roles/hg-web/files/requirements-tools-py3.txt
 
 # mod_wsgi is a Python package. But pip-compile doesn't like it for some reason.
 # We install it into the virtualenv as a one-off.
 - name: download mod_wsgi
   get_url: url=https://s3-us-west-2.amazonaws.com/moz-packages/mod_wsgi-4.5.24.tar.gz
            dest=/var/tmp/mod_wsgi-4.5.24.tar.gz
--- a/create-test-environment
+++ b/create-test-environment
@@ -23,17 +23,17 @@ do
 
     # Set environment variables according to Python version being installed
     if [[ ${PYTHON_VERSION_NUMBER} -eq 2 ]]
     then
         VENV=${ROOT}/venv
         PYTHON_VERSION=python2.7
     else
         VENV=${ROOT}/venv/py3
-        PYTHON_VERSION=python3.6
+        PYTHON_VERSION=python3
     fi
 
     if [ ! -d ${VENV}/lib ]; then
       # There is a bug in pycrypto where it tries to invoke `configure` instead
       # of a path qualified configure. So if there is a "configure" on $PATH
       # it will invoke the wrong one.
       configure=which configure &>/dev/null || true
       if [ "x" != "x${configure}" ]; then
new file mode 100644
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,61 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# To use docker-compose for running the test environment you must
+# create a `.env` file with the following contents:
+#     UID=<uid-of-your-host-machine-user>
+#     GID=<gid-of-your-host-machine-user>
+#     DOCKER_GID=<gid-of-the-host-machine-docker-group>
+#
+# The values for UID, GID, and DOCKER_GID can be found using the
+# following respectively.
+#     $ id -u <username>
+#     $ id -g <username>
+#     $ echo "Group docker with GID="$(cut -d: -f3 < <(getent group docker))""
+#
+# Here is an example of my `.env` file at time of writing:
+#     UID=1000
+#     GID=1000
+#     DOCKER_GID=975
+#
+# The environment may be built by running:
+# `$ docker-compose build`
+#
+# Non-docker tests may be run with:
+# `$ docker-compose run --rm test-runner`
+#
+# The test-runner docker image runs ./create-test-environment
+# with NO_DOCKER=1, since that relys on the host docker daemon.
+# The containers for docker tests can be built by running:
+# `$ docker-compose run --rm test-runner ./d0cker build-all`
+#
+# The full test-suite with docker tests can be run with:
+# `$ docker-compose run --rm test-runner /vct/run-tests`
+
+version: '3'
+services:
+  test-runner:
+    build:
+      context: ./
+      dockerfile: ./testing/docker/test-runner/Dockerfile
+      args:
+        USER_ID: ${UID}
+        GROUP_ID: ${GID}
+        DOCKER_GID: ${DOCKER_GID}
+    image: vct-test-runner:built
+    command: ["/vct/run-tests", "--no-docker"]
+    network_mode: "host"
+    user: vct
+    environment:
+      - NO_DOCKER=1
+      - USER=vct
+    volumes:
+      - ./:/vct
+      - test_runner_venv:/vct/venv
+      - test_runner_cache:/vct/.cache
+      - /var/run/docker.sock:/var/run/docker.sock
+
+volumes:
+  test_runner_venv:
+  test_runner_cache:
--- a/pylib/mercurial-support/run-tests.py
+++ b/pylib/mercurial-support/run-tests.py
@@ -1026,20 +1026,25 @@ class Test(unittest.TestCase):
             self._portmap(1),
             self._portmap(2),
             self._portmap(3),
             self._portmap(4),
             self._portmap(5),
             self._portmap(6),
             self._portmap(7),
             self._portmap(8),
-            (br'\b%s\b' % re.escape(self._localhostname()), br'$LOCALHOST'),
             (br'([^0-9])%s' % re.escape(self._localip()), br'\1$LOCALIP'),
             (br'\bHG_TXNID=TXN:[a-f0-9]{40}\b', br'HG_TXNID=TXN:$ID$'),
-            ]
+        ] + [
+            (br'\b%s\b' % re.escape(hostname), br'$LOCALHOST')
+            for hostname in
+            # Sort by most '.' characters first so that we properly handle
+            # aliases that are substrings.
+            sorted(self._localhostnames(), key=lambda x: x.count(b'.'), reverse=True)
+        ]
         r.append((self._escapepath(self._testtmp), b'$TESTTMP'))
 
         # TRACKING MOZ - docker stuff
         if 'DOCKER_HOSTNAME' in os.environ:
             r.append((re.escape(os.environ['DOCKER_HOSTNAME'].encode('utf-8')),
                       b'$DOCKER_HOSTNAME'))
 
         replacementfile = os.path.join(self._testdir, b'common-pattern.py')
@@ -1071,17 +1076,22 @@ class Test(unittest.TestCase):
     def _localip(self):
         if self._useipv6:
             return b'::1'
         else:
             return b'127.0.0.1'
 
     # Tracking MOZ - add _localhostname for cross-platform compat
     def _localhostname(self):
-        return b'%s' % socket.getfqdn(self._localip())
+        return self._localhostnames()[0]
+
+    # Tracking MOZ - add _localhostnames for cross-platform compat
+    def _localhostnames(self):
+        hostname, aliaslist, _ = socket.gethostbyaddr(self._localip())
+        return [b'%s' % hostname] + [b'%s' % alias for alias in aliaslist]
 
     def _genrestoreenv(self, testenv):
         """Generate a script that can be used by tests to restore the original
         environment."""
         # Put the restoreenv script inside self._threadtmp
         scriptpath = os.path.join(self._threadtmp, b'restoreenv.sh')
         testenv['HGTEST_RESTOREENV'] = _strpath(scriptpath)
 
--- a/scripts/update-repo-mtimes
+++ b/scripts/update-repo-mtimes
@@ -1,9 +1,9 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import argparse
 import concurrent.futures
 import os
 import pathlib
--- a/scripts/upgrade-hg-repos.py
+++ b/scripts/upgrade-hg-repos.py
@@ -1,9 +1,9 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 """Upgrade Mercurial repositories.
 
 Given a list of repository paths on stdin, run `hg debugupgraderepo` on those
 repositories.
new file mode 100644
--- /dev/null
+++ b/testing/docker/test-runner/Dockerfile
@@ -0,0 +1,44 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+FROM buildpack-deps:latest
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -yq \
+    docker \
+    docker-compose \
+    libldap2-dev \
+    libsasl2-dev \
+    libssl-dev \
+    python \
+    python-dev \
+    python3 \
+    python3-dev \
+    rsync \
+    sqlite3
+
+ARG USER_ID
+ARG GROUP_ID
+ARG DOCKER_GID
+RUN echo 'EXTRA_GROUPS="docker host-docker"' >> /etc/adduser.conf
+RUN addgroup --gid ${GROUP_ID} vct \
+    && addgroup --gid ${DOCKER_GID} host-docker \
+    && adduser \
+        --disabled-password \
+        --uid ${USER_ID} \
+        --gid ${GROUP_ID} \
+        --add_extra_groups \
+        --home /vct \
+        --gecos "vct,,," \
+        vct
+
+ADD ./ /vct/
+WORKDIR /vct/
+
+RUN chown -R vct:vct /vct
+
+USER vct
+RUN NO_DOCKER=1 /vct/create-test-environment
+
+CMD ["sh"]