Merge back from production.
authorBen Hearsum <bhearsum@mozilla.com>
Mon, 04 Jun 2018 11:06:43 -0400
changeset 6615 cbdb17db2fa78db6595305a7c39825b504d2793a
parent 6612 e2bbbba1e62504794fba51e90876bfc0fbb7eb2b (current diff)
parent 6614 2f772867ed630d08feb343b0e8815895889b9a8e (diff)
child 6616 8b9852f23c90d0c00b5f4a25f145de3eca0c3ca9
push id5320
push userbhearsum@mozilla.com
push dateMon, 04 Jun 2018 17:20:36 +0000
Merge back from production.
--- a/modules/addon_scriptworker/files/requirements.txt
+++ b/modules/addon_scriptworker/files/requirements.txt
@@ -1,10 +1,11 @@
+# python_version: 36
 PyYAML==3.12
-addonscript==1.0
+addonscript==1.0  # puppet: nodownload
 aiohttp==3.2.1
 arrow==0.12.1
 async_timeout==3.0.0
 attrs==18.1.0
 certifi==2018.4.16
 chardet==3.0.4
 defusedxml==0.5.0
 dictdiffer==0.7.1
--- a/modules/aws_manager/files/requirements.txt
+++ b/modules/aws_manager/files/requirements.txt
@@ -1,17 +1,18 @@
+# python_version: 27
 # fabric 2 contains api breaking changes.
-Fabric==1.14.0 # pyup: ignore
+Fabric==1.14.0  # pyup: ignore
 IPy==0.83
 MySQL-python==1.2.5
 SQLAlchemy==1.2.7
 argparse==1.4.0
 boto==2.48.0
 ecdsa==0.13
-invtool==4.4
+invtool==4.4  # puppet: nodownload
 iso8601==0.1.12
 paramiko==2.4.1
 pycrypto==2.6.1
 repoze.lru==0.7
 requests==2.18.4
 simplejson==3.15.0
 ssh==1.8.0
 wsgiref==0.1.2
--- a/modules/balrog_scriptworker/files/requirements-27.txt
+++ b/modules/balrog_scriptworker/files/requirements-27.txt
@@ -1,15 +1,16 @@
+# python_version: 27
 # install six before cryptography
 six==1.11.0
 arrow==0.12.1
 asn1crypto==0.24.0
 backports.functools-lru-cache==1.5
 backports.lzma==0.0.10
-balrogscript==3.1.0
+balrogscript==3.1.0  # puppet: nodownload
 certifi==2018.1.18
 cffi==1.11.4
 chardet==3.0.4
 click==6.7
 construct==2.9.27
 cryptography==2.1.4
 enum34==1.1.6
 functools32==3.2.3-2
--- a/modules/balrog_scriptworker/files/requirements-3.txt
+++ b/modules/balrog_scriptworker/files/requirements-3.txt
@@ -1,8 +1,9 @@
+# python_version: 36
 PyYAML==3.12
 aiohttp==3.2.1
 arrow==0.12.1
 async_timeout==3.0.0
 attrs==18.1.0
 certifi==2018.4.16
 chardet==3.0.4
 click==6.7
--- a/modules/beetmover_scriptworker/files/requirements.txt
+++ b/modules/beetmover_scriptworker/files/requirements.txt
@@ -1,16 +1,17 @@
+# python_version: 36
 Jinja2==2.10
 MarkupSafe==1.0
 PyYAML==3.12
 aiohttp==3.2.1
 arrow==0.12.1
 async_timeout==3.0.0
 attrs==18.1.0
-beetmoverscript==7.2.3
+beetmoverscript==7.2.3  # puppet: nodownload
 boto3==1.7.22
 botocore==1.10.22
 certifi==2018.4.16
 chardet==3.0.4
 click==6.7
 defusedxml==0.5.0
 dictdiffer==0.7.1
 docutils==0.14
--- a/modules/bouncer_check/files/requirements.txt
+++ b/modules/bouncer_check/files/requirements.txt
@@ -1,4 +1,5 @@
+# python_version: 27
 argparse==1.4.0
-nagios-tools==0.6
+nagios-tools==0.6  # puppet: nodownload
 nagiosplugin==1.2.4
 wsgiref==0.1.2
--- a/modules/bouncer_scriptworker/files/requirements.txt
+++ b/modules/bouncer_scriptworker/files/requirements.txt
@@ -1,13 +1,14 @@
+# python_version: 36
 PyYAML==3.12
 aiohttp==3.2.1
 arrow==0.12.1
 async_timeout==3.0.0
-bouncerscript==1.3.0
+bouncerscript==1.3.0  # puppet: nodownload
 attrs==18.1.0
 certifi==2018.4.16
 chardet==3.0.4
 defusedxml==0.5.0
 dictdiffer==0.7.1
 frozendict==1.2
 idna==2.6
 idna_ssl==1.0.1
--- a/modules/buildbot_bridge/files/requirements.txt
+++ b/modules/buildbot_bridge/files/requirements.txt
@@ -1,8 +1,9 @@
+# python_version: 27
 requests==2.18.4
 arrow==0.12.1
 taskcluster==3.0.1
 sqlalchemy==1.2.7
 kombu==4.2.0
 redo==1.6
 mysql-python==1.2.5
 amqp==2.2.2
--- a/modules/buildbot_bridge2/files/requirements.txt
+++ b/modules/buildbot_bridge2/files/requirements.txt
@@ -1,18 +1,19 @@
+# python_version: 36
 aiohttp==1.3.5
 appdirs==1.4.3
 arrow==0.10.0
 async-timeout==1.2.0
 chardet==2.3.0
 click==6.7
 jsonschema==2.6.0
 mohawk==0.3.4
 multidict==2.1.4
-mysql-connector-python==2.0.4
+mysql-connector-python==2.0.4  # puppet: nodownload - because there's no package for this version on pypi. remove me when we update it
 py==1.4.33
 pyparsing==2.2.0
 python-dateutil==2.6.0
 PyYAML==3.12
 Represent==1.5.1
 requests==2.13.0
 six==1.10.0
 slugid==1.0.7
--- a/modules/buildduty_tools/files/requirements.txt
+++ b/modules/buildduty_tools/files/requirements.txt
@@ -1,8 +1,9 @@
+# python_version: 27
 # install six before cryptography
 six==1.11.0
 Jinja2==2.10
 MarkupSafe==1.0
 Twisted==18.4.0
 argparse==1.4.0
 cffi==1.11.5
 cryptography==2.2.2
--- a/modules/buildmaster/files/db_maintenance_requirements.txt
+++ b/modules/buildmaster/files/db_maintenance_requirements.txt
@@ -1,2 +1,3 @@
+# python_version: 27
 SQLAlchemy==1.2.7
 MySQL-python==1.2.5
--- a/modules/buildmaster/files/queue_requirements.txt
+++ b/modules/buildmaster/files/queue_requirements.txt
@@ -1,7 +1,8 @@
-buildbot==0.8.4-pre-moz2 # pyup: ignore
+# python_version: 27
+buildbot==0.8.4-pre-moz2  # pyup: ignore, puppet: nodownload
 MozillaPulse==1.3
 amqp==2.2.2
 anyjson==0.3.3
 kombu==4.2.0
 pytz==2018.4
 vine==1.1.4
--- a/modules/buildslave/files/requirements.txt
+++ b/modules/buildslave/files/requirements.txt
@@ -1,5 +1,6 @@
+# python_version: 27
 # buildbot infrastructure is going away soon, and these packages are too risky to upgrade
-zope.interface==3.6.1 # pyup: ignore
-Twisted==10.2.0 # pyup: ignore
+zope.interface==3.6.1  # pyup: ignore
+Twisted==10.2.0  # pyup: ignore
 # this is required for some mozilla custom classes
-simplejson==2.1.3 # pyup: ignore
+simplejson==2.1.3  # pyup: ignore
--- a/modules/cleanslate/files/requirements.txt
+++ b/modules/cleanslate/files/requirements.txt
@@ -1,1 +1,2 @@
-cleanslate==1.3
+# python_version: 27
+cleanslate==1.3  # puppet: nodownload
--- a/modules/cruncher/files/allthethings_requirements.txt
+++ b/modules/cruncher/files/allthethings_requirements.txt
@@ -1,14 +1,15 @@
+# python_version: 27
 # install six before cryptography
 six==1.11.0
 Jinja2==2.10
 MarkupSafe==1.0
 # doesn't work with our version of buildbot
-Twisted==10.1.0 # pyup: ignore
+Twisted==10.1.0  # pyup: ignore
 argparse==1.4.0
 cffi==1.11.5
 cryptography==2.2.2
 pyOpenSSL==18.0.0
 pyasn1==0.4.3
 pycparser==2.18
 pycrypto==2.6.1
 simplejson==3.15.0
--- a/modules/cruncher/files/slave_health_requirements.txt
+++ b/modules/cruncher/files/slave_health_requirements.txt
@@ -1,8 +1,9 @@
+# python_version: 27
 MySQL-python==1.2.5
 SQLAlchemy==1.2.7
 pytz==2018.4
 # Pinning to avoid investigating if an update would break
 # https://bugzilla.mozilla.org/show_bug.cgi?id=1289822#c7
 requests==2.18.4
 simplejson==3.15.0
 wsgiref==0.1.2
--- a/modules/puppetmaster/manifests/data.pp
+++ b/modules/puppetmaster/manifests/data.pp
@@ -84,9 +84,56 @@ class puppetmaster::data {
                 content => template("${module_name}/puppetmaster-upstream-rsync.sh.erb");
         }
     } else {
         file {
             [ $crontab, $script ]:
                 ensure => absent;
         }
     }
+
+    $python2_crontab = '/etc/cron.d/puppetmaster-pip2-download'
+    $python3_crontab = '/etc/cron.d/puppetmaster-pip3-download'
+    $python_script = '/etc/puppet/puppetmaster-pip-download.sh'
+    if ($puppetmaster::settings::is_distinguished) {
+        include packages::mozilla::python27
+        include packages::mozilla::python3
+        # To make pip shut up when downloading the mysql-python package
+        include packages::mysql
+        include packages::mysql_devel
+
+        # Due to https://github.com/pypa/pip/issues/5369, we have to run
+        # "pip download" for each target version of Python that we want
+        # packages for. When this bug is fixed, we should be able to use
+        # Python2 for all of the downloads, and pass "--python-version"
+        # to specify the target version.
+        python27::virtualenv {
+            $puppetmaster::settings::pip2_download_virtualenv:
+                python          => $packages::mozilla::python27::python,
+                rebuild_trigger => Class['packages::mozilla::python27'],
+                require         => Class['packages::mozilla::python27'];
+        }
+
+        python3::virtualenv {
+            $puppetmaster::settings::pip3_download_virtualenv:
+                python3         => $packages::mozilla::python3::python3,
+                rebuild_trigger => Class['packages::mozilla::python3'],
+                require         => Class['packages::mozilla::python3'];
+        }
+
+        file {
+            # We need to sync these at least as often as puppet manifests are updated - which is every 5 minutes.
+            $python2_crontab:
+                content => "*/5 * * * * root ${python_script} 27\n";
+            $python3_crontab:
+                content => "*/5 * * * * root ${python_script} 36\n";
+            $python_script:
+                mode    => '0755',
+                content => template("${module_name}/puppetmaster-pip-download.sh.erb");
+        }
+    } else {
+        file {
+            [ $python2_crontab, $python3_crontab, $python_script, $puppetmaster::settings::pip2_download_virtualenv,
+              $puppetmaster::settings::pip3_download_virtualenv ]:
+                ensure => absent;
+        }
+    }
 }
--- a/modules/puppetmaster/manifests/settings.pp
+++ b/modules/puppetmaster/manifests/settings.pp
@@ -1,19 +1,21 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 class puppetmaster::settings {
     include ::config
 
-    $data_root         = '/data'
-    $puppetmaster_root = '/var/lib/puppetmaster'
-    $puppetsync_home   = '/var/lib/puppetsync-home'
-    $deploy_dir        = '/var/lib/puppetmaster/deploy'
+    $data_root                = '/data'
+    $puppetmaster_root        = '/var/lib/puppetmaster'
+    $puppetsync_home          = '/var/lib/puppetsync-home'
+    $deploy_dir               = '/var/lib/puppetmaster/deploy'
+    $pip2_download_virtualenv = '/etc/puppet/pip2-venv'
+    $pip3_download_virtualenv = '/etc/puppet/pip3-venv'
 
     # how often to check and update the puppet manifests and files
     $puppet_check_interval_mins = 5
     $puppet_check_splay_secs    = 200
 
     # copy some useful values from config to this module
     $all_masters           = $::config::puppet_servers
     $distinguished_master  = $::config::distinguished_puppetmaster
new file mode 100644
--- /dev/null
+++ b/modules/puppetmaster/templates/puppetmaster-pip-download.sh.erb
@@ -0,0 +1,81 @@
+#! /bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+distinguished_master=<%= scope.lookupvar('puppetmaster::settings::distinguished_master') %>
+data_root=<%= scope.lookupvar('puppetmaster::settings::data_root') %>
+mailto=<%= scope.lookupvar('::config::puppet_notif_email') %>
+hostname=<%= scope.lookupvar('::fqdn') %>
+
+python_version=$1
+pip_binary=<%= scope.lookupvar('puppetmaster::settings::pip2_download_virtualenv') %>/bin/pip
+download_dir="${data_root}/python/packages"
+if [ "${python_version}" != "27" ]; then
+    pip_binary=<%= scope.lookupvar('puppetmaster::settings::pip3_download_virtualenv') %>/bin/pip
+    download_dir="${data_root}/python/packages-3.x"
+fi
+
+# make logfile that puppetsync can write to
+logfile=$(mktemp)
+
+# set the default subject line for our output email
+subject="[PuppetAagin Changes] Downloaded new python-${python_version} packages"
+
+cd /etc/puppet
+
+if [ -f ${logfile} ]; then
+    LOCKFILE="/dev/shm/puppetmaster-pypi-download-log-${python_version}"
+    if ! lockfile -r 0 "${LOCKFILE}" >> ${logfile} 2>&1; then
+        echo "${LOCKFILE} already exists; not running pip download" >> ${logfile}
+        subject="[PuppetAgain Errors] pip download already running on ${hostname}"
+    else
+        trap "rm -f ${LOCKFILE}; exit" SIGHUP SIGINT SIGTERM EXIT
+
+        # To minimize the number of times we need to call "pip download", we collect
+        # all of the unique dependencies from all of the requirements files first
+        # and then call "pip download" on each one. If we were to call "pip download"
+        # on each requirements files, many dependencies would be processed multiple times
+        # which is pretty expensive.
+        all_deps=""
+        for req_file in `find . -wholename "*files*requirements*.txt"`; do
+            # Skip requirements files that aren't for the Python version that we're
+            # operating on.
+            # xargs is to remove leading/trailing whitespace
+            req_python_version=$(grep python_version $req_file | cut -d: -f2 | xargs)
+            if [ "${python_version}" == "${req_python_version}" ]; then
+                while read dependency; do
+                    # Apparently this construct is bash for "does not contain substring"
+                    if [[ "${all_deps}" != *"${dependency}"* ]]; then
+                        all_deps="${all_deps} ${dependency}"
+                    fi
+                # read requirements file, filtering away any comments, and packages that
+                # are explicitly marked as something we shouldn't download (these are typically
+                # internal packages that don't exist on pypi).
+                done < <(cat $req_file | grep -v '^#' | grep -v 'puppet: nodownload' | sed -e 's/.\?#.*//')
+            fi
+        done
+        for dep in $all_deps; do
+            # -q to avoid sending unnecessary email when nothing is downloaded
+            # --cache-dir because pip doesn't cache in a way that can be shared by multiple python versions
+            # --no-deps beacuse we don't want to download implicit dependencies (we'd rather have errors, and make them explicit)
+            # --python-version to make sure we're getting the correct wheels
+            # -d to make sure the packages go to the right place
+            ${pip_binary} download -q --cache-dir=/root/.cache/pip-${python_version} --no-deps --python-version ${python_version} -d $download_dir $dep >> $logfile 2>&1
+        done
+    fi
+
+    # if anything was logged..
+
+    if [ -s ${logfile} ]; then
+        # if we're interactive, show the results
+        tty -s && cat ${logfile}
+
+        # email it
+        mail -s "${subject}" "${mailto}" < "${logfile}"
+    fi
+
+    rm ${logfile}
+else
+    echo "Cannot create log file for pip-download-${python_version} on ${hostname}." | mail -s "[PuppetAgain Errors] pip-download-${python_version} failed on ${hostname}" "${mailto}"
+fi
--- a/modules/pushapk_scriptworker/files/requirements.txt
+++ b/modules/pushapk_scriptworker/files/requirements.txt
@@ -1,8 +1,9 @@
+# python_version: 36
 # install six and setuptools before cryptography
 six==1.11.0
 PyYAML==3.12
 Pygments==2.2.0
 aiohttp==3.2.1
 androguard==3.2.0
 appnope==0.1.0
 arrow==0.12.1
@@ -60,9 +61,9 @@ slugid==1.0.7
 taskcluster==3.0.1
 traitlets==4.3.2
 uritemplate==3.0.0
 urllib3==1.22
 virtualenv==15.2.0
 voluptuous==0.11.1
 wcwidth==0.1.7
 yarl==1.2.4
-pushapkscript==0.7.0
+pushapkscript==0.7.0  # puppet: nodownload
--- a/modules/pushsnap_scriptworker/files/requirements.txt
+++ b/modules/pushsnap_scriptworker/files/requirements.txt
@@ -1,8 +1,9 @@
+# python_version: 36
 PyNaCl==1.2.1
 PyYAML==3.12
 aiohttp==3.2.1
 arrow==0.12.1
 async_timeout==3.0.0
 attrs==18.1.0
 certifi==2018.4.16
 cffi==1.11.5
@@ -35,9 +36,9 @@ scriptworker==12.0.1
 simplejson==3.15.0
 six==1.11.0
 slugid==1.0.7
 tabulate==0.8.2
 taskcluster==3.0.1
 urllib3==1.22
 virtualenv==15.2.0
 yarl==1.2.4
-pushsnapscript==0.2.1
+pushsnapscript==0.2.1  # puppet: nodownload
--- a/modules/releaserunner/files/requirements.txt
+++ b/modules/releaserunner/files/requirements.txt
@@ -1,21 +1,22 @@
+# python_version: 27
 # install six before cryptography
 six==1.11.0
 Fabric==2.0.1
 Jinja2==2.10
 PGPy==0.4.3
 PyHawk-with-a-single-extra-commit==0.1.5
 PyYAML==3.12
 SQLAlchemy==1.2.7
 Tempita==0.5.2
 # don't upgrade twisted because it doesn't work with our buildbot
-Twisted==12.3.0 # pyup: ignore
+Twisted==12.3.0  # pyup: ignore
 arrow==0.12.1
-buildbot==0.8.7p1 # pyup: ignore
+buildbot==0.8.7p1  # pyup: ignore, puppet: nodownload
 certifi==2018.4.16
 chunkify==1.2
 cryptography==2.2.2
 decorator==4.2.1
 ecdsa==0.13
 enum34==1.1.6
 futures==3.2.0
 mohawk==0.3.4
--- a/modules/releaserunner3/files/requirements.txt
+++ b/modules/releaserunner3/files/requirements.txt
@@ -1,11 +1,12 @@
+# python_version: 27
 PyYAML==3.12
 # don't upgrade twisted because it doesn't work with our buildbot
-Twisted==12.3.0 # pyup: ignore
+Twisted==12.3.0  # pyup: ignore
 certifi==2018.4.16
 chardet==3.0.4
 idna==2.6
 json-e==2.5.0
 mohawk==0.3.4
 requests==2.18.4
 simplejson==3.15.0
 six==1.11.0
--- a/modules/selfserve_agent/files/requirements.txt
+++ b/modules/selfserve_agent/files/requirements.txt
@@ -1,30 +1,31 @@
+# python_version: 27
 anyjson==0.3.3
 Beaker==1.9.1
 FormEncode==1.3.1
 Mako==1.0.7
 MarkupSafe==1.0
 MySQL-python==1.2.5
 Paste==2.0.3
 PasteDeploy==1.5.2
 PasteScript==2.0.2
 Pygments==2.2.0
 Pylons==1.0.3
 amqp==2.2.2
-buildapi==0.3.25
-buildbot==0.8.4-pre-moz2 # pyup: ignore
+buildapi==0.3.25  # puppet: nodownload
+buildbot==0.8.4-pre-moz2  # pyup: ignore, puppet: nodownload
 decorator==4.2.1
 #distribute==0.7.3
 kombu==4.2.0
 Routes==2.4.1
 SQLAlchemy==1.2.7
 Tempita==0.5.2
 # don't upgrade twisted because it doesn't work with our buildbot
-Twisted==10.1.0 # pyup: ignore
+Twisted==10.1.0  # pyup: ignore
 WebError==0.13.1
 WebHelpers==1.3
 WebOb==1.8.1
 WebTest==2.0.29
 meld3==1.0.2
 nose==1.3.7
 pytz==2018.4
 redis==2.10.6
--- a/modules/shipit_scriptworker/files/requirements.txt
+++ b/modules/shipit_scriptworker/files/requirements.txt
@@ -1,8 +1,9 @@
+# python_version: 36
 PyYAML==3.12
 aiohttp==3.2.1
 arrow==0.12.1
 async_timeout==3.0.0
 attrs==18.1.0
 certifi==2018.4.16
 chardet==3.0.4
 defusedxml==0.5.0
@@ -23,9 +24,9 @@ requests==2.18.4
 scriptworker==12.0.1
 shipitapi==1.0.0
 six==1.11.0
 slugid==1.0.7
 taskcluster==3.0.1
 urllib3==1.22
 virtualenv==15.2.0
 yarl==1.2.4
-shipitscript==1.0.0
+shipitscript==1.0.0  # puppet: nodownload
--- a/modules/signing_scriptworker/files/requirements.txt
+++ b/modules/signing_scriptworker/files/requirements.txt
@@ -1,8 +1,9 @@
+# python_version: 36
 PyYAML==3.12
 aiohttp==3.2.1
 arrow==0.12.1
 async_timeout==3.0.0
 attrs==18.1.0
 certifi==2018.4.16
 chardet==3.0.4
 datadog==0.17.0
@@ -22,17 +23,17 @@ pexpect==4.5.0
 ptyprocess==0.5.2
 pyasn1==0.4.2
 python-dateutil==2.7.3
 python-gnupg==0.4.2
 python-jose==3.0.0
 requests==2.18.4
 rsa==3.4.2
 scriptworker==12.0.1
-signingscript==6.1.0
+signingscript==6.1.0  # puppet: nodownload
 signtool==3.2.0
 simplejson==3.14.0
 six==1.11.0
 slugid==1.0.7
 taskcluster==3.0.1
 urllib3==1.22
 virtualenv==15.2.0
 yarl==1.2.4
--- a/modules/signingserver/files/linux_requirements.txt
+++ b/modules/signingserver/files/linux_requirements.txt
@@ -1,16 +1,17 @@
+# python_version: 27
 gevent==1.3.1
 WebOb==1.8.1
 poster==0.8.1
 IPy==0.83
 greenlet==0.4.13
 redis==2.10.6
 # fluf.lock 3.0+ requires python 3
-flufl.lock==2.4.1 # pyup: <3.0
+flufl.lock==2.4.1  # pyup: <3.0
 pexpect==4.5.0
 # widevine: osx cant build cryptography and doesnt need these!
 ## install six and setuptools before cryptography
 setuptools==39.2.0
 six==1.11.0
 argparse==1.4.0
 enum34==1.1.6
 ipaddress==1.0.21
--- a/modules/signingserver/files/mac_requirements.txt
+++ b/modules/signingserver/files/mac_requirements.txt
@@ -1,10 +1,11 @@
+# python_version: 27
 gevent==1.3.1
 WebOb==1.8.1
 poster==0.8.1
 IPy==0.83
 greenlet==0.4.13
 redis==2.10.6
 # fluf.lock 3.0+ requires python 3
-flufl.lock==2.4.1 # pyup: <3.0
+flufl.lock==2.4.1  # pyup: <3.0
 pexpect==4.5.0
 ptyprocess==0.5.2
--- a/modules/signingworker/files/requirements.txt
+++ b/modules/signingworker/files/requirements.txt
@@ -1,8 +1,9 @@
+# python_version: 27
 PyHawk-with-a-single-extra-commit==0.1.5
 amqp==2.2.2
 anyjson==0.3.3
 argparse==1.4.0
 arrow==0.12.1
 backports.functools_lru_cache==1.5
 certifi==2018.4.16
 chardet==3.0.4
--- a/modules/slaveapi/files/requirements.txt
+++ b/modules/slaveapi/files/requirements.txt
@@ -1,8 +1,9 @@
+# python_version: 27
 gevent==1.3.1
 greenlet==0.4.13
 pycrypto==2.6.1
 # unused, but one of the vendor libraries (Flask) requires it
 Jinja2==2.10
 MarkupSafe==1.0
 WebOb==1.8.1
 requests==2.18.4
@@ -20,17 +21,17 @@ orderedmultidict==0.7.11
 pytz==2018.4
 python-dateutil==2.7.3
 # for aws cloud-tools
 MySQL-python==1.2.5
 PyYAML==3.12
 SQLAlchemy==1.2.7
 cfn-pyplates>=0.5.0
 ecdsa==0.13
-invtool==4.4
+invtool==4.4  # puppet: nodownload
 netaddr==0.7.19
 ordereddict==1.1
 testrepository==0.0.20
 python-subunit==1.3.0
 extras==1.0.0
 testtools==2.3.0
 pbr==4.0.3
 schema==0.6.7
--- a/modules/slaverebooter/files/requirements.txt
+++ b/modules/slaverebooter/files/requirements.txt
@@ -1,5 +1,6 @@
+# python_version: 27
 furl==1.0.1
 requests==2.18.4
 docopt==0.6.2
 buildtools==1.0.6
 orderedmultidict==0.7.11
--- a/modules/transparency_scriptworker/files/requirements.txt
+++ b/modules/transparency_scriptworker/files/requirements.txt
@@ -1,8 +1,9 @@
+# python_version: 36
 PyYAML==3.12
 aiohttp==3.2.1
 arrow==0.12.1
 async_timeout==3.0.0
 attrs==18.1.0
 certifi==2018.4.16
 chardet==3.0.4
 defusedxml==0.5.0
@@ -22,9 +23,9 @@ redo==1.6
 requests==2.18.4
 scriptworker==12.0.1
 six==1.11.0
 slugid==1.0.7
 taskcluster==3.0.1
 urllib3==1.22
 virtualenv==15.2.0
 yarl==1.2.4
-transparencyscript==0.0.4
+transparencyscript==0.0.4  # puppet: nodownload
--- a/modules/tree_scriptworker/files/requirements.txt
+++ b/modules/tree_scriptworker/files/requirements.txt
@@ -1,8 +1,9 @@
+# python_version: 36
 PyYAML==3.12
 aiohttp==3.2.1
 arrow==0.12.1
 async_timeout==3.0.0
 attrs==18.1.0
 certifi==2018.4.16
 chardet==3.0.4
 defusedxml==0.5.0
@@ -18,12 +19,12 @@ pexpect==4.6.0
 ptyprocess==0.5.2
 python-dateutil==2.7.3
 python-gnupg==0.4.2
 requests==2.18.4
 scriptworker==12.0.1
 six==1.11.0
 slugid==1.0.7
 taskcluster==3.0.1
-treescript==0.4
+treescript==0.4  # puppet: nodownload
 urllib3==1.22
 virtualenv==16.0.0
 yarl==1.2.4