Bug 1512065 Remove ClamAV scanning from partials input r=tomprince
authorSimon Fraser <sfraser@mozilla.com>
Wed, 05 Dec 2018 16:41:03 +0000
changeset 508644 7bb91650c76349aa0fda487c1223914b1672bd0c
parent 508643 c42984729af7fb8986e08f81ea457575b5d1669a
child 508645 4a88ba63269cb0460cf9dd658ed4298399d453fe
push id1905
push userffxbld-merge
push dateMon, 21 Jan 2019 12:33:13 +0000
treeherdermozilla-release@c2fca1944d8c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstomprince
bugs1512065
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1512065 Remove ClamAV scanning from partials input r=tomprince Differential Revision: https://phabricator.services.mozilla.com/D13761
taskcluster/docker/funsize-update-generator/Dockerfile
taskcluster/docker/funsize-update-generator/scripts/funsize.py
--- a/taskcluster/docker/funsize-update-generator/Dockerfile
+++ b/taskcluster/docker/funsize-update-generator/Dockerfile
@@ -3,31 +3,28 @@ MAINTAINER Simon Fraser <sfraser@mozilla
 
 # Required software
 ENV DEBIAN_FRONTEND noninteractive
 # Chain apt-get commands with apt-get clean in a single docker RUN
 # to make sure that files are removed within a single docker layer
 RUN apt-get update -q && \
     apt-get install -yyq --no-install-recommends \
     python3.6 python3-setuptools python3-cryptography libgetopt-simple-perl \
-    bzip2 clamav clamav-freshclam python3-requests python3-sh curl \
+    bzip2 python3-requests python3-sh curl \
     python3-dev gcc liblzma-dev xz-utils jq libdpkg-perl locales && \
     apt-get clean
 RUN useradd -d /home/worker -s /bin/bash -m worker
 COPY Pipfile Pipfile.lock /
 
 RUN locale-gen en_CA.UTF-8
 ENV LANG en_CA.UTF-8
 ENV LANGUAGE en_CA.UTF-8
 ENV LANG_ALL en_CA.UTF-8
 ENV LC_ALL en_CA.UTF-8
 
-# Freshclam may be flaky, retry if it fails
-RUN for i in 1 2 3 4 5; do freshclam --verbose && break || sleep 15; done
-
 # python-pip installs a lot of dependencies increasing the size of an image
 # drastically. Install it like this saves us almost 200M.
 RUN bash -c "curl -L https://bootstrap.pypa.io/get-pip.py | python3"
 RUN ["pip", "install", "pipenv<2018.10.9", "pip==18.0"]
 
 # scripts
 RUN mkdir /home/worker/bin
 COPY scripts/* /home/worker/bin/
--- a/taskcluster/docker/funsize-update-generator/scripts/funsize.py
+++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py
@@ -315,22 +315,16 @@ async def manage_partial(partial_def, wo
                                  section="App", option="Version")
             major = int(version.split(".")[0])
             # The updater for versions less than 56.0 requires BZ2
             # compressed MAR files
             if major < 56:
                 use_old_format = True
                 log.info("Forcing BZ2 compression for %s", f)
 
-        log.info("AV-scanning %s ...", unpack_dir)
-        metric_tags = [
-            "platform:{}".format(partial_def['platform']),
-        ]
-        with ddstats.timer('mar.clamscan.time', tags=metric_tags):
-            await run_command("clamscan -r {}".format(unpack_dir), label='clamscan')
         log.info("Done.")
 
     to_path = os.path.join(work_env.workdir, "to")
     from_path = os.path.join(work_env.workdir, "from")
 
     mar_data = {
         "ACCEPTED_MAR_CHANNEL_IDS": get_option(
             to_path, filename="update-settings.ini", section="Settings",
@@ -450,18 +444,16 @@ def main():
                         type=argparse.FileType('r'))
     parser.add_argument("--allow-staging-prefixes",
                         action="store_true",
                         default=strtobool(
                             os.environ.get('FUNSIZE_ALLOW_STAGING_PREFIXES', "false")),
                         help="Allow files from staging buckets.")
     parser.add_argument("--filename-template",
                         default=DEFAULT_FILENAME_TEMPLATE)
-    parser.add_argument("--no-freshclam", action="store_true", default=False,
-                        help="Do not refresh ClamAV DB")
     parser.add_argument("-q", "--quiet", dest="log_level",
                         action="store_const", const=logging.WARNING,
                         default=logging.DEBUG)
     args = parser.parse_args()
 
     logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
     log.setLevel(args.log_level)
 
@@ -486,27 +478,16 @@ def main():
         log.info("Starting metric collection")
         initialize(**dd_options)
         ddstats.start(flush_interval=1)
         # For use in shell scripts.
         write_dogrc(dd_api_key)
     else:
         log.info("No metric collection")
 
-    if args.no_freshclam:
-        log.info("Skipping freshclam")
-    else:
-        log.info("Refreshing clamav db...")
-        try:
-            redo.retry(lambda: sh.freshclam("--stdout", "--verbose",
-                                            _timeout=300, _err_to_out=True))
-            log.info("Done.")
-        except sh.ErrorReturnCode:
-            log.warning("Freshclam failed, skipping DB update")
-
     loop = asyncio.get_event_loop()
     manifest = loop.run_until_complete(async_main(args, signing_certs))
     loop.close()
 
     manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
     with open(manifest_file, "w") as fp:
         json.dump(manifest, fp, indent=2, sort_keys=True)