Bug 1252908 - [beetmover] refresh AV database on every run r=rail a=testing DONTBUILD
authorKim Moir <kmoir@mozilla.com>
Wed, 02 Mar 2016 16:36:31 -0500
changeset 304355 c160afabf0e7486f41f9f0c0c1b11d9d510f7a9e
parent 304354 2caad913ec805e729ce802279d46ab087e82a565
child 304356 0bee186afe5ab070ca8adbba46bcea934adbb254
push id9175
push userraliiev@mozilla.com
push dateThu, 03 Mar 2016 03:39:52 +0000
treeherdermozilla-aurora@0bee186afe5a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrail, testing
bugs1252908
milestone46.0a2
Bug 1252908 - [beetmover] refresh AV database on every run r=rail a=testing DONTBUILD MozReview-Commit-ID: Ceo1dF3tyVS
release/docker/beet-mover/Dockerfile
release/docker/beet-mover/requirements.txt
testing/mozharness/scripts/release/beet_mover.py
--- a/release/docker/beet-mover/Dockerfile
+++ b/release/docker/beet-mover/Dockerfile
@@ -11,9 +11,11 @@ RUN apt-get -q update \
     libyaml-dev \
     libmysqlclient-dev \
     clamav \
     clamav-freshclam \
     curl \
     wget \
     && apt-get clean
 
+COPY requirements.txt /tmp/
+RUN pip install -r /tmp/requirements.txt
 RUN freshclam --verbose
new file mode 100644
--- /dev/null
+++ b/release/docker/beet-mover/requirements.txt
@@ -0,0 +1,2 @@
+sh
+redo
--- a/testing/mozharness/scripts/release/beet_mover.py
+++ b/testing/mozharness/scripts/release/beet_mover.py
@@ -10,16 +10,18 @@ downloads artifacts, scans them and uplo
 """
 import hashlib
 import sys
 import os
 import pprint
 import re
 from os import listdir
 from os.path import isfile, join
+import sh
+import redo
 
 sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
 from mozharness.base.log import FATAL
 from mozharness.base.python import VirtualenvMixin
 from mozharness.base.script import BaseScript
 from mozharness.mozilla.aws import pop_aws_auth_from_env
 import mozharness
 
@@ -304,16 +306,20 @@ class BeetMover(BaseScript, VirtualenvMi
                 self.copyfile(os.path.join(dirs['abs_work_dir'], file_name), os.path.join(self.dest_dir,file_name))
         self._scan_files()
         self.info('Emptying {}'.format(self.dest_dir))
         self.rmtree(self.dest_dir)
 
     def _scan_files(self):
         """Scan the files we've collected. We do the download and scan concurrently to make
         it easier to have a coherent log afterwards. Uses the venv python."""
+        self.info("Refreshing clamav db...")
+        redo.retry(lambda:
+            sh.freshclam("--stdout", "--verbose", _timeout=300, _err_to_out=True))
+        self.info("Done.")
         external_tools_path = os.path.join(
                               os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))), 'external_tools')
         self.run_command([self.query_python_path(), os.path.join(external_tools_path,'extract_and_run_command.py'),
                          '-j{}'.format(self.config['scan_parallelization']),
                          'clamscan', '--no-summary', '--', self.dest_dir])
 
     def _matches_exclude(self, keyname):
          return any(re.search(exclude, keyname) for exclude in self.excludes)