Bug 1252908 - [beetmover] refresh AV database on every run r=rail a=testing DONTBUILD
authorKim Moir <kmoir@mozilla.com>
Wed, 02 Mar 2016 16:36:31 -0500
changeset 322747 4ea7408b3eef059aa248f4b00328f8fdb4475112
parent 322746 52cfda4c6bae5c28ab09709655435da6ecda9e2a
child 322781 7e43bdd93e439b8c8d6e62d48d9de3e293655560
push id5913
push userjlund@mozilla.com
push dateMon, 25 Apr 2016 16:57:49 +0000
treeherdermozilla-beta@dcaf0a6fa115 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrail, testing
bugs1252908
milestone47.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1252908 - [beetmover] refresh AV database on every run r=rail a=testing DONTBUILD MozReview-Commit-ID: Ceo1dF3tyVS
release/docker/beet-mover/Dockerfile
release/docker/beet-mover/requirements.txt
testing/mozharness/scripts/release/beet_mover.py
--- a/release/docker/beet-mover/Dockerfile
+++ b/release/docker/beet-mover/Dockerfile
@@ -11,9 +11,11 @@ RUN apt-get -q update \
     libyaml-dev \
     libmysqlclient-dev \
     clamav \
     clamav-freshclam \
     curl \
     wget \
     && apt-get clean
 
+COPY requirements.txt /tmp/
+RUN pip install -r /tmp/requirements.txt
 RUN freshclam --verbose
new file mode 100644
--- /dev/null
+++ b/release/docker/beet-mover/requirements.txt
@@ -0,0 +1,2 @@
+sh
+redo
--- a/testing/mozharness/scripts/release/beet_mover.py
+++ b/testing/mozharness/scripts/release/beet_mover.py
@@ -10,16 +10,18 @@ downloads artifacts, scans them and uplo
 """
 import hashlib
 import sys
 import os
 import pprint
 import re
 from os import listdir
 from os.path import isfile, join
+import sh
+import redo
 
 sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
 from mozharness.base.log import FATAL
 from mozharness.base.python import VirtualenvMixin
 from mozharness.base.script import BaseScript
 from mozharness.mozilla.aws import pop_aws_auth_from_env
 import mozharness
 
@@ -304,16 +306,20 @@ class BeetMover(BaseScript, VirtualenvMi
                 self.copyfile(os.path.join(dirs['abs_work_dir'], file_name), os.path.join(self.dest_dir,file_name))
         self._scan_files()
         self.info('Emptying {}'.format(self.dest_dir))
         self.rmtree(self.dest_dir)
 
     def _scan_files(self):
         """Scan the files we've collected. We do the download and scan concurrently to make
         it easier to have a coherent log afterwards. Uses the venv python."""
+        self.info("Refreshing clamav db...")
+        redo.retry(lambda:
+            sh.freshclam("--stdout", "--verbose", _timeout=300, _err_to_out=True))
+        self.info("Done.")
         external_tools_path = os.path.join(
                               os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))), 'external_tools')
         self.run_command([self.query_python_path(), os.path.join(external_tools_path,'extract_and_run_command.py'),
                          '-j{}'.format(self.config['scan_parallelization']),
                          'clamscan', '--no-summary', '--', self.dest_dir])
 
     def _matches_exclude(self, keyname):
          return any(re.search(exclude, keyname) for exclude in self.excludes)