Bug 1259423 - freshclam fails to update the DB r=rail a=release DONTBUILD
authorKim Moir <kmoir@mozilla.com>
Tue, 29 Mar 2016 10:27:32 -0400
changeset 323700 ab7eecfe1bba31fb8f477bd018ed6983607323e5
parent 323699 f1f3b63c5381ab1df1103f1729a6aef11442862c
child 323701 52315caa0dae41bf65da870154f3f5ac84d15883
push id5913
push userjlund@mozilla.com
push dateMon, 25 Apr 2016 16:57:49 +0000
treeherdermozilla-beta@dcaf0a6fa115 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrail, release
bugs1259423
milestone47.0a2
Bug 1259423 - freshclam fails to update the DB r=rail a=release DONTBUILD
release/docker/funsize-update-generator/scripts/funsize.py
testing/mozharness/scripts/release/beet_mover.py
--- a/release/docker/funsize-update-generator/scripts/funsize.py
+++ b/release/docker/funsize-update-generator/scripts/funsize.py
@@ -176,19 +176,22 @@ def main():
     args = parser.parse_args()
 
     logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
                         level=args.log_level)
     task = json.load(args.task_definition)
     # TODO: verify task["extra"]["funsize"]["partials"] with jsonschema
 
     log.info("Refreshing clamav db...")
-    redo.retry(lambda:
-               sh.freshclam("--stdout", "--verbose", _timeout=300, _err_to_out=True))
-    log.info("Done.")
+    try:
+        redo.retry(lambda:
+                   sh.freshclam("--stdout", "--verbose", _timeout=300, _err_to_out=True))
+        log.info("Done.")
+    except sh.ErrorReturnCode:
+        log.warning("Freshclam failed, skipping DB update")
     manifest = []
     for e in task["extra"]["funsize"]["partials"]:
         for mar in (e["from_mar"], e["to_mar"]):
             verify_allowed_url(mar)
 
         work_env = WorkEnv()
         # TODO: run setup once
         work_env.setup()
--- a/testing/mozharness/scripts/release/beet_mover.py
+++ b/testing/mozharness/scripts/release/beet_mover.py
@@ -113,16 +113,17 @@ class BeetMover(BaseScript, VirtualenvMi
     def __init__(self, aws_creds):
         beetmover_kwargs = {
             'config_options': CONFIG_OPTIONS,
             'all_actions': [
                 # 'clobber',
                 'create-virtualenv',
                 'activate-virtualenv',
                 'generate-candidates-manifest',
+                'refresh-antivirus',
                 'verify-bits',  # beets
                 'download-bits', # beets
                 'scan-bits',     # beets
                 'upload-bits',  # beets
             ],
             'require_config_file': False,
             # Default configuration
             'config': {
@@ -208,16 +209,25 @@ class BeetMover(BaseScript, VirtualenvMi
 
     def verify_bits(self):
         """
         inspects each artifact and verifies that they were created by trustworthy tasks
         """
         # TODO
         self.log('skipping verification. unimplemented...')
 
+    def refresh_antivirus(self):
+       self.info("Refreshing clamav db...")
+       try:
+           redo.retry(lambda:
+                      sh.freshclam("--stdout", "--verbose", _timeout=300, _err_to_out=True))
+           self.info("Done.")
+       except sh.ErrorReturnCode:
+           self.warning("Freshclam failed, skipping DB update")
+
     def download_bits(self):
         """
         downloads list of artifacts to self.dest_dir dir based on a given manifest
         """
         self.log('downloading and uploading artifacts to self_dest_dir...')
 
         # TODO - do we want to mirror/upload to more than one region?
         dirs = self.query_abs_dirs()
@@ -300,20 +310,16 @@ class BeetMover(BaseScript, VirtualenvMi
                 self.copyfile(os.path.join(dirs['abs_work_dir'], file_name), os.path.join(self.dest_dir,file_name))
         self._scan_files()
         self.info('Emptying {}'.format(self.dest_dir))
         self.rmtree(self.dest_dir)
 
     def _scan_files(self):
         """Scan the files we've collected. We do the download and scan concurrently to make
         it easier to have a coherent log afterwards. Uses the venv python."""
-        self.info("Refreshing clamav db...")
-        redo.retry(lambda:
-            sh.freshclam("--stdout", "--verbose", _timeout=300, _err_to_out=True))
-        self.info("Done.")
         external_tools_path = os.path.join(
                               os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))), 'external_tools')
         self.run_command([self.query_python_path(), os.path.join(external_tools_path,'extract_and_run_command.py'),
                          '-j{}'.format(self.config['scan_parallelization']),
                          'clamscan', '--no-summary', '--', self.dest_dir])
 
     def _matches_exclude(self, keyname):
          return any(re.search(exclude, keyname) for exclude in self.excludes)