Do not fail on FS failures
authorRail Aliiev <rail@mozilla.com>
Mon, 01 Dec 2014 04:03:24 -0500
changeset 577 0442d92d16a5ec156bc516ce2ce3847facf53676
parent 576 4e1b8556c3fe4b1f773b281cb7254fe35bb1da05
child 578 01b34efd06724c9f5e0785b3f38a1ff67ad02342
push id568
push userraliiev@mozilla.com
push dateMon, 01 Dec 2014 09:03:28 +0000
Do not fail on FS failures
scripts/aws_process_cloudtrail_logs.py
--- a/scripts/aws_process_cloudtrail_logs.py
+++ b/scripts/aws_process_cloudtrail_logs.py
@@ -28,21 +28,24 @@ def move_to_bad_logs(filename):
 
 def process_cloudtrail(discard_bad_logs, events_dir, filename):
     """extracts data from filename"""
     try:
         data = get_data_from_gz_file(filename)
         data = json.loads(data)
     except (ValueError, IOError):
         log.debug('cannot decode JSON from %s', filename)
-        if discard_bad_logs:
-            log.debug('%s is not valid, deleting it', filename)
-            os.remove(filename)
-        else:
-            move_to_bad_logs(filename)
+        try:
+            if discard_bad_logs:
+                log.debug('%s is not valid, deleting it', filename)
+                os.remove(filename)
+            else:
+                move_to_bad_logs(filename)
+        except Exception:
+            pass
         return
 
     log.debug('processing: %s', filename)
     for record in data['Records']:
         eventName = record['eventName']
         # just process stop events, skip StartInstances and TerminateInstances
         if eventName in ('StopInstances',):
             process_start_stop_record(events_dir, record)
@@ -93,17 +96,19 @@ if __name__ == '__main__':
     parser.add_argument("-v", "--verbose", action="store_true",
                         help="Increase logging verbosity")
     parser.add_argument("--cloudtrail-dir", metavar="cloudtrail_dir",
                         required=True,
                         help="Cloutrail logs directory")
     parser.add_argument("--events-dir", metavar="events_dir", required=True,
                         help="directory where events logs will be stored")
     parser.add_argument("--discard-bad-logs", action="store_true",
-                        help="delete bad log files, if not provided, bad log files will be moved into bad_logs_dir (next to --event-dir)")
+                        help="delete bad log files, if not provided, bad log "
+                        "files will be moved into bad_logs_dir (next to "
+                        "--event-dir)")
     args = parser.parse_args()
 
     logging.basicConfig(format="%(asctime)s - %(message)s")
     if args.verbose:
         log.setLevel(logging.DEBUG)
     else:
         log.setLevel(logging.INFO)
 
@@ -114,16 +119,14 @@ if __name__ == '__main__':
     cloudtrail_files = []
     for dirpath, dirnames, filenames in os.walk(args.cloudtrail_dir):
         for log_file in filenames:
             cloudtrail_files.append(os.path.join(dirpath, log_file))
 
     # process_cloud_tails requires 3 arguments: discard_bad_logs,
     # events_dir and cloudtrail_file, maps() accepts only 2 parameters,
     # function name and an iterable, let's use partials
-    process_cloudtrail_partial = partial(process_cloudtrail,
-                                         args.discard_bad_logs)
-    process_cloudtrail_partial_2 = partial(process_cloudtrail_partial,
-                                           args.events_dir)
+    process_cloudtrail_partial = partial(
+        process_cloudtrail, args.discard_bad_logs, args.events_dir)
     pool = Pool()
-    pool.map(process_cloudtrail_partial_2, cloudtrail_files)
+    pool.map(process_cloudtrail_partial, cloudtrail_files)
     pool.close()
     pool.join()