Bug 1222872 - Upload *_info.txt for Socorro, r=rail DONTBUILD
authorNick Thomas <nthomas@mozilla.com>
Mon, 23 Nov 2015 15:50:55 +1300
changeset 273659 ec8eaf3dbb459ec97a3a8b68419b82fad834b607
parent 273658 3f51676191a6be10a6321edd49662a73a21e381d
child 273660 c0bbcab29901ca287180a3b9f89fd438d825cb55
push id68350
push usernthomas@mozilla.com
push dateMon, 23 Nov 2015 02:51:57 +0000
treeherdermozilla-inbound@ec8eaf3dbb45 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrail
bugs1222872
milestone45.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1222872 - Upload *_info.txt for Socorro, r=rail DONTBUILD
testing/mozharness/scripts/release/generate-checksums.py
--- a/testing/mozharness/scripts/release/generate-checksums.py
+++ b/testing/mozharness/scripts/release/generate-checksums.py
@@ -1,13 +1,14 @@
 from multiprocessing.pool import ThreadPool
 import os
 from os import path
 import re
 import sys
+import posixpath
 
 sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
 
 from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
 from mozharness.base.script import BaseScript
 from mozharness.base.vcs.vcsbase import VCSMixin
 from mozharness.mozilla.checksums import parse_checksums_file
 from mozharness.mozilla.signing import SigningMixin
@@ -47,82 +48,69 @@ class ChecksumsGenerator(BaseScript, Vir
             "default": [],
             "action": "append",
             "help": "List of patterns to include in big checksums file. See script source for default.",
         }],
         [["--tools-repo"], {
             "dest": "tools_repo",
             "default": "https://hg.mozilla.org/build/tools",
         }],
-        [["--gecko-repo"], {
-            "dest": "gecko_repo",
-            "help": "Gecko repo to get upload.py from, eg: https://hg.mozilla.org/releases/mozilla-beta",
-        }],
-        [["--gecko-revision"], {
-            "dest": "gecko_revision",
-            "help": "Revision of gecko repo to use when getting upload.py. Must be a valid hg revision identifier.",
-        }],
-        [["--upload-host"], {
-            "dest": "upload_host",
-            "help": "Host to upload big checksums file to, eg: upload.ffxbld.productdelivery.prod.mozaws.net",
-        }],
-        [["--upload-user"], {
-            "dest": "upload_user",
-            "help": "Username to use when uploading, eg: ffxbld",
-        }],
-        [["--upload-ssh-key"], {
-            "dest": "upload_ssh_key",
-            "help": "SSH Key to use when uploading, eg: ~/.ssh/ffxbld_dsa",
+        [["--credentials"], {
+            "dest": "credentials",
+            "help": "File containing access key and secret access key for S3",
         }],
     ] + virtualenv_config_options
 
     def __init__(self):
         BaseScript.__init__(self,
             config_options=self.config_options,
             require_config_file=False,
             config={
                 "virtualenv_modules": [
                     "boto",
-                    "redo",
                 ],
                 "virtualenv_path": "venv",
             },
             all_actions=[
                 "create-virtualenv",
-                "activate-virtualenv",
                 "collect-individual-checksums",
                 "create-big-checksums",
                 "sign",
-                "get-upload-script",
                 "upload",
+                "copy-info-files",
             ],
             default_actions=[
                 "create-virtualenv",
-                "activate-virtualenv",
                 "collect-individual-checksums",
                 "create-big-checksums",
                 "sign",
-                "get-upload-script",
                 "upload",
             ],
         )
 
         self.checksums = {}
+        self.bucket = None
+        self.bucket_name = self._get_bucket_name()
+        self.file_prefix = self._get_file_prefix()
+        # set the env var for boto to read our special config file
+        # rather than anything else we have at ~/.boto
+        os.environ["BOTO_CONFIG"] = os.path.abspath(self.config["credentials"])
 
     def _pre_config_lock(self, rw_config):
         super(ChecksumsGenerator, self)._pre_config_lock(rw_config)
 
         # These defaults are set here rather in the config because default
         # lists cannot be completely overidden, only appended to.
         if not self.config.get("formats"):
             self.config["formats"] = ["sha512"]
 
         if not self.config.get("includes"):
             self.config["includes"] = [
                 "^.*\.tar\.bz2$",
+                "^.*\.tar\.xz$",
                 "^.*\.dmg$",
                 "^.*\.bundle$",
                 "^.*\.mar$",
                 "^.*Setup.*\.exe$",
                 "^.*\.xpi$",
             ]
 
     def _get_bucket_name(self):
@@ -136,44 +124,47 @@ class ChecksumsGenerator(BaseScript, Vir
     def _get_file_prefix(self):
         return "pub/{}/candidates/{}-candidates/build{}".format(
             self.config["stage_product"], self.config["version"], self.config["build_number"]
         )
 
     def _get_sums_filename(self, format_):
         return "{}SUMS".format(format_.upper())
 
+    def _get_bucket(self):
+        if not self.bucket:
+            self.activate_virtualenv()
+            from boto.s3.connection import S3Connection
+
+            self.info("Connecting to S3")
+            conn = S3Connection()
+            self.debug("Successfully connected to S3")
+            self.info("Connecting to bucket {}".format(self.bucket_name))
+            self.bucket = conn.get_bucket(self.bucket_name)
+        return self.bucket
+
     def collect_individual_checksums(self):
         """This step grabs all of the small checksums files for the release,
         filters out any unwanted files from within them, and adds the remainder
         to self.checksums for subsequent steps to use."""
-        from boto.s3.connection import S3Connection
-
-        bucket_name = self._get_bucket_name()
-        file_prefix = self._get_file_prefix()
-        self.info("Bucket name is: {}".format(bucket_name))
-        self.info("File prefix is: {}".format(file_prefix))
-
-        self.info("Connecting to S3")
-        conn = S3Connection(anon=True)
-        self.debug("Successfully connected to S3")
-        candidates = conn.get_bucket(bucket_name)
+        bucket = self._get_bucket()
+        self.info("File prefix is: {}".format(self.file_prefix))
 
         # Temporary holding place for checksums
         raw_checksums = []
         def worker(item):
             self.debug("Downloading {}".format(item))
             # TODO: It would be nice to download the associated .asc file
             # and verify against it.
-            sums = candidates.get_key(item).get_contents_as_string()
+            sums = bucket.get_key(item).get_contents_as_string()
             raw_checksums.append(sums)
 
         def find_checksums_files():
             self.info("Getting key names from bucket")
-            for key in candidates.list(prefix=file_prefix):
+            for key in bucket.list(prefix=self.file_prefix):
                 if key.key.endswith(".checksums"):
                     self.debug("Found checksums file: {}".format(key.key))
                     yield key.key
                 else:
                     self.debug("Ignoring non-checksums file: {}".format(key.key))
 
         pool = ThreadPool(self.config["parallelization"])
         pool.map(worker, find_checksums_files())
@@ -214,43 +205,38 @@ class ChecksumsGenerator(BaseScript, Vir
 
         for fmt in self.config["formats"]:
             sums = self._get_sums_filename(fmt)
             self.info("Signing big checksums file: {}".format(sums))
             retval = self.run_command(sign_cmd + [sums])
             if retval != 0:
                 self.fatal("Failed to sign {}".format(sums))
 
-    def get_upload_script(self):
-        remote_file = "{}/raw-file/{}/build/upload.py".format(self.config["gecko_repo"], self.config["gecko_revision"])
-        self.download_file(remote_file, file_name="upload.py")
-
     def upload(self):
-        dirs = self.query_abs_dirs()
-
-        sys.path.append(path.dirname(path.abspath("upload.py")))
-        from upload import UploadFiles
-
         files = []
         for fmt in self.config["formats"]:
             files.append(self._get_sums_filename(fmt))
             files.append("{}.asc".format(self._get_sums_filename(fmt)))
 
-        post_upload_cmd = " ".join([
-            "post_upload.py", "-p", self.config["stage_product"], "-n", self.config["build_number"],
-            "-v", self.config["version"], "--release-to-candidates-dir", "--signed",
-            "--bucket-prefix", self.config["bucket_name_prefix"],
-        ])
+        bucket = self._get_bucket()
+        for f in files:
+            dest = posixpath.join(self.file_prefix, f)
+            self.info("Uploading {} to {}".format(f, dest))
+            key = bucket.new_key(dest)
+            key.set_contents_from_filename(f, headers={'Content-Type': 'text/plain'})
+
+    def copy_info_files(self):
+        bucket = self._get_bucket()
 
-        UploadFiles(
-            self.config["upload_user"],
-            self.config["upload_host"],
-            None,
-            files,
-            ssh_key=self.config["upload_ssh_key"],
-            base_path=dirs["abs_work_dir"],
-            upload_to_temp_dir=True,
-            post_upload_command=post_upload_cmd,
-        )
+        for key in bucket.list(prefix=self.file_prefix):
+            if re.search(r'/en-US/android.*_info\.txt$', key.name):
+                self.info("Found      {}".format(key.name))
+                dest = posixpath.join(self.file_prefix, posixpath.basename(key.name))
+                self.info("Copying to {}".format(dest))
+                bucket.copy_key(new_key_name=dest,
+                                src_bucket_name=self.bucket_name,
+                                src_key_name=key.name,
+                                metadata={'Content-Type': 'text/plain'})
+
 
 if __name__ == "__main__":
     myScript = ChecksumsGenerator()
     myScript.run_and_exit()