Bug 1555970 - switch mapper auth to taskcluster. r=dhouse default
authorRok Garbas <rgarbas@mozilla.com>
Tue, 06 Aug 2019 07:50:21 -0600
changeset 4484 8dfd0cc64322f241ddd5921d24169994384eec6e
parent 4482 28e41a10ce4f010e7721476b4aa6874bdcd82c5b
child 4485 319939e658c9b6ac062812ee686c6e3d5c8352e5
push id3588
push userdhouse@mozilla.com
push dateTue, 06 Aug 2019 14:03:02 +0000
reviewersdhouse
bugs1555970
Bug 1555970 - switch mapper auth to taskcluster. r=dhouse
configs/vcs_sync/beagle.py
configs/vcs_sync/build-repos.py
scripts/vcs-sync/vcs_sync.py
--- a/configs/vcs_sync/beagle.py
+++ b/configs/vcs_sync/beagle.py
@@ -266,16 +266,17 @@ config = {
         "ordereddict==1.1",
         "hg-git==0.4.0-moz2",
         "mapper==0.1",
         "mercurial==3.7.3",
         "mozfile==0.9",
         "mozinfo==0.5",
         "mozprocess==0.11",
         "requests==2.8.1",
+        "mohawk==1.0.0",
     ],
     "find_links": [
         "http://pypi.pvt.build.mozilla.org/pub",
         "http://pypi.pub.build.mozilla.org/pub",
     ],
     "pip_index": False,
 
     "default_notify_from": "developer-services+%s@mozilla.org" % hostname,
--- a/configs/vcs_sync/build-repos.py
+++ b/configs/vcs_sync/build-repos.py
@@ -159,16 +159,17 @@ config = {
         "ordereddict==1.1",
         "hg-git==0.4.0-moz2",
         "mapper==0.1",
         "mercurial==3.7.3",
         "mozfile==0.9",
         "mozinfo==0.5",
         "mozprocess==0.11",
         "requests==2.8.1",
+        "mohawk==1.0.0",
     ],
     "find_links": [
         "http://pypi.pub.build.mozilla.org/pub"
     ],
     "pip_index": False,
 
     "default_notify_from": "developer-services+%s@mozilla.org" % hostname,
     "notify_config": [{
--- a/scripts/vcs-sync/vcs_sync.py
+++ b/scripts/vcs-sync/vcs_sync.py
@@ -978,20 +978,17 @@ intree=1
                     sys.path.append(site_packages_path)
                 try:
                     import requests
                 except ImportError as e:
                     self.error("Can't import requests: %s\nDid you create-virtualenv?" % str(e))
                 mapper_url = mapper_config['url']
                 mapper_project = mapper_config['project']
                 insert_url = "%s/%s/insert/ignoredups" % (mapper_url, mapper_project)
-                headers = {
-                    'Content-Type': 'text/plain',
-                    'Authentication': 'Bearer %s' % os.environ["RELENGAPI_INSERT_HGGIT_MAPPINGS_AUTH_TOKEN"]
-                }
+
                 all_new_mappings = []
                 all_new_mappings.extend(self.pull_out_new_sha_lookups(published_to_mapper, complete_mapfile))
                 self.write_to_file(delta_for_mapper, "".join(all_new_mappings))
                 # bug 1193011 says there are problems on occasion, independently
                 # check calculation of additions and save off mapfiles
                 dirs = self.query_abs_dirs()
                 for mapfile in [delta_for_mapper, published_to_mapper, complete_mapfile]:
                     self.copyfile(src=mapfile, dest=os.path.join('logs',
@@ -1013,44 +1010,95 @@ intree=1
                     elif retcode:
                         cmd = kwargs.get("args")
                         if cmd is None:
                             cmd = popenargs[0]
                         error = subprocess.CalledProcessError(retcode, cmd)
                         error.output = output
                         raise error
                     return output
-                lines_last_time = int(check_output('wc -l <%s' % published_to_mapper, shell=True))
-                lines_this_time = int(check_output('wc -l <%s' % complete_mapfile, shell=True))
+                lines_last_time = 0
+                if os.path.exists(published_to_mapper):
+                    lines_last_time = int(check_output('wc -l <%s' % published_to_mapper, shell=True))
+                lines_this_time = 0
+                if os.path.exists(complete_mapfile):
+                    lines_this_time = int(check_output('wc -l <%s' % complete_mapfile, shell=True))
                 if lines_this_time - lines_last_time != len(all_new_mappings):
                     self.error("Bad calc of new mappings: last %d, now %d, diff %d, calc %d"
                                % (lines_last_time, lines_this_time, lines_this_time - lines_last_time,
                                   len(all_new_mappings)))
                 # correct # of entries, but are they the correct
                 # entries? None of the lines in delta_for_mapper should
                 # be in published_to_mapper. grep -Ff can help verify
                 retcode = subprocess.call(['grep', '-Ff',
                     delta_for_mapper, published_to_mapper])
                 if retcode != 1:
                     self.error("Bad selection of new mappings, some already there")
 
+                # create authentication headers
+                content_type = 'text/plain'
+                tc_client_id = os.environ.get(
+                    'RELENGAPI_INSERT_HGGIT_MAPPINGS_TASKCLUSTER_CLIENT_ID')
+                tc_access_token = os.environ.get(
+                    'RELENGAPI_INSERT_HGGIT_MAPPINGS_TASKCLUSTER_ACCESS_TOKEN')
+                relengapi_token = os.environ.get(
+                    'RELENGAPI_INSERT_HGGIT_MAPPINGS_AUTH_TOKEN')
+
+                # For taskcluster auth, we only import mohawk since we need
+                # content to create the header
+                try:
+                    import mohawk
+                except ImportError as e:
+                    self.fatal("Can't import mohawk: %s\nDid you create-virtualenv?" % str(e))
+
                 # due to timeouts on load balancer, we only push 200 lines at a time
                 # this means that we should get http response back within 30 seconds
                 # including the time it takes to insert the mappings in the database
                 publish_successful = True
+
                 for i in range(0, len(all_new_mappings), 200):
-                    r = requests.post(insert_url, data="".join(all_new_mappings[i:i+200]), headers=headers)
+                    data = "".join(all_new_mappings[i:i+200])
+
+                    if tc_client_id and tc_access_token:
+                        headers = {
+                            'Content-Type': content_type,
+                            'Authentication': mohawk.Sender(
+                                credentials=dict(
+                                    id=tc_client_id,
+                                    key=tc_access_token,
+                                    algorithm='sha256',
+                                ),
+                                ext=dict(),
+                                url=insert_url,
+                                content=data,
+                                content_type=content_type,
+                                method='POST',
+                            ).request_header,
+                        }
+                    elif relengapi_token:
+                        headers = {
+                            'Content-Type': content_type,
+                            'Authentication': 'Bearer %s' % relengapi_token,
+                        }
+                    else:
+                        self.fatal(
+                            "Please provide either:\n"
+                            "- RELENGAPI_INSERT_HGGIT_MAPPINGS_AUTH_TOKEN\n"
+                            "- RELENGAPI_INSERT_HGGIT_MAPPINGS_TASKCLUSTER_ACCESS_TOKEN and RELENGAPI_INSERT_HGGIT_MAPPINGS_TASKCLUSTER_CLIENT_ID")
+
+                    r = requests.post(insert_url, data=data, headers=headers)
                     if (r.status_code != 200):
                         self.error("Could not publish mapfile ('%s') line range [%s, %s] to mapper (%s) - received http %s code" % (delta_for_mapper, i, i+200, insert_url, r.status_code))
                         publish_successful = False
                         # we won't break out, since we may be able to publish other mappings
                         # and duplicates are allowed, so we will push the whole lot again next
                         # time anyway
                     else:
                         self.info("Published mapfile ('%s') line range [%s, %s] to mapper (%s)" % (delta_for_mapper, i, i+200, insert_url))
+
                 if publish_successful:
                     # bug 1193011 says there are problems on occasion
                     # with delta uploads. Check that items are in db in
                     # an effort to find the root cause.
                     global publish_verified
                     publish_verified = True
                     try:
                         # previously checked first, last only. No errors