Bug 1019962 - Allow to download files outside of releng network with http authentication. r=aki
authorArmen Zambrano Gasparnian <armenzg@mozilla.com>
Fri, 11 Jul 2014 13:46:47 -0400
changeset 2946 500783a50ea4
parent 2945 58debbd1383a
child 2947 1b571af3fd9d
child 2948 ba4d1b9608fd
child 3009 b8e0225a9544
push id2215
push userarmenzg@mozilla.com
push date2014-07-11 17:46 +0000
reviewersaki
bugs1019962
Bug 1019962 - Allow to download files outside of releng network with http authentication. r=aki
mozharness/base/script.py
mozharness/mozilla/testing/testbase.py
--- a/mozharness/base/script.py
+++ b/mozharness/base/script.py
@@ -172,22 +172,28 @@ class ScriptMixin(object):
 
     def get_filename_from_url(self, url):
         parsed = urlparse.urlsplit(url.rstrip('/'))
         if parsed.path != '':
             return parsed.path.rsplit('/', 1)[-1]
         else:
             return parsed.netloc
 
+    def _urlopen(self, url, **kwargs):
+        """ This method can be overwritten to extend its complexity
+        """
+        return urllib2.urlopen(url, **kwargs)
+
     def _download_file(self, url, file_name):
         """ Helper script for download_file()
-            """
+        """
         try:
             f_length = None
-            f = urllib2.urlopen(url, timeout=30)
+            f = self._urlopen(url, timeout=30)
+
             if f.info().get('content-length') is not None:
                 f_length = int(f.info()['content-length'])
                 got_length = 0
             local_file = open(file_name, 'wb')
             while True:
                 block = f.read(1024 ** 2)
                 if not block:
                     if f_length is not None and got_length != f_length:
--- a/mozharness/mozilla/testing/testbase.py
+++ b/mozharness/mozilla/testing/testbase.py
@@ -3,16 +3,18 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 
 import copy
 import os
 import platform
+import urllib2
+import getpass
 
 from mozharness.base.config import ReadOnlyDict, parse_config_file
 from mozharness.base.errors import BaseErrorList
 from mozharness.base.log import FATAL
 from mozharness.base.python import (
     ResourceMonitoringMixin,
     VirtualenvMixin,
     virtualenv_config_options,
@@ -119,16 +121,75 @@ class TestingMixin(VirtualenvMixin, Buil
             self.fatal("Can't figure out symbols_url without an installer_url!")
         for suffix in INSTALLER_SUFFIXES:
             if self.installer_url.endswith(suffix):
                 self.symbols_url = self.installer_url[:-len(suffix)] + '.crashreporter-symbols.zip'
                 return self.symbols_url
         else:
             self.fatal("Can't figure out symbols_url from installer_url %s!" % self.installer_url)
 
+    def _query_outside_vpn(self, url):
+        ''' This function determines if the file needs basic http authentication.
+            If it needs it is because we're retrieving the file outside of the releng vpn.
+        '''
+        try:
+            response = urllib2.urlopen(url)
+            return False
+        except urllib2.URLError, e:
+            self.debug("We are running outside the releng network: %s" % str(e))
+            return True
+
+    def _urlopen(self, url, **kwargs):
+        '''
+        This function helps dealing with downloading files while outside
+        of the releng network.
+        '''
+        def _get_credentials():
+            if not hasattr(self, "https_username"):
+                self.https_username = raw_input("Please enter your full LDAP email address: ")
+                self.info("Please enter your LDAP password: ")
+                self.https_password = getpass.getpass()
+            return self.https_username, self.https_password
+
+        # Code based on http://code.activestate.com/recipes/305288-http-basic-authentication
+        def _urlopen_basic_auth(url, **kwargs):
+            self.info("NOTICE: Files downloaded from Release Engineering network require LDAP credentials.")
+            self.info("        We want to download this file %s" % url)
+            username, password = _get_credentials()
+            # This creates a password manager
+            passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
+            # Because we have put None at the start it will use this username/password combination from here on
+            passman.add_password(None, url, username, password)
+            authhandler = urllib2.HTTPBasicAuthHandler(passman)
+
+            return urllib2.build_opener(authhandler).open(url, **kwargs)
+
+        # We first try urlopen() normally; If we have http authentication failure,
+        # we will ask for credentials
+        try:
+            return urllib2.urlopen(url, **kwargs)
+        except urllib2.HTTPError, e:
+            if e.code == 401:
+                return _urlopen_basic_auth(url, **kwargs)
+            raise
+
+    def download_file(self, url, *args, **kwargs):
+        ''' To be able to reach pvtbuilds files we need to hit a different domain
+        '''
+        if url.startswith("http://pvtbuilds.pvt.build") and self._query_outside_vpn(url):
+            url = url.replace("http://pvtbuilds.pvt.build", "https://pvtbuilds")
+        return super(TestingMixin, self).download_file(url, *args, **kwargs)
+
+    def _download_file(self, url, filename):
+        ''' To be able to reach pvtbuilds files we need to hit a different domain
+        '''
+        if url.startswith("http://pvtbuilds.pvt.build") and self._query_outside_vpn(url):
+            url = url.replace("http://pvtbuilds.pvt.build", "https://pvtbuilds")
+        return super(TestingMixin, self)._download_file(url, filename)
+
     # read_buildbot_config is in BuildbotMixin.
 
     def postflight_read_buildbot_config(self):
         """
         Determine which files to download from the buildprops.json file
         created via the buildbot ScriptFactory.
         """
         if self.buildbot_config: