Bug 1328301 - handle push/pull directory semantics changes in adb 1.0.36 for devicemanagerADB, r=gbrown
authorBob Clary <bclary@bclary.com>
Sat, 21 Jan 2017 09:49:56 -0800
changeset 377830 92f215ed4df341c2dc316ac7c6ff6fb4735fb95c
parent 377829 e78066796efe05df7b2229e759fa9eb63dc3ec14
child 377831 e329450e4d25998bc0556c6f0f1526259d69a7c3
push id1419
push userjlund@mozilla.com
push dateMon, 10 Apr 2017 20:44:07 +0000
treeherdermozilla-release@5e6801b73ef6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgbrown
bugs1328301
milestone53.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1328301 - handle push/pull directory semantics changes in adb 1.0.36 for devicemanagerADB, r=gbrown
testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py
--- a/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py
+++ b/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py
@@ -1,20 +1,21 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import logging
 import re
 import os
-import shutil
 import tempfile
 import time
 import traceback
 
+from distutils import dir_util
+
 from devicemanager import DeviceManager, DMError
 from mozprocess import ProcessHandler
 import mozfile
 
 
 class DeviceManagerADB(DeviceManager):
     """
     Implementation of DeviceManager interface that uses the Android "adb"
@@ -27,16 +28,17 @@ class DeviceManagerADB(DeviceManager):
     _haveSu = None
     _suModifier = None
     _lsModifier = None
     _useZip = False
     _logcatNeedsRoot = False
     _pollingInterval = 0.01
     _packageName = None
     _tempDir = None
+    _adb_version = None
     connected = False
 
     def __init__(self, host=None, port=5555, retryLimit=5, packageName='fennec',
                  adbPath=None, deviceSerial=None, deviceRoot=None,
                  logLevel=logging.ERROR, autoconnect=True, runAdbAsRoot=False,
                  serverHost=None, serverPort=None, **kwargs):
         DeviceManager.__init__(self, logLevel=logLevel,
                                deviceRoot=deviceRoot)
@@ -266,28 +268,52 @@ class DeviceManagerADB(DeviceManager):
                 if re.search("unzip: exiting", data) or re.search("Operation not permitted", data):
                     raise Exception("unzip failed, or permissions error")
             except:
                 self._logger.warning(traceback.format_exc())
                 self._logger.warning("zip/unzip failure: falling back to normal push")
                 self._useZip = False
                 self.pushDir(localDir, remoteDir, retryLimit=retryLimit, timeout=timeout)
         else:
-            # If the remote directory exists, newer implementations of
-            # "adb push" will create a sub-directory, while older versions
-            # will not! Bug 1285040
-            self.mkDirs(remoteDir + "/x")
-            self.removeDir(remoteDir)
-            tmpDir = tempfile.mkdtemp()
-            # copytree's target dir must not already exist, so create a subdir
-            tmpDirTarget = os.path.join(tmpDir, "tmp")
-            shutil.copytree(localDir, tmpDirTarget)
-            self._checkCmd(["push", tmpDirTarget, remoteDir],
-                           retryLimit=retryLimit, timeout=timeout)
-            mozfile.remove(tmpDir)
+            localDir = os.path.normpath(localDir)
+            remoteDir = os.path.normpath(remoteDir)
+            copyRequired = False
+            if self._adb_version >= '1.0.36' and \
+               os.path.isdir(localDir) and self.dirExists(remoteDir):
+                # See do_sync_push in
+                # https://android.googlesource.com/platform/system/core/+/master/adb/file_sync_client.cpp
+                # Work around change in behavior in adb 1.0.36 where if
+                # the remote destination directory exists, adb push will
+                # copy the source directory *into* the destination
+                # directory otherwise it will copy the source directory
+                # *onto* the destination directory.
+                #
+                # If the destination directory does exist, push to its
+                # parent directory.  If the source and destination leaf
+                # directory names are different, copy the source directory
+                # to a temporary directory with the same leaf name as the
+                # destination so that when we push to the parent, the
+                # source is copied onto the destination directory.
+                localName = os.path.basename(localDir)
+                remoteName = os.path.basename(remoteDir)
+                if localName != remoteName:
+                    copyRequired = True
+                    tempParent = tempfile.mkdtemp()
+                    newLocal = os.path.join(tempParent, remoteName)
+                    dir_util.copy_tree(localDir, newLocal)
+                    localDir = newLocal
+                remoteDir = '/'.join(remoteDir.rstrip('/').split('/')[:-1])
+            try:
+                self._checkCmd(["push", localDir, remoteDir],
+                               retryLimit=retryLimit, timeout=timeout)
+            except:
+                raise
+            finally:
+                if copyRequired:
+                    mozfile.remove(tempParent)
 
     def dirExists(self, remotePath):
         self._detectLsModifier()
         data = self._runCmd(["shell", "ls", self._lsModifier, remotePath + '/'],
                             timeout=self.short_timeout).output
 
         if len(data) == 1:
             res = data[0]
@@ -459,17 +485,47 @@ class DeviceManagerADB(DeviceManager):
                     return tf2.read()
                 else:
                     return tf2.read()
 
     def getFile(self, remoteFile, localFile):
         self._runPull(remoteFile, localFile)
 
     def getDirectory(self, remoteDir, localDir, checkDir=True):
+        localDir = os.path.normpath(localDir)
+        remoteDir = os.path.normpath(remoteDir)
+        copyRequired = False
+        originalLocal = localDir
+        if self._adb_version >= '1.0.36' and \
+           os.path.isdir(localDir) and self.dirExists(remoteDir):
+            # See do_sync_pull in
+            # https://android.googlesource.com/platform/system/core/+/master/adb/file_sync_client.cpp
+            # Work around change in behavior in adb 1.0.36 where if
+            # the local destination directory exists, adb pull will
+            # copy the source directory *into* the destination
+            # directory otherwise it will copy the source directory
+            # *onto* the destination directory.
+            #
+            # If the destination directory does exist, pull to its
+            # parent directory. If the source and destination leaf
+            # directory names are different, pull the source directory
+            # into a temporary directory and then copy the temporary
+            # directory onto the destination.
+            localName = os.path.basename(localDir)
+            remoteName = os.path.basename(remoteDir)
+            if localName != remoteName:
+                copyRequired = True
+                tempParent = tempfile.mkdtemp()
+                localDir = os.path.join(tempParent, remoteName)
+            else:
+                localDir = '/'.join(localDir.rstrip('/').split('/')[:-1])
         self._runCmd(["pull", remoteDir, localDir]).wait()
+        if copyRequired:
+            dir_util.copy_tree(localDir, originalLocal)
+            mozfile.remove(tempParent)
 
     def validateFile(self, remoteFile, localFile):
         md5Remote = self._getRemoteHash(remoteFile)
         md5Local = self._getLocalHash(localFile)
         if md5Remote is None or md5Local is None:
             return None
         return md5Remote == md5Local
 
@@ -677,17 +733,20 @@ class DeviceManagerADB(DeviceManager):
         """
         Check to see if adb itself can be executed.
         """
         if self._adbPath != 'adb':
             if not os.access(self._adbPath, os.X_OK):
                 raise DMError("invalid adb path, or adb not executable: %s" % self._adbPath)
 
         try:
-            self._checkCmd(["version"], timeout=self.short_timeout)
+            proc = self._runCmd(["version"], timeout=self.short_timeout)
+            re_version = re.compile(r'Android Debug Bridge version (.*)')
+            self._adb_version = re_version.match(proc.output[0]).group(1)
+            self._logger.info("Detected adb %s", self._adb_version)
         except os.error as err:
             raise DMError(
                 "unable to execute ADB (%s): ensure Android SDK is installed "
                 "and adb is in your $PATH" % err)
 
     def _verifyDevice(self):
         # If there is a device serial number, see if adb is connected to it
         if self._deviceSerial: