Backed out 4 changesets (bug 1529894) for xperf failures on a CLOSED TREE.
authorGurzau Raul <rgurzau@mozilla.com>
Wed, 06 Mar 2019 13:06:13 +0200
changeset 520458 24e79b0a187fdff462d085bc50b1adcc269e3509
parent 520457 3f01af64f1bb74cf43ded6a381ae4c2a625fcf9a
child 520459 c6b7a7d1835236a71bdc762a42f7bad27bcff4d0
push id10862
push userffxbld-merge
push dateMon, 11 Mar 2019 13:01:11 +0000
treeherdermozilla-beta@a2e7f5c935da [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1529894
milestone67.0a1
backs outd31f88692394c2212c2eca1321930b5c71a8d67f
e2d7b59776a26e24783e4eb07665e0f0cdd46aaf
3c37fd9d30d50b66b9b899c3f85758079f8e31d1
e93f4871731b8e658c7febf013cd985738ad1435
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out 4 changesets (bug 1529894) for xperf failures on a CLOSED TREE. Backed out changeset d31f88692394 (bug 1529894) Backed out changeset e2d7b59776a2 (bug 1529894) Backed out changeset 3c37fd9d30d5 (bug 1529894) Backed out changeset e93f4871731b (bug 1529894)
modules/libjar/nsZipArchive.cpp
python/mozbuild/mozpack/mozjar.py
python/mozbuild/mozpack/test/test_mozjar.py
toolkit/mozapps/installer/packager.py
--- a/modules/libjar/nsZipArchive.cpp
+++ b/modules/libjar/nsZipArchive.cpp
@@ -17,17 +17,16 @@
 #endif
 #include "nsISupportsUtils.h"
 #include "prio.h"
 #include "plstr.h"
 #include "mozilla/Attributes.h"
 #include "mozilla/Logging.h"
 #include "mozilla/UniquePtrExtensions.h"
 #include "stdlib.h"
-#include "nsDirectoryService.h"
 #include "nsWildCard.h"
 #include "nsXULAppAPI.h"
 #include "nsZipArchive.h"
 #include "nsString.h"
 #include "prenv.h"
 #if defined(XP_WIN)
 #  include <windows.h>
 #endif
@@ -74,18 +73,24 @@ static const uint16_t kSyntheticTime = 0
 static const uint16_t kSyntheticDate = (1 + (1 << 5) + (0 << 9));
 
 static uint16_t xtoint(const uint8_t *ii);
 static uint32_t xtolong(const uint8_t *ll);
 static uint32_t HashName(const char *aName, uint16_t nameLen);
 
 class ZipArchiveLogger {
  public:
-  void Init(const char *env) {
+  void Write(const nsACString &zip, const char *entry) const {
+    if (!XRE_IsParentProcess()) {
+      return;
+    }
     if (!fd) {
+      char *env = PR_GetEnv("MOZ_JAR_LOG_FILE");
+      if (!env) return;
+
       nsCOMPtr<nsIFile> logFile;
       nsresult rv = NS_NewLocalFile(NS_ConvertUTF8toUTF16(env), false,
                                     getter_AddRefs(logFile));
       if (NS_FAILED(rv)) return;
 
       // Create the log file and its parent directory (in case it doesn't exist)
       rv = logFile->Create(nsIFile::NORMAL_FILE_TYPE, 0644);
       if (NS_FAILED(rv)) return;
@@ -106,26 +111,21 @@ class ZipArchiveLogger {
       if (!file) return;
 #else
       rv = logFile->OpenNSPRFileDesc(PR_WRONLY | PR_CREATE_FILE | PR_APPEND,
                                      0644, &file);
       if (NS_FAILED(rv)) return;
 #endif
       fd = file;
     }
-  }
-
-  void Write(const nsACString &zip, const char *entry) const {
-    if (fd) {
-      nsCString buf(zip);
-      buf.Append(' ');
-      buf.Append(entry);
-      buf.Append('\n');
-      PR_Write(fd, buf.get(), buf.Length());
-    }
+    nsCString buf(zip);
+    buf.Append(' ');
+    buf.Append(entry);
+    buf.Append('\n');
+    PR_Write(fd, buf.get(), buf.Length());
   }
 
   void AddRef() {
     MOZ_ASSERT(refCnt >= 0);
     ++refCnt;
   }
 
   void Release() {
@@ -133,17 +133,17 @@ class ZipArchiveLogger {
     if ((0 == --refCnt) && fd) {
       PR_Close(fd);
       fd = nullptr;
     }
   }
 
  private:
   int refCnt;
-  PRFileDesc *fd;
+  mutable PRFileDesc *fd;
 };
 
 static ZipArchiveLogger zipLog;
 
 //***********************************************************
 // For every inflation the following allocations are done:
 // malloc(1 * 9520)
 // malloc(32768 * 1)
@@ -331,62 +331,17 @@ nsZipHandle::~nsZipHandle() {
 //  nsZipArchive::OpenArchive
 //---------------------------------------------
 nsresult nsZipArchive::OpenArchive(nsZipHandle *aZipHandle, PRFileDesc *aFd) {
   mFd = aZipHandle;
 
   //-- get table of contents for archive
   nsresult rv = BuildFileList(aFd);
   if (NS_SUCCEEDED(rv)) {
-    if (aZipHandle->mFile && XRE_IsParentProcess()) {
-      static char *env = PR_GetEnv("MOZ_JAR_LOG_FILE");
-      if (env) {
-        zipLog.Init(env);
-        // We only log accesses in jar/zip archives within the NS_GRE_DIR
-        // and/or the APK on Android. For the former, we log the archive path
-        // relative to NS_GRE_DIR, and for the latter, the nested-archive
-        // path within the APK. This makes the path match the path of the
-        // archives relative to the packaged dist/$APP_NAME directory in a
-        // build.
-        if (aZipHandle->mFile.IsZip()) {
-          // Nested archive, likely omni.ja in APK.
-          aZipHandle->mFile.GetPath(mURI);
-        } else if (nsDirectoryService::gService) {
-          // We can reach here through the initialization of Omnijar from
-          // XRE_InitCommandLine, which happens before the directory service
-          // is initialized. When that happens, it means the opened archive is
-          // the APK, and we don't care to log that one, so we just skip
-          // when the directory service is not initialized.
-          nsCOMPtr<nsIFile> dir = aZipHandle->mFile.GetBaseFile();
-          nsCOMPtr<nsIFile> gre_dir;
-          nsAutoCString path;
-          if (NS_SUCCEEDED(nsDirectoryService::gService->Get(
-                  NS_GRE_DIR, NS_GET_IID(nsIFile), getter_AddRefs(gre_dir)))) {
-            nsAutoCString leaf;
-            nsCOMPtr<nsIFile> parent;
-            while (NS_SUCCEEDED(dir->GetNativeLeafName(leaf)) &&
-                   NS_SUCCEEDED(dir->GetParent(getter_AddRefs(parent)))) {
-              if (!parent) {
-                break;
-              }
-              dir = parent;
-              if (path.Length()) {
-                path.Insert('/', 0);
-              }
-              path.Insert(leaf, 0);
-              bool equals;
-              if (NS_SUCCEEDED(dir->Equals(gre_dir, &equals)) && equals) {
-                mURI.Assign(path);
-                break;
-              }
-            }
-          }
-        }
-      }
-    }
+    if (aZipHandle->mFile) aZipHandle->mFile.GetURIString(mURI);
   }
   return rv;
 }
 
 nsresult nsZipArchive::OpenArchive(nsIFile *aFile) {
   RefPtr<nsZipHandle> handle;
 #if defined(XP_WIN)
   mozilla::AutoFDClose fd;
@@ -467,19 +422,17 @@ nsZipItem *nsZipArchive::GetItem(const c
     }
     MOZ_WIN_MEM_TRY_BEGIN
     nsZipItem *item = mFiles[HashName(aEntryName, len)];
     while (item) {
       if ((len == item->nameLength) &&
           (!memcmp(aEntryName, item->Name(), len))) {
         // Successful GetItem() is a good indicator that the file is about to be
         // read
-        if (mURI.Length()) {
-          zipLog.Write(mURI, aEntryName);
-        }
+        zipLog.Write(mURI, aEntryName);
         return item;  //-- found it
       }
       item = item->next;
     }
     MOZ_WIN_MEM_TRY_CATCH(return nullptr)
   }
   return nullptr;
 }
--- a/python/mozbuild/mozpack/mozjar.py
+++ b/python/mozbuild/mozpack/mozjar.py
@@ -9,16 +9,17 @@ import struct
 import subprocess
 import zlib
 import os
 from zipfile import (
     ZIP_STORED,
     ZIP_DEFLATED,
 )
 from collections import OrderedDict
+from urlparse import urlparse, ParseResult
 import mozpack.path as mozpath
 from mozbuild.util import memoize
 
 
 JAR_STORED = ZIP_STORED
 JAR_DEFLATED = ZIP_DEFLATED
 JAR_BROTLI = 0x81
 MAX_WBITS = 15
@@ -827,23 +828,61 @@ class BrotliCompress(object):
 
     def flush(self):
         return Brotli.compress(self._buf.getvalue())
 
 
 class JarLog(dict):
     '''
     Helper to read the file Gecko generates when setting MOZ_JAR_LOG_FILE.
-    The jar log is then available as a dict with the jar path as key, and
-    the corresponding access log as a list value. Only the first access to
-    a given member of a jar is stored.
+    The jar log is then available as a dict with the jar path as key (see
+    canonicalize for more details on the key value), and the corresponding
+    access log as a list value. Only the first access to a given member of
+    a jar is stored.
     '''
 
     def __init__(self, file=None, fileobj=None):
         if not fileobj:
             fileobj = open(file, 'r')
+        urlmap = {}
         for line in fileobj:
-            jar, path = line.strip().split(None, 1)
-            if not jar or not path:
+            url, path = line.strip().split(None, 1)
+            if not url or not path:
                 continue
+            if url not in urlmap:
+                urlmap[url] = JarLog.canonicalize(url)
+            jar = urlmap[url]
             entry = self.setdefault(jar, [])
             if path not in entry:
                 entry.append(path)
+
+    @staticmethod
+    def canonicalize(url):
+        '''
+        The jar path is stored in a MOZ_JAR_LOG_FILE log as a url. This method
+        returns a unique value corresponding to such urls.
+        - file:///{path} becomes {path}
+        - jar:file:///{path}!/{subpath} becomes ({path}, {subpath})
+        - jar:jar:file:///{path}!/{subpath}!/{subpath2} becomes
+           ({path}, {subpath}, {subpath2})
+        '''
+        if not isinstance(url, ParseResult):
+            # Assume that if it doesn't start with jar: or file:, it's a path.
+            if not url.startswith(('jar:', 'file:')):
+                url = 'file:///' + os.path.abspath(url)
+            url = urlparse(url)
+        assert url.scheme
+        assert url.scheme in ('jar', 'file')
+        if url.scheme == 'jar':
+            path = JarLog.canonicalize(url.path)
+            if isinstance(path, tuple):
+                return path[:-1] + tuple(path[-1].split('!/', 1))
+            return tuple(path.split('!/', 1))
+        if url.scheme == 'file':
+            assert os.path.isabs(url.path)
+            path = url.path
+            # On Windows, url.path will be /drive:/path ; on Unix systems,
+            # /path. As we want drive:/path instead of /drive:/path on Windows,
+            # remove the leading /.
+            if os.path.isabs(path[1:]):
+                path = path[1:]
+            path = os.path.realpath(path)
+            return mozpath.normsep(os.path.normcase(path))
--- a/python/mozbuild/mozpack/test/test_mozjar.py
+++ b/python/mozbuild/mozpack/test/test_mozjar.py
@@ -12,16 +12,17 @@ from mozpack.mozjar import (
     Deflater,
     JarLog,
 )
 from collections import OrderedDict
 from mozpack.test.test_files import MockDest
 import unittest
 import mozunit
 from cStringIO import StringIO
+from urllib import pathname2url
 import mozpack.path as mozpath
 import os
 
 
 test_data_path = mozpath.abspath(mozpath.dirname(__file__))
 test_data_path = mozpath.join(test_data_path, 'data')
 
 
@@ -284,38 +285,61 @@ class TestPreload(unittest.TestCase):
 
         self.assertEqual(files[0].filename, 'baz/qux')
         self.assertEqual(files[1].filename, 'bar')
         self.assertEqual(files[2].filename, 'foo')
 
 
 class TestJarLog(unittest.TestCase):
     def test_jarlog(self):
+        base = 'file:' + pathname2url(os.path.abspath(os.curdir))
         s = StringIO('\n'.join([
-            'bar/baz.jar first',
-            'bar/baz.jar second',
-            'bar/baz.jar third',
-            'bar/baz.jar second',
-            'bar/baz.jar second',
-            'omni.ja stuff',
-            'bar/baz.jar first',
-            'omni.ja other/stuff',
-            'omni.ja stuff',
-            'bar/baz.jar third',
+            base + '/bar/baz.jar first',
+            base + '/bar/baz.jar second',
+            base + '/bar/baz.jar third',
+            base + '/bar/baz.jar second',
+            base + '/bar/baz.jar second',
+            'jar:' + base + '/qux.zip!/omni.ja stuff',
+            base + '/bar/baz.jar first',
+            'jar:' + base + '/qux.zip!/omni.ja other/stuff',
+            'jar:' + base + '/qux.zip!/omni.ja stuff',
+            base + '/bar/baz.jar third',
+            'jar:jar:' + base + '/qux.zip!/baz/baz.jar!/omni.ja nested/stuff',
+            'jar:jar:jar:' + base + '/qux.zip!/baz/baz.jar!/foo.zip!/omni.ja' +
+            ' deeply/nested/stuff',
         ]))
         log = JarLog(fileobj=s)
+
+        def canonicalize(p):
+            return mozpath.normsep(os.path.normcase(os.path.realpath(p)))
+
+        baz_jar = canonicalize('bar/baz.jar')
+        qux_zip = canonicalize('qux.zip')
         self.assertEqual(set(log.keys()), set([
-            'bar/baz.jar',
-            'omni.ja',
+            baz_jar,
+            (qux_zip, 'omni.ja'),
+            (qux_zip, 'baz/baz.jar', 'omni.ja'),
+            (qux_zip, 'baz/baz.jar', 'foo.zip', 'omni.ja'),
         ]))
-        self.assertEqual(log['bar/baz.jar'], [
+        self.assertEqual(log[baz_jar], [
             'first',
             'second',
             'third',
         ])
-        self.assertEqual(log['omni.ja'], [
+        self.assertEqual(log[(qux_zip, 'omni.ja')], [
             'stuff',
             'other/stuff',
         ])
+        self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'omni.ja')],
+                         ['nested/stuff'])
+        self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'foo.zip',
+                              'omni.ja')], ['deeply/nested/stuff'])
+
+        # The above tests also indirectly check the value returned by
+        # JarLog.canonicalize for various jar: and file: urls, but
+        # JarLog.canonicalize also supports plain paths.
+        self.assertEqual(JarLog.canonicalize(os.path.abspath('bar/baz.jar')),
+                         baz_jar)
+        self.assertEqual(JarLog.canonicalize('bar/baz.jar'), baz_jar)
 
 
 if __name__ == '__main__':
     mozunit.main()
--- a/toolkit/mozapps/installer/packager.py
+++ b/toolkit/mozapps/installer/packager.py
@@ -319,18 +319,19 @@ def main():
 
     # Setup preloading
     if args.jarlog and os.path.exists(args.jarlog):
         from mozpack.mozjar import JarLog
         log = JarLog(args.jarlog)
         for p, f in copier:
             if not isinstance(f, Jarrer):
                 continue
-            if p in log:
-                f.preload(log[p])
+            key = JarLog.canonicalize(os.path.join(args.destination, p))
+            if key in log:
+                f.preload(log[key])
 
     copier.copy(args.destination)
     generate_precomplete(os.path.normpath(os.path.join(args.destination,
                                                        respath)))
 
 
 if __name__ == '__main__':
     main()