☠☠ backed out by a0c57dffd179 ☠ ☠ | |
author | Mike Hommey <mh+mozilla@glandium.org> |
Tue, 19 Feb 2013 11:02:12 +0100 | |
changeset 122284 | 0577eb1893c44ccce6f6085603f4b12021c01d5f |
parent 122283 | 3f0f2fc4bd0f4c9df2061a1ce9fd462a1a86fa4c |
child 122285 | b8b9100f6c418c164a8a2a1d6e70bed8019af0f9 |
child 127319 | 00f552950d2c18220622750860cc16b7a3707d90 |
push id | 24329 |
push user | mh@glandium.org |
push date | Tue, 19 Feb 2013 10:02:33 +0000 |
treeherder | mozilla-central@0577eb1893c4 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | taras, gps |
bugs | 840094 |
milestone | 21.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/build/pgo/profileserver.py +++ b/build/pgo/profileserver.py @@ -15,17 +15,17 @@ from datetime import datetime SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))) sys.path.insert(0, SCRIPT_DIR) from automation import Automation from automationutils import getDebuggerInfo, addCommonOptions PORT = 8888 PROFILE_DIRECTORY = os.path.abspath(os.path.join(SCRIPT_DIR, "./pgoprofile")) -MOZ_JAR_LOG_DIR = os.path.abspath(os.getenv("JARLOG_DIR")) +MOZ_JAR_LOG_FILE = os.path.abspath(os.getenv("JARLOG_FILE")) os.chdir(SCRIPT_DIR) class EasyServer(SocketServer.TCPServer): allow_reuse_address = True if __name__ == '__main__': from optparse import OptionParser automation = Automation() @@ -42,17 +42,17 @@ if __name__ == '__main__': t = threading.Thread(target=httpd.serve_forever) t.setDaemon(True) # don't hang on exit t.start() automation.setServerInfo("localhost", PORT) automation.initializeProfile(PROFILE_DIRECTORY) browserEnv = automation.environment() browserEnv["XPCOM_DEBUG_BREAK"] = "warn" - browserEnv["MOZ_JAR_LOG_DIR"] = MOZ_JAR_LOG_DIR + browserEnv["MOZ_JAR_LOG_FILE"] = MOZ_JAR_LOG_FILE url = "http://localhost:%d/index.html" % PORT appPath = os.path.join(SCRIPT_DIR, automation.DEFAULT_APP) status = automation.runApp(url, browserEnv, appPath, PROFILE_DIRECTORY, {}, debuggerInfo=debuggerInfo, # the profiling HTML doesn't output anything, # so let's just run this without a timeout timeout = None)
--- a/client.mk +++ b/client.mk @@ -198,17 +198,18 @@ ifdef MOZ_OBJDIR PGO_OBJDIR = $(MOZ_OBJDIR) else PGO_OBJDIR := $(TOPSRCDIR) endif profiledbuild:: $(MAKE) -f $(TOPSRCDIR)/client.mk realbuild MOZ_PROFILE_GENERATE=1 MOZ_PGO_INSTRUMENTED=1 $(MAKE) -C $(PGO_OBJDIR) package MOZ_PGO_INSTRUMENTED=1 MOZ_INTERNAL_SIGNING_FORMAT= MOZ_EXTERNAL_SIGNING_FORMAT= - MOZ_PGO_INSTRUMENTED=1 OBJDIR=${PGO_OBJDIR} JARLOG_DIR=${PGO_OBJDIR}/jarlog/en-US $(PROFILE_GEN_SCRIPT) + rm -f ${PGO_OBJDIR}/jarlog/en-US.log + MOZ_PGO_INSTRUMENTED=1 OBJDIR=${PGO_OBJDIR} JARLOG_FILE=${PGO_OBJDIR}/jarlog/en-US.log $(PROFILE_GEN_SCRIPT) $(MAKE) -f $(TOPSRCDIR)/client.mk maybe_clobber_profiledbuild $(MAKE) -f $(TOPSRCDIR)/client.mk realbuild MOZ_PROFILE_USE=1 ##################################################### # Build date unification ifdef MOZ_UNIFY_BDATE ifndef MOZ_BUILD_DATE
--- a/modules/libjar/nsZipArchive.cpp +++ b/modules/libjar/nsZipArchive.cpp @@ -50,31 +50,99 @@ # ifndef S_IFLNK # define S_IFLNK 0120000 # endif # ifndef PATH_MAX # define PATH_MAX 1024 # endif #endif /* XP_UNIX */ +#ifdef XP_WIN +#include "private/pprio.h" // To get PR_ImportFile +#endif using namespace mozilla; static const uint32_t kMaxNameLength = PATH_MAX; /* Maximum name length */ // For synthetic zip entries. Date/time corresponds to 1980-01-01 00:00. static const uint16_t kSyntheticTime = 0; static const uint16_t kSyntheticDate = (1 + (1 << 5) + (0 << 9)); static uint16_t xtoint(const uint8_t *ii); static uint32_t xtolong(const uint8_t *ll); static uint32_t HashName(const char* aName, uint16_t nameLen); #ifdef XP_UNIX static nsresult ResolveSymlink(const char *path); #endif +class ZipArchiveLogger { +public: + void Write(const nsACString &zip, const char *entry) const { + if (!fd) { + char *env = PR_GetEnv("MOZ_JAR_LOG_FILE"); + if (!env) + return; + + nsCOMPtr<nsIFile> logFile; + nsresult rv = NS_NewLocalFile(NS_ConvertUTF8toUTF16(env), false, getter_AddRefs(logFile)); + if (NS_FAILED(rv)) + return; + + // Create the log file and its parent directory (in case it doesn't exist) + logFile->Create(nsIFile::NORMAL_FILE_TYPE, 0644); + + PRFileDesc* file; +#ifdef XP_WIN + // PR_APPEND is racy on Windows, so open a handle ourselves with flags that + // will work, and use PR_ImportFile to make it a PRFileDesc. + // This can go away when bug 840435 is fixed. + nsAutoString path; + logFile->GetPath(path); + if (path.IsEmpty()) + return; + HANDLE handle = CreateFileW(path.get(), FILE_APPEND_DATA, FILE_SHARE_WRITE, + NULL, OPEN_ALWAYS, 0, NULL); + if (handle == INVALID_HANDLE_VALUE) + return; + file = PR_ImportFile((PROsfd)handle); + if (!file) + return; +#else + rv = logFile->OpenNSPRFileDesc(PR_WRONLY|PR_CREATE_FILE|PR_APPEND, 0644, &file); + if (NS_FAILED(rv)) + return; +#endif + fd = file; + } + nsCString buf(zip); + buf.Append(" "); + buf.Append(entry); + buf.Append('\n'); + PR_Write(fd, buf.get(), buf.Length()); + } + + void AddRef() { + MOZ_ASSERT(refCnt >= 0); + ++refCnt; + } + + void Release() { + MOZ_ASSERT(refCnt > 0); + if ((0 == --refCnt) && fd) { + PR_Close(fd); + fd = NULL; + } + } +private: + int refCnt; + mutable PRFileDesc *fd; +}; + +static ZipArchiveLogger zipLog; + //*********************************************************** // For every inflation the following allocations are done: // malloc(1 * 9520) // malloc(32768 * 1) //*********************************************************** nsresult gZlibInit(z_stream *zs) { @@ -184,37 +252,20 @@ nsresult nsZipArchive::OpenArchive(nsZip { mFd = aZipHandle; // Initialize our arena PL_INIT_ARENA_POOL(&mArena, "ZipArena", ZIP_ARENABLOCKSIZE); //-- get table of contents for archive nsresult rv = BuildFileList(); - char *env = PR_GetEnv("MOZ_JAR_LOG_DIR"); - if (env && NS_SUCCEEDED(rv) && aZipHandle->mFile) { - nsCOMPtr<nsIFile> logFile; - nsresult rv2 = NS_NewLocalFile(NS_ConvertUTF8toUTF16(env), false, getter_AddRefs(logFile)); - - if (!NS_SUCCEEDED(rv2)) - return rv; - - // Create a directory for the log (in case it doesn't exist) - logFile->Create(nsIFile::DIRECTORY_TYPE, 0700); - - nsAutoString name; - nsCOMPtr<nsIFile> file = aZipHandle->mFile.GetBaseFile(); - file->GetLeafName(name); - name.Append(NS_LITERAL_STRING(".log")); - logFile->Append(name); - - PRFileDesc* fd; - rv2 = logFile->OpenNSPRFileDesc(PR_WRONLY|PR_CREATE_FILE|PR_APPEND, 0644, &fd); - if (NS_SUCCEEDED(rv2)) - mLog = fd; + if (NS_SUCCEEDED(rv)) { + zipLog.AddRef(); + if (aZipHandle->mFile) + aZipHandle->mFile.GetURIString(mURI); } return rv; } nsresult nsZipArchive::OpenArchive(nsIFile *aFile) { nsRefPtr<nsZipHandle> handle; nsresult rv = nsZipHandle::Init(aFile, getter_AddRefs(handle)); @@ -271,16 +322,17 @@ nsresult nsZipArchive::CloseArchive() // We don't need to delete each of the nsZipItem as the memory for // the zip item and the filename it holds are both allocated from the Arena. // Hence, destroying the Arena is like destroying all the memory // for all the nsZipItem in one shot. But if the ~nsZipItem is doing // anything more than cleaning up memory, we should start calling it. // Let us also cleanup the mFiles table for re-use on the next 'open' call memset(mFiles, 0, sizeof(mFiles)); mBuiltSynthetics = false; + zipLog.Release(); return NS_OK; } //--------------------------------------------- // nsZipArchive::GetItem //--------------------------------------------- nsZipItem* nsZipArchive::GetItem(const char * aEntryName) { @@ -295,23 +347,18 @@ nsZipItem* nsZipArchive::GetItem(const } } MOZ_WIN_MEM_TRY_BEGIN nsZipItem* item = mFiles[ HashName(aEntryName, len) ]; while (item) { if ((len == item->nameLength) && (!memcmp(aEntryName, item->Name(), len))) { - if (mLog) { - // Successful GetItem() is a good indicator that the file is about to be read - char *tmp = PL_strdup(aEntryName); - tmp[len]='\n'; - PR_Write(mLog, tmp, len+1); - PL_strfree(tmp); - } + // Successful GetItem() is a good indicator that the file is about to be read + zipLog.Write(mURI, aEntryName); return item; //-- found it } item = item->next; } MOZ_WIN_MEM_TRY_CATCH(return nullptr) } return nullptr; }
--- a/modules/libjar/nsZipArchive.h +++ b/modules/libjar/nsZipArchive.h @@ -212,19 +212,18 @@ private: uint16_t mCommentLen; // Whether we synthesized the directory entries bool mBuiltSynthetics; // file handle nsRefPtr<nsZipHandle> mFd; - // logging handle - mozilla::AutoFDClose mLog; - + // file URI, for logging + nsCString mURI; private: //--- private methods --- nsZipItem* CreateZipItem(); nsresult BuildFileList(); nsresult BuildSynthetics(); nsZipArchive& operator=(const nsZipArchive& rhs) MOZ_DELETE;
--- a/python/mozbuild/mozpack/chrome/manifest.py +++ b/python/mozbuild/mozpack/chrome/manifest.py @@ -344,17 +344,17 @@ def parse_manifest(root, path, fileobj=N Parse a manifest file. ''' base = mozpack.path.dirname(path) if root: path = os.path.normpath(os.path.abspath(os.path.join(root, path))) if not fileobj: fileobj = open(path) linenum = 0 - for line in fileobj.readlines(): + for line in fileobj: linenum += 1 with errors.context(path, linenum): e = parse_manifest_line(base, line) if e: yield e def is_manifest(path):
--- a/python/mozbuild/mozpack/mozjar.py +++ b/python/mozbuild/mozpack/mozjar.py @@ -6,16 +6,18 @@ from io import BytesIO import struct import zlib import os from zipfile import ( ZIP_STORED, ZIP_DEFLATED, ) from collections import OrderedDict +from urlparse import urlparse, ParseResult +import mozpack.path JAR_STORED = ZIP_STORED JAR_DEFLATED = ZIP_DEFLATED MAX_WBITS = 15 class JarReaderError(Exception): '''Error type for Jar reader errors.''' @@ -274,16 +276,22 @@ class JarFileReader(object): def readlines(self): ''' Return a list containing all the lines of data in the uncompressed data. ''' return self.read().splitlines(True) + def __iter__(self): + ''' + Iterator, to support the "for line in fileobj" constructs. + ''' + return iter(self.readlines()) + def seek(self, pos, whence=os.SEEK_SET): ''' Change the current position in the uncompressed data. Subsequent reads will start from there. ''' return self.uncompressed_data.seek(pos, whence) def close(self): @@ -727,8 +735,65 @@ class Deflater(object): ''' Return the compressed data, if the data should be compressed (real compressed size smaller than the uncompressed size), or the uncompressed data otherwise. ''' if self.compressed: return self._deflated.getvalue() return self._data.getvalue() + + +class JarLog(dict): + ''' + Helper to read the file Gecko generates when setting MOZ_JAR_LOG_FILE. + The jar log is then available as a dict with the jar path as key (see + canonicalize for more details on the key value), and the corresponding + access log as a list value. Only the first access to a given member of + a jar is stored. + ''' + def __init__(self, file=None, fileobj=None): + if not fileobj: + fileobj = open(file, 'r') + urlmap = {} + for line in fileobj: + url, path = line.strip().split(None, 1) + if not url or not path: + continue + if url not in urlmap: + urlmap[url] = JarLog.canonicalize(url) + jar = urlmap[url] + entry = self.setdefault(jar, []) + if path not in entry: + entry.append(path) + + @staticmethod + def canonicalize(url): + ''' + The jar path is stored in a MOZ_JAR_LOG_FILE log as a url. This method + returns a unique value corresponding to such urls. + - file:///{path} becomes {path} + - jar:file:///{path}!/{subpath} becomes ({path}, {subpath}) + - jar:jar:file:///{path}!/{subpath}!/{subpath2} becomes + ({path}, {subpath}, {subpath2}) + ''' + if not isinstance(url, ParseResult): + # Assume that if it doesn't start with jar: or file:, it's a path. + if not url.startswith(('jar:', 'file:')): + url = 'file:///' + os.path.abspath(url) + url = urlparse(url) + assert url.scheme + assert url.scheme in ('jar', 'file') + if url.scheme == 'jar': + path = JarLog.canonicalize(url.path) + if isinstance(path, tuple): + return path[:-1] + tuple(path[-1].split('!/', 1)) + return tuple(path.split('!/', 1)) + if url.scheme == 'file': + assert os.path.isabs(url.path) + path = url.path + # On Windows, url.path will be /drive:/path ; on Unix systems, + # /path. As we want drive:/path instead of /drive:/path on Windows, + # remove the leading /. + if os.path.isabs(path[1:]): + path = path[1:] + path = os.path.realpath(path) + return mozpack.path.normsep(os.path.normcase(path))
--- a/python/mozbuild/mozpack/test/test_mozjar.py +++ b/python/mozbuild/mozpack/test/test_mozjar.py @@ -4,21 +4,26 @@ from mozpack.mozjar import ( JarReaderError, JarWriterError, JarStruct, JarReader, JarWriter, Deflater, - OrderedDict, + JarLog, ) +from collections import OrderedDict from mozpack.test.test_files import MockDest import unittest import mozunit +from cStringIO import StringIO +from urllib import pathname2url +import mozpack.path +import os class TestJarStruct(unittest.TestCase): class Foo(JarStruct): MAGIC = 0x01020304 STRUCT = OrderedDict([ ('foo', 'uint32'), ('bar', 'uint16'), @@ -250,10 +255,61 @@ class TestPreload(unittest.TestCase): self.assertEqual(jar.last_preloaded, 'bar') files = [j for j in jar] self.assertEqual(files[0].filename, 'baz/qux') self.assertEqual(files[1].filename, 'bar') self.assertEqual(files[2].filename, 'foo') +class TestJarLog(unittest.TestCase): + def test_jarlog(self): + base = 'file:' + pathname2url(os.path.abspath(os.curdir)) + s = StringIO('\n'.join([ + base + '/bar/baz.jar first', + base + '/bar/baz.jar second', + base + '/bar/baz.jar third', + base + '/bar/baz.jar second', + base + '/bar/baz.jar second', + 'jar:' + base + '/qux.zip!/omni.ja stuff', + base + '/bar/baz.jar first', + 'jar:' + base + '/qux.zip!/omni.ja other/stuff', + 'jar:' + base + '/qux.zip!/omni.ja stuff', + base + '/bar/baz.jar third', + 'jar:jar:' + base + '/qux.zip!/baz/baz.jar!/omni.ja nested/stuff', + 'jar:jar:jar:' + base + '/qux.zip!/baz/baz.jar!/foo.zip!/omni.ja' + + ' deeply/nested/stuff', + ])) + log = JarLog(fileobj=s) + canonicalize = lambda p: \ + mozpack.path.normsep(os.path.normcase(os.path.realpath(p))) + baz_jar = canonicalize('bar/baz.jar') + qux_zip = canonicalize('qux.zip') + self.assertEqual(set(log.keys()), set([ + baz_jar, + (qux_zip, 'omni.ja'), + (qux_zip, 'baz/baz.jar', 'omni.ja'), + (qux_zip, 'baz/baz.jar', 'foo.zip', 'omni.ja'), + ])) + self.assertEqual(log[baz_jar], [ + 'first', + 'second', + 'third', + ]) + self.assertEqual(log[(qux_zip, 'omni.ja')], [ + 'stuff', + 'other/stuff', + ]) + self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'omni.ja')], + ['nested/stuff']) + self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'foo.zip', + 'omni.ja')], ['deeply/nested/stuff']) + + # The above tests also indirectly check the value returned by + # JarLog.canonicalize for various jar: and file: urls, but + # JarLog.canonicalize also supports plain paths. + self.assertEqual(JarLog.canonicalize(os.path.abspath('bar/baz.jar')), + baz_jar) + self.assertEqual(JarLog.canonicalize('bar/baz.jar'), baz_jar) + + if __name__ == '__main__': mozunit.main()
--- a/toolkit/mozapps/installer/packager.mk +++ b/toolkit/mozapps/installer/packager.mk @@ -93,17 +93,17 @@ JSSHELL_BINS += \ $(NULL) endif endif # MOZ_NATIVE_NSPR MAKE_JSSHELL = $(ZIP) -9j $(PKG_JSSHELL) $(JSSHELL_BINS) endif # LIBXUL_SDK _ABS_DIST = $(call core_abspath,$(DIST)) JARLOG_DIR = $(call core_abspath,$(DEPTH)/jarlog/) -JARLOG_DIR_AB_CD = $(JARLOG_DIR)/$(AB_CD) +JARLOG_FILE_AB_CD = $(JARLOG_DIR)/$(AB_CD).log TAR_CREATE_FLAGS := --exclude=.mkdir.done $(TAR_CREATE_FLAGS) CREATE_FINAL_TAR = $(TAR) -c --owner=0 --group=0 --numeric-owner \ --mode="go-w" --exclude=.mkdir.done -f UNPACK_TAR = tar -xf- ifeq ($(MOZ_PKG_FORMAT),TAR) PKG_SUFFIX = .tar @@ -586,17 +586,17 @@ endif export NO_PKG_FILES USE_ELF_HACK ELF_HACK_FLAGS _BINPATH stage-package: $(MOZ_PKG_MANIFEST) @rm -rf $(DIST)/$(PKG_PATH)$(PKG_BASENAME).tar $(DIST)/$(PKG_PATH)$(PKG_BASENAME).dmg $@ $(EXCLUDE_LIST) $(PYTHON) $(MOZILLA_DIR)/toolkit/mozapps/installer/packager.py $(DEFINES) \ --format $(MOZ_PACKAGER_FORMAT) \ $(addprefix --removals ,$(MOZ_PKG_REMOVALS)) \ $(if $(filter-out 0,$(MOZ_PKG_FATAL_WARNINGS)),,--ignore-errors) \ $(if $(MOZ_PACKAGER_MINIFY),--minify) \ - $(if $(JARLOG_DIR),--jarlogs $(JARLOG_DIR_AB_CD)) \ + $(if $(JARLOG_DIR),$(addprefix --jarlog ,$(wildcard $(JARLOG_FILE_AB_CD)))) \ $(if $(OPTIMIZEJARS),--optimizejars) \ $(addprefix --unify ,$(UNIFY_DIST)) \ $(MOZ_PKG_MANIFEST) $(DIST) $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR) \ $(if $(filter omni,$(MOZ_PACKAGER_FORMAT)),$(if $(NON_OMNIJAR_FILES),--non-resource $(NON_OMNIJAR_FILES))) $(PYTHON) $(MOZILLA_DIR)/toolkit/mozapps/installer/find-dupes.py $(DIST)/$(MOZ_PKG_DIR) ifndef LIBXUL_SDK ifdef MOZ_PACKAGE_JSSHELL # Package JavaScript Shell
--- a/toolkit/mozapps/installer/packager.py +++ b/toolkit/mozapps/installer/packager.py @@ -225,18 +225,18 @@ def main(): help='Choose the chrome format for packaging ' + '(omni, jar or flat ; default: %(default)s)') parser.add_argument('--removals', default=None, help='removed-files source file') parser.add_argument('--ignore-errors', action='store_true', default=False, help='Transform errors into warnings.') parser.add_argument('--minify', action='store_true', default=False, help='Make some files more compact while packaging') - parser.add_argument('--jarlogs', default='', help='Base directory where ' + - 'to find jar content access logs') + parser.add_argument('--jarlog', default='', help='File containing jar ' + + 'access logs') parser.add_argument('--optimizejars', action='store_true', default=False, help='Enable jar optimizations') parser.add_argument('--unify', default='', help='Base directory of another build to unify with') parser.add_argument('manifest', default=None, nargs='?', help='Manifest file name') parser.add_argument('source', help='Source directory') parser.add_argument('destination', help='Destination directory') @@ -326,23 +326,25 @@ def main(): % (buildconfig.substs['DLL_PREFIX'], lib) libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX']) if copier.contains(libname): copier.add(libbase + '.chk', LibSignFile(os.path.join(args.destination, libname))) # Setup preloading - if args.jarlogs: - jarlogs = FileFinder(args.jarlogs) - for p, log in jarlogs: - if p.endswith('.log'): - p = p[:-4] - if copier.contains(p) and isinstance(copier[p], Jarrer): - copier[p].preload([l.strip() for l in log.open().readlines()]) + if args.jarlog and os.path.exists(args.jarlog): + from mozpack.mozjar import JarLog + log = JarLog(args.jarlog) + for p, f in copier: + if not isinstance(f, Jarrer): + continue + key = JarLog.canonicalize(os.path.join(args.destination, p)) + if key in log: + f.preload(log[key]) # Fill startup cache if isinstance(formatter, OmniJarFormatter) and launcher.can_launch(): if buildconfig.substs['LIBXUL_SDK']: gre_path = buildconfig.substs['LIBXUL_DIST'] else: gre_path = None for base in sorted([[p for p in [mozpack.path.join('bin', b), b]