Bug 683127 part 4 - Use the new ELF linker Zip reader in the old linker. r=tglek
authorMike Hommey <mh+mozilla@glandium.org>
Wed, 11 Jan 2012 11:11:02 +0100
changeset 85484 52edf42878930241de0331a601481eca9d410360
parent 85483 ea78ef72f47fe254cd6beea23425417af9d83490
child 85485 37c00830f5991e2f0283631bd51288d9a03b34c2
push id805
push userakeybl@mozilla.com
push dateWed, 01 Feb 2012 18:17:35 +0000
treeherdermozilla-aurora@6fb3bf232436 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstglek
bugs683127
milestone12.0a1
Bug 683127 part 4 - Use the new ELF linker Zip reader in the old linker. r=tglek
mozglue/android/APKOpen.cpp
mozglue/android/Makefile.in
toolkit/mozapps/installer/packager.mk
--- a/mozglue/android/APKOpen.cpp
+++ b/mozglue/android/APKOpen.cpp
@@ -56,92 +56,36 @@
 #include <endian.h>
 #include <unistd.h>
 #include <zlib.h>
 #include <linux/ashmem.h>
 #include "dlfcn.h"
 #include "APKOpen.h"
 #include <sys/time.h>
 #include <sys/resource.h>
+#include "Zip.h"
 
 /* Android headers don't define RUSAGE_THREAD */
 #ifndef RUSAGE_THREAD
 #define RUSAGE_THREAD 1
 #endif
 
-/* compression methods */
-#define STORE    0
-#define DEFLATE  8
-#define LZMA    14
-
 enum StartupEvent {
 #define mozilla_StartupTimeline_Event(ev, z) ev,
 #include "StartupTimeline.h"
 #undef mozilla_StartupTimeline_Event
 };
 
 static uint64_t *sStartupTimeline;
 
 void StartupTimeline_Record(StartupEvent ev, struct timeval *tm)
 {
   sStartupTimeline[ev] = (((uint64_t)tm->tv_sec * 1000000LL) + (uint64_t)tm->tv_usec);
 }
 
-struct local_file_header {
-  uint32_t signature;
-  uint16_t min_version;
-  uint16_t general_flag;
-  uint16_t compression;
-  uint16_t lastmod_time;
-  uint16_t lastmod_date;
-  uint32_t crc32;
-  uint32_t compressed_size;
-  uint32_t uncompressed_size;
-  uint16_t filename_size;
-  uint16_t extra_field_size;
-  char     data[0];
-} __attribute__((__packed__));
-
-struct cdir_entry {
-  uint32_t signature;
-  uint16_t creator_version;
-  uint16_t min_version;
-  uint16_t general_flag;
-  uint16_t compression;
-  uint16_t lastmod_time;
-  uint16_t lastmod_date;
-  uint32_t crc32;
-  uint32_t compressed_size;
-  uint32_t uncompressed_size;
-  uint16_t filename_size;
-  uint16_t extra_field_size;
-  uint16_t file_comment_size;
-  uint16_t disk_num;
-  uint16_t internal_attr;
-  uint32_t external_attr;
-  uint32_t offset;
-  char     data[0];
-} __attribute__((__packed__));
-
-#define CDIR_END_SIG 0x06054b50
-
-struct cdir_end {
-  uint32_t signature;
-  uint16_t disk_num;
-  uint16_t cdir_disk;
-  uint16_t disk_entries;
-  uint16_t cdir_entries;
-  uint32_t cdir_size;
-  uint32_t cdir_offset;
-  uint16_t comment_size;
-  char     comment[0];
-} __attribute__((__packed__));
-
-static size_t zip_size;
-static int zip_fd;
 static struct mapping_info * lib_mapping = NULL;
 
 NS_EXPORT const struct mapping_info *
 getLibraryMapping()
 {
   return lib_mapping;
 }
 
@@ -159,62 +103,16 @@ createAshmem(size_t bytes, const char *n
 
   if (!ioctl(fd, ASHMEM_SET_SIZE, bytes))
     return fd;
 
   close(fd);
   return -1;
 }
 
-static void * map_file (const char *file)
-{
-  int fd = open(file, O_RDONLY);
-  if (fd == -1) {
-    __android_log_print(ANDROID_LOG_ERROR, "GeckoMapFile", "map_file open %s", strerror(errno));
-    return NULL;
-  }
-
-  zip_fd = fd;
-  struct stat s;
-  if (fstat(fd, &s) == -1) {
-    __android_log_print(ANDROID_LOG_ERROR, "GeckoMapFile", "map_file fstat %s", strerror(errno));
-    return NULL;
-  }
-
-  zip_size = s.st_size;
-  void *addr = mmap(NULL, zip_size, PROT_READ, MAP_SHARED, fd, 0);
-  if (addr == MAP_FAILED) {
-    __android_log_print(ANDROID_LOG_ERROR, "GeckoMapFile", "map_file mmap %s", strerror(errno));
-    return NULL;
-  }
-
-  return addr;
-}
-
-static uint32_t cdir_entry_size (struct cdir_entry *entry)
-{
-  return sizeof(*entry) +
-         letoh16(entry->filename_size) +
-         letoh16(entry->extra_field_size) +
-         letoh16(entry->file_comment_size);
-}
-
-static struct cdir_entry *
-find_cdir_entry (struct cdir_entry *entry, int count, const char *name)
-{
-  size_t name_size = strlen(name);
-  while (count--) {
-    if (letoh16(entry->filename_size) == name_size &&
-        !memcmp(entry->data, name, name_size))
-      return entry;
-    entry = (struct cdir_entry *)((char *)entry + cdir_entry_size(entry));
-  }
-  return NULL;
-}
-
 #define SHELL_WRAPPER0(name) \
 typedef void (*name ## _t)(JNIEnv *, jclass); \
 static name ## _t f_ ## name; \
 extern "C" NS_EXPORT void JNICALL \
 Java_org_mozilla_gecko_GeckoAppShell_ ## name(JNIEnv *jenv, jclass jc) \
 { \
   f_ ## name(jenv, jc); \
 }
@@ -305,19 +203,19 @@ static void * xul_handle = NULL;
 static time_t apk_mtime = 0;
 #ifdef DEBUG
 extern "C" int extractLibs = 1;
 #else
 extern "C" int extractLibs = 0;
 #endif
 
 static void
-extractFile(const char * path, const struct cdir_entry *entry, void * data)
+extractFile(const char * path, Zip::Stream &s)
 {
-  uint32_t size = letoh32(entry->uncompressed_size);
+  uint32_t size = s.GetUncompressedSize();
 
   struct stat status;
   if (!stat(path, &status) &&
       status.st_size == size &&
       apk_mtime < status.st_mtime)
     return;
 
   int fd = open(path, O_CREAT | O_NOATIME | O_TRUNC | O_RDWR, S_IRWXU);
@@ -336,18 +234,18 @@ extractFile(const char * path, const str
                     MAP_SHARED, fd, 0);
   if (buf == (void *)-1) {
     __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "Couldn't mmap decompression buffer");
     close(fd);
     return;
   }
 
   z_stream strm = {
-    next_in: (Bytef *)data,
-    avail_in: letoh32(entry->compressed_size),
+    next_in: (Bytef *)s.GetBuffer(),
+    avail_in: s.GetSize(),
     total_in: 0,
 
     next_out: (Bytef *)buf,
     avail_out: size,
     total_out: 0
   };
 
   int ret;
@@ -370,43 +268,43 @@ extractFile(const char * path, const str
   /* We just extracted data that is going to be executed in the future.
    * We thus need to ensure Instruction and Data cache coherency. */
   cacheflush((unsigned) buf, (unsigned) buf + size, 0);
 #endif
   munmap(buf, size);
 }
 
 static void
-extractLib(const struct cdir_entry *entry, void * data, void * dest)
+extractLib(Zip::Stream &s, void * dest)
 {
   z_stream strm = {
-    next_in: (Bytef *)data,
-    avail_in: letoh32(entry->compressed_size),
+    next_in: (Bytef *)s.GetBuffer(),
+    avail_in: s.GetSize(),
     total_in: 0,
 
     next_out: (Bytef *)dest,
-    avail_out: letoh32(entry->uncompressed_size),
+    avail_out: s.GetUncompressedSize(),
     total_out: 0
   };
 
   int ret;
   ret = inflateInit2(&strm, -MAX_WBITS);
   if (ret != Z_OK)
     __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "inflateInit failed: %s", strm.msg);
 
   ret = inflate(&strm, Z_SYNC_FLUSH);
   if (ret != Z_STREAM_END)
     __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "inflate failed: %s", strm.msg);
 
   ret = inflateEnd(&strm);
   if (ret != Z_OK)
     __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "inflateEnd failed: %s", strm.msg);
 
-  if (strm.total_out != letoh32(entry->uncompressed_size))
-    __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "File not fully uncompressed! %d / %d", strm.total_out, letoh32(entry->uncompressed_size));
+  if (strm.total_out != s.GetUncompressedSize())
+    __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "File not fully uncompressed! %d / %d", strm.total_out, s.GetUncompressedSize());
 }
 
 static int cache_count = 0;
 static struct lib_cache_info *cache_mapping = NULL;
 
 NS_EXPORT const struct lib_cache_info *
 getLibraryCache()
 {
@@ -475,55 +373,53 @@ addLibCacheFd(const char *libName, int f
 
   struct lib_cache_info *info = &cache_mapping[cache_count++];
   strncpy(info->name, libName, MAX_LIB_CACHE_NAME_LEN - 1);
   info->fd = fd;
   info->lib_size = lib_size;
   info->buffer = buffer;
 }
 
-static void * mozload(const char * path, void *zip,
-                      struct cdir_entry *cdir_start, uint16_t cdir_entries)
+static void * mozload(const char * path, Zip *zip)
 {
 #ifdef DEBUG
   struct timeval t0, t1;
   gettimeofday(&t0, 0);
 #endif
 
-  struct cdir_entry *entry = find_cdir_entry(cdir_start, cdir_entries, path);
-  struct local_file_header *file = (struct local_file_header *)((char *)zip + letoh32(entry->offset));
-  void * data = ((char *)&file->data) + letoh16(file->filename_size) + letoh16(file->extra_field_size);
-  void * handle;
+  void *handle;
+  Zip::Stream s;
+  if (!zip->GetStream(path, &s))
+    return NULL;
 
   if (extractLibs) {
     char fullpath[PATH_MAX];
     snprintf(fullpath, PATH_MAX, "%s/%s", getenv("CACHE_PATH"), path);
     __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "resolved %s to %s", path, fullpath);
-    extractFile(fullpath, entry, data);
+    extractFile(fullpath, s);
     handle = __wrap_dlopen(fullpath, RTLD_LAZY);
     if (!handle)
       __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "Couldn't load %s because %s", fullpath, __wrap_dlerror());
 #ifdef DEBUG
     gettimeofday(&t1, 0);
     __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "%s: spent %d", path,
                         (((long long)t1.tv_sec * 1000000LL) +
                           (long long)t1.tv_usec) -
                         (((long long)t0.tv_sec * 1000000LL) +
                           (long long)t0.tv_usec));
 #endif
     return handle;
   }
 
-  size_t offset = letoh32(entry->offset) + sizeof(*file) + letoh16(file->filename_size) + letoh16(file->extra_field_size);
   bool skipLibCache = false;
-  int fd = zip_fd;
+  int fd;
   void * buf = NULL;
-  uint32_t lib_size = letoh32(entry->uncompressed_size);
+  uint32_t lib_size = s.GetUncompressedSize();
   int cache_fd = 0;
-  if (letoh16(file->compression) == DEFLATE) {
+  if (s.GetType() == Zip::Stream::DEFLATE) {
     cache_fd = lookupLibCacheFd(path);
     fd = cache_fd;
     if (fd < 0)
       fd = createAshmem(lib_size, path);
 #ifdef DEBUG
     else
       __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "Loading %s from cache", path);
 #endif
@@ -535,72 +431,68 @@ static void * mozload(const char * path,
                PROT_READ | PROT_WRITE,
                MAP_SHARED, fd, 0);
     if (buf == (void *)-1) {
       __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "Couldn't mmap decompression buffer");
       close(fd);
       return NULL;
     }
 
-    offset = 0;
-
     if (cache_fd < 0) {
-      extractLib(entry, data, buf);
+      extractLib(s, buf);
 #ifdef ANDROID_ARM_LINKER
       /* We just extracted data that is going to be executed in the future.
        * We thus need to ensure Instruction and Data cache coherency. */
-      cacheflush((unsigned) buf, (unsigned) buf + entry->uncompressed_size, 0);
+      cacheflush((unsigned) buf, (unsigned) buf + s.GetUncompressedSize(), 0);
 #endif
       addLibCacheFd(path, fd, lib_size, buf);
     }
 
     // preload libxul, to avoid slowly demand-paging it
     if (!strcmp(path, "libxul.so"))
-      madvise(buf, entry->uncompressed_size, MADV_WILLNEED);
-    data = buf;
+      madvise(buf, s.GetUncompressedSize(), MADV_WILLNEED);
   }
 
 #ifdef DEBUG
   __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "Loading %s with len %d (0x%08x) and offset %d (0x%08x)", path, lib_size, lib_size, offset, offset);
 #endif
 
-  handle = moz_mapped_dlopen(path, RTLD_LAZY, fd, data,
-                             lib_size, offset);
+  handle = moz_mapped_dlopen(path, RTLD_LAZY, fd, buf,
+                             lib_size, 0);
   if (!handle)
     __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "Couldn't load %s because %s", path, __wrap_dlerror());
 
   if (buf)
     munmap(buf, lib_size);
 
 #ifdef DEBUG
   gettimeofday(&t1, 0);
   __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "%s: spent %d", path, (((long long)t1.tv_sec * 1000000LL) + (long long)t1.tv_usec) -
                (((long long)t0.tv_sec * 1000000LL) + (long long)t0.tv_usec));
 #endif
 
   return handle;
 }
 
 static void *
-extractBuf(const char * path, void *zip,
-           struct cdir_entry *cdir_start, uint16_t cdir_entries)
+extractBuf(const char * path, Zip *zip)
 {
-  struct cdir_entry *entry = find_cdir_entry(cdir_start, cdir_entries, path);
-  struct local_file_header *file = (struct local_file_header *)((char *)zip + letoh32(entry->offset));
-  void * data = ((char *)&file->data) + letoh16(file->filename_size) + letoh16(file->extra_field_size);
+  Zip::Stream s;
+  if (!zip->GetStream(path, &s))
+    return NULL;
 
-  void * buf = malloc(letoh32(entry->uncompressed_size));
+  void * buf = malloc(s.GetUncompressedSize());
   if (buf == (void *)-1) {
     __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "Couldn't alloc decompression buffer for %s", path);
     return NULL;
   }
-  if (letoh16(file->compression) == DEFLATE)
-    extractLib(entry, data, buf);
+  if (s.GetType() == Zip::Stream::DEFLATE)
+    extractLib(s, buf);
   else
-    memcpy(buf, data, letoh32(entry->uncompressed_size));
+    memcpy(buf, s.GetBuffer(), s.GetUncompressedSize());
 
   return buf;
 }
 
 static int mapping_count = 0;
 static char *file_ids = NULL;
 
 #define MAX_MAPPING_INFO 32
@@ -635,54 +527,41 @@ loadLibs(const char *apkName)
   if (!stat(apkName, &status))
     apk_mtime = status.st_mtime;
 
   struct timeval t0, t1;
   gettimeofday(&t0, 0);
   struct rusage usage1;
   getrusage(RUSAGE_THREAD, &usage1);
   
-  void *zip = map_file(apkName);
-  struct cdir_end *dirend = (struct cdir_end *)((char *)zip + zip_size - sizeof(*dirend));
-  while ((void *)dirend > zip &&
-         letoh32(dirend->signature) != CDIR_END_SIG)
-    dirend = (struct cdir_end *)((char *)dirend - 1);
-  if (letoh32(dirend->signature) != CDIR_END_SIG) {
-    __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "Couldn't find end of central directory record");
-    return;
-  }
-
-  uint32_t cdir_offset = letoh32(dirend->cdir_offset);
-  uint16_t cdir_entries = letoh16(dirend->cdir_entries);
-
-  struct cdir_entry *cdir_start = (struct cdir_entry *)((char *)zip + cdir_offset);
+  Zip *zip = new Zip(apkName);
 
   lib_mapping = (struct mapping_info *)calloc(MAX_MAPPING_INFO, sizeof(*lib_mapping));
 #ifdef MOZ_CRASHREPORTER
-  file_ids = (char *)extractBuf("lib.id", zip, cdir_start, cdir_entries);
+  file_ids = (char *)extractBuf("lib.id", zip);
 #endif
 
-#define MOZLOAD(name) mozload("lib" name ".so", zip, cdir_start, cdir_entries)
+#define MOZLOAD(name) mozload("lib" name ".so", zip)
   MOZLOAD("mozalloc");
   MOZLOAD("nspr4");
   MOZLOAD("plc4");
   MOZLOAD("plds4");
   MOZLOAD("mozsqlite3");
   MOZLOAD("nssutil3");
   MOZLOAD("nss3");
   MOZLOAD("ssl3");
   MOZLOAD("smime3");
   xul_handle = MOZLOAD("xul");
   MOZLOAD("xpcom");
   MOZLOAD("nssckbi");
   MOZLOAD("freebl3");
   MOZLOAD("softokn3");
 #undef MOZLOAD
 
-  close(zip_fd);
+  delete zip;
 
 #ifdef MOZ_CRASHREPORTER
   free(file_ids);
   file_ids = NULL;
 #endif
 
   if (!xul_handle)
     __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "Couldn't get a handle to libxul!");
--- a/mozglue/android/Makefile.in
+++ b/mozglue/android/Makefile.in
@@ -50,16 +50,17 @@ DEFINES += \
   -DANDROID_PACKAGE_NAME='"$(ANDROID_PACKAGE_NAME)"' \
   $(NULL)
 
 CPPSRCS = \
   nsGeckoUtils.cpp \
   APKOpen.cpp \
   $(NULL)
 
+LOCAL_INCLUDES += -I$(srcdir)/../linker
 LOCAL_INCLUDES += -I$(topsrcdir)/toolkit/components/startup
 ifdef MOZ_OLD_LINKER
 LOCAL_INCLUDES += -I$(topsrcdir)/other-licenses/android
 ifeq ($(CPU_ARCH),arm)
 DEFINES += -DANDROID_ARM_LINKER
 endif
 endif
 
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -260,41 +260,47 @@ endif #Create an RPM file
 
 ifeq ($(MOZ_PKG_FORMAT),APK)
 
 JAVA_CLASSPATH = $(ANDROID_SDK)/android.jar
 include $(topsrcdir)/config/android-common.mk
 
 JARSIGNER ?= echo
 
-DIST_FILES = \
-  resources.arsc \
-  AndroidManifest.xml \
-  chrome \
-  components \
-  defaults \
-  modules \
-  hyphenation \
-  res \
-  lib \
-  lib.id \
+DIST_FILES =
+
+# Place the files in the order they are going to be opened by the linker
+ifdef MOZ_CRASHREPORTER
+DIST_FILES += lib.id
+endif
+
+DIST_FILES += \
   libmozalloc.so \
   libnspr4.so \
   libplc4.so \
   libplds4.so \
   libmozsqlite3.so \
   libnssutil3.so \
   libnss3.so \
   libssl3.so \
   libsmime3.so \
   libxul.so \
   libxpcom.so \
   libnssckbi.so \
   libfreebl3.so \
   libsoftokn3.so \
+  resources.arsc \
+  AndroidManifest.xml \
+  chrome \
+  components \
+  defaults \
+  modules \
+  hyphenation \
+  res \
+  lib \
   extensions \
   application.ini \
   package-name.txt \
   platform.ini \
   greprefs.js \
   browserconfig.properties \
   blocklist.xml \
   chrome.manifest \