Bug 683127 part 9 - Allow to temporarily extract Elf files from a Zip archive for e.g. valgrind. r=tglek,a=blassey
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 20 Jan 2012 09:48:50 +0100
changeset 85124 c390403ee7cb06922fe3ba9205a638c526a1c097
parent 85123 88c009db8205340e9dcf9a986ba074a45e9a2518
child 85125 9d162c7fb4642e94852381ab23bdcf627711c517
push id519
push userakeybl@mozilla.com
push dateWed, 01 Feb 2012 00:38:35 +0000
treeherdermozilla-beta@788ea1ef610b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstglek, blassey
bugs683127
milestone11.0a2
Bug 683127 part 9 - Allow to temporarily extract Elf files from a Zip archive for e.g. valgrind. r=tglek,a=blassey
mozglue/linker/ElfLoader.cpp
mozglue/linker/ElfLoader.h
mozglue/linker/Mappable.cpp
mozglue/linker/Mappable.h
mozglue/linker/Utils.h
mozglue/linker/Zip.h
--- a/mozglue/linker/ElfLoader.cpp
+++ b/mozglue/linker/ElfLoader.cpp
@@ -198,18 +198,28 @@ ElfLoader::Load(const char *path, int fl
   RefPtr<Zip> zip;
   const char *subpath;
   if ((subpath = strchr(path, '!'))) {
     char *zip_path = strndup(path, subpath - path);
     while (*(++subpath) == '/') { }
     zip = zips.GetZip(zip_path);
     Zip::Stream s;
     if (zip && zip->GetStream(subpath, &s)) {
-      if (s.GetType() == Zip::Stream::DEFLATE)
-        mappable = MappableDeflate::Create(name, zip, &s);
+      if (s.GetType() == Zip::Stream::DEFLATE) {
+        /* When the MOZ_LINKER_EXTRACT environment variable is set to "1",
+         * compressed libraries are going to be (temporarily) extracted as
+         * files, in the directory pointed by the MOZ_LINKER_CACHE
+         * environment variable. */
+        const char *extract = getenv("MOZ_LINKER_EXTRACT");
+        if (extract && !strncmp(extract, "1", 2 /* Including '\0' */))
+          mappable = MappableExtractFile::Create(name, &s);
+        /* The above may fail in some cases. */
+        if (!mappable)
+          mappable = MappableDeflate::Create(name, zip, &s);
+      }
     }
   }
   /* If we couldn't load above, try with a MappableFile */
   if (!mappable && !zip)
     mappable = MappableFile::Create(path);
 
   /* Try loading with the custom linker if we have a Mappable */
   if (mappable)
--- a/mozglue/linker/ElfLoader.h
+++ b/mozglue/linker/ElfLoader.h
@@ -1,16 +1,17 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef ElfLoader_h
 #define ElfLoader_h
 
 #include <vector>
+#include <dlfcn.h>
 /* Until RefPtr.h stops using JS_Assert */
 #undef DEBUG
 #include "mozilla/RefPtr.h"
 #include "Zip.h"
 
 /**
  * dlfcn.h replacement functions
  */
--- a/mozglue/linker/Mappable.cpp
+++ b/mozglue/linker/Mappable.cpp
@@ -1,21 +1,23 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include <fcntl.h>
 #include <unistd.h>
 #include <sys/mman.h>
+#include <sys/stat.h>
 #include <cstring>
 #include <cstdlib>
 #include "Mappable.h"
 #ifdef ANDROID
 #include <linux/ashmem.h>
 #endif
+#include "ElfLoader.h"
 #include "Logging.h"
 
 #ifndef PAGE_SIZE
 #define PAGE_SIZE 4096
 #endif
 
 #ifndef PAGE_MASK
 #define PAGE_MASK (~ (PAGE_SIZE - 1))
@@ -54,16 +56,80 @@ MappableFile::mmap(const void *addr, siz
 
 void
 MappableFile::finalize()
 {
   /* Close file ; equivalent to close(fd.forget()) */
   fd = -1;
 }
 
+MappableExtractFile *
+MappableExtractFile::Create(const char *name, Zip::Stream *stream)
+{
+  const char *cachePath = getenv("MOZ_LINKER_CACHE");
+  if (!cachePath || !*cachePath) {
+    log("Warning: MOZ_LINKER_EXTRACT is set, but not MOZ_LINKER_CACHE; "
+        "not extracting");
+    return NULL;
+  }
+  AutoDeleteArray<char> path = new char[strlen(cachePath) + strlen(name) + 2];
+  sprintf(path, "%s/%s", cachePath, name);
+  debug("Extracting to %s", (char *)path);
+  AutoCloseFD fd = open(path, O_TRUNC | O_RDWR | O_CREAT | O_NOATIME,
+                              S_IRUSR | S_IWUSR);
+  if (fd == -1) {
+    log("Couldn't open %s to decompress library", path.get());
+    return NULL;
+  }
+  AutoUnlinkFile file = path.forget();
+  if (ftruncate(fd, stream->GetUncompressedSize()) == -1) {
+    log("Couldn't ftruncate %s to decompress library", file.get());
+    return NULL;
+  }
+  /* Map the temporary file for use as inflate buffer */
+  MappedPtr buffer(::mmap(NULL, stream->GetUncompressedSize(), PROT_WRITE,
+                          MAP_SHARED, fd, 0), stream->GetUncompressedSize());
+  if (buffer == MAP_FAILED) {
+    log("Couldn't map %s to decompress library", file.get());
+    return NULL;
+  }
+  z_stream zStream = stream->GetZStream(buffer);
+
+  /* Decompress */
+  if (inflateInit2(&zStream, -MAX_WBITS) != Z_OK) {
+    log("inflateInit failed: %s", zStream.msg);
+    return NULL;
+  }
+  if (inflate(&zStream, Z_FINISH) != Z_STREAM_END) {
+    log("inflate failed: %s", zStream.msg);
+    return NULL;
+  }
+  if (inflateEnd(&zStream) != Z_OK) {
+    log("inflateEnd failed: %s", zStream.msg);
+    return NULL;
+  }
+  if (zStream.total_out != stream->GetUncompressedSize()) {
+    log("File not fully uncompressed! %ld / %d", zStream.total_out,
+        static_cast<unsigned int>(stream->GetUncompressedSize()));
+    return NULL;
+  }
+
+  return new MappableExtractFile(fd.forget(), file.forget());
+}
+
+MappableExtractFile::~MappableExtractFile()
+{
+  /* When destroying from a forked process, we don't want the file to be
+   * removed, as the main process is still using the file. Although it
+   * doesn't really matter, it helps e.g. valgrind that the file is there.
+   * The string still needs to be delete[]d, though */
+  if (pid != getpid())
+    delete [] path.forget();
+}
+
 /**
  * _MappableBuffer is a buffer which content can be mapped at different
  * locations in the virtual address space.
  * On Linux, uses a (deleted) temporary file on a tmpfs for sharable content.
  * On Android, uses ashmem.
  */
 class _MappableBuffer: public MappedPtr
 {
@@ -160,33 +226,19 @@ MappableDeflate::Create(const char *name
   _MappableBuffer *buf = _MappableBuffer::Create(name, stream->GetUncompressedSize());
   if (buf)
     return new MappableDeflate(buf, zip, stream);
   return NULL;
 }
 
 MappableDeflate::MappableDeflate(_MappableBuffer *buf, Zip *zip,
                                  Zip::Stream *stream)
-: zip(zip), buffer(buf)
-{
-  /* Initialize Zlib data with zip stream info and decompression buffer */
-  memset(&zStream, 0, sizeof(zStream));
-  zStream.avail_in = stream->GetSize();
-  zStream.next_in = const_cast<Bytef *>(
-                    reinterpret_cast<const Bytef *>(stream->GetBuffer()));
-  zStream.total_in = 0;
-  zStream.avail_out = stream->GetUncompressedSize();
-  zStream.next_out = static_cast<Bytef*>(*buffer);
-  zStream.total_out = 0;
-}
+: zip(zip), buffer(buf), zStream(stream->GetZStream(*buf)) { }
 
-MappableDeflate::~MappableDeflate()
-{
-  delete buffer;
-}
+MappableDeflate::~MappableDeflate() { }
 
 void *
 MappableDeflate::mmap(const void *addr, size_t length, int prot, int flags, off_t offset)
 {
   // ASSERT(buffer)
   // ASSERT(! flags & MAP_SHARED)
   flags |= MAP_PRIVATE;
 
@@ -235,13 +287,12 @@ MappableDeflate::mmap(const void *addr, 
 
   return buffer->mmap(addr, length, prot, flags, offset);
 }
 
 void
 MappableDeflate::finalize()
 {
   /* Free decompression buffer */
-  delete buffer;
   buffer = NULL;
   /* Remove reference to Zip archive */
   zip = NULL;
 }
--- a/mozglue/linker/Mappable.h
+++ b/mozglue/linker/Mappable.h
@@ -1,15 +1,16 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef Mappable_h
 #define Mappable_h
 
+#include <sys/types.h>
 #include "Zip.h"
 #include "mozilla/RefPtr.h"
 #include "zlib.h"
 
 /**
  * Abstract class to handle mmap()ing from various kind of entities, such as
  * plain files or Zip entries. The virtual members are meant to act as the
  * equivalent system functions, with a few differences:
@@ -47,27 +48,75 @@ public:
    * Create a MappableFile instance for the given file path.
    */
   static MappableFile *Create(const char *path);
 
   /* Inherited from Mappable */
   virtual void *mmap(const void *addr, size_t length, int prot, int flags, off_t offset);
   virtual void finalize();
 
-private:
+protected:
   MappableFile(int fd): fd(fd) { }
 
+private:
   /* File descriptor */
   AutoCloseFD fd;
 };
 
+/**
+ * Mappable implementation for deflated stream in a Zip archive
+ * Inflates the complete stream into a cache file.
+ */
+class MappableExtractFile: public MappableFile
+{
+public:
+  ~MappableExtractFile();
+
+  /**
+   * Create a MappableExtractFile instance for the given Zip stream. The name
+   * argument is used to create the cache file in the cache directory.
+   */
+  static MappableExtractFile *Create(const char *name, Zip::Stream *stream);
+
+  /**
+   * Returns the path of the extracted file.
+   */
+  char *GetPath() {
+    return path;
+  }
+private:
+  MappableExtractFile(int fd, char *path)
+  : MappableFile(fd), path(path), pid(getpid()) { }
+
+  /**
+   * AutoUnlinkFile keeps track or a file name and removes (unlinks) the file
+   * when the instance is destroyed.
+   */
+  struct AutoUnlinkFileTraits: public AutoDeleteArrayTraits<char>
+  {
+    static void clean(char *value)
+    {
+      unlink(value);
+      AutoDeleteArrayTraits<char>::clean(value);
+    }
+  };
+  typedef AutoClean<AutoUnlinkFileTraits> AutoUnlinkFile;
+
+  /* Extracted file */
+  AutoUnlinkFile path;
+
+  /* Id of the process that initialized the instance */
+  pid_t pid;
+};
+
 class _MappableBuffer;
 
 /**
- * Mappable implementation for deflated stream in a Zip archive
+ * Mappable implementation for deflated stream in a Zip archive.
+ * Inflates the mapped bits in a temporary buffer. 
  */
 class MappableDeflate: public Mappable
 {
 public:
   ~MappableDeflate();
 
   /**
    * Create a MappableDeflate instance for the given Zip stream. The name
@@ -82,15 +131,15 @@ public:
 
 private:
   MappableDeflate(_MappableBuffer *buf, Zip *zip, Zip::Stream *stream);
 
   /* Zip reference */
   mozilla::RefPtr<Zip> zip;
 
   /* Decompression buffer */
-  _MappableBuffer *buffer;
+  AutoDeletePtr<_MappableBuffer> buffer;
 
   /* Zlib data */
   z_stream zStream;
 };
 
 #endif /* Mappable_h */
--- a/mozglue/linker/Utils.h
+++ b/mozglue/linker/Utils.h
@@ -46,57 +46,133 @@ private:
 /**
  * Type definitions
  */
 typedef le_to_cpu<unsigned char> le_uint16;
 typedef le_to_cpu<le_uint16> le_uint32;
 #endif
 
 /**
- * AutoCloseFD is a RAII wrapper for POSIX file descriptors
+ * AutoClean is a helper to create RAII wrappers
+ * The Traits class is expected to look like the following:
+ *   struct Traits {
+ *     // Define the type of the value stored in the wrapper
+ *     typedef value_type type;
+ *     // Returns the value corresponding to the uninitialized or freed state
+ *     const static type None();
+ *     // Cleans up resources corresponding to the wrapped value
+ *     const static void clean(type);
+ *   }
  */
-class AutoCloseFD
+template <typename Traits>
+class AutoClean
 {
+  typedef typename Traits::type T;
 public:
-  AutoCloseFD(): fd(-1) { }
-  AutoCloseFD(int fd): fd(fd) { }
-  ~AutoCloseFD()
+  AutoClean(): value(Traits::None()) { }
+  AutoClean(const T& value): value(value) { }
+  ~AutoClean()
   {
-    if (fd != -1)
-      close(fd);
-  }
-
-  operator int() const
-  {
-    return fd;
+    if (value != Traits::None())
+      Traits::clean(value);
   }
 
-  int forget()
+  operator const T&() const { return value; }
+  const T& operator->() const { return value; }
+  const T& get() const { return value; }
+
+  T forget()
   {
-    int _fd = fd;
-    fd = -1;
-    return _fd;
+    T _value = value;
+    value = Traits::None();
+    return _value;
   }
 
-  bool operator ==(int other) const
+  bool operator ==(T other) const
   {
-    return fd == other;
+    return value == other;
   }
 
-  int operator =(int other)
+  AutoClean& operator =(T other)
   {
-    if (fd != -1)
-      close(fd);
-    fd = other;
-    return fd;
+    if (value != Traits::None())
+      Traits::clean(value);
+    value = other;
+    return *this;
   }
 
 private:
-  int fd;
+  T value;
+};
+
+/**
+ * AUTOCLEAN_TEMPLATE defines a templated class derived from AutoClean
+ * This allows to implement templates such as AutoFreePtr.
+ */
+#define AUTOCLEAN_TEMPLATE(name, Traits) \
+template <typename T> \
+struct name: public AutoClean<Traits<T> > \
+{ \
+  using AutoClean<Traits<T> >::operator =; \
+  name(): AutoClean<Traits<T> >() { } \
+  name(typename Traits<T>::type ptr): AutoClean<Traits<T> >(ptr) { } \
+}
+
+/**
+ * AutoCloseFD is a RAII wrapper for POSIX file descriptors
+ */
+struct AutoCloseFDTraits
+{
+  typedef int type;
+  static int None() { return -1; }
+  static void clean(int fd) { close(fd); }
 };
+typedef AutoClean<AutoCloseFDTraits> AutoCloseFD;
+
+/**
+ * AutoFreePtr is a RAII wrapper for pointers that need to be free()d.
+ *
+ *   struct S { ... };
+ *   AutoFreePtr<S> foo = malloc(sizeof(S));
+ *   AutoFreePtr<char> bar = strdup(str);
+ */
+template <typename T>
+struct AutoFreePtrTraits
+{
+  typedef T *type;
+  static T *None() { return NULL; }
+  static void clean(T *ptr) { free(ptr); }
+};
+AUTOCLEAN_TEMPLATE(AutoFreePtr, AutoFreePtrTraits);
+
+/**
+ * AutoDeletePtr is a RAII wrapper for pointers that need to be deleted.
+ *
+ *   struct S { ... };
+ *   AutoDeletePtr<S> foo = new S();
+ */
+template <typename T>
+struct AutoDeletePtrTraits: public AutoFreePtrTraits<T>
+{
+  static void clean(T *ptr) { delete ptr; }
+};
+AUTOCLEAN_TEMPLATE(AutoDeletePtr, AutoDeletePtrTraits);
+
+/**
+ * AutoDeleteArray is a RAII wrapper for pointers that need to be delete[]ed.
+ *
+ *   struct S { ... };
+ *   AutoDeleteArray<S> foo = new S[42];
+ */
+template <typename T>
+struct AutoDeleteArrayTraits: public AutoFreePtrTraits<T>
+{
+  static void clean(T *ptr) { delete [] ptr; }
+};
+AUTOCLEAN_TEMPLATE(AutoDeleteArray, AutoDeleteArrayTraits);
 
 /**
  * MappedPtr is a RAII wrapper for mmap()ed memory. It can be used as
  * a simple void * or unsigned char *.
  *
  * It is defined as a derivative of a template that allows to use a
  * different unmapping strategy.
  */
--- a/mozglue/linker/Zip.h
+++ b/mozglue/linker/Zip.h
@@ -3,16 +3,17 @@
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef Zip_h
 #define Zip_h
 
 #include <cstring>
 #include <stdint.h>
 #include <vector>
+#include <zlib.h>
 #include "Utils.h"
 /* Until RefPtr.h stops using JS_Assert */
 #undef DEBUG
 #include "mozilla/RefPtr.h"
 
 /**
  * Forward declaration
  */
@@ -56,21 +57,41 @@ public:
     };
 
     /**
      * Constructor
      */
     Stream(): compressedBuf(NULL), compressedSize(0), uncompressedSize(0)
             , type(STORE) { }
 
+    /**
+     * Getters
+     */
     const void *GetBuffer() { return compressedBuf; }
     size_t GetSize() { return compressedSize; }
     size_t GetUncompressedSize() { return uncompressedSize; }
     Type GetType() { return type; }
 
+    /**
+     * Returns a z_stream for use with inflate functions using the given
+     * buffer as inflate output. The caller is expected to allocate enough
+     * memory for the Stream uncompressed size.
+     */
+    z_stream GetZStream(void *buf)
+    {
+      z_stream zStream;
+      memset(&zStream, 0, sizeof(zStream));
+      zStream.avail_in = compressedSize;
+      zStream.next_in = reinterpret_cast<Bytef *>(
+                        const_cast<void *>(compressedBuf));
+      zStream.avail_out = uncompressedSize;
+      zStream.next_out = static_cast<Bytef *>(buf);
+      return zStream;
+    }
+
   protected:
     friend class Zip;
     const void *compressedBuf;
     size_t compressedSize;
     size_t uncompressedSize;
     Type type;
   };