Bug 683127 part 9 - Allow to temporarily extract Elf files from a Zip archive for e.g. valgrind. r=tglek
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 20 Jan 2012 09:48:50 +0100
changeset 84941 229140e62d7b49f34cb140a639d21f92aaa39d97
parent 84940 bd752f4935d978c6e81f04c632ea054701f64a08
child 84942 41c7ad654949db5393d22a95d8dd4d233d47f244
push idunknown
push userunknown
push dateunknown
reviewerstglek
bugs683127
milestone12.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 683127 part 9 - Allow to temporarily extract Elf files from a Zip archive for e.g. valgrind. r=tglek
mozglue/linker/ElfLoader.cpp
mozglue/linker/ElfLoader.h
mozglue/linker/Mappable.cpp
mozglue/linker/Mappable.h
mozglue/linker/Utils.h
mozglue/linker/Zip.h
--- a/mozglue/linker/ElfLoader.cpp
+++ b/mozglue/linker/ElfLoader.cpp
@@ -198,18 +198,28 @@ ElfLoader::Load(const char *path, int fl
   RefPtr<Zip> zip;
   const char *subpath;
   if ((subpath = strchr(path, '!'))) {
     char *zip_path = strndup(path, subpath - path);
     while (*(++subpath) == '/') { }
     zip = zips.GetZip(zip_path);
     Zip::Stream s;
     if (zip && zip->GetStream(subpath, &s)) {
-      if (s.GetType() == Zip::Stream::DEFLATE)
-        mappable = MappableDeflate::Create(name, zip, &s);
+      if (s.GetType() == Zip::Stream::DEFLATE) {
+        /* When the MOZ_LINKER_EXTRACT environment variable is set to "1",
+         * compressed libraries are going to be (temporarily) extracted as
+         * files, in the directory pointed by the MOZ_LINKER_CACHE
+         * environment variable. */
+        const char *extract = getenv("MOZ_LINKER_EXTRACT");
+        if (extract && !strncmp(extract, "1", 2 /* Including '\0' */))
+          mappable = MappableExtractFile::Create(name, &s);
+        /* The above may fail in some cases. */
+        if (!mappable)
+          mappable = MappableDeflate::Create(name, zip, &s);
+      }
     }
   }
   /* If we couldn't load above, try with a MappableFile */
   if (!mappable && !zip)
     mappable = MappableFile::Create(path);
 
   /* Try loading with the custom linker if we have a Mappable */
   if (mappable)
--- a/mozglue/linker/ElfLoader.h
+++ b/mozglue/linker/ElfLoader.h
@@ -1,16 +1,17 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef ElfLoader_h
 #define ElfLoader_h
 
 #include <vector>
+#include <dlfcn.h>
 /* Until RefPtr.h stops using JS_Assert */
 #undef DEBUG
 #include "mozilla/RefPtr.h"
 #include "Zip.h"
 
 /**
  * dlfcn.h replacement functions
  */
--- a/mozglue/linker/Mappable.cpp
+++ b/mozglue/linker/Mappable.cpp
@@ -1,21 +1,23 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include <fcntl.h>
 #include <unistd.h>
 #include <sys/mman.h>
+#include <sys/stat.h>
 #include <cstring>
 #include <cstdlib>
 #include "Mappable.h"
 #ifdef ANDROID
 #include <linux/ashmem.h>
 #endif
+#include "ElfLoader.h"
 #include "Logging.h"
 
 #ifndef PAGE_SIZE
 #define PAGE_SIZE 4096
 #endif
 
 #ifndef PAGE_MASK
 #define PAGE_MASK (~ (PAGE_SIZE - 1))
@@ -54,16 +56,80 @@ MappableFile::mmap(const void *addr, siz
 
 void
 MappableFile::finalize()
 {
   /* Close file ; equivalent to close(fd.forget()) */
   fd = -1;
 }
 
+MappableExtractFile *
+MappableExtractFile::Create(const char *name, Zip::Stream *stream)
+{
+  const char *cachePath = getenv("MOZ_LINKER_CACHE");
+  if (!cachePath || !*cachePath) {
+    log("Warning: MOZ_LINKER_EXTRACT is set, but not MOZ_LINKER_CACHE; "
+        "not extracting");
+    return NULL;
+  }
+  AutoDeleteArray<char> path = new char[strlen(cachePath) + strlen(name) + 2];
+  sprintf(path, "%s/%s", cachePath, name);
+  debug("Extracting to %s", (char *)path);
+  AutoCloseFD fd = open(path, O_TRUNC | O_RDWR | O_CREAT | O_NOATIME,
+                              S_IRUSR | S_IWUSR);
+  if (fd == -1) {
+    log("Couldn't open %s to decompress library", path.get());
+    return NULL;
+  }
+  AutoUnlinkFile file = path.forget();
+  if (ftruncate(fd, stream->GetUncompressedSize()) == -1) {
+    log("Couldn't ftruncate %s to decompress library", file.get());
+    return NULL;
+  }
+  /* Map the temporary file for use as inflate buffer */
+  MappedPtr buffer(::mmap(NULL, stream->GetUncompressedSize(), PROT_WRITE,
+                          MAP_SHARED, fd, 0), stream->GetUncompressedSize());
+  if (buffer == MAP_FAILED) {
+    log("Couldn't map %s to decompress library", file.get());
+    return NULL;
+  }
+  z_stream zStream = stream->GetZStream(buffer);
+
+  /* Decompress */
+  if (inflateInit2(&zStream, -MAX_WBITS) != Z_OK) {
+    log("inflateInit failed: %s", zStream.msg);
+    return NULL;
+  }
+  if (inflate(&zStream, Z_FINISH) != Z_STREAM_END) {
+    log("inflate failed: %s", zStream.msg);
+    return NULL;
+  }
+  if (inflateEnd(&zStream) != Z_OK) {
+    log("inflateEnd failed: %s", zStream.msg);
+    return NULL;
+  }
+  if (zStream.total_out != stream->GetUncompressedSize()) {
+    log("File not fully uncompressed! %ld / %d", zStream.total_out,
+        static_cast<unsigned int>(stream->GetUncompressedSize()));
+    return NULL;
+  }
+
+  return new MappableExtractFile(fd.forget(), file.forget());
+}
+
+MappableExtractFile::~MappableExtractFile()
+{
+  /* When destroying from a forked process, we don't want the file to be
+   * removed, as the main process is still using the file. Although it
+   * doesn't really matter, it helps e.g. valgrind that the file is there.
+   * The string still needs to be delete[]d, though */
+  if (pid != getpid())
+    delete [] path.forget();
+}
+
 /**
  * _MappableBuffer is a buffer which content can be mapped at different
  * locations in the virtual address space.
  * On Linux, uses a (deleted) temporary file on a tmpfs for sharable content.
  * On Android, uses ashmem.
  */
 class _MappableBuffer: public MappedPtr
 {
@@ -160,33 +226,19 @@ MappableDeflate::Create(const char *name
   _MappableBuffer *buf = _MappableBuffer::Create(name, stream->GetUncompressedSize());
   if (buf)
     return new MappableDeflate(buf, zip, stream);
   return NULL;
 }
 
 MappableDeflate::MappableDeflate(_MappableBuffer *buf, Zip *zip,
                                  Zip::Stream *stream)
-: zip(zip), buffer(buf)
-{
-  /* Initialize Zlib data with zip stream info and decompression buffer */
-  memset(&zStream, 0, sizeof(zStream));
-  zStream.avail_in = stream->GetSize();
-  zStream.next_in = const_cast<Bytef *>(
-                    reinterpret_cast<const Bytef *>(stream->GetBuffer()));
-  zStream.total_in = 0;
-  zStream.avail_out = stream->GetUncompressedSize();
-  zStream.next_out = static_cast<Bytef*>(*buffer);
-  zStream.total_out = 0;
-}
+: zip(zip), buffer(buf), zStream(stream->GetZStream(*buf)) { }
 
-MappableDeflate::~MappableDeflate()
-{
-  delete buffer;
-}
+MappableDeflate::~MappableDeflate() { }
 
 void *
 MappableDeflate::mmap(const void *addr, size_t length, int prot, int flags, off_t offset)
 {
   // ASSERT(buffer)
   // ASSERT(! flags & MAP_SHARED)
   flags |= MAP_PRIVATE;
 
@@ -235,13 +287,12 @@ MappableDeflate::mmap(const void *addr, 
 
   return buffer->mmap(addr, length, prot, flags, offset);
 }
 
 void
 MappableDeflate::finalize()
 {
   /* Free decompression buffer */
-  delete buffer;
   buffer = NULL;
   /* Remove reference to Zip archive */
   zip = NULL;
 }
--- a/mozglue/linker/Mappable.h
+++ b/mozglue/linker/Mappable.h
@@ -1,15 +1,16 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef Mappable_h
 #define Mappable_h
 
+#include <sys/types.h>
 #include "Zip.h"
 #include "mozilla/RefPtr.h"
 #include "zlib.h"
 
 /**
  * Abstract class to handle mmap()ing from various kind of entities, such as
  * plain files or Zip entries. The virtual members are meant to act as the
  * equivalent system functions, with a few differences:
@@ -47,27 +48,75 @@ public:
    * Create a MappableFile instance for the given file path.
    */
   static MappableFile *Create(const char *path);
 
   /* Inherited from Mappable */
   virtual void *mmap(const void *addr, size_t length, int prot, int flags, off_t offset);
   virtual void finalize();
 
-private:
+protected:
   MappableFile(int fd): fd(fd) { }
 
+private:
   /* File descriptor */
   AutoCloseFD fd;
 };
 
+/**
+ * Mappable implementation for deflated stream in a Zip archive
+ * Inflates the complete stream into a cache file.
+ */
+class MappableExtractFile: public MappableFile
+{
+public:
+  ~MappableExtractFile();
+
+  /**
+   * Create a MappableExtractFile instance for the given Zip stream. The name
+   * argument is used to create the cache file in the cache directory.
+   */
+  static MappableExtractFile *Create(const char *name, Zip::Stream *stream);
+
+  /**
+   * Returns the path of the extracted file.
+   */
+  char *GetPath() {
+    return path;
+  }
+private:
+  MappableExtractFile(int fd, char *path)
+  : MappableFile(fd), path(path), pid(getpid()) { }
+
+  /**
+   * AutoUnlinkFile keeps track or a file name and removes (unlinks) the file
+   * when the instance is destroyed.
+   */
+  struct AutoUnlinkFileTraits: public AutoDeleteArrayTraits<char>
+  {
+    static void clean(char *value)
+    {
+      unlink(value);
+      AutoDeleteArrayTraits<char>::clean(value);
+    }
+  };
+  typedef AutoClean<AutoUnlinkFileTraits> AutoUnlinkFile;
+
+  /* Extracted file */
+  AutoUnlinkFile path;
+
+  /* Id of the process that initialized the instance */
+  pid_t pid;
+};
+
 class _MappableBuffer;
 
 /**
- * Mappable implementation for deflated stream in a Zip archive
+ * Mappable implementation for deflated stream in a Zip archive.
+ * Inflates the mapped bits in a temporary buffer. 
  */
 class MappableDeflate: public Mappable
 {
 public:
   ~MappableDeflate();
 
   /**
    * Create a MappableDeflate instance for the given Zip stream. The name
@@ -82,15 +131,15 @@ public:
 
 private:
   MappableDeflate(_MappableBuffer *buf, Zip *zip, Zip::Stream *stream);
 
   /* Zip reference */
   mozilla::RefPtr<Zip> zip;
 
   /* Decompression buffer */
-  _MappableBuffer *buffer;
+  AutoDeletePtr<_MappableBuffer> buffer;
 
   /* Zlib data */
   z_stream zStream;
 };
 
 #endif /* Mappable_h */
--- a/mozglue/linker/Utils.h
+++ b/mozglue/linker/Utils.h
@@ -46,57 +46,133 @@ private:
 /**
  * Type definitions
  */
 typedef le_to_cpu<unsigned char> le_uint16;
 typedef le_to_cpu<le_uint16> le_uint32;
 #endif
 
 /**
- * AutoCloseFD is a RAII wrapper for POSIX file descriptors
+ * AutoClean is a helper to create RAII wrappers
+ * The Traits class is expected to look like the following:
+ *   struct Traits {
+ *     // Define the type of the value stored in the wrapper
+ *     typedef value_type type;
+ *     // Returns the value corresponding to the uninitialized or freed state
+ *     const static type None();
+ *     // Cleans up resources corresponding to the wrapped value
+ *     const static void clean(type);
+ *   }
  */
-class AutoCloseFD
+template <typename Traits>
+class AutoClean
 {
+  typedef typename Traits::type T;
 public:
-  AutoCloseFD(): fd(-1) { }
-  AutoCloseFD(int fd): fd(fd) { }
-  ~AutoCloseFD()
+  AutoClean(): value(Traits::None()) { }
+  AutoClean(const T& value): value(value) { }
+  ~AutoClean()
   {
-    if (fd != -1)
-      close(fd);
-  }
-
-  operator int() const
-  {
-    return fd;
+    if (value != Traits::None())
+      Traits::clean(value);
   }
 
-  int forget()
+  operator const T&() const { return value; }
+  const T& operator->() const { return value; }
+  const T& get() const { return value; }
+
+  T forget()
   {
-    int _fd = fd;
-    fd = -1;
-    return _fd;
+    T _value = value;
+    value = Traits::None();
+    return _value;
   }
 
-  bool operator ==(int other) const
+  bool operator ==(T other) const
   {
-    return fd == other;
+    return value == other;
   }
 
-  int operator =(int other)
+  AutoClean& operator =(T other)
   {
-    if (fd != -1)
-      close(fd);
-    fd = other;
-    return fd;
+    if (value != Traits::None())
+      Traits::clean(value);
+    value = other;
+    return *this;
   }
 
 private:
-  int fd;
+  T value;
+};
+
+/**
+ * AUTOCLEAN_TEMPLATE defines a templated class derived from AutoClean
+ * This allows to implement templates such as AutoFreePtr.
+ */
+#define AUTOCLEAN_TEMPLATE(name, Traits) \
+template <typename T> \
+struct name: public AutoClean<Traits<T> > \
+{ \
+  using AutoClean<Traits<T> >::operator =; \
+  name(): AutoClean<Traits<T> >() { } \
+  name(typename Traits<T>::type ptr): AutoClean<Traits<T> >(ptr) { } \
+}
+
+/**
+ * AutoCloseFD is a RAII wrapper for POSIX file descriptors
+ */
+struct AutoCloseFDTraits
+{
+  typedef int type;
+  static int None() { return -1; }
+  static void clean(int fd) { close(fd); }
 };
+typedef AutoClean<AutoCloseFDTraits> AutoCloseFD;
+
+/**
+ * AutoFreePtr is a RAII wrapper for pointers that need to be free()d.
+ *
+ *   struct S { ... };
+ *   AutoFreePtr<S> foo = malloc(sizeof(S));
+ *   AutoFreePtr<char> bar = strdup(str);
+ */
+template <typename T>
+struct AutoFreePtrTraits
+{
+  typedef T *type;
+  static T *None() { return NULL; }
+  static void clean(T *ptr) { free(ptr); }
+};
+AUTOCLEAN_TEMPLATE(AutoFreePtr, AutoFreePtrTraits);
+
+/**
+ * AutoDeletePtr is a RAII wrapper for pointers that need to be deleted.
+ *
+ *   struct S { ... };
+ *   AutoDeletePtr<S> foo = new S();
+ */
+template <typename T>
+struct AutoDeletePtrTraits: public AutoFreePtrTraits<T>
+{
+  static void clean(T *ptr) { delete ptr; }
+};
+AUTOCLEAN_TEMPLATE(AutoDeletePtr, AutoDeletePtrTraits);
+
+/**
+ * AutoDeleteArray is a RAII wrapper for pointers that need to be delete[]ed.
+ *
+ *   struct S { ... };
+ *   AutoDeleteArray<S> foo = new S[42];
+ */
+template <typename T>
+struct AutoDeleteArrayTraits: public AutoFreePtrTraits<T>
+{
+  static void clean(T *ptr) { delete [] ptr; }
+};
+AUTOCLEAN_TEMPLATE(AutoDeleteArray, AutoDeleteArrayTraits);
 
 /**
  * MappedPtr is a RAII wrapper for mmap()ed memory. It can be used as
  * a simple void * or unsigned char *.
  *
  * It is defined as a derivative of a template that allows to use a
  * different unmapping strategy.
  */
--- a/mozglue/linker/Zip.h
+++ b/mozglue/linker/Zip.h
@@ -3,16 +3,17 @@
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef Zip_h
 #define Zip_h
 
 #include <cstring>
 #include <stdint.h>
 #include <vector>
+#include <zlib.h>
 #include "Utils.h"
 /* Until RefPtr.h stops using JS_Assert */
 #undef DEBUG
 #include "mozilla/RefPtr.h"
 
 /**
  * Forward declaration
  */
@@ -56,21 +57,41 @@ public:
     };
 
     /**
      * Constructor
      */
     Stream(): compressedBuf(NULL), compressedSize(0), uncompressedSize(0)
             , type(STORE) { }
 
+    /**
+     * Getters
+     */
     const void *GetBuffer() { return compressedBuf; }
     size_t GetSize() { return compressedSize; }
     size_t GetUncompressedSize() { return uncompressedSize; }
     Type GetType() { return type; }
 
+    /**
+     * Returns a z_stream for use with inflate functions using the given
+     * buffer as inflate output. The caller is expected to allocate enough
+     * memory for the Stream uncompressed size.
+     */
+    z_stream GetZStream(void *buf)
+    {
+      z_stream zStream;
+      memset(&zStream, 0, sizeof(zStream));
+      zStream.avail_in = compressedSize;
+      zStream.next_in = reinterpret_cast<Bytef *>(
+                        const_cast<void *>(compressedBuf));
+      zStream.avail_out = uncompressedSize;
+      zStream.next_out = static_cast<Bytef *>(buf);
+      return zStream;
+    }
+
   protected:
     friend class Zip;
     const void *compressedBuf;
     size_t compressedSize;
     size_t uncompressedSize;
     Type type;
   };