Bug 686805 part 4 - Make the linker load libraries with on-demand decompression when they are seekable compressed streams. r=tglek,r=sewardj
--- a/mozglue/linker/CustomElf.h
+++ b/mozglue/linker/CustomElf.h
@@ -321,16 +321,17 @@ private:
{
/* C++ doesn't allow direct conversion between pointer-to-object
* and pointer-to-function. */
union {
void *ptr;
void (*func)(void);
} f;
f.ptr = ptr;
+ debug("%s: Calling function @%p", GetPath(), ptr);
f.func();
}
/**
* Call a function given a an address relative to the library base
*/
void CallFunction(Elf::Addr addr) const
{
--- a/mozglue/linker/ElfLoader.cpp
+++ b/mozglue/linker/ElfLoader.cpp
@@ -221,27 +221,29 @@ ElfLoader::Load(const char *path, int fl
RefPtr<Zip> zip;
const char *subpath;
if ((subpath = strchr(path, '!'))) {
char *zip_path = strndup(path, subpath - path);
while (*(++subpath) == '/') { }
zip = zips.GetZip(zip_path);
Zip::Stream s;
if (zip && zip->GetStream(subpath, &s)) {
- if (s.GetType() == Zip::Stream::DEFLATE) {
- /* When the MOZ_LINKER_EXTRACT environment variable is set to "1",
- * compressed libraries are going to be (temporarily) extracted as
- * files, in the directory pointed by the MOZ_LINKER_CACHE
- * environment variable. */
- const char *extract = getenv("MOZ_LINKER_EXTRACT");
- if (extract && !strncmp(extract, "1", 2 /* Including '\0' */))
- mappable = MappableExtractFile::Create(name, &s);
- /* The above may fail in some cases. */
- if (!mappable)
+ /* When the MOZ_LINKER_EXTRACT environment variable is set to "1",
+ * compressed libraries are going to be (temporarily) extracted as
+ * files, in the directory pointed by the MOZ_LINKER_CACHE
+ * environment variable. */
+ const char *extract = getenv("MOZ_LINKER_EXTRACT");
+ if (extract && !strncmp(extract, "1", 2 /* Including '\0' */))
+ mappable = MappableExtractFile::Create(name, &s);
+ if (!mappable) {
+ if (s.GetType() == Zip::Stream::DEFLATE) {
mappable = MappableDeflate::Create(name, zip, &s);
+ } else if (s.GetType() == Zip::Stream::STORE) {
+ mappable = MappableSeekableZStream::Create(name, zip, &s);
+ }
}
}
}
/* If we couldn't load above, try with a MappableFile */
if (!mappable && !zip)
mappable = MappableFile::Create(path);
/* Try loading with the custom linker if we have a Mappable */
--- a/mozglue/linker/Makefile.in
+++ b/mozglue/linker/Makefile.in
@@ -29,12 +29,13 @@ HOST_CPPSRCS = \
VPATH += $(topsrcdir)/mfbt
HOST_LIBS = -lz
CPPSRCS += \
ElfLoader.cpp \
CustomElf.cpp \
Mappable.cpp \
+ SeekableZStream.cpp \
$(NULL)
endif
include $(topsrcdir)/config/rules.mk
--- a/mozglue/linker/Mappable.cpp
+++ b/mozglue/linker/Mappable.cpp
@@ -8,16 +8,17 @@
#include <sys/stat.h>
#include <cstring>
#include <cstdlib>
#include "Mappable.h"
#ifdef ANDROID
#include <linux/ashmem.h>
#endif
#include "ElfLoader.h"
+#include "SeekableZStream.h"
#include "Logging.h"
#ifndef PAGE_SIZE
#define PAGE_SIZE 4096
#endif
#ifndef PAGE_MASK
#define PAGE_MASK (~ (PAGE_SIZE - 1))
@@ -75,45 +76,71 @@ MappableExtractFile::Create(const char *
debug("Extracting to %s", (char *)path);
AutoCloseFD fd = open(path, O_TRUNC | O_RDWR | O_CREAT | O_NOATIME,
S_IRUSR | S_IWUSR);
if (fd == -1) {
log("Couldn't open %s to decompress library", path.get());
return NULL;
}
AutoUnlinkFile file = path.forget();
- if (ftruncate(fd, stream->GetUncompressedSize()) == -1) {
- log("Couldn't ftruncate %s to decompress library", file.get());
- return NULL;
- }
- /* Map the temporary file for use as inflate buffer */
- MappedPtr buffer(::mmap(NULL, stream->GetUncompressedSize(), PROT_WRITE,
- MAP_SHARED, fd, 0), stream->GetUncompressedSize());
- if (buffer == MAP_FAILED) {
- log("Couldn't map %s to decompress library", file.get());
- return NULL;
- }
- z_stream zStream = stream->GetZStream(buffer);
+ if (stream->GetType() == Zip::Stream::DEFLATE) {
+ if (ftruncate(fd, stream->GetUncompressedSize()) == -1) {
+ log("Couldn't ftruncate %s to decompress library", file.get());
+ return NULL;
+ }
+ /* Map the temporary file for use as inflate buffer */
+ MappedPtr buffer(::mmap(NULL, stream->GetUncompressedSize(), PROT_WRITE,
+ MAP_SHARED, fd, 0), stream->GetUncompressedSize());
+ if (buffer == MAP_FAILED) {
+ log("Couldn't map %s to decompress library", file.get());
+ return NULL;
+ }
+
+ z_stream zStream = stream->GetZStream(buffer);
- /* Decompress */
- if (inflateInit2(&zStream, -MAX_WBITS) != Z_OK) {
- log("inflateInit failed: %s", zStream.msg);
- return NULL;
- }
- if (inflate(&zStream, Z_FINISH) != Z_STREAM_END) {
- log("inflate failed: %s", zStream.msg);
- return NULL;
- }
- if (inflateEnd(&zStream) != Z_OK) {
- log("inflateEnd failed: %s", zStream.msg);
- return NULL;
- }
- if (zStream.total_out != stream->GetUncompressedSize()) {
- log("File not fully uncompressed! %ld / %d", zStream.total_out,
- static_cast<unsigned int>(stream->GetUncompressedSize()));
+ /* Decompress */
+ if (inflateInit2(&zStream, -MAX_WBITS) != Z_OK) {
+ log("inflateInit failed: %s", zStream.msg);
+ return NULL;
+ }
+ if (inflate(&zStream, Z_FINISH) != Z_STREAM_END) {
+ log("inflate failed: %s", zStream.msg);
+ return NULL;
+ }
+ if (inflateEnd(&zStream) != Z_OK) {
+ log("inflateEnd failed: %s", zStream.msg);
+ return NULL;
+ }
+ if (zStream.total_out != stream->GetUncompressedSize()) {
+ log("File not fully uncompressed! %ld / %d", zStream.total_out,
+ static_cast<unsigned int>(stream->GetUncompressedSize()));
+ return NULL;
+ }
+ } else if (stream->GetType() == Zip::Stream::STORE) {
+ SeekableZStream zStream;
+ if (!zStream.Init(stream->GetBuffer())) {
+ log("Couldn't initialize SeekableZStream for %s", name);
+ return NULL;
+ }
+ if (ftruncate(fd, zStream.GetUncompressedSize()) == -1) {
+ log("Couldn't ftruncate %s to decompress library", file.get());
+ return NULL;
+ }
+ MappedPtr buffer(::mmap(NULL, zStream.GetUncompressedSize(), PROT_WRITE,
+ MAP_SHARED, fd, 0), zStream.GetUncompressedSize());
+ if (buffer == MAP_FAILED) {
+ log("Couldn't map %s to decompress library", file.get());
+ return NULL;
+ }
+
+ if (!zStream.Decompress(buffer, 0, zStream.GetUncompressedSize())) {
+ log("%s: failed to decompress", name);
+ return NULL;
+ }
+ } else {
return NULL;
}
return new MappableExtractFile(fd.forget(), file.forget());
}
MappableExtractFile::~MappableExtractFile()
{
@@ -291,8 +318,183 @@ MappableDeflate::mmap(const void *addr,
void
MappableDeflate::finalize()
{
/* Free decompression buffer */
buffer = NULL;
/* Remove reference to Zip archive */
zip = NULL;
}
+
+MappableSeekableZStream *
+MappableSeekableZStream::Create(const char *name, Zip *zip,
+ Zip::Stream *stream)
+{
+ MOZ_ASSERT(stream->GetType() == Zip::Stream::STORE);
+ AutoDeletePtr<MappableSeekableZStream> mappable =
+ new MappableSeekableZStream(zip);
+
+ if (pthread_mutex_init(&mappable->mutex, NULL))
+ return NULL;
+
+ if (!mappable->zStream.Init(stream->GetBuffer()))
+ return NULL;
+
+ mappable->buffer = _MappableBuffer::Create(name,
+ mappable->zStream.GetUncompressedSize());
+ if (!mappable->buffer)
+ return NULL;
+
+ mappable->chunkAvail = new unsigned char[mappable->zStream.GetChunksNum()];
+ memset(mappable->chunkAvail, 0, mappable->zStream.GetChunksNum());
+
+ return mappable.forget();
+}
+
+MappableSeekableZStream::MappableSeekableZStream(Zip *zip)
+: zip(zip) { }
+
+MappableSeekableZStream::~MappableSeekableZStream()
+{
+ pthread_mutex_destroy(&mutex);
+}
+
+void *
+MappableSeekableZStream::mmap(const void *addr, size_t length, int prot,
+ int flags, off_t offset)
+{
+ /* Map with PROT_NONE so that accessing the mapping would segfault, and
+ * bring us to ensure() */
+ void *res = buffer->mmap(addr, length, PROT_NONE, flags, offset);
+ if (res == MAP_FAILED)
+ return MAP_FAILED;
+
+ /* Store the mapping, ordered by offset and length */
+ std::vector<LazyMap>::reverse_iterator it;
+ for (it = lazyMaps.rbegin(); it < lazyMaps.rend(); ++it) {
+ if ((it->offset < offset) ||
+ ((it->offset == offset) && (it->length < length)))
+ break;
+ }
+ LazyMap map = { res, length, prot, offset };
+ lazyMaps.insert(it.base(), map);
+ return res;
+}
+
+void
+MappableSeekableZStream::munmap(void *addr, size_t length)
+{
+ std::vector<LazyMap>::iterator it;
+ for (it = lazyMaps.begin(); it < lazyMaps.end(); ++it)
+ if ((it->addr = addr) && (it->length == length)) {
+ lazyMaps.erase(it);
+ ::munmap(addr, length);
+ return;
+ }
+ MOZ_NOT_REACHED("munmap called with unknown mapping");
+}
+
+void
+MappableSeekableZStream::finalize() { }
+
+class AutoLock {
+public:
+ AutoLock(pthread_mutex_t *mutex): mutex(mutex)
+ {
+ if (pthread_mutex_lock(mutex))
+ MOZ_NOT_REACHED("pthread_mutex_lock failed");
+ }
+ ~AutoLock()
+ {
+ if (pthread_mutex_unlock(mutex))
+ MOZ_NOT_REACHED("pthread_mutex_unlock failed");
+ }
+private:
+ pthread_mutex_t *mutex;
+};
+
+bool
+MappableSeekableZStream::ensure(const void *addr)
+{
+ debug("ensure @%p", addr);
+ void *addrPage = reinterpret_cast<void *>
+ (reinterpret_cast<uintptr_t>(addr) & PAGE_MASK);
+ /* Find the mapping corresponding to the given page */
+ std::vector<LazyMap>::iterator map;
+ for (map = lazyMaps.begin(); map < lazyMaps.end(); ++map) {
+ if (map->Contains(addrPage))
+ break;
+ }
+ if (map == lazyMaps.end())
+ return false;
+
+ /* Find corresponding chunk */
+ off_t mapOffset = map->offsetOf(addrPage);
+ size_t chunk = mapOffset / zStream.GetChunkSize();
+
+ /* In the typical case, we just need to decompress the chunk entirely. But
+ * when the current mapping ends in the middle of the chunk, we want to
+ * stop there. However, if another mapping needs the last part of the
+ * chunk, we still need to continue. As mappings are ordered by offset
+ * and length, we don't need to scan the entire list of mappings.
+ * It is safe to run through lazyMaps here because the linker is never
+ * going to call mmap (which adds lazyMaps) while this function is
+ * called. */
+ size_t length = zStream.GetChunkSize(chunk);
+ size_t chunkStart = chunk * zStream.GetChunkSize();
+ size_t chunkEnd = chunkStart + length;
+ std::vector<LazyMap>::iterator it;
+ for (it = map; it < lazyMaps.end(); ++it) {
+ if (chunkEnd <= it->endOffset())
+ break;
+ }
+ if ((it == lazyMaps.end()) || (chunkEnd > it->endOffset())) {
+ /* The mapping "it" points at now is past the interesting one */
+ --it;
+ length = it->endOffset() - chunkStart;
+ }
+
+ AutoLock lock(&mutex);
+
+ /* The very first page is mapped and accessed separately of the rest, and
+ * as such, only the first page of the first chunk is decompressed this way.
+ * When we fault in the remaining pages of that chunk, we want to decompress
+ * the complete chunk again. Short of doing that, we would end up with
+ * no data between PAGE_SIZE and chunkSize, which would effectively corrupt
+ * symbol resolution in the underlying library. */
+ if (chunkAvail[chunk] < (length + PAGE_SIZE - 1) / PAGE_SIZE) {
+ if (!zStream.DecompressChunk(*buffer + chunkStart, chunk, length))
+ return false;
+
+#if defined(ANDROID) && defined(__arm__)
+ if (map->prot & PROT_EXEC) {
+ /* We just extracted data that may be executed in the future.
+ * We thus need to ensure Instruction and Data cache coherency. */
+ debug("cacheflush(%p, %p)", *buffer + chunkStart, *buffer + (chunkStart + length));
+ cacheflush(reinterpret_cast<uintptr_t>(*buffer + chunkStart),
+ reinterpret_cast<uintptr_t>(*buffer + (chunkStart + length)), 0);
+ }
+#endif
+ chunkAvail[chunk] = (length + PAGE_SIZE - 1) / PAGE_SIZE;
+ }
+
+ /* Flip the chunk mapping protection to the recorded flags. We could
+ * also flip the protection for other mappings of the same chunk,
+ * but it's easier to skip that and let further segfaults call
+ * ensure again. */
+ const void *chunkAddr = reinterpret_cast<const void *>
+ (reinterpret_cast<uintptr_t>(addrPage)
+ - mapOffset % zStream.GetChunkSize());
+ const void *chunkEndAddr = reinterpret_cast<const void *>
+ (reinterpret_cast<uintptr_t>(chunkAddr) + length);
+
+ const void *start = std::max(map->addr, chunkAddr);
+ const void *end = std::min(map->end(), chunkEndAddr);
+ length = reinterpret_cast<uintptr_t>(end)
+ - reinterpret_cast<uintptr_t>(start);
+
+ debug("mprotect @%p, 0x%x, 0x%x", start, length, map->prot);
+ if (mprotect(const_cast<void *>(start), length, map->prot) == 0)
+ return true;
+
+ log("mprotect failed");
+ return false;
+}
--- a/mozglue/linker/Mappable.h
+++ b/mozglue/linker/Mappable.h
@@ -1,17 +1,19 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef Mappable_h
#define Mappable_h
#include <sys/types.h>
+#include <pthread.h>
#include "Zip.h"
+#include "SeekableZStream.h"
#include "mozilla/RefPtr.h"
#include "zlib.h"
/**
* Abstract class to handle mmap()ing from various kind of entities, such as
* plain files or Zip entries. The virtual members are meant to act as the
* equivalent system functions, with a few differences:
* - mapped memory is always MAP_PRIVATE, even though a given implementation
@@ -150,9 +152,89 @@ private:
/* Decompression buffer */
AutoDeletePtr<_MappableBuffer> buffer;
/* Zlib data */
z_stream zStream;
};
+/**
+ * Mappable implementation for seekable zStreams.
+ * Inflates the mapped bits in a temporary buffer, on demand.
+ */
+class MappableSeekableZStream: public Mappable
+{
+public:
+ ~MappableSeekableZStream();
+
+ /**
+ * Create a MappableSeekableZStream instance for the given Zip stream. The
+ * name argument is used for an appropriately named temporary file, and the
+ * Zip instance is given for the MappableSeekableZStream to keep a reference
+ * of it.
+ */
+ static MappableSeekableZStream *Create(const char *name, Zip *zip,
+ Zip::Stream *stream);
+
+ /* Inherited from Mappable */
+ virtual void *mmap(const void *addr, size_t length, int prot, int flags, off_t offset);
+ virtual void munmap(void *addr, size_t length);
+ virtual void finalize();
+ virtual bool ensure(const void *addr);
+
+private:
+ MappableSeekableZStream(Zip *zip);
+
+ /* Zip reference */
+ mozilla::RefPtr<Zip> zip;
+
+ /* Decompression buffer */
+ AutoDeletePtr<_MappableBuffer> buffer;
+
+ /* Seekable ZStream */
+ SeekableZStream zStream;
+
+ /* Keep track of mappings performed with MappableSeekableZStream::mmap so
+ * that they can be realized by MappableSeekableZStream::ensure.
+ * Values stored in the struct are those passed to mmap */
+ struct LazyMap
+ {
+ const void *addr;
+ size_t length;
+ int prot;
+ off_t offset;
+
+ /* Returns addr + length, as a pointer */
+ const void *end() const {
+ return reinterpret_cast<const void *>
+ (reinterpret_cast<const unsigned char *>(addr) + length);
+ }
+
+ /* Returns offset + length */
+ const off_t endOffset() const {
+ return offset + length;
+ }
+
+ /* Returns the offset corresponding to the given address */
+ const off_t offsetOf(const void *ptr) const {
+ return reinterpret_cast<uintptr_t>(ptr)
+ - reinterpret_cast<uintptr_t>(addr) + offset;
+ }
+
+ /* Returns whether the given address is in the LazyMap range */
+ const bool Contains(const void *ptr) const {
+ return (ptr >= addr) && (ptr < end());
+ }
+ };
+
+ /* List of all mappings */
+ std::vector<LazyMap> lazyMaps;
+
+ /* Array keeping track of which chunks have already been decompressed.
+ * Each value is the number of pages decompressed for the given chunk. */
+ AutoDeleteArray<unsigned char> chunkAvail;
+
+ /* Mutex protecting decompression */
+ pthread_mutex_t mutex;
+};
+
#endif /* Mappable_h */
new file mode 100644
--- /dev/null
+++ b/mozglue/linker/SeekableZStream.cpp
@@ -0,0 +1,100 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <algorithm>
+#include "SeekableZStream.h"
+#include "Logging.h"
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE 4096
+#endif
+
+#ifndef PAGE_MASK
+#define PAGE_MASK (~ (PAGE_SIZE - 1))
+#endif
+
+bool
+SeekableZStream::Init(const void *buf)
+{
+ const SeekableZStreamHeader *header = SeekableZStreamHeader::validate(buf);
+ if (!header) {
+ log("Not a seekable zstream");
+ return false;
+ }
+
+ buffer = reinterpret_cast<const unsigned char *>(buf);
+ totalSize = header->totalSize;
+ chunkSize = header->chunkSize;
+ lastChunkSize = header->lastChunkSize;
+ offsetTable.Init(&header[1], header->nChunks);
+
+ /* Sanity check */
+ if ((chunkSize == 0) ||
+ (chunkSize % PAGE_SIZE) ||
+ (chunkSize > 8 * PAGE_SIZE) ||
+ (offsetTable.numElements() < 1) ||
+ (lastChunkSize == 0) ||
+ (lastChunkSize > chunkSize)) {
+ log("Malformed or broken seekable zstream");
+ return false;
+ }
+
+ return true;
+}
+
+bool
+SeekableZStream::Decompress(void *where, size_t chunk, size_t length)
+{
+ while (length) {
+ size_t len = std::min(length, static_cast<size_t>(chunkSize));
+ if (!DecompressChunk(where, chunk, len))
+ return false;
+ where = reinterpret_cast<unsigned char *>(where) + len;
+ length -= len;
+ chunk++;
+ }
+ return true;
+}
+
+bool
+SeekableZStream::DecompressChunk(void *where, size_t chunk, size_t length)
+{
+ if (chunk >= offsetTable.numElements()) {
+ log("DecompressChunk: chunk #%ld out of range [0-%ld)",
+ chunk, offsetTable.numElements());
+ return false;
+ }
+
+ bool isLastChunk = (chunk == offsetTable.numElements() - 1);
+
+ size_t chunkLen = isLastChunk ? lastChunkSize : chunkSize;
+
+ if (length == 0 || length > chunkLen)
+ length = chunkLen;
+
+ debug("DecompressChunk #%ld @%p (%ld/%ld)", chunk, where, length, chunkLen);
+ z_stream zStream;
+ memset(&zStream, 0, sizeof(zStream));
+ zStream.avail_in = (isLastChunk ? totalSize : uint32_t(offsetTable[chunk + 1]))
+ - uint32_t(offsetTable[chunk]);
+ zStream.next_in = const_cast<Bytef *>(buffer + uint32_t(offsetTable[chunk]));
+ zStream.avail_out = length;
+ zStream.next_out = reinterpret_cast<Bytef *>(where);
+
+ /* Decompress chunk */
+ if (inflateInit(&zStream) != Z_OK) {
+ log("inflateInit failed: %s", zStream.msg);
+ return false;
+ }
+ if (inflate(&zStream, (length == chunkLen) ? Z_FINISH : Z_SYNC_FLUSH)
+ != (length == chunkLen) ? Z_STREAM_END : Z_OK) {
+ log("inflate failed: %s", zStream.msg);
+ return false;
+ }
+ if (inflateEnd(&zStream) != Z_OK) {
+ log("inflateEnd failed: %s", zStream.msg);
+ return false;
+ }
+ return true;
+}
--- a/mozglue/linker/SeekableZStream.h
+++ b/mozglue/linker/SeekableZStream.h
@@ -40,9 +40,63 @@ struct SeekableZStreamHeader: public Zip
/* Size of last chunk (> 0, <= Chunk size) */
le_uint32 lastChunkSize;
};
#pragma pack()
MOZ_STATIC_ASSERT(sizeof(SeekableZStreamHeader) == 5 * 4,
"SeekableZStreamHeader should be 5 32-bits words");
+/**
+ * Helper class used to decompress Seekable ZStreams.
+ */
+class SeekableZStream {
+public:
+ /* Initialize from the given buffer. Returns whether initialization
+ * succeeded (true) or failed (false). */
+ bool Init(const void *buf);
+
+ /* Decompresses starting from the given chunk. The decompressed data is
+ * stored at the given location. The given length, in bytes, indicates
+ * how much data to decompress. If length is 0, then exactly one chunk
+ * is decompressed.
+ * Returns whether decompression succeeded (true) or failed (false). */
+ bool Decompress(void *where, size_t chunk, size_t length = 0);
+
+ /* Decompresses the given chunk at the given address. If a length is given,
+ * only decompresses that amount of data instead of the entire chunk.
+ * Returns whether decompression succeeded (true) or failed (false). */
+ bool DecompressChunk(void *where, size_t chunk, size_t length = 0);
+
+ /* Returns the uncompressed size of the complete zstream */
+ const size_t GetUncompressedSize() const
+ {
+ return (offsetTable.numElements() - 1) * chunkSize + lastChunkSize;
+ }
+
+ /* Returns the chunk size of the given chunk */
+ const size_t GetChunkSize(size_t chunk = 0) const {
+ return (chunk == offsetTable.numElements() - 1) ? lastChunkSize : chunkSize;
+ }
+
+ /* Returns the number of chunks */
+ const size_t GetChunksNum() const {
+ return offsetTable.numElements();
+ }
+
+private:
+ /* RAW Seekable SZtream buffer */
+ const unsigned char *buffer;
+
+ /* Total size of the stream, including the 4 magic bytes. */
+ uint32_t totalSize;
+
+ /* Chunk size */
+ uint32_t chunkSize;
+
+ /* Size of last chunk (> 0, <= Chunk size) */
+ uint32_t lastChunkSize;
+
+ /* Offsets table */
+ Array<le_uint32> offsetTable;
+};
+
#endif /* SeekableZStream_h */