author Mike Hommey <>
Wed, 24 Jan 2018 11:18:13 +0900
changeset 455913 d7d6780f2b42e0a434803bb2a734c7b8564713a1
parent 442424 ab03b4c381d4e1768d4a2af7cec65003d89a4b12
child 507921 04f0bbf40bf36957dc1f72a8aae9916df0e3222f
permissions -rw-r--r--
Bug 1432390 - Use zstandard and requests modules instead of spawning curl | zstd in docker.load_image. r=dustin The zstd command we spawn, if available at all, might be the wrong version: zstd changed its stream format in an incompatible way at some point, and the version shipped in e.g. Ubuntu 16.04 uses the old format, while the version taskcluster relies on uses the new format. Relying on gps's zstandard library allows to ensure we use the right version. Another advantage is that we can trivially pip install it in a virtualenv if it isn't available on the system running the command. If we're ridding ourselves of the subprocess spawning for zstd, we might as well cover curl as well. Especially considering the error handling when subprocesses are involved is not trivial, such that the current error handling code is actually broken and leads to dead-lock conditions, when, for example, curl is still waiting for the python side to read data, but the python side is not reading data anymore because an exception was thrown in the tar reading loop.

/* -*-  Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at */
#ifndef nsStartupCacheUtils_h_
#define nsStartupCacheUtils_h_

#include "nsString.h"
#include "nsIStorageStream.h"
#include "nsIObjectInputStream.h"
#include "nsIObjectOutputStream.h"
#include "mozilla/UniquePtr.h"

class nsIURI;

namespace mozilla {
namespace scache {

NewObjectInputStreamFromBuffer(UniquePtr<char[]> buffer, uint32_t len,
                               nsIObjectInputStream** stream);

// We can't retrieve the wrapped stream from the objectOutputStream later,
// so we return it here. We give callers in debug builds the option
// to wrap the outputstream in a debug stream, which will detect if
// non-singleton objects are written out multiple times during a serialization.
// This could cause them to be deserialized incorrectly (as multiple copies
// instead of references).
NewObjectOutputWrappedStorageStream(nsIObjectOutputStream **wrapperStream,
                                    nsIStorageStream** stream,
                                    bool wantDebugStream);

// Creates a buffer for storing the stream into the cache. The buffer is
// allocated with 'new []'.  After calling this function, the caller would
// typically call StartupCache::PutBuffer with the returned buffer.
NewBufferFromStorageStream(nsIStorageStream *storageStream,
                           UniquePtr<char[]>* buffer, uint32_t* len);

ResolveURI(nsIURI *in, nsIURI **out);

PathifyURI(nsIURI *in, nsACString &out);
} // namespace scache
} // namespace mozilla

#endif //nsStartupCacheUtils_h_