startupcache/StartupCacheUtils.cpp
author Mike Hommey <mh+mozilla@glandium.org>
Wed, 24 Jan 2018 11:18:13 +0900
changeset 455913 d7d6780f2b42e0a434803bb2a734c7b8564713a1
parent 439792 4f6ff8f40729503c749f1e7f966182807fa7993f
child 477586 b54db66223586b4e04f5cb926fccdacf8a176b91
permissions -rw-r--r--
Bug 1432390 - Use zstandard and requests modules instead of spawning curl | zstd in docker.load_image. r=dustin The zstd command we spawn, if available at all, might be the wrong version: zstd changed its stream format in an incompatible way at some point, and the version shipped in e.g. Ubuntu 16.04 uses the old format, while the version taskcluster relies on uses the new format. Relying on gps's zstandard library allows to ensure we use the right version. Another advantage is that we can trivially pip install it in a virtualenv if it isn't available on the system running the command. If we're ridding ourselves of the subprocess spawning for zstd, we might as well cover curl as well. Especially considering the error handling when subprocesses are involved is not trivial, such that the current error handling code is actually broken and leads to dead-lock conditions, when, for example, curl is still waiting for the python side to read data, but the python side is not reading data anymore because an exception was thrown in the tar reading loop.

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "nsCOMPtr.h"
#include "nsIInputStream.h"
#include "nsIStringStream.h"
#include "nsNetUtil.h"
#include "nsIFileURL.h"
#include "nsIJARURI.h"
#include "nsIResProtocolHandler.h"
#include "nsIChromeRegistry.h"
#include "nsAutoPtr.h"
#include "nsStringStream.h"
#include "StartupCacheUtils.h"
#include "mozilla/scache/StartupCache.h"
#include "mozilla/Omnijar.h"

namespace mozilla {
namespace scache {

nsresult
NewObjectInputStreamFromBuffer(UniquePtr<char[]> buffer, uint32_t len,
                               nsIObjectInputStream** stream)
{
  nsCOMPtr<nsIInputStream> stringStream;
  nsresult rv = NS_NewByteInputStream(getter_AddRefs(stringStream),
                                      buffer.release(), len,
                                      NS_ASSIGNMENT_ADOPT);
  MOZ_ALWAYS_SUCCEEDS(rv);

  nsCOMPtr<nsIObjectInputStream> objectInput =
    NS_NewObjectInputStream(stringStream);

  objectInput.forget(stream);
  return NS_OK;
}

nsresult
NewObjectOutputWrappedStorageStream(nsIObjectOutputStream **wrapperStream,
                                    nsIStorageStream** stream,
                                    bool wantDebugStream)
{
  nsCOMPtr<nsIStorageStream> storageStream;

  nsresult rv = NS_NewStorageStream(256, UINT32_MAX, getter_AddRefs(storageStream));
  NS_ENSURE_SUCCESS(rv, rv);

  nsCOMPtr<nsIOutputStream> outputStream
    = do_QueryInterface(storageStream);

  nsCOMPtr<nsIObjectOutputStream> objectOutput
    = NS_NewObjectOutputStream(outputStream);

#ifdef DEBUG
  if (wantDebugStream) {
    // Wrap in debug stream to detect unsupported writes of
    // multiply-referenced non-singleton objects
    StartupCache* sc = StartupCache::GetSingleton();
    NS_ENSURE_TRUE(sc, NS_ERROR_UNEXPECTED);
    nsCOMPtr<nsIObjectOutputStream> debugStream;
    sc->GetDebugObjectOutputStream(objectOutput, getter_AddRefs(debugStream));
    debugStream.forget(wrapperStream);
  } else {
    objectOutput.forget(wrapperStream);
  }
#else
  objectOutput.forget(wrapperStream);
#endif

  storageStream.forget(stream);
  return NS_OK;
}

nsresult
NewBufferFromStorageStream(nsIStorageStream *storageStream,
                           UniquePtr<char[]>* buffer, uint32_t* len)
{
  nsresult rv;
  nsCOMPtr<nsIInputStream> inputStream;
  rv = storageStream->NewInputStream(0, getter_AddRefs(inputStream));
  NS_ENSURE_SUCCESS(rv, rv);

  uint64_t avail64;
  rv = inputStream->Available(&avail64);
  NS_ENSURE_SUCCESS(rv, rv);
  NS_ENSURE_TRUE(avail64 <= UINT32_MAX, NS_ERROR_FILE_TOO_BIG);

  uint32_t avail = (uint32_t)avail64;
  auto temp = MakeUnique<char[]>(avail);
  uint32_t read;
  rv = inputStream->Read(temp.get(), avail, &read);
  if (NS_SUCCEEDED(rv) && avail != read)
    rv = NS_ERROR_UNEXPECTED;

  if (NS_FAILED(rv)) {
    return rv;
  }

  *len = avail;
  *buffer = Move(temp);
  return NS_OK;
}

static const char baseName[2][5] = { "gre/", "app/" };

static inline bool
canonicalizeBase(nsAutoCString &spec,
                 nsACString &out)
{
    nsAutoCString greBase, appBase;
    nsresult rv = mozilla::Omnijar::GetURIString(mozilla::Omnijar::GRE, greBase);
    if (NS_FAILED(rv) || !greBase.Length())
        return false;

    rv = mozilla::Omnijar::GetURIString(mozilla::Omnijar::APP, appBase);
    if (NS_FAILED(rv))
        return false;

    bool underGre = !greBase.Compare(spec.get(), false, greBase.Length());
    bool underApp = appBase.Length() &&
                    !appBase.Compare(spec.get(), false, appBase.Length());

    if (!underGre && !underApp)
        return false;

    /**
     * At this point, if both underGre and underApp are true, it can be one
     * of the two following cases:
     * - the GRE directory points to a subdirectory of the APP directory,
     *   meaning spec points under GRE.
     * - the APP directory points to a subdirectory of the GRE directory,
     *   meaning spec points under APP.
     * Checking the GRE and APP path length is enough to know in which case
     * we are.
     */
    if (underGre && underApp && greBase.Length() < appBase.Length())
        underGre = false;

    out.AppendLiteral("/resource/");
    out.Append(baseName[underGre ? mozilla::Omnijar::GRE : mozilla::Omnijar::APP]);
    out.Append(Substring(spec, underGre ? greBase.Length() : appBase.Length()));
    return true;
}

/**
 * ResolveURI transforms a chrome: or resource: URI into the URI for its
 * underlying resource, or returns any other URI unchanged.
 */
nsresult
ResolveURI(nsIURI *in, nsIURI **out)
{
    bool equals;
    nsresult rv;

    // Resolve resource:// URIs. At the end of this if/else block, we
    // have both spec and uri variables identifying the same URI.
    if (NS_SUCCEEDED(in->SchemeIs("resource", &equals)) && equals) {
        nsCOMPtr<nsIIOService> ioService = do_GetIOService(&rv);
        NS_ENSURE_SUCCESS(rv, rv);

        nsCOMPtr<nsIProtocolHandler> ph;
        rv = ioService->GetProtocolHandler("resource", getter_AddRefs(ph));
        NS_ENSURE_SUCCESS(rv, rv);

        nsCOMPtr<nsIResProtocolHandler> irph(do_QueryInterface(ph, &rv));
        NS_ENSURE_SUCCESS(rv, rv);

        nsAutoCString spec;
        rv = irph->ResolveURI(in, spec);
        NS_ENSURE_SUCCESS(rv, rv);

        return ioService->NewURI(spec, nullptr, nullptr, out);
    } else if (NS_SUCCEEDED(in->SchemeIs("chrome", &equals)) && equals) {
        nsCOMPtr<nsIChromeRegistry> chromeReg =
            mozilla::services::GetChromeRegistryService();
        if (!chromeReg)
            return NS_ERROR_UNEXPECTED;

        return chromeReg->ConvertChromeURL(in, out);
    }

    *out = do_AddRef(in).take();
    return NS_OK;
}

/**
 * PathifyURI transforms uris into useful zip paths
 * to make it easier to manipulate startup cache entries
 * using standard zip tools.
 * Transformations applied:
 *  * resource:// URIs are resolved to their corresponding file/jar URI to
 *    canonicalize resources URIs other than gre and app.
 *  * Paths under GRE or APP directory have their base path replaced with
 *    resource/gre or resource/app to avoid depending on install location.
 *  * jar:file:///path/to/file.jar!/sub/path urls are replaced with
 *    /path/to/file.jar/sub/path
 *
 *  The result is appended to the string passed in. Adding a prefix before
 *  calling is recommended to avoid colliding with other cache users.
 *
 * For example, in the js loader (string is prefixed with jsloader by caller):
 *  resource://gre/modules/XPCOMUtils.jsm or
 *  file://$GRE_DIR/modules/XPCOMUtils.jsm or
 *  jar:file://$GRE_DIR/omni.jar!/modules/XPCOMUtils.jsm becomes
 *     jsloader/resource/gre/modules/XPCOMUtils.jsm
 *  file://$PROFILE_DIR/extensions/{uuid}/components/component.js becomes
 *     jsloader/$PROFILE_DIR/extensions/%7Buuid%7D/components/component.js
 *  jar:file://$PROFILE_DIR/extensions/some.xpi!/components/component.js becomes
 *     jsloader/$PROFILE_DIR/extensions/some.xpi/components/component.js
 */
nsresult
PathifyURI(nsIURI *in, nsACString &out)
{
    bool equals;
    nsresult rv;

    nsCOMPtr<nsIURI> uri;
    rv = ResolveURI(in, getter_AddRefs(uri));
    NS_ENSURE_SUCCESS(rv, rv);

    nsAutoCString spec;
    rv = uri->GetSpec(spec);
    NS_ENSURE_SUCCESS(rv, rv);

    if (!canonicalizeBase(spec, out)) {
        if (NS_SUCCEEDED(uri->SchemeIs("file", &equals)) && equals) {
            nsCOMPtr<nsIFileURL> baseFileURL;
            baseFileURL = do_QueryInterface(uri, &rv);
            NS_ENSURE_SUCCESS(rv, rv);

            nsAutoCString path;
            rv = baseFileURL->GetPathQueryRef(path);
            NS_ENSURE_SUCCESS(rv, rv);

            out.Append(path);
        } else if (NS_SUCCEEDED(uri->SchemeIs("jar", &equals)) && equals) {
            nsCOMPtr<nsIJARURI> jarURI = do_QueryInterface(uri, &rv);
            NS_ENSURE_SUCCESS(rv, rv);

            nsCOMPtr<nsIURI> jarFileURI;
            rv = jarURI->GetJARFile(getter_AddRefs(jarFileURI));
            NS_ENSURE_SUCCESS(rv, rv);

            rv = PathifyURI(jarFileURI, out);
            NS_ENSURE_SUCCESS(rv, rv);

            nsAutoCString path;
            rv = jarURI->GetJAREntry(path);
            NS_ENSURE_SUCCESS(rv, rv);
            out.Append('/');
            out.Append(path);
        } else { // Very unlikely
            rv = uri->GetSpec(spec);
            NS_ENSURE_SUCCESS(rv, rv);

            out.Append('/');
            out.Append(spec);
        }
    }
    return NS_OK;
}

} // namespace scache
} // namespace mozilla