Bug 1333980 - Introduce L10nRegistry.jsm. r?mossop draft
authorZibi Braniecki <zbraniecki@mozilla.com>
Fri, 02 Jun 2017 10:36:08 +0200
changeset 642757 7f08ccc04ec97a036e34bc0aae247edc8e46d976
parent 642756 70929e33c3d2163616ee30294126c5ebf2658dea
child 642758 d47ccca79579f72a0df92984f957372fbc5c3a95
push id72852
push userbmo:gandalf@aviary.pl
push dateTue, 08 Aug 2017 17:54:38 +0000
reviewersmossop
bugs1333980
milestone57.0a1
Bug 1333980 - Introduce L10nRegistry.jsm. r?mossop This patch introduces a new registry for localization resources to replace ChromeRegistry. It uses asynchronous I/O and iterators to generate permutations of possible sources of language resources for use in the new Localization API. In the initial form the API handles packages resources and has API for interacting with the AddonsManager which will report language packs. MozReview-Commit-ID: LfERDYMROh
intl/l10n/L10nRegistry.jsm
intl/l10n/moz.build
intl/l10n/test/test_l10nregistry.js
intl/l10n/test/xpcshell.ini
python/mozbuild/mozpack/packager/formats.py
new file mode 100644
--- /dev/null
+++ b/intl/l10n/L10nRegistry.jsm
@@ -0,0 +1,346 @@
+const { Services } = Components.utils.import('resource://gre/modules/Services.jsm', {});
+const { MessageContext } = Components.utils.import("resource://gre/modules/MessageContext.jsm", {});
+Components.utils.importGlobalProperties(["fetch"]); /* globals fetch */
+
+/**
+ * L10nRegistry is a localization resource management system for Gecko.
+ *
+ * It manages the list of resource sources provided with the app and allows
+ * for additional sources to be added and updated.
+ *
+ * It's primary purpose is to allow for building an iterator over MessageContext objects
+ * that will be utilized by a localization API.
+ *
+ * The generator creates all possible permutations of locales and sources to allow for
+ * complete fallbacking.
+ *
+ * Example:
+ *
+ *   FileSource1:
+ *     name: 'app'
+ *     locales: ['en-US', 'de']
+ *     resources: [
+ *       '/browser/menu.ftl',
+ *       '/platform/toolkit.ftl',
+ *     ]
+ *   FileSource2:
+ *     name: 'platform'
+ *     locales: ['en-US', 'de']
+ *     resources: [
+ *       '/platform/toolkit.ftl',
+ *     ]
+ *
+ * If the user will request:
+ *   L10nRegistry.generateContexts(['de', 'en-US'], [
+ *     '/browser/menu.ftl',
+ *     '/platform/toolkit.ftl'
+ *   ]);
+ *
+ * the generator will return an iterator over the following contexts:
+ *
+ *   {
+ *     locale: 'de',
+ *     resources: [
+ *       ['app', '/browser/menu.ftl'],
+ *       ['app', '/platform/toolkit.ftl'],
+ *     ]
+ *   },
+ *   {
+ *     locale: 'de',
+ *     resources: [
+ *       ['app', '/browser/menu.ftl'],
+ *       ['platform', '/platform/toolkit.ftl'],
+ *     ]
+ *   },
+ *   {
+ *     locale: 'en-US',
+ *     resources: [
+ *       ['app', '/browser/menu.ftl'],
+ *       ['app', '/platform/toolkit.ftl'],
+ *     ]
+ *   },
+ *   {
+ *     locale: 'en-US',
+ *     resources: [
+ *       ['app', '/browser/menu.ftl'],
+ *       ['platform', '/platform/toolkit.ftl'],
+ *     ]
+ *   }
+ *
+ * This allows the localization API to consume the MessageContext and lazily fallback
+ * on the next in case of a missing string or error.
+ *
+ * If during the life-cycle of the app a new source is added, the generator can be called again
+ * and will produce a new set of permutations placing the language pack provided resources
+ * at the top.
+ */
+
+const L10nRegistry = {
+  sources: new Map(),
+  ctxCache: new Map(),
+
+  /**
+   * Based on the list of requested languages and resource Ids,
+   * this function returns an lazy iterator over message context permutations.
+   *
+   * @param {Array} requestedLangs
+   * @param {Array} resourceIds
+   * @returns {Iterator<MessageContext>}
+   */
+  * generateContexts(requestedLangs, resourceIds) {
+    const sourcesOrder = Array.from(this.sources.keys()).reverse();
+    for (const locale of requestedLangs) {
+      yield * generateContextsForLocale(locale, sourcesOrder, resourceIds);
+    }
+  },
+
+  /**
+   * Adds a new resource source to the L10nRegistry.
+   *
+   * @param {FileSource} source
+   */
+  registerSource(source) {
+    if (this.sources.has(source.name)) {
+      throw new Error(`Source with name "${source.name}" already registered.`);
+    }
+    this.sources.set(source.name, source);
+    Services.obs.notifyObservers(null, 'l10n:available-locales-changed', null);
+  },
+
+  /**
+   * Updates an existing source in the L10nRegistry
+   *
+   * That will usually happen when a new version of a source becomes
+   * available (for example, an updated version of a language pack).
+   *
+   * @param {FileSource} source
+   */
+  updateSource(source) {
+    if (!this.sources.has(source.name)) {
+      throw new Error(`Source with name "${source.name}" is not registered.`);
+    }
+    this.sources.set(source.name, source);
+    this.ctxCache.clear();
+    Services.obs.notifyObservers(null, 'l10n:available-locales-changed', null);
+  },
+
+  /**
+   * Removes a source from the L10nRegistry.
+   *
+   * @param {String} sourceId
+   */
+  removeSource(sourceName) {
+    this.sources.delete(sourceName);
+    Services.obs.notifyObservers(null, 'l10n:available-locales-changed', null);
+  },
+
+  /**
+   * Returns a list of locales for which at least one source
+   * has resources.
+   *
+   * @returns {Array<String>}
+   */
+  getAvailableLocales() {
+    const locales = new Set();
+
+    for (const source of this.sources.values()) {
+      for (const locale of source.locales) {
+        locales.add(locale);
+      }
+    }
+    return Array.from(locales);
+  },
+};
+
+/**
+ * A helper function for generating unique context ID used for caching
+ * MessageContexts.
+ *
+ * @param {String} locale
+ * @param {Array} sourcesOrder
+ * @param {Array} resourceIds
+ * @returns {String}
+ */
+function generateContextID(locale, sourcesOrder, resourceIds) {
+  const sources = sourcesOrder.join(',');
+  const ids = resourceIds.join(',');
+  return `${locale}|${sources}|${ids}`;
+}
+
+/**
+ * This function generates an iterator over MessageContexts for a single locale
+ * for a given list of resourceIds for all possible combinations of sources.
+ *
+ * This function is called recursively to generate all possible permutations
+ * and uses the last, optional parameter, to pass the already resolved
+ * sources order.
+ *
+ * @param {String} locale
+ * @param {Array} sourcesOrder
+ * @param {Array} resourceIds
+ * @param {Array} [resolvedOrder]
+ * @returns {Iterator<MessageContext>}
+ */
+function* generateContextsForLocale(locale, sourcesOrder, resourceIds, resolvedOrder = []) {
+  const resolvedLength = resolvedOrder.length;
+  const resourcesLength = resourceIds.length;
+
+  // Inside that loop we have a list of resources and the sources for them, like this:
+  //   ['test.ftl', 'menu.ftl', 'foo.ftl']
+  //   ['app', 'platform', 'app']
+  for (const sourceName of sourcesOrder) {
+    const order = resolvedOrder.concat(sourceName);
+
+    // We bail only if the hasFile returns a strict false here,
+    // because for FileSource it may also return undefined, which means
+    // that we simply don't know if the source contains the file and we'll
+    // have to perform the I/O to learn.
+    if (L10nRegistry.sources.get(sourceName).hasFile(locale, resourceIds[resolvedOrder.length]) === false) {
+      continue;
+    }
+
+    // If the number of resolved sources equals the number of resources,
+    // create the right context and return it if it loads.
+    if (resolvedLength + 1 === resourcesLength) {
+      yield generateContext(locale, order, resourceIds);
+    } else {
+      // otherwise recursively load another generator that walks over the
+      // partially resolved list of sources.
+      yield * generateContextsForLocale(locale, sourcesOrder, resourceIds, order);
+    }
+  }
+}
+
+/**
+ * Generates a single MessageContext by loading all resources
+ * from the listed sources for a given locale.
+ *
+ * @param {String} locale
+ * @param {Array} sourcesOrder
+ * @param {Array} resourceIds
+ * @returns {Promise<MessageContext>}
+ */
+async function generateContext(locale, sourcesOrder, resourceIds) {
+  const ctxId = generateContextID(locale, sourcesOrder, resourceIds);
+  if (!L10nRegistry.ctxCache.has(ctxId)) {
+    const ctx = new MessageContext(locale);
+    for (let i = 0; i < resourceIds.length; i++) {
+      const data = await L10nRegistry.sources.get(sourcesOrder[i]).fetchFile(locale, resourceIds[i]);
+      if (data === null) {
+        return false;
+      }
+      ctx.addMessages(data);
+    }
+    L10nRegistry.ctxCache.set(ctxId, ctx);
+  }
+  return L10nRegistry.ctxCache.get(ctxId);
+}
+
+/**
+ * This is a basic Source for L10nRegistry.
+ * It registers its own locales and a pre-path, and when asked for a file
+ * it attempts to download and cache it.
+ *
+ * The Source caches the downloaded files so any consecutive loads will
+ * come from the cache.
+ **/
+class FileSource {
+  constructor(name, locales, prePath) {
+    this.name = name;
+    this.locales = locales;
+    this.prePath = prePath;
+    this.cache = {};
+  }
+
+  getPath(locale, path) {
+    return (this.prePath + path).replace(/\{locale\}/g, locale);
+  }
+
+  hasFile(locale, path) {
+    if (!this.locales.includes(locale)) {
+      return false;
+    }
+
+    const fullPath = this.getPath(locale, path);
+    if (!this.cache.hasOwnProperty(fullPath)) {
+      return undefined;
+    }
+
+    if (this.cache[fullPath] === null) {
+      return false;
+    }
+    return true;
+  }
+
+  async fetchFile(locale, path) {
+    if (!this.locales.includes(locale)) {
+      return null;
+    }
+
+    const fullPath = this.getPath(locale, path);
+    if (this.hasFile(locale, path) === undefined) {
+      let file = await L10nRegistry.load(fullPath);
+
+      if (file === undefined) {
+        this.cache[fullPath] = null;
+      } else {
+        this.cache[fullPath] = file;
+      }
+    }
+    return this.cache[fullPath];
+  }
+}
+
+/**
+ * This is an extension of the FileSource which should be used
+ * for sources that can provide the list of files available in the source.
+ *
+ * This allows for a faster lookup in cases where the source does not
+ * contain most of the files that the app will request for (e.g. an addon).
+ **/
+class IndexedFileSource extends FileSource {
+  constructor(name, locales, prePath, paths) {
+    super(name, locales, prePath);
+    this.paths = paths;
+  }
+
+  hasFile(locale, path) {
+    if (!this.locales.includes(locale)) {
+      return false;
+    }
+    const fullPath = this.getPath(locale, path);
+    return this.paths.includes(fullPath);
+  }
+
+  async fetchFile(locale, path) {
+    if (!this.locales.includes(locale)) {
+      return null;
+    }
+
+    const fullPath = this.getPath(locale, path);
+    if (this.paths.includes(fullPath)) {
+      let file = await L10nRegistry.load(fullPath);
+
+      if (file === undefined) {
+        return null;
+      } else {
+        return file;
+      }
+    } else {
+      return null;
+    }
+  }
+}
+
+/**
+ * We keep it as a method to make it easier to override for testing purposes.
+ **/
+L10nRegistry.load = function(url) {
+  return fetch(url).then(data => data.text()).catch(() => undefined);
+};
+
+this.L10nRegistry = L10nRegistry;
+this.FileSource = FileSource;
+this.IndexedFileSource = IndexedFileSource;
+
+this.EXPORTED_SYMBOLS = [];
--- a/intl/l10n/moz.build
+++ b/intl/l10n/moz.build
@@ -1,13 +1,14 @@
 # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 EXTRA_JS_MODULES += [
+    'L10nRegistry.jsm',
     'MessageContext.jsm',
 ]
 
 XPCSHELL_TESTS_MANIFESTS += ['test/xpcshell.ini']
 
 FINAL_LIBRARY = 'xul'
new file mode 100644
--- /dev/null
+++ b/intl/l10n/test/test_l10nregistry.js
@@ -0,0 +1,312 @@
+/* Any copyrighequal dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const {
+  L10nRegistry,
+  FileSource,
+  IndexedFileSource
+} = Components.utils.import("resource://gre/modules/L10nRegistry.jsm", {});
+
+let fs;
+L10nRegistry.load = async function(url) {
+  return fs[url];
+}
+
+add_task(function test_methods_presence() {
+  equal(typeof L10nRegistry.generateContexts, "function");
+  equal(typeof L10nRegistry.getAvailableLocales, "function");
+  equal(typeof L10nRegistry.registerSource, "function");
+  equal(typeof L10nRegistry.updateSource, "function");
+});
+
+/**
+ * This test tests generation of a proper context for a single
+ * source scenario
+ */
+add_task(async function test_methods_calling() {
+  fs = {
+    '/localization/en-US/browser/menu.ftl': 'key = Value',
+  };
+  const originalLoad = L10nRegistry.load;
+
+  const source = new FileSource('test', ['en-US'], '/localization/{locale}');
+  L10nRegistry.registerSource(source);
+
+  const ctxs = L10nRegistry.generateContexts(['en-US'], ['/browser/menu.ftl']);
+
+  const ctx = await ctxs.next().value;
+
+  equal(ctx.hasMessage('key'), true);
+
+  // cleanup
+  L10nRegistry.sources.clear();
+  L10nRegistry.ctxCache.clear();
+});
+
+/**
+ * This test verifies that the public methods return expected values
+ * for the single source scenario
+ */
+add_task(async function test_has_one_source() {
+  let oneSource = new FileSource('app', ['en-US'], './app/data/locales/{locale}/');
+  fs = {
+    './app/data/locales/en-US/test.ftl': 'key = value en-US'
+  };
+  L10nRegistry.registerSource(oneSource);
+
+
+  // has one source
+
+  equal(L10nRegistry.sources.size, 1);
+  equal(L10nRegistry.sources.has('app'), true);
+
+
+  // returns a single context
+
+  let ctxs = L10nRegistry.generateContexts(['en-US'], ['test.ftl']);
+  let ctx0 = await ctxs.next().value;
+  equal(ctx0.hasMessage('key'), true);
+
+  equal(ctxs.next().done, true);
+
+
+  // returns no contexts for missing locale
+
+  ctxs = L10nRegistry.generateContexts(['pl'], ['test.ftl']);
+
+  equal(ctxs.next().done, true);
+
+  // cleanup
+  L10nRegistry.sources.clear();
+  L10nRegistry.ctxCache.clear();
+});
+
+/**
+ * This test verifies that public methods return expected values
+ * for the dual source scenario.
+ */
+add_task(async function test_has_two_sources() {
+  let oneSource = new FileSource('platform', ['en-US'], './platform/data/locales/{locale}/');
+  L10nRegistry.registerSource(oneSource);
+
+  let secondSource = new FileSource('app', ['pl'], './app/data/locales/{locale}/');
+  L10nRegistry.registerSource(secondSource);
+  fs = {
+    './platform/data/locales/en-US/test.ftl': 'key = platform value',
+    './app/data/locales/pl/test.ftl': 'key = app value'
+  };
+
+
+  // has two sources
+
+  equal(L10nRegistry.sources.size, 2);
+  equal(L10nRegistry.sources.has('app'), true);
+  equal(L10nRegistry.sources.has('platform'), true);
+
+
+  // returns correct contexts for en-US
+
+  let ctxs = L10nRegistry.generateContexts(['en-US'], ['test.ftl']);
+  let ctx0 = await ctxs.next().value;
+
+  equal(ctx0.hasMessage('key'), true);
+  let msg = ctx0.getMessage('key');
+  equal(ctx0.format(msg), 'platform value');
+
+  equal(ctxs.next().done, true);
+
+
+  // returns correct contexts for [pl, en-US]
+
+  ctxs = L10nRegistry.generateContexts(['pl', 'en-US'], ['test.ftl']);
+  ctx0 = await ctxs.next().value;
+  equal(ctx0.locales[0], 'pl');
+  equal(ctx0.hasMessage('key'), true);
+  let msg0 = ctx0.getMessage('key');
+  equal(ctx0.format(msg0), 'app value');
+
+  let ctx1 = await ctxs.next().value;
+  equal(ctx1.locales[0], 'en-US');
+  equal(ctx1.hasMessage('key'), true);
+  let msg1 = ctx1.getMessage('key');
+  equal(ctx1.format(msg1), 'platform value');
+
+  equal(ctxs.next().done, true);
+
+  // cleanup
+  L10nRegistry.sources.clear();
+  L10nRegistry.ctxCache.clear();
+});
+
+/**
+ * This test verifies that behavior specific to the IndexedFileSource
+ * works correctly.
+ *
+ * In particular it tests that IndexedFileSource correctly returns
+ * missing files as `false` instead of `undefined`.
+ */
+add_task(async function test_indexed() {
+  let oneSource = new IndexedFileSource('langpack-pl', ['pl'], '/data/locales/{locale}/', [
+    '/data/locales/pl/test.ftl',
+  ]);
+  L10nRegistry.registerSource(oneSource);
+  fs = {
+    '/data/locales/pl/test.ftl': 'key = value'
+  };
+
+  equal(L10nRegistry.sources.size, 1);
+  equal(L10nRegistry.sources.has('langpack-pl'), true);
+
+  equal(oneSource.getPath('pl', 'test.ftl'), '/data/locales/pl/test.ftl');
+  equal(oneSource.hasFile('pl', 'test.ftl'), true);
+  equal(oneSource.hasFile('pl', 'missing.ftl'), false);
+
+  // cleanup
+  L10nRegistry.sources.clear();
+  L10nRegistry.ctxCache.clear();
+});
+
+/**
+ * This test checks if the correct order of contexts is used for
+ * scenarios where a new file source is added on top of the default one.
+ */
+add_task(async function test_override() {
+  let fileSource = new FileSource('app', ['pl'], '/app/data/locales/{locale}/');
+  L10nRegistry.registerSource(fileSource);
+
+  let oneSource = new IndexedFileSource('langpack-pl', ['pl'], '/data/locales/{locale}/', [
+    '/data/locales/pl/test.ftl'
+  ]);
+  L10nRegistry.registerSource(oneSource);
+
+  fs = {
+    '/app/data/locales/pl/test.ftl': 'key = value',
+    '/data/locales/pl/test.ftl': 'key = addon value'
+  };
+
+  equal(L10nRegistry.sources.size, 2);
+  equal(L10nRegistry.sources.has('langpack-pl'), true);
+
+  let ctxs = L10nRegistry.generateContexts(['pl'], ['test.ftl']);
+  let ctx0 = await ctxs.next().value;
+  equal(ctx0.locales[0], 'pl');
+  equal(ctx0.hasMessage('key'), true);
+  let msg0 = ctx0.getMessage('key');
+  equal(ctx0.format(msg0), 'addon value');
+
+  let ctx1 = await ctxs.next().value;
+  equal(ctx1.locales[0], 'pl');
+  equal(ctx1.hasMessage('key'), true);
+  let msg1 = ctx1.getMessage('key');
+  equal(ctx1.format(msg1), 'value');
+
+  equal(ctxs.next().done, true);
+
+  // cleanup
+  L10nRegistry.sources.clear();
+  L10nRegistry.ctxCache.clear();
+});
+
+/**
+ * This test verifies that new contexts are returned
+ * after source update.
+ */
+add_task(async function test_updating() {
+  let oneSource = new IndexedFileSource('langpack-pl', ['pl'], '/data/locales/{locale}/', [
+    '/data/locales/pl/test.ftl',
+  ]);
+  L10nRegistry.registerSource(oneSource);
+  fs = {
+    '/data/locales/pl/test.ftl': 'key = value'
+  };
+
+  let ctxs = L10nRegistry.generateContexts(['pl'], ['test.ftl']);
+  let ctx0 = await ctxs.next().value;
+  equal(ctx0.locales[0], 'pl');
+  equal(ctx0.hasMessage('key'), true);
+  let msg0 = ctx0.getMessage('key');
+  equal(ctx0.format(msg0), 'value');
+
+
+  const newSource = new IndexedFileSource('langpack-pl', ['pl'], '/data/locales/{locale}/', [
+    '/data/locales/pl/test.ftl'
+  ]);
+  fs['/data/locales/pl/test.ftl'] = 'key = new value';
+  L10nRegistry.updateSource(newSource);
+
+  equal(L10nRegistry.sources.size, 1);
+  ctxs = L10nRegistry.generateContexts(['pl'], ['test.ftl']);
+  ctx0 = await ctxs.next().value;
+  msg0 = ctx0.getMessage('key');
+  equal(ctx0.format(msg0), 'new value');
+
+  // cleanup
+  L10nRegistry.sources.clear();
+  L10nRegistry.ctxCache.clear();
+});
+
+/**
+ * This test verifies that generated contexts return correct values
+ * after sources are being removed.
+ */
+add_task(async function test_removing() {
+  let fileSource = new FileSource('app', ['pl'], '/app/data/locales/{locale}/');
+  L10nRegistry.registerSource(fileSource);
+
+  let oneSource = new IndexedFileSource('langpack-pl', ['pl'], '/data/locales/{locale}/', [
+    '/data/locales/pl/test.ftl'
+  ]);
+  L10nRegistry.registerSource(oneSource);
+
+  fs = {
+    '/app/data/locales/pl/test.ftl': 'key = value',
+    '/data/locales/pl/test.ftl': 'key = addon value'
+  };
+
+  equal(L10nRegistry.sources.size, 2);
+  equal(L10nRegistry.sources.has('langpack-pl'), true);
+
+  let ctxs = L10nRegistry.generateContexts(['pl'], ['test.ftl']);
+  let ctx0 = await ctxs.next().value;
+  equal(ctx0.locales[0], 'pl');
+  equal(ctx0.hasMessage('key'), true);
+  let msg0 = ctx0.getMessage('key');
+  equal(ctx0.format(msg0), 'addon value');
+
+  let ctx1 = await ctxs.next().value;
+  equal(ctx1.locales[0], 'pl');
+  equal(ctx1.hasMessage('key'), true);
+  let msg1 = ctx1.getMessage('key');
+  equal(ctx1.format(msg1), 'value');
+
+  equal(ctxs.next().done, true);
+
+  // Remove langpack
+
+  L10nRegistry.removeSource('langpack-pl');
+
+  equal(L10nRegistry.sources.size, 1);
+  equal(L10nRegistry.sources.has('langpack-pl'), false);
+
+  ctxs = L10nRegistry.generateContexts(['pl'], ['test.ftl']);
+  ctx0 = await ctxs.next().value;
+  equal(ctx0.locales[0], 'pl');
+  equal(ctx0.hasMessage('key'), true);
+  msg0 = ctx0.getMessage('key');
+  equal(ctx0.format(msg0), 'value');
+
+  equal(ctxs.next().done, true);
+
+  // Remove app source
+
+  L10nRegistry.removeSource('app');
+
+  equal(L10nRegistry.sources.size, 0);
+
+  ctxs = L10nRegistry.generateContexts(['pl'], ['test.ftl']);
+  equal(ctxs.next().done, true);
+
+  // cleanup
+  L10nRegistry.sources.clear();
+  L10nRegistry.ctxCache.clear();
+});
--- a/intl/l10n/test/xpcshell.ini
+++ b/intl/l10n/test/xpcshell.ini
@@ -1,4 +1,5 @@
 [DEFAULT]
 head =
 
+[test_l10nregistry.js]
 [test_messagecontext.js]
--- a/python/mozbuild/mozpack/packager/formats.py
+++ b/python/mozbuild/mozpack/packager/formats.py
@@ -333,10 +333,11 @@ class OmniJarSubFormatter(PiecemealForma
         if path[0] == 'defaults':
             return len(path) != 3 or \
                 not (path[2] == 'channel-prefs.js' and
                      path[1] in ['pref', 'preferences'])
         return path[0] in [
             'modules',
             'greprefs.js',
             'hyphenation',
+            'localization',
             'update.locale',
         ]