author | Staś Małolepszy <stas@mozilla.com> |
Wed, 27 Mar 2019 20:43:33 +0000 | |
changeset 466774 | 62c0925f8e1dd9b6f17016ccf553ac7b0248a304 |
parent 466773 | bb3a9a19e1083bf1418b72bc9617fce9de83178e |
child 466775 | 5a70120ae2d349d2ad8a8ac3530f4bd390677d92 |
push id | 35780 |
push user | opoprus@mozilla.com |
push date | Fri, 29 Mar 2019 21:53:01 +0000 |
treeherder | mozilla-central@414f37afbe07 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | zbraniecki |
bugs | 1539192 |
milestone | 68.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
intl/l10n/Fluent.jsm | file | annotate | diff | comparison | revisions | |
intl/l10n/FluentSyntax.jsm | file | annotate | diff | comparison | revisions |
--- a/intl/l10n/Fluent.jsm +++ b/intl/l10n/Fluent.jsm @@ -1,27 +1,27 @@ /* vim: set ts=2 et sw=2 tw=80 filetype=javascript: */ -/* Copyright 2017 Mozilla Foundation and others +/* Copyright 2019 Mozilla Foundation and others * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -/* fluent@0.10.0 */ +/* fluent@0.12.0 */ /* global Intl */ /** * The `FluentType` class is the base of Fluent's type system. * * Fluent types wrap JavaScript values and store additional configuration for * them, which can then be used in the `toString` method together with a proper @@ -133,79 +133,34 @@ function merge(argopts, opts) { function values(opts) { const unwrapped = {}; for (const [name, opt] of Object.entries(opts)) { unwrapped[name] = opt.valueOf(); } return unwrapped; } -/** - * @overview - * - * The role of the Fluent resolver is to format a translation object to an - * instance of `FluentType` or an array of instances. - * - * Translations can contain references to other messages or variables, - * conditional logic in form of select expressions, traits which describe their - * grammatical features, and can use Fluent builtins which make use of the - * `Intl` formatters to format numbers, dates, lists and more into the - * bundle's language. See the documentation of the Fluent syntax for more - * information. - * - * In case of errors the resolver will try to salvage as much of the - * translation as possible. In rare situations where the resolver didn't know - * how to recover from an error it will return an instance of `FluentNone`. - * - * `MessageReference`, `VariantExpression`, `AttributeExpression` and - * `SelectExpression` resolve to raw Runtime Entries objects and the result of - * the resolution needs to be passed into `Type` to get their real value. - * This is useful for composing expressions. Consider: - * - * brand-name[nominative] - * - * which is a `VariantExpression` with properties `id: MessageReference` and - * `key: Keyword`. If `MessageReference` was resolved eagerly, it would - * instantly resolve to the value of the `brand-name` message. Instead, we - * want to get the message object and look for its `nominative` variant. - * - * All other expressions (except for `FunctionReference` which is only used in - * `CallExpression`) resolve to an instance of `FluentType`. The caller should - * use the `toString` method to convert the instance to a native value. - * - * - * All functions in this file pass around a special object called `env`. - * This object stores a set of elements used by all resolve functions: - * - * * {FluentBundle} bundle - * bundle for which the given resolution is happening - * * {Object} args - * list of developer provided arguments that can be used - * * {Array} errors - * list of errors collected while resolving - * * {WeakSet} dirty - * Set of patterns already encountered during this resolution. - * This is used to prevent cyclic resolutions. - */ +/* global Intl */ // Prevent expansion of too long placeables. const MAX_PLACEABLE_LENGTH = 2500; // Unicode bidi isolation characters. const FSI = "\u2068"; const PDI = "\u2069"; // Helper: match a variant key to the given selector. function match(bundle, selector, key) { if (key === selector) { // Both are strings. return true; } + // XXX Consider comparing options too, e.g. minimumFractionDigits. if (key instanceof FluentNumber && selector instanceof FluentNumber && key.value === selector.value) { return true; } if (selector instanceof FluentNumber && typeof key === "string") { let category = bundle @@ -215,104 +170,102 @@ function match(bundle, selector, key) { return true; } } return false; } // Helper: resolve the default variant from a list of variants. -function getDefault(env, variants, star) { +function getDefault(scope, variants, star) { if (variants[star]) { - return Type(env, variants[star]); + return Type(scope, variants[star]); } - const { errors } = env; - errors.push(new RangeError("No default")); + scope.errors.push(new RangeError("No default")); return new FluentNone(); } // Helper: resolve arguments to a call expression. -function getArguments(env, args) { +function getArguments(scope, args) { const positional = []; const named = {}; - if (args) { - for (const arg of args) { - if (arg.type === "narg") { - named[arg.name] = Type(env, arg.value); - } else { - positional.push(Type(env, arg)); - } + for (const arg of args) { + if (arg.type === "narg") { + named[arg.name] = Type(scope, arg.value); + } else { + positional.push(Type(scope, arg)); } } return [positional, named]; } // Resolve an expression to a Fluent type. -function Type(env, expr) { +function Type(scope, expr) { // A fast-path for strings which are the most common case. Since they // natively have the `toString` method they can be used as if they were // a FluentType instance without incurring the cost of creating one. if (typeof expr === "string") { - return env.bundle._transform(expr); + return scope.bundle._transform(expr); } // A fast-path for `FluentNone` which doesn't require any additional logic. if (expr instanceof FluentNone) { return expr; } // The Runtime AST (Entries) encodes patterns (complex strings with // placeables) as Arrays. if (Array.isArray(expr)) { - return Pattern(env, expr); + return Pattern(scope, expr); } switch (expr.type) { case "str": return expr.value; case "num": - return new FluentNumber(expr.value); + return new FluentNumber(expr.value, { + minimumFractionDigits: expr.precision, + }); case "var": - return VariableReference(env, expr); + return VariableReference(scope, expr); + case "mesg": + return MessageReference(scope, expr); case "term": - return TermReference({...env, args: {}}, expr); - case "ref": - return expr.args - ? FunctionReference(env, expr) - : MessageReference(env, expr); + return TermReference(scope, expr); + case "func": + return FunctionReference(scope, expr); case "select": - return SelectExpression(env, expr); + return SelectExpression(scope, expr); case undefined: { // If it's a node with a value, resolve the value. if (expr.value !== null && expr.value !== undefined) { - return Type(env, expr.value); + return Type(scope, expr.value); } - const { errors } = env; - errors.push(new RangeError("No value")); + scope.errors.push(new RangeError("No value")); return new FluentNone(); } default: return new FluentNone(); } } // Resolve a reference to a variable. -function VariableReference(env, {name}) { - const { args, errors } = env; - - if (!args || !args.hasOwnProperty(name)) { - errors.push(new ReferenceError(`Unknown variable: ${name}`)); +function VariableReference(scope, {name}) { + if (!scope.args || !scope.args.hasOwnProperty(name)) { + if (scope.insideTermReference === false) { + scope.errors.push(new ReferenceError(`Unknown variable: ${name}`)); + } return new FluentNone(`$${name}`); } - const arg = args[name]; + const arg = scope.args[name]; // Return early if the argument already is an instance of FluentType. if (arg instanceof FluentType) { return arg; } // Convert the argument to a Fluent type. switch (typeof arg) { @@ -320,184 +273,158 @@ function VariableReference(env, {name}) return arg; case "number": return new FluentNumber(arg); case "object": if (arg instanceof Date) { return new FluentDateTime(arg); } default: - errors.push( + scope.errors.push( new TypeError(`Unsupported variable type: ${name}, ${typeof arg}`) ); return new FluentNone(`$${name}`); } } // Resolve a reference to another message. -function MessageReference(env, {name, attr}) { - const {bundle, errors} = env; - const message = bundle._messages.get(name); +function MessageReference(scope, {name, attr}) { + const message = scope.bundle._messages.get(name); if (!message) { const err = new ReferenceError(`Unknown message: ${name}`); - errors.push(err); + scope.errors.push(err); return new FluentNone(name); } if (attr) { const attribute = message.attrs && message.attrs[attr]; if (attribute) { - return Type(env, attribute); + return Type(scope, attribute); } - errors.push(new ReferenceError(`Unknown attribute: ${attr}`)); - return Type(env, message); + scope.errors.push(new ReferenceError(`Unknown attribute: ${attr}`)); + return Type(scope, message); } - return Type(env, message); + return Type(scope, message); } // Resolve a call to a Term with key-value arguments. -function TermReference(env, {name, attr, selector, args}) { - const {bundle, errors} = env; - +function TermReference(scope, {name, attr, args}) { const id = `-${name}`; - const term = bundle._terms.get(id); + const term = scope.bundle._terms.get(id); if (!term) { const err = new ReferenceError(`Unknown term: ${id}`); - errors.push(err); + scope.errors.push(err); return new FluentNone(id); } // Every TermReference has its own args. - const [, keyargs] = getArguments(env, args); - const local = {...env, args: keyargs}; + const [, keyargs] = getArguments(scope, args); + const local = {...scope, args: keyargs, insideTermReference: true}; if (attr) { const attribute = term.attrs && term.attrs[attr]; if (attribute) { return Type(local, attribute); } - errors.push(new ReferenceError(`Unknown attribute: ${attr}`)); + scope.errors.push(new ReferenceError(`Unknown attribute: ${attr}`)); return Type(local, term); } - const variantList = getVariantList(term); - if (selector && variantList) { - return SelectExpression(local, {...variantList, selector}); - } - return Type(local, term); } -// Helper: convert a value into a variant list, if possible. -function getVariantList(term) { - const value = term.value || term; - return Array.isArray(value) - && value[0].type === "select" - && value[0].selector === null - ? value[0] - : null; -} - // Resolve a call to a Function with positional and key-value arguments. -function FunctionReference(env, {name, args}) { +function FunctionReference(scope, {name, args}) { // Some functions are built-in. Others may be provided by the runtime via // the `FluentBundle` constructor. - const {bundle: {_functions}, errors} = env; - const func = _functions[name] || builtins[name]; - + const func = scope.bundle._functions[name] || builtins[name]; if (!func) { - errors.push(new ReferenceError(`Unknown function: ${name}()`)); + scope.errors.push(new ReferenceError(`Unknown function: ${name}()`)); return new FluentNone(`${name}()`); } if (typeof func !== "function") { - errors.push(new TypeError(`Function ${name}() is not callable`)); + scope.errors.push(new TypeError(`Function ${name}() is not callable`)); return new FluentNone(`${name}()`); } try { - return func(...getArguments(env, args)); + return func(...getArguments(scope, args)); } catch (e) { // XXX Report errors. return new FluentNone(); } } // Resolve a select expression to the member object. -function SelectExpression(env, {selector, variants, star}) { - if (selector === null) { - return getDefault(env, variants, star); - } - - let sel = Type(env, selector); +function SelectExpression(scope, {selector, variants, star}) { + let sel = Type(scope, selector); if (sel instanceof FluentNone) { - const variant = getDefault(env, variants, star); - return Type(env, variant); + const variant = getDefault(scope, variants, star); + return Type(scope, variant); } // Match the selector against keys of each variant, in order. for (const variant of variants) { - const key = Type(env, variant.key); - if (match(env.bundle, sel, key)) { - return Type(env, variant); + const key = Type(scope, variant.key); + if (match(scope.bundle, sel, key)) { + return Type(scope, variant); } } - const variant = getDefault(env, variants, star); - return Type(env, variant); + const variant = getDefault(scope, variants, star); + return Type(scope, variant); } // Resolve a pattern (a complex string with placeables). -function Pattern(env, ptn) { - const { bundle, dirty, errors } = env; - - if (dirty.has(ptn)) { - errors.push(new RangeError("Cyclic reference")); +function Pattern(scope, ptn) { + if (scope.dirty.has(ptn)) { + scope.errors.push(new RangeError("Cyclic reference")); return new FluentNone(); } // Tag the pattern as dirty for the purpose of the current resolution. - dirty.add(ptn); + scope.dirty.add(ptn); const result = []; // Wrap interpolations with Directional Isolate Formatting characters // only when the pattern has more than one element. - const useIsolating = bundle._useIsolating && ptn.length > 1; + const useIsolating = scope.bundle._useIsolating && ptn.length > 1; for (const elem of ptn) { if (typeof elem === "string") { - result.push(bundle._transform(elem)); + result.push(scope.bundle._transform(elem)); continue; } - const part = Type(env, elem).toString(bundle); + const part = Type(scope, elem).toString(scope.bundle); if (useIsolating) { result.push(FSI); } if (part.length > MAX_PLACEABLE_LENGTH) { - errors.push( + scope.errors.push( new RangeError( "Too many characters in placeable " + `(${part.length}, max allowed is ${MAX_PLACEABLE_LENGTH})` ) ); result.push(part.slice(MAX_PLACEABLE_LENGTH)); } else { result.push(part); } if (useIsolating) { result.push(PDI); } } - dirty.delete(ptn); + scope.dirty.delete(ptn); return result.join(""); } /** * Format a translation into a string. * * @param {FluentBundle} bundle * A FluentBundle instance which will be used to resolve the @@ -507,36 +434,39 @@ function Pattern(env, ptn) { * from the message. * @param {Object} message * An object with the Message to be resolved. * @param {Array} errors * An error array that any encountered errors will be appended to. * @returns {FluentType} */ function resolve(bundle, args, message, errors = []) { - const env = { + const scope = { bundle, args, errors, dirty: new WeakSet(), + // TermReferences are resolved in a new scope. + insideTermReference: false, }; - return Type(env, message).toString(bundle); + return Type(scope, message).toString(bundle); } class FluentError extends Error {} // This regex is used to iterate through the beginnings of messages and terms. // With the /m flag, the ^ matches at the beginning of every line. const RE_MESSAGE_START = /^(-?[a-zA-Z][\w-]*) *= */mg; // Both Attributes and Variants are parsed in while loops. These regexes are // used to break out of them. const RE_ATTRIBUTE_START = /\.([a-zA-Z][\w-]*) *= */y; const RE_VARIANT_START = /\*?\[/y; -const RE_NUMBER_LITERAL = /(-?[0-9]+(\.[0-9]+)?)/y; +const RE_NUMBER_LITERAL = /(-?[0-9]+(?:\.([0-9]+))?)/y; const RE_IDENTIFIER = /([a-zA-Z][\w-]*)/y; const RE_REFERENCE = /([$-])?([a-zA-Z][\w-]*)(?:\.([a-zA-Z][\w-]*))?/y; +const RE_FUNCTION_NAME = /^[A-Z][A-Z0-9_-]*$/; // A "run" is a sequence of text or string literal characters which don't // require any special handling. For TextElements such special characters are: { // (starts a placeable), and line breaks which require additional logic to check // if the next line is indented. For StringLiterals they are: \ (starts an // escape sequence), " (ends the literal), and line breaks which are not allowed // in StringLiterals. Note that string runs may be empty; text runs may not. const RE_TEXT_RUN = /([^{}\n\r]+)/y; @@ -786,23 +716,16 @@ class FluentResource extends Map { } } return baked; } function parsePlaceable() { consumeToken(TOKEN_BRACE_OPEN, FluentError); - // VariantLists are parsed as selector-less SelectExpressions. - let onlyVariants = parseVariants(); - if (onlyVariants) { - consumeToken(TOKEN_BRACE_CLOSE, FluentError); - return {type: "select", selector: null, ...onlyVariants}; - } - let selector = parseInlineExpression(); if (consumeToken(TOKEN_BRACE_CLOSE)) { return selector; } if (consumeToken(TOKEN_ARROW)) { let variants = parseVariants(); consumeToken(TOKEN_BRACE_CLOSE, FluentError); @@ -815,28 +738,42 @@ class FluentResource extends Map { function parseInlineExpression() { if (source[cursor] === "{") { // It's a nested placeable. return parsePlaceable(); } if (test(RE_REFERENCE)) { let [, sigil, name, attr = null] = match(RE_REFERENCE); - let type = {"$": "var", "-": "term"}[sigil] || "ref"; - if (source[cursor] === "[") { - // DEPRECATED VariantExpressions will be removed before 1.0. - return {type, name, selector: parseVariantKey()}; + if (sigil === "$") { + return {type: "var", name}; } if (consumeToken(TOKEN_PAREN_OPEN)) { - return {type, name, attr, args: parseArguments()}; + let args = parseArguments(); + + if (sigil === "-") { + // A parameterized term: -term(...). + return {type: "term", name, attr, args}; + } + + if (RE_FUNCTION_NAME.test(name)) { + return {type: "func", name, args}; + } + + throw new FluentError("Function names must be all upper-case"); } - return {type, name, attr, args: null}; + if (sigil === "-") { + // A non-parameterized term: -term. + return {type: "term", name, attr, args: []}; + } + + return {type: "mesg", name, attr}; } return parseLiteral(); } function parseArguments() { let args = []; while (true) { @@ -850,28 +787,28 @@ class FluentResource extends Map { args.push(parseArgument()); // Commas between arguments are treated as whitespace. consumeToken(TOKEN_COMMA); } } function parseArgument() { - let ref = parseInlineExpression(); - if (ref.type !== "ref") { - return ref; + let expr = parseInlineExpression(); + if (expr.type !== "mesg") { + return expr; } if (consumeToken(TOKEN_COLON)) { // The reference is the beginning of a named argument. - return {type: "narg", name: ref.name, value: parseLiteral()}; + return {type: "narg", name: expr.name, value: parseLiteral()}; } // It's a regular message reference. - return ref; + return expr; } function parseVariants() { let variants = []; let count = 0; let star; while (test(RE_VARIANT_START)) { @@ -915,17 +852,19 @@ class FluentResource extends Map { if (source[cursor] === "\"") { return parseStringLiteral(); } throw new FluentError("Invalid expression"); } function parseNumberLiteral() { - return {type: "num", value: match1(RE_NUMBER_LITERAL)}; + let [, value, fraction = ""] = match(RE_NUMBER_LITERAL); + let precision = fraction.length; + return {type: "num", value: parseFloat(value), precision}; } function parseStringLiteral() { consumeChar("\"", FluentError); let value = ""; while (true) { value += match1(RE_STRING_RUN); @@ -1047,16 +986,17 @@ class FluentBundle { * * Available options: * * - `functions` - an object of additional functions available to * translations as builtins. * * - `useIsolating` - boolean specifying whether to use Unicode isolation * marks (FSI, PDI) for bidi interpolations. + * Default: true * * - `transform` - a function used to transform string parts of patterns. * * @param {string|Array<string>} locales - Locale or locales of the bundle * @param {Object} [options] * @returns {FluentBundle} */ constructor(locales, { @@ -1113,59 +1053,89 @@ class FluentBundle { * the bundle and each translation unit (message) will be available in the * bundle by its identifier. * * bundle.addMessages('foo = Foo'); * bundle.getMessage('foo'); * * // Returns a raw representation of the 'foo' message. * + * bundle.addMessages('bar = Bar'); + * bundle.addMessages('bar = Newbar', { allowOverrides: true }); + * bundle.getMessage('bar'); + * + * // Returns a raw representation of the 'bar' message: Newbar. + * * Parsed entities should be formatted with the `format` method in case they * contain logic (references, select expressions etc.). * + * Available options: + * + * - `allowOverrides` - boolean specifying whether it's allowed to override + * an existing message or term with a new value. + * Default: false + * * @param {string} source - Text resource with translations. + * @param {Object} [options] * @returns {Array<Error>} */ - addMessages(source) { + addMessages(source, options) { const res = FluentResource.fromString(source); - return this.addResource(res); + return this.addResource(res, options); } /** * Add a translation resource to the bundle. * * The translation resource must be an instance of FluentResource, * e.g. parsed by `FluentResource.fromString`. * * let res = FluentResource.fromString("foo = Foo"); * bundle.addResource(res); * bundle.getMessage('foo'); * * // Returns a raw representation of the 'foo' message. * + * let res = FluentResource.fromString("bar = Bar"); + * bundle.addResource(res); + * res = FluentResource.fromString("bar = Newbar"); + * bundle.addResource(res, { allowOverrides: true }); + * bundle.getMessage('bar'); + * + * // Returns a raw representation of the 'bar' message: Newbar. + * * Parsed entities should be formatted with the `format` method in case they * contain logic (references, select expressions etc.). * + * Available options: + * + * - `allowOverrides` - boolean specifying whether it's allowed to override + * an existing message or term with a new value. + * Default: false + * * @param {FluentResource} res - FluentResource object. + * @param {Object} [options] * @returns {Array<Error>} */ - addResource(res) { + addResource(res, { + allowOverrides = false, + } = {}) { const errors = []; for (const [id, value] of res) { if (id.startsWith("-")) { // Identifiers starting with a dash (-) define terms. Terms are private // and cannot be retrieved from FluentBundle. - if (this._terms.has(id)) { + if (allowOverrides === false && this._terms.has(id)) { errors.push(`Attempt to override an existing term: "${id}"`); continue; } this._terms.set(id, value); } else { - if (this._messages.has(id)) { + if (allowOverrides === false && this._messages.has(id)) { errors.push(`Attempt to override an existing message: "${id}"`); continue; } this._messages.set(id, value); } } return errors; @@ -1228,11 +1198,27 @@ class FluentBundle { cache[id] = new ctor(this.locales, opts); this._intls.set(ctor, cache); } return cache[id]; } } -this.FluentBundle = FluentBundle; -this.FluentResource = FluentResource; -var EXPORTED_SYMBOLS = ["FluentBundle", "FluentResource"]; +/* + * @module fluent + * @overview + * + * `fluent` is a JavaScript implementation of Project Fluent, a localization + * framework designed to unleash the expressive power of the natural language. + * + */ + +this.EXPORTED_SYMBOLS = [ + ...Object.keys({ + FluentBundle, + FluentResource, + FluentError, + FluentType, + FluentNumber, + FluentDateTime, + }), +];
--- a/intl/l10n/FluentSyntax.jsm +++ b/intl/l10n/FluentSyntax.jsm @@ -11,27 +11,88 @@ * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -/* fluent-syntax@0.10.0 */ +/* fluent-syntax@0.12.0 */ /* * Base class for all Fluent AST nodes. * * All productions described in the ASDL subclass BaseNode, including Span and * Annotation. * */ class BaseNode { constructor() {} + + equals(other, ignoredFields = ["span"]) { + const thisKeys = new Set(Object.keys(this)); + const otherKeys = new Set(Object.keys(other)); + if (ignoredFields) { + for (const fieldName of ignoredFields) { + thisKeys.delete(fieldName); + otherKeys.delete(fieldName); + } + } + if (thisKeys.size !== otherKeys.size) { + return false; + } + for (const fieldName of thisKeys) { + if (!otherKeys.has(fieldName)) { + return false; + } + const thisVal = this[fieldName]; + const otherVal = other[fieldName]; + if (typeof thisVal !== typeof otherVal) { + return false; + } + if (thisVal instanceof Array) { + if (thisVal.length !== otherVal.length) { + return false; + } + for (let i = 0; i < thisVal.length; ++i) { + if (!scalarsEqual(thisVal[i], otherVal[i], ignoredFields)) { + return false; + } + } + } else if (!scalarsEqual(thisVal, otherVal, ignoredFields)) { + return false; + } + } + return true; + } + + clone() { + function visit(value) { + if (value instanceof BaseNode) { + return value.clone(); + } + if (Array.isArray(value)) { + return value.map(visit); + } + return value; + } + const clone = Object.create(this.constructor.prototype); + for (const prop of Object.keys(this)) { + clone[prop] = visit(this[prop]); + } + return clone; + } +} + +function scalarsEqual(thisVal, otherVal, ignoredFields) { + if (thisVal instanceof BaseNode) { + return thisVal.equals(otherVal, ignoredFields); + } + return thisVal === otherVal; } /* * Base class for AST nodes which can have Spans. */ class SyntaxNode extends BaseNode { addSpan(start, end) { this.span = new Span(start, end); @@ -68,24 +129,16 @@ class Term extends Entry { this.type = "Term"; this.id = id; this.value = value; this.attributes = attributes; this.comment = comment; } } -class VariantList extends SyntaxNode { - constructor(variants) { - super(); - this.type = "VariantList"; - this.variants = variants; - } -} - class Pattern extends SyntaxNode { constructor(elements) { super(); this.type = "Pattern"; this.elements = elements; } } @@ -110,97 +163,130 @@ class Placeable extends PatternElement { } } /* * An abstract base class for expressions. */ class Expression extends SyntaxNode {} -class StringLiteral extends Expression { - constructor(raw, value) { +// An abstract base class for Literals. +class Literal extends Expression { + constructor(value) { super(); - this.type = "StringLiteral"; - this.raw = raw; + // The "value" field contains the exact contents of the literal, + // character-for-character. this.value = value; } + + parse() { + return {value: this.value}; + } } -class NumberLiteral extends Expression { +class StringLiteral extends Literal { constructor(value) { - super(); + super(value); + this.type = "StringLiteral"; + } + + parse() { + // Backslash backslash, backslash double quote, uHHHH, UHHHHHH. + const KNOWN_ESCAPES = + /(?:\\\\|\\"|\\u([0-9a-fA-F]{4})|\\U([0-9a-fA-F]{6}))/g; + + function from_escape_sequence(match, codepoint4, codepoint6) { + switch (match) { + case "\\\\": + return "\\"; + case "\\\"": + return "\""; + default: + let codepoint = parseInt(codepoint4 || codepoint6, 16); + if (codepoint <= 0xD7FF || 0xE000 <= codepoint) { + // It's a Unicode scalar value. + return String.fromCodePoint(codepoint); + } + // Escape sequences reresenting surrogate code points are + // well-formed but invalid in Fluent. Replace them with U+FFFD + // REPLACEMENT CHARACTER. + return "�"; + } + } + + let value = this.value.replace(KNOWN_ESCAPES, from_escape_sequence); + return {value}; + } +} + +class NumberLiteral extends Literal { + constructor(value) { + super(value); this.type = "NumberLiteral"; - this.value = value; + } + + parse() { + let value = parseFloat(this.value); + let decimal_position = this.value.indexOf("."); + let precision = decimal_position > 0 + ? this.value.length - decimal_position - 1 + : 0; + return {value, precision}; } } class MessageReference extends Expression { - constructor(id) { + constructor(id, attribute = null) { super(); this.type = "MessageReference"; this.id = id; + this.attribute = attribute; } } class TermReference extends Expression { - constructor(id) { + constructor(id, attribute = null, args = null) { super(); this.type = "TermReference"; this.id = id; + this.attribute = attribute; + this.arguments = args; } } class VariableReference extends Expression { constructor(id) { super(); this.type = "VariableReference"; this.id = id; } } class FunctionReference extends Expression { - constructor(id) { + constructor(id, args) { super(); this.type = "FunctionReference"; this.id = id; + this.arguments = args; } } class SelectExpression extends Expression { constructor(selector, variants) { super(); this.type = "SelectExpression"; this.selector = selector; this.variants = variants; } } -class AttributeExpression extends Expression { - constructor(ref, name) { - super(); - this.type = "AttributeExpression"; - this.ref = ref; - this.name = name; - } -} - -class VariantExpression extends Expression { - constructor(ref, key) { +class CallArguments extends SyntaxNode { + constructor(positional = [], named = []) { super(); - this.type = "VariantExpression"; - this.ref = ref; - this.key = key; - } -} - -class CallExpression extends Expression { - constructor(callee, positional = [], named = []) { - super(); - this.type = "CallExpression"; - this.callee = callee; + this.type = "CallArguments"; this.positional = positional; this.named = named; } } class Attribute extends SyntaxNode { constructor(id, value) { super(); @@ -287,42 +373,41 @@ class Span extends BaseNode { } } class Annotation extends SyntaxNode { constructor(code, args = [], message) { super(); this.type = "Annotation"; this.code = code; - this.args = args; + this.arguments = args; this.message = message; } } const ast = ({ + BaseNode: BaseNode, Resource: Resource, Entry: Entry, Message: Message, Term: Term, - VariantList: VariantList, Pattern: Pattern, PatternElement: PatternElement, TextElement: TextElement, Placeable: Placeable, Expression: Expression, + Literal: Literal, StringLiteral: StringLiteral, NumberLiteral: NumberLiteral, MessageReference: MessageReference, TermReference: TermReference, VariableReference: VariableReference, FunctionReference: FunctionReference, SelectExpression: SelectExpression, - AttributeExpression: AttributeExpression, - VariantExpression: VariantExpression, - CallExpression: CallExpression, + CallArguments: CallArguments, Attribute: Attribute, Variant: Variant, NamedArgument: NamedArgument, Identifier: Identifier, BaseComment: BaseComment, Comment: Comment, GroupComment: GroupComment, ResourceComment: ResourceComment, @@ -363,17 +448,17 @@ function getErrorMessage(code, args) { const [id] = args; return `Expected term "-${id}" to have a value`; } case "E0007": return "Keyword cannot end with a whitespace"; case "E0008": return "The callee has to be an upper-case identifier or a term"; case "E0009": - return "The key has to be a simple identifier"; + return "The argument name has to be a simple identifier"; case "E0010": return "Expected one of the variants to be marked as default (*)"; case "E0011": return 'Expected at least one variant after "->"'; case "E0012": return "Expected value"; case "E0013": return "Expected variant key"; @@ -783,20 +868,19 @@ class FluentParser { constructor({ withSpans = true, } = {}) { this.withSpans = withSpans; // Poor man's decorators. const methodNames = [ "getComment", "getMessage", "getTerm", "getAttribute", "getIdentifier", - "getVariant", "getNumber", "getPattern", "getVariantList", - "getTextElement", "getPlaceable", "getExpression", - "getInlineExpression", "getCallArgument", "getString", - "getSimpleExpression", "getLiteral", + "getVariant", "getNumber", "getPattern", "getTextElement", + "getPlaceable", "getExpression", "getInlineExpression", + "getCallArgument", "getCallArguments", "getString", "getLiteral", ]; for (const name of methodNames) { this[name] = withSpan(this[name]); } } parse(source) { const ps = new FluentParserStream(source); @@ -989,19 +1073,17 @@ class FluentParser { getTerm(ps) { ps.expectChar("-"); const id = this.getIdentifier(ps); ps.skipBlankInline(); ps.expectChar("="); - // Syntax 0.8 compat: VariantLists are supported but deprecated. They can - // only be found as values of Terms. Nested VariantLists are not allowed. - const value = this.maybeGetVariantList(ps) || this.maybeGetPattern(ps); + const value = this.maybeGetPattern(ps); if (value === null) { throw new ParseError("E0006", id.name); } const attrs = this.getAttributes(ps); return new Term(id, value, attrs); } @@ -1127,32 +1209,31 @@ class FluentParser { if (num.length === 0) { throw new ParseError("E0004", "0-9"); } return num; } getNumber(ps) { - let num = ""; + let value = ""; if (ps.currentChar === "-") { - num += "-"; ps.next(); + value += `-${this.getDigits(ps)}`; + } else { + value += this.getDigits(ps); } - num = `${num}${this.getDigits(ps)}`; - if (ps.currentChar === ".") { - num += "."; ps.next(); - num = `${num}${this.getDigits(ps)}`; + value += `.${this.getDigits(ps)}`; } - return new NumberLiteral(num); + return new NumberLiteral(value); } // maybeGetPattern distinguishes between patterns which start on the same line // as the identifier (a.k.a. inline signleline patterns and inline multiline // patterns) and patterns which start on a new line (a.k.a. block multiline // patterns). The distinction is important for the dedentation logic: the // indent of the first line of a block pattern must be taken into account when // calculating the maximum common indent. @@ -1167,57 +1248,27 @@ class FluentParser { if (ps.isValueContinuation()) { ps.skipToPeek(); return this.getPattern(ps, {isBlock: true}); } return null; } - // Deprecated in Syntax 0.8. VariantLists are only allowed as values of Terms. - // Values of Messages, Attributes and Variants must be Patterns. This method - // is only used in getTerm. - maybeGetVariantList(ps) { - ps.peekBlank(); - if (ps.currentPeek === "{") { - const start = ps.peekOffset; - ps.peek(); - ps.peekBlankInline(); - if (ps.currentPeek === EOL) { - ps.peekBlank(); - if (ps.isVariantStart()) { - ps.resetPeek(start); - ps.skipToPeek(); - return this.getVariantList(ps); - } - } - } - - ps.resetPeek(); - return null; - } - - getVariantList(ps) { - ps.expectChar("{"); - var variants = this.getVariants(ps); - ps.expectChar("}"); - return new VariantList(variants); - } - getPattern(ps, {isBlock}) { const elements = []; if (isBlock) { // A block pattern is a pattern which starts on a new line. Store and // measure the indent of this first line for the dedentation logic. const blankStart = ps.index; const firstIndent = ps.skipBlankInline(); elements.push(this.getIndent(ps, firstIndent, blankStart)); var commonIndentLength = firstIndent.length; } else { - var commonIndentLength = Infinity; + commonIndentLength = Infinity; } let ch; elements: while ((ch = ps.currentChar)) { switch (ch) { case EOL: { const blankStart = ps.index; const blankLines = ps.peekBlankBlock(); @@ -1338,17 +1389,17 @@ class FluentParser { getEscapeSequence(ps) { const next = ps.currentChar; switch (next) { case "\\": case "\"": ps.next(); - return [`\\${next}`, next]; + return `\\${next}`; case "u": return this.getUnicodeEscapeSequence(ps, next, 4); case "U": return this.getUnicodeEscapeSequence(ps, next, 6); default: throw new ParseError("E0025", next); } } @@ -1363,25 +1414,17 @@ class FluentParser { if (!ch) { throw new ParseError( "E0026", `\\${u}${sequence}${ps.currentChar}`); } sequence += ch; } - const codepoint = parseInt(sequence, 16); - const unescaped = codepoint <= 0xD7FF || 0xE000 <= codepoint - // It's a Unicode scalar value. - ? String.fromCodePoint(codepoint) - // Escape sequences reresenting surrogate code points are well-formed - // but invalid in Fluent. Replace them with U+FFFD REPLACEMENT - // CHARACTER. - : "�"; - return [`\\${u}${sequence}`, unescaped]; + return `\\${u}${sequence}`; } getPlaceable(ps) { ps.expectChar("{"); ps.skipBlank(); const expression = this.getExpression(ps); ps.expectChar("}"); return new Placeable(expression); @@ -1393,116 +1436,49 @@ class FluentParser { if (ps.currentChar === "-") { if (ps.peek() !== ">") { ps.resetPeek(); return selector; } if (selector.type === "MessageReference") { - throw new ParseError("E0016"); - } - - if (selector.type === "AttributeExpression" - && selector.ref.type === "MessageReference") { - throw new ParseError("E0018"); + if (selector.attribute === null) { + throw new ParseError("E0016"); + } else { + throw new ParseError("E0018"); + } } - if (selector.type === "TermReference" - || selector.type === "VariantExpression") { - throw new ParseError("E0017"); - } - - if (selector.type === "CallExpression" - && selector.callee.type === "TermReference") { + if (selector.type === "TermReference" && selector.attribute === null) { throw new ParseError("E0017"); } ps.next(); ps.next(); ps.skipBlankInline(); ps.expectLineEnd(); const variants = this.getVariants(ps); return new SelectExpression(selector, variants); } - if (selector.type === "AttributeExpression" - && selector.ref.type === "TermReference") { - throw new ParseError("E0019"); - } - - if (selector.type === "CallExpression" - && selector.callee.type === "AttributeExpression") { + if (selector.type === "TermReference" && selector.attribute !== null) { throw new ParseError("E0019"); } return selector; } getInlineExpression(ps) { if (ps.currentChar === "{") { return this.getPlaceable(ps); } - let expr = this.getSimpleExpression(ps); - switch (expr.type) { - case "NumberLiteral": - case "StringLiteral": - case "VariableReference": - return expr; - case "MessageReference": { - if (ps.currentChar === ".") { - ps.next(); - const attr = this.getIdentifier(ps); - return new AttributeExpression(expr, attr); - } - - if (ps.currentChar === "(") { - // It's a Function. Ensure it's all upper-case. - if (!/^[A-Z][A-Z_?-]*$/.test(expr.id.name)) { - throw new ParseError("E0008"); - } - - const func = new FunctionReference(expr.id); - if (this.withSpans) { - func.addSpan(expr.span.start, expr.span.end); - } - return new CallExpression(func, ...this.getCallArguments(ps)); - } - - return expr; - } - case "TermReference": { - if (ps.currentChar === "[") { - ps.next(); - const key = this.getVariantKey(ps); - ps.expectChar("]"); - return new VariantExpression(expr, key); - } - - if (ps.currentChar === ".") { - ps.next(); - const attr = this.getIdentifier(ps); - expr = new AttributeExpression(expr, attr); - } - - if (ps.currentChar === "(") { - return new CallExpression(expr, ...this.getCallArguments(ps)); - } - - return expr; - } - default: - throw new ParseError("E0028"); - } - } - - getSimpleExpression(ps) { if (ps.isNumberStart()) { return this.getNumber(ps); } if (ps.currentChar === '"') { return this.getString(ps); } @@ -1510,45 +1486,75 @@ class FluentParser { ps.next(); const id = this.getIdentifier(ps); return new VariableReference(id); } if (ps.currentChar === "-") { ps.next(); const id = this.getIdentifier(ps); - return new TermReference(id); + + let attr; + if (ps.currentChar === ".") { + ps.next(); + attr = this.getIdentifier(ps); + } + + let args; + if (ps.currentChar === "(") { + args = this.getCallArguments(ps); + } + + return new TermReference(id, attr, args); } if (ps.isIdentifierStart()) { const id = this.getIdentifier(ps); - return new MessageReference(id); + + if (ps.currentChar === "(") { + // It's a Function. Ensure it's all upper-case. + if (!/^[A-Z][A-Z0-9_-]*$/.test(id.name)) { + throw new ParseError("E0008"); + } + + let args = this.getCallArguments(ps); + return new FunctionReference(id, args); + } + + let attr; + if (ps.currentChar === ".") { + ps.next(); + attr = this.getIdentifier(ps); + } + + return new MessageReference(id, attr); } + throw new ParseError("E0028"); } getCallArgument(ps) { const exp = this.getInlineExpression(ps); ps.skipBlank(); if (ps.currentChar !== ":") { return exp; } - if (exp.type !== "MessageReference") { - throw new ParseError("E0009"); + if (exp.type === "MessageReference" && exp.attribute === null) { + ps.next(); + ps.skipBlank(); + + const value = this.getLiteral(ps); + return new NamedArgument(exp.id, value); } - ps.next(); - ps.skipBlank(); - - const value = this.getLiteral(ps); - return new NamedArgument(exp.id, value); + throw new ParseError("E0009"); } getCallArguments(ps) { const positional = []; const named = []; const argumentNames = new Set(); ps.expectChar("("); @@ -1579,44 +1585,39 @@ class FluentParser { ps.skipBlank(); continue; } break; } ps.expectChar(")"); - return [positional, named]; + return new CallArguments(positional, named); } getString(ps) { - let raw = ""; + ps.expectChar("\""); let value = ""; - ps.expectChar("\""); - let ch; while ((ch = ps.takeChar(x => x !== '"' && x !== EOL))) { if (ch === "\\") { - const [sequence, unescaped] = this.getEscapeSequence(ps); - raw += sequence; - value += unescaped; + value += this.getEscapeSequence(ps); } else { - raw += ch; value += ch; } } if (ps.currentChar === EOL) { throw new ParseError("E0020"); } ps.expectChar("\""); - return new StringLiteral(raw, value); + return new StringLiteral(value); } getLiteral(ps) { if (ps.isNumberStart()) { return this.getNumber(ps); } if (ps.currentChar === '"') { @@ -1635,17 +1636,16 @@ function includesNewLine(elem) { return elem.type === "TextElement" && includes(elem.value, "\n"); } function isSelectExpr(elem) { return elem.type === "Placeable" && elem.expression.type === "SelectExpression"; } -// Bit masks representing the state of the serializer. const HAS_ENTRIES = 1; class FluentSerializer { constructor({ withJunk = false } = {}) { this.withJunk = withJunk; } serialize(resource) { @@ -1690,20 +1690,16 @@ class FluentSerializer { } return `${serializeComment(entry, "###")}\n`; case "Junk": return serializeJunk(entry); default : throw new Error(`Unknown entry type: ${entry.type}`); } } - - serializeExpression(expr) { - return serializeExpression(expr); - } } function serializeComment(comment, prefix = "#") { const prefixed = comment.content.split("\n").map( line => line.length ? `${prefix} ${line}` : prefix ).join("\n"); // Add the trailing newline. @@ -1721,17 +1717,17 @@ function serializeMessage(message) { if (message.comment) { parts.push(serializeComment(message.comment)); } parts.push(`${message.id.name} =`); if (message.value) { - parts.push(serializeValue(message.value)); + parts.push(serializePattern(message.value)); } for (const attribute of message.attributes) { parts.push(serializeAttribute(attribute)); } parts.push("\n"); return parts.join(""); @@ -1741,190 +1737,221 @@ function serializeMessage(message) { function serializeTerm(term) { const parts = []; if (term.comment) { parts.push(serializeComment(term.comment)); } parts.push(`-${term.id.name} =`); - parts.push(serializeValue(term.value)); + parts.push(serializePattern(term.value)); for (const attribute of term.attributes) { parts.push(serializeAttribute(attribute)); } parts.push("\n"); return parts.join(""); } function serializeAttribute(attribute) { - const value = indent(serializeValue(attribute.value)); + const value = indent(serializePattern(attribute.value)); return `\n .${attribute.id.name} =${value}`; } -function serializeValue(value) { - switch (value.type) { - case "Pattern": - return serializePattern(value); - case "VariantList": - return serializeVariantList(value); - default: - throw new Error(`Unknown value type: ${value.type}`); - } -} - - function serializePattern(pattern) { const content = pattern.elements.map(serializeElement).join(""); const startOnNewLine = pattern.elements.some(isSelectExpr) || pattern.elements.some(includesNewLine); if (startOnNewLine) { return `\n ${indent(content)}`; } return ` ${content}`; } -function serializeVariantList(varlist) { - const content = varlist.variants.map(serializeVariant).join(""); - return `\n {${indent(content)}\n }`; -} - - -function serializeVariant(variant) { - const key = serializeVariantKey(variant.key); - const value = indent(serializeValue(variant.value)); - - if (variant.default) { - return `\n *[${key}]${value}`; - } - - return `\n [${key}]${value}`; -} - - function serializeElement(element) { switch (element.type) { case "TextElement": return element.value; case "Placeable": return serializePlaceable(element); default: throw new Error(`Unknown element type: ${element.type}`); } } function serializePlaceable(placeable) { const expr = placeable.expression; - switch (expr.type) { case "Placeable": return `{${serializePlaceable(expr)}}`; case "SelectExpression": // Special-case select expression to control the whitespace around the // opening and the closing brace. - return `{ ${serializeSelectExpression(expr)}}`; + return `{ ${serializeExpression(expr)}}`; default: return `{ ${serializeExpression(expr)} }`; } } function serializeExpression(expr) { switch (expr.type) { case "StringLiteral": - return `"${expr.raw}"`; + return `"${expr.value}"`; case "NumberLiteral": return expr.value; - case "MessageReference": - case "FunctionReference": - return expr.id.name; - case "TermReference": - return `-${expr.id.name}`; case "VariableReference": return `$${expr.id.name}`; - case "AttributeExpression": - return serializeAttributeExpression(expr); - case "VariantExpression": - return serializeVariantExpression(expr); - case "CallExpression": - return serializeCallExpression(expr); - case "SelectExpression": - return serializeSelectExpression(expr); + case "TermReference": { + let out = `-${expr.id.name}`; + if (expr.attribute) { + out += `.${expr.attribute.name}`; + } + if (expr.arguments) { + out += serializeCallArguments(expr.arguments); + } + return out; + } + case "MessageReference": { + let out = expr.id.name; + if (expr.attribute) { + out += `.${expr.attribute.name}`; + } + return out; + } + case "FunctionReference": + return `${expr.id.name}${serializeCallArguments(expr.arguments)}`; + case "SelectExpression": { + let out = `${serializeExpression(expr.selector)} ->`; + for (let variant of expr.variants) { + out += serializeVariant(variant); + } + return `${out}\n`; + } case "Placeable": return serializePlaceable(expr); default: throw new Error(`Unknown expression type: ${expr.type}`); } } -function serializeSelectExpression(expr) { - const parts = []; - const selector = `${serializeExpression(expr.selector)} ->`; - parts.push(selector); +function serializeVariant(variant) { + const key = serializeVariantKey(variant.key); + const value = indent(serializePattern(variant.value)); - for (const variant of expr.variants) { - parts.push(serializeVariant(variant)); + if (variant.default) { + return `\n *[${key}]${value}`; } - parts.push("\n"); - return parts.join(""); + return `\n [${key}]${value}`; } -function serializeAttributeExpression(expr) { - const ref = serializeExpression(expr.ref); - return `${ref}.${expr.name.name}`; -} - - -function serializeVariantExpression(expr) { - const ref = serializeExpression(expr.ref); - const key = serializeVariantKey(expr.key); - return `${ref}[${key}]`; -} - - -function serializeCallExpression(expr) { - const callee = serializeExpression(expr.callee); +function serializeCallArguments(expr) { const positional = expr.positional.map(serializeExpression).join(", "); const named = expr.named.map(serializeNamedArgument).join(", "); if (expr.positional.length > 0 && expr.named.length > 0) { - return `${callee}(${positional}, ${named})`; + return `(${positional}, ${named})`; } - return `${callee}(${positional || named})`; + return `(${positional || named})`; } function serializeNamedArgument(arg) { const value = serializeExpression(arg.value); return `${arg.name.name}: ${value}`; } function serializeVariantKey(key) { switch (key.type) { case "Identifier": return key.name; + case "NumberLiteral": + return key.value; default: - return serializeExpression(key); + throw new Error(`Unknown variant key type: ${key.type}`); + } +} + +/* + * Abstract Visitor pattern + */ +class Visitor { + visit(node) { + if (Array.isArray(node)) { + node.forEach(child => this.visit(child)); + return; + } + if (!(node instanceof BaseNode)) { + return; + } + const visit = this[`visit${node.type}`] || this.genericVisit; + visit.call(this, node); + } + + genericVisit(node) { + for (const propname of Object.keys(node)) { + this.visit(node[propname]); + } } } +/* + * Abstract Transformer pattern + */ +class Transformer extends Visitor { + visit(node) { + if (!(node instanceof BaseNode)) { + return node; + } + const visit = this[`visit${node.type}`] || this.genericVisit; + return visit.call(this, node); + } + + genericVisit(node) { + for (const propname of Object.keys(node)) { + const propvalue = node[propname]; + if (Array.isArray(propvalue)) { + const newvals = propvalue + .map(child => this.visit(child)) + .filter(newchild => newchild !== undefined); + node[propname] = newvals; + } + if (propvalue instanceof BaseNode) { + const new_val = this.visit(propvalue); + if (new_val === undefined) { + delete node[propname]; + } else { + node[propname] = new_val; + } + } + } + return node; + } +} + +const visitor = ({ + Visitor: Visitor, + Transformer: Transformer +}); + /* eslint object-shorthand: "off", - no-unused-vars: "off", - no-redeclare: "off", comma-dangle: "off", no-labels: "off" */ this.EXPORTED_SYMBOLS = [ - "FluentParser", - "FluentSerializer", + ...Object.keys({ + FluentParser, + FluentSerializer, + }), ...Object.keys(ast), + ...Object.keys(visitor), ];