Bug 1231764 - part 4 - add pytoml to the virtualenv; r=chmanchester
☠☠ backed out by 5d2cfad6ba81 ☠ ☠
authorNathan Froyd <froydnj@gmail.com>
Wed, 03 Aug 2016 19:40:41 -0400
changeset 308076 f9b204c2f5a7c2375276b49253ff9188c9bc32d7
parent 308075 f933119e57e43696881327fd2b3c730388b301b5
child 308077 234d8a930afaf01a3cc4e3cd9da9e7b81630e3a3
push id30528
push usercbook@mozilla.com
push dateThu, 04 Aug 2016 13:58:28 +0000
treeherdermozilla-central@0ba72e8027cf [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschmanchester
bugs1231764
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1231764 - part 4 - add pytoml to the virtualenv; r=chmanchester We need to parse Cargo.toml files from moz.build to determine crate package names.
build/virtualenv_packages.txt
python/pytoml/PKG-INFO
python/pytoml/pytoml/__init__.py
python/pytoml/pytoml/core.py
python/pytoml/pytoml/parser.py
python/pytoml/pytoml/writer.py
python/pytoml/setup.cfg
python/pytoml/setup.py
python/pytoml/test/test.py
--- a/build/virtualenv_packages.txt
+++ b/build/virtualenv_packages.txt
@@ -31,8 +31,9 @@ pyasn1_modules.pth:python/pyasn1-modules
 bitstring.pth:python/bitstring
 redo.pth:python/redo
 requests.pth:python/requests
 rsa.pth:python/rsa
 futures.pth:python/futures
 ecc.pth:python/PyECC
 xpcshell.pth:testing/xpcshell
 pyyaml.pth:python/pyyaml/lib
+pytoml.pth:python/pytoml
new file mode 100644
--- /dev/null
+++ b/python/pytoml/PKG-INFO
@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: pytoml
+Version: 0.1.10
+Summary: A parser for TOML-0.4.0
+Home-page: https://github.com/avakar/pytoml
+Author: Martin Vejnár
+Author-email: avakar@ratatanek.cz
+License: MIT
+Description: UNKNOWN
+Platform: UNKNOWN
new file mode 100644
--- /dev/null
+++ b/python/pytoml/pytoml/__init__.py
@@ -0,0 +1,3 @@
+from .core import TomlError
+from .parser import load, loads
+from .writer import dump, dumps
new file mode 100644
--- /dev/null
+++ b/python/pytoml/pytoml/core.py
@@ -0,0 +1,13 @@
+class TomlError(RuntimeError):
+    def __init__(self, message, line, col, filename):
+        RuntimeError.__init__(self, message, line, col, filename)
+        self.message = message
+        self.line = line
+        self.col = col
+        self.filename = filename
+
+    def __str__(self):
+        return '{}({}, {}): {}'.format(self.filename, self.line, self.col, self.message)
+
+    def __repr__(self):
+        return 'TomlError({!r}, {!r}, {!r}, {!r})'.format(self.message, self.line, self.col, self.filename)
new file mode 100644
--- /dev/null
+++ b/python/pytoml/pytoml/parser.py
@@ -0,0 +1,366 @@
+import string, re, sys, datetime
+from .core import TomlError
+
+if sys.version_info[0] == 2:
+    _chr = unichr
+else:
+    _chr = chr
+
+def load(fin, translate=lambda t, x, v: v):
+    return loads(fin.read(), translate=translate, filename=fin.name)
+
+def loads(s, filename='<string>', translate=lambda t, x, v: v):
+    if isinstance(s, bytes):
+        s = s.decode('utf-8')
+
+    s = s.replace('\r\n', '\n')
+
+    root = {}
+    tables = {}
+    scope = root
+
+    src = _Source(s, filename=filename)
+    ast = _p_toml(src)
+
+    def error(msg):
+        raise TomlError(msg, pos[0], pos[1], filename)
+
+    def process_value(v):
+        kind, text, value, pos = v
+        if kind == 'str' and value.startswith('\n'):
+            value = value[1:]
+        if kind == 'array':
+            if value and any(k != value[0][0] for k, t, v, p in value[1:]):
+                error('array-type-mismatch')
+            value = [process_value(item) for item in value]
+        elif kind == 'table':
+            value = dict([(k, process_value(value[k])) for k in value])
+        return translate(kind, text, value)
+
+    for kind, value, pos in ast:
+        if kind == 'kv':
+            k, v = value
+            if k in scope:
+                error('duplicate_keys. Key "{0}" was used more than once.'.format(k))
+            scope[k] = process_value(v)
+        else:
+            is_table_array = (kind == 'table_array')
+            cur = tables
+            for name in value[:-1]:
+                if isinstance(cur.get(name), list):
+                    d, cur = cur[name][-1]
+                else:
+                    d, cur = cur.setdefault(name, (None, {}))
+
+            scope = {}
+            name = value[-1]
+            if name not in cur:
+                if is_table_array:
+                    cur[name] = [(scope, {})]
+                else:
+                    cur[name] = (scope, {})
+            elif isinstance(cur[name], list):
+                if not is_table_array:
+                    error('table_type_mismatch')
+                cur[name].append((scope, {}))
+            else:
+                if is_table_array:
+                    error('table_type_mismatch')
+                old_scope, next_table = cur[name]
+                if old_scope is not None:
+                    error('duplicate_tables')
+                cur[name] = (scope, next_table)
+
+    def merge_tables(scope, tables):
+        if scope is None:
+            scope = {}
+        for k in tables:
+            if k in scope:
+                error('key_table_conflict')
+            v = tables[k]
+            if isinstance(v, list):
+                scope[k] = [merge_tables(sc, tbl) for sc, tbl in v]
+            else:
+                scope[k] = merge_tables(v[0], v[1])
+        return scope
+
+    return merge_tables(root, tables)
+
+class _Source:
+    def __init__(self, s, filename=None):
+        self.s = s
+        self._pos = (1, 1)
+        self._last = None
+        self._filename = filename
+        self.backtrack_stack = []
+
+    def last(self):
+        return self._last
+
+    def pos(self):
+        return self._pos
+
+    def fail(self):
+        return self._expect(None)
+
+    def consume_dot(self):
+        if self.s:
+            self._last = self.s[0]
+            self.s = self[1:]
+            self._advance(self._last)
+            return self._last
+        return None
+
+    def expect_dot(self):
+        return self._expect(self.consume_dot())
+
+    def consume_eof(self):
+        if not self.s:
+            self._last = ''
+            return True
+        return False
+
+    def expect_eof(self):
+        return self._expect(self.consume_eof())
+
+    def consume(self, s):
+        if self.s.startswith(s):
+            self.s = self.s[len(s):]
+            self._last = s
+            self._advance(s)
+            return True
+        return False
+
+    def expect(self, s):
+        return self._expect(self.consume(s))
+
+    def consume_re(self, re):
+        m = re.match(self.s)
+        if m:
+            self.s = self.s[len(m.group(0)):]
+            self._last = m
+            self._advance(m.group(0))
+            return m
+        return None
+
+    def expect_re(self, re):
+        return self._expect(self.consume_re(re))
+
+    def __enter__(self):
+        self.backtrack_stack.append((self.s, self._pos))
+
+    def __exit__(self, type, value, traceback):
+        if type is None:
+            self.backtrack_stack.pop()
+        else:
+            self.s, self._pos = self.backtrack_stack.pop()
+        return type == TomlError
+
+    def commit(self):
+        self.backtrack_stack[-1] = (self.s, self._pos)
+
+    def _expect(self, r):
+        if not r:
+            raise TomlError('msg', self._pos[0], self._pos[1], self._filename)
+        return r
+
+    def _advance(self, s):
+        suffix_pos = s.rfind('\n')
+        if suffix_pos == -1:
+            self._pos = (self._pos[0], self._pos[1] + len(s))
+        else:
+            self._pos = (self._pos[0] + s.count('\n'), len(s) - suffix_pos)
+
+_ews_re = re.compile(r'(?:[ \t]|#[^\n]*\n|#[^\n]*\Z|\n)*')
+def _p_ews(s):
+    s.expect_re(_ews_re)
+
+_ws_re = re.compile(r'[ \t]*')
+def _p_ws(s):
+    s.expect_re(_ws_re)
+
+_escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"', '\'': '\'',
+    '\\': '\\', '/': '/', 'f': '\f' }
+
+_basicstr_re = re.compile(r'[^"\\\000-\037]*')
+_short_uni_re = re.compile(r'u([0-9a-fA-F]{4})')
+_long_uni_re = re.compile(r'U([0-9a-fA-F]{8})')
+_escapes_re = re.compile('[bnrt"\'\\\\/f]')
+_newline_esc_re = re.compile('\n[ \t\n]*')
+def _p_basicstr_content(s, content=_basicstr_re):
+    res = []
+    while True:
+        res.append(s.expect_re(content).group(0))
+        if not s.consume('\\'):
+            break
+        if s.consume_re(_newline_esc_re):
+            pass
+        elif s.consume_re(_short_uni_re) or s.consume_re(_long_uni_re):
+            res.append(_chr(int(s.last().group(1), 16)))
+        else:
+            s.expect_re(_escapes_re)
+            res.append(_escapes[s.last().group(0)])
+    return ''.join(res)
+
+_key_re = re.compile(r'[0-9a-zA-Z-_]+')
+def _p_key(s):
+    with s:
+        s.expect('"')
+        r = _p_basicstr_content(s, _basicstr_re)
+        s.expect('"')
+        return r
+    return s.expect_re(_key_re).group(0)
+
+_float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]?(?:\d(?:_?\d)*))?')
+_datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))')
+
+_basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*')
+_litstr_re = re.compile(r"[^'\000-\037]*")
+_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\011\013-\037]))*")
+def _p_value(s):
+    pos = s.pos()
+
+    if s.consume('true'):
+        return 'bool', s.last(), True, pos
+    if s.consume('false'):
+        return 'bool', s.last(), False, pos
+
+    if s.consume('"'):
+        if s.consume('""'):
+            r = _p_basicstr_content(s, _basicstr_ml_re)
+            s.expect('"""')
+        else:
+            r = _p_basicstr_content(s, _basicstr_re)
+            s.expect('"')
+        return 'str', r, r, pos
+
+    if s.consume('\''):
+        if s.consume('\'\''):
+            r = s.expect_re(_litstr_ml_re).group(0)
+            s.expect('\'\'\'')
+        else:
+            r = s.expect_re(_litstr_re).group(0)
+            s.expect('\'')
+        return 'str', r, r, pos
+
+    if s.consume_re(_datetime_re):
+        m = s.last()
+        s0 = m.group(0)
+        r = map(int, m.groups()[:6])
+        if m.group(7):
+            micro = float(m.group(7))
+        else:
+            micro = 0
+
+        if m.group(8):
+            g = int(m.group(8), 10) * 60 + int(m.group(9), 10)
+            tz = _TimeZone(datetime.timedelta(0, g * 60))
+        else:
+            tz = _TimeZone(datetime.timedelta(0, 0))
+
+        y, m, d, H, M, S = r
+        dt = datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz)
+        return 'datetime', s0, dt, pos
+
+    if s.consume_re(_float_re):
+        m = s.last().group(0)
+        r = m.replace('_','')
+        if '.' in m or 'e' in m or 'E' in m:
+            return 'float', m, float(r), pos
+        else:
+            return 'int', m, int(r, 10), pos
+
+    if s.consume('['):
+        items = []
+        with s:
+            while True:
+                _p_ews(s)
+                items.append(_p_value(s))
+                s.commit()
+                _p_ews(s)
+                s.expect(',')
+                s.commit()
+        _p_ews(s)
+        s.expect(']')
+        return 'array', None, items, pos
+
+    if s.consume('{'):
+        _p_ws(s)
+        items = {}
+        if not s.consume('}'):
+            k = _p_key(s)
+            _p_ws(s)
+            s.expect('=')
+            _p_ws(s)
+            items[k] = _p_value(s)
+            _p_ws(s)
+            while s.consume(','):
+                _p_ws(s)
+                k = _p_key(s)
+                _p_ws(s)
+                s.expect('=')
+                _p_ws(s)
+                items[k] = _p_value(s)
+                _p_ws(s)
+            s.expect('}')
+        return 'table', None, items, pos
+
+    s.fail()
+
+def _p_stmt(s):
+    pos = s.pos()
+    if s.consume(   '['):
+        is_array = s.consume('[')
+        _p_ws(s)
+        keys = [_p_key(s)]
+        _p_ws(s)
+        while s.consume('.'):
+            _p_ws(s)
+            keys.append(_p_key(s))
+            _p_ws(s)
+        s.expect(']')
+        if is_array:
+            s.expect(']')
+        return 'table_array' if is_array else 'table', keys, pos
+
+    key = _p_key(s)
+    _p_ws(s)
+    s.expect('=')
+    _p_ws(s)
+    value = _p_value(s)
+    return 'kv', (key, value), pos
+
+_stmtsep_re = re.compile(r'(?:[ \t]*(?:#[^\n]*)?\n)+[ \t]*')
+def _p_toml(s):
+    stmts = []
+    _p_ews(s)
+    with s:
+        stmts.append(_p_stmt(s))
+        while True:
+            s.commit()
+            s.expect_re(_stmtsep_re)
+            stmts.append(_p_stmt(s))
+    _p_ews(s)
+    s.expect_eof()
+    return stmts
+
+class _TimeZone(datetime.tzinfo):
+    def __init__(self, offset):
+        self._offset = offset
+
+    def utcoffset(self, dt):
+        return self._offset
+
+    def dst(self, dt):
+        return None
+
+    def tzname(self, dt):
+        m = self._offset.total_seconds() // 60
+        if m < 0:
+            res = '-'
+            m = -m
+        else:
+            res = '+'
+        h = m // 60
+        m = m - h * 60
+        return '{}{:.02}{:.02}'.format(res, h, m)
new file mode 100644
--- /dev/null
+++ b/python/pytoml/pytoml/writer.py
@@ -0,0 +1,120 @@
+from __future__ import unicode_literals
+import io, datetime, sys
+
+if sys.version_info[0] == 3:
+    long = int
+    unicode = str
+
+
+def dumps(obj, sort_keys=False):
+    fout = io.StringIO()
+    dump(fout, obj, sort_keys=sort_keys)
+    return fout.getvalue()
+
+
+_escapes = {'\n': 'n', '\r': 'r', '\\': '\\', '\t': 't', '\b': 'b', '\f': 'f', '"': '"'}
+
+
+def _escape_string(s):
+    res = []
+    start = 0
+
+    def flush():
+        if start != i:
+            res.append(s[start:i])
+        return i + 1
+
+    i = 0
+    while i < len(s):
+        c = s[i]
+        if c in '"\\\n\r\t\b\f':
+            start = flush()
+            res.append('\\' + _escapes[c])
+        elif ord(c) < 0x20:
+            start = flush()
+            res.append('\\u%04x' % ord(c))
+        i += 1
+
+    flush()
+    return '"' + ''.join(res) + '"'
+
+
+def _escape_id(s):
+    if any(not c.isalnum() and c not in '-_' for c in s):
+        return _escape_string(s)
+    return s
+
+
+def _format_list(v):
+    return '[{0}]'.format(', '.join(_format_value(obj) for obj in v))
+
+# Formula from:
+#   https://docs.python.org/2/library/datetime.html#datetime.timedelta.total_seconds
+# Once support for py26 is dropped, this can be replaced by td.total_seconds()
+def _total_seconds(td):
+    return ((td.microseconds
+             + (td.seconds + td.days * 24 * 3600) * 10**6) / 10.0**6)
+
+def _format_value(v):
+    if isinstance(v, bool):
+        return 'true' if v else 'false'
+    if isinstance(v, int) or isinstance(v, long):
+        return unicode(v)
+    if isinstance(v, float):
+        return '{0:.17f}'.format(v)
+    elif isinstance(v, unicode) or isinstance(v, bytes):
+        return _escape_string(v)
+    elif isinstance(v, datetime.datetime):
+        offs = v.utcoffset()
+        offs = _total_seconds(offs) // 60 if offs is not None else 0
+
+        if offs == 0:
+            suffix = 'Z'
+        else:
+            if offs > 0:
+                suffix = '+'
+            else:
+                suffix = '-'
+                offs = -offs
+            suffix = '{0}{1:.02}{2:.02}'.format(suffix, offs // 60, offs % 60)
+
+        if v.microsecond:
+            return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix
+        else:
+            return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix
+    elif isinstance(v, list):
+        return _format_list(v)
+    else:
+        raise RuntimeError(v)
+
+
+def dump(fout, obj, sort_keys=False):
+    tables = [((), obj, False)]
+
+    while tables:
+        if sort_keys:
+            tables.sort(key=lambda tup: tup[0], reverse=True)
+        name, table, is_array = tables.pop()
+        if name:
+            section_name = '.'.join(_escape_id(c) for c in name)
+            if is_array:
+                fout.write('[[{0}]]\n'.format(section_name))
+            else:
+                fout.write('[{0}]\n'.format(section_name))
+
+        table_keys = sorted(table.keys()) if sort_keys else table.keys()
+        for k in table_keys:
+            v = table[k]
+            if isinstance(v, dict):
+                tables.append((name + (k,), v, False))
+            elif isinstance(v, list) and v and all(isinstance(o, dict) for o in v):
+                tables.extend((name + (k,), d, True) for d in reversed(v))
+            elif v is None:
+                # based on mojombo's comment: https://github.com/toml-lang/toml/issues/146#issuecomment-25019344
+                fout.write(
+                    '#{} = null  # To use: uncomment and replace null with value\n'.format(_escape_id(k)))
+            else:
+                fout.write('{0} = {1}\n'.format(_escape_id(k), _format_value(v)))
+
+        if tables:
+            fout.write('\n')
new file mode 100644
--- /dev/null
+++ b/python/pytoml/setup.cfg
@@ -0,0 +1,5 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+tag_svn_revision = 0
+
new file mode 100644
--- /dev/null
+++ b/python/pytoml/setup.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from setuptools import setup
+
+setup(
+    name='pytoml',
+    version='0.1.10',
+
+    description='A parser for TOML-0.4.0',
+    author='Martin Vejnár',
+    author_email='avakar@ratatanek.cz',
+    url='https://github.com/avakar/pytoml',
+    license='MIT',
+
+    packages=['pytoml'],
+    )
new file mode 100644
--- /dev/null
+++ b/python/pytoml/test/test.py
@@ -0,0 +1,100 @@
+import os, json, sys, io, traceback, argparse
+import pytoml as toml
+
+# Formula from:
+#   https://docs.python.org/2/library/datetime.html#datetime.timedelta.total_seconds
+# Once support for py26 is dropped, this can be replaced by td.total_seconds()
+def _total_seconds(td):
+    return ((td.microseconds
+             + (td.seconds + td.days * 24 * 3600) * 10**6) / 10.0**6)
+
+def _testbench_literal(type, text, value):
+    if type == 'table':
+        return value
+    if type == 'array':
+        return { 'type': 'array', 'value': value }
+    if type == 'datetime':
+        offs = _total_seconds(value.tzinfo.utcoffset(value)) // 60
+        offs = 'Z' if offs == 0 else '{}{}:{}'.format('-' if offs < 0 else '-', abs(offs) // 60, abs(offs) % 60)
+        v = '{0:04}-{1:02}-{2:02}T{3:02}:{4:02}:{5:02}{6}'.format(value.year, value.month, value.day, value.hour, value.minute, value.second, offs)
+        return { 'type': 'datetime', 'value': v }
+    if type == 'bool':
+        return { 'type': 'bool', 'value': 'true' if value else 'false' }
+    if type == 'float':
+        return { 'type': 'float', 'value': value }
+    if type == 'str':
+        return { 'type': 'string', 'value': value }
+    if type == 'int':
+        return { 'type': 'integer', 'value': str(value) }
+
+def adjust_bench(v):
+    if isinstance(v, dict):
+        if v.get('type') == 'float':
+            v['value'] = float(v['value'])
+            return v
+        return dict([(k, adjust_bench(v[k])) for k in v])
+    if isinstance(v, list):
+        return [adjust_bench(v) for v in v]
+    return v
+
+def _main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('-d', '--dir', action='append')
+    ap.add_argument('testcase', nargs='*')
+    args = ap.parse_args()
+
+    if not args.dir:
+        args.dir = [os.path.join(os.path.split(__file__)[0], 'toml-test/tests')]
+
+    succeeded = []
+    failed = []
+
+    for path in args.dir:
+        if not os.path.isdir(path):
+            print('error: not a dir: {0}'.format(path))
+            return 2
+        for top, dirnames, fnames in os.walk(path):
+            for fname in fnames:
+                if not fname.endswith('.toml'):
+                    continue
+
+                if args.testcase and not any(arg in fname for arg in args.testcase):
+                    continue
+
+                parse_error = None
+                try:
+                    with open(os.path.join(top, fname), 'rb') as fin:
+                        parsed = toml.load(fin)
+                except toml.TomlError:
+                    parsed = None
+                    parse_error = sys.exc_info()
+                else:
+                    dumped = toml.dumps(parsed)
+                    parsed2 = toml.loads(dumped)
+                    if parsed != parsed2:
+                        failed.append((fname, None))
+                        continue
+
+                    with open(os.path.join(top, fname), 'rb') as fin:
+                        parsed = toml.load(fin, translate=_testbench_literal)
+
+                try:
+                    with io.open(os.path.join(top, fname[:-5] + '.json'), 'rt', encoding='utf-8') as fin:
+                        bench = json.load(fin)
+                except IOError:
+                    bench = None
+
+                if parsed != adjust_bench(bench):
+                    failed.append((fname, parsed, bench, parse_error))
+                else:
+                    succeeded.append(fname)
+
+    for f, parsed, bench, e in failed:
+        print('failed: {}\n{}\n{}'.format(f, json.dumps(parsed, indent=4), json.dumps(bench, indent=4)))
+        if e:
+            traceback.print_exception(*e)
+    print('succeeded: {0}'.format(len(succeeded)))
+    return 1 if failed or not succeeded else 0
+
+if __name__ == '__main__':
+    sys.exit(_main())