Bug 1255450 - [mach] Simplify managing of locale documentation for settings, r=gps
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 23 Mar 2016 17:49:15 -0400
changeset 330688 b552927e1a1fe9f916811236eb30352be72eaaaf
parent 330687 99146ae25db725f0a1b37b624ec47fea07366e03
child 330689 84ba3a5bc33cc4af94f5a46465a6f921d794041b
push id6048
push userkmoir@mozilla.com
push dateMon, 06 Jun 2016 19:02:08 +0000
treeherdermozilla-beta@46d72a56c57d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1255450
milestone48.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1255450 - [mach] Simplify managing of locale documentation for settings, r=gps This adds a |mach settings locale-gen| subcommand to automatically generate locale specific documentation for settings. It also refactors |mach settings-create| to |mach settings| and moves |mach settings| to |mach settings -l|. Finally it performs some misc cleanup mostly related to locales. MozReview-Commit-ID: 1VWLcb9ehAH
python/mach/docs/settings.rst
python/mach/mach/commands/settings.py
python/mach/mach/config.py
python/mach/mach/test/test_config.py
--- a/python/mach/docs/settings.rst
+++ b/python/mach/docs/settings.rst
@@ -69,16 +69,31 @@ This allows configuration files like thi
 
 .. parsed-literal::
 
     [foo]
     arbitrary1 = some string
     arbitrary2 = some other string
 
 
+Documenting Settings
+====================
+
+All settings must at least be documented in the en_US locale. Otherwise,
+running ``mach settings`` will raise. Mach uses gettext to perform localization.
+
+A handy command exists to generate the localization files:
+
+.. parsed-literal::
+
+    mach settings locale-gen <section>
+
+You'll be prompted to add documentation for all options in section with the
+en_US locale. To add documentation in another locale, pass in ``--locale``.
+
 
 Accessing Settings
 ==================
 
 Now that the settings are defined and documented, they're accessible from
 individual mach commands if the command receives a context in its constructor.
 For example:
 
--- a/python/mach/mach/commands/settings.py
+++ b/python/mach/mach/commands/settings.py
@@ -1,50 +1,132 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import os
 from textwrap import TextWrapper
 
 from mach.decorators import (
+    CommandArgument,
     CommandProvider,
     Command,
+    SubCommand,
 )
 
+POLIB_NOT_FOUND = """
+Could not detect the 'polib' package on the local system.
+Please run:
 
-#@CommandProvider
+    pip install polib
+""".lstrip()
+
+
+@CommandProvider
 class Settings(object):
     """Interact with settings for mach.
 
     Currently, we only provide functionality to view what settings are
     available. In the future, this module will be used to modify settings, help
     people create configs via a wizard, etc.
     """
     def __init__(self, context):
-        self.settings = context.settings
+        self._settings = context.settings
+
+    @Command('settings', category='devenv',
+             description='Show available config settings.')
+    @CommandArgument('-l', '--list', dest='short', action='store_true',
+                     help='Show settings in a concise list')
+    def settings(self, short=None):
+        """List available settings."""
+        if short:
+            for section in sorted(self._settings):
+                for option in sorted(self._settings[section]._settings):
+                    short, full = self._settings.option_help(section, option)
+                    print('%s.%s -- %s' % (section, option, short))
+            return
+
+        wrapper = TextWrapper(initial_indent='# ', subsequent_indent='# ')
+        for section in sorted(self._settings):
+            print('[%s]' % section)
+
+            for option in sorted(self._settings[section]._settings):
+                short, full = self._settings.option_help(section, option)
+                print(wrapper.fill(full))
 
-    @Command('settings-list', category='devenv',
-        description='Show available config settings.')
-    def list_settings(self):
-        """List available settings in a concise list."""
-        for section in sorted(self.settings):
-            for option in sorted(self.settings[section]):
-                short, full = self.settings.option_help(section, option)
-                print('%s.%s -- %s' % (section, option, short))
+                if option != '*':
+                    print(';%s =' % option)
+                    print('')
+
+    @SubCommand('settings', 'locale-gen',
+                description='Generate or update gettext .po and .mo locale files.')
+    @CommandArgument('sections', nargs='*',
+                     help='A list of strings in the form of either <section> or '
+                          '<section>.<option> to translate. By default, prompt to '
+                          'translate all applicable options.')
+    @CommandArgument('--locale', default='en_US',
+                     help='Locale to generate, defaults to en_US.')
+    @CommandArgument('--overwrite', action='store_true', default=False,
+                     help='Overwrite pre-existing entries in .po files.')
+    def locale_gen(self, sections, **kwargs):
+        try:
+            import polib
+        except ImportError:
+            print(POLIB_NOT_FOUND)
+            return 1
+
+        self.was_prompted = False
+
+        sections = sections or self._settings
+        for section in sections:
+            self._gen_section(section, **kwargs)
 
-    @Command('settings-create', category='devenv',
-        description='Print a new settings file with usage info.')
-    def create(self):
-        """Create an empty settings file with full documentation."""
-        wrapper = TextWrapper(initial_indent='# ', subsequent_indent='# ')
+        if not self.was_prompted:
+            print("All specified options already have an {} translation. "
+                  "To overwrite existing translations, pass --overwrite."
+                  .format(kwargs['locale']))
+
+    def _gen_section(self, section, **kwargs):
+        if '.' in section:
+            section, option = section.split('.')
+            return self._gen_option(section, option, **kwargs)
+
+        for option in sorted(self._settings[section]._settings):
+            self._gen_option(section, option, **kwargs)
+
+    def _gen_option(self, section, option, locale, overwrite):
+        import polib
+
+        meta = self._settings[section]._settings[option]
+
+        localedir = os.path.join(meta['localedir'], locale, 'LC_MESSAGES')
+        if not os.path.isdir(localedir):
+            os.makedirs(localedir)
 
-        for section in sorted(self.settings):
-            print('[%s]' % section)
-            print('')
+        path = os.path.join(localedir, '{}.po'.format(section))
+        if os.path.isfile(path):
+            po = polib.pofile(path)
+        else:
+            po = polib.POFile()
+
+        optionid = '{}.{}'.format(section, option)
+        for name in ('short', 'full'):
+            msgid = '{}.{}'.format(optionid, name)
+            entry = po.find(msgid)
+            if not entry:
+                entry = polib.POEntry(msgid=msgid)
+                po.append(entry)
 
-            for option in sorted(self.settings[section]):
-                short, full = self.settings.option_help(section, option)
+            if entry in po.translated_entries() and not overwrite:
+                continue
+
+            self.was_prompted = True
 
-                print(wrapper.fill(full))
-                print(';%s =' % option)
-                print('')
+            msgstr = raw_input("Translate {} to {}:\n"
+                               .format(msgid, locale))
+            entry.msgstr = msgstr
+
+        if self.was_prompted:
+            mopath = os.path.join(localedir, '{}.mo'.format(section))
+            po.save(path)
+            po.save_as_mofile(mopath)
--- a/python/mach/mach/config.py
+++ b/python/mach/mach/config.py
@@ -15,19 +15,18 @@ settings are available.
 
 Descriptions of individual config options can be translated to multiple
 languages using gettext. Each option has associated with it a domain and locale
 directory. By default, the domain is the section the option is in and the
 locale directory is the "locale" directory beneath the directory containing the
 module that defines it.
 
 People implementing ConfigProvider instances are expected to define a complete
-gettext .po and .mo file for the en-US locale. You can use the gettext-provided
-msgfmt binary to perform this conversion. Generation of the original .po file
-can be done via the write_pot() of ConfigSettings.
+gettext .po and .mo file for the en_US locale. The |mach settings locale-gen|
+command can be used to populate these files.
 """
 
 from __future__ import absolute_import, unicode_literals
 
 import collections
 import gettext
 import os
 import sys
@@ -36,16 +35,24 @@ from functools import wraps
 if sys.version_info[0] == 3:
     from configparser import RawConfigParser, NoSectionError
     str_type = str
 else:
     from ConfigParser import RawConfigParser, NoSectionError
     str_type = basestring
 
 
+TRANSLATION_NOT_FOUND = """
+No translation files detected for {section}, there must at least be a
+translation for the 'en_US' locale. To generate these files, run:
+
+    mach settings locale-gen {section}
+""".lstrip()
+
+
 class ConfigException(Exception):
     pass
 
 
 class ConfigType(object):
     """Abstract base class for config values."""
 
     @staticmethod
@@ -292,45 +299,42 @@ class ConfigSettings(collections.Mapping
 
 
     def __init__(self):
         self._config = RawConfigParser()
 
         self._settings = {}
         self._sections = {}
         self._finalized = False
-        self._loaded_filenames = set()
+        self.loaded_files = set()
 
     def load_file(self, filename):
         self.load_files([filename])
 
     def load_files(self, filenames):
         """Load a config from files specified by their paths.
 
         Files are loaded in the order given. Subsequent files will overwrite
         values from previous files. If a file does not exist, it will be
         ignored.
         """
         filtered = [f for f in filenames if os.path.exists(f)]
 
         fps = [open(f, 'rt') for f in filtered]
         self.load_fps(fps)
-        self._loaded_filenames.update(set(filtered))
+        self.loaded_files.update(set(filtered))
         for fp in fps:
             fp.close()
 
     def load_fps(self, fps):
         """Load config data by reading file objects."""
 
         for fp in fps:
             self._config.readfp(fp)
 
-    def loaded_files(self):
-        return self._loaded_filenames
-
     def write(self, fh):
         """Write the config to a file object."""
         self._config.write(fh)
 
     @classmethod
     def _format_metadata(cls, provider, section, option, type_cls,
                          default=DefaultValue, extra=None):
         """Formats and returns the metadata for a setting.
@@ -394,48 +398,34 @@ class ConfigSettings(collections.Mapping
             config_settings[section][option] = meta
 
         for section_name, settings in config_settings.items():
             section = self._settings.get(section_name, {})
 
             for k, v in settings.items():
                 if k in section:
                     raise ConfigException('Setting already registered: %s.%s' %
-                        section_name, k)
+                                          section_name, k)
 
                 section[k] = v
 
             self._settings[section_name] = section
 
-    def write_pot(self, fh):
-        """Write a pot gettext translation file."""
-
-        for section in sorted(self):
-            fh.write('# Section %s\n\n' % section)
-            for option in sorted(self[section]):
-                fh.write('msgid "%s.%s.short"\n' % (section, option))
-                fh.write('msgstr ""\n\n')
-
-                fh.write('msgid "%s.%s.full"\n' % (section, option))
-                fh.write('msgstr ""\n\n')
-
-            fh.write('# End of section %s\n\n' % section)
-
     def option_help(self, section, option):
         """Obtain the translated help messages for an option."""
 
         meta = self[section].get_meta(option)
 
-        # Providers should always have an en-US translation. If they don't,
+        # Providers should always have an en_US translation. If they don't,
         # they are coded wrong and this will raise.
         default = gettext.translation(meta['domain'], meta['localedir'],
-            ['en-US'])
+                                      ['en_US'])
 
         t = gettext.translation(meta['domain'], meta['localedir'],
-            fallback=True)
+                                fallback=True)
         t.add_fallback(default)
 
         short = t.ugettext('%s.%s.short' % (section, option))
         full = t.ugettext('%s.%s.full' % (section, option))
 
         return (short, full)
 
     def _finalize(self):
--- a/python/mach/mach/test/test_config.py
+++ b/python/mach/mach/test/test_config.py
@@ -287,21 +287,11 @@ class TestConfigSettings(unittest.TestCa
         s2 = ConfigSettings()
         s2.register_provider(Provider2)
 
         s2.load_file(temp.name)
 
         self.assertEqual(s.a.string, s2.a.string)
         self.assertEqual(s.a.boolean, s2.a.boolean)
 
-    def test_write_pot(self):
-        s = ConfigSettings()
-        s.register_provider(Provider1)
-        s.register_provider(Provider2)
-
-        # Just a basic sanity test.
-        temp = NamedTemporaryFile('wt')
-        s.write_pot(temp)
-        temp.flush()
-
 
 if __name__ == '__main__':
     main()