Bug 1395459 - Store the version of the locale data in a langpack based on HG push timestamp. r=gps
authorZibi Braniecki <zbraniecki@mozilla.com>
Sun, 10 Sep 2017 19:57:34 -0700
changeset 383273 e73300d22ba597dc397b34fc901fa3ab8f2fda56
parent 383272 e8737573b0e0c8515487812b5f01aaba865bbef8
child 383274 9e57229ccdb9c231c9506e115fd7f64fd84cb1b6
push id95539
push userkwierso@gmail.com
push dateThu, 28 Sep 2017 00:01:12 +0000
treeherdermozilla-inbound@72de90e66155 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1395459
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1395459 - Store the version of the locale data in a langpack based on HG push timestamp. r=gps MozReview-Commit-ID: ELx9VW81s6Q
python/mozbuild/mozbuild/action/langpack_manifest.py
toolkit/locales/l10n.mk
--- a/python/mozbuild/mozbuild/action/langpack_manifest.py
+++ b/python/mozbuild/mozbuild/action/langpack_manifest.py
@@ -10,29 +10,101 @@
 ###
 from __future__ import absolute_import
 
 import argparse
 import sys
 import os
 import json
 import io
+import datetime
+import requests
+import mozversioncontrol
 from mozpack.chrome.manifest import (
     Manifest,
     ManifestLocale,
     parse_manifest,
 )
 from mozbuild.preprocessor import Preprocessor
 
 
 def write_file(path, content):
     with io.open(path, 'w', encoding='utf-8') as out:
         out.write(content + '\n')
 
 
+pushlog_api_url = "{0}/json-rev/{1}"
+
+
+###
+# Retrievers a UTC datetime of the push for the current commit
+# from a mercurial clone directory.
+#
+# Args:
+#    path (str) - path to a directory
+#
+# Returns:
+#    (datetime) - a datetime object
+#
+# Example:
+#    dt = get_dt_from_hg("/var/vcs/l10n-central/pl")
+#    dt == datetime(2017, 10, 11, 23, 31, 54, 0)
+###
+def get_dt_from_hg(path):
+    with mozversioncontrol.get_repository_object(path=path) as repo:
+        repo_url = repo._run_in_client(["paths", "default"])
+        repo_url = repo_url.strip().replace("ssh://", "https://")
+        repo_url = repo_url.replace("hg://", "https://")
+        cs = repo._run_in_client(["log", "-r", ".", "-T" "{node}"])
+
+    url = pushlog_api_url.format(repo_url, cs)
+    session = requests.Session()
+    try:
+        response = session.get(url)
+    except Exception as e:
+        msg = "Failed to retrieve push timestamp using {}\nError: {}".format(url, e)
+        raise Exception(msg)
+
+    data = response.json()
+
+    date = data['pushdate'][0]
+
+    return datetime.datetime.utcfromtimestamp(date)
+
+
+###
+# Generates timestamp for a locale based on its path.
+# If possible, will use the commit timestamp from HG repository,
+# and if that fails, will generate the timestamp for `now`.
+#
+# The timestamp format is "{year}{month}{day}{hour}{minute}{second}" and
+# the datetime stored in it is using UTC timezone.
+#
+# Args:
+#    path (str) - path to the locale directory
+#
+# Returns:
+#    (str) - a timestamp string
+#
+# Example:
+#    ts = get_timestamp_for_locale("/var/vcs/l10n-central/pl")
+#    ts == "20170914215617"
+###
+def get_timestamp_for_locale(path):
+    dt = None
+    if os.path.isdir(os.path.join(path, '.hg')):
+        dt = get_dt_from_hg(path)
+
+    if dt is None:
+        dt = datetime.datetime.utcnow()
+
+    dt = dt.replace(microsecond=0)
+    return dt.strftime("%Y%m%d%H%M%S")
+
+
 ###
 # Parses multiple defines files into a single key-value pair object.
 #
 # Args:
 #    paths (str) - a comma separated list of paths to defines files
 #
 # Returns:
 #    (dict) - a key-value dict with defines
@@ -233,16 +305,17 @@ def parse_chrome_manifest(path, base_pat
 #
 # Example:
 #    manifest = create_webmanifest(
 #      ['pl'],
 #      '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}',
 #      '57.0',
 #      '57.0.*',
 #      'Firefox',
+#      '/var/vcs/l10n-central',
 #      {'MOZ_LANG_TITLE': 'Polski'},
 #      chrome_entries
 #    )
 #    manifest == {
 #        'languages': {
 #            'pl': {
 #                'version': '201709121481',
 #                'chrome_resources': {
@@ -272,17 +345,17 @@ def parse_chrome_manifest(path, base_pat
 #            }
 #        },
 #        'version': '57.0',
 #        'name': 'Polski Language Pack',
 #        ...
 #    }
 ###
 def create_webmanifest(locstr, min_app_ver, max_app_ver, app_name,
-                       defines, chrome_entries):
+                       l10n_basedir, defines, chrome_entries):
     locales = map(lambda loc: loc.strip(), locstr.split(','))
     main_locale = locales[0]
 
     author = build_author_string(
         defines['MOZ_LANGPACK_CREATOR'],
         defines['MOZ_LANGPACK_CONTRIBUTORS']
     )
 
@@ -320,33 +393,35 @@ def create_webmanifest(locstr, min_app_v
             else:
                 assert entry['alias'] not in cr
                 cr[entry['alias']] = entry['path']
         else:
             raise Exception('Unknown type {0}'.format(entry['type']))
 
     for loc in locales:
         manifest['languages'][loc] = {
-            'version': min_app_ver,
+            'version': get_timestamp_for_locale(os.path.join(l10n_basedir, loc)),
             'chrome_resources': cr
         }
 
     return json.dumps(manifest, indent=2, ensure_ascii=False, encoding='utf8')
 
 
 def main(args):
     parser = argparse.ArgumentParser()
     parser.add_argument('--locales',
                         help='List of language codes provided by the langpack')
     parser.add_argument('--min-app-ver',
                         help='Min version of the application the langpack is for')
     parser.add_argument('--max-app-ver',
                         help='Max version of the application the langpack is for')
     parser.add_argument('--app-name',
                         help='Name of the application the langpack is for')
+    parser.add_argument('--l10n-basedir',
+                        help='Base directory for locales used in the language pack')
     parser.add_argument('--defines', default=[], nargs='+',
                         help='List of defines files to load data from')
     parser.add_argument('--input',
                         help='Langpack directory.')
 
     args = parser.parse_args(args)
 
     chrome_entries = []
@@ -355,16 +430,17 @@ def main(args):
 
     defines = parse_defines(args.defines)
 
     res = create_webmanifest(
         args.locales,
         args.min_app_ver,
         args.max_app_ver,
         args.app_name,
+        args.l10n_basedir,
         defines,
         chrome_entries
     )
     write_file(os.path.join(args.input, 'manifest.json'), res)
 
 
 if __name__ == '__main__':
     main(sys.argv[1:])
--- a/toolkit/locales/l10n.mk
+++ b/toolkit/locales/l10n.mk
@@ -220,17 +220,17 @@ langpack-%: libs-%
 	$(call py_action,zip,-C $(DIST)/xpi-stage/locale-$(AB_CD) $(LANGPACK_FILE) install.rdf $(PKG_ZIP_DIRS) chrome.manifest)
 
 langpack-webext-%: LANGPACK_FILE=$(ABS_DIST)/$(PKG_LANGPACK_PATH)$(PKG_LANGPACK_BASENAME).xpi
 langpack-webext-%: AB_CD=$*
 langpack-webext-%: XPI_NAME=locale-$*
 langpack-webext-%: libs-%
 	@echo 'Making new-langpack $(LANGPACK_FILE)'
 	$(NSINSTALL) -D $(DIST)/$(PKG_LANGPACK_PATH)
-	$(call py_action,langpack_manifest,--locales $(AB_CD) --min-app-ver $(MOZ_APP_VERSION) --max-app-ver $(MOZ_APP_MAXVERSION) --app-name "$(MOZ_APP_DISPLAYNAME)" --defines $(NEW_APP_DEFINES) --input $(DIST)/xpi-stage/locale-$(AB_CD))
+	$(call py_action,langpack_manifest,--locales $(AB_CD) --min-app-ver $(MOZ_APP_VERSION) --max-app-ver $(MOZ_APP_MAXVERSION) --app-name "$(MOZ_APP_DISPLAYNAME)" --l10n-basedir "$(L10NBASEDIR)" --defines $(NEW_APP_DEFINES) --input $(DIST)/xpi-stage/locale-$(AB_CD))
 	$(call py_action,zip,-C $(DIST)/xpi-stage/locale-$(AB_CD) -x **/*.manifest -x **/*.js -x **/*.ini $(LANGPACK_FILE) $(PKG_ZIP_DIRS) manifest.json)
 
 # This variable is to allow the wget-en-US target to know which ftp server to download from
 ifndef EN_US_BINARY_URL 
 EN_US_BINARY_URL = $(error You must set EN_US_BINARY_URL)
 endif
 # In taskcluster the installer comes from another location
 ifndef EN_US_INSTALLER_BINARY_URL