Bug 1533481 - Part 4: Prepare update to ICU 64. r=jwalden!
authorAndré Bargull <andre.bargull@gmail.com>
Thu, 04 Apr 2019 11:52:54 +0300
changeset 526749 912fb96c21b3946b50e462b5ff00fda9f55050d9
parent 526748 de8c3ab271f7771205c9f7b8d51055a72c1a3cf4
child 526750 cedfd85187269b0d17924be2fec2014af77cf93a
push id11265
push userffxbld-merge
push dateMon, 13 May 2019 10:53:39 +0000
treeherdermozilla-beta@77e0fe8dbdd3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjwalden
bugs1533481
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1533481 - Part 4: Prepare update to ICU 64. r=jwalden! Summary: - ICU no longer supports removing resource files manually, instead the new data filter builder needs to be used. - This includes test data, which also must be preserved in ICU 64. - umutex.h contains an implicit call to `new` in ICU64 which needs to be allowed in check_vanilla_allocations.py. - Drive-by: Make ICU with multiple jobs for faster rebuilds. Reviewers: jwalden Reviewed By: jwalden Bug #: 1533481 Differential Revision: https://phabricator.services.mozilla.com/D25264
config/check_vanilla_allocations.py
intl/icu/data_filter.json
intl/icu_sources_data.py
intl/update-icu.sh
--- a/config/check_vanilla_allocations.py
+++ b/config/check_vanilla_allocations.py
@@ -145,16 +145,24 @@ def main():
         if "Fuzzer" in filename:
             continue
 
         # Ignore the profiling pseudo-stack, since it needs to run even when
         # SpiderMonkey's allocator isn't initialized.
         if "ProfilingStack" in filename:
             continue
 
+        # Ignore implicit call to operator new in std::condition_variable_any.
+        #
+        # From intl/icu/source/common/umutex.h:
+        # On Linux, the default constructor of std::condition_variable_any
+        # produces an in-line reference to global operator new(), [...].
+        if filename == 'umutex.o':
+            continue
+
         fn = m.group(2)
         if filename == 'jsutil.o':
             jsutil_cpp.add(fn)
         else:
             # An allocation is present in a non-special file.  Fail!
             fail("'" + fn + "' present in " + filename)
             # Try to give more precise information about the offending code.
             emit_line_info = True
new file mode 100644
--- /dev/null
+++ b/intl/icu/data_filter.json
@@ -0,0 +1,44 @@
+{
+    "featureFilters": {
+        "brkitr_dictionaries": {
+            "filterType": "exclude"
+        },
+        "brkitr_rules": {
+            "filterType": "exclude"
+        },
+        "brkitr_tree": {
+            "filterType": "exclude"
+        },
+        "conversion_mappings": {
+            "whitelist": [
+                "ibm-37_P100-1995",
+                "ibm-1047_P100-1995"
+            ]
+        },
+        "lang_tree": {
+            "filterType": "exclude"
+        },
+        "rbnf_tree": {
+            "filterType": "exclude"
+        },
+        "region_tree": {
+            "filterType": "exclude"
+        },
+        "translit": {
+            "filterType": "exclude"
+        },
+        "unit_tree": {
+            "filterType": "exclude"
+        }
+    },
+    "resourceFilters": [
+        {
+            "categories": [
+                "zone_tree"
+            ],
+            "rules": [
+                "-/zoneStrings/*/ec"
+            ]
+        }
+    ]
+}
--- a/intl/icu_sources_data.py
+++ b/intl/icu_sources_data.py
@@ -8,16 +8,17 @@
 # system to generate a new copy of the in-tree ICU data file.
 #
 # This script expects to be run from `update-icu.sh` after the in-tree
 # copy of ICU has been updated.
 
 from __future__ import print_function
 
 import glob
+import multiprocessing
 import os
 import sets
 import shutil
 import subprocess
 import sys
 import tempfile
 
 from mozpack import path as mozpath
@@ -212,16 +213,23 @@ def update_data_file(topsrcdir):
     env.update({
         'CPPFLAGS': ('-DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 ' +
                      '-DUCONFIG_NO_LEGACY_CONVERSION ' +
                      '-DUCONFIG_NO_TRANSLITERATION ' +
                      '-DUCONFIG_NO_REGULAR_EXPRESSIONS ' +
                      '-DUCONFIG_NO_BREAK_ITERATION ' +
                      '-DU_CHARSET_IS_UTF8')
     })
+
+    # Exclude data that we currently don't need.
+    #
+    # The file format for ICU's data build tool is described at
+    # <https://github.com/unicode-org/icu/blob/master/docs/userguide/icu_data/buildtool.md>.
+    env["ICU_DATA_FILTER_FILE"] = mozpath.join(topsrcdir, 'intl/icu/data_filter.json')
+
     print('Running ICU configure...')
     if not try_run(
             'icu-configure',
             ['sh', configure,
              '--with-data-packaging=archive',
              '--enable-static',
              '--disable-shared',
              '--disable-extras',
@@ -230,17 +238,22 @@ def update_data_file(topsrcdir):
              '--disable-layoutex',
              '--disable-tests',
              '--disable-samples',
              '--disable-strict'],
             cwd=objdir,
             env=env):
         return False
     print('Running ICU make...')
-    if not try_run('icu-make', ['make'], cwd=objdir):
+    if not try_run(
+            'icu-make',
+            ['make',
+             '--jobs=%d' % multiprocessing.cpu_count(),
+             '--output-sync'],
+            cwd=objdir):
         return False
     print('Copying ICU data file...')
     tree_data_path = mozpath.join(topsrcdir,
                                   'config/external/icu/data/')
     old_data_file = get_data_file(tree_data_path)
     if not old_data_file:
         print('Error: no ICU data file in %s' % tree_data_path,
               file=sys.stderr)
--- a/intl/update-icu.sh
+++ b/intl/update-icu.sh
@@ -39,53 +39,21 @@ cp -r ${tmpclonedir}/icu4c/source ${icu_
 # (This ensures that if ICU modifications are performed properly, it's always
 # possible to run the command at the top of this script and make no changes to
 # the tree.)
 git -C ${tmpclonedir} log -1 > ${icu_dir}/GIT-INFO
 
 # Clean up after ourselves.
 rm -rf ${tmpclonedir}
 
-# remove layoutex, tests, and samples, but leave makefiles in place
+# Remove layoutex, tests, and samples, but leave makefiles and test data in place.
 find ${icu_dir}/source/layoutex -name '*Makefile.in' -prune -or -type f -print | xargs rm
-find ${icu_dir}/source/test -name '*Makefile.in' -prune -or -type f -print | xargs rm
+find ${icu_dir}/source/test -name '*Makefile.in' -prune -or -name 'testdata' -prune -or -type f -print | xargs rm
 find ${icu_dir}/source/samples -name '*Makefile.in' -prune -or -type f -print | xargs rm
 
-# remove data that we currently don't need
-rm -rf ${icu_dir}/source/data/brkitr/*
-rm ${icu_dir}/source/data/lang/*.mk
-rm ${icu_dir}/source/data/lang/*.txt
-rm ${icu_dir}/source/data/mappings/*.mk
-find ${icu_dir}/source/data/mappings \
-    -name ibm-37_P100-1995.ucm -prune -or \
-    -name ibm-1047_P100-1995.ucm -prune -or \
-    -name '*.ucm' -print | xargs rm
-rm ${icu_dir}/source/data/rbnf/*
-rm ${icu_dir}/source/data/region/*.mk
-rm ${icu_dir}/source/data/region/*.txt
-rm ${icu_dir}/source/data/translit/*
-rm ${icu_dir}/source/data/unit/*.mk
-rm ${icu_dir}/source/data/unit/*.txt
-
-# Remove all exemplar cities 'ec'. (bug 1225401 and bug 1345336)
-find ${icu_dir}/source/data/zone \
-    -name root.txt -prune -or \
-    -name tzdbNames.txt -prune -or \
-    -name '*.txt' -print | xargs sed -i '/^\s*ec{\".*\"}$/ { d }'
-# Remove empty time zone entries after exemplar cities removal.
-find ${icu_dir}/source/data/zone \
-    -name root.txt -prune -or \
-    -name tzdbNames.txt -prune -or \
-    -name '*.txt' -print | xargs sed -i '/^\s*\"[A-Z][a-zA-Z:_-]*\"{/{N; s/^\s*\"[A-Z][a-zA-Z:_-]*\"{\n\s*}// }; /^$/d'
-# And finally remove any empty 'zoneStrings' entries.
-find ${icu_dir}/source/data/zone \
-    -name root.txt -prune -or \
-    -name tzdbNames.txt -prune -or \
-    -name '*.txt' -print | xargs sed -i '/^\s*zoneStrings{/{N; s/^\s*zoneStrings{\n\s*}// }; /^$/d'
-
 for patch in \
  bug-915735 \
  suppress-warnings.diff \
  bug-1172609-timezone-recreateDefault.diff \
  bug-1198952-workaround-make-3.82-bug.diff \
 ; do
   echo "Applying local patch $patch"
   patch -d ${icu_dir}/../../ -p1 --no-backup-if-mismatch < ${icu_dir}/../icu-patches/$patch