[Windows only] patch 2 - convert preprocess-locale.pl to python rev3 for Bug 570689 - Convert preprocess-locale.pl to a python script. r=ted.mielczarek, approval2.0=benjamin
authorRobert Strong <robert.bugzilla@gmail.com>
Thu, 29 Jul 2010 22:39:25 -0700
changeset 48395 e1161df6af197e9ac4ac337b74a020b31f59459f
parent 48394 92c3a3a024057b65a4b6a4b9b42335f320c01a1a
child 48396 bf4cec83e4d242cf2aea5869b349743f5cf8ed3f
push id14712
push userrstrong@mozilla.com
push dateFri, 30 Jul 2010 05:47:51 +0000
treeherdermozilla-central@aee9fa2b6e9d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted.mielczarek
bugs570689
milestone2.0b3pre
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
[Windows only] patch 2 - convert preprocess-locale.pl to python rev3 for Bug 570689 - Convert preprocess-locale.pl to a python script. r=ted.mielczarek, approval2.0=benjamin
browser/installer/windows/Makefile.in
configure.in
toolkit/mozapps/installer/windows/nsis/locales.nsi
toolkit/mozapps/installer/windows/nsis/makensis.mk
toolkit/mozapps/installer/windows/nsis/preprocess-locale.py
--- a/browser/installer/windows/Makefile.in
+++ b/browser/installer/windows/Makefile.in
@@ -49,35 +49,25 @@ include $(topsrcdir)/toolkit/mozapps/ins
 
 CONFIG_DIR = instgen
 SFX_MODULE = $(topsrcdir)/other-licenses/7zstub/firefox/7zSD.sfx
 APP_VERSION := $(shell cat $(srcdir)/../../config/version.txt)
 DEFINES += -DAPP_VERSION=$(APP_VERSION)
 PRE_RELEASE_SUFFIX := $(shell $(PYTHON) $(topsrcdir)/config/printprereleasesuffix.py $(APP_VERSION))
 DEFINES += -DPRE_RELEASE_SUFFIX="$(PRE_RELEASE_SUFFIX)"
 
-# All script and locale files used by the Unicode version of NSIS need to be
-# converted from UTF-8 to UTF-16LE
-INSTALLER_FILES_CONV = \
+INSTALLER_FILES = \
+	app.tag \
 	nsis/installer.nsi \
 	nsis/uninstaller.nsi \
 	nsis/shared.nsh \
 	$(NULL)
 
-INSTALLER_FILES = \
-	app.tag \
-	$(NULL)
-
-# All script and locale files used by the Unicode version of NSIS need to be
-# converted from UTF-8 to UTF-16LE
-BRANDING_FILES_CONV = \
+BRANDING_FILES = \
 	branding.nsi \
-	$(NULL)
-
-BRANDING_FILES = \
 	wizHeader.bmp \
 	wizHeaderRTL.bmp \
 	wizWatermark.bmp \
 	$(NULL)
 
 DEFINES += \
 	-DAB_CD=$(AB_CD) \
 	-DMOZ_APP_NAME=$(MOZ_APP_NAME) \
@@ -90,52 +80,28 @@ include $(topsrcdir)/config/config.mk
 installer::
 	$(MAKE) -C .. installer-stage
 	$(MAKE) $(CONFIG_DIR)/setup.exe
 
 # For building the uninstaller during the application build so it can be
 # included for mar file generation.
 uninstaller::
 	$(RM) -rf $(CONFIG_DIR) && mkdir $(CONFIG_DIR)
-	for i in $(INSTALLER_FILES_CONV); do \
-	  iconv -f UTF-8 -t UTF-16LE $(srcdir)/$$i | \
-	    cat $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin - > \
-	    $(CONFIG_DIR)/`basename $$i`; \
-	done
 	$(INSTALL) $(addprefix $(srcdir)/,$(INSTALLER_FILES)) $(CONFIG_DIR)
-	for i in $(BRANDING_FILES_CONV); do \
-	  iconv -f UTF-8 -t UTF-16LE $(DIST)/branding/$$i | \
-	    cat $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin - > \
-	    $(CONFIG_DIR)/$$i; \
-	done
 	$(INSTALL) $(addprefix $(DIST)/branding/,$(BRANDING_FILES)) $(CONFIG_DIR)
 	$(PYTHON) $(topsrcdir)/config/Preprocessor.py -Fsubstitution $(DEFINES) $(ACDEFINES) \
-	  $(srcdir)/nsis/defines.nsi.in | iconv -f UTF-8 -t UTF-16LE | \
-	  cat $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin - > \
-	  $(CONFIG_DIR)/defines.nsi
-	$(PERL) $(topsrcdir)/toolkit/mozapps/installer/windows/nsis/preprocess-locale.pl \
-	  $(topsrcdir) $(call EXPAND_LOCALE_SRCDIR,browser/locales)/installer $(AB_CD) \
-	  $(CONFIG_DIR)
+	  $(srcdir)/nsis/defines.nsi.in > $(CONFIG_DIR)/defines.nsi
+	$(PYTHON) $(topsrcdir)/toolkit/mozapps/installer/windows/nsis/preprocess-locale.py \
+	  --preprocess-locale $(topsrcdir) \
+	  $(call EXPAND_LOCALE_SRCDIR,browser/locales)/installer $(AB_CD) $(CONFIG_DIR)
 
 $(CONFIG_DIR)/setup.exe::
 	$(RM) -rf $(CONFIG_DIR) && mkdir $(CONFIG_DIR)
-	for i in $(INSTALLER_FILES_CONV); do \
-	  iconv -f UTF-8 -t UTF-16LE $(srcdir)/$$i | \
-	    cat $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin - > \
-	    $(CONFIG_DIR)/`basename $$i`; \
-	done
 	$(INSTALL) $(addprefix $(srcdir)/,$(INSTALLER_FILES)) $(CONFIG_DIR)
-	for i in $(BRANDING_FILES_CONV); do \
-	  iconv -f UTF-8 -t UTF-16LE $(DIST)/branding/$$i | \
-	    cat $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin - > \
-	    $(CONFIG_DIR)/$$i; \
-	done
 	$(INSTALL) $(addprefix $(DIST)/branding/,$(BRANDING_FILES)) $(CONFIG_DIR)
 	$(PYTHON) $(topsrcdir)/config/Preprocessor.py -Fsubstitution $(DEFINES) $(ACDEFINES) \
-	  $(srcdir)/nsis/defines.nsi.in | iconv -f UTF-8 -t UTF-16LE | \
-	  cat $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin - > \
-	  $(CONFIG_DIR)/defines.nsi
-	$(PERL) $(topsrcdir)/toolkit/mozapps/installer/windows/nsis/preprocess-locale.pl \
-	  $(topsrcdir) $(call EXPAND_LOCALE_SRCDIR,browser/locales)/installer $(AB_CD) \
-	  $(CONFIG_DIR)
+	  $(srcdir)/nsis/defines.nsi.in > $(CONFIG_DIR)/defines.nsi
+	$(PYTHON) $(topsrcdir)/toolkit/mozapps/installer/windows/nsis/preprocess-locale.py \
+	  --preprocess-locale $(topsrcdir) \
+	  $(call EXPAND_LOCALE_SRCDIR,browser/locales)/installer $(AB_CD) $(CONFIG_DIR)
 
 include $(topsrcdir)/config/rules.mk
 include $(topsrcdir)/toolkit/mozapps/installer/windows/nsis/makensis.mk
--- a/configure.in
+++ b/configure.in
@@ -6437,23 +6437,16 @@ if test -n "$MOZ_INSTALLER" -a "$OS_ARCH
     if test "$MAKENSISU_VER" == "" -o \
        ! "$MAKENSISU_MAJOR_VER" == "$REQ_NSIS_MAJOR_VER" -o \
        ! "$MAKENSISU_MINOR_VER" -ge $MIN_NSIS_MINOR_VER; then
         AC_MSG_RESULT([no])
         AC_MSG_ERROR([To build the installer you must have the latest MozillaBuild or Unicode NSIS with a major version of $REQ_NSIS_MAJOR_VER and a minimum minor version of $MIN_NSIS_MINOR_VER in your path. To build without the installer reconfigure using --disable-installer.])
     fi
     AC_MSG_RESULT([yes])
     MAKENSISU="${CYGWIN_WRAPPER} $MAKENSISU"
-
-    # The Windows build for NSIS requires the iconv command line utility to
-    # convert the charset of the locale files.
-    MOZ_PATH_PROGS(HOST_ICONV, $HOST_ICONV "iconv", "")
-    if test -z "$HOST_ICONV"; then
-        AC_MSG_ERROR([To build the installer iconv is required in your path. To build without the installer reconfigure using --disable-installer.])
-    fi
 fi
 
 AC_SUBST(MOZ_INSTALLER)
 
 AC_MSG_CHECKING([for tar archiver])
 AC_CHECK_PROGS(TAR, gnutar gtar tar, "")
 if test -z "$TAR"; then
     AC_MSG_ERROR([no tar archiver found in \$PATH])
--- a/toolkit/mozapps/installer/windows/nsis/locales.nsi
+++ b/toolkit/mozapps/installer/windows/nsis/locales.nsi
@@ -31,18 +31,17 @@
 # and other provisions required by the GPL or the LGPL. If you do not delete
 # the provisions above, a recipient may use your version of this file under
 # the terms of any one of the MPL, the GPL or the LGPL.
 #
 # ***** END LICENSE BLOCK *****
 
 /**
  * "One off" locale configuration settings for RTL (e.g. locale text is read
- * right to left). Fonts should not only be specified if Win9x systems are
- * supported for East Asian locales.
+ * right to left).
  */
 
 ; Arabic
 !define ar_rtl
 
 ; Hebrew
 !define he_rtl
 
--- a/toolkit/mozapps/installer/windows/nsis/makensis.mk
+++ b/toolkit/mozapps/installer/windows/nsis/makensis.mk
@@ -40,40 +40,36 @@
 ifndef CONFIG_DIR
 $(error CONFIG_DIR must be set before including makensis.mk)
 endif
 
 ABS_CONFIG_DIR := $(shell pwd)/$(CONFIG_DIR)
 
 SFX_MODULE ?= $(error SFX_MODULE is not defined)
 
-# UTF-8 encoded files that need to be converted to UTF-16LE for NSIS
-TOOLKIT_NSIS_FILES_CONV = \
+TOOLKIT_NSIS_FILES = \
 	common.nsh \
 	locales.nsi \
 	overrides.nsh \
+	setup.ico \
 	version.nsh \
 	$(NULL)
 
 CUSTOM_NSIS_PLUGINS = \
 	AccessControl.dll \
 	AppAssocReg.dll \
 	ApplicationID.dll \
 	nsProcess.dll \
 	ShellLink.dll \
 	UAC.dll \
 	$(NULL)
 
 $(CONFIG_DIR)/setup.exe::
-	for i in $(TOOLKIT_NSIS_FILES_CONV); do \
-	  iconv -f UTF-8 -t UTF-16LE $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/$$i | \
-	    cat $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin - > $(CONFIG_DIR)/$$i; \
-	done
+	$(INSTALL) $(addprefix $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/,$(TOOLKIT_NSIS_FILES)) $(CONFIG_DIR)
 	$(INSTALL) $(addprefix $(MOZILLA_DIR)/other-licenses/nsis/Plugins/,$(CUSTOM_NSIS_PLUGINS)) $(CONFIG_DIR)
-	$(INSTALL) $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/setup.ico $(CONFIG_DIR)
 	cd $(CONFIG_DIR) && $(MAKENSISU) installer.nsi
 # Support for building the uninstaller when repackaging locales
 ifeq ($(CONFIG_DIR),l10ngen)
 	cd $(CONFIG_DIR) && $(MAKENSISU) uninstaller.nsi
 endif
 
 $(CONFIG_DIR)/7zSD.sfx:
 	$(CYGWIN_WRAPPER) upx --best -o $(CONFIG_DIR)/7zSD.sfx $(SFX_MODULE)
@@ -84,17 +80,13 @@ installer::
 	$(MAKE) $(CONFIG_DIR)/7zSD.sfx
 	$(NSINSTALL) -D $(DIST)/$(PKG_INST_PATH)
 	cat $(CONFIG_DIR)/7zSD.sfx $(CONFIG_DIR)/app.tag $(CONFIG_DIR)/app.7z > "$(DIST)/$(PKG_INST_PATH)$(PKG_INST_BASENAME).exe"
 	chmod 0755 "$(DIST)/$(PKG_INST_PATH)$(PKG_INST_BASENAME).exe"
 
 # For building the uninstaller during the application build so it can be
 # included for mar file generation.
 uninstaller::
-	for i in $(TOOLKIT_NSIS_FILES_CONV); do \
-	  iconv -f UTF-8 -t UTF-16LE $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/$$i | \
-	    cat $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin - > $(CONFIG_DIR)/$$i; \
-	done
+	$(INSTALL) $(addprefix $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/,$(TOOLKIT_NSIS_FILES)) $(CONFIG_DIR)
 	$(INSTALL) $(addprefix $(MOZILLA_DIR)/other-licenses/nsis/Plugins/,$(CUSTOM_NSIS_PLUGINS)) $(CONFIG_DIR)
-	$(INSTALL) $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/setup.ico $(CONFIG_DIR)
 	cd $(CONFIG_DIR) && $(MAKENSISU) uninstaller.nsi
 	$(NSINSTALL) -D $(DIST)/bin/uninstall
 	cp $(CONFIG_DIR)/helper.exe $(DIST)/bin/uninstall
new file mode 100644
--- /dev/null
+++ b/toolkit/mozapps/installer/windows/nsis/preprocess-locale.py
@@ -0,0 +1,201 @@
+# preprocess-locale.py
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+# preprocess-locale.py provides two functions depending on the arguments passed
+# to it when invoked.
+#
+# Preprocesses installer locale properties files and creates a basic NSIS nlf
+# file when invoked with --preprocess-locale.
+# 
+# Converts a UTF-8 file to a new UTF-16LE file when invoked with
+# --convert-utf8-utf16le.
+
+from codecs import BOM_UTF16_LE
+from os.path import join
+import sys
+
+def preprocess_locale(argv):
+    """
+    Validates command line arguments and displays usage if necessary
+    """
+    if len(argv) < 1 or (argv[0] != '--convert-utf8-utf16le' and argv[0] != '--preprocess-locale'):
+        sys.stderr.write("""
+preprocess-locale.py
+
+Commands:
+ --convert-utf8-utf16le - preprocesses installer locale properties files and
+                          creates a basic NSIS nlf file
+ --preprocess-locale    - converts a UTF-8 file to a new UTF-16LE file
+
+use "preprocess-locale.py <command>" to see the usage for each command
+""")
+        sys.exit(1)
+
+    if argv[0] == '--convert-utf8-utf16le':
+        if len(argv) != 3:
+            sys.stderr.write("""
+Converts a UTF-8 file to UTF-16LE
+
+Usage: preprocess-locale.py --convert-utf8-utf16le <src> <dest>
+
+Arguments:
+ <src> \tthe path to the UTF-8 source file to convert
+ <dest>\tthe path to the UTF-16LE destination file to create
+""")
+            sys.exit(1)
+	convert_utf8_utf16le(argv[1], argv[2])
+
+    if argv[0] == '--preprocess-locale':
+        if len(argv) != 5:
+            sys.stderr.write("""
+Preprocesses the installer localized properties files into the format
+required by NSIS and creates a basic NSIS nlf file.
+
+Usage: preprocess-locale.py --preprocess-locale <src> <locale> <code> <dest>
+
+Arguments:
+ <src>   \tthe path to top source directory for the toolkit source
+ <locale>\tthe path to the installer's locale files
+ <code>  \tthe locale code
+ <dest>  \tthe path to the destination directory
+""")
+            sys.exit(1)
+        preprocess_locale_files(argv[1], argv[2], argv[4], argv[4])
+
+def open_utf16le_file(path):
+    """
+    Returns an opened file object with a a UTF-16LE byte order mark.
+    """
+    fp = open(path, "w+b")
+    fp.write(BOM_UTF16_LE)
+    return fp
+
+def get_locale_strings(path, prefix, middle, add_cr):
+    """
+    Returns a string created by converting an installer locale properties file
+    into the format required by NSIS locale files.
+
+    Parameters:
+    path   - the path to the installer locale properties file to preprocess
+    prefix - a string to prefix each line with
+    middle - a string to insert between the name and value for each line
+    add_cr - boolean for whether to add an NSIS carriage return before NSIS
+             linefeeds when there isn't one already
+    """
+    output = ""
+    fp = open(path, "r")
+    for line in fp:
+        line = line.strip()
+        if line == "" or line[0] == "#":
+            continue
+
+        name, value = line.split("=", 1)
+        value = value.strip() # trim whitespace from the start and end
+        if value[-1] == "\"" and value[0] == "\"":
+            value = value[1:-1] # remove " from the start and end
+
+        if add_cr:
+            value = value.replace("\\n", "\\r\\n") # prefix $\n with $\r
+            value = value.replace("\\r\\r", "\\r") # replace $\r$\r with $\r
+
+        value = value.replace("\"", "$\\\"") # prefix " with $\
+        value = value.replace("\\r", "$\\r") # prefix \r with $
+        value = value.replace("\\n", "$\\n") # prefix \n with $
+        value = value.replace("\\t", "$\\t") # prefix \t with $
+
+        output += prefix + name.strip() + middle + " \"" + value + "\"\n"
+    fp.close()
+    return output
+
+def preprocess_locale_files(moz_dir, locale_dir, ab_cd, config_dir):
+    """
+    Preprocesses the installer localized properties files into the format
+    required by NSIS and creates a basic NSIS nlf file.
+
+    Parameters:
+    moz_dir    - the path to top source directory for the toolkit source
+    locale_dir - the path to the installer's locale files
+    ab_cd      - the locale code
+    config_dir - the path to the destination directory
+    """
+
+    # Set the language ID to 0 to make this locale the default locale. An
+    # actual ID will need to be used to create a multi-language installer
+    # (e.g. for CD distributions, etc.).
+    lang_id = "0"
+    rtl = "-"
+
+    # Check whether the locale is right to left from locales.nsi.
+    fp = open(join(moz_dir, "toolkit/mozapps/installer/windows/nsis/locales.nsi"), "r")
+    for line in fp:
+        line = line.strip()
+        if line == "!define " + ab_cd + "_rtl":
+            rtl = "RTL"
+            break
+
+    fp.close()
+
+    # Create the main NSIS language file with RTL for right to left locales
+    # along with the default codepage, font name, and font size represented
+    # by the '-' character.
+    fp = open_utf16le_file(join(config_dir, "baseLocale.nlf"))
+    fp.write((u"""# Header, don't edit
+NLF v6
+# Start editing here
+# Language ID
+%s
+# Font and size - dash (-) means default
+-
+-
+# Codepage - dash (-) means ANSI code page
+-
+# RTL - anything else than RTL means LTR
+%s
+""" % (lang_id, rtl)).encode("utf-16-le"))
+    fp.close()
+
+    # Create the main NSIS language file
+    fp = open_utf16le_file(join(config_dir, "overrideLocale.nsh"))
+    locale_strings = get_locale_strings(join(locale_dir, "override.properties"),
+                                        "LangString ^", " " + lang_id + " ", False)
+    fp.write(unicode(locale_strings, "utf-8").encode("utf-16-le"))
+    fp.close()
+
+    # Create the Modern User Interface language file
+    fp = open_utf16le_file(join(config_dir, "baseLocale.nsh"))
+    fp.write((u""";NSIS Modern User Interface - Language File
+;Compatible with Modern UI 1.68
+;Language: baseLocale (%s)
+!insertmacro MOZ_MUI_LANGUAGEFILE_BEGIN \"baseLocale\"
+!define MUI_LANGNAME \"baseLocale\"
+""" % (lang_id)).encode("utf-16-le"))
+    locale_strings = get_locale_strings(join(locale_dir, "mui.properties"),
+                                        "!define ", " ", True)
+    fp.write(unicode(locale_strings, "utf-8").encode("utf-16-le"))
+    fp.write(u"!insertmacro MOZ_MUI_LANGUAGEFILE_END\n".encode("utf-16-le"))
+    fp.close()
+
+    # Create the custom language file for our custom strings
+    fp = open_utf16le_file(join(config_dir, "customLocale.nsh"))
+    locale_strings = get_locale_strings(join(locale_dir, "custom.properties"),
+                        "LangString ", " " + lang_id + " ", True)
+    fp.write(unicode(locale_strings, "utf-8").encode("utf-16-le"))
+    fp.close()
+
+def convert_utf8_utf16le(in_file_path, out_file_path):
+    """
+    Converts a UTF-8 file to a new UTF-16LE file
+
+    Arguments:
+    in_file_path  - the path to the UTF-8 source file to convert
+    out_file_path - the path to the UTF-16LE destination file to create
+    """
+    in_fp = open(in_file_path, "r")
+    out_fp = open_utf16le_file(out_file_path)
+    out_fp.write(unicode(in_fp.read(), "utf-8").encode("utf-16-le"))
+    in_fp.close()
+    out_fp.close()
+
+if __name__ == '__main__':
+    sys.exit(preprocess_locale(sys.argv[1:]))