Bug 479520: Implement IDNA2008 standard for International Domain Names, patch 1: build system changes, r=ted; code changes, r=jfkthame
authorSimon Montagu <smontagu@smontagu.org>
Wed, 28 Oct 2015 07:05:15 -0700
changeset 270002 ac843b13053736adf0fb793ac435e454f08a7dd2
parent 270001 692dbb16a94976c90cd7700604bb02be54f3d20b
child 270003 dd3d6c83f3546eedbb4dffc515134176c5ea5c2d
push id29596
push userkwierso@gmail.com
push dateThu, 29 Oct 2015 00:00:32 +0000
treeherdermozilla-central@1e700005a0dd [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted, code, jfkthame
bugs479520
milestone44.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 479520: Implement IDNA2008 standard for International Domain Names, patch 1: build system changes, r=ted; code changes, r=jfkthame
CLOBBER
build/autoconf/icu.m4
netwerk/dns/moz.build
netwerk/dns/nsIDNService.cpp
netwerk/dns/nsIDNService.h
netwerk/standalone/moz.build
--- a/CLOBBER
+++ b/CLOBBER
@@ -17,9 +17,9 @@
 #
 # Modifying this file will now automatically clobber the buildbot machines \o/
 #
 
 # Are you updating CLOBBER because you think it's needed for your WebIDL
 # changes to stick? As of bug 928195, this shouldn't be necessary! Please
 # don't change CLOBBER for WebIDL changes any more.
 
-Bug 1217261 - Update mp4parse to v0.1.2
+Bug 479520 - Implement IDNA2008 standard for International Domain Names
--- a/build/autoconf/icu.m4
+++ b/build/autoconf/icu.m4
@@ -137,17 +137,16 @@ if test -z "$BUILDING_JS" -o -n "$JS_STA
         ICU_CPPFLAGS="$ICU_CPPFLAGS -DU_USING_ICU_NAMESPACE=0"
         # don't include obsolete header files
         ICU_CPPFLAGS="$ICU_CPPFLAGS -DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1"
         # remove chunks of the library that we don't need (yet)
         ICU_CPPFLAGS="$ICU_CPPFLAGS -DUCONFIG_NO_LEGACY_CONVERSION"
         ICU_CPPFLAGS="$ICU_CPPFLAGS -DUCONFIG_NO_TRANSLITERATION"
         ICU_CPPFLAGS="$ICU_CPPFLAGS -DUCONFIG_NO_REGULAR_EXPRESSIONS"
         ICU_CPPFLAGS="$ICU_CPPFLAGS -DUCONFIG_NO_BREAK_ITERATION"
-        ICU_CPPFLAGS="$ICU_CPPFLAGS -DUCONFIG_NO_IDNA"
         # we don't need to pass data to and from legacy char* APIs
         ICU_CPPFLAGS="$ICU_CPPFLAGS -DU_CHARSET_IS_UTF8"
         # make sure to not accidentally pick up system-icu headers
         ICU_CPPFLAGS="$ICU_CPPFLAGS -I$icudir/common -I$icudir/i18n"
 
         ICU_CROSS_BUILD_OPT=""
 
         if test "$CROSS_COMPILE"; then
--- a/netwerk/dns/moz.build
+++ b/netwerk/dns/moz.build
@@ -35,17 +35,16 @@ SOURCES += [
 
 UNIFIED_SOURCES += [
     'ChildDNSService.cpp',
     'DNS.cpp',
     'DNSListenerProxy.cpp',
     'DNSRequestChild.cpp',
     'DNSRequestParent.cpp',
     'GetAddrInfo.cpp',
-    'nameprep.c',
     'nsDNSService2.cpp',
     'nsIDNService.cpp',
     'punycode.c',
 ]
 
 IPDL_SOURCES = [
     'PDNSRequest.ipdl',
     'PDNSRequestParams.ipdlh',
@@ -64,8 +63,18 @@ etld_data.inputs = ['effective_tld_names
 
 # need to include etld_data.inc
 LOCAL_INCLUDES += [
     '/netwerk/base',
 ]
 
 if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'gonk' and CONFIG['ANDROID_VERSION'] > '19':
     CXXFLAGS += ['-I%s/bionic/libc/dns/include' % CONFIG['ANDROID_SOURCE']]
+
+if CONFIG['ENABLE_INTL_API']:
+    DEFINES['IDNA2008'] = True
+    CXXFLAGS += CONFIG['MOZ_ICU_CFLAGS']
+    CFLAGS += CONFIG['MOZ_ICU_CFLAGS']
+    USE_LIBS += ['icu']
+else:
+    UNIFIED_SOURCES += [
+        'nameprep.c',
+    ]
--- a/netwerk/dns/nsIDNService.cpp
+++ b/netwerk/dns/nsIDNService.cpp
@@ -12,16 +12,26 @@
 #include "harfbuzz/hb.h"
 #include "nsIServiceManager.h"
 #include "nsIPrefService.h"
 #include "nsIPrefBranch.h"
 #include "nsIObserverService.h"
 #include "nsISupportsPrimitives.h"
 #include "punycode.h"
 
+#ifdef IDNA2008
+// Currently we use the transitional processing option -- see
+// http://unicode.org/reports/tr46/
+// To switch to non-transitional processing, change the value of this flag
+// and kTransitionalProcessing in netwerk/test/unit/test_idna2008.js to false
+// (patch in bug 1218179).
+const bool kIDNA2008_TransitionalProcessing = true;
+
+#include "ICUUtils.h"
+#endif
 
 using namespace mozilla::unicode;
 
 //-----------------------------------------------------------------------------
 // RFC 1034 - 3.1. Name space specifications and terminology
 static const uint32_t kMaxDNSNodeLen = 63;
 // RFC 3490 - 5.   ACE prefix
 static const char kACEPrefix[] = "xn--";
@@ -118,28 +128,101 @@ void nsIDNService::prefsChanged(nsIPrefB
     } else {
       mRestrictionProfile = eASCIIOnlyProfile;
     }
   }
 }
 
 nsIDNService::nsIDNService()
 {
+#ifdef IDNA2008
+  uint32_t IDNAOptions = UIDNA_CHECK_BIDI | UIDNA_CHECK_CONTEXTJ;
+  if (!kIDNA2008_TransitionalProcessing) {
+    IDNAOptions |= UIDNA_NONTRANSITIONAL_TO_UNICODE;
+  }
+  UErrorCode errorCode = U_ZERO_ERROR;
+  mIDNA = uidna_openUTS46(IDNAOptions, &errorCode);
+#else
   if (idn_success != idn_nameprep_create(nullptr, &mNamePrepHandle))
     mNamePrepHandle = nullptr;
 
   mNormalizer = do_GetService(NS_UNICODE_NORMALIZER_CONTRACTID);
   /* member initializers and constructor code */
+#endif
 }
 
 nsIDNService::~nsIDNService()
 {
+#ifdef IDNA2008
+  uidna_close(mIDNA);
+#else
   idn_nameprep_destroy(mNamePrepHandle);
+#endif
 }
 
+#ifdef IDNA2008
+nsresult
+nsIDNService::IDNA2008ToUnicode(const nsACString& input, nsAString& output)
+{
+  NS_ConvertUTF8toUTF16 inputStr(input);
+  UIDNAInfo info = UIDNA_INFO_INITIALIZER;
+  UErrorCode errorCode = U_ZERO_ERROR;
+  int32_t inLen = inputStr.Length();
+  int32_t outMaxLen = inLen - kACEPrefixLen + 1;
+  UChar outputBuffer[kMaxDNSNodeLen + 1];
+
+  int32_t outLen = uidna_labelToUnicode(mIDNA, (const UChar*)inputStr.get(),
+                                        inLen, outputBuffer, outMaxLen,
+                                        &info, &errorCode);
+  if (info.errors != 0) {
+    return NS_ERROR_FAILURE;
+  }
+
+  if (U_SUCCESS(errorCode)) {
+    ICUUtils::AssignUCharArrayToString(outputBuffer, outLen, output);
+  }
+
+  return ICUUtils::UErrorToNsResult(errorCode);
+}
+
+nsresult
+nsIDNService::IDNA2008StringPrep(const nsAString& input,
+                                 nsAString& output,
+                                 stringPrepFlag flag)
+{
+  UIDNAInfo info = UIDNA_INFO_INITIALIZER;
+  UErrorCode errorCode = U_ZERO_ERROR;
+  int32_t inLen = input.Length();
+  int32_t outMaxLen = kMaxDNSNodeLen + 1;
+  UChar outputBuffer[kMaxDNSNodeLen + 1];
+
+  int32_t outLen =
+    uidna_labelToUnicode(mIDNA, (const UChar*)PromiseFlatString(input).get(),
+                         inLen, outputBuffer, outMaxLen, &info, &errorCode);
+  nsresult rv = ICUUtils::UErrorToNsResult(errorCode);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // Output the result of nameToUnicode even if there were errors
+  ICUUtils::AssignUCharArrayToString(outputBuffer, outLen, output);
+
+  if (flag == eStringPrepIgnoreErrors) {
+    return NS_OK;
+  }
+
+  if (info.errors != 0) {
+    if (flag == eStringPrepForDNS) {
+      output.Truncate();
+    }
+    rv = NS_ERROR_FAILURE;
+  }
+
+  return rv;
+}
+#endif
+
 NS_IMETHODIMP nsIDNService::ConvertUTF8toACE(const nsACString & input, nsACString & ace)
 {
   return UTF8toACE(input, ace, eStringPrepForDNS);
 }
 
 nsresult nsIDNService::UTF8toACE(const nsACString & input, nsACString & ace,
                                  stringPrepFlag flag)
 {
@@ -391,28 +474,30 @@ static nsresult utf16ToUcs4(const nsAStr
     if (i >= outBufLen)
       return NS_ERROR_FAILURE;
   }
   out[i] = (uint32_t)'\0';
   *outLen = i;
   return NS_OK;
 }
 
+#ifndef IDNA2008
 static void ucs4toUtf16(const uint32_t *in, nsAString& out)
 {
   while (*in) {
     if (!IS_IN_BMP(*in)) {
       out.Append((char16_t) H_SURROGATE(*in));
       out.Append((char16_t) L_SURROGATE(*in));
     }
     else
       out.Append((char16_t) *in);
     in++;
   }
 }
+#endif
 
 static nsresult punycode(const nsAString& in, nsACString& out)
 {
   uint32_t ucs4Buf[kMaxDNSNodeLen + 1];
   uint32_t ucs4Len;
   nsresult rv = utf16ToUcs4(in, ucs4Buf, kMaxDNSNodeLen, &ucs4Len);
   NS_ENSURE_SUCCESS(rv, rv);
 
@@ -457,16 +542,19 @@ static nsresult punycode(const nsAString
 //
 // 5) Check unassigned code points -- If allowUnassigned is false, check for
 // any unassigned Unicode points and if any are found return an error.
 // This is described in section 7.
 //
 nsresult nsIDNService::stringPrep(const nsAString& in, nsAString& out,
                                   stringPrepFlag flag)
 {
+#ifdef IDNA2008
+  return IDNA2008StringPrep(in, out, flag);
+#else
   if (!mNamePrepHandle || !mNormalizer)
     return NS_ERROR_FAILURE;
 
   uint32_t ucs4Buf[kMaxDNSNodeLen + 1];
   uint32_t ucs4Len;
   nsresult rv = utf16ToUcs4(in, ucs4Buf, kMaxDNSNodeLen, &ucs4Len);
   NS_ENSURE_SUCCESS(rv, rv);
 
@@ -518,16 +606,17 @@ nsresult nsIDNService::stringPrep(const 
     }
   }
 
   if (flag == eStringPrepForDNS && NS_FAILED(rv)) {
     out.Truncate();
   }
 
   return rv;
+#endif
 }
 
 nsresult nsIDNService::stringPrepAndACE(const nsAString& in, nsACString& out,
                                         stringPrepFlag flag)
 {
   nsresult rv = NS_OK;
 
   out.Truncate();
@@ -607,16 +696,21 @@ nsresult nsIDNService::decodeACE(const n
 {
   bool isAce;
   IsACE(in, &isAce);
   if (!isAce) {
     out.Assign(in);
     return NS_OK;
   }
 
+  nsAutoString utf16;
+#ifdef IDNA2008
+  nsresult result = IDNA2008ToUnicode(in, utf16);
+  NS_ENSURE_SUCCESS(result, result);
+#else
   // RFC 3490 - 4.2 ToUnicode
   // The ToUnicode output never contains more code points than its input.
   punycode_uint output_length = in.Length() - kACEPrefixLen + 1;
   punycode_uint *output = new punycode_uint[output_length];
   NS_ENSURE_TRUE(output, NS_ERROR_OUT_OF_MEMORY);
 
   enum punycode_status status = punycode_decode(in.Length() - kACEPrefixLen,
                                                 PromiseFlatCString(in).get() + kACEPrefixLen,
@@ -625,19 +719,19 @@ nsresult nsIDNService::decodeACE(const n
                                                 nullptr);
   if (status != punycode_success) {
     delete [] output;
     return NS_ERROR_FAILURE;
   }
 
   // UCS4 -> UTF8
   output[output_length] = 0;
-  nsAutoString utf16;
   ucs4toUtf16(output, utf16);
   delete [] output;
+#endif
   if (flag != eStringPrepForUI || isLabelSafe(utf16)) {
     CopyUTF16toUTF8(utf16, out);
   } else {
     out.Assign(in);
     return NS_OK;
   }
 
   // Validation: encode back to ACE and compare the strings
--- a/netwerk/dns/nsIDNService.h
+++ b/netwerk/dns/nsIDNService.h
@@ -5,18 +5,24 @@
 
 #ifndef nsIDNService_h__
 #define nsIDNService_h__
 
 #include "nsIIDNService.h"
 #include "nsCOMPtr.h"
 #include "nsIObserver.h"
 #include "nsWeakReference.h"
+
+#ifdef IDNA2008
+#include "unicode/uidna.h"
+#else
 #include "nsIUnicodeNormalizer.h"
 #include "nsIDNKitInterface.h"
+#endif
+
 #include "nsString.h"
 
 class nsIPrefBranch;
 
 //-----------------------------------------------------------------------------
 // nsIDNService
 //-----------------------------------------------------------------------------
 
@@ -138,18 +144,33 @@ private:
    *  Latin + Han + Bopomofo; or
    *  Latin + Han + Hangul
    *
    * For the "Moderately restrictive" profile, Latin is also allowed
    *  with other scripts except Cyrillic and Greek
    */
   bool illegalScriptCombo(int32_t script, int32_t& savedScript);
 
+#ifdef IDNA2008
+  /**
+   * Convert a DNS label from ASCII to Unicode using IDNA2008
+   */
+  nsresult IDNA2008ToUnicode(const nsACString& input, nsAString& output);
+
+  /**
+   * Convert a DNS label to a normalized form conforming to IDNA2008
+   */
+  nsresult IDNA2008StringPrep(const nsAString& input, nsAString& output,
+                              stringPrepFlag flag);
+
+  UIDNA* mIDNA;
+#else
   idn_nameprep_t mNamePrepHandle;
   nsCOMPtr<nsIUnicodeNormalizer> mNormalizer;
+#endif
   nsXPIDLString mIDNBlacklist;
 
   /**
    * Flag set by the pref network.IDN_show_punycode. When it is true,
    * IDNs containing non-ASCII characters are always displayed to the
    * user in punycode
    */
   bool mShowPunycode;
--- a/netwerk/standalone/moz.build
+++ b/netwerk/standalone/moz.build
@@ -38,13 +38,14 @@ src_list += [
 SOURCES += sorted(src_list)
 
 LOCAL_INCLUDES = [
     '../base',
     '../build',
     '../dns',
 ]
 
+DEFINES['IDNA2008'] = False
 DEFINES['MOZILLA_INTERNAL_API'] = True
 DEFINES['MOZILLA_XPCOMRT_API'] = True
 DEFINES['MOZILLA_EXTERNAL_LINKAGE'] = True
 
 include('/ipc/chromium/chromium-config.mozbuild')