Bug 1377287: Add URIUtils.getHostSecondLevelDomain and friends. r=liuche
authorMichael Comella <michael.l.comella@gmail.com>
Mon, 17 Jul 2017 18:10:16 -0700
changeset 419907 e25baed29efd7119609ad7838f46e33ee5ecebf7
parent 419906 be3b083e82b8a0eb76d1094f63d3290a8ee8e2bf
child 419908 cdc31f6fc0840fb5fa76a9fefed1801c3625ce96
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersliuche
bugs1377287
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1377287: Add URIUtils.getHostSecondLevelDomain and friends. r=liuche This is used by the iOS implementation of AS to get the highlight titles. MozReview-Commit-ID: 1p5Lf9OBcfD
mobile/android/base/java/org/mozilla/gecko/util/URIUtils.java
mobile/android/base/moz.build
mobile/android/tests/background/junit4/src/org/mozilla/gecko/util/TestURIUtils.java
new file mode 100644
--- /dev/null
+++ b/mobile/android/base/java/org/mozilla/gecko/util/URIUtils.java
@@ -0,0 +1,82 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.util;
+
+import android.content.Context;
+import android.support.annotation.NonNull;
+import android.support.annotation.Nullable;
+import android.support.annotation.WorkerThread;
+import ch.boye.httpclientandroidlib.util.TextUtils;
+import org.mozilla.gecko.util.publicsuffix.PublicSuffix;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/** Utilities for operating on URLs. */
+public class URIUtils {
+    private URIUtils() {}
+
+    /**
+     * Returns the second level domain (SLD) of a url. It removes any subdomain/TLD.
+     * e.g. https://m.foo.com/bar/baz?noo=abc#123  => foo
+     *
+     * This implementation is taken from Firefox for iOS:
+     *   https://github.com/mozilla-mobile/firefox-ios/blob/deb9736c905cdf06822ecc4a20152df7b342925d/Shared/Extensions/NSURLExtensions.swift#L152
+     *
+     * @param uriString A url from which to extract the second level domain.
+     * @return The second level domain of the url.
+     */
+    @WorkerThread // PublicSuffix methods can touch the disk.
+    public static String getHostSecondLevelDomain(@NonNull final Context context, @NonNull final String uriString)
+            throws URISyntaxException {
+        if (context == null) { throw new NullPointerException("Expected non-null Context argument"); }
+        if (uriString == null) { throw new NullPointerException("Expected non-null uri argument"); }
+
+        final URI uri = new URI(uriString);
+        final String baseDomain = getBaseDomain(context, uri);
+        if (baseDomain == null) {
+            final String normalizedHost = StringUtils.stripCommonSubdomains(uri.getHost());
+            return !TextUtils.isEmpty(normalizedHost) ? normalizedHost : uriString;
+        }
+
+        return PublicSuffix.stripPublicSuffix(context, baseDomain);
+    }
+
+    /**
+     * Returns the base domain from a given hostname. The base domain name is defined as the public domain suffix
+     * with the base private domain attached to the front. For example, for the URL www.bbc.co.uk, the base domain
+     * would be bbc.co.uk. The base domain includes the public suffix (co.uk) + one level down (bbc).
+     *
+     * IPv4 & IPv6 urls are not supported and will return null.
+     *
+     * This implementation is taken from Firefox for iOS:
+     *   https://github.com/mozilla-mobile/firefox-ios/blob/deb9736c905cdf06822ecc4a20152df7b342925d/Shared/Extensions/NSURLExtensions.swift#L205
+     *
+     * @param uri The uri to find the base domain of
+     * @return The base domain string for the given host name, or null if not applicable.
+     */
+    @Nullable
+    @WorkerThread // PublicSuffix methods can touch the disk.
+    public static String getBaseDomain(@NonNull final Context context, final URI uri) {
+        final String host = uri.getHost();
+        if (isIPv6(uri) || TextUtils.isEmpty(host)) {
+            return null;
+        }
+
+        // If this is just a hostname and not a FQDN, use the entire hostname.
+        if (!host.contains(".")) {
+            return host;
+        }
+
+        final String publicSuffixWithDomain = PublicSuffix.getPublicSuffix(context, host, 1);
+        return !TextUtils.isEmpty(publicSuffixWithDomain) ? publicSuffixWithDomain : null;
+    }
+
+    // impl via FFiOS: https://github.com/mozilla-mobile/firefox-ios/blob/deb9736c905cdf06822ecc4a20152df7b342925d/Shared/Extensions/NSURLExtensions.swift#L292
+    private static boolean isIPv6(final URI uri) {
+        final String host = uri.getHost();
+        return !TextUtils.isEmpty(host) && host.contains(":");
+    }
+}
--- a/mobile/android/base/moz.build
+++ b/mobile/android/base/moz.build
@@ -943,16 +943,17 @@ gbjar.sources += ['java/org/mozilla/geck
     'updater/PostUpdateHandler.java',
     'updater/UpdateService.java',
     'updater/UpdateServiceHelper.java',
     'util/ColorUtil.java',
     'util/DrawableUtil.java',
     'util/JavaUtil.java',
     'util/ResourceDrawableUtils.java',
     'util/TouchTargetUtil.java',
+    'util/URIUtils.java',
     'util/ViewUtil.java',
     'webapps/WebAppActivity.java',
     'webapps/WebAppIndexer.java',
     'webapps/WebApps.java',
     'widget/ActionModePresenter.java',
     'widget/ActivityChooserModel.java',
     'widget/AllCapsTextView.java',
     'widget/AnchoredPopup.java',
new file mode 100644
--- /dev/null
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/gecko/util/TestURIUtils.java
@@ -0,0 +1,117 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mozilla.gecko.background.testhelpers.TestRunner;
+import org.robolectric.RuntimeEnvironment;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+
+@RunWith(TestRunner.class)
+public class TestURIUtils {
+
+    private final String BUGZILLA_URL = "https://bugzilla.mozilla.org/enter_bug.cgi?format=guided#h=dupes%7CData%20%26%20BI%20Services%20Team%7C";
+
+    @Test
+    public void testGetHostSecondLevelDomain() throws Exception {
+        assertGetHostSLD("https://www.example.com/index.html", "example");
+        assertGetHostSLD("https://m.foo.com/bar/baz?noo=abc#123", "foo");
+        assertGetHostSLD("https://user:pass@m.foo.com/bar/baz?noo=abc#123", "foo");
+    }
+
+    @Test
+    public void testGetHostSecondLevelDomainIPv4() throws Exception {
+        assertGetHostSLD("http://192.168.1.1", "192.168.1.1");
+    }
+
+    @Test
+    public void testGetHostSecondLevelDomainIPv6() throws Exception {
+        assertGetHostSLD("http://[3ffe:1900:4545:3:200:f8ff:fe21:67cf]", "[3ffe:1900:4545:3:200:f8ff:fe21:67cf]");
+    }
+
+    @Test(expected = URISyntaxException.class)
+    public void testGetHostSecondLevelDomainNonURI() throws Exception {
+        URIUtils.getHostSecondLevelDomain(RuntimeEnvironment.application, "this  -is  -not-a-uri");
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void testGetHostSecondLevelDomainNullContextThrows() throws Exception {
+        URIUtils.getHostSecondLevelDomain(null, "http://google.com");
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void testGetHostSecondLevelDomainNullURIThrows() throws Exception {
+        URIUtils.getHostSecondLevelDomain(RuntimeEnvironment.application, null);
+    }
+
+    // SLD = second level domain.
+    private void assertGetHostSLD(final String input, final String expected) throws Exception {
+        Assert.assertEquals("for input:" + input + "||", expected,
+                URIUtils.getHostSecondLevelDomain(RuntimeEnvironment.application, input));
+    }
+
+    @Test
+    public void testGetBaseDomainNormal() throws Exception {
+        assertGetBaseDomain("http://bbc.co.uk", "bbc.co.uk");
+    }
+
+    @Test
+    public void testGetBaseDomainNormalWithAdditionalSubdomain() throws Exception {
+        assertGetBaseDomain("http://a.bbc.co.uk", "bbc.co.uk");
+        assertGetBaseDomain(BUGZILLA_URL, "mozilla.org");
+    }
+
+    @Test
+    public void testGetBaseDomainWilcardDomain() throws Exception {
+        // TLD entry: *.kawasaki.jp
+        assertGetBaseDomain("http://a.b.kawasaki.jp", "a.b.kawasaki.jp");
+    }
+
+    @Test
+    public void testGetBaseDomainWilcardDomainWithAdditionalSubdomain() throws Exception {
+        // TLD entry: *.kawasaki.jp
+        assertGetBaseDomain("http://a.b.c.kawasaki.jp", "b.c.kawasaki.jp");
+    }
+
+    @Test
+    public void testGetBaseDomainExceptionDomain() throws Exception {
+        // TLD entry: !city.kawasaki.jp
+        assertGetBaseDomain("http://city.kawasaki.jp", "city.kawasaki.jp");
+    }
+
+    @Test
+    public void testGetBaseDomainExceptionDomainWithAdditionalSubdomain() throws Exception {
+        // TLD entry: !city.kawasaki.jp
+        assertGetBaseDomain("http://a.city.kawasaki.jp", "city.kawasaki.jp");
+    }
+
+    @Test
+    public void testGetBaseDomainExceptionDomainBugzillaURL() throws Exception {
+        // TLD entry: !city.kawasaki.jp
+        assertGetBaseDomain("http://a.city.kawasaki.jp", "city.kawasaki.jp");
+    }
+
+    @Test
+    public void testGetBaseDomainIPv4() throws Exception {
+        assertGetBaseDomain("http://192.168.1.1", null);
+    }
+
+    @Test
+    public void testGetBaseDomainIPv6() throws Exception {
+        assertGetBaseDomain("http://[3ffe:1900:4545:3:200:f8ff:fe21:67cf]", null);
+    }
+
+    private void assertGetBaseDomain(final String input, final String expected) throws Exception {
+        Assert.assertEquals("for input:" + input + "||",
+                expected,
+                URIUtils.getBaseDomain(RuntimeEnvironment.application, new URI(input)));
+    }
+}
\ No newline at end of file