Bug 1012462 - Part 5: Add distribution support in SuggestedSites (r=rnewman)
authorLucas Rocha <lucasr@mozilla.com>
Tue, 15 Jul 2014 20:54:27 +0100
changeset 216238 6cd4b8d5036c120827894ca0befb96ca789ddf22
parent 216237 7811f4c944c5092ad38281d2d94824dee2c19568
child 216239 13b4b1b388e270cba618a1432410be4ab484e838
push id515
push userraliiev@mozilla.com
push dateMon, 06 Oct 2014 12:51:51 +0000
treeherdermozilla-release@267c7a481bef [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrnewman
bugs1012462
milestone33.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1012462 - Part 5: Add distribution support in SuggestedSites (r=rnewman)
mobile/android/base/db/SuggestedSites.java
mobile/android/tests/browser/junit3/src/tests/TestSuggestedSites.java
--- a/mobile/android/base/db/SuggestedSites.java
+++ b/mobile/android/base/db/SuggestedSites.java
@@ -1,40 +1,49 @@
 /* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 package org.mozilla.gecko.db;
 
 import android.content.Context;
+import android.content.ContentResolver;
 import android.content.SharedPreferences;
 import android.database.Cursor;
 import android.database.MatrixCursor;
 import android.database.MatrixCursor.RowBuilder;
 import android.net.Uri;
 import android.text.TextUtils;
 import android.util.Log;
 
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.OutputStreamWriter;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Scanner;
 import java.util.Set;
 
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 
+import org.mozilla.gecko.BrowserLocaleManager;
 import org.mozilla.gecko.GeckoSharedPrefs;
+import org.mozilla.gecko.GeckoProfile;
 import org.mozilla.gecko.R;
+import org.mozilla.gecko.distribution.Distribution;
 import org.mozilla.gecko.db.BrowserContract;
 import org.mozilla.gecko.mozglue.RobocopTarget;
 import org.mozilla.gecko.preferences.GeckoPreferences;
 import org.mozilla.gecko.util.RawResource;
 import org.mozilla.gecko.util.ThreadUtils;
 
 /**
  * {@code SuggestedSites} provides API to get a list of locale-specific
@@ -57,16 +66,19 @@ import org.mozilla.gecko.util.ThreadUtil
  */
 @RobocopTarget
 public class SuggestedSites {
     private static final String LOGTAG = "GeckoSuggestedSites";
 
     // SharedPreference key for suggested sites that should be hidden.
     public static final String PREF_SUGGESTED_SITES_HIDDEN = "suggestedSites.hidden";
 
+    // File in profile dir with the list of suggested sites.
+    private static final String FILENAME = "suggestedsites.json";
+
     private static final String[] COLUMNS = new String[] {
         BrowserContract.SuggestedSites._ID,
         BrowserContract.SuggestedSites.URL,
         BrowserContract.SuggestedSites.TITLE
     };
 
     private static final String JSON_KEY_URL = "url";
     private static final String JSON_KEY_TITLE = "title";
@@ -124,82 +136,239 @@ public class SuggestedSites {
             json.put(JSON_KEY_IMAGE_URL, imageUrl);
             json.put(JSON_KEY_BG_COLOR, bgColor);
 
             return json;
         }
     }
 
     private final Context context;
+    private final Distribution distribution;
+    private final File file;
     private Map<String, Site> cachedSites;
     private Locale cachedLocale;
     private Set<String> cachedBlacklist;
 
     public SuggestedSites(Context appContext) {
-        context = appContext;
+        this(appContext, null);
+    }
+
+    public SuggestedSites(Context appContext, Distribution distribution) {
+        this(appContext, distribution,
+             GeckoProfile.get(appContext).getFile(FILENAME));
+    }
+
+    public SuggestedSites(Context appContext, Distribution distribution, File file) {
+        this.context = appContext;
+        this.distribution = distribution;
+        this.file = file;
     }
 
-    private Map<String, Site> loadSites(String jsonString) {
+    /**
+     * Return the current locale and its fallback (en_US) in order.
+     */
+    private static List<Locale> getAcceptableLocales() {
+        final List<Locale> locales = new ArrayList<Locale>();
+
+        final Locale defaultLocale = Locale.getDefault();
+        locales.add(defaultLocale);
+
+        if (!defaultLocale.equals(Locale.US)) {
+            locales.add(Locale.US);
+        }
+
+        return locales;
+    }
+
+    private static Map<String, Site> loadSites(File f) throws IOException {
+        Scanner scanner = null;
+
+        try {
+            scanner = new Scanner(f, "UTF-8");
+            return loadSites(scanner.useDelimiter("\\A").next());
+        } finally {
+            if (scanner != null) {
+                scanner.close();
+            }
+        }
+    }
+
+    private static Map<String, Site> loadSites(String jsonString) {
         if (TextUtils.isEmpty(jsonString)) {
             return null;
         }
 
         Map<String, Site> sites = null;
 
         try {
             final JSONArray jsonSites = new JSONArray(jsonString);
             sites = new LinkedHashMap<String, Site>(jsonSites.length());
 
             final int count = jsonSites.length();
             for (int i = 0; i < count; i++) {
-                final Site site = new Site((JSONObject) jsonSites.get(i));
+                final Site site = new Site(jsonSites.getJSONObject(i));
                 sites.put(site.url, site);
             }
         } catch (Exception e) {
             Log.e(LOGTAG, "Failed to refresh suggested sites", e);
             return null;
         }
 
         return sites;
     }
 
-    private Map<String, Site> loadFromFile() {
-        // Do nothing for now
+    /**
+     * Saves suggested sites file to disk. Access to this method should
+     * be synchronized on 'file'.
+     */
+    private static void saveSites(File f, Map<String, Site> sites) {
+        ThreadUtils.assertNotOnUiThread();
+
+        if (sites == null || sites.isEmpty()) {
+            return;
+        }
+
+        OutputStreamWriter osw = null;
+
+        try {
+            final JSONArray jsonSites = new JSONArray();
+            for (Site site : sites.values()) {
+                jsonSites.put(site.toJSON());
+            }
+
+            osw = new OutputStreamWriter(new FileOutputStream(f), "UTF-8");
+
+            final String jsonString = jsonSites.toString();
+            osw.write(jsonString, 0, jsonString.length());
+        } catch (Exception e) {
+            Log.e(LOGTAG, "Failed to save suggested sites", e);
+        } finally {
+            if (osw != null) {
+                try {
+                    osw.close();
+                } catch (IOException e) {
+                    // Ignore.
+                }
+            }
+        }
+    }
+
+    private void maybeWaitForDistribution() {
+        if (distribution == null) {
+            return;
+        }
+
+        distribution.addOnDistributionReadyCallback(new Runnable() {
+            @Override
+            public void run() {
+                Log.d(LOGTAG, "Running post-distribution task: suggested sites.");
+
+                // If distribution doesn't exist, simply continue to load
+                // suggested sites directly from resources. See refresh().
+                if (!distribution.exists()) {
+                    return;
+                }
+
+                // Merge suggested sites from distribution with the
+                // default ones. Distribution takes precedence.
+                Map<String, Site> sites = loadFromDistribution(distribution);
+                if (sites == null) {
+                    sites = new LinkedHashMap<String, Site>();
+                }
+                sites.putAll(loadFromResource());
+
+                // Update cached list of sites.
+                setCachedSites(sites);
+
+                // Save the result to disk.
+                synchronized (file) {
+                    saveSites(file, sites);
+                }
+
+                // Then notify any active loaders about the changes.
+                final ContentResolver cr = context.getContentResolver();
+                cr.notifyChange(BrowserContract.SuggestedSites.CONTENT_URI, null);
+            }
+        });
+    }
+
+    /**
+     * Loads suggested sites from a distribution file either matching the
+     * current locale or with the fallback locale (en-US).
+     *
+     * It's assumed that the given distribution instance is ready to be
+     * used and exists.
+     */
+    private static Map<String, Site> loadFromDistribution(Distribution dist) {
+        for (Locale locale : getAcceptableLocales()) {
+            try {
+                final String languageTag = BrowserLocaleManager.getLanguageTag(locale);
+                final String path = String.format("suggestedsites/locales/%s/%s",
+                                                  languageTag, FILENAME);
+
+                final File f = dist.getDistributionFile(path);
+                if (f == null) {
+                    Log.d(LOGTAG, "No suggested sites for locale: " + languageTag);
+                    continue;
+                }
+
+                return loadSites(f);
+            } catch (Exception e) {
+                Log.e(LOGTAG, "Failed to open suggested sites for locale " +
+                              locale + " in distribution.", e);
+            }
+        }
+
+        return null;
+    }
+
+    private Map<String, Site> loadFromProfile() {
+        try {
+            synchronized (file) {
+                return loadSites(file);
+            }
+        } catch (FileNotFoundException e) {
+            maybeWaitForDistribution();
+        } catch (IOException e) {
+            // Fall through, return null.
+        }
+
         return null;
     }
 
     private Map<String, Site> loadFromResource() {
         try {
             return loadSites(RawResource.getAsString(context, R.raw.suggestedsites));
         } catch (IOException e) {
             return null;
         }
     }
 
+    private synchronized void setCachedSites(Map<String, Site> sites) {
+        cachedSites = Collections.unmodifiableMap(sites);
+        cachedLocale = Locale.getDefault();
+    }
+
     /**
      * Refreshes the cached list of sites either from the default raw
      * source or standard file location. This will be called on every
      * cache miss during a {@code get()} call.
      */
     private void refresh() {
-        Log.d(LOGTAG, "Refreshing tiles from file");
+        Log.d(LOGTAG, "Refreshing suggested sites from file");
 
-        Map<String, Site> sites = loadFromFile();
+        Map<String, Site> sites = loadFromProfile();
         if (sites == null) {
             sites = loadFromResource();
         }
 
-        // Nothing to cache, bail.
-        if (sites == null) {
-            return;
+        // Update cached list of sites.
+        if (sites != null) {
+            setCachedSites(sites);
         }
-
-        // Update cached list of sites.
-        cachedSites = Collections.unmodifiableMap(sites);
-        cachedLocale = Locale.getDefault();
     }
 
     private boolean isEnabled() {
         final SharedPreferences prefs = GeckoSharedPrefs.forApp(context);
         return prefs.getBoolean(GeckoPreferences.PREFS_SUGGESTED_SITES, true);
     }
 
     private synchronized Site getSiteForUrl(String url) {
--- a/mobile/android/tests/browser/junit3/src/tests/TestSuggestedSites.java
+++ b/mobile/android/tests/browser/junit3/src/tests/TestSuggestedSites.java
@@ -1,36 +1,50 @@
 /* Any copyright is dedicated to the Public Domain.
    http://creativecommons.org/publicdomain/zero/1.0/ */
 
 package org.mozilla.gecko.browser.tests;
 
 import android.content.Context;
+import android.content.ContentResolver;
 import android.content.res.Resources;
 import android.content.SharedPreferences;
 import android.database.Cursor;
+import android.database.ContentObserver;
 import android.net.Uri;
+import android.os.Handler;
+import android.os.SystemClock;
 import android.test.mock.MockResources;
 import android.test.RenamingDelegatingContext;
 
 import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.InputStream;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.jar.JarInputStream;
+import java.util.Map;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
 
 import org.json.JSONArray;
 import org.json.JSONObject;
 
+import org.mozilla.gecko.BrowserLocaleManager;
 import org.mozilla.gecko.R;
 import org.mozilla.gecko.db.BrowserContract;
 import org.mozilla.gecko.db.SuggestedSites;
+import org.mozilla.gecko.distribution.Distribution;
 import org.mozilla.gecko.GeckoSharedPrefs;
 import org.mozilla.gecko.preferences.GeckoPreferences;
 
 public class TestSuggestedSites extends BrowserTestCase {
     private static class TestContext extends RenamingDelegatingContext {
         private static final String PREFIX = "TestSuggestedSites-";
 
         private final Resources resources;
@@ -74,20 +88,74 @@ public class TestSuggestedSites extends 
             return null;
         }
 
         public void setSuggestedSitesResource(String suggestedSites) {
             this.suggestedSites = suggestedSites;
         }
     }
 
+    private static class TestDistribution extends Distribution {
+        private final Context context;
+        private final Map<Locale, File> filesPerLocale;
+
+        public TestDistribution(Context context) {
+            super(context);
+            this.context = context;
+            this.filesPerLocale = new HashMap<Locale, File>();
+        }
+
+        @Override
+        public File getDistributionFile(String name) {
+            for (Locale locale : filesPerLocale.keySet()) {
+                if (name.startsWith("suggestedsites/locales/" + BrowserLocaleManager.getLanguageTag(locale))) {
+                    return filesPerLocale.get(locale);
+                }
+            }
+
+            return null;
+        }
+
+        @Override
+        public boolean exists() {
+            return true;
+        }
+
+        public void setFileForLocale(Locale locale, File file) {
+            filesPerLocale.put(locale, file);
+        }
+
+        public void start() {
+            doInit();
+        }
+    }
+
+    class TestObserver extends ContentObserver {
+        private final Object changeLock;
+
+        public TestObserver(Object changeLock) {
+            super(null);
+            this.changeLock = changeLock;
+        }
+
+        @Override
+        public void onChange(boolean selfChange) {
+            synchronized(changeLock) {
+                changeLock.notifyAll();
+            }
+        }
+    }
+
     private static final int DEFAULT_LIMIT = 6;
 
+    private static final String DIST_PREFIX = "dist";
+
     private TestContext context;
     private TestResources resources;
+    private List<File> tempFiles;
 
     private String generateSites(int n) {
         return generateSites(n, "");
     }
 
     private String generateSites(int n, String prefix) {
         JSONArray sites = new JSONArray();
 
@@ -103,34 +171,64 @@ public class TestSuggestedSites extends 
             }
         } catch (Exception e) {
             return "";
         }
 
         return sites.toString();
     }
 
+    private File createDistSuggestedSitesFile(int n) {
+        FileOutputStream fos = null;
+
+        try {
+            File distFile = File.createTempFile("distrosites", ".json",
+                                                context.getCacheDir());
+
+            fos = new FileOutputStream(distFile);
+            fos.write(generateSites(n, DIST_PREFIX).getBytes());
+
+            return distFile;
+        } catch (IOException e) {
+            fail("Failed to create temp suggested sites file");
+        } finally {
+            if (fos != null) {
+                try {
+                    fos.close();
+                } catch (IOException e) {
+                    // Ignore.
+                }
+            }
+        }
+
+        return null;
+    }
+
     private void checkCursorCount(String content, int expectedCount) {
         checkCursorCount(content, expectedCount, DEFAULT_LIMIT);
     }
 
     private void checkCursorCount(String content, int expectedCount, int limit) {
         resources.setSuggestedSitesResource(content);
         Cursor c = new SuggestedSites(context).get(limit);
         assertEquals(expectedCount, c.getCount());
         c.close();
     }
 
     protected void setUp() {
         context = new TestContext(getApplicationContext());
         resources = (TestResources) context.getResources();
+        tempFiles = new ArrayList<File>();
     }
 
     protected void tearDown() {
         context.clearUsedPrefs();
+        for (File f : tempFiles) {
+            f.delete();
+        }
     }
 
     public void testCount() {
         // Empty array = empty cursor
         checkCursorCount(generateSites(0), 0);
 
         // 2 items = cursor with 2 rows
         checkCursorCount(generateSites(2), 2);
@@ -303,9 +401,100 @@ public class TestSuggestedSites extends 
         assertEquals(3, c.getCount());
         c.close();
 
         // Changing the locale forces the cached list to be refreshed.
         c = suggestedSites.get(DEFAULT_LIMIT, Locale.US);
         assertEquals(5, c.getCount());
         c.close();
     }
+
+    public void testDistribution() {
+        final int DIST_COUNT = 2;
+        final int DEFAULT_COUNT = 3;
+
+        File sitesFile = new File(context.getCacheDir(),
+                                  "suggestedsites-" + SystemClock.uptimeMillis() + ".json");
+        tempFiles.add(sitesFile);
+        assertFalse(sitesFile.exists());
+
+        File distFile = createDistSuggestedSitesFile(DIST_COUNT);
+        tempFiles.add(distFile);
+        assertTrue(distFile.exists());
+
+        // Init distribution with the mock file.
+        TestDistribution distribution = new TestDistribution(context);
+        distribution.setFileForLocale(Locale.getDefault(), distFile);
+        distribution.start();
+
+        // Init suggested sites with default values.
+        resources.setSuggestedSitesResource(generateSites(DEFAULT_COUNT));
+        SuggestedSites suggestedSites =
+                new SuggestedSites(context, distribution, sitesFile);
+
+        Object changeLock = new Object();
+
+        // Watch for change notifications on suggested sites.
+        ContentResolver cr = context.getContentResolver();
+        ContentObserver observer = new TestObserver(changeLock);
+        cr.registerContentObserver(BrowserContract.SuggestedSites.CONTENT_URI,
+                                   false, observer);
+
+        // The initial query will not contain the distribution sites
+        // yet. This will happen asynchronously once the distribution
+        // is installed.
+        Cursor c1 = null;
+        try {
+            c1 = suggestedSites.get(DEFAULT_LIMIT);
+            assertEquals(DEFAULT_COUNT, c1.getCount());
+        } finally {
+            if (c1 != null) {
+                c1.close();
+            }
+        }
+
+        synchronized(changeLock) {
+            try {
+                changeLock.wait(5000);
+            } catch (InterruptedException ie) {
+                fail("No change notification after fetching distribution file");
+            }
+        }
+
+        // Target file should exist after distribution is deployed.
+        assertTrue(sitesFile.exists());
+        cr.unregisterContentObserver(observer);
+
+        Cursor c2 = null;
+        try {
+            c2 = suggestedSites.get(DEFAULT_LIMIT);
+
+            // The next query should contain the distribution contents.
+            assertEquals(DIST_COUNT + DEFAULT_COUNT, c2.getCount());
+
+            // The first items should be from the distribution
+            for (int i = 0; i < DIST_COUNT; i++) {
+                c2.moveToPosition(i);
+
+                String url = c2.getString(c2.getColumnIndexOrThrow(BrowserContract.SuggestedSites.URL));
+                assertEquals(DIST_PREFIX +  "url" + i, url);
+
+                String title = c2.getString(c2.getColumnIndexOrThrow(BrowserContract.SuggestedSites.TITLE));
+                assertEquals(DIST_PREFIX +  "title" + i, title);
+            }
+
+            // The remaining items should be the default ones
+            for (int i = 0; i < c2.getCount() - DIST_COUNT; i++) {
+                c2.moveToPosition(i + DIST_COUNT);
+
+                String url = c2.getString(c2.getColumnIndexOrThrow(BrowserContract.SuggestedSites.URL));
+                assertEquals("url" + i, url);
+
+                String title = c2.getString(c2.getColumnIndexOrThrow(BrowserContract.SuggestedSites.TITLE));
+                assertEquals("title" + i, title);
+            }
+        } finally {
+            if (c2 != null) {
+                c2.close();
+            }
+        }
+    }
 }