Bug 1013024 - Part 1: catch install intent and deliver it to the distribution handler, processing the distribution file dynamically. r=mfinkle

This commit is contained in:
Richard Newman 2014-07-03 19:45:24 -07:00
parent c01a19a6cd
commit eae3673f45
9 changed files with 473 additions and 70 deletions

View File

@ -290,7 +290,9 @@
</intent-filter>
</receiver>
<receiver android:name="org.mozilla.gecko.ReferrerReceiver" android:exported="true">
<!-- Catch install referrer so we can do post-install work. -->
<receiver android:name="org.mozilla.gecko.distribution.ReferrerReceiver"
android:exported="true">
<intent-filter>
<action android:name="com.android.vending.INSTALL_REFERRER" />
</intent-filter>

View File

@ -1,62 +0,0 @@
/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko;
import org.json.JSONException;
import org.json.JSONObject;
import android.content.BroadcastReceiver;
import android.content.Context;
import android.content.Intent;
import android.util.Log;
import java.net.URLDecoder;
import java.util.HashMap;
public class ReferrerReceiver
extends BroadcastReceiver
{
private static final String LOGTAG = "GeckoReferrerReceiver";
public static final String ACTION_INSTALL_REFERRER = "com.android.vending.INSTALL_REFERRER";
public static final String UTM_SOURCE = "mozilla";
@Override
public void onReceive(Context context, Intent intent) {
if (ACTION_INSTALL_REFERRER.equals(intent.getAction())) {
String referrer = intent.getStringExtra("referrer");
if (referrer == null)
return;
HashMap<String, String> values = new HashMap<String, String>();
try {
String referrers[] = referrer.split("&");
for (String referrerValue : referrers) {
String keyValue[] = referrerValue.split("=");
values.put(URLDecoder.decode(keyValue[0]), URLDecoder.decode(keyValue[1]));
}
} catch (Exception e) {
}
String source = values.get("utm_source");
String campaign = values.get("utm_campaign");
if (source != null && UTM_SOURCE.equals(source) && campaign != null) {
try {
JSONObject data = new JSONObject();
data.put("id", "playstore");
data.put("version", campaign);
// Try to make sure the prefs are written as a group
GeckoEvent event = GeckoEvent.createBroadcastEvent("Campaign:Set", data.toString());
GeckoAppShell.sendEventToGecko(event);
} catch (JSONException e) {
Log.e(LOGTAG, "Error setting distribution", e);
}
}
}
}
}

View File

@ -5,12 +5,19 @@
package org.mozilla.gecko.distribution;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.ProtocolException;
import java.net.SocketException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
@ -19,21 +26,28 @@ import java.util.Map;
import java.util.Queue;
import java.util.Scanner;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.jar.JarEntry;
import java.util.jar.JarInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import javax.net.ssl.SSLException;
import org.apache.http.protocol.HTTP;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.mozilla.gecko.GeckoAppShell;
import org.mozilla.gecko.GeckoEvent;
import org.mozilla.gecko.GeckoSharedPrefs;
import org.mozilla.gecko.Telemetry;
import org.mozilla.gecko.mozglue.RobocopTarget;
import org.mozilla.gecko.util.ThreadUtils;
import android.app.Activity;
import android.content.Context;
import android.content.SharedPreferences;
import android.os.SystemClock;
import android.util.Log;
/**
@ -47,6 +61,56 @@ public final class Distribution {
private static final int STATE_NONE = 1;
private static final int STATE_SET = 2;
private static final String FETCH_PROTOCOL = "https";
private static final String FETCH_HOSTNAME = "distro-download.cdn.mozilla.net";
private static final String FETCH_PATH = "/android/1/";
private static final String FETCH_EXTENSION = ".jar";
private static final String EXPECTED_CONTENT_TYPE = "application/java-archive";
private static final String DISTRIBUTION_PATH = "distribution/";
/**
* Telemetry constants.
*/
private static final String HISTOGRAM_REFERRER_INVALID = "FENNEC_DISTRIBUTION_REFERRER_INVALID";
private static final String HISTOGRAM_DOWNLOAD_TIME_MS = "FENNEC_DISTRIBUTION_DOWNLOAD_TIME_MS";
private static final String HISTOGRAM_CODE_CATEGORY = "FENNEC_DISTRIBUTION_CODE_CATEGORY";
/**
* Success/failure codes. Don't exceed the maximum listed in Histograms.json.
*/
private static final int CODE_CATEGORY_STATUS_OUT_OF_RANGE = 0;
// HTTP status 'codes' run from 1 to 5.
private static final int CODE_CATEGORY_OFFLINE = 6;
private static final int CODE_CATEGORY_FETCH_EXCEPTION = 7;
// It's a post-fetch exception if we were able to download, but not
// able to extract.
private static final int CODE_CATEGORY_POST_FETCH_EXCEPTION = 8;
private static final int CODE_CATEGORY_POST_FETCH_SECURITY_EXCEPTION = 9;
// It's a malformed distribution if we could extract, but couldn't
// process the contents.
private static final int CODE_CATEGORY_MALFORMED_DISTRIBUTION = 10;
// Specific fetch errors.
private static final int CODE_CATEGORY_FETCH_SOCKET_ERROR = 11;
private static final int CODE_CATEGORY_FETCH_SSL_ERROR = 12;
private static final int CODE_CATEGORY_FETCH_NON_SUCCESS_RESPONSE = 13;
private static final int CODE_CATEGORY_FETCH_INVALID_CONTENT_TYPE = 14;
// Corresponds to the high value in Histograms.json.
private static final long MAX_DOWNLOAD_TIME_MSEC = 40000; // 40 seconds.
/**
* Used as a drop-off point for ReferrerReceiver. Checked when we process
* first-run distribution.
*/
private static volatile ReferrerDescriptor referrer;
private static Distribution instance;
private final Context context;
@ -166,6 +230,17 @@ public final class Distribution {
this(context, context.getPackageResourcePath(), null);
}
/**
* This method is called by ReferrerReceiver when we receive a post-install
* notification from Google Play.
*
* @param ref a parsed referrer value from the store-supplied intent.
*/
public static void onReceivedReferrer(ReferrerDescriptor ref) {
// Track the referrer object for distribution handling.
referrer = ref;
}
/**
* Helper to grab a file in the distribution directory.
*
@ -214,9 +289,11 @@ public final class Distribution {
} catch (IOException e) {
Log.e(LOGTAG, "Error getting distribution descriptor file.", e);
Telemetry.HistogramAdd(HISTOGRAM_CODE_CATEGORY, CODE_CATEGORY_MALFORMED_DISTRIBUTION);
return null;
} catch (JSONException e) {
Log.e(LOGTAG, "Error parsing preferences.json", e);
Telemetry.HistogramAdd(HISTOGRAM_CODE_CATEGORY, CODE_CATEGORY_MALFORMED_DISTRIBUTION);
return null;
}
}
@ -232,11 +309,13 @@ public final class Distribution {
return new JSONArray(getFileContents(bookmarks));
} catch (IOException e) {
Log.e(LOGTAG, "Error getting bookmarks", e);
Telemetry.HistogramAdd(HISTOGRAM_CODE_CATEGORY, CODE_CATEGORY_MALFORMED_DISTRIBUTION);
return null;
} catch (JSONException e) {
Log.e(LOGTAG, "Error parsing bookmarks.json", e);
Telemetry.HistogramAdd(HISTOGRAM_CODE_CATEGORY, CODE_CATEGORY_MALFORMED_DISTRIBUTION);
return null;
}
return null;
}
/**
@ -274,8 +353,9 @@ public final class Distribution {
return true;
}
// We try the APK, then the system directory.
// We try the install intent, then the APK, then the system directory.
final boolean distributionSet =
checkIntentDistribution() ||
checkAPKDistribution() ||
checkSystemDistribution();
@ -286,6 +366,149 @@ public final class Distribution {
return distributionSet;
}
/**
* If applicable, download and select the distribution specified in
* the referrer intent.
*
* @return true if a referrer-supplied distribution was selected.
*/
private boolean checkIntentDistribution() {
if (referrer == null) {
return false;
}
URI uri = getReferredDistribution(referrer);
if (uri == null) {
return false;
}
long start = SystemClock.uptimeMillis();
Log.v(LOGTAG, "Downloading referred distribution: " + uri);
try {
HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection();
connection.setRequestProperty(HTTP.USER_AGENT, GeckoAppShell.getGeckoInterface().getDefaultUAString());
connection.setRequestProperty("Accept", EXPECTED_CONTENT_TYPE);
try {
final JarInputStream distro;
try {
distro = fetchDistribution(uri, connection);
} catch (Exception e) {
Log.e(LOGTAG, "Error fetching distribution from network.", e);
recordFetchTelemetry(e);
return false;
}
long end = SystemClock.uptimeMillis();
final long duration = end - start;
Log.d(LOGTAG, "Distro fetch took " + duration + "ms; result? " + (distro != null));
Telemetry.HistogramAdd(HISTOGRAM_DOWNLOAD_TIME_MS, clamp(MAX_DOWNLOAD_TIME_MSEC, duration));
if (distro == null) {
// Nothing to do.
return false;
}
// Try to copy distribution files from the fetched stream.
try {
Log.d(LOGTAG, "Copying files from fetched zip.");
if (copyFilesFromStream(distro)) {
// We always copy to the data dir, and we only copy files from
// a 'distribution' subdirectory. Track our dist dir now that
// we know it.
this.distributionDir = new File(getDataDir(), DISTRIBUTION_PATH);
return true;
}
} catch (SecurityException e) {
Log.e(LOGTAG, "Security exception copying files. Corrupt or malicious?", e);
Telemetry.HistogramAdd(HISTOGRAM_CODE_CATEGORY, CODE_CATEGORY_POST_FETCH_SECURITY_EXCEPTION);
} catch (Exception e) {
Log.e(LOGTAG, "Error copying files from distribution.", e);
Telemetry.HistogramAdd(HISTOGRAM_CODE_CATEGORY, CODE_CATEGORY_POST_FETCH_EXCEPTION);
} finally {
distro.close();
}
} finally {
connection.disconnect();
}
} catch (IOException e) {
Log.e(LOGTAG, "Error copying distribution files from network.", e);
recordFetchTelemetry(e);
}
return false;
}
private static final int clamp(long v, long c) {
return (int) Math.min(c, v);
}
/**
* Fetch the provided URI, returning a {@link JarInputStream} if the response body
* is appropriate.
*
* @return the entity body as a stream, or null on failure.
*/
private JarInputStream fetchDistribution(URI uri, HttpURLConnection connection) throws IOException {
final int status = connection.getResponseCode();
Log.d(LOGTAG, "Distribution fetch: " + status);
// We record HTTP statuses as 2xx, 3xx, 4xx, 5xx => 2, 3, 4, 5.
final int value;
if (status > 599 || status < 100) {
Log.wtf(LOGTAG, "Unexpected HTTP status code: " + status);
value = CODE_CATEGORY_STATUS_OUT_OF_RANGE;
} else {
value = status / 100;
}
Telemetry.HistogramAdd(HISTOGRAM_CODE_CATEGORY, value);
if (status != 200) {
Log.w(LOGTAG, "Got status " + status + " fetching distribution.");
Telemetry.HistogramAdd(HISTOGRAM_CODE_CATEGORY, CODE_CATEGORY_FETCH_NON_SUCCESS_RESPONSE);
return null;
}
final String contentType = connection.getContentType();
if (contentType == null || !contentType.startsWith(EXPECTED_CONTENT_TYPE)) {
Log.w(LOGTAG, "Malformed response: invalid Content-Type.");
Telemetry.HistogramAdd(HISTOGRAM_CODE_CATEGORY, CODE_CATEGORY_FETCH_INVALID_CONTENT_TYPE);
return null;
}
return new JarInputStream(new BufferedInputStream(connection.getInputStream()), true);
}
private static void recordFetchTelemetry(final Exception exception) {
if (exception == null) {
// Should never happen.
Telemetry.HistogramAdd(HISTOGRAM_CODE_CATEGORY, CODE_CATEGORY_FETCH_EXCEPTION);
return;
}
if (exception instanceof UnknownHostException) {
// Unknown host => we're offline.
Telemetry.HistogramAdd(HISTOGRAM_CODE_CATEGORY, CODE_CATEGORY_OFFLINE);
return;
}
if (exception instanceof SSLException) {
Telemetry.HistogramAdd(HISTOGRAM_CODE_CATEGORY, CODE_CATEGORY_FETCH_SSL_ERROR);
return;
}
if (exception instanceof ProtocolException ||
exception instanceof SocketException) {
Telemetry.HistogramAdd(HISTOGRAM_CODE_CATEGORY, CODE_CATEGORY_FETCH_SOCKET_ERROR);
return;
}
Telemetry.HistogramAdd(HISTOGRAM_CODE_CATEGORY, CODE_CATEGORY_FETCH_EXCEPTION);
}
/**
* Execute tasks that wanted to run when we were done loading
* the distribution. These tasks are expected to call {@link #exists()}
@ -308,7 +531,7 @@ public final class Distribution {
// We always copy to the data dir, and we only copy files from
// a 'distribution' subdirectory. Track our dist dir now that
// we know it.
this.distributionDir = new File(getDataDir(), "distribution/");
this.distributionDir = new File(getDataDir(), DISTRIBUTION_PATH);
return true;
}
} catch (IOException e) {
@ -330,6 +553,41 @@ public final class Distribution {
return false;
}
/**
* Unpack distribution files from a downloaded jar stream.
*
* The caller is responsible for closing the provided stream.
*/
private boolean copyFilesFromStream(JarInputStream jar) throws FileNotFoundException, IOException {
final byte[] buffer = new byte[1024];
boolean distributionSet = false;
JarEntry entry;
while ((entry = jar.getNextJarEntry()) != null) {
final String name = entry.getName();
if (entry.isDirectory()) {
// We'll let getDataFile deal with creating the directory hierarchy.
// Yes, we can do better, but it can wait.
continue;
}
if (!name.startsWith(DISTRIBUTION_PATH)) {
continue;
}
File outFile = getDataFile(name);
if (outFile == null) {
continue;
}
distributionSet = true;
writeStream(jar, outFile, entry.getTime(), buffer);
}
return distributionSet;
}
/**
* Copies the /distribution folder out of the APK and into the app's data directory.
* Returns true if distribution files were found and copied.
@ -352,7 +610,7 @@ public final class Distribution {
continue;
}
if (!name.startsWith("distribution/")) {
if (!name.startsWith(DISTRIBUTION_PATH)) {
continue;
}
@ -413,6 +671,29 @@ public final class Distribution {
return outFile;
}
private URI getReferredDistribution(ReferrerDescriptor descriptor) {
final String content = descriptor.content;
if (content == null) {
return null;
}
// We restrict here to avoid injection attacks. After all,
// we're downloading a distribution payload based on intent input.
if (!content.matches("^[a-zA-Z0-9]+$")) {
Log.e(LOGTAG, "Invalid referrer content: " + content);
Telemetry.HistogramAdd(HISTOGRAM_REFERRER_INVALID, 1);
return null;
}
try {
return new URI(FETCH_PROTOCOL, FETCH_HOSTNAME, FETCH_PATH + content + FETCH_EXTENSION, null);
} catch (URISyntaxException e) {
// This should never occur.
Log.wtf(LOGTAG, "Invalid URI with content " + content + "!");
return null;
}
}
/**
* After calling this method, either <code>distributionDir</code>
* will be set, or there is no distribution in use.
@ -432,7 +713,7 @@ public final class Distribution {
// the APK, or it exists in /system/.
// Look in each location in turn.
// (This could be optimized by caching the path in shared prefs.)
File copied = new File(getDataDir(), "distribution/");
File copied = new File(getDataDir(), DISTRIBUTION_PATH);
if (copied.exists()) {
return this.distributionDir = copied;
}

View File

@ -0,0 +1,47 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.distribution;
import android.net.Uri;
/**
* Encapsulates access to values encoded in the "referrer" extra of an install intent.
*
* This object is immutable.
*
* Example input:
*
* "utm_source=campsource&utm_medium=campmed&utm_term=term%2Bhere&utm_content=content&utm_campaign=name"
*/
public class ReferrerDescriptor {
public final String source;
public final String medium;
public final String term;
public final String content;
public final String campaign;
public ReferrerDescriptor(final String referrer) {
if (referrer == null) {
source = null;
medium = null;
term = null;
content = null;
campaign = null;
return;
}
final Uri u = new Uri.Builder()
.scheme("http")
.authority("local")
.path("/")
.encodedQuery(referrer).build();
source = u.getQueryParameter("utm_source");
medium = u.getQueryParameter("utm_medium");
term = u.getQueryParameter("utm_term");
content = u.getQueryParameter("utm_content");
campaign = u.getQueryParameter("utm_campaign");
}
}

View File

@ -0,0 +1,75 @@
/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.distribution;
import org.json.JSONException;
import org.json.JSONObject;
import org.mozilla.gecko.GeckoAppShell;
import org.mozilla.gecko.GeckoEvent;
import android.content.BroadcastReceiver;
import android.content.Context;
import android.content.Intent;
import android.text.TextUtils;
import android.util.Log;
public class ReferrerReceiver extends BroadcastReceiver {
private static final String LOGTAG = "GeckoReferrerReceiver";
private static final String ACTION_INSTALL_REFERRER = "com.android.vending.INSTALL_REFERRER";
/**
* If the install intent has this source, we'll track the campaign ID.
*/
private static final String MOZILLA_UTM_SOURCE = "mozilla";
/**
* If the install intent has this campaign, we'll load the specified distribution.
*/
private static final String DISTRIBUTION_UTM_CAMPAIGN = "distribution";
@Override
public void onReceive(Context context, Intent intent) {
if (!ACTION_INSTALL_REFERRER.equals(intent.getAction())) {
// This should never happen.
return;
}
ReferrerDescriptor referrer = new ReferrerDescriptor(intent.getStringExtra("referrer"));
// Track the referrer object for distribution handling.
if (TextUtils.equals(referrer.campaign, DISTRIBUTION_UTM_CAMPAIGN)) {
Distribution.onReceivedReferrer(referrer);
} else {
Log.d(LOGTAG, "Not downloading distribution: non-matching campaign.");
}
// If this is a Mozilla campaign, pass the campaign along to Gecko.
if (TextUtils.equals(referrer.source, MOZILLA_UTM_SOURCE)) {
propagateMozillaCampaign(referrer);
}
}
private void propagateMozillaCampaign(ReferrerDescriptor referrer) {
if (referrer.campaign == null) {
return;
}
try {
final JSONObject data = new JSONObject();
data.put("id", "playstore");
data.put("version", referrer.campaign);
String payload = data.toString();
// Try to make sure the prefs are written as a group.
final GeckoEvent event = GeckoEvent.createBroadcastEvent("Campaign:Set", payload);
GeckoAppShell.sendEventToGecko(event);
} catch (JSONException e) {
Log.e(LOGTAG, "Error propagating campaign identifier.", e);
}
}
}

View File

@ -156,6 +156,8 @@ gbjar.sources += [
'db/TabsProvider.java',
'db/TopSitesCursorWrapper.java',
'distribution/Distribution.java',
'distribution/ReferrerDescriptor.java',
'distribution/ReferrerReceiver.java',
'DoorHangerPopup.java',
'DynamicToolbar.java',
'EditBookmarkDialog.java',
@ -354,7 +356,6 @@ gbjar.sources += [
'prompts/PromptService.java',
'prompts/TabInput.java',
'ReaderModeUtils.java',
'ReferrerReceiver.java',
'Restarter.java',
'ScrollAnimator.java',
'ServiceNotificationClient.java',

View File

@ -11,6 +11,7 @@ jar.sources += [
'src/harness/BrowserInstrumentationTestRunner.java',
'src/harness/BrowserTestListener.java',
'src/tests/BrowserTestCase.java',
'src/tests/TestDistribution.java',
'src/tests/TestGeckoSharedPrefs.java',
'src/tests/TestJarReader.java',
'src/tests/TestRawResource.java',

View File

@ -0,0 +1,36 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.browser.tests;
import org.mozilla.gecko.distribution.ReferrerDescriptor;
public class TestDistribution extends BrowserTestCase {
private static final String TEST_REFERRER_STRING = "utm_source=campsource&utm_medium=campmed&utm_term=term%2Bhere&utm_content=content&utm_campaign=name";
private static final String TEST_MALFORMED_REFERRER_STRING = "utm_source=campsource&utm_medium=campmed&utm_term=term%2";
public void testReferrerParsing() {
ReferrerDescriptor good = new ReferrerDescriptor(TEST_REFERRER_STRING);
assertEquals("campsource", good.source);
assertEquals("campmed", good.medium);
assertEquals("term+here", good.term);
assertEquals("content", good.content);
assertEquals("name", good.campaign);
// Uri.Builder is permissive.
ReferrerDescriptor bad = new ReferrerDescriptor(TEST_MALFORMED_REFERRER_STRING);
assertEquals("campsource", bad.source);
assertEquals("campmed", bad.medium);
assertFalse("term+here".equals(bad.term));
assertNull(bad.content);
assertNull(bad.campaign);
ReferrerDescriptor ugly = new ReferrerDescriptor(null);
assertNull(ugly.source);
assertNull(ugly.medium);
assertNull(ugly.term);
assertNull(ugly.content);
assertNull(ugly.campaign);
}
}

View File

@ -2959,6 +2959,28 @@
"extended_statistics_ok": true,
"description": "PLACES: Time to calculate the md5 hash for a backup"
},
"FENNEC_DISTRIBUTION_REFERRER_INVALID": {
"expires_in_version": "never",
"kind": "flag",
"description": "Whether the referrer intent specified an invalid distribution name",
"cpp_guard": "ANDROID"
},
"FENNEC_DISTRIBUTION_CODE_CATEGORY": {
"expires_in_version": "never",
"kind": "enumerated",
"n_values": 20,
"description": "First digit of HTTP result code, or error category, during distribution download",
"cpp_guard": "ANDROID"
},
"FENNEC_DISTRIBUTION_DOWNLOAD_TIME_MS": {
"expires_in_version": "never",
"kind": "exponential",
"low": 100,
"high": "40000",
"n_buckets": 30,
"description": "Time taken to download a specified distribution file (msec)",
"cpp_guard": "ANDROID"
},
"FENNEC_FAVICONS_COUNT": {
"expires_in_version": "never",
"kind": "exponential",