/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at
http://mozilla.org/MPL/2.0/. */
/*
* This module is responsible for uploading pings to the server and persisting
* pings that can't be send now.
* Those pending pings are persisted on disk and sent at the next opportunity,
* newest first.
*/
import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs";
import { ClientID } from "resource://gre/modules/ClientID.sys.mjs";
import { Log } from "resource://gre/modules/Log.sys.mjs";
import { ServiceRequest } from "resource://gre/modules/ServiceRequest.sys.mjs";
import { TelemetryUtils } from "resource://gre/modules/TelemetryUtils.sys.mjs";
import { clearTimeout, setTimeout } from "resource://gre/modules/Timer.sys.mjs";
const lazy = {};
ChromeUtils.defineESModuleGetters(lazy, {
NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs",
TelemetryHealthPing: "resource://gre/modules/HealthPing.sys.mjs",
TelemetryReportingPolicy:
"resource://gre/modules/TelemetryReportingPolicy.sys.mjs",
TelemetryStorage: "resource://gre/modules/TelemetryStorage.sys.mjs",
});
const Utils = TelemetryUtils;
const PREF_TELEMETRY_ENABLED = "toolkit.telemetry.enabled";
const LOGGER_NAME = "Toolkit.Telemetry";
const LOGGER_PREFIX = "TelemetrySend::";
const TOPIC_IDLE_DAILY = "idle-daily";
// The following topics are notified when Firefox is closing
// because the OS is shutting down.
const TOPIC_QUIT_APPLICATION_GRANTED = "quit-application-granted";
const TOPIC_QUIT_APPLICATION_FORCED = "quit-application-forced";
const PREF_CHANGED_TOPIC = "nsPref:changed";
const TOPIC_PROFILE_CHANGE_NET_TEARDOWN = "profile-change-net-teardown";
// Whether the FHR/Telemetry unification features are enabled.
// Changing this pref requires a restart.
const IS_UNIFIED_TELEMETRY = Services.prefs.getBoolPref(
TelemetryUtils.Preferences.Unified,
false
);
const MS_IN_A_MINUTE = 60 * 1000;
const PING_TYPE_DELETION_REQUEST = "deletion-request";
// We try to spread "midnight" pings out over this interval.
const MIDNIGHT_FUZZING_INTERVAL_MS = 60 * MS_IN_A_MINUTE;
// We delay sending "midnight" pings on this client by this interval.
const MIDNIGHT_FUZZING_DELAY_MS = Math.random() * MIDNIGHT_FUZZING_INTERVAL_MS;
// Timeout after which we consider a ping submission failed.
export const PING_SUBMIT_TIMEOUT_MS = 1.5 * MS_IN_A_MINUTE;
// To keep resource usage in check, we limit ping sending to a maximum number
// of pings per minute.
const MAX_PING_SENDS_PER_MINUTE = 10;
// If we have more pending pings then we can send right now, we schedule the next
// send for after SEND_TICK_DELAY.
const SEND_TICK_DELAY = 1 * MS_IN_A_MINUTE;
// If we had any ping send failures since the last ping, we use a backoff timeout
// for the next ping sends. We increase the delay exponentially up to a limit of
// SEND_MAXIMUM_BACKOFF_DELAY_MS.
// This exponential backoff will be reset by external ping submissions & idle-daily.
const SEND_MAXIMUM_BACKOFF_DELAY_MS = 120 * MS_IN_A_MINUTE;
// Strings to map from XHR.errorCode to TELEMETRY_SEND_FAILURE_TYPE.
// Echoes XMLHttpRequestMainThread's ErrorType enum.
// Make sure that any additions done to XHR_ERROR_TYPE enum are also mirrored in
// TELEMETRY_SEND_FAILURE_TYPE and TELEMETRY_SEND_FAILURE_TYPE_PER_PING's labels.
const XHR_ERROR_TYPE = [
"eOK",
"eRequest",
"eUnreachable",
"eChannelOpen",
"eRedirect",
"eTerminated",
];
/**
* This is a policy object used to override behavior within this module.
* Tests override properties on this object to allow for control of behavior
* that would otherwise be very hard to cover.
*/
export var Policy = {
now: () => new Date(),
midnightPingFuzzingDelay: () => MIDNIGHT_FUZZING_DELAY_MS,
pingSubmissionTimeout: () => PING_SUBMIT_TIMEOUT_MS,
setSchedulerTickTimeout: (callback, delayMs) => setTimeout(callback, delayMs),
clearSchedulerTickTimeout: id => clearTimeout(id),
gzipCompressString: data => gzipCompressString(data),
};
/**
* Determine if the ping has the new v4 ping format or the legacy v2 one or earlier.
*/
function isV4PingFormat(aPing) {
return (
"id" in aPing &&
"application" in aPing &&
"version" in aPing &&
aPing.version >= 2
);
}
/**
* Check if the provided ping is a deletion-request ping.
* @param {Object} aPing The ping to check.
* @return {Boolean} True if the ping is a deletion-request ping, false otherwise.
*/
function isDeletionRequestPing(aPing) {
return isV4PingFormat(aPing) && aPing.type == PING_TYPE_DELETION_REQUEST;
}
/**
* Generate a string suitable for including in a profiler marker as a ping description.
* @param {Object} aPing The ping to describe.
*/
function getPingMarkerString(aPing) {
let markerString = aPing.type;
let reason = aPing.payload?.info?.reason || aPing.payload?.reason;
if (reason) {
markerString += ", reason: " + reason;
}
return markerString;
}
/**
* Save the provided ping as a pending ping.
* @param {Object} aPing The ping to save.
* @return {Promise} A promise resolved when the ping is saved.
*/
function savePing(aPing) {
let startTime = Cu.now();
let promise = lazy.TelemetryStorage.savePendingPing(aPing);
promise.then(() => {
ChromeUtils.addProfilerMarker(
"Ping Save",
{ category: "Telemetry", startTime },
getPingMarkerString(aPing)
);
});
return promise;
}
function arrayToString(array) {
let buffer = "";
// String.fromCharCode can only deal with 500,000 characters at
// a time, so chunk the result into parts of that size.
const chunkSize = 500000;
for (let offset = 0; offset < array.length; offset += chunkSize) {
buffer += String.fromCharCode.apply(
String,
array.slice(offset, offset + chunkSize)
);
}
return buffer;
}
/**
* @return {String} This returns a string with the gzip compressed data.
*/
export function gzipCompressString(string) {
let observer = {
buffer: null,
onStreamComplete(loader, context, status, length, result) {
this.buffer = arrayToString(result);
},
};
let scs = Cc["@mozilla.org/streamConverters;1"].getService(
Ci.nsIStreamConverterService
);
let listener = Cc["@mozilla.org/network/stream-loader;1"].createInstance(
Ci.nsIStreamLoader
);
listener.init(observer);
let converter = scs.asyncConvertData("uncompressed", "gzip", listener, null);
let stringStream = Cc["@mozilla.org/io/string-input-stream;1"].createInstance(
Ci.nsIStringInputStream
);
stringStream.setByteStringData(string);
converter.onStartRequest(null, null);
converter.onDataAvailable(null, stringStream, 0, string.length);
converter.onStopRequest(null, null, null);
return observer.buffer;
}
const STANDALONE_PING_TIMEOUT = 30 * 1000; // 30 seconds
export function sendStandalonePing(endpoint, payload, extraHeaders = {}) {
return new Promise((resolve, reject) => {
let request = new ServiceRequest({ mozAnon: true });
request.mozBackgroundRequest = true;
request.timeout = STANDALONE_PING_TIMEOUT;
request.open("POST", endpoint, true);
request.overrideMimeType("text/plain");
request.setRequestHeader("Content-Type", "application/json; charset=UTF-8");
request.setRequestHeader("Content-Encoding", "gzip");
request.setRequestHeader("Date", new Date().toUTCString());
for (let header in extraHeaders) {
request.setRequestHeader(header, extraHeaders[header]);
}
request.onload = event => {
if (request.status !== 200) {
reject(event);
} else {
resolve(event);
}
};
request.onerror = reject;
request.onabort = reject;
request.ontimeout = reject;
let payloadStream = Cc[
"@mozilla.org/io/string-input-stream;1"
].createInstance(Ci.nsIStringInputStream);
const utf8Payload = new TextEncoder().encode(payload);
payloadStream.setByteStringData(
gzipCompressString(arrayToString(utf8Payload))
);
request.sendInputStream(payloadStream);
});
}
export var TelemetrySend = {
get pendingPingCount() {
return TelemetrySendImpl.pendingPingCount;
},
/**
* Partial setup that runs immediately at startup. This currently triggers
* the crash report annotations.
*/
earlyInit() {
TelemetrySendImpl.earlyInit();
},
/**
* Initializes this module.
*
* @param {Boolean} testing Whether this is run in a test. This changes some behavior
* to enable proper testing.
* @return {Promise} Resolved when setup is finished.
*/
setup(testing = false) {
return TelemetrySendImpl.setup(testing);
},
/**
* Shutdown this module - this will cancel any pending ping tasks and wait for
* outstanding async activity like network and disk I/O.
*
* @return {Promise} Promise that is resolved when shutdown is finished.
*/
shutdown() {
return TelemetrySendImpl.shutdown();
},
/**
* Flushes all pings to pingsender that were both
* 1. submitted after profile-change-net-teardown, and
* 2. wanting to be sent using pingsender.
*/
flushPingSenderBatch() {
TelemetrySendImpl.flushPingSenderBatch();
},
/**
* Submit a ping for sending. This will:
* - send the ping right away if possible or
* - save the ping to disk and send it at the next opportunity
*
* @param {Object} ping The ping data to send, must be serializable to JSON.
* @param {Object} [aOptions] Options object.
* @param {Boolean} [options.usePingSender=false] if true, send the ping using the PingSende
r.
* @return {Promise} Test-only - a promise that is resolved when the ping is sent or saved.
*/
submitPing(ping, options = {}) {
options.usePingSender = options.usePingSender || false;
return TelemetrySendImpl.submitPing(ping, options);
},
/**
* Check if sending is disabled. If Telemetry is not allowed to upload,
* pings are not sent to the server.
* If trying to send a deletion-request ping, don't block it.
*
* @param {Object} [ping=null] A ping to be checked.
* @return {Boolean} True if pings can be send to the servers, false otherwise.
*/
sendingEnabled(ping = null) {
return TelemetrySendImpl.sendingEnabled(ping);
},
/**
* Notify that we can start submitting data to the servers.
*/
notifyCanUpload() {
return TelemetrySendImpl.notifyCanUpload();
},
/**
* Only used in tests. Used to reset the module data to emulate a restart.
*/
reset() {
return TelemetrySendImpl.reset();
},
/**
* Only used in tests.
*/
setServer(server) {
return TelemetrySendImpl.setServer(server);
},
/**
* Clear out unpersisted, yet to be sent, pings and cancel outgoing ping requests.
*/
clearCurrentPings() {
return TelemetrySendImpl.clearCurrentPings();
},
/**
* Only used in tests to wait on outgoing pending pings.
*/
testWaitOnOutgoingPings() {
return TelemetrySendImpl.promisePendingPingActivity();
},
/**
* Only used in tests to set whether it is too late in shutdown to send pings.
*/
testTooLateToSend(tooLate) {
TelemetrySendImpl._tooLateToSend = tooLate;
},
/**
* Test-only - this allows overriding behavior to enable ping sending in debug builds.
*/
setTestModeEnabled(testing) {
TelemetrySendImpl.setTestModeEnabled(testing);
},
/**
* This returns state info for this module for AsyncShutdown timeout diagnostics.
*/
getShutdownState() {
return TelemetrySendImpl.getShutdownState();
},
/**
* Send a ping using the ping sender.
* This method will not wait for the ping to be sent, instead it will return
* as soon as the pingsender program has been launched.
*
* This method is currently exposed here only for testing purposes as it's
* only used internally.
*
* @param {Array}<Object> pings An array of objects holding url / path pairs
* for each ping to be sent. The URL represent the telemetry server the
* ping will be sent to and the path points to the ping data. The ping
* data files will be deleted if the pings have been submitted
* successfully.
* @param {callback} observer A function called with parameters
* (subject, topic, data) and a topic of "process-finished" or
* "process-failed" after pingsender completion.
*
* @throws NS_ERROR_FAILURE if we couldn't find or run the pingsender
* executable.
* @throws NS_ERROR_NOT_IMPLEMENTED on Android as the pingsender is not
* available.
*/
testRunPingSender(pings, observer) {
return TelemetrySendImpl.runPingSender(pings, observer);
},
};
var CancellableTimeout = {
_deferred: null,
_timer: null,
/**
* This waits until either the given timeout passed or the timeout was cancelled.
*
* @param {Number} timeoutMs The timeout in ms.
* @return {Promise<bool>} Promise that is resolved with false if the timeout was cancelled,
* false otherwise.
*/
promiseWaitOnTimeout(timeoutMs) {
if (!this._deferred) {
this._deferred = Promise.withResolvers();
this._timer = Policy.setSchedulerTickTimeout(
() => this._onTimeout(),
timeoutMs
);
}
return this._deferred.promise;
},
_onTimeout() {
if (this._deferred) {
this._deferred.resolve(false);
this._timer = null;
this._deferred = null;
}
},
cancelTimeout() {
if (this._deferred) {
Policy.clearSchedulerTickTimeout(this._timer);
this._deferred.resolve(true);
this._timer = null;
this._deferred = null;
}
},
};
/**
* SendScheduler implements the timer & scheduling behavior for ping sends.
*/
export var SendScheduler = {
// Whether any ping sends failed since the last tick. If yes, we start with our exponential
// backoff timeout.
_sendsFailed: false,
// The current retry delay after ping send failures. We use this for the exponential backoff,
// increasing this value everytime we had send failures since the last tick.
_backoffDelay: SEND_TICK_DELAY,
_shutdown: false,
_sendTask: null,
// A string that tracks the last seen send task state, null if it never ran.
_sendTaskState: null,
_logger: null,
get _log() {
if (!this._logger) {
this._logger = Log.repository.getLoggerWithMessagePrefix(
LOGGER_NAME,
LOGGER_PREFIX + "Scheduler::"
);
}
return this._logger;
},
shutdown() {
this._log.trace("shutdown");
this._shutdown = true;
CancellableTimeout.cancelTimeout();
return Promise.resolve(this._sendTask);
},
start() {
this._log.trace("start");
this._sendsFailed = false;
this._backoffDelay = SEND_TICK_DELAY;
this._shutdown = false;
},
/**
* Only used for testing, resets the state to emulate a restart.
*/
reset() {
this._log.trace("reset");
return this.shutdown().then(() => this.start());
},
/**
* Notify the scheduler of a failure in sending out pings that warrants retrying.
* This will trigger the exponential backoff timer behavior on the next tick.
*/
notifySendsFailed() {
this._log.trace("notifySendsFailed");
if (this._sendsFailed) {
return;
}
this._sendsFailed = true;
this._log.trace("notifySendsFailed - had send failures");
},
/**
* Returns whether ping submissions are currently throttled.
*/
isThrottled() {
const now = Policy.now();
const nextPingSendTime = this._getNextPingSendTime(now);
return nextPingSendTime > now.getTime();
},
waitOnSendTask() {
return Promise.resolve(this._sendTask);
},
triggerSendingPings(immediately) {
this._log.trace(
"triggerSendingPings - active send task: " +
!!this._sendTask +
", immediately: " +
immediately
);
if (!this._sendTask) {
this._sendTask = this._doSendTask();
let clear = () => (this._sendTask = null);
this._sendTask.then(clear, clear);
} else if (immediately) {
CancellableTimeout.cancelTimeout();
}
return this._sendTask;
},
async _doSendTask() {
this._sendTaskState = "send task started";
this._backoffDelay = SEND_TICK_DELAY;
this._sendsFailed = false;
const resetBackoffTimer = () => {
this._backoffDelay = SEND_TICK_DELAY;
};
for (;;) {
this._log.trace("_doSendTask iteration");
this._sendTaskState = "start iteration";
if (this._shutdown) {
this._log.trace("_doSendTask - shutting down, bailing out");
this._sendTaskState = "bail out - shutdown check";
return;
}
// Get a list of pending pings, sorted by last modified, descending.
// Filter out all the pings we can't send now. This addresses scenarios like "deletion-request" pings
// which can be sent even when upload is disabled.
let pending = lazy.TelemetryStorage.getPendingPingList();
let current = TelemetrySendImpl.getUnpersistedPings();
this._log.trace(
"_doSendTask - pending: " +
pending.length +
", current: " +
current.length
);
// Note that the two lists contain different kind of data. |pending| only holds ping
// info, while |current| holds actual ping data.
if (!TelemetrySendImpl.sendingEnabled()) {
// If sending is disabled, only handle deletion-request pings
pending = [];
current = current.filter(p => isDeletionRequestPing(p));
}
this._log.trace(
"_doSendTask - can send - pending: " +
pending.length +
", current: " +
current.length
);
// Bail out if there is nothing to send.
if (!pending.length && !current.length) {
this._log.trace("_doSendTask - no pending pings, bailing out");
this._sendTaskState = "bail out - no pings to send";
return;
}
// If we are currently throttled (e.g. fuzzing to avoid midnight spikes), wait for the next send window.
const now = Policy.now();
if (this.isThrottled()) {
const nextPingSendTime = this._getNextPingSendTime(now);
this._log.trace(
"_doSendTask - throttled, delaying ping send to " +
new Date(nextPingSendTime)
);
this._sendTaskState = "wait for throttling to pass";
const delay = nextPingSendTime - now.getTime();
const cancelled = await CancellableTimeout.promiseWaitOnTimeout(delay);
if (cancelled) {
this._log.trace(
"_doSendTask - throttling wait was cancelled, resetting backoff timer"
);
resetBackoffTimer();
}
continue;
}
let sending = pending.slice(0, MAX_PING_SENDS_PER_MINUTE);
pending = pending.slice(MAX_PING_SENDS_PER_MINUTE);
this._log.trace(
"_doSendTask - triggering sending of " +
sending.length +
" pings now" +
", " +
pending.length +
" pings waiting"
);
this._sendsFailed = false;
const sendStartTime = Policy.now();
this._sendTaskState = "wait on ping sends";
await TelemetrySendImpl.sendPings(
current,
sending.map(p => p.id)
);
if (this._shutdown || TelemetrySend.pendingPingCount == 0) {
this._log.trace(
"_doSendTask - bailing out after sending, shutdown: " +
this._shutdown +
", pendingPingCount: " +
TelemetrySend.pendingPingCount
);
this._sendTaskState = "bail out - shutdown & pending check after send";
return;
}
// Calculate the delay before sending the next batch of pings.
// We start with a delay that makes us send max. 1 batch per minute.
// If we had send failures in the last batch, we will override this with
// a backoff delay.
const timeSinceLastSend = Policy.now() - sendStartTime;
let nextSendDelay = Math.max(0, SEND_TICK_DELAY - timeSinceLastSend);
if (!this._sendsFailed) {
this._log.trace(
"_doSendTask - had no send failures, resetting backoff timer"
);
resetBackoffTimer();
} else {
const newDelay = Math.min(
SEND_MAXIMUM_BACKOFF_DELAY_MS,
this._backoffDelay * 2
);
this._log.trace(
"_doSendTask - had send failures, backing off -" +
" old timeout: " +
this._backoffDelay +
", new timeout: " +
newDelay
);
this._backoffDelay = newDelay;
nextSendDelay = this._backoffDelay;
}
this._log.trace(
"_doSendTask - waiting for next send opportunity, timeout is " +
nextSendDelay
);
this._sendTaskState = "wait on next send opportunity";
const cancelled =
await CancellableTimeout.promiseWaitOnTimeout(nextSendDelay);
if (cancelled) {
this._log.trace(
"_doSendTask - batch send wait was cancelled, resetting backoff timer"
);
resetBackoffTimer();
}
}
},
/**
* This helper calculates the next time that we can send pings at.
* Currently this mostly redistributes ping sends from midnight until one hour after
* to avoid submission spikes around local midnight for daily pings.
*
* @param now Date The current time.
* @return Number The next time (ms from UNIX epoch) when we can send pings.
*/
_getNextPingSendTime(now) {
// 1. First we check if the pref is set to skip any delay and send immediately.
// 2. Next we check if the time is between 0am and 1am. If it's not, we send
// immediately.
// 3. If we confirmed the time is indeed between 0am and 1am in step 1, we disallow
// sending before (midnight + fuzzing delay), which is a random time between 0am-1am
// (decided at startup).
let disableFuzzingDelay = Services.prefs.getBoolPref(
TelemetryUtils.Preferences.DisableFuzzingDelay,
false
);
if (disableFuzzingDelay) {
return now.getTime();
}
const midnight = Utils.truncateToDays(now);
// Don't delay pings if we are not within the fuzzing interval.
if (now.getTime() - midnight.getTime() > MIDNIGHT_FUZZING_INTERVAL_MS) {
return now.getTime();
}
// Delay ping send if we are within the midnight fuzzing range.
// We spread those ping sends out between |midnight| and |midnight + midnightPingFuzzingDelay|.
return midnight.getTime() + Policy.midnightPingFuzzingDelay();
},
getShutdownState() {
return {
shutdown: this._shutdown,
hasSendTask: !!this._sendTask,
sendsFailed: this._sendsFailed,
sendTaskState: this._sendTaskState,
backoffDelay: this._backoffDelay,
};
},
};
export var TelemetrySendImpl = {
_sendingEnabled: false,
// Tracks the shutdown state.
_shutdown: false,
_logger: null,
// This tracks all pending ping requests to the server.
_pendingPingRequests: new Map(),
// This tracks all the pending async ping activity.
_pendingPingActivity: new Set(),
// This is true when running in the test infrastructure.
_testMode: false,
// This holds pings that we currently try and haven't persisted yet.
_currentPings: new Map(),
// Used to skip spawning the pingsender if OS is shutting down.
_isOSShutdown: false,
// Has the network shut down, making it too late to send pings?
_tooLateToSend: false,
// Array of {url, path} awaiting flushPingSenderBatch().
_pingSenderBatch: [],
OBSERVER_TOPICS: [
TOPIC_IDLE_DAILY,
TOPIC_QUIT_APPLICATION_GRANTED,
TOPIC_QUIT_APPLICATION_FORCED,
TOPIC_PROFILE_CHANGE_NET_TEARDOWN,
],
OBSERVED_PREFERENCES: [
TelemetryUtils.Preferences.TelemetryEnabled,
TelemetryUtils.Preferences.FhrUploadEnabled,
],
// Whether sending pings has been overridden.
get _overrideOfficialCheck() {
return Services.prefs.getBoolPref(
TelemetryUtils.Preferences.OverrideOfficialCheck,
false
);
},
get _log() {
if (!this._logger) {
this._logger = Log.repository.getLoggerWithMessagePrefix(
LOGGER_NAME,
LOGGER_PREFIX
);
}
return this._logger;
},
get pendingPingRequests() {
return this._pendingPingRequests;
},
get pendingPingCount() {
return (
lazy.TelemetryStorage.getPendingPingList().length +
this._currentPings.size
);
},
setTestModeEnabled(testing) {
this._testMode = testing;
},
earlyInit() {
this._annotateCrashReport();
// Install the observer to detect OS shutdown early enough, so
// that we catch this before the delayed setup happens.
Services.obs.addObserver(this, TOPIC_QUIT_APPLICATION_FORCED);
Services.obs.addObserver(this, TOPIC_QUIT_APPLICATION_GRANTED);
},
QueryInterface: ChromeUtils.generateQI(["nsISupportsWeakReference"]),
async setup(testing) {
this._log.trace("setup");
this._testMode = testing;
Services.obs.addObserver(this, TOPIC_IDLE_DAILY);
Services.obs.addObserver(this, TOPIC_PROFILE_CHANGE_NET_TEARDOWN);
this._server = Services.prefs.getStringPref(
TelemetryUtils.Preferences.Server,
undefined
);
this._sendingEnabled = true;
// Annotate crash reports so that crash pings are sent correctly and listen
// to pref changes to adjust the annotations accordingly.
for (let pref of this.OBSERVED_PREFERENCES) {
Services.prefs.addObserver(pref, this, true);
}
this._annotateCrashReport();
// Check the pending pings on disk now.
try {
await this._checkPendingPings();
} catch (ex) {
this._log.error("setup - _checkPendingPings rejected", ex);
}
// Enforce the pending pings storage quota. It could take a while so don't
// block on it.
lazy.TelemetryStorage.runEnforcePendingPingsQuotaTask();
// Start sending pings, but don't block on this.
SendScheduler.triggerSendingPings(true);
},
/**
* Triggers the crash report annotations depending on the current
* configuration. This communicates to the crash reporter if it can send a
* crash ping or not. This method can be called safely before setup() has
* been called.
*/
_annotateCrashReport() {
try {
const cr = Cc["@mozilla.org/toolkit/crash-reporter;1"];
if (cr) {
// This needs to use nsICrashReporter because test_TelemetrySend.js
// replaces the crash reporter service, which we can't access here
// as Services caches it.
// eslint-disable-next-line mozilla/use-services
const crs = cr.getService(Ci.nsICrashReporter);
let clientId = ClientID.getCachedClientID();
let profileGroupId = ClientID.getCachedProfileGroupID();
let server =
this._server ||
Services.prefs.getStringPref(
TelemetryUtils.Preferences.Server,
undefined
);
if (
!this.sendingEnabled() ||
!lazy.TelemetryReportingPolicy.canUpload()
) {
// If we cannot send pings then clear the crash annotations
crs.removeCrashReportAnnotation("TelemetryClientId");
crs.removeCrashReportAnnotation("TelemetryProfileGroupId");
crs.removeCrashReportAnnotation("TelemetryServerURL");
} else {
crs.annotateCrashReport("TelemetryClientId", clientId);
crs.annotateCrashReport("TelemetryProfileGroupId", profileGroupId);
crs.annotateCrashReport("TelemetryServerURL", server);
}
}
} catch (e) {
// Ignore errors when crash reporting is disabled
}
},
/**
* Discard old pings from the pending pings and detect overdue ones.
* @return {Boolean} True if we have overdue pings, false otherwise.
*/
async _checkPendingPings() {
// Scan the pending pings - that gives us a list sorted by last modified, descending.
let infos = await lazy.TelemetryStorage.loadPendingPingList();
this._log.info("_checkPendingPings - pending ping count: " + infos.length);
if (!infos.length) {
this._log.trace("_checkPendingPings - no pending pings");
return;
}
const now = Policy.now();
// Submit the age of the pending pings.
for (let pingInfo of infos) {
const ageInDays = Utils.millisecondsToDays(
Math.abs(now.getTime() - pingInfo.lastModificationDate)
);
Services.telemetry
.getHistogramById("TELEMETRY_PENDING_PINGS_AGE")
.add(ageInDays);
}
},
async shutdown() {
this._shutdown = true;
for (let pref of this.OBSERVED_PREFERENCES) {
// FIXME: When running tests this causes errors to be printed out if
// TelemetrySend.shutdown() is called twice in a row without calling
// TelemetrySend.setup() in-between.
Services.prefs.removeObserver(pref, this);
}
for (let topic of this.OBSERVER_TOPICS) {
try {
Services.obs.removeObserver(this, topic);
} catch (ex) {
this._log.error(
"shutdown - failed to remove observer for " + topic,
ex
);
}
}
// We can't send anymore now.
this._sendingEnabled = false;
// Cancel any outgoing requests.
await this._cancelOutgoingRequests();
// Stop any active send tasks.
await SendScheduler.shutdown();
// Wait for any outstanding async ping activity.
await this.promisePendingPingActivity();
// Save any outstanding pending pings to disk.
await this._persistCurrentPings();
},
flushPingSenderBatch() {
if (this._pingSenderBatch.length === 0) {
return;
}
this._log.trace(
`flushPingSenderBatch - Sending ${this._pingSenderBatch.length} pings.`
);
this.runPingSender(this._pingSenderBatch);
},
reset() {
this._log.trace("reset");
this._shutdown = false;
this._currentPings = new Map();
this._tooLateToSend = false;
this._isOSShutdown = false;
this._sendingEnabled = true;
const histograms = [
"TELEMETRY_SUCCESS",
"TELEMETRY_SEND_SUCCESS",
"TELEMETRY_SEND_FAILURE",
"TELEMETRY_SEND_FAILURE_TYPE",
];
histograms.forEach(h => Services.telemetry.getHistogramById(h).clear());
const keyedHistograms = ["TELEMETRY_SEND_FAILURE_TYPE_PER_PING"];
keyedHistograms.forEach(h =>
Services.telemetry.getKeyedHistogramById(h).clear()
);
return SendScheduler.reset();
},
/**
* Notify that we can start submitting data to the servers.
*/
notifyCanUpload() {
if (!this._sendingEnabled) {
this._log.trace(
"notifyCanUpload - notifying before sending is enabled. Ignoring."
);
return Promise.resolve();
}
// Let the scheduler trigger sending pings if possible, also inform the
// crash reporter that it can send crash pings if appropriate.
SendScheduler.triggerSendingPings(true);
this._annotateCrashReport();
return this.promisePendingPingActivity();
},
observe(subject, topic, data) {
let setOSShutdown = () => {
this._log.trace("setOSShutdown - in OS shutdown");
this._isOSShutdown = true;
};
switch (topic) {
case TOPIC_IDLE_DAILY:
SendScheduler.triggerSendingPings(true);
break;
case TOPIC_QUIT_APPLICATION_FORCED:
setOSShutdown();
break;
case TOPIC_QUIT_APPLICATION_GRANTED:
if (data == "syncShutdown") {
setOSShutdown();
}
break;
case PREF_CHANGED_TOPIC:
if (this.OBSERVED_PREFERENCES.includes(data)) {
this._annotateCrashReport();
}
break;
case TOPIC_PROFILE_CHANGE_NET_TEARDOWN:
this._tooLateToSend = true;
break;
}
},
/**
* Spawn the PingSender process that sends a ping. This function does
* not return an error or throw, it only logs an error.
*
* Even if the function doesn't fail, it doesn't mean that the ping was
* successfully sent, as we have no control over the spawned process. If it,
* succeeds, the ping is eventually removed from the disk to prevent duplicated
* submissions.
*
* @param {String} pingId The id of the ping to send.
* @param {String} submissionURL The complete Telemetry-compliant URL for the ping.
*/
_sendWithPingSender(pingId, submissionURL) {
this._log.trace(
"_sendWithPingSender - sending " + pingId + " to " + submissionURL
);
try {
const pingPath = PathUtils.join(
lazy.TelemetryStorage.pingDirectoryPath,
pingId
);
if (this._tooLateToSend) {
// We're in shutdown. Batch pings destined for pingsender.
this._log.trace("_sendWithPingSender - too late to send. Batching.");
this._pingSenderBatch.push({ url: submissionURL, path: pingPath });
return;
}
this.runPingSender([{ url: submissionURL, path: pingPath }]);
} catch (e) {
this._log.error("_sendWithPingSender - failed to submit ping", e);
}
},
submitPing(ping, options) {
ChromeUtils.addProfilerMarker(
"Ping Submit",
{ category: "Telemetry" },
getPingMarkerString(ping) + `, options: ${JSON.stringify(options)}`
);
this._log.trace(
"submitPing - ping id: " +
ping.id +
", options: " +
JSON.stringify(options)
);
if (!this.sendingEnabled(ping)) {
this._log.trace("submitPing - Telemetry is not allowed to send pings.");
return Promise.resolve();
}
// Send the ping using the PingSender, if requested and the user was
// notified of our policy. We don't support the pingsender on Android,
// so ignore this option on that platform (see bug 1335917).
// Moreover, if the OS is shutting down, we don't want to spawn the
// pingsender as it could unnecessarily slow down OS shutdown.
// Additionally, it could be be killed before it can complete its tasks,
// for example after successfully sending the ping but before removing
// the copy from the disk, resulting in receiving duplicate pings when
// Firefox restarts.
if (
options.usePingSender &&
!this._isOSShutdown &&
lazy.TelemetryReportingPolicy.canUpload() &&
AppConstants.platform != "android"
) {
const url = this._buildSubmissionURL(ping);
// Serialize the ping to the disk and then spawn the PingSender.
return savePing(ping).then(() => this._sendWithPingSender(ping.id, url));
}
if (!this.canSendNow) {
// Sending is disabled or throttled, add this to the persisted pending pings.
this._log.trace(
"submitPing - can't send ping now, persisting to disk - " +
"canSendNow: " +
this.canSendNow
);
return savePing(ping);
}
// Let the scheduler trigger sending pings if possible.
// As a safety mechanism, this resets any currently active throttling.
this._log.trace("submitPing - can send pings, trying to send now");
this._currentPings.set(ping.id, ping);
SendScheduler.triggerSendingPings(true);
return Promise.resolve();
},
/**
* Only used in tests.
*/
setServer(server) {
this._log.trace("setServer", server);
this._server = server;
},
/**
* Clear out unpersisted, yet to be sent, pings and cancel outgoing ping requests.
*/
async clearCurrentPings() {
if (this._shutdown) {
this._log.trace("clearCurrentPings - in shutdown, bailing out");
return;
}
// Temporarily disable the scheduler. It must not try to reschedule ping sending
// while we're deleting them.
await SendScheduler.shutdown();
// Now that the ping activity has settled, abort outstanding ping requests.
this._cancelOutgoingRequests();
// Also, purge current pings.
this._currentPings.clear();
// We might have been interrupted and shutdown could have been started.
// We need to bail out in that case to avoid triggering send activity etc.
// at unexpected times.
if (this._shutdown) {
this._log.trace(
"clearCurrentPings - in shutdown, not spinning SendScheduler up again"
);
return;
}
// Enable the scheduler again and spin the send task.
SendScheduler.start();
SendScheduler.triggerSendingPings(true);
},
_cancelOutgoingRequests() {
// Abort any pending ping XHRs.
for (let [id, request] of this._pendingPingRequests) {
this._log.trace(
"_cancelOutgoingRequests - aborting ping request for id " + id
);
try {
request.abort();
} catch (e) {
this._log.error(
"_cancelOutgoingRequests - failed to abort request for id " + id,
e
);
}
}
this._pendingPingRequests.clear();
},
sendPings(currentPings, persistedPingIds) {
let pingSends = [];
// Prioritize health pings to enable low-latency monitoring.
currentPings = [
...currentPings.filter(ping => ping.type === "health"),
...currentPings.filter(ping => ping.type !== "health"),
];
for (let current of currentPings) {
let ping = current;
let p = (async () => {
try {
await this._doPing(ping, ping.id, false);
} catch (ex) {
this._log.info(
"sendPings - ping " + ping.id + " not sent, saving to disk",
ex
);
await savePing(ping);
} finally {
this._currentPings.delete(ping.id);
}
})();
this._trackPendingPingTask(p);
pingSends.push(p);
}
if (persistedPingIds.length) {
pingSends.push(
this._sendPersistedPings(persistedPingIds).catch(ex => {
this._log.info("sendPings - persisted pings not sent", ex);
})
);
}
return Promise.all(pingSends);
},
/**
* Send the persisted pings to the server.
*
* @param {Array<string>} List of ping ids that should be sent.
*
* @return Promise A promise that is resolved when all pings finished sending or failed.
*/
async _sendPersistedPings(pingIds) {
this._log.trace("sendPersistedPings");
if (this.pendingPingCount < 1) {
this._log.trace("_sendPersistedPings - no pings to send");
return;
}
if (pingIds.length < 1) {
this._log.trace("sendPersistedPings - no pings to send");
return;
}
// We can send now.
// If there are any send failures, _doPing() sets up handlers that e.g. trigger backoff timer behavior.
this._log.trace(
"sendPersistedPings - sending " + pingIds.length + " pings"
);
let pingSendPromises = [];
for (let pingId of pingIds) {
const id = pingId;
pingSendPromises.push(
lazy.TelemetryStorage.loadPendingPing(id)
.then(data => this._doPing(data, id, true))
.catch(e =>
this._log.error("sendPersistedPings - failed to send ping " + id, e)
)
);
}
let promise = Promise.all(pingSendPromises);
this._trackPendingPingTask(promise);
await promise;
},
_onPingRequestFinished(success, startTime, id, isPersisted) {
this._log.trace(
"_onPingRequestFinished - success: " +
success +
", persisted: " +
isPersisted
);
let time = Utils.monotonicNow() - startTime;
if (success) {
Services.telemetry.getHistogramById("TELEMETRY_SEND_SUCCESS").add(time);
} else {
Services.telemetry.getHistogramById("TELEMETRY_SEND_FAILURE").add(time);
}
Services.telemetry.getHistogramById("TELEMETRY_SUCCESS").add(success);
if (!success) {
// Let the scheduler know about send failures for triggering backoff timeouts.
SendScheduler.notifySendsFailed();
}
if (success && isPersisted) {
return lazy.TelemetryStorage.removePendingPing(id);
}
return Promise.resolve();
},
_buildSubmissionURL(ping) {
const version = isV4PingFormat(ping)
? AppConstants.TELEMETRY_PING_FORMAT_VERSION
: 1;
return this._server + this._getSubmissionPath(ping) + "?v=" + version;
},
_getSubmissionPath(ping) {
// The new ping format contains an "application" section, the old one doesn't.
let pathComponents;
if (isV4PingFormat(ping)) {
// We insert the Ping id in the URL to simplify server handling of duplicated
// pings.
let app = ping.application;
pathComponents = [
ping.id,
ping.type,
app.name,
app.version,
app.channel,
app.buildId,
];
} else {
// This is a ping in the old format.
if (!("slug" in ping)) {
// That's odd, we don't have a slug. Generate one so that TelemetryStorage.sys.mjs works.
ping.slug = Utils.generateUUID();
}
// Do we have enough info to build a submission URL?
let payload = "payload" in ping ? ping.payload : null;
if (payload && "info" in payload) {
let info = ping.payload.info;
pathComponents = [
ping.slug,
info.reason,
info.appName,
info.appVersion,
info.appUpdateChannel,
info.appBuildID,
];
} else {
// Only use the UUID as the slug.
pathComponents = [ping.slug];
}
}
let slug = pathComponents.join("/");
return "/submit/telemetry/" + slug;
},
_doPingRequest(ping, id, url, options, errorHandler, onloadHandler) {
// Don't send cookies with these requests.
let request = new ServiceRequest({ mozAnon: true });
request.mozBackgroundRequest = true;
request.timeout = Policy.pingSubmissionTimeout();
request.open("POST", url, options);
request.overrideMimeType("text/plain");
request.setRequestHeader("Content-Type", "application/json; charset=UTF-8");
request.setRequestHeader("Date", Policy.now().toUTCString());
request.setRequestHeader("Content-Encoding", "gzip");
request.onerror = errorHandler;
request.ontimeout = errorHandler;
request.onabort = errorHandler;
request.onload = onloadHandler;
this._pendingPingRequests.set(id, request);
let startTime = Utils.monotonicNow();
// If that's a legacy ping format, just send its payload.
let networkPayload = isV4PingFormat(ping) ? ping : ping.payload;
const utf8Payload = new TextEncoder().encode(
JSON.stringify(networkPayload)
);
Services.telemetry
.getHistogramById("TELEMETRY_STRINGIFY")
.add(Utils.monotonicNow() - startTime);
let payloadStream = Cc[
"@mozilla.org/io/string-input-stream;1"
].createInstance(Ci.nsIStringInputStream);
startTime = Utils.monotonicNow();
const compressedPing = Policy.gzipCompressString(
arrayToString(utf8Payload)
);
payloadStream.setByteStringData(compressedPing);
// Check the size and drop pings which are too big.
const compressedPingSizeBytes = compressedPing.length;
if (compressedPingSizeBytes > lazy.TelemetryStorage.MAXIMUM_PING_SIZE) {
this._log.error(
"_doPing - submitted ping exceeds the size limit, size: " +
compressedPingSizeBytes
);
Services.telemetry
.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_SEND")
.add();
Services.telemetry
.getHistogramById("TELEMETRY_DISCARDED_SEND_PINGS_SIZE_MB")
.add(Math.floor(compressedPingSizeBytes / 1024 / 1024));
// We don't need to call |request.abort()| as it was not sent yet.
this._pendingPingRequests.delete(id);
lazy.TelemetryHealthPing.recordDiscardedPing(ping.type);
return { promise: lazy.TelemetryStorage.removePendingPing(id) };
}
Services.telemetry
.getHistogramById("TELEMETRY_COMPRESS")
.add(Utils.monotonicNow() - startTime);
request.sendInputStream(payloadStream);
return { payloadStream };
},
_doPing(ping, id, isPersisted) {
let startTime = Cu.now();
if (!this.sendingEnabled(ping)) {
// We can't send the pings to the server, so don't try to.
this._log.trace("_doPing - Can't send ping " + ping.id);
return Promise.resolve();
}
if (this._tooLateToSend) {
// Too late to send now. Reject so we pend the ping to send it next time.
this._log.trace("_doPing - Too late to send ping " + ping.id);
Services.telemetry
.getHistogramById("TELEMETRY_SEND_FAILURE_TYPE")
.add("eTooLate");
Services.telemetry
.getKeyedHistogramById("TELEMETRY_SEND_FAILURE_TYPE_PER_PING")
.add(ping.type, "eTooLate");
return Promise.reject();
}
this._log.trace(
"_doPing - server: " +
this._server +
", persisted: " +
isPersisted +
", id: " +
id
);
const url = this._buildSubmissionURL(ping);
const monotonicStartTime = Utils.monotonicNow();
let deferred = Promise.withResolvers();
let onRequestFinished = (success, event) => {
let onCompletion = () => {
if (success) {
ChromeUtils.addProfilerMarker(
"Ping Send",
{ category: "Telemetry", startTime },
getPingMarkerString(ping)
);
deferred.resolve();
} else {
deferred.reject(event);
}
};
this._pendingPingRequests.delete(id);
this._onPingRequestFinished(
success,
monotonicStartTime,
id,
isPersisted
).then(
() => onCompletion(),
error => {
this._log.error(
"_doPing - request success: " + success + ", error: " + error
);
onCompletion();
}
);
};
let retryRequest = request => {
if (
this._shutdown ||
ServiceRequest.isOffline ||
Services.startup.shuttingDown ||
!request.bypassProxyEnabled ||
this._tooLateToSend ||
request.bypassProxy ||
!request.isProxied
) {
return false;
}
ServiceRequest.logProxySource(request.channel, "telemetry.send");
// If the request failed, and it's using a proxy, automatically
// attempt without proxy.
let { payloadStream } = this._doPingRequest(
ping,
id,
url,
{ bypassProxy: true },
errorHandler,
onloadHandler
);
this.payloadStream = payloadStream;
return true;
};
let errorHandler = event => {
let request = event.target;
if (retryRequest(request)) {
return;
}
let failure = event.type;
if (failure === "error") {
failure = XHR_ERROR_TYPE[request.errorCode];
}
lazy.TelemetryHealthPing.recordSendFailure(failure);
Services.telemetry
.getHistogramById("TELEMETRY_SEND_FAILURE_TYPE")
.add(failure);
Services.telemetry
.getKeyedHistogramById("TELEMETRY_SEND_FAILURE_TYPE_PER_PING")
.add(ping.type, failure);
this._log.error(
"_doPing - error making request to " + url + ": " + failure
);
onRequestFinished(false, event);
};
let onloadHandler = event => {
let request = event.target;
let status = request.status;
let statusClass = status - (status % 100);
let success = false;
if (statusClass === 200) {
// We can treat all 2XX as success.
this._log.info("_doPing - successfully loaded, status: " + status);
success = true;
} else if (statusClass === 400) {
// 4XX means that something with the request was broken.
this._log.error(
"_doPing - error submitting to " +
url +
", status: " +
status +
" - ping request broken?"
);
Services.telemetry
.getHistogramById("TELEMETRY_PING_EVICTED_FOR_SERVER_ERRORS")
.add();
// TODO: we should handle this better, but for now we should avoid resubmitting
// broken requests by pretending success.
success = true;
} else if (statusClass === 500) {
// 5XX means there was a server-side error and we should try again later.
this._log.error(
"_doPing - error submitting to " +
url +
", status: " +
status +
" - server error, should retry later"
);
} else {
// We received an unexpected status code.
this._log.error(
"_doPing - error submitting to " +
url +
", status: " +
status +
", type: " +
event.type
);
}
if (!success && retryRequest(request)) {
return;
}
onRequestFinished(success, event);
};
let { payloadStream, promise } = this._doPingRequest(
ping,
id,
url,
{},
errorHandler,
onloadHandler
);
if (promise) {
return promise;
}
this.payloadStream = payloadStream;
return deferred.promise;
},
/**
* Check if sending is temporarily disabled.
* @return {Boolean} True if we can send pings to the server right now, false if
* sending is temporarily disabled.
*/
get canSendNow() {
// If the reporting policy was not accepted yet, don't send pings.
if (!lazy.TelemetryReportingPolicy.canUpload()) {
return false;
}
return this._sendingEnabled;
},
/**
* Check if sending is disabled. If Telemetry is not allowed to upload,
* pings are not sent to the server.
* If trying to send a "deletion-request" ping, don't block it.
* If unified telemetry is off, don't send pings if Telemetry is disabled.
*
* @param {Object} [ping=null] A ping to be checked.
* @return {Boolean} True if pings can be send to the servers, false otherwise.
*/
sendingEnabled(ping = null) {
// We only send pings from official builds, but allow overriding this for tests.
if (
!Services.telemetry.isOfficialTelemetry &&
!this._testMode &&
!this._overrideOfficialCheck
) {
return false;
}
// With unified Telemetry, the FHR upload setting controls whether we can send pings.
// The Telemetry pref enables sending extended data sets instead.
if (IS_UNIFIED_TELEMETRY) {
// "deletion-request" pings are sent once even if the upload is disabled.
if (ping && isDeletionRequestPing(ping)) {
return true;
}
return Services.prefs.getBoolPref(
TelemetryUtils.Preferences.FhrUploadEnabled,
false
);
}
// Without unified Telemetry, the Telemetry enabled pref controls ping sending.
return Services.prefs.getBoolPref(PREF_TELEMETRY_ENABLED, false) === true;
},
/**
* Track any pending ping send and save tasks through the promise passed here.
* This is needed to block shutdown on any outstanding ping activity.
*/
_trackPendingPingTask(promise) {
let clear = () => this._pendingPingActivity.delete(promise);
promise.then(clear, clear);
this._pendingPingActivity.add(promise);
},
/**
* Return a promise that allows to wait on pending pings.
* @return {Object<Promise>} A promise resolved when all the pending pings promises
* are resolved.
*/
promisePendingPingActivity() {
this._log.trace("promisePendingPingActivity - Waiting for ping task");
let p = Array.from(this._pendingPingActivity, p =>
p.catch(ex => {
this._log.error(
"promisePendingPingActivity - ping activity had an error",
ex
);
})
);
p.push(SendScheduler.waitOnSendTask());
return Promise.all(p);
},
async _persistCurrentPings() {
for (let [id, ping] of this._currentPings) {
try {
await savePing(ping);
this._log.trace("_persistCurrentPings - saved ping " + id);
} catch (ex) {
this._log.error("_persistCurrentPings - failed to save ping " + id, ex);
} finally {
this._currentPings.delete(id);
}
}
},
/**
* Returns the current pending, not yet persisted, pings, newest first.
*/
getUnpersistedPings() {
let current = [...this._currentPings.values()];
current.reverse();
return current;
},
getShutdownState() {
return {
sendingEnabled: this._sendingEnabled,
pendingPingRequestCount: this._pendingPingRequests.size,
pendingPingActivityCount: this._pendingPingActivity.size,
unpersistedPingCount: this._currentPings.size,
persistedPingCount: lazy.TelemetryStorage.getPendingPingList().length,
schedulerState: SendScheduler.getShutdownState(),
};
},
runPingSender(pings, observer) {
if (AppConstants.platform === "android") {
throw Components.Exception("", Cr.NS_ERROR_NOT_IMPLEMENTED);
}
let suppressPingsender = Services.prefs.getBoolPref(
"toolkit.telemetry.testing.suppressPingsender",
false
);
if (suppressPingsender) {
this._log.trace("Silently skipping pingsender call in automation");
return;
}
// By default, invoke `pingsender[.exe] URL path ...`.
let exeName =
AppConstants.platform === "win" ? "pingsender.exe" : "pingsender";
let params = [];
if (lazy.NimbusFeatures.pingsender.getVariable("backgroundTaskEnabled")) {
// If using pingsender background task, invoke `firefox[.exe] --backgroundtask pingsender URL path ...`.
exeName =
AppConstants.MOZ_APP_NAME +
(AppConstants.platform === "win" ? ".exe" : "");
params = ["--backgroundtask", "pingsender"];
}
this._log.info(
`Invoking '${exeName}${params.length ? " " + params.join(" ") : ""} ...'`
);
let exe = Services.dirsvc.get("GreBinD", Ci.nsIFile);
exe.append(exeName);
params.push(...pings.flatMap(ping => [ping.url, ping.path]));
let process = Cc["@mozilla.org/process/util;1"].createInstance(
Ci.nsIProcess
);
process.init(exe);
process.startHidden = true;
process.noShell = true;
process.runAsync(params, params.length, observer);
},
};
