You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

594 lines
19 KiB

/*
* This Source Code is subject to the terms of the Mozilla Public License
* version 2.0 (the "License"). You can obtain a copy of the License at
* http://mozilla.org/MPL/2.0/.
*/
#filter substitution
/**
* @fileOverview Manages synchronization of filter subscriptions.
*/
var EXPORTED_SYMBOLS = ["Synchronizer"];
const Cc = Components.classes;
const Ci = Components.interfaces;
const Cr = Components.results;
const Cu = Components.utils;
let baseURL = "resource://@ADDON_CHROME_NAME@/modules/";
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import(baseURL + "TimeLine.jsm");
Cu.import(baseURL + "Utils.jsm");
Cu.import(baseURL + "FilterStorage.jsm");
Cu.import(baseURL + "FilterNotifier.jsm");
Cu.import(baseURL + "FilterClasses.jsm");
Cu.import(baseURL + "SubscriptionClasses.jsm");
Cu.import(baseURL + "Prefs.jsm");
const MILLISECONDS_IN_SECOND = 1000;
const SECONDS_IN_MINUTE = 60;
const SECONDS_IN_HOUR = 60 * SECONDS_IN_MINUTE;
const SECONDS_IN_DAY = 24 * SECONDS_IN_HOUR;
const INITIAL_DELAY = 6 * SECONDS_IN_MINUTE;
const CHECK_INTERVAL = SECONDS_IN_HOUR;
const MIN_EXPIRATION_INTERVAL = 1 * SECONDS_IN_DAY;
const MAX_EXPIRATION_INTERVAL = 14 * SECONDS_IN_DAY;
const MAX_ABSENSE_INTERVAL = 1 * SECONDS_IN_DAY;
let timer = null;
/**
* Map of subscriptions currently being downloaded, all currently downloaded
* URLs are keys of that map.
*/
let executing = {__proto__: null};
/**
* This object is responsible for downloading filter subscriptions whenever
* necessary.
* @class
*/
var Synchronizer =
{
/**
* Called on module startup.
*/
startup: function()
{
TimeLine.enter("Entered Synchronizer.startup()");
let callback = function()
{
timer.delay = CHECK_INTERVAL * MILLISECONDS_IN_SECOND;
checkSubscriptions();
};
timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
timer.initWithCallback(callback, INITIAL_DELAY * MILLISECONDS_IN_SECOND, Ci.nsITimer.TYPE_REPEATING_SLACK);
TimeLine.leave("Synchronizer.startup() done");
},
/**
* Checks whether a subscription is currently being downloaded.
* @param {String} url URL of the subscription
* @return {Boolean}
*/
isExecuting: function(url)
{
return url in executing;
},
/**
* Starts the download of a subscription.
* @param {DownloadableSubscription} subscription Subscription to be downloaded
* @param {Boolean} manual true for a manually started download (should not trigger fallback requests)
* @param {Boolean} forceDownload if true, the subscription will even be redownloaded if it didn't change on the server
*/
execute: function(subscription, manual, forceDownload)
{
// Delay execution, SeaMonkey 2.1 won't fire request's event handlers
// otherwise if the window that called us is closed.
Utils.runAsync(this.executeInternal, this, subscription, manual, forceDownload);
},
executeInternal: function(subscription, manual, forceDownload)
{
let url = subscription.url;
if (url in executing)
return;
let newURL = subscription.nextURL;
let hadTemporaryRedirect = false;
subscription.nextURL = null;
let curVersion = "3.5.0";
let loadFrom = newURL;
let isBaseLocation = true;
if (!loadFrom)
loadFrom = url;
if (loadFrom == url)
{
if (subscription.alternativeLocations)
{
// We have alternative download locations, choose one. "Regular"
// subscription URL always goes in with weight 1.
let options = [[1, url]];
let totalWeight = 1;
for each (let alternative in subscription.alternativeLocations.split(','))
{
if (!/^https?:\/\//.test(alternative))
continue;
let weight = 1;
let match = /;q=([\d\.]+)$/.exec(alternative);
if (match)
{
weight = parseFloat(match[1]);
if (isNaN(weight) || !isFinite(weight) || weight < 0)
weight = 1;
if (weight > 10)
weight = 10;
alternative = alternative.substr(0, match.index);
}
options.push([weight, alternative]);
totalWeight += weight;
}
let choice = Math.random() * totalWeight;
for each (let [weight, alternative] in options)
{
choice -= weight;
if (choice < 0)
{
loadFrom = alternative;
break;
}
}
isBaseLocation = (loadFrom == url);
}
}
else
{
// Ignore modification date if we are downloading from a different location
forceDownload = true;
}
loadFrom = loadFrom.replace(/%VERSION%/, "ABP" + "3.5.0");
let request = null;
function errorCallback(error)
{
let channelStatus = -1;
try
{
channelStatus = request.channel.status;
} catch (e) {}
let responseStatus = "";
try
{
responseStatus = request.channel.QueryInterface(Ci.nsIHttpChannel).responseStatus;
} catch (e) {}
setError(subscription, error, channelStatus, responseStatus, loadFrom, isBaseLocation, manual);
}
try
{
request = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"].createInstance(Ci.nsIXMLHttpRequest);
request.mozBackgroundRequest = true;
request.open("GET", loadFrom);
}
catch (e)
{
errorCallback("synchronize_invalid_url");
return;
}
try {
request.overrideMimeType("text/plain");
request.channel.loadFlags = request.channel.loadFlags |
request.channel.INHIBIT_CACHING |
request.channel.VALIDATE_ALWAYS;
// Override redirect limit from preferences, user might have set it to 1
if (request.channel instanceof Ci.nsIHttpChannel)
request.channel.redirectionLimit = 5;
var oldNotifications = request.channel.notificationCallbacks;
var oldEventSink = null;
request.channel.notificationCallbacks =
{
QueryInterface: XPCOMUtils.generateQI([Ci.nsIInterfaceRequestor, Ci.nsIChannelEventSink]),
getInterface: function(iid)
{
if (iid.equals(Ci.nsIChannelEventSink))
{
try {
oldEventSink = oldNotifications.QueryInterface(iid);
} catch(e) {}
return this;
}
if (oldNotifications)
return oldNotifications.QueryInterface(iid);
else
throw Cr.NS_ERROR_NO_INTERFACE;
},
asyncOnChannelRedirect: function(oldChannel, newChannel, flags, callback)
{
if (isBaseLocation && !hadTemporaryRedirect && oldChannel instanceof Ci.nsIHttpChannel)
{
try
{
subscription.alternativeLocations = oldChannel.getResponseHeader("X-Alternative-Locations");
}
catch (e)
{
subscription.alternativeLocations = null;
}
}
if (flags & Ci.nsIChannelEventSink.REDIRECT_TEMPORARY)
hadTemporaryRedirect = true;
else if (!hadTemporaryRedirect)
newURL = newChannel.URI.spec;
if (oldEventSink)
oldEventSink.asyncOnChannelRedirect(oldChannel, newChannel, flags, callback);
else
callback.onRedirectVerifyCallback(Cr.NS_OK);
}
}
}
catch (e)
{
Cu.reportError(e)
}
if (subscription.lastModified && !forceDownload)
request.setRequestHeader("If-Modified-Since", subscription.lastModified);
request.addEventListener("error", function(ev)
{
delete executing[url];
try {
request.channel.notificationCallbacks = null;
} catch (e) {}
errorCallback("synchronize_connection_error");
}, false);
request.addEventListener("load", function(ev)
{
delete executing[url];
try {
request.channel.notificationCallbacks = null;
} catch (e) {}
// Status will be 0 for non-HTTP requests
if (request.status && request.status != 200 && request.status != 304)
{
errorCallback("synchronize_connection_error");
return;
}
let newFilters = null;
if (request.status != 304)
{
newFilters = readFilters(subscription, request.responseText, errorCallback);
if (!newFilters)
return;
subscription.lastModified = request.getResponseHeader("Last-Modified");
}
if (isBaseLocation && !hadTemporaryRedirect)
subscription.alternativeLocations = request.getResponseHeader("X-Alternative-Locations");
subscription.lastSuccess = subscription.lastDownload = Math.round(Date.now() / MILLISECONDS_IN_SECOND);
subscription.downloadStatus = "synchronize_ok";
subscription.errors = 0;
// Expiration header is relative to server time - use Date header if it exists, otherwise local time
let now = Math.round((new Date(request.getResponseHeader("Date")).getTime() || Date.now()) / MILLISECONDS_IN_SECOND);
let expires = Math.round(new Date(request.getResponseHeader("Expires")).getTime() / MILLISECONDS_IN_SECOND) || 0;
let expirationInterval = (expires ? expires - now : 0);
for each (let filter in newFilters || subscription.filters)
{
if (!(filter instanceof CommentFilter))
continue;
let match = /\bExpires\s*(?::|after)\s*(\d+)\s*(h)?/i.exec(filter.text);
if (match)
{
let interval = parseInt(match[1], 10);
if (match[2])
interval *= SECONDS_IN_HOUR;
else
interval *= SECONDS_IN_DAY;
if (interval > expirationInterval)
expirationInterval = interval;
}
}
// Expiration interval should be within allowed range
expirationInterval = Math.min(Math.max(expirationInterval, MIN_EXPIRATION_INTERVAL), MAX_EXPIRATION_INTERVAL);
// Hard expiration: download immediately after twice the expiration interval
subscription.expires = (subscription.lastDownload + expirationInterval * 2);
// Soft expiration: use random interval factor between 0.8 and 1.2
subscription.softExpiration = (subscription.lastDownload + Math.round(expirationInterval * (Math.random() * 0.4 + 0.8)));
// Process some special filters and remove them
if (newFilters)
{
let fixedTitle = false;
for (let i = 0; i < newFilters.length; i++)
{
let filter = newFilters[i];
if (!(filter instanceof CommentFilter))
continue;
let match = /^!\s*(\w+)\s*:\s*(.*)/.exec(filter.text);
if (match)
{
let keyword = match[1].toLowerCase();
let value = match[2];
let known = true;
if (keyword == "redirect")
{
if (isBaseLocation && value != url)
subscription.nextURL = value;
}
else if (keyword == "homepage")
{
let uri = Utils.makeURI(value);
if (uri && (uri.scheme == "http" || uri.scheme == "https"))
subscription.homepage = uri.spec;
}
else if (keyword == "title")
{
if (value)
{
subscription.title = value;
fixedTitle = true;
}
}
else
known = false;
if (known)
newFilters.splice(i--, 1);
}
}
subscription.fixedTitle = fixedTitle;
}
if (isBaseLocation && newURL && newURL != url)
{
let listed = (subscription.url in FilterStorage.knownSubscriptions);
if (listed)
FilterStorage.removeSubscription(subscription);
url = newURL;
let newSubscription = Subscription.fromURL(url);
for (let key in newSubscription)
delete newSubscription[key];
for (let key in subscription)
newSubscription[key] = subscription[key];
delete Subscription.knownSubscriptions[subscription.url];
newSubscription.oldSubscription = subscription;
subscription = newSubscription;
subscription.url = url;
if (!(subscription.url in FilterStorage.knownSubscriptions) && listed)
FilterStorage.addSubscription(subscription);
}
if (newFilters)
FilterStorage.updateSubscriptionFilters(subscription, newFilters);
delete subscription.oldSubscription;
}, false);
executing[url] = true;
FilterNotifier.triggerListeners("subscription.downloadStatus", subscription);
try
{
request.send(null);
}
catch (e)
{
delete executing[url];
errorCallback("synchronize_connection_error");
return;
}
}
};
/**
* Checks whether any subscriptions need to be downloaded and starts the download
* if necessary.
*/
function checkSubscriptions()
{
if (!Prefs.subscriptions_autoupdate)
return;
let time = Math.round(Date.now() / MILLISECONDS_IN_SECOND);
for each (let subscription in FilterStorage.subscriptions)
{
if (!(subscription instanceof DownloadableSubscription))
continue;
if (subscription.lastCheck && time - subscription.lastCheck > MAX_ABSENSE_INTERVAL)
{
// No checks for a long time interval - user must have been offline, e.g.
// during a weekend. Increase soft expiration to prevent load peaks on the
// server.
subscription.softExpiration += time - subscription.lastCheck;
}
subscription.lastCheck = time;
// Sanity check: do expiration times make sense? Make sure people changing
// system clock don't get stuck with outdated subscriptions.
if (subscription.expires - time > MAX_EXPIRATION_INTERVAL)
subscription.expires = time + MAX_EXPIRATION_INTERVAL;
if (subscription.softExpiration - time > MAX_EXPIRATION_INTERVAL)
subscription.softExpiration = time + MAX_EXPIRATION_INTERVAL;
if (subscription.softExpiration > time && subscription.expires > time)
continue;
// Do not retry downloads more often than MIN_EXPIRATION_INTERVAL
if (time - subscription.lastDownload >= MIN_EXPIRATION_INTERVAL)
Synchronizer.execute(subscription, false);
}
}
/**
* Extracts a list of filters from text returned by a server.
* @param {DownloadableSubscription} subscription subscription the info should be placed into
* @param {String} text server response
* @param {Function} errorCallback function to be called on error
* @return {Array of Filter}
*/
function readFilters(subscription, text, errorCallback)
{
let lines = text.split(/[\r\n]+/);
let match = /\[Adblock(?:\s*Plus\s*([\d\.]+)?)?\]/i.exec(lines[0]);
if (!match)
{
errorCallback("synchronize_invalid_data");
return null;
}
let minVersion = match[1];
for (let i = 0; i < lines.length; i++)
{
let match = /!\s*checksum[\s\-:]+([\w\+\/]+)/i.exec(lines[i]);
if (match)
{
lines.splice(i, 1);
let checksum = Utils.generateChecksum(lines);
if (checksum && checksum != match[1])
{
errorCallback("synchronize_checksum_mismatch");
return null;
}
break;
}
}
delete subscription.requiredVersion;
delete subscription.upgradeRequired;
if (minVersion)
{
subscription.requiredVersion = minVersion;
if (Utils.versionComparator.compare(minVersion, "3.5.0") > 0)
subscription.upgradeRequired = true;
}
lines.shift();
let result = [];
for each (let line in lines)
{
let filter = Filter.fromText(Filter.normalize(line));
if (filter)
result.push(filter);
}
return result;
}
/**
* Handles an error during a subscription download.
* @param {DownloadableSubscription} subscription subscription that failed to download
* @param {Integer} channelStatus result code of the download channel
* @param {String} responseStatus result code as received from server
* @param {String} downloadURL the URL used for download
* @param {String} error error ID in global.properties
* @param {Boolean} isBaseLocation false if the subscription was downloaded from a location specified in X-Alternative-Locations header
* @param {Boolean} manual true for a manually started download (should not trigger fallback requests)
*/
function setError(subscription, error, channelStatus, responseStatus, downloadURL, isBaseLocation, manual)
{
// If download from an alternative location failed, reset the list of
// alternative locations - have to get an updated list from base location.
if (!isBaseLocation)
subscription.alternativeLocations = null;
try {
Cu.reportError("Adblock Plus: Downloading filter subscription " + subscription.title + " failed (" + Utils.getString(error) + ")\n" +
"Download address: " + downloadURL + "\n" +
"Channel status: " + channelStatus + "\n" +
"Server response: " + responseStatus);
} catch(e) {}
subscription.lastDownload = Math.round(Date.now() / MILLISECONDS_IN_SECOND);
subscription.downloadStatus = error;
// Request fallback URL if necessary - for automatic updates only
if (!manual)
{
if (error == "synchronize_checksum_mismatch")
{
// No fallback for successful download with checksum mismatch, reset error counter
subscription.errors = 0;
}
else
subscription.errors++;
if (subscription.errors >= Prefs.subscriptions_fallbackerrors && /^https?:\/\//i.test(subscription.url))
{
subscription.errors = 0;
let fallbackURL = Prefs.subscriptions_fallbackurl;
fallbackURL = fallbackURL.replace(/%VERSION%/g, encodeURIComponent("3.5.0"));
fallbackURL = fallbackURL.replace(/%SUBSCRIPTION%/g, encodeURIComponent(subscription.url));
fallbackURL = fallbackURL.replace(/%URL%/g, encodeURIComponent(downloadURL));
fallbackURL = fallbackURL.replace(/%ERROR%/g, encodeURIComponent(error));
fallbackURL = fallbackURL.replace(/%CHANNELSTATUS%/g, encodeURIComponent(channelStatus));
fallbackURL = fallbackURL.replace(/%RESPONSESTATUS%/g, encodeURIComponent(responseStatus));
let request = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"].createInstance(Ci.nsIXMLHttpRequest);
request.mozBackgroundRequest = true;
request.open("GET", fallbackURL);
request.overrideMimeType("text/plain");
request.channel.loadFlags = request.channel.loadFlags |
request.channel.INHIBIT_CACHING |
request.channel.VALIDATE_ALWAYS;
request.addEventListener("load", function(ev)
{
if (!(subscription.url in FilterStorage.knownSubscriptions))
return;
let match = /^(\d+)(?:\s+(\S+))?$/.exec(request.responseText);
if (match && match[1] == "301" && match[2]) // Moved permanently
subscription.nextURL = match[2];
else if (match && match[1] == "410") // Gone
{
let data = "[Adblock]\n" + subscription.filters.map(function(f) f.text).join("\n");
let url = "data:text/plain," + encodeURIComponent(data);
let newSubscription = Subscription.fromURL(url);
newSubscription.title = subscription.title;
newSubscription.disabled = subscription.disabled;
FilterStorage.removeSubscription(subscription);
FilterStorage.addSubscription(newSubscription);
Synchronizer.execute(newSubscription);
}
}, false);
request.send(null);
}
}
}