lib/downloader.js - Issue 29335650: Issue 2595 - Use the core code from adblockpluscore

Side by Side Diff: lib/downloader.js

Issue 29335650: Issue 2595 - Use the core code from adblockpluscore (Closed)

Patch Set: Created Feb. 4, 2016, 6:35 p.m.

Left:
Right:

Use n/p to move between diff chunks; N/P to move between comments.

Jump to:

View unified diff | Download patch

OLD	NEW
	(Empty)
1 /*

2 * This file is part of Adblock Plus <https://adblockplus.org/>,

3 * Copyright (C) 2006-2016 Eyeo GmbH

4 *

5 * Adblock Plus is free software: you can redistribute it and/or modify

6 * it under the terms of the GNU General Public License version 3 as

7 * published by the Free Software Foundation.

8 *

9 * Adblock Plus is distributed in the hope that it will be useful,

10 * but WITHOUT ANY WARRANTY; without even the implied warranty of

11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

12 * GNU General Public License for more details.

13 *

14 * You should have received a copy of the GNU General Public License

15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.

16 */

17

18 /**

19 * @fileOverview Downloads a set of URLs in regular time intervals.

20 */

21

22 let {Utils} = require("utils");

23

24 let MILLIS_IN_SECOND = exports.MILLIS_IN_SECOND = 1000;

25 let MILLIS_IN_MINUTE = exports.MILLIS_IN_MINUTE = 60 * MILLIS_IN_SECOND;

26 let MILLIS_IN_HOUR = exports.MILLIS_IN_HOUR = 60 * MILLIS_IN_MINUTE;

27 let MILLIS_IN_DAY = exports.MILLIS_IN_DAY = 24 * MILLIS_IN_HOUR;

28

29 /**

30 * Creates a new downloader instance.

31 * @param {Function} dataSource Function that will yield downloadable objects o n each check

32 * @param {Integer} initialDelay Number of milliseconds to wait before the firs t check

33 * @param {Integer} checkInterval Interval between the checks

34 * @constructor

35 */

36 let Downloader = exports.Downloader = function Downloader(dataSource, initialDel ay, checkInterval)

37 {

38 this.dataSource = dataSource;

39 this._timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);

40 this._timer.initWithCallback(function()

41 {

42 this._timer.delay = checkInterval;

43 this._doCheck();

44 }.bind(this), initialDelay, Ci.nsITimer.TYPE_REPEATING_SLACK);

45 this._downloading = Object.create(null);

46 }

47 Downloader.prototype =

48 {

49 /**

50 * Timer triggering the downloads.

51 * @type nsITimer

52 */

53 _timer: null,

54

55 /**

56 * Map containing the URLs of objects currently being downloaded as its keys.

57 */

58 _downloading: null,

59

60 /**

61 * Function that will yield downloadable objects on each check.

62 * @type Function

63 */

64 dataSource: null,

65

66 /**

67 * Maximal time interval that the checks can be left out until the soft

68 * expiration interval increases.

69 * @type Integer

70 */

71 maxAbsenceInterval: 1 * MILLIS_IN_DAY,

72

73 /**

74 * Minimal time interval before retrying a download after an error.

75 * @type Integer

76 */

77 minRetryInterval: 1 * MILLIS_IN_DAY,

78

79 /**

80 * Maximal allowed expiration interval, larger expiration intervals will be

81 * corrected.

82 * @type Integer

83 */

84 maxExpirationInterval: 14 * MILLIS_IN_DAY,

85

86 /**

87 * Maximal number of redirects before the download is considered as failed.

88 * @type Integer

89 */

90 maxRedirects: 5,

91

92 /**

93 * Called whenever expiration intervals for an object need to be adapted.

94 * @type Function

95 */

96 onExpirationChange: null,

97

98 /**

99 * Callback to be triggered whenever a download starts.

100 * @type Function

101 */

102 onDownloadStarted: null,

103

104 /**

105 * Callback to be triggered whenever a download finishes successfully. The

106 * callback can return an error code to indicate that the data is wrong.

107 * @type Function

108 */

109 onDownloadSuccess: null,

110

111 /**

112 * Callback to be triggered whenever a download fails.

113 * @type Function

114 */

115 onDownloadError: null,

116

117 /**

118 * Checks whether anything needs downloading.

119 */

120 _doCheck: function()

121 {

122 let now = Date.now();

123 for (let downloadable of this.dataSource())

124 {

125 if (downloadable.lastCheck && now - downloadable.lastCheck > this.maxAbsen ceInterval)

126 {

127 // No checks for a long time interval - user must have been offline, e.g .

128 // during a weekend. Increase soft expiration to prevent load peaks on t he

129 // server.

130 downloadable.softExpiration += now - downloadable.lastCheck;

131 }

132 downloadable.lastCheck = now;

133

134 // Sanity check: do expiration times make sense? Make sure people changing

135 // system clock don't get stuck with outdated subscriptions.

136 if (downloadable.hardExpiration - now > this.maxExpirationInterval)

137 downloadable.hardExpiration = now + this.maxExpirationInterval;

138 if (downloadable.softExpiration - now > this.maxExpirationInterval)

139 downloadable.softExpiration = now + this.maxExpirationInterval;

140

141 // Notify the caller about changes to expiration parameters

142 if (this.onExpirationChange)

143 this.onExpirationChange(downloadable);

144

145 // Does that object need downloading?

146 if (downloadable.softExpiration > now && downloadable.hardExpiration > now )

147 continue;

148

149 // Do not retry downloads too often

150 if (downloadable.lastError && now - downloadable.lastError < this.minRetry Interval)

151 continue;

152

153 this._download(downloadable, 0);

154 }

155 },

156

157 /**

158 * Stops the periodic checks.

159 */

160 cancel: function()

161 {

162 this._timer.cancel();

163 },

164

165 /**

166 * Checks whether an address is currently being downloaded.

167 */

168 isDownloading: function(/*String/ url) /*Boolean/

169 {

170 return url in this._downloading;

171 },

172

173 /**

174 * Starts downloading for an object.

175 */

176 download: function(/*Downloadable/ downloadable)

177 {

178 // Make sure to detach download from the current execution context

179 Utils.runAsync(this._download.bind(this, downloadable, 0));

180 },

181

182 /**

183 * Generates the real download URL for an object by appending various

184 * parameters.

185 */

186 getDownloadUrl: function(/*Downloadable/ downloadable) /** String*/

187 {

188 let {addonName, addonVersion, application, applicationVersion, platform, pla tformVersion} = require("info");

189 let url = downloadable.redirectURL \|\| downloadable.url;

190 if (url.indexOf("?") >= 0)

191 url += "&";

192 else

193 url += "?";

194 // We limit the download count to 4+ to keep the request anonymized

195 let downloadCount = downloadable.downloadCount;

196 if (downloadCount > 4)

197 downloadCount = "4+";

198 url += "addonName=" + encodeURIComponent(addonName) +

199 "&addonVersion=" + encodeURIComponent(addonVersion) +

200 "&application=" + encodeURIComponent(application) +

201 "&applicationVersion=" + encodeURIComponent(applicationVersion) +

202 "&platform=" + encodeURIComponent(platform) +

203 "&platformVersion=" + encodeURIComponent(platformVersion) +

204 "&lastVersion=" + encodeURIComponent(downloadable.lastVersion) +

205 "&downloadCount=" + encodeURIComponent(downloadCount);

206 return url;

207 },

208

209 _download: function(downloadable, redirects)

210 {

211 if (this.isDownloading(downloadable.url))

212 return;

213

214 let downloadUrl = this.getDownloadUrl(downloadable);

215 let request = null;

216

217 let errorCallback = function errorCallback(error)

218 {

219 let channelStatus = -1;

220 try

221 {

222 channelStatus = request.channel.status;

223 } catch (e) {}

224

225 let responseStatus = request.status;

226

227 Cu.reportError("Adblock Plus: Downloading URL " + downloadable.url + " fai led (" + error + ")\n" +

228 "Download address: " + downloadUrl + "\n" +

229 "Channel status: " + channelStatus + "\n" +

230 "Server response: " + responseStatus);

231

232 if (this.onDownloadError)

233 {

234 // Allow one extra redirect if the error handler gives us a redirect URL

235 let redirectCallback = null;

236 if (redirects <= this.maxRedirects)

237 {

238 redirectCallback = function redirectCallback(url)

239 {

240 downloadable.redirectURL = url;

241 this._download(downloadable, redirects + 1);

242 }.bind(this);

243 }

244

245 this.onDownloadError(downloadable, downloadUrl, error, channelStatus, re sponseStatus, redirectCallback);

246 }

247 }.bind(this);

248

249 try

250 {

251 request = new XMLHttpRequest();

252 request.mozBackgroundRequest = true;

253 request.open("GET", downloadUrl);

254 }

255 catch (e)

256 {

257 errorCallback("synchronize_invalid_url");

258 return;

259 }

260

261 try {

262 request.overrideMimeType("text/plain");

263 request.channel.loadFlags = request.channel.loadFlags \|

264 request.channel.INHIBIT_CACHING \|

265 request.channel.VALIDATE_ALWAYS;

266

267 // Override redirect limit from preferences, user might have set it to 1

268 if (request.channel instanceof Ci.nsIHttpChannel)

269 request.channel.redirectionLimit = this.maxRedirects;

270 }

271 catch (e)

272 {

273 Cu.reportError(e)

274 }

275

276 request.addEventListener("error", function(event)

277 {

278 if (onShutdown.done)

279 return;

280

281 delete this._downloading[downloadable.url];

282 errorCallback("synchronize_connection_error");

283 }.bind(this), false);

284

285 request.addEventListener("load", function(event)

286 {

287 if (onShutdown.done)

288 return;

289

290 delete this._downloading[downloadable.url];

291

292 // Status will be 0 for non-HTTP requests

293 if (request.status && request.status != 200)

294 {

295 errorCallback("synchronize_connection_error");

296 return;

297 }

298

299 downloadable.downloadCount++;

300

301 this.onDownloadSuccess(downloadable, request.responseText, errorCallback, function redirectCallback(url)

302 {

303 if (redirects >= this.maxRedirects)

304 errorCallback("synchronize_connection_error");

305 else

306 {

307 downloadable.redirectURL = url;

308 this._download(downloadable, redirects + 1);

309 }

310 }.bind(this));

311 }.bind(this), false);

312

313 request.send(null);

314

315 this._downloading[downloadable.url] = true;

316 if (this.onDownloadStarted)

317 this.onDownloadStarted(downloadable);

318 },

319

320 /**

321 * Produces a soft and a hard expiration interval for a given supplied

322 * expiration interval.

323 * @return {Array} soft and hard expiration interval

324 */

325 processExpirationInterval: function(/*Integer/ interval)

326 {

327 interval = Math.min(Math.max(interval, 0), this.maxExpirationInterval);

328 let soft = Math.round(interval * (Math.random() * 0.4 + 0.8));

329 let hard = interval * 2;

330 let now = Date.now();

331 return [now + soft, now + hard];

332 }

333 };

334

335 /**

336 * An object that can be downloaded by the downloadable

337 * @param {String} url URL that has to be requested for the object

338 * @constructor

339 */

340 let Downloadable = exports.Downloadable = function Downloadable(url)

341 {

342 this.url = url;

343 }

344 Downloadable.prototype =

345 {

346 /**

347 * URL that has to be requested for the object.

348 * @type String

349 */

350 url: null,

351

352 /**

353 * URL that the download was redirected to if any.

354 * @type String

355 */

356 redirectURL: null,

357

358 /**

359 * Time of last download error or 0 if the last download was successful.

360 * @type Integer

361 */

362 lastError: 0,

363

364 /**

365 * Time of last check whether the object needs downloading.

366 * @type Integer

367 */

368 lastCheck: 0,

369

370 /**

371 * Object version corresponding to the last successful download.

372 * @type Integer

373 */

374 lastVersion: 0,

375

376 /**

377 * Soft expiration interval, will increase if no checks are performed for a

378 * while.

379 * @type Integer

380 */

381 softExpiration: 0,

382

383 /**

384 * Hard expiration interval, this is fixed.

385 * @type Integer

386 */

387 hardExpiration: 0,

388

389 /**

390 * Number indicating how often the object was downloaded.

391 * @type Integer

392 */

393 downloadCount: 0,

394 };

OLD	NEW

« no previous file with comments | « lib/cssRules.js ('k') | lib/elemHide.js » ('j') | no next file with comments »