| OLD | NEW | 
|---|
| 1 /* | 1 /* | 
| 2  * This file is part of Adblock Plus <https://adblockplus.org/>, | 2  * This file is part of Adblock Plus <https://adblockplus.org/>, | 
| 3  * Copyright (C) 2006-present eyeo GmbH | 3  * Copyright (C) 2006-present eyeo GmbH | 
| 4  * | 4  * | 
| 5  * Adblock Plus is free software: you can redistribute it and/or modify | 5  * Adblock Plus is free software: you can redistribute it and/or modify | 
| 6  * it under the terms of the GNU General Public License version 3 as | 6  * it under the terms of the GNU General Public License version 3 as | 
| 7  * published by the Free Software Foundation. | 7  * published by the Free Software Foundation. | 
| 8  * | 8  * | 
| 9  * Adblock Plus is distributed in the hope that it will be useful, | 9  * Adblock Plus is distributed in the hope that it will be useful, | 
| 10  * but WITHOUT ANY WARRANTY; without even the implied warranty of | 10  * but WITHOUT ANY WARRANTY; without even the implied warranty of | 
| (...skipping 10 matching lines...) Expand all  Loading... | 
| 21  * @fileOverview Downloads a set of URLs in regular time intervals. | 21  * @fileOverview Downloads a set of URLs in regular time intervals. | 
| 22  */ | 22  */ | 
| 23 | 23 | 
| 24 const {Utils} = require("utils"); | 24 const {Utils} = require("utils"); | 
| 25 | 25 | 
| 26 const MILLIS_IN_SECOND = exports.MILLIS_IN_SECOND = 1000; | 26 const MILLIS_IN_SECOND = exports.MILLIS_IN_SECOND = 1000; | 
| 27 const MILLIS_IN_MINUTE = exports.MILLIS_IN_MINUTE = 60 * MILLIS_IN_SECOND; | 27 const MILLIS_IN_MINUTE = exports.MILLIS_IN_MINUTE = 60 * MILLIS_IN_SECOND; | 
| 28 const MILLIS_IN_HOUR = exports.MILLIS_IN_HOUR = 60 * MILLIS_IN_MINUTE; | 28 const MILLIS_IN_HOUR = exports.MILLIS_IN_HOUR = 60 * MILLIS_IN_MINUTE; | 
| 29 const MILLIS_IN_DAY = exports.MILLIS_IN_DAY = 24 * MILLIS_IN_HOUR; | 29 const MILLIS_IN_DAY = exports.MILLIS_IN_DAY = 24 * MILLIS_IN_HOUR; | 
| 30 | 30 | 
| 31 let Downloader = | 31 class Downloader | 
| 32 /** |  | 
| 33  * Creates a new downloader instance. |  | 
| 34  * @param {Function} dataSource |  | 
| 35  *   Function that will yield downloadable objects on each check |  | 
| 36  * @param {number} initialDelay |  | 
| 37  *   Number of milliseconds to wait before the first check |  | 
| 38  * @param {number} checkInterval |  | 
| 39  *   Interval between the checks |  | 
| 40  * @constructor |  | 
| 41  */ |  | 
| 42 exports.Downloader = function(dataSource, initialDelay, checkInterval) |  | 
| 43 { |  | 
| 44   this.dataSource = dataSource; |  | 
| 45   this._timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); |  | 
| 46   this._timer.initWithCallback(() => |  | 
| 47   { |  | 
| 48     this._timer.delay = checkInterval; |  | 
| 49     this._doCheck(); |  | 
| 50   }, initialDelay, Ci.nsITimer.TYPE_REPEATING_SLACK); |  | 
| 51   this._downloading = new Set(); |  | 
| 52 }; |  | 
| 53 Downloader.prototype = |  | 
| 54 { | 32 { | 
| 55   /** | 33   /** | 
| 56    * Timer triggering the downloads. | 34    * Creates a new downloader instance. | 
| 57    * @type {nsITimer} | 35    * @param {function} dataSource | 
|  | 36    *   Function that will yield downloadable objects on each check | 
|  | 37    * @param {number} initialDelay | 
|  | 38    *   Number of milliseconds to wait before the first check | 
|  | 39    * @param {number} checkInterval | 
|  | 40    *   Interval between the checks | 
| 58    */ | 41    */ | 
| 59   _timer: null, | 42   constructor(dataSource, initialDelay, checkInterval) | 
|  | 43   { | 
|  | 44     /** | 
|  | 45      * Maximal time interval that the checks can be left out until the soft | 
|  | 46      * expiration interval increases. | 
|  | 47      * @type {number} | 
|  | 48      */ | 
|  | 49     this.maxAbsenceInterval = 1 * MILLIS_IN_DAY; | 
| 60 | 50 | 
| 61   /** | 51     /** | 
| 62    * Set containing the URLs of objects currently being downloaded. | 52      * Minimal time interval before retrying a download after an error. | 
| 63    * @type {Set.<string>} | 53      * @type {number} | 
| 64    */ | 54      */ | 
| 65   _downloading: null, | 55     this.minRetryInterval = 1 * MILLIS_IN_DAY; | 
| 66 | 56 | 
| 67   /** | 57     /** | 
| 68    * Function that will yield downloadable objects on each check. | 58      * Maximal allowed expiration interval; larger expiration intervals will be | 
| 69    * @type {Function} | 59      * corrected. | 
| 70    */ | 60      * @type {number} | 
| 71   dataSource: null, | 61      */ | 
|  | 62     this.maxExpirationInterval = 14 * MILLIS_IN_DAY; | 
| 72 | 63 | 
| 73   /** | 64     /** | 
| 74    * Maximal time interval that the checks can be left out until the soft | 65      * Maximal number of redirects before the download is considered as failed. | 
| 75    * expiration interval increases. | 66      * @type {number} | 
| 76    * @type {number} | 67      */ | 
| 77    */ | 68     this.maxRedirects = 5; | 
| 78   maxAbsenceInterval: 1 * MILLIS_IN_DAY, |  | 
| 79 | 69 | 
| 80   /** | 70     /** | 
| 81    * Minimal time interval before retrying a download after an error. | 71      * Called whenever expiration intervals for an object need to be adapted. | 
| 82    * @type {number} | 72      * @type {function?} | 
| 83    */ | 73      */ | 
| 84   minRetryInterval: 1 * MILLIS_IN_DAY, | 74     this.onExpirationChange = null; | 
| 85 | 75 | 
| 86   /** | 76     /** | 
| 87    * Maximal allowed expiration interval, larger expiration intervals will be | 77      * Callback to be triggered whenever a download starts. | 
| 88    * corrected. | 78      * @type {function?} | 
| 89    * @type {number} | 79      */ | 
| 90    */ | 80     this.onDownloadStarted = null; | 
| 91   maxExpirationInterval: 14 * MILLIS_IN_DAY, |  | 
| 92 | 81 | 
| 93   /** | 82     /** | 
| 94    * Maximal number of redirects before the download is considered as failed. | 83      * Callback to be triggered whenever a download finishes successfully. The | 
| 95    * @type {number} | 84      * callback can return an error code to indicate that the data is wrong. | 
| 96    */ | 85      * @type {function?} | 
| 97   maxRedirects: 5, | 86      */ | 
|  | 87     this.onDownloadSuccess = null; | 
| 98 | 88 | 
| 99   /** | 89     /** | 
| 100    * Called whenever expiration intervals for an object need to be adapted. | 90      * Callback to be triggered whenever a download fails. | 
| 101    * @type {Function} | 91      * @type {function?} | 
| 102    */ | 92      */ | 
| 103   onExpirationChange: null, | 93     this.onDownloadError = null; | 
| 104 | 94 | 
| 105   /** | 95     /** | 
| 106    * Callback to be triggered whenever a download starts. | 96      * Function that will yield downloadable objects on each check. | 
| 107    * @type {Function} | 97      * @type {function} | 
| 108    */ | 98      */ | 
| 109   onDownloadStarted: null, | 99     this.dataSource = dataSource; | 
| 110 | 100 | 
| 111   /** | 101     /** | 
| 112    * Callback to be triggered whenever a download finishes successfully. The | 102      * Timer triggering the downloads. | 
| 113    * callback can return an error code to indicate that the data is wrong. | 103      * @type {nsITimer} | 
| 114    * @type {Function} | 104      */ | 
| 115    */ | 105     this._timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); | 
| 116   onDownloadSuccess: null, | 106     this._timer.initWithCallback(() => | 
|  | 107     { | 
|  | 108       this._timer.delay = checkInterval; | 
|  | 109       this._doCheck(); | 
|  | 110     }, initialDelay, Ci.nsITimer.TYPE_REPEATING_SLACK); | 
| 117 | 111 | 
| 118   /** | 112     /** | 
| 119    * Callback to be triggered whenever a download fails. | 113      * Set containing the URLs of objects currently being downloaded. | 
| 120    * @type {Function} | 114      * @type {Set.<string>} | 
| 121    */ | 115      */ | 
| 122   onDownloadError: null, | 116     this._downloading = new Set(); | 
|  | 117   } | 
| 123 | 118 | 
| 124   /** | 119   /** | 
| 125    * Checks whether anything needs downloading. | 120    * Checks whether anything needs downloading. | 
| 126    */ | 121    */ | 
| 127   _doCheck() | 122   _doCheck() | 
| 128   { | 123   { | 
| 129     let now = Date.now(); | 124     let now = Date.now(); | 
| 130     for (let downloadable of this.dataSource()) | 125     for (let downloadable of this.dataSource()) | 
| 131     { | 126     { | 
| 132       if (downloadable.lastCheck && | 127       if (downloadable.lastCheck && | 
| (...skipping 26 matching lines...) Expand all  Loading... | 
| 159 | 154 | 
| 160       // Do not retry downloads too often | 155       // Do not retry downloads too often | 
| 161       if (downloadable.lastError && | 156       if (downloadable.lastError && | 
| 162           now - downloadable.lastError < this.minRetryInterval) | 157           now - downloadable.lastError < this.minRetryInterval) | 
| 163       { | 158       { | 
| 164         continue; | 159         continue; | 
| 165       } | 160       } | 
| 166 | 161 | 
| 167       this._download(downloadable, 0); | 162       this._download(downloadable, 0); | 
| 168     } | 163     } | 
| 169   }, | 164   } | 
| 170 | 165 | 
| 171   /** | 166   /** | 
| 172    * Stops the periodic checks. | 167    * Stops the periodic checks. | 
| 173    */ | 168    */ | 
| 174   cancel() | 169   cancel() | 
| 175   { | 170   { | 
| 176     this._timer.cancel(); | 171     this._timer.cancel(); | 
| 177   }, | 172   } | 
| 178 | 173 | 
| 179   /** | 174   /** | 
| 180    * Checks whether an address is currently being downloaded. | 175    * Checks whether an address is currently being downloaded. | 
| 181    * @param {string} url | 176    * @param {string} url | 
| 182    * @return {boolean} | 177    * @returns {boolean} | 
| 183    */ | 178    */ | 
| 184   isDownloading(url) | 179   isDownloading(url) | 
| 185   { | 180   { | 
| 186     return this._downloading.has(url); | 181     return this._downloading.has(url); | 
| 187   }, | 182   } | 
| 188 | 183 | 
| 189   /** | 184   /** | 
| 190    * Starts downloading for an object. | 185    * Starts downloading for an object. | 
| 191    * @param {Downloadable} downloadable | 186    * @param {Downloadable} downloadable | 
| 192    */ | 187    */ | 
| 193   download(downloadable) | 188   download(downloadable) | 
| 194   { | 189   { | 
| 195     // Make sure to detach download from the current execution context | 190     // Make sure to detach download from the current execution context | 
| 196     Utils.runAsync(this._download.bind(this, downloadable, 0)); | 191     Utils.runAsync(this._download.bind(this, downloadable, 0)); | 
| 197   }, | 192   } | 
| 198 | 193 | 
| 199   /** | 194   /** | 
| 200    * Generates the real download URL for an object by appending various | 195    * Generates the real download URL for an object by appending various | 
| 201    * parameters. | 196    * parameters. | 
| 202    * @param {Downloadable} downloadable | 197    * @param {Downloadable} downloadable | 
| 203    * @return {string} | 198    * @returns {string} | 
| 204    */ | 199    */ | 
| 205   getDownloadUrl(downloadable) | 200   getDownloadUrl(downloadable) | 
| 206   { | 201   { | 
| 207     const {addonName, addonVersion, application, applicationVersion, | 202     const {addonName, addonVersion, application, applicationVersion, | 
| 208            platform, platformVersion} = require("info"); | 203            platform, platformVersion} = require("info"); | 
| 209     let url = downloadable.redirectURL || downloadable.url; | 204     let url = downloadable.redirectURL || downloadable.url; | 
| 210     if (url.includes("?")) | 205     if (url.includes("?")) | 
| 211       url += "&"; | 206       url += "&"; | 
| 212     else | 207     else | 
| 213       url += "?"; | 208       url += "?"; | 
| 214     // We limit the download count to 4+ to keep the request anonymized | 209     // We limit the download count to 4+ to keep the request anonymized | 
| 215     let {downloadCount} = downloadable; | 210     let {downloadCount} = downloadable; | 
| 216     if (downloadCount > 4) | 211     if (downloadCount > 4) | 
| 217       downloadCount = "4+"; | 212       downloadCount = "4+"; | 
| 218     url += "addonName=" + encodeURIComponent(addonName) + | 213     url += "addonName=" + encodeURIComponent(addonName) + | 
| 219         "&addonVersion=" + encodeURIComponent(addonVersion) + | 214         "&addonVersion=" + encodeURIComponent(addonVersion) + | 
| 220         "&application=" + encodeURIComponent(application) + | 215         "&application=" + encodeURIComponent(application) + | 
| 221         "&applicationVersion=" + encodeURIComponent(applicationVersion) + | 216         "&applicationVersion=" + encodeURIComponent(applicationVersion) + | 
| 222         "&platform=" + encodeURIComponent(platform) + | 217         "&platform=" + encodeURIComponent(platform) + | 
| 223         "&platformVersion=" + encodeURIComponent(platformVersion) + | 218         "&platformVersion=" + encodeURIComponent(platformVersion) + | 
| 224         "&lastVersion=" + encodeURIComponent(downloadable.lastVersion) + | 219         "&lastVersion=" + encodeURIComponent(downloadable.lastVersion) + | 
| 225         "&downloadCount=" + encodeURIComponent(downloadCount); | 220         "&downloadCount=" + encodeURIComponent(downloadCount); | 
| 226     return url; | 221     return url; | 
| 227   }, | 222   } | 
| 228 | 223 | 
| 229   _download(downloadable, redirects) | 224   _download(downloadable, redirects) | 
| 230   { | 225   { | 
| 231     if (this.isDownloading(downloadable.url)) | 226     if (this.isDownloading(downloadable.url)) | 
| 232       return; | 227       return; | 
| 233 | 228 | 
| 234     let downloadUrl = this.getDownloadUrl(downloadable); | 229     let downloadUrl = this.getDownloadUrl(downloadable); | 
| 235     let request = null; | 230     let request = null; | 
| 236 | 231 | 
| 237     let errorCallback = function errorCallback(error) | 232     let errorCallback = function errorCallback(error) | 
| (...skipping 97 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
| 335           } | 330           } | 
| 336         } | 331         } | 
| 337       ); | 332       ); | 
| 338     }); | 333     }); | 
| 339 | 334 | 
| 340     request.send(null); | 335     request.send(null); | 
| 341 | 336 | 
| 342     this._downloading.add(downloadable.url); | 337     this._downloading.add(downloadable.url); | 
| 343     if (this.onDownloadStarted) | 338     if (this.onDownloadStarted) | 
| 344       this.onDownloadStarted(downloadable); | 339       this.onDownloadStarted(downloadable); | 
| 345   }, | 340   } | 
| 346 | 341 | 
| 347   /** | 342   /** | 
| 348    * Produces a soft and a hard expiration interval for a given supplied | 343    * Produces a soft and a hard expiration interval for a given supplied | 
| 349    * expiration interval. | 344    * expiration interval. | 
| 350    * @param {number} interval | 345    * @param {number} interval | 
| 351    * @return {Array} soft and hard expiration interval | 346    * @returns {Array.<number>} soft and hard expiration interval | 
| 352    */ | 347    */ | 
| 353   processExpirationInterval(interval) | 348   processExpirationInterval(interval) | 
| 354   { | 349   { | 
| 355     interval = Math.min(Math.max(interval, 0), this.maxExpirationInterval); | 350     interval = Math.min(Math.max(interval, 0), this.maxExpirationInterval); | 
| 356     let soft = Math.round(interval * (Math.random() * 0.4 + 0.8)); | 351     let soft = Math.round(interval * (Math.random() * 0.4 + 0.8)); | 
| 357     let hard = interval * 2; | 352     let hard = interval * 2; | 
| 358     let now = Date.now(); | 353     let now = Date.now(); | 
| 359     return [now + soft, now + hard]; | 354     return [now + soft, now + hard]; | 
| 360   } | 355   } | 
| 361 }; | 356 } | 
| 362 | 357 | 
| 363 /** | 358 exports.Downloader = Downloader; | 
| 364  * An object that can be downloaded by the downloadable | 359 | 
| 365  * @param {string} url  URL that has to be requested for the object | 360 class Downloadable | 
| 366  * @constructor |  | 
| 367  */ |  | 
| 368 let Downloadable = exports.Downloadable = function Downloadable(url) |  | 
| 369 { |  | 
| 370   this.url = url; |  | 
| 371 }; |  | 
| 372 Downloadable.prototype = |  | 
| 373 { | 361 { | 
| 374   /** | 362   /** | 
| 375    * URL that has to be requested for the object. | 363    * Creates an object that can be downloaded by the downloader. | 
| 376    * @type {string} | 364    * @param {string} url  URL that has to be requested for the object | 
| 377    */ | 365    */ | 
| 378   url: null, | 366   constructor(url) | 
|  | 367   { | 
|  | 368     /** | 
|  | 369      * URL that the download was redirected to if any. | 
|  | 370      * @type {string?} | 
|  | 371      */ | 
|  | 372     this.redirectURL = null; | 
| 379 | 373 | 
| 380   /** | 374     /** | 
| 381    * URL that the download was redirected to if any. | 375      * Time of last download error or 0 if the last download was successful. | 
| 382    * @type {string} | 376      * @type {number} | 
| 383    */ | 377      */ | 
| 384   redirectURL: null, | 378     this.lastError = 0; | 
| 385 | 379 | 
| 386   /** | 380     /** | 
| 387    * Time of last download error or 0 if the last download was successful. | 381      * Time of last check whether the object needs downloading. | 
| 388    * @type {number} | 382      * @type {number} | 
| 389    */ | 383      */ | 
| 390   lastError: 0, | 384     this.lastCheck = 0; | 
| 391 | 385 | 
| 392   /** | 386     /** | 
| 393    * Time of last check whether the object needs downloading. | 387      * Object version corresponding to the last successful download. | 
| 394    * @type {number} | 388      * @type {number} | 
| 395    */ | 389      */ | 
| 396   lastCheck: 0, | 390     this.lastVersion = 0; | 
| 397 | 391 | 
| 398   /** | 392     /** | 
| 399    * Object version corresponding to the last successful download. | 393      * Soft expiration interval; will increase if no checks are performed for a | 
| 400    * @type {number} | 394      * while. | 
| 401    */ | 395      * @type {number} | 
| 402   lastVersion: 0, | 396      */ | 
|  | 397     this.softExpiration = 0; | 
| 403 | 398 | 
| 404   /** | 399     /** | 
| 405    * Soft expiration interval, will increase if no checks are performed for a | 400      * Hard expiration interval; this is fixed. | 
| 406    * while. | 401      * @type {number} | 
| 407    * @type {number} | 402      */ | 
| 408    */ | 403     this.hardExpiration = 0; | 
| 409   softExpiration: 0, |  | 
| 410 | 404 | 
| 411   /** | 405     /** | 
| 412    * Hard expiration interval, this is fixed. | 406      * Number indicating how often the object was downloaded. | 
| 413    * @type {number} | 407      * @type {number} | 
| 414    */ | 408      */ | 
| 415   hardExpiration: 0, | 409     this.downloadCount = 0; | 
| 416 | 410 | 
| 417   /** | 411     /** | 
| 418    * Number indicating how often the object was downloaded. | 412      * URL that has to be requested for the object. | 
| 419    * @type {number} | 413      * @type {string} | 
| 420    */ | 414      */ | 
| 421   downloadCount: 0 | 415     this.url = url; | 
| 422 }; | 416   } | 
|  | 417 } | 
|  | 418 | 
|  | 419 exports.Downloadable = Downloadable; | 
| OLD | NEW | 
|---|