Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: lib/filterStorage.js

Issue 29408742: Issue 5059 - Simplify I/O API and FilterStorage implementation (Closed) Base URL: https://hg.adblockplus.org/adblockpluscore
Patch Set: Replaced inner function by an arrow function Created April 11, 2017, 3:11 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | test/filterStorage_readwrite.js » ('j') | test/stub-modules/io.js » ('J')
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 /* 1 /*
2 * This file is part of Adblock Plus <https://adblockplus.org/>, 2 * This file is part of Adblock Plus <https://adblockplus.org/>,
3 * Copyright (C) 2006-2017 eyeo GmbH 3 * Copyright (C) 2006-2017 eyeo GmbH
4 * 4 *
5 * Adblock Plus is free software: you can redistribute it and/or modify 5 * Adblock Plus is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 3 as 6 * it under the terms of the GNU General Public License version 3 as
7 * published by the Free Software Foundation. 7 * published by the Free Software Foundation.
8 * 8 *
9 * Adblock Plus is distributed in the hope that it will be useful, 9 * Adblock Plus is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 15 * along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
16 */ 16 */
17 17
18 "use strict"; 18 "use strict";
19 19
20 /** 20 /**
21 * @fileOverview FilterStorage class responsible for managing user's 21 * @fileOverview FilterStorage class responsible for managing user's
22 * subscriptions and filters. 22 * subscriptions and filters.
23 */ 23 */
24 24
25 const {FileUtils} = Cu.import("resource://gre/modules/FileUtils.jsm", {});
26 const {Services} = Cu.import("resource://gre/modules/Services.jsm", {});
27
28 const {IO} = require("io"); 25 const {IO} = require("io");
29 const {Prefs} = require("prefs"); 26 const {Prefs} = require("prefs");
30 const {Filter, ActiveFilter} = require("filterClasses"); 27 const {Filter, ActiveFilter} = require("filterClasses");
31 const {Subscription, SpecialSubscription, 28 const {Subscription, SpecialSubscription,
32 ExternalSubscription} = require("subscriptionClasses"); 29 ExternalSubscription} = require("subscriptionClasses");
33 const {FilterNotifier} = require("filterNotifier"); 30 const {FilterNotifier} = require("filterNotifier");
34 const {Utils} = require("utils");
35 31
36 /** 32 /**
37 * Version number of the filter storage file format. 33 * Version number of the filter storage file format.
38 * @type {number} 34 * @type {number}
39 */ 35 */
40 let formatVersion = 4; 36 let formatVersion = 4;
41 37
42 /** 38 /**
43 * This class reads user's filters from disk, manages them in memory 39 * This class reads user's filters from disk, manages them in memory
44 * and writes them back. 40 * and writes them back.
(...skipping 10 matching lines...) Expand all
55 /** 51 /**
56 * Version number of the patterns.ini format used. 52 * Version number of the patterns.ini format used.
57 * @type {number} 53 * @type {number}
58 */ 54 */
59 get formatVersion() 55 get formatVersion()
60 { 56 {
61 return formatVersion; 57 return formatVersion;
62 }, 58 },
63 59
64 /** 60 /**
65 * File that the filter list has been loaded from and should be saved to 61 * File containing the filter list
66 * @type {nsIFile} 62 * @type {string}
67 */ 63 */
68 get sourceFile() 64 get sourceFile()
69 { 65 {
70 let file = null; 66 return "patterns.ini";
71 if (Prefs.patternsfile)
72 {
73 // Override in place, use it instead of placing the file in the
74 // regular data dir
75 file = IO.resolveFilePath(Prefs.patternsfile);
76 }
77 if (!file)
78 {
79 // Place the file in the data dir
80 file = IO.resolveFilePath(Prefs.data_directory);
81 if (file)
82 file.append("patterns.ini");
83 }
84 if (!file)
85 {
86 // Data directory pref misconfigured? Try the default value
87 try
88 {
89 let dir = Services.prefs.getDefaultBranch("extensions.adblockplus.")
90 .getCharPref("data_directory");
91 file = IO.resolveFilePath(dir);
92 if (file)
93 file.append("patterns.ini");
94 }
95 catch (e) {}
96 }
97
98 if (!file)
99 {
100 Cu.reportError("Adblock Plus: Failed to resolve filter file location " +
101 "from extensions.adblockplus.patternsfile preference");
102 }
103
104 // Property is configurable because of the test suite.
105 Object.defineProperty(this, "sourceFile",
106 {value: file, configurable: true});
107 return file;
108 }, 67 },
109 68
110 /** 69 /**
111 * Will be set to true if no patterns.ini file exists. 70 * Will be set to true if no patterns.ini file exists.
112 * @type {boolean} 71 * @type {boolean}
113 */ 72 */
114 firstRun: false, 73 firstRun: false,
115 74
116 /** 75 /**
117 * Map of properties listed in the filter storage file before the sections 76 * Map of properties listed in the filter storage file before the sections
(...skipping 284 matching lines...) Expand 10 before | Expand all | Expand 10 after
402 } 361 }
403 }, 362 },
404 363
405 /** 364 /**
406 * @callback TextSink 365 * @callback TextSink
407 * @param {string?} line 366 * @param {string?} line
408 */ 367 */
409 368
410 /** 369 /**
411 * Allows importing previously serialized filter data. 370 * Allows importing previously serialized filter data.
371 * @param {boolean} silent
372 * If true, no "load" notification will be sent out.
412 * @return {TextSink} 373 * @return {TextSink}
413 * Function to be called for each line of data. Calling it with null as 374 * Function to be called for each line of data. Calling it with null as
414 * parameter finalizes the import and replaces existing data. No changes 375 * parameter finalizes the import and replaces existing data. No changes
415 * will be applied before finalization, so import can be "aborted" by 376 * will be applied before finalization, so import can be "aborted" by
416 * forgetting this callback. 377 * forgetting this callback.
417 */ 378 */
418 importData() 379 importData(silent)
419 { 380 {
420 let parser = new INIParser(); 381 let parser = new INIParser();
421 return line => 382 return line =>
422 { 383 {
423 parser.process(line); 384 parser.process(line);
424 if (line === null) 385 if (line === null)
425 { 386 {
426 // Old special groups might have been converted, remove them if 387 // Old special groups might have been converted, remove them if
427 // they are empty 388 // they are empty
428 let specialMap = new Set(["~il~", "~wl~", "~fl~", "~eh~"]); 389 let specialMap = new Set(["~il~", "~wl~", "~fl~", "~eh~"]);
(...skipping 16 matching lines...) Expand all
445 this.knownSubscriptions = knownSubscriptions; 406 this.knownSubscriptions = knownSubscriptions;
446 Filter.knownFilters = parser.knownFilters; 407 Filter.knownFilters = parser.knownFilters;
447 Subscription.knownSubscriptions = parser.knownSubscriptions; 408 Subscription.knownSubscriptions = parser.knownSubscriptions;
448 409
449 if (parser.userFilters) 410 if (parser.userFilters)
450 { 411 {
451 for (let filter of parser.userFilters) 412 for (let filter of parser.userFilters)
452 this.addFilter(Filter.fromText(filter), null, undefined, true); 413 this.addFilter(Filter.fromText(filter), null, undefined, true);
453 } 414 }
454 415
455 FilterNotifier.triggerListeners("load"); 416 if (!silent)
417 FilterNotifier.triggerListeners("load");
456 } 418 }
457 }; 419 };
458 }, 420 },
459 421
460 /** 422 /**
461 * Loads all subscriptions from the disk 423 * Loads all subscriptions from the disk.
424 * @return {Promise} promise resolved or rejected when loading is complete
462 */ 425 */
463 loadFromDisk() 426 loadFromDisk()
464 { 427 {
465 let readFile = () =>
466 {
467 let parser = {
468 process: this.importData()
469 };
470 IO.readFromFile(this.sourceFile, parser, readFromFileException =>
471 {
472 this.initialized = true;
473
474 if (!readFromFileException && this.subscriptions.length == 0)
475 {
476 // No filter subscriptions in the file, this isn't right.
477 readFromFileException = new Error("No data in the file");
478 }
479
480 if (readFromFileException)
481 Cu.reportError(readFromFileException);
482
483 if (readFromFileException)
484 tryBackup(1);
485 });
486 };
487
488 let tryBackup = backupIndex => 428 let tryBackup = backupIndex =>
489 { 429 {
490 this.restoreBackup(backupIndex).then(() => 430 return this.restoreBackup(backupIndex, true).then(() =>
491 { 431 {
492 if (this.subscriptions.length == 0) 432 if (this.subscriptions.length == 0)
493 tryBackup(backupIndex + 1); 433 return tryBackup(backupIndex + 1);
494 }).catch(error => 434 }).catch(error =>
495 { 435 {
496 // Give up 436 // Give up
497 }); 437 });
498 }; 438 };
499 439
500 IO.statFile(this.sourceFile, (statError, statData) => 440 return IO.statFile(this.sourceFile).then(statData =>
501 { 441 {
502 if (statError || !statData.exists) 442 if (!statData.exists)
503 { 443 {
504 this.firstRun = true; 444 this.firstRun = true;
505 this.initialized = true; 445 return;
506 FilterNotifier.triggerListeners("load");
507 } 446 }
508 else 447
509 readFile(); 448 let parser = this.importData(true);
449 return IO.readFromFile(this.sourceFile, parser).then(() =>
450 {
451 parser(null);
452 if (this.subscriptions.length == 0)
453 {
454 // No filter subscriptions in the file, this isn't right.
455 throw new Error("No data in the file");
456 }
457 });
458 }).catch(error =>
459 {
460 Cu.reportError(error);
461 return tryBackup(1);
kzar 2017/04/20 06:49:25 So we return a promise here, which might recursive
Wladimir Palant 2017/04/20 07:17:06 Yes, this is the intended behavior - we already re
462 }).then(() =>
463 {
464 this.initialized = true;
465 FilterNotifier.triggerListeners("load");
510 }); 466 });
511 }, 467 },
512 468
513 /** 469 /**
470 * Constructs the file name for a patterns.ini backup.
471 * @param {number} backupIndex
472 * number of the backup file (1 being the most recent)
473 * @return {string} backup file name
474 */
475 getBackupName(backupIndex)
476 {
477 let [name, extension] = this.sourceFile.split(".", 2);
478 return (name + "-backup" + backupIndex + "." + extension);
479 },
480
481 /**
514 * Restores an automatically created backup. 482 * Restores an automatically created backup.
515 * @param {number} backupIndex 483 * @param {number} backupIndex
516 * number of the backup to restore (1 being the most recent) 484 * number of the backup to restore (1 being the most recent)
485 * @param {boolean} silent
486 * If true, no "load" notification will be sent out.
517 * @return {Promise} promise resolved or rejected when restoring is complete 487 * @return {Promise} promise resolved or rejected when restoring is complete
518 */ 488 */
519 restoreBackup(backupIndex) 489 restoreBackup(backupIndex, silent)
520 { 490 {
521 return new Promise((resolve, reject) => 491 let backupFile = this.getBackupName(backupIndex);
492 let parser = this.importData(silent);
493 return IO.readFromFile(backupFile, parser).then(() =>
522 { 494 {
523 // Attempt to load a backup 495 parser(null);
524 let [, part1, part2] = /^(.*)(\.\w+)$/.exec( 496 return this.saveToDisk();
525 this.sourceFile.leafName
526 ) || [null, this.sourceFile.leafName, ""];
527
528 let backupFile = this.sourceFile.clone();
529 backupFile.leafName = (part1 + "-backup" + backupIndex + part2);
530
531 let parser = {
532 process: this.importData()
533 };
534 IO.readFromFile(backupFile, parser, error =>
535 {
536 if (error)
537 reject(error);
538 else
539 {
540 this.saveToDisk();
541 resolve();
542 }
543 });
544 }); 497 });
545 }, 498 },
546 499
547 /** 500 /**
548 * Generator serializing filter data and yielding it line by line. 501 * Generator serializing filter data and yielding it line by line.
549 */ 502 */
550 *exportData() 503 *exportData()
551 { 504 {
552 // Do not persist external subscriptions 505 // Do not persist external subscriptions
553 let subscriptions = this.subscriptions.filter( 506 let subscriptions = this.subscriptions.filter(
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after
601 554
602 /** 555 /**
603 * Will be set to true if a saveToDisk() call arrives while saveToDisk() is 556 * Will be set to true if a saveToDisk() call arrives while saveToDisk() is
604 * already running (delayed execution). 557 * already running (delayed execution).
605 * @type {boolean} 558 * @type {boolean}
606 */ 559 */
607 _needsSave: false, 560 _needsSave: false,
608 561
609 /** 562 /**
610 * Saves all subscriptions back to disk 563 * Saves all subscriptions back to disk
564 * @return {Promise} promise resolved or rejected when saving is complete
611 */ 565 */
612 saveToDisk() 566 saveToDisk()
613 { 567 {
614 if (this._saving) 568 if (this._saving)
615 { 569 {
616 this._needsSave = true; 570 this._needsSave = true;
617 return; 571 return;
618 } 572 }
619 573
620 // Make sure the file's parent directory exists
621 let targetFile = this.sourceFile;
622 try
623 {
624 targetFile.parent.create(Ci.nsIFile.DIRECTORY_TYPE,
625 FileUtils.PERMS_DIRECTORY);
626 }
627 catch (e) {}
628
629 let writeFilters = () =>
630 {
631 IO.writeToFile(targetFile, this.exportData(), e =>
632 {
633 this._saving = false;
634
635 if (e)
636 Cu.reportError(e);
637
638 if (this._needsSave)
639 {
640 this._needsSave = false;
641 this.saveToDisk();
642 }
643 else
644 FilterNotifier.triggerListeners("save");
645 });
646 };
647
648 let checkBackupRequired = (callbackNotRequired, callbackRequired) =>
649 {
650 if (Prefs.patternsbackups <= 0)
651 callbackNotRequired();
652 else
653 {
654 IO.statFile(targetFile, (statFileException, statData) =>
655 {
656 if (statFileException || !statData.exists)
657 callbackNotRequired();
658 else
659 {
660 let [, part1, part2] = /^(.*)(\.\w+)$/.exec(targetFile.leafName) ||
661 [null, targetFile.leafName, ""];
662 let newestBackup = targetFile.clone();
663 newestBackup.leafName = part1 + "-backup1" + part2;
664 IO.statFile(
665 newestBackup,
666 (statBackupFileException, statBackupData) =>
667 {
668 if (!statBackupFileException && (!statBackupData.exists ||
669 (Date.now() - statBackupData.lastModified) /
670 3600000 >= Prefs.patternsbackupinterval))
671 {
672 callbackRequired(part1, part2);
673 }
674 else
675 callbackNotRequired();
676 }
677 );
678 }
679 });
680 }
681 };
682
683 let removeLastBackup = (part1, part2) =>
684 {
685 let file = targetFile.clone();
686 file.leafName = part1 + "-backup" + Prefs.patternsbackups + part2;
687 IO.removeFile(
688 file, e => renameBackup(part1, part2, Prefs.patternsbackups - 1)
689 );
690 };
691
692 let renameBackup = (part1, part2, index) =>
693 {
694 if (index > 0)
695 {
696 let fromFile = targetFile.clone();
697 fromFile.leafName = part1 + "-backup" + index + part2;
698
699 let toName = part1 + "-backup" + (index + 1) + part2;
700
701 IO.renameFile(fromFile, toName, e => renameBackup(part1, part2,
702 index - 1));
703 }
704 else
705 {
706 let toFile = targetFile.clone();
707 toFile.leafName = part1 + "-backup" + (index + 1) + part2;
708
709 IO.copyFile(targetFile, toFile, writeFilters);
710 }
711 };
712
713 this._saving = true; 574 this._saving = true;
714 575
715 checkBackupRequired(writeFilters, removeLastBackup); 576 return Promise.resolve().then(() =>
577 {
578 // First check whether we need to create a backup
579 if (Prefs.patternsbackups <= 0)
580 return false;
581
582 return IO.statFile(this.sourceFile).then(statData =>
583 {
584 if (!statData.exists)
585 return false;
586
587 return IO.statFile(this.getBackupName(1)).then(backupStatData =>
588 {
589 if (backupStatData.exists &&
590 (Date.now() - backupStatData.lastModified) / 3600000 <
591 Prefs.patternsbackupinterval)
592 {
593 return false;
594 }
595 return true;
596 });
597 });
598 }).then(backupRequired =>
599 {
600 if (!backupRequired)
601 return;
602
603 let ignoreErrors = error =>
604 {
605 // Expected error, backup file doesn't exist.
606 };
607
608 let renameBackup = index =>
609 {
610 if (index > 0)
611 {
612 return IO.renameFile(this.getBackupName(index),
613 this.getBackupName(index + 1))
614 .catch(ignoreErrors)
615 .then(() => renameBackup(index - 1));
616 }
617
618 return IO.renameFile(this.sourceFile, this.getBackupName(1))
619 .catch(ignoreErrors);
620 };
621
622 // Rename existing files
623 return renameBackup(Prefs.patternsbackups - 1);
624 }).catch(error =>
625 {
626 // Errors during backup creation shouldn't prevent writing filters.
627 Cu.reportError(error);
628 }).then(() =>
629 {
630 return IO.writeToFile(this.sourceFile, this.exportData());
631 }).then(() =>
632 {
633 FilterNotifier.triggerListeners("save");
634 }).catch(error =>
635 {
636 // If saving failed, report error but continue - we still have to process
637 // flags.
638 Cu.reportError(error);
639 }).then(() =>
640 {
641 this._saving = false;
642 if (this._needsSave)
643 {
644 this._needsSave = false;
645 this.saveToDisk();
646 }
647 });
716 }, 648 },
717 649
718 /** 650 /**
719 * @typedef FileInfo 651 * @typedef FileInfo
720 * @type {object} 652 * @type {object}
721 * @property {nsIFile} file 653 * @property {nsIFile} file
722 * @property {number} lastModified 654 * @property {number} lastModified
723 */ 655 */
724 656
725 /** 657 /**
726 * Returns a promise resolving in a list of existing backup files. 658 * Returns a promise resolving in a list of existing backup files.
727 * @return {Promise.<FileInfo[]>} 659 * @return {Promise.<FileInfo[]>}
728 */ 660 */
729 getBackupFiles() 661 getBackupFiles()
730 { 662 {
731 let backups = []; 663 let backups = [];
732 664
733 let [, part1, part2] = /^(.*)(\.\w+)$/.exec( 665 let checkBackupFile = index =>
734 FilterStorage.sourceFile.leafName 666 {
735 ) || [null, FilterStorage.sourceFile.leafName, ""]; 667 return IO.statFile(this.getBackupName(index)).then(statData =>
668 {
669 if (!statData.exists)
670 return backups;
736 671
737 function checkBackupFile(index) 672 backups.push({
738 { 673 index,
739 return new Promise((resolve, reject) => 674 lastModified: statData.lastModified
675 });
676 return checkBackupFile(index + 1);
677 }).catch(error =>
740 { 678 {
741 let file = FilterStorage.sourceFile.clone(); 679 // Something went wrong, return whatever data we got so far.
742 file.leafName = part1 + "-backup" + index + part2; 680 Cu.reportError(error);
743 681 return backups;
744 IO.statFile(file, (error, result) =>
745 {
746 if (!error && result.exists)
747 {
748 backups.push({
749 index,
750 lastModified: result.lastModified
751 });
752 resolve(checkBackupFile(index + 1));
753 }
754 else
755 resolve(backups);
756 });
757 }); 682 });
758 } 683 };
759 684
760 return checkBackupFile(1); 685 return checkBackupFile(1);
761 } 686 }
762 }; 687 };
763 688
764 /** 689 /**
765 * Joins subscription's filters to the subscription without any notifications. 690 * Joins subscription's filters to the subscription without any notifications.
766 * @param {Subscription} subscription 691 * @param {Subscription} subscription
767 * filter subscription that should be connected to its filters 692 * filter subscription that should be connected to its filters
768 */ 693 */
(...skipping 18 matching lines...) Expand all
787 712
788 for (let filter of subscription.filters) 713 for (let filter of subscription.filters)
789 { 714 {
790 let i = filter.subscriptions.indexOf(subscription); 715 let i = filter.subscriptions.indexOf(subscription);
791 if (i >= 0) 716 if (i >= 0)
792 filter.subscriptions.splice(i, 1); 717 filter.subscriptions.splice(i, 1);
793 } 718 }
794 } 719 }
795 720
796 /** 721 /**
797 * IO.readFromFile() listener to parse filter data. 722 * Listener returned by FilterStorage.importData(), parses filter data.
798 * @constructor 723 * @constructor
799 */ 724 */
800 function INIParser() 725 function INIParser()
801 { 726 {
802 this.fileProperties = this.curObj = {}; 727 this.fileProperties = this.curObj = {};
803 this.subscriptions = []; 728 this.subscriptions = [];
804 this.knownFilters = Object.create(null); 729 this.knownFilters = Object.create(null);
805 this.knownSubscriptions = Object.create(null); 730 this.knownSubscriptions = Object.create(null);
806 } 731 }
807 INIParser.prototype = 732 INIParser.prototype =
(...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after
890 } 815 }
891 } 816 }
892 else if (this.wantObj === false && val) 817 else if (this.wantObj === false && val)
893 this.curObj.push(val.replace(/\\\[/g, "[")); 818 this.curObj.push(val.replace(/\\\[/g, "["));
894 } 819 }
895 finally 820 finally
896 { 821 {
897 Filter.knownFilters = origKnownFilters; 822 Filter.knownFilters = origKnownFilters;
898 Subscription.knownSubscriptions = origKnownSubscriptions; 823 Subscription.knownSubscriptions = origKnownSubscriptions;
899 } 824 }
900
901 // Allow events to be processed every now and then.
902 // Note: IO.readFromFile() will deal with the potential reentrance here.
903 this.linesProcessed++;
904 if (this.linesProcessed % 1000 == 0)
905 return Utils.yield();
906 } 825 }
907 }; 826 };
OLDNEW
« no previous file with comments | « no previous file | test/filterStorage_readwrite.js » ('j') | test/stub-modules/io.js » ('J')

Powered by Google App Engine
This is Rietveld