Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: lib/storage.js

Issue 9615013: Crawler, first version (Closed)
Patch Set: Created March 6, 2013, 4:05 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« .hgignore ('K') | « lib/logger.js ('k') | lib/task.js » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 Cu.import("resource://gre/modules/FileUtils.jsm"); 1 let {Encoding} = require( "encoding" );
2 2 let {Logger} = require( "logger" );
3 let outputStream; 3
4 let converterOutputStream; 4 Cu.import( "resource://gre/modules/FileUtils.jsm" );
5 5
6 function createTemporaryFile(name) 6 //-------------------------------------------------------
7 { 7 // Storage_class
8 let file = FileUtils.getFile("TmpD", [name]); 8 //-------------------------------------------------------
9 file.createUnique(Ci.nsIFile.NORMAL_FILE_TYPE, FileUtils.PERMS_FILE); 9 /**
10 return file; 10 * Base class for particular storage mechanisms. Implements all storage framewor k methods. Necessary methods have
11 } 11 * implementations that throw if not overridden. Optional methods have no-op imp lementations.
12 12 * @constructor
13 function openOutputStream(file) 13 */
14 { 14 var Storage_class = function()
15 let outputStream = FileUtils.openSafeFileOutputStream(file); 15 {
16 let converterOutputStream = Cc["@mozilla.org/intl/converter-output-stream;1"] 16 };
17 .createInstance(Ci.nsIConverterOutputStream); 17
18 converterOutputStream.init(outputStream, "UTF-8", 0, 0); 18 /**
19 return [outputStream, converterOutputStream]; 19 * Return a string of the form "[object ...]" like the built-in types do.
20 } 20 * <p/>
21 21 * Mandatory framework function.
22 let Storage = exports.Storage = {}; 22 */
23 23 Storage_class.prototype.toJSON = function()
24 Storage.init = function() 24 {
25 { 25 throw new Error( "Subclass of Storage_class did not implement 'toJSON()'." ) ;
26 Storage.dataFile = createTemporaryFile("crawler-data"); 26 };
27 [outputStream, converterOutputStream] = openOutputStream(Storage.dataFile); 27
28 }; 28 /**
29 29 * Called when the crawl first begins, before any targets have been loaded.
30 Storage.write = function(data) 30 * <p/>
31 { 31 * Mandatory framework function.
32 let line = JSON.stringify(data) + "\n"; 32 */
33 converterOutputStream.writeString(line); 33 Storage_class.prototype.write = function()
34 }; 34 {
35 35 throw new Error( "Subclass of Storage_class did not implement 'write()'." );
36 Storage.finish = function() 36 };
37 { 37
38 converterOutputStream.flush(); 38 /**
39 FileUtils.closeSafeFileOutputStream(outputStream); 39 * Called when the crawl ends, whether normally or abnormally.
40 }; 40 */
41 41 Storage_class.prototype.close = function()
42 Storage.destroy = function() 42 {
43 { 43 };
44 Storage.dataFile.remove(true); 44
45 }; 45 Storage_class.prototype.writer = function()
46 {
47 return this.write.bind( this );
48 };
49
50 //-------------------------------------------------------
51 // Storage.Local_File
52 //-------------------------------------------------------
53 /**
54 * Storage to the local file system.
55 * <p>
56 * This class writes directly to the given file. It does not write to a temporar y file first, say, using 'createUnique',
57 * and then renaming it.
58 *
59 * @param {nsIFile} file
60 * File object to which to write.
61 * @constructor
62 */
63 var Local_File = function( file )
64 {
65 /**
66 * @type {nsIFile}
67 */
68 this.file = file;
69
70 this.output = FileUtils.openSafeFileOutputStream( file );
71 this.converter_output = Cc["@mozilla.org/intl/converter-output-stream;1"]
72 .createInstance( Ci.nsIConverterOutputStream );
73 this.converter_output.init( this.output, "UTF-8", 0, 0 );
74 };
75 Local_File.prototype = new Storage_class();
76
77 Local_File.prototype.toJSON = function()
78 {
79 return "local file " + this.file.path;
80 };
81
82 Local_File.prototype.close = function()
83 {
84 this.converter_output.flush();
85 FileUtils.closeSafeFileOutputStream( this.output );
86 };
87
88 Local_File.prototype.write = function( s )
89 {
90 this.converter_output.writeString( s );
91 };
92
93 //-------------------------------------------------------
94 // Storage.Bit_Bucket
95 //-------------------------------------------------------
96
97 var Bit_Bucket = function()
98 {
99 };
100 Bit_Bucket.prototype = new Storage_class();
101
102 /**
103 * The only way that this differs from the do-nothing base class is that we prov ide a name.
104 */
105 Bit_Bucket.prototype.toJSON = function()
106 {
107 return "bit bucket";
108 };
109 Bit_Bucket.prototype.write = function()
110 {
111 };
112
113 //-------------------------------------------------------
114 // Storage.Multiple
115 //-------------------------------------------------------
116 /**
117 * Combine one or more Storage_class objects into a single one. This is ordinari ly used for logging activity to the
118 * screen during the run. It also allows enables summary statistics to be gather ed simultaneously with a crawl to
119 * support filter list construction.
120 * @constructor
121 * @param {Array} stores
122 * An array of Storage_class objects.
123 * @param {boolean} [omit_first]
124 * Suppress first element in JSON display.
125 */
126 var Multiple = function( stores, omit_first )
127 {
128 this.stores = stores;
129 this.omit_first = omit_first;
130 };
131 Multiple.prototype = new Storage_class();
132
133 /**
134 * @override
135 */
136 Multiple.prototype.toJSON = function()
137 {
138 if ( this.omit_first && this.stores.length == 2 )
139 {
140 /*
141 * Special case: Pretend we're not here.
142 */
143 return this.stores[ 1 ].toJSON();
144 }
145 var s = "Multiple[";
146 var first = (this.omit_first) ? 1 : 0;
147 if ( this.first < this.stores.length )
148 {
149 for ( let i = first ; i < this.stores.length ; ++i )
150 {
151 if ( i > first ) s += ",";
152 s += " " + JSON.stringify( this.stores[i] );
153 }
154 s += " ";
155 }
156 s += "]";
157 return s;
158 };
159
160 /**
161 * @override
162 */
163 Multiple.prototype.write = function( s )
164 {
165 for ( let i = 0 ; i < this.stores.length ; ++i )
166 {
167 this.stores[i].write( s );
168 }
169 };
170
171 //-------------------------------------------------------
172 // Storage.Display_Log
173 //-------------------------------------------------------
174 /**
175 * A display log in the crawler UI, presumably a simple textbox. Used to monitor progress during a crawl, if desired.
176 * @constructor
177 * @param [display]
178 * A display object from the crawler UI.
179 */
180 var Display_Log = function( display )
181 {
182 this.display = display;
183 };
184 Display_Log.prototype = new Storage_class();
185
186 /**
187 * @override
188 */
189 Display_Log.prototype.toJSON = function()
190 {
191 return "Display_Log";
192 };
193
194 /**
195 * @override
196 */
197 Display_Log.prototype.write = function( s )
198 {
199 this.display.write( s );
200 };
201
202 //-------------------------------------------------------
203 // Storage
204 //-------------------------------------------------------
205 /**
206 * Export variable for this module. There's no single class named "Storage" as s uch.
207 */
208 exports.Storage = {
209 Local_File: Local_File,
210 Bit_Bucket: Bit_Bucket,
211 Multiple: Multiple,
212 Display_Log: Display_Log
213 };
OLDNEW
« .hgignore ('K') | « lib/logger.js ('k') | lib/task.js » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld