OLD | NEW |
1 # This file is part of Adblock Plus <https://adblockplus.org/>, | 1 # This file is part of Adblock Plus <https://adblockplus.org/>, |
2 # Copyright (C) 2006-2016 Eyeo GmbH | 2 # Copyright (C) 2006-2016 Eyeo GmbH |
3 # | 3 # |
4 # Adblock Plus is free software: you can redistribute it and/or modify | 4 # Adblock Plus is free software: you can redistribute it and/or modify |
5 # it under the terms of the GNU General Public License version 3 as | 5 # it under the terms of the GNU General Public License version 3 as |
6 # published by the Free Software Foundation. | 6 # published by the Free Software Foundation. |
7 # | 7 # |
8 # Adblock Plus is distributed in the hope that it will be useful, | 8 # Adblock Plus is distributed in the hope that it will be useful, |
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of | 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
52 | 52 |
53 :returns: a tuple `(lines_iterator, inherited_source)` where | 53 :returns: a tuple `(lines_iterator, inherited_source)` where |
54 `inherited_source` is the default source to use for included | 54 `inherited_source` is the default source to use for included |
55 fragments. | 55 fragments. |
56 """ | 56 """ |
57 if ':' in name: | 57 if ':' in name: |
58 source_name, name_in_source = name.split(':', 1) | 58 source_name, name_in_source = name.split(':', 1) |
59 try: | 59 try: |
60 source = sources[source_name] | 60 source = sources[source_name] |
61 except KeyError: | 61 except KeyError: |
62 raise IncludeError('Unknown source: \'{}\''.format(source_name), | 62 raise IncludeError("Unknown source: '{}'".format(source_name), |
63 include_stack) | 63 include_stack) |
64 else: | 64 else: |
65 source, name_in_source = default_source, name | 65 source, name_in_source = default_source, name |
66 | 66 |
67 if source is None: | 67 if source is None: |
68 raise IncludeError('Source name is absent in: \'{}\''.format(name), | 68 raise IncludeError("Source name is absent in: '{}'".format(name), |
69 include_stack) | 69 include_stack) |
70 | 70 |
71 return (parse_filterlist(source.get(name_in_source)), | 71 return (parse_filterlist(source.get(name_in_source)), |
72 source if source.is_inheritable else None) | 72 source if source.is_inheritable else None) |
73 | 73 |
74 | 74 |
75 def _process_includes(sources, default_source, parent_include_stack, lines): | 75 def _process_includes(sources, default_source, parent_include_stack, lines): |
76 """Replace include instructions with the lines of included fragment.""" | 76 """Replace include instructions with the lines of included fragment.""" |
77 for line in lines: | 77 for line in lines: |
78 if line.type == 'include': | 78 if line.type == 'include': |
(...skipping 15 matching lines...) Expand all Loading... |
94 except (NotFound, ValueError) as exc: | 94 except (NotFound, ValueError) as exc: |
95 raise IncludeError(exc, include_stack) | 95 raise IncludeError(exc, include_stack) |
96 else: | 96 else: |
97 yield line | 97 yield line |
98 | 98 |
99 | 99 |
100 def _process_timestamps(lines): | 100 def _process_timestamps(lines): |
101 """Convert timestamp markers into actual timestamps.""" | 101 """Convert timestamp markers into actual timestamps.""" |
102 for line in lines: | 102 for line in lines: |
103 if line.type == 'comment' and '%timestamp%' in line.text: | 103 if line.type == 'comment' and '%timestamp%' in line.text: |
104 timestamp = time.strftime("%d %b %Y %H:%M UTC", time.gmtime()) | 104 timestamp = time.strftime('%d %b %Y %H:%M UTC', time.gmtime()) |
105 yield Comment(text=line.text.replace('%timestamp%', timestamp)) | 105 yield Comment(text=line.text.replace('%timestamp%', timestamp)) |
106 else: | 106 else: |
107 yield line | 107 yield line |
108 | 108 |
109 | 109 |
110 def _first_and_rest(iterable): | 110 def _first_and_rest(iterable): |
111 """Return the first item from the iterable and the rest as an iterator.""" | 111 """Return the first item from the iterable and the rest as an iterator.""" |
112 iterator = iter(iterable) | 112 iterator = iter(iterable) |
113 first_item = next(iterator) | 113 first_item = next(iterator) |
114 return first_item, iterator | 114 return first_item, iterator |
115 | 115 |
116 | 116 |
117 def _insert_version(lines): | 117 def _insert_version(lines): |
118 """Insert metadata comment with version (a.k.a. date).""" | 118 """Insert metadata comment with version (a.k.a. date).""" |
119 first_line, rest = _first_and_rest(lines) | 119 first_line, rest = _first_and_rest(lines) |
120 version = Metadata('Version', time.strftime("%Y%m%d%H%M", time.gmtime())) | 120 version = Metadata('Version', time.strftime('%Y%m%d%H%M', time.gmtime())) |
121 return itertools.chain([first_line, version], rest) | 121 return itertools.chain([first_line, version], rest) |
122 | 122 |
123 | 123 |
124 def _remove_duplicates(lines): | 124 def _remove_duplicates(lines): |
125 """Remove duplicate metadata and headers.""" | 125 """Remove duplicate metadata and headers.""" |
126 # Always remove checksum -- a checksum coming from a fragment | 126 # Always remove checksum -- a checksum coming from a fragment |
127 # will not match for the rendered list. | 127 # will not match for the rendered list. |
128 seen = {'Checksum'} | 128 seen = {'Checksum'} |
129 for i, line in enumerate(lines): | 129 for i, line in enumerate(lines): |
130 if line.type == 'metadata': | 130 if line.type == 'metadata': |
(...skipping 13 matching lines...) Expand all Loading... |
144 See https://adblockplus.org/filters#special-comments for description | 144 See https://adblockplus.org/filters#special-comments for description |
145 of the checksum algorithm. | 145 of the checksum algorithm. |
146 """ | 146 """ |
147 md5sum = hashlib.md5() | 147 md5sum = hashlib.md5() |
148 | 148 |
149 for line in lines: | 149 for line in lines: |
150 if line.type != 'emptyline': | 150 if line.type != 'emptyline': |
151 md5sum.update(line.to_string().encode('utf-8') + b'\n') | 151 md5sum.update(line.to_string().encode('utf-8') + b'\n') |
152 yield line | 152 yield line |
153 | 153 |
154 sum = base64.b64encode(md5sum.digest()).rstrip(b'=') | 154 checksum = base64.b64encode(md5sum.digest()).rstrip(b'=') |
155 yield Metadata('Checksum', sum.decode('utf-8')) | 155 yield Metadata('Checksum', checksum.decode('utf-8')) |
156 | 156 |
157 | 157 |
158 def _validate(lines): | 158 def _validate(lines): |
159 """Validate the final list.""" | 159 """Validate the final list.""" |
160 first_line, rest = _first_and_rest(lines) | 160 first_line, rest = _first_and_rest(lines) |
161 if first_line.type != 'header': | 161 if first_line.type != 'header': |
162 raise MissingHeader('No header found at the beginning of the input.') | 162 raise MissingHeader('No header found at the beginning of the input.') |
163 return itertools.chain([first_line], rest) | 163 return itertools.chain([first_line], rest) |
164 | 164 |
165 | 165 |
(...skipping 10 matching lines...) Expand all Loading... |
176 :raises MissingHeader: If the top level fragment doesn't start with a valid | 176 :raises MissingHeader: If the top level fragment doesn't start with a valid |
177 header. | 177 header. |
178 """ | 178 """ |
179 _logger.info('Rendering: %s', name) | 179 _logger.info('Rendering: %s', name) |
180 lines, default_source = _get_and_parse_fragment(name, sources, top_source) | 180 lines, default_source = _get_and_parse_fragment(name, sources, top_source) |
181 lines = _process_includes(sources, default_source, [name], lines) | 181 lines = _process_includes(sources, default_source, [name], lines) |
182 for proc in [_process_timestamps, _insert_version, _remove_duplicates, | 182 for proc in [_process_timestamps, _insert_version, _remove_duplicates, |
183 _insert_checksum, _validate]: | 183 _insert_checksum, _validate]: |
184 lines = proc(lines) | 184 lines = proc(lines) |
185 return lines | 185 return lines |
OLD | NEW |