| OLD | NEW |
| 1 # This file is part of Adblock Plus <https://adblockplus.org/>, | 1 # This file is part of Adblock Plus <https://adblockplus.org/>, |
| 2 # Copyright (C) 2006-2016 Eyeo GmbH | 2 # Copyright (C) 2006-2016 Eyeo GmbH |
| 3 # | 3 # |
| 4 # Adblock Plus is free software: you can redistribute it and/or modify | 4 # Adblock Plus is free software: you can redistribute it and/or modify |
| 5 # it under the terms of the GNU General Public License version 3 as | 5 # it under the terms of the GNU General Public License version 3 as |
| 6 # published by the Free Software Foundation. | 6 # published by the Free Software Foundation. |
| 7 # | 7 # |
| 8 # Adblock Plus is distributed in the hope that it will be useful, | 8 # Adblock Plus is distributed in the hope that it will be useful, |
| 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of | 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| (...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 52 | 52 |
| 53 :returns: a tuple `(lines_iterator, inherited_source)` where | 53 :returns: a tuple `(lines_iterator, inherited_source)` where |
| 54 `inherited_source` is the default source to use for included | 54 `inherited_source` is the default source to use for included |
| 55 fragments. | 55 fragments. |
| 56 """ | 56 """ |
| 57 if ':' in name: | 57 if ':' in name: |
| 58 source_name, name_in_source = name.split(':', 1) | 58 source_name, name_in_source = name.split(':', 1) |
| 59 try: | 59 try: |
| 60 source = sources[source_name] | 60 source = sources[source_name] |
| 61 except KeyError: | 61 except KeyError: |
| 62 raise IncludeError('Unknown source: \'{}\''.format(source_name), | 62 raise IncludeError("Unknown source: '{}'".format(source_name), |
| 63 include_stack) | 63 include_stack) |
| 64 else: | 64 else: |
| 65 source, name_in_source = default_source, name | 65 source, name_in_source = default_source, name |
| 66 | 66 |
| 67 if source is None: | 67 if source is None: |
| 68 raise IncludeError('Source name is absent in: \'{}\''.format(name), | 68 raise IncludeError("Source name is absent in: '{}'".format(name), |
| 69 include_stack) | 69 include_stack) |
| 70 | 70 |
| 71 return (parse_filterlist(source.get(name_in_source)), | 71 return (parse_filterlist(source.get(name_in_source)), |
| 72 source if source.is_inheritable else None) | 72 source if source.is_inheritable else None) |
| 73 | 73 |
| 74 | 74 |
| 75 def _process_includes(sources, default_source, parent_include_stack, lines): | 75 def _process_includes(sources, default_source, parent_include_stack, lines): |
| 76 """Replace include instructions with the lines of included fragment.""" | 76 """Replace include instructions with the lines of included fragment.""" |
| 77 for line in lines: | 77 for line in lines: |
| 78 if line.type == 'include': | 78 if line.type == 'include': |
| (...skipping 15 matching lines...) Expand all Loading... |
| 94 except (NotFound, ValueError) as exc: | 94 except (NotFound, ValueError) as exc: |
| 95 raise IncludeError(exc, include_stack) | 95 raise IncludeError(exc, include_stack) |
| 96 else: | 96 else: |
| 97 yield line | 97 yield line |
| 98 | 98 |
| 99 | 99 |
| 100 def _process_timestamps(lines): | 100 def _process_timestamps(lines): |
| 101 """Convert timestamp markers into actual timestamps.""" | 101 """Convert timestamp markers into actual timestamps.""" |
| 102 for line in lines: | 102 for line in lines: |
| 103 if line.type == 'comment' and '%timestamp%' in line.text: | 103 if line.type == 'comment' and '%timestamp%' in line.text: |
| 104 timestamp = time.strftime("%d %b %Y %H:%M UTC", time.gmtime()) | 104 timestamp = time.strftime('%d %b %Y %H:%M UTC', time.gmtime()) |
| 105 yield Comment(text=line.text.replace('%timestamp%', timestamp)) | 105 yield Comment(text=line.text.replace('%timestamp%', timestamp)) |
| 106 else: | 106 else: |
| 107 yield line | 107 yield line |
| 108 | 108 |
| 109 | 109 |
| 110 def _first_and_rest(iterable): | 110 def _first_and_rest(iterable): |
| 111 """Return the first item from the iterable and the rest as an iterator.""" | 111 """Return the first item from the iterable and the rest as an iterator.""" |
| 112 iterator = iter(iterable) | 112 iterator = iter(iterable) |
| 113 first_item = next(iterator) | 113 first_item = next(iterator) |
| 114 return first_item, iterator | 114 return first_item, iterator |
| 115 | 115 |
| 116 | 116 |
| 117 def _insert_version(lines): | 117 def _insert_version(lines): |
| 118 """Insert metadata comment with version (a.k.a. date).""" | 118 """Insert metadata comment with version (a.k.a. date).""" |
| 119 first_line, rest = _first_and_rest(lines) | 119 first_line, rest = _first_and_rest(lines) |
| 120 version = Metadata('Version', time.strftime("%Y%m%d%H%M", time.gmtime())) | 120 version = Metadata('Version', time.strftime('%Y%m%d%H%M', time.gmtime())) |
| 121 return itertools.chain([first_line, version], rest) | 121 return itertools.chain([first_line, version], rest) |
| 122 | 122 |
| 123 | 123 |
| 124 def _remove_duplicates(lines): | 124 def _remove_duplicates(lines): |
| 125 """Remove duplicate metadata and headers.""" | 125 """Remove duplicate metadata and headers.""" |
| 126 # Always remove checksum -- a checksum coming from a fragment | 126 # Always remove checksum -- a checksum coming from a fragment |
| 127 # will not match for the rendered list. | 127 # will not match for the rendered list. |
| 128 seen = {'Checksum'} | 128 seen = {'Checksum'} |
| 129 for i, line in enumerate(lines): | 129 for i, line in enumerate(lines): |
| 130 if line.type == 'metadata': | 130 if line.type == 'metadata': |
| (...skipping 13 matching lines...) Expand all Loading... |
| 144 See https://adblockplus.org/filters#special-comments for description | 144 See https://adblockplus.org/filters#special-comments for description |
| 145 of the checksum algorithm. | 145 of the checksum algorithm. |
| 146 """ | 146 """ |
| 147 md5sum = hashlib.md5() | 147 md5sum = hashlib.md5() |
| 148 | 148 |
| 149 for line in lines: | 149 for line in lines: |
| 150 if line.type != 'emptyline': | 150 if line.type != 'emptyline': |
| 151 md5sum.update(line.to_string().encode('utf-8') + b'\n') | 151 md5sum.update(line.to_string().encode('utf-8') + b'\n') |
| 152 yield line | 152 yield line |
| 153 | 153 |
| 154 sum = base64.b64encode(md5sum.digest()).rstrip(b'=') | 154 checksum = base64.b64encode(md5sum.digest()).rstrip(b'=') |
| 155 yield Metadata('Checksum', sum.decode('utf-8')) | 155 yield Metadata('Checksum', checksum.decode('utf-8')) |
| 156 | 156 |
| 157 | 157 |
| 158 def _validate(lines): | 158 def _validate(lines): |
| 159 """Validate the final list.""" | 159 """Validate the final list.""" |
| 160 first_line, rest = _first_and_rest(lines) | 160 first_line, rest = _first_and_rest(lines) |
| 161 if first_line.type != 'header': | 161 if first_line.type != 'header': |
| 162 raise MissingHeader('No header found at the beginning of the input.') | 162 raise MissingHeader('No header found at the beginning of the input.') |
| 163 return itertools.chain([first_line], rest) | 163 return itertools.chain([first_line], rest) |
| 164 | 164 |
| 165 | 165 |
| (...skipping 10 matching lines...) Expand all Loading... |
| 176 :raises MissingHeader: If the top level fragment doesn't start with a valid | 176 :raises MissingHeader: If the top level fragment doesn't start with a valid |
| 177 header. | 177 header. |
| 178 """ | 178 """ |
| 179 _logger.info('Rendering: %s', name) | 179 _logger.info('Rendering: %s', name) |
| 180 lines, default_source = _get_and_parse_fragment(name, sources, top_source) | 180 lines, default_source = _get_and_parse_fragment(name, sources, top_source) |
| 181 lines = _process_includes(sources, default_source, [name], lines) | 181 lines = _process_includes(sources, default_source, [name], lines) |
| 182 for proc in [_process_timestamps, _insert_version, _remove_duplicates, | 182 for proc in [_process_timestamps, _insert_version, _remove_duplicates, |
| 183 _insert_checksum, _validate]: | 183 _insert_checksum, _validate]: |
| 184 lines = proc(lines) | 184 lines = proc(lines) |
| 185 return lines | 185 return lines |
| OLD | NEW |