| OLD | NEW | 
|    1 # This file is part of Adblock Plus <https://adblockplus.org/>, |    1 # This file is part of Adblock Plus <https://adblockplus.org/>, | 
|    2 # Copyright (C) 2006-2016 Eyeo GmbH |    2 # Copyright (C) 2006-2016 Eyeo GmbH | 
|    3 # |    3 # | 
|    4 # Adblock Plus is free software: you can redistribute it and/or modify |    4 # Adblock Plus is free software: you can redistribute it and/or modify | 
|    5 # it under the terms of the GNU General Public License version 3 as |    5 # it under the terms of the GNU General Public License version 3 as | 
|    6 # published by the Free Software Foundation. |    6 # published by the Free Software Foundation. | 
|    7 # |    7 # | 
|    8 # Adblock Plus is distributed in the hope that it will be useful, |    8 # Adblock Plus is distributed in the hope that it will be useful, | 
|    9 # but WITHOUT ANY WARRANTY; without even the implied warranty of |    9 # but WITHOUT ANY WARRANTY; without even the implied warranty of | 
|   10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the |   10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the | 
| (...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|   52  |   52  | 
|   53     :returns: a tuple `(lines_iterator, inherited_source)` where |   53     :returns: a tuple `(lines_iterator, inherited_source)` where | 
|   54         `inherited_source` is the default source to use for included |   54         `inherited_source` is the default source to use for included | 
|   55         fragments. |   55         fragments. | 
|   56     """ |   56     """ | 
|   57     if ':' in name: |   57     if ':' in name: | 
|   58         source_name, name_in_source = name.split(':', 1) |   58         source_name, name_in_source = name.split(':', 1) | 
|   59         try: |   59         try: | 
|   60             source = sources[source_name] |   60             source = sources[source_name] | 
|   61         except KeyError: |   61         except KeyError: | 
|   62             raise IncludeError('Unknown source: \'{}\''.format(source_name), |   62             raise IncludeError("Unknown source: '{}'".format(source_name), | 
|   63                                include_stack) |   63                                include_stack) | 
|   64     else: |   64     else: | 
|   65         source, name_in_source = default_source, name |   65         source, name_in_source = default_source, name | 
|   66  |   66  | 
|   67     if source is None: |   67     if source is None: | 
|   68         raise IncludeError('Source name is absent in: \'{}\''.format(name), |   68         raise IncludeError("Source name is absent in: '{}'".format(name), | 
|   69                            include_stack) |   69                            include_stack) | 
|   70  |   70  | 
|   71     return (parse_filterlist(source.get(name_in_source)), |   71     return (parse_filterlist(source.get(name_in_source)), | 
|   72             source if source.is_inheritable else None) |   72             source if source.is_inheritable else None) | 
|   73  |   73  | 
|   74  |   74  | 
|   75 def _process_includes(sources, default_source, parent_include_stack, lines): |   75 def _process_includes(sources, default_source, parent_include_stack, lines): | 
|   76     """Replace include instructions with the lines of included fragment.""" |   76     """Replace include instructions with the lines of included fragment.""" | 
|   77     for line in lines: |   77     for line in lines: | 
|   78         if line.type == 'include': |   78         if line.type == 'include': | 
| (...skipping 15 matching lines...) Expand all  Loading... | 
|   94             except (NotFound, ValueError) as exc: |   94             except (NotFound, ValueError) as exc: | 
|   95                 raise IncludeError(exc, include_stack) |   95                 raise IncludeError(exc, include_stack) | 
|   96         else: |   96         else: | 
|   97             yield line |   97             yield line | 
|   98  |   98  | 
|   99  |   99  | 
|  100 def _process_timestamps(lines): |  100 def _process_timestamps(lines): | 
|  101     """Convert timestamp markers into actual timestamps.""" |  101     """Convert timestamp markers into actual timestamps.""" | 
|  102     for line in lines: |  102     for line in lines: | 
|  103         if line.type == 'comment' and '%timestamp%' in line.text: |  103         if line.type == 'comment' and '%timestamp%' in line.text: | 
|  104             timestamp = time.strftime("%d %b %Y %H:%M UTC", time.gmtime()) |  104             timestamp = time.strftime('%d %b %Y %H:%M UTC', time.gmtime()) | 
|  105             yield Comment(text=line.text.replace('%timestamp%', timestamp)) |  105             yield Comment(text=line.text.replace('%timestamp%', timestamp)) | 
|  106         else: |  106         else: | 
|  107             yield line |  107             yield line | 
|  108  |  108  | 
|  109  |  109  | 
|  110 def _first_and_rest(iterable): |  110 def _first_and_rest(iterable): | 
|  111     """Return the first item from the iterable and the rest as an iterator.""" |  111     """Return the first item from the iterable and the rest as an iterator.""" | 
|  112     iterator = iter(iterable) |  112     iterator = iter(iterable) | 
|  113     first_item = next(iterator) |  113     first_item = next(iterator) | 
|  114     return first_item, iterator |  114     return first_item, iterator | 
|  115  |  115  | 
|  116  |  116  | 
|  117 def _insert_version(lines): |  117 def _insert_version(lines): | 
|  118     """Insert metadata comment with version (a.k.a. date).""" |  118     """Insert metadata comment with version (a.k.a. date).""" | 
|  119     first_line, rest = _first_and_rest(lines) |  119     first_line, rest = _first_and_rest(lines) | 
|  120     version = Metadata('Version', time.strftime("%Y%m%d%H%M", time.gmtime())) |  120     version = Metadata('Version', time.strftime('%Y%m%d%H%M', time.gmtime())) | 
|  121     return itertools.chain([first_line, version], rest) |  121     return itertools.chain([first_line, version], rest) | 
|  122  |  122  | 
|  123  |  123  | 
|  124 def _remove_duplicates(lines): |  124 def _remove_duplicates(lines): | 
|  125     """Remove duplicate metadata and headers.""" |  125     """Remove duplicate metadata and headers.""" | 
|  126     # Always remove checksum -- a checksum coming from a fragment |  126     # Always remove checksum -- a checksum coming from a fragment | 
|  127     # will not match for the rendered list. |  127     # will not match for the rendered list. | 
|  128     seen = {'Checksum'} |  128     seen = {'Checksum'} | 
|  129     for i, line in enumerate(lines): |  129     for i, line in enumerate(lines): | 
|  130         if line.type == 'metadata': |  130         if line.type == 'metadata': | 
| (...skipping 13 matching lines...) Expand all  Loading... | 
|  144     See https://adblockplus.org/filters#special-comments for description |  144     See https://adblockplus.org/filters#special-comments for description | 
|  145     of the checksum algorithm. |  145     of the checksum algorithm. | 
|  146     """ |  146     """ | 
|  147     md5sum = hashlib.md5() |  147     md5sum = hashlib.md5() | 
|  148  |  148  | 
|  149     for line in lines: |  149     for line in lines: | 
|  150         if line.type != 'emptyline': |  150         if line.type != 'emptyline': | 
|  151             md5sum.update(line.to_string().encode('utf-8') + b'\n') |  151             md5sum.update(line.to_string().encode('utf-8') + b'\n') | 
|  152         yield line |  152         yield line | 
|  153  |  153  | 
|  154     sum = base64.b64encode(md5sum.digest()).rstrip(b'=') |  154     checksum = base64.b64encode(md5sum.digest()).rstrip(b'=') | 
|  155     yield Metadata('Checksum', sum.decode('utf-8')) |  155     yield Metadata('Checksum', checksum.decode('utf-8')) | 
|  156  |  156  | 
|  157  |  157  | 
|  158 def _validate(lines): |  158 def _validate(lines): | 
|  159     """Validate the final list.""" |  159     """Validate the final list.""" | 
|  160     first_line, rest = _first_and_rest(lines) |  160     first_line, rest = _first_and_rest(lines) | 
|  161     if first_line.type != 'header': |  161     if first_line.type != 'header': | 
|  162         raise MissingHeader('No header found at the beginning of the input.') |  162         raise MissingHeader('No header found at the beginning of the input.') | 
|  163     return itertools.chain([first_line], rest) |  163     return itertools.chain([first_line], rest) | 
|  164  |  164  | 
|  165  |  165  | 
| (...skipping 10 matching lines...) Expand all  Loading... | 
|  176     :raises MissingHeader: If the top level fragment doesn't start with a valid |  176     :raises MissingHeader: If the top level fragment doesn't start with a valid | 
|  177         header. |  177         header. | 
|  178     """ |  178     """ | 
|  179     _logger.info('Rendering: %s', name) |  179     _logger.info('Rendering: %s', name) | 
|  180     lines, default_source = _get_and_parse_fragment(name, sources, top_source) |  180     lines, default_source = _get_and_parse_fragment(name, sources, top_source) | 
|  181     lines = _process_includes(sources, default_source, [name], lines) |  181     lines = _process_includes(sources, default_source, [name], lines) | 
|  182     for proc in [_process_timestamps, _insert_version, _remove_duplicates, |  182     for proc in [_process_timestamps, _insert_version, _remove_duplicates, | 
|  183                  _insert_checksum, _validate]: |  183                  _insert_checksum, _validate]: | 
|  184         lines = proc(lines) |  184         lines = proc(lines) | 
|  185     return lines |  185     return lines | 
| OLD | NEW |