| OLD | NEW |
| 1 # This file is part of the Adblock Plus web scripts, | 1 # This file is part of the Adblock Plus web scripts, |
| 2 # Copyright (C) 2006-2016 Eyeo GmbH | 2 # Copyright (C) 2006-2016 Eyeo GmbH |
| 3 # | 3 # |
| 4 # Adblock Plus is free software: you can redistribute it and/or modify | 4 # Adblock Plus is free software: you can redistribute it and/or modify |
| 5 # it under the terms of the GNU General Public License version 3 as | 5 # it under the terms of the GNU General Public License version 3 as |
| 6 # published by the Free Software Foundation. | 6 # published by the Free Software Foundation. |
| 7 # | 7 # |
| 8 # Adblock Plus is distributed in the hope that it will be useful, | 8 # Adblock Plus is distributed in the hope that it will be useful, |
| 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of | 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| (...skipping 25 matching lines...) Expand all Loading... |
| 36 except KeyError: | 36 except KeyError: |
| 37 return memoized.setdefault(args, func(*args)) | 37 return memoized.setdefault(args, func(*args)) |
| 38 wrapper.clear_cache = memoized.clear | 38 wrapper.clear_cache = memoized.clear |
| 39 return wrapper | 39 return wrapper |
| 40 | 40 |
| 41 | 41 |
| 42 def generate_pages(repo, output_dir): | 42 def generate_pages(repo, output_dir): |
| 43 known_files = set() | 43 known_files = set() |
| 44 | 44 |
| 45 def write_file(path_parts, contents, binary=False): | 45 def write_file(path_parts, contents, binary=False): |
| 46 encoding = None if binary else "utf-8" | 46 encoding = None if binary else 'utf-8' |
| 47 outfile = os.path.join(output_dir, *path_parts) | 47 outfile = os.path.join(output_dir, *path_parts) |
| 48 if outfile in known_files: | 48 if outfile in known_files: |
| 49 logging.warning("File %s has multiple sources", outfile) | 49 logging.warning('File %s has multiple sources', outfile) |
| 50 return | 50 return |
| 51 known_files.add(outfile) | 51 known_files.add(outfile) |
| 52 | 52 |
| 53 if os.path.exists(outfile): | 53 if os.path.exists(outfile): |
| 54 with codecs.open(outfile, "rb", encoding=encoding) as handle: | 54 with codecs.open(outfile, 'rb', encoding=encoding) as handle: |
| 55 if handle.read() == contents: | 55 if handle.read() == contents: |
| 56 return | 56 return |
| 57 | 57 |
| 58 try: | 58 try: |
| 59 os.makedirs(os.path.dirname(outfile)) | 59 os.makedirs(os.path.dirname(outfile)) |
| 60 except OSError, e: | 60 except OSError, e: |
| 61 if e.errno != errno.EEXIST: | 61 if e.errno != errno.EEXIST: |
| 62 raise | 62 raise |
| 63 | 63 |
| 64 with codecs.open(outfile, "wb", encoding=encoding) as handle: | 64 with codecs.open(outfile, 'wb', encoding=encoding) as handle: |
| 65 handle.write(contents) | 65 handle.write(contents) |
| 66 | 66 |
| 67 with MercurialSource(repo) as source: | 67 with MercurialSource(repo) as source: |
| 68 # Cache the result for some functions - we can assume here that the data | 68 # Cache the result for some functions - we can assume here that the data |
| 69 # never changes | 69 # never changes |
| 70 source.resolve_link = memoize(source.resolve_link) | 70 source.resolve_link = memoize(source.resolve_link) |
| 71 source.read_config = memoize(source.read_config) | 71 source.read_config = memoize(source.read_config) |
| 72 source.read_template = memoize(source.read_template) | 72 source.read_template = memoize(source.read_template) |
| 73 source.read_locale = memoize(source.read_locale) | 73 source.read_locale = memoize(source.read_locale) |
| 74 source.read_include = memoize(source.read_include) | 74 source.read_include = memoize(source.read_include) |
| 75 source.exec_file = memoize(source.exec_file) | 75 source.exec_file = memoize(source.exec_file) |
| 76 | 76 |
| 77 config = source.read_config() | 77 config = source.read_config() |
| 78 defaultlocale = config.get("general", "defaultlocale") | 78 defaultlocale = config.get('general', 'defaultlocale') |
| 79 locales = list(source.list_locales()) | 79 locales = list(source.list_locales()) |
| 80 if defaultlocale not in locales: | 80 if defaultlocale not in locales: |
| 81 locales.append(defaultlocale) | 81 locales.append(defaultlocale) |
| 82 | 82 |
| 83 # First pass: compile the list of pages with given translation level | 83 # First pass: compile the list of pages with given translation level |
| 84 def get_locale_file(page): | 84 def get_locale_file(page): |
| 85 try: | 85 try: |
| 86 return config.get("locale_overrides", page) | 86 return config.get('locale_overrides', page) |
| 87 except ConfigParser.Error: | 87 except ConfigParser.Error: |
| 88 return page | 88 return page |
| 89 | 89 |
| 90 pagelist = set() | 90 pagelist = set() |
| 91 blacklist = set() | 91 blacklist = set() |
| 92 for page, format in source.list_pages(): | 92 for page, format in source.list_pages(): |
| 93 for locale in locales: | 93 for locale in locales: |
| 94 if locale == defaultlocale: | 94 if locale == defaultlocale: |
| 95 pagelist.add((locale, page)) | 95 pagelist.add((locale, page)) |
| 96 else: | 96 else: |
| 97 params = get_page_params(source, locale, page, format) | 97 params = get_page_params(source, locale, page, format) |
| 98 if params["translation_ratio"] >= MIN_TRANSLATED: | 98 if params['translation_ratio'] >= MIN_TRANSLATED: |
| 99 pagelist.add((locale, page)) | 99 pagelist.add((locale, page)) |
| 100 else: | 100 else: |
| 101 blacklist.add((locale, get_locale_file(page))) | 101 blacklist.add((locale, get_locale_file(page))) |
| 102 | 102 |
| 103 # Override existance check to avoid linking to pages we don't generate | 103 # Override existance check to avoid linking to pages we don't generate |
| 104 orig_has_locale = source.has_locale | 104 orig_has_locale = source.has_locale |
| 105 | 105 |
| 106 def has_locale(locale, page): | 106 def has_locale(locale, page): |
| 107 page = get_locale_file(page) | 107 page = get_locale_file(page) |
| 108 if (locale, page) in blacklist: | 108 if (locale, page) in blacklist: |
| 109 return False | 109 return False |
| 110 return orig_has_locale(locale, page) | 110 return orig_has_locale(locale, page) |
| 111 source.has_locale = has_locale | 111 source.has_locale = has_locale |
| 112 source.resolve_link.clear_cache() | 112 source.resolve_link.clear_cache() |
| 113 | 113 |
| 114 # Second pass: actually generate pages this time | 114 # Second pass: actually generate pages this time |
| 115 for locale, page in pagelist: | 115 for locale, page in pagelist: |
| 116 pagedata = process_page(source, locale, page) | 116 pagedata = process_page(source, locale, page) |
| 117 | 117 |
| 118 # Make sure links to static files are versioned | 118 # Make sure links to static files are versioned |
| 119 pagedata = re.sub(r'(<script\s[^<>]*\bsrc="/[^"<>]+)', r"\1?%s" % so
urce.version, pagedata) | 119 pagedata = re.sub(r'(<script\s[^<>]*\bsrc="/[^"<>]+)', r'\1?%s' % so
urce.version, pagedata) |
| 120 pagedata = re.sub(r'(<link\s[^<>]*\bhref="/[^"<>]+)', r"\1?%s" % sou
rce.version, pagedata) | 120 pagedata = re.sub(r'(<link\s[^<>]*\bhref="/[^"<>]+)', r'\1?%s' % sou
rce.version, pagedata) |
| 121 pagedata = re.sub(r'(<img\s[^<>]*\bsrc="/[^"<>]+)', r"\1?%s" % sourc
e.version, pagedata) | 121 pagedata = re.sub(r'(<img\s[^<>]*\bsrc="/[^"<>]+)', r'\1?%s' % sourc
e.version, pagedata) |
| 122 | 122 |
| 123 write_file([locale] + page.split("/"), pagedata) | 123 write_file([locale] + page.split('/'), pagedata) |
| 124 | 124 |
| 125 for filename in source.list_localizable_files(): | 125 for filename in source.list_localizable_files(): |
| 126 for locale in locales: | 126 for locale in locales: |
| 127 if source.has_localizable_file(locale, filename): | 127 if source.has_localizable_file(locale, filename): |
| 128 filedata = source.read_localizable_file(locale, filename) | 128 filedata = source.read_localizable_file(locale, filename) |
| 129 write_file([locale] + filename.split("/"), filedata, binary=
True) | 129 write_file([locale] + filename.split('/'), filedata, binary=
True) |
| 130 | 130 |
| 131 for filename in source.list_static(): | 131 for filename in source.list_static(): |
| 132 write_file(filename.split("/"), source.read_static(filename), binary
=True) | 132 write_file(filename.split('/'), source.read_static(filename), binary
=True) |
| 133 | 133 |
| 134 def remove_unknown(dir): | 134 def remove_unknown(dir): |
| 135 files = os.listdir(dir) | 135 files = os.listdir(dir) |
| 136 for filename in files: | 136 for filename in files: |
| 137 path = os.path.join(dir, filename) | 137 path = os.path.join(dir, filename) |
| 138 if os.path.isfile(path) and path not in known_files: | 138 if os.path.isfile(path) and path not in known_files: |
| 139 os.remove(path) | 139 os.remove(path) |
| 140 elif os.path.isdir(path): | 140 elif os.path.isdir(path): |
| 141 remove_unknown(path) | 141 remove_unknown(path) |
| 142 if not os.listdir(path): | 142 if not os.listdir(path): |
| 143 os.rmdir(path) | 143 os.rmdir(path) |
| 144 remove_unknown(output_dir) | 144 remove_unknown(output_dir) |
| 145 | 145 |
| 146 if __name__ == "__main__": | 146 if __name__ == '__main__': |
| 147 if len(sys.argv) < 3: | 147 if len(sys.argv) < 3: |
| 148 print >>sys.stderr, "Usage: %s source_repository output_dir" % sys.argv[
0] | 148 print >>sys.stderr, 'Usage: %s source_repository output_dir' % sys.argv[
0] |
| 149 sys.exit(1) | 149 sys.exit(1) |
| 150 | 150 |
| 151 repo, output_dir = sys.argv[1:3] | 151 repo, output_dir = sys.argv[1:3] |
| 152 generate_pages(repo, output_dir) | 152 generate_pages(repo, output_dir) |
| OLD | NEW |