OLD | NEW |
1 # This file is part of the Adblock Plus web scripts, | 1 # This file is part of the Adblock Plus web scripts, |
2 # Copyright (C) 2006-2016 Eyeo GmbH | 2 # Copyright (C) 2006-2016 Eyeo GmbH |
3 # | 3 # |
4 # Adblock Plus is free software: you can redistribute it and/or modify | 4 # Adblock Plus is free software: you can redistribute it and/or modify |
5 # it under the terms of the GNU General Public License version 3 as | 5 # it under the terms of the GNU General Public License version 3 as |
6 # published by the Free Software Foundation. | 6 # published by the Free Software Foundation. |
7 # | 7 # |
8 # Adblock Plus is distributed in the hope that it will be useful, | 8 # Adblock Plus is distributed in the hope that it will be useful, |
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of | 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
(...skipping 25 matching lines...) Expand all Loading... |
36 except KeyError: | 36 except KeyError: |
37 return memoized.setdefault(args, func(*args)) | 37 return memoized.setdefault(args, func(*args)) |
38 wrapper.clear_cache = memoized.clear | 38 wrapper.clear_cache = memoized.clear |
39 return wrapper | 39 return wrapper |
40 | 40 |
41 | 41 |
42 def generate_pages(repo, output_dir): | 42 def generate_pages(repo, output_dir): |
43 known_files = set() | 43 known_files = set() |
44 | 44 |
45 def write_file(path_parts, contents, binary=False): | 45 def write_file(path_parts, contents, binary=False): |
46 encoding = None if binary else "utf-8" | 46 encoding = None if binary else 'utf-8' |
47 outfile = os.path.join(output_dir, *path_parts) | 47 outfile = os.path.join(output_dir, *path_parts) |
48 if outfile in known_files: | 48 if outfile in known_files: |
49 logging.warning("File %s has multiple sources", outfile) | 49 logging.warning('File %s has multiple sources', outfile) |
50 return | 50 return |
51 known_files.add(outfile) | 51 known_files.add(outfile) |
52 | 52 |
53 if os.path.exists(outfile): | 53 if os.path.exists(outfile): |
54 with codecs.open(outfile, "rb", encoding=encoding) as handle: | 54 with codecs.open(outfile, 'rb', encoding=encoding) as handle: |
55 if handle.read() == contents: | 55 if handle.read() == contents: |
56 return | 56 return |
57 | 57 |
58 try: | 58 try: |
59 os.makedirs(os.path.dirname(outfile)) | 59 os.makedirs(os.path.dirname(outfile)) |
60 except OSError, e: | 60 except OSError, e: |
61 if e.errno != errno.EEXIST: | 61 if e.errno != errno.EEXIST: |
62 raise | 62 raise |
63 | 63 |
64 with codecs.open(outfile, "wb", encoding=encoding) as handle: | 64 with codecs.open(outfile, 'wb', encoding=encoding) as handle: |
65 handle.write(contents) | 65 handle.write(contents) |
66 | 66 |
67 with MercurialSource(repo) as source: | 67 with MercurialSource(repo) as source: |
68 # Cache the result for some functions - we can assume here that the data | 68 # Cache the result for some functions - we can assume here that the data |
69 # never changes | 69 # never changes |
70 source.resolve_link = memoize(source.resolve_link) | 70 source.resolve_link = memoize(source.resolve_link) |
71 source.read_config = memoize(source.read_config) | 71 source.read_config = memoize(source.read_config) |
72 source.read_template = memoize(source.read_template) | 72 source.read_template = memoize(source.read_template) |
73 source.read_locale = memoize(source.read_locale) | 73 source.read_locale = memoize(source.read_locale) |
74 source.read_include = memoize(source.read_include) | 74 source.read_include = memoize(source.read_include) |
75 source.exec_file = memoize(source.exec_file) | 75 source.exec_file = memoize(source.exec_file) |
76 | 76 |
77 config = source.read_config() | 77 config = source.read_config() |
78 defaultlocale = config.get("general", "defaultlocale") | 78 defaultlocale = config.get('general', 'defaultlocale') |
79 locales = list(source.list_locales()) | 79 locales = list(source.list_locales()) |
80 if defaultlocale not in locales: | 80 if defaultlocale not in locales: |
81 locales.append(defaultlocale) | 81 locales.append(defaultlocale) |
82 | 82 |
83 # First pass: compile the list of pages with given translation level | 83 # First pass: compile the list of pages with given translation level |
84 def get_locale_file(page): | 84 def get_locale_file(page): |
85 try: | 85 try: |
86 return config.get("locale_overrides", page) | 86 return config.get('locale_overrides', page) |
87 except ConfigParser.Error: | 87 except ConfigParser.Error: |
88 return page | 88 return page |
89 | 89 |
90 pagelist = set() | 90 pagelist = set() |
91 blacklist = set() | 91 blacklist = set() |
92 for page, format in source.list_pages(): | 92 for page, format in source.list_pages(): |
93 for locale in locales: | 93 for locale in locales: |
94 if locale == defaultlocale: | 94 if locale == defaultlocale: |
95 pagelist.add((locale, page)) | 95 pagelist.add((locale, page)) |
96 else: | 96 else: |
97 params = get_page_params(source, locale, page, format) | 97 params = get_page_params(source, locale, page, format) |
98 if params["translation_ratio"] >= MIN_TRANSLATED: | 98 if params['translation_ratio'] >= MIN_TRANSLATED: |
99 pagelist.add((locale, page)) | 99 pagelist.add((locale, page)) |
100 else: | 100 else: |
101 blacklist.add((locale, get_locale_file(page))) | 101 blacklist.add((locale, get_locale_file(page))) |
102 | 102 |
103 # Override existance check to avoid linking to pages we don't generate | 103 # Override existance check to avoid linking to pages we don't generate |
104 orig_has_locale = source.has_locale | 104 orig_has_locale = source.has_locale |
105 | 105 |
106 def has_locale(locale, page): | 106 def has_locale(locale, page): |
107 page = get_locale_file(page) | 107 page = get_locale_file(page) |
108 if (locale, page) in blacklist: | 108 if (locale, page) in blacklist: |
109 return False | 109 return False |
110 return orig_has_locale(locale, page) | 110 return orig_has_locale(locale, page) |
111 source.has_locale = has_locale | 111 source.has_locale = has_locale |
112 source.resolve_link.clear_cache() | 112 source.resolve_link.clear_cache() |
113 | 113 |
114 # Second pass: actually generate pages this time | 114 # Second pass: actually generate pages this time |
115 for locale, page in pagelist: | 115 for locale, page in pagelist: |
116 pagedata = process_page(source, locale, page) | 116 pagedata = process_page(source, locale, page) |
117 | 117 |
118 # Make sure links to static files are versioned | 118 # Make sure links to static files are versioned |
119 pagedata = re.sub(r'(<script\s[^<>]*\bsrc="/[^"<>]+)', r"\1?%s" % so
urce.version, pagedata) | 119 pagedata = re.sub(r'(<script\s[^<>]*\bsrc="/[^"<>]+)', r'\1?%s' % so
urce.version, pagedata) |
120 pagedata = re.sub(r'(<link\s[^<>]*\bhref="/[^"<>]+)', r"\1?%s" % sou
rce.version, pagedata) | 120 pagedata = re.sub(r'(<link\s[^<>]*\bhref="/[^"<>]+)', r'\1?%s' % sou
rce.version, pagedata) |
121 pagedata = re.sub(r'(<img\s[^<>]*\bsrc="/[^"<>]+)', r"\1?%s" % sourc
e.version, pagedata) | 121 pagedata = re.sub(r'(<img\s[^<>]*\bsrc="/[^"<>]+)', r'\1?%s' % sourc
e.version, pagedata) |
122 | 122 |
123 write_file([locale] + page.split("/"), pagedata) | 123 write_file([locale] + page.split('/'), pagedata) |
124 | 124 |
125 for filename in source.list_localizable_files(): | 125 for filename in source.list_localizable_files(): |
126 for locale in locales: | 126 for locale in locales: |
127 if source.has_localizable_file(locale, filename): | 127 if source.has_localizable_file(locale, filename): |
128 filedata = source.read_localizable_file(locale, filename) | 128 filedata = source.read_localizable_file(locale, filename) |
129 write_file([locale] + filename.split("/"), filedata, binary=
True) | 129 write_file([locale] + filename.split('/'), filedata, binary=
True) |
130 | 130 |
131 for filename in source.list_static(): | 131 for filename in source.list_static(): |
132 write_file(filename.split("/"), source.read_static(filename), binary
=True) | 132 write_file(filename.split('/'), source.read_static(filename), binary
=True) |
133 | 133 |
134 def remove_unknown(dir): | 134 def remove_unknown(dir): |
135 files = os.listdir(dir) | 135 files = os.listdir(dir) |
136 for filename in files: | 136 for filename in files: |
137 path = os.path.join(dir, filename) | 137 path = os.path.join(dir, filename) |
138 if os.path.isfile(path) and path not in known_files: | 138 if os.path.isfile(path) and path not in known_files: |
139 os.remove(path) | 139 os.remove(path) |
140 elif os.path.isdir(path): | 140 elif os.path.isdir(path): |
141 remove_unknown(path) | 141 remove_unknown(path) |
142 if not os.listdir(path): | 142 if not os.listdir(path): |
143 os.rmdir(path) | 143 os.rmdir(path) |
144 remove_unknown(output_dir) | 144 remove_unknown(output_dir) |
145 | 145 |
146 if __name__ == "__main__": | 146 if __name__ == '__main__': |
147 if len(sys.argv) < 3: | 147 if len(sys.argv) < 3: |
148 print >>sys.stderr, "Usage: %s source_repository output_dir" % sys.argv[
0] | 148 print >>sys.stderr, 'Usage: %s source_repository output_dir' % sys.argv[
0] |
149 sys.exit(1) | 149 sys.exit(1) |
150 | 150 |
151 repo, output_dir = sys.argv[1:3] | 151 repo, output_dir = sys.argv[1:3] |
152 generate_pages(repo, output_dir) | 152 generate_pages(repo, output_dir) |
OLD | NEW |