Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: cms/bin/generate_static_pages.py

Issue 5242593268989952: Issue 2340 - Don`t generate pages if less than 30% have been translated (Closed)
Left Patch Set: Addressed comments Created April 17, 2015, 6:20 p.m.
Right Patch Set: Addressed comments Created May 6, 2015, 5:20 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « no previous file | cms/converters.py » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 # coding: utf-8 1 # coding: utf-8
2 2
3 # This file is part of the Adblock Plus web scripts, 3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2015 Eyeo GmbH 4 # Copyright (C) 2006-2015 Eyeo GmbH
5 # 5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify 6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as 7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation. 8 # published by the Free Software Foundation.
9 # 9 #
10 # Adblock Plus is distributed in the hope that it will be useful, 10 # Adblock Plus is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details. 13 # GNU General Public License for more details.
14 # 14 #
15 # You should have received a copy of the GNU General Public License 15 # You should have received a copy of the GNU General Public License
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
17 17
18 import sys, os, re, errno, codecs 18 import sys
19 from ..utils import process_page 19 import os
20 from ..sources import MercurialSource 20 import re
21 import errno
22 import codecs
23 import ConfigParser
24 import logging
25
26 from cms.utils import get_page_params, process_page
27 from cms.sources import MercurialSource
21 28
22 MIN_TRANSLATED = 0.3 29 MIN_TRANSLATED = 0.3
23 30
24 def memoize(func): 31 def memoize(func):
25 memoized = {} 32 memoized = {}
26 def wrapper(*args): 33 def wrapper(*args):
27 try: 34 try:
28 return memoized[args] 35 return memoized[args]
29 except KeyError: 36 except KeyError:
30 return memoized.setdefault(args, func(*args)) 37 return memoized.setdefault(args, func(*args))
38 wrapper.clear_cache = memoized.clear
31 return wrapper 39 return wrapper
32 40
33 def generate_pages(repo, output_dir): 41 def generate_pages(repo, output_dir):
34 known_files = set() 42 known_files = set()
35 43
36 def write_file(path_parts, contents, binary=False): 44 def write_file(path_parts, contents, binary=False):
37 encoding = None if binary else "utf-8" 45 encoding = None if binary else "utf-8"
38 outfile = os.path.join(output_dir, *path_parts) 46 outfile = os.path.join(output_dir, *path_parts)
39 if outfile in known_files: 47 if outfile in known_files:
40 print >>sys.stderr, "Warning: File %s has multiple sources" % outfile 48 logging.warning("File %s has multiple sources", outfile)
41 return 49 return
42 known_files.add(outfile) 50 known_files.add(outfile)
43 51
44 if os.path.exists(outfile): 52 if os.path.exists(outfile):
45 with codecs.open(outfile, "rb", encoding=encoding) as handle: 53 with codecs.open(outfile, "rb", encoding=encoding) as handle:
46 if handle.read() == contents: 54 if handle.read() == contents:
47 return 55 return
48 56
49 try: 57 try:
50 os.makedirs(os.path.dirname(outfile)) 58 os.makedirs(os.path.dirname(outfile))
(...skipping 11 matching lines...) Expand all
62 source.read_config = memoize(source.read_config) 70 source.read_config = memoize(source.read_config)
63 source.read_template = memoize(source.read_template) 71 source.read_template = memoize(source.read_template)
64 source.read_locale = memoize(source.read_locale) 72 source.read_locale = memoize(source.read_locale)
65 source.read_include = memoize(source.read_include) 73 source.read_include = memoize(source.read_include)
66 74
67 config = source.read_config() 75 config = source.read_config()
68 defaultlocale = config.get("general", "defaultlocale") 76 defaultlocale = config.get("general", "defaultlocale")
69 locales = list(source.list_locales()) 77 locales = list(source.list_locales())
70 if defaultlocale not in locales: 78 if defaultlocale not in locales:
71 locales.append(defaultlocale) 79 locales.append(defaultlocale)
80
81 # First pass: compile the list of pages with given translation level
82 pagelist = set()
83 blacklist = set()
72 for page, format in source.list_pages(): 84 for page, format in source.list_pages():
73 for locale in locales: 85 for locale in locales:
74 if locale == defaultlocale or source.has_locale(locale, page): 86 if locale == defaultlocale:
75 pagedata = process_page(source, locale, page, format, 87 pagelist.add((locale, page))
76 min_translated=MIN_TRANSLATED) 88 elif source.has_locale(locale, page):
77 if pagedata: 89 params = get_page_params(source, locale, page, format)
78 # Make sure links to static files are versioned 90 if params["translation_ratio"] >= MIN_TRANSLATED:
79 pagedata = re.sub(r'(<script\s[^<>]*\bsrc="/[^"<>]+)', r"\1?%s" % so urce.version, pagedata) 91 pagelist.add((locale, page))
80 pagedata = re.sub(r'(<link\s[^<>]*\bhref="/[^"<>]+)', r"\1?%s" % sou rce.version, pagedata) 92 else:
81 pagedata = re.sub(r'(<img\s[^<>]*\bsrc="/[^"<>]+)', r"\1?%s" % sourc e.version, pagedata) 93 blacklist.add((locale, page))
82 94
83 write_file([locale] + page.split("/"), pagedata) 95 # Override existance check to avoid linking to pages we don't generate
96 orig_has_locale = source.has_locale
97 def has_locale(locale, page):
98 try:
99 page = config.get("locale_overrides", page)
100 except ConfigParser.Error:
101 pass
102 if (locale, page) in blacklist:
103 return False
104 return orig_has_locale(locale, page)
105 source.has_locale = has_locale
106 source.resolve_link.clear_cache()
107
108 # Second pass: actually generate pages this time
109 for locale, page in pagelist:
110 pagedata = process_page(source, locale, page)
111
112 # Make sure links to static files are versioned
113 pagedata = re.sub(r'(<script\s[^<>]*\bsrc="/[^"<>]+)', r"\1?%s" % source.v ersion, pagedata)
114 pagedata = re.sub(r'(<link\s[^<>]*\bhref="/[^"<>]+)', r"\1?%s" % source.ve rsion, pagedata)
115 pagedata = re.sub(r'(<img\s[^<>]*\bsrc="/[^"<>]+)', r"\1?%s" % source.vers ion, pagedata)
116
117 write_file([locale] + page.split("/"), pagedata)
84 118
85 for filename in source.list_localizable_files(): 119 for filename in source.list_localizable_files():
86 for locale in locales: 120 for locale in locales:
87 if source.has_localizable_file(locale, filename): 121 if source.has_localizable_file(locale, filename):
88 filedata = source.read_localizable_file(locale, filename) 122 filedata = source.read_localizable_file(locale, filename)
89 write_file([locale] + filename.split("/"), filedata, binary=True) 123 write_file([locale] + filename.split("/"), filedata, binary=True)
90 124
91 for filename in source.list_static(): 125 for filename in source.list_static():
92 write_file(filename.split("/"), source.read_static(filename), binary=True) 126 write_file(filename.split("/"), source.read_static(filename), binary=True)
93 127
94 def remove_unknown(dir): 128 def remove_unknown(dir):
95 files = os.listdir(dir) 129 files = os.listdir(dir)
96 for filename in files: 130 for filename in files:
97 path = os.path.join(dir, filename) 131 path = os.path.join(dir, filename)
98 if os.path.isfile(path) and path not in known_files: 132 if os.path.isfile(path) and path not in known_files:
99 os.remove(path) 133 os.remove(path)
100 elif os.path.isdir(path): 134 elif os.path.isdir(path):
101 remove_unknown(path) 135 remove_unknown(path)
102 if not os.listdir(path): 136 if not os.listdir(path):
103 os.rmdir(path) 137 os.rmdir(path)
104 remove_unknown(output_dir) 138 remove_unknown(output_dir)
105 139
106 if __name__ == "__main__": 140 if __name__ == "__main__":
107 if len(sys.argv) < 3: 141 if len(sys.argv) < 3:
108 print >>sys.stderr, "Usage: %s source_repository output_dir" % sys.argv[0] 142 print >>sys.stderr, "Usage: %s source_repository output_dir" % sys.argv[0]
109 sys.exit(1) 143 sys.exit(1)
110 144
111 repo, output_dir = sys.argv[1:3] 145 repo, output_dir = sys.argv[1:3]
112 generate_pages(repo, output_dir) 146 generate_pages(repo, output_dir)
LEFTRIGHT
« no previous file | cms/converters.py » ('j') | Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Toggle Comments ('s')

Powered by Google App Engine
This is Rietveld