Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: cms/bin/generate_static_pages.py

Issue 29555839: Issue 5336 - Allow additional include, page, and template paths using CMS (Closed)
Patch Set: Address comments on PS1 Created Sept. 27, 2017, 11:33 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | cms/bin/test_server.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # This file is part of the Adblock Plus web scripts, 1 # This file is part of the Adblock Plus web scripts,
2 # Copyright (C) 2006-present eyeo GmbH 2 # Copyright (C) 2006-present eyeo GmbH
3 # 3 #
4 # Adblock Plus is free software: you can redistribute it and/or modify 4 # Adblock Plus is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License version 3 as 5 # it under the terms of the GNU General Public License version 3 as
6 # published by the Free Software Foundation. 6 # published by the Free Software Foundation.
7 # 7 #
8 # Adblock Plus is distributed in the hope that it will be useful, 8 # Adblock Plus is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details. 11 # GNU General Public License for more details.
12 # 12 #
13 # You should have received a copy of the GNU General Public License 13 # You should have received a copy of the GNU General Public License
14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
15 15
16 import os 16 import os
17 import re 17 import re
18 import errno 18 import errno
19 import codecs 19 import codecs
20 import ConfigParser 20 import ConfigParser
21 import logging 21 import logging
22 from argparse import ArgumentParser 22 from argparse import ArgumentParser
23 23
24 from cms.utils import get_page_params, process_page 24 from cms.utils import get_page_params, process_page
25 from cms.sources import MercurialSource 25 from cms.sources import create_source
26 26
27 MIN_TRANSLATED = 0.3 27 MIN_TRANSLATED = 0.3
28 28
29 29
30 def memoize(func):
31 memoized = {}
32
33 def wrapper(*args):
34 try:
35 return memoized[args]
36 except KeyError:
37 return memoized.setdefault(args, func(*args))
38 wrapper.clear_cache = memoized.clear
39 return wrapper
40
41
42 def generate_pages(repo, output_dir, revision): 30 def generate_pages(repo, output_dir, revision):
43 known_files = set() 31 known_files = set()
44 32
45 def write_file(path_parts, contents, binary=False): 33 def write_file(path_parts, contents, binary=False):
46 encoding = None if binary else 'utf-8' 34 encoding = None if binary else 'utf-8'
47 outfile = os.path.join(output_dir, *path_parts) 35 outfile = os.path.join(output_dir, *path_parts)
48 if outfile in known_files: 36 if outfile in known_files:
49 logging.warning('File %s has multiple sources', outfile) 37 logging.warning('File %s has multiple sources', outfile)
50 return 38 return
51 known_files.add(outfile) 39 known_files.add(outfile)
52 40
53 if os.path.exists(outfile): 41 if os.path.exists(outfile):
54 with codecs.open(outfile, 'rb', encoding=encoding) as handle: 42 with codecs.open(outfile, 'rb', encoding=encoding) as handle:
55 if handle.read() == contents: 43 if handle.read() == contents:
56 return 44 return
57 45
58 try: 46 try:
59 os.makedirs(os.path.dirname(outfile)) 47 os.makedirs(os.path.dirname(outfile))
60 except OSError as e: 48 except OSError as e:
61 if e.errno != errno.EEXIST: 49 if e.errno != errno.EEXIST:
62 raise 50 raise
63 51
64 with codecs.open(outfile, 'wb', encoding=encoding) as handle: 52 with codecs.open(outfile, 'wb', encoding=encoding) as handle:
65 handle.write(contents) 53 handle.write(contents)
66 54
67 with MercurialSource(repo, revision) as source: 55 with create_source(repo, static=True, revision=revision) as source:
68 # Cache the result for some functions - we can assume here that the data
69 # never changes
70 source.resolve_link = memoize(source.resolve_link)
71 source.read_config = memoize(source.read_config)
72 source.read_template = memoize(source.read_template)
73 source.read_locale = memoize(source.read_locale)
74 source.read_include = memoize(source.read_include)
75 source.exec_file = memoize(source.exec_file)
76
77 config = source.read_config() 56 config = source.read_config()
78 defaultlocale = config.get('general', 'defaultlocale') 57 defaultlocale = config.get('general', 'defaultlocale')
79 locales = list(source.list_locales()) 58 locales = list(source.list_locales())
80 if defaultlocale not in locales: 59 if defaultlocale not in locales:
81 locales.append(defaultlocale) 60 locales.append(defaultlocale)
82 61
83 # First pass: compile the list of pages with given translation level 62 # First pass: compile the list of pages with given translation level
84 def get_locale_file(page): 63 def get_locale_file(page):
85 try: 64 try:
86 return config.get('locale_overrides', page) 65 return config.get('locale_overrides', page)
(...skipping 15 matching lines...) Expand all
102 81
103 # Override existance check to avoid linking to pages we don't generate 82 # Override existance check to avoid linking to pages we don't generate
104 orig_has_locale = source.has_locale 83 orig_has_locale = source.has_locale
105 84
106 def has_locale(locale, page): 85 def has_locale(locale, page):
107 page = get_locale_file(page) 86 page = get_locale_file(page)
108 if (locale, page) in blacklist: 87 if (locale, page) in blacklist:
109 return False 88 return False
110 return orig_has_locale(locale, page) 89 return orig_has_locale(locale, page)
111 source.has_locale = has_locale 90 source.has_locale = has_locale
112 source.resolve_link.clear_cache() 91 source.resolve_link.cache_clear()
113 92
114 # Second pass: actually generate pages this time 93 # Second pass: actually generate pages this time
115 for locale, page in pagelist: 94 for locale, page in pagelist:
116 pagedata = process_page(source, locale, page) 95 pagedata = process_page(source, locale, page)
117 96
118 # Make sure links to static files are versioned 97 # Make sure links to static files are versioned
119 pagedata = re.sub(r'(<script\s[^<>]*\bsrc="/[^"<>]+)', r'\1?%s' % so urce.version, pagedata) 98 pagedata = re.sub(r'(<script\s[^<>]*\bsrc="/[^"<>]+)', r'\1?%s' % so urce.version, pagedata)
120 pagedata = re.sub(r'(<link\s[^<>]*\bhref="/[^"<>]+)', r'\1?%s' % sou rce.version, pagedata) 99 pagedata = re.sub(r'(<link\s[^<>]*\bhref="/[^"<>]+)', r'\1?%s' % sou rce.version, pagedata)
121 pagedata = re.sub(r'(<img\s[^<>]*\bsrc="/[^"<>]+)', r'\1?%s' % sourc e.version, pagedata) 100 pagedata = re.sub(r'(<img\s[^<>]*\bsrc="/[^"<>]+)', r'\1?%s' % sourc e.version, pagedata)
122 101
(...skipping 23 matching lines...) Expand all
146 if __name__ == '__main__': 125 if __name__ == '__main__':
147 parser = ArgumentParser('Convert website source to static website') 126 parser = ArgumentParser('Convert website source to static website')
148 parser.add_argument('-r', '--rev', 127 parser.add_argument('-r', '--rev',
149 help=('Specify which revision to generate from. ' 128 help=('Specify which revision to generate from. '
150 'See "hg help revisions" for details.'), 129 'See "hg help revisions" for details.'),
151 default='default') 130 default='default')
152 parser.add_argument('source', help="Path to website's repository") 131 parser.add_argument('source', help="Path to website's repository")
153 parser.add_argument('output', help='Path to desired output directory') 132 parser.add_argument('output', help='Path to desired output directory')
154 args = parser.parse_args() 133 args = parser.parse_args()
155 generate_pages(args.source, args.output, args.rev) 134 generate_pages(args.source, args.output, args.rev)
OLDNEW
« no previous file with comments | « no previous file | cms/bin/test_server.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld