Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: cms/bin/generate_static_pages.py

Issue 29887585: Issue #5352 - generate_static_pages cannot deal with directories being turned into regular pages (Closed)
Patch Set: Addressed comments from Patch Set #4 Created Oct. 5, 2018, 11:28 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « .hgignore ('k') | tests/expected_output/en/sitemap » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # This file is part of the Adblock Plus web scripts, 1 # This file is part of the Adblock Plus web scripts,
2 # Copyright (C) 2006-present eyeo GmbH 2 # Copyright (C) 2006-present eyeo GmbH
3 # 3 #
4 # Adblock Plus is free software: you can redistribute it and/or modify 4 # Adblock Plus is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License version 3 as 5 # it under the terms of the GNU General Public License version 3 as
6 # published by the Free Software Foundation. 6 # published by the Free Software Foundation.
7 # 7 #
8 # Adblock Plus is distributed in the hope that it will be useful, 8 # Adblock Plus is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details. 11 # GNU General Public License for more details.
12 # 12 #
13 # You should have received a copy of the GNU General Public License 13 # You should have received a copy of the GNU General Public License
14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
15 15
16 import os 16 import os
17 import re 17 import re
18 import errno
19 import codecs 18 import codecs
20 import ConfigParser 19 import ConfigParser
21 import logging 20 import logging
22 from argparse import ArgumentParser 21 from argparse import ArgumentParser
22 import shutil
23 23
24 from cms.utils import get_page_params, process_page 24 from cms.utils import get_page_params, process_page
25 from cms.sources import create_source 25 from cms.sources import create_source
26 26
27 MIN_TRANSLATED = 0.3 27 MIN_TRANSLATED = 0.3
28 28
29 29
30 def ensure_dirs(partial_path, path_parts):
31 """Create an entire path of directories.
32
33 This is a recursive function, that also treats these special cases, if
34 the partial path we reached so far exists and is:
35
36 1. a directory - do nothing, just move on,
37 2. a file - remove the file, create a directory in its place, and move on.
38 3. neither a file, nor a directory - raise and exception.
39
40 Parameters
41 ----------
42 partial_path: str
43 The path to resolve at this step.
44 path_parts: iterable
45 The remaining directories that will be created.
46
47 Raises
48 -------
49 Exception
50 If the path we want to resolve at this step exists and is neither a
51 file, nor a directory.
52
53 """
54 if os.path.isfile(partial_path):
55 os.remove(partial_path)
56 elif os.path.exists(partial_path) and not os.path.isdir(partial_path):
57 raise Exception('The object at {} is not recognisable! It is neither '
58 'a file, nor a directory!'.format(partial_path))
59
60 if not os.path.isdir(partial_path):
61 os.mkdir(partial_path)
62
63 if len(path_parts) == 0:
64 return
65
66 ensure_dirs(os.path.join(partial_path, path_parts[0]), path_parts[1:])
67
68
69 def is_in_previous_version(path, new_contents, encoding):
70 """Test if a file we try to create already is in the output directory.
71
72 It tests if the pre-existent file has all the expected content.
73 It also handles the following two cases:
74
75 1. The path is a directory - If this happens, it removes the directory from
76 the file tree.
77 2. The path exists, but it's neither a file, nor a directory. - If this
78 happens, it will raise an exception.
79
80 Parameters
81 ----------
82 path: str
83 The path we want to test for existence.
84 new_contents: bytes
85 The contents we want to write to the file in the new version of the
86 website.
87 encoding: str
88 The encoding to open the file in (if the path exists and is a file.
89
90 Returns
91 -------
92 bool
93 True - if the file exists and has the same contents.
94 False - if the path doesn't exist/ is not a file.
95
96 Raises
97 ------
98 Exception
99 If the path exists, but is neither a file, nor a directory.
100
101 """
102 if os.path.isfile(path):
103 with codecs.open(path, 'rb', encoding=encoding) as handle:
104 if handle.read() == new_contents:
105 return True
106 elif os.path.isdir(path):
107 shutil.rmtree(path)
108 elif os.path.exists(path):
109 raise Exception('The object at {} is not recognisable! It is '
110 'neither a file, nor a directory!'.format(path))
111
112 return False
113
114
30 def generate_pages(repo, output_dir): 115 def generate_pages(repo, output_dir):
31 known_files = set() 116 known_files = set()
32 117
33 def write_file(path_parts, contents, binary=False): 118 def write_file(path_parts, contents, binary=False):
34 encoding = None if binary else 'utf-8' 119 encoding = None if binary else 'utf-8'
35 outfile = os.path.join(output_dir, *path_parts) 120 outfile = os.path.join(output_dir, *path_parts)
36 if outfile in known_files: 121 if outfile in known_files:
37 logging.warning('File %s has multiple sources', outfile) 122 logging.warning('File %s has multiple sources', outfile)
38 return 123 return
39 known_files.add(outfile) 124 known_files.add(outfile)
40 125
41 if os.path.exists(outfile): 126 if is_in_previous_version(outfile, contents, encoding):
42 with codecs.open(outfile, 'rb', encoding=encoding) as handle: 127 return
43 if handle.read() == contents:
44 return
45 128
46 try: 129 ensure_dirs(output_dir, path_parts[:-1])
47 os.makedirs(os.path.dirname(outfile))
48 except OSError as e:
49 if e.errno != errno.EEXIST:
50 raise
51 130
52 with codecs.open(outfile, 'wb', encoding=encoding) as handle: 131 with codecs.open(outfile, 'wb', encoding=encoding) as handle:
53 handle.write(contents) 132 handle.write(contents)
54 133
55 with create_source(repo, cached=True) as source: 134 with create_source(repo, cached=True) as source:
56 config = source.read_config() 135 config = source.read_config()
57 defaultlocale = config.get('general', 'defaultlocale') 136 defaultlocale = config.get('general', 'defaultlocale')
58 locales = list(source.list_locales()) 137 locales = list(source.list_locales())
59 if defaultlocale not in locales: 138 if defaultlocale not in locales:
60 locales.append(defaultlocale) 139 locales.append(defaultlocale)
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after
122 os.rmdir(path) 201 os.rmdir(path)
123 remove_unknown(output_dir) 202 remove_unknown(output_dir)
124 203
125 204
126 if __name__ == '__main__': 205 if __name__ == '__main__':
127 parser = ArgumentParser('Convert website source to static website') 206 parser = ArgumentParser('Convert website source to static website')
128 parser.add_argument('source', help="Path to website's repository") 207 parser.add_argument('source', help="Path to website's repository")
129 parser.add_argument('output', help='Path to desired output directory') 208 parser.add_argument('output', help='Path to desired output directory')
130 args = parser.parse_args() 209 args = parser.parse_args()
131 generate_pages(args.source, args.output) 210 generate_pages(args.source, args.output)
OLDNEW
« no previous file with comments | « .hgignore ('k') | tests/expected_output/en/sitemap » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld