Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: cms/translations/xtm/utils.py

Issue 29886648: Issue #6942 - Add XTM integration in CMS (Closed)
Left Patch Set: Created Sept. 20, 2018, 4:24 p.m.
Right Patch Set: Addressed comments from Patch Set #4 Created Oct. 5, 2018, 4:23 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Right: Side by side diff | Download
LEFTRIGHT
(no file at all)
1 # This file is part of the Adblock Plus web scripts,
2 # Copyright (C) 2006-present eyeo GmbH
3 #
4 # Adblock Plus is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License version 3 as
6 # published by the Free Software Foundation.
7 #
8 # Adblock Plus is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
12 #
13 # You should have received a copy of the GNU General Public License
14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
15
16 import collections
17 import logging
18 import os
19 import time
20 import json
21
22 from cms.utils import process_page
23 import cms.translations.xtm.constants as const
24 from cms.translations.xtm.xtm_api import XTMCloudException
25
26
27 __all__ = [
28 'extract_strings', 'resolve_locales', 'map_locales', 'local_to_remote',
29 'remote_to_local', 'run_and_wait', 'sanitize_project_name', 'read_token',
30 'get_files_to_upload', 'log_resulting_jobs', 'clear_files', 'input_fn',
31 ]
32
33
34 def log_resulting_jobs(jobs):
35 """Log the jobs created as a result of uploading files/ creating projects.
36
37 Parameters
38 ----------
39 jobs: iterable
40 Of dicts, as returned by XTM.
41
42 """
43 if len(jobs) == 0:
44 logging.info(const.InfoMessages.NO_JOBS_CREATED)
45 return
46
47 for job in jobs:
48 logging.info(
49 const.InfoMessages.CREATED_JOB.format(
50 job['jobId'], job['fileName'], job['targetLanguage'],
51 ),
52 )
53
54
55 def get_files_to_upload(source):
56 """Return the files to upload in the format supported by XTMCloudAPI.
57
58 It performs the following tasks:
59 1. Extracts the translation strings from the website.
60 2. Cleanup the extracted strings (i.e. ignore all empty ones).
61 3. Convert local file names to remote ones.
62 4. Construct a dictionary with the format required by XTMCloudAPI
63
64 Parameters
65 ----------
66 source: cms.sources.Source
67
68 Returns
69 -------
70 dict
71 With the following format:
72 <remote_filename>: <file_data>
73
74 """
75 # 1. Extracting strings.
76 raw_strings = extract_strings(source)
77 page_strings = {}
78
79 # 2. Cleaning up.
80 for page, string in raw_strings.iteritems():
81 if string:
82 page_strings[page] = string
83
84 # 3. Converting local file names.
85 remote_names = local_to_remote(page_strings)
86
87 # 4. Constructing final data structure
88 files_to_upload = {}
89 for file in remote_names:
90 files_to_upload[remote_names[file]] = json.dumps(page_strings[file])
91
92 return files_to_upload
93
94
95 def read_token():
96 """Read token from pre-defined environment variable."""
97 token = os.environ.get(const.Token.ENV_VAR)
98 if token:
99 return token
100
101 raise Exception(const.ErrorMessages.NO_TOKEN_PROVIDED.format(
102 const.Token.CREATION_CMD,
103 ))
104
105
106 def sanitize_project_name(name):
107 """Handle project name conflicts.
108
109 Parameters
110 ----------
111 name: str
112 The name of the project.
113
114 Returns
115 -------
116 str
117 The new name of the project, with the length cut down to
118 const.ProjectName.MAX_LENGTH and invalid characters replaced with
119 const.ProjectName.NAME_WILDCARD.
120
121 """
122 valid_name = ''.join(
123 [const.ProjectName.NAME_WILDCARD
124 if c in const.ProjectName.INVALID_CHARS else c for c in name],
125 )
126
127 return valid_name[:const.ProjectName.MAX_LENGTH]
128
129
130 def extract_strings(source):
131 """Extract strings from a website.
132
133 Parameters
134 ----------
135 source: cms.sources.Source
136 The source representing the website.
137
138 Returns
139 -------
140 dict
141 With the extracted strings.
142
143 """
144 logging.info(const.InfoMessages.EXTRACTING_STRINGS)
145 page_strings = collections.defaultdict(collections.OrderedDict)
146
147 defaultlocale = source.read_config().get(
148 const.Config.MAIN_SECTION, const.Config.DEFAULT_LOCALE_OPTION,
149 )
150
151 def record_string(page, locale, name, value, comment, fixed_strings):
152 if locale != defaultlocale:
153 return
154
155 store = page_strings[page]
156 store[name] = {'message': value}
157
158 if fixed_strings:
159 comment = comment + '\n' if comment else ''
160 comment += ', '.join('{{{0}}}: {1}'.format(*i_s)
161 for i_s in enumerate(fixed_strings, 1))
162 if comment:
163 store[name]['description'] = comment
164
165 for page, page_format in source.list_pages():
166 process_page(source, defaultlocale, page, format=page_format,
167 localized_string_callback=record_string)
168
169 return page_strings
170
171
172 def resolve_locales(api, source):
173 """Sync a website's locales with the target languages of the API.
174
175 Parameters
176 ----------
177 api: cms.bin.xtm_translations.xtm_api.XTMCloudAPI
178 Handler used to make requests to the API.
179 source: cms.sources.Source
180 Source representing a website.
181
182 """
183 logging.info(const.InfoMessages.RESOLVING_LOCALES)
184 local_locales = map_locales(source)
185 project_id = source.read_config().get(
186 const.Config.XTM_SECTION, const.Config.PROJECT_OPTION,
187 )
188
189 languages = run_and_wait(
190 api.get_target_languages,
191 XTMCloudException,
192 const.UNDER_ANALYSIS_MESSAGE,
193 const.InfoMessages.WAITING_FOR_PROJECT,
194 project_id=project_id,
195 )
196
197 enabled_locales = {l.encode('utf-8') for l in languages}
198
199 if len(enabled_locales - local_locales) != 0:
200 raise Exception(const.ErrorMessages.LOCALES_NOT_PRESENT.format(
201 enabled_locales - local_locales, project_id,
202 ))
203
204 if not local_locales == enabled_locales:
205 # Add languages to the project
206 langs_to_add = list(local_locales - enabled_locales)
207 logging.info(const.InfoMessages.ADDING_LANGUAGES.format(
208 project_id, langs_to_add,
209 ))
210 run_and_wait(
211 api.add_target_languages,
212 XTMCloudException,
213 const.UNDER_ANALYSIS_MESSAGE,
214 const.InfoMessages.WAITING_FOR_PROJECT,
215 project_id=project_id,
216 target_languages=langs_to_add,
217 )
218
219
220 def map_locales(source):
221 """Map website locale to target languages supported by XTM.
222
223 Parameters
224 ----------
225 source: cms.sources.Source
226 Source representing a website.
227
228 Returns
229 -------
230 set
231 Of the resulting mapped locales.
232
233 """
234 config = source.read_config()
235 defaultlocale = config.get(const.Config.MAIN_SECTION,
236 const.Config.DEFAULT_LOCALE_OPTION)
237 locales = source.list_locales() - {defaultlocale}
238
239 mapped_locales = set()
240
241 for locale in locales:
242 if locale in const.SUPPORTED_LOCALES:
243 mapped_locales.add(locale)
244 else:
245 xtm_locale = '{0}_{1}'.format(locale, locale.upper())
246 if xtm_locale in const.SUPPORTED_LOCALES:
247 mapped_locales.add(xtm_locale)
248 else:
249 logging.warning(
250 const.WarningMessages.LOCALE_NOT_SUPPORTED.format(locale),
251 )
252
253 return mapped_locales
254
255
256 def local_to_remote(local_names):
257 """Convert local file names to valid remote ones.
258
259 Parameters
260 ----------
261 local_names: iterable
262 The local file names (without any extension).
263
264 Returns
265 -------
266 dict
267 With the local files. Each element in the set is a tuple with the
268 format:
269 <local_filename>: <remote_filename>
270
271 """
272 files = {}
273
274 for page in local_names:
275 remote_name = '{}.json'.format(
276 page.replace(os.path.sep, const.FileNames.PATH_SEP_REP),
277 )
278 if len(remote_name) > const.FileNames.MAX_LENGTH:
279 raise Exception(
280 const.ErrorMessages.FILENAME_TOO_LONG.format(
281 '{}.json'.format(page), const.FileNames.MAX_LENGTH,
282 ))
283 files[page] = remote_name
284
285 return files
286
287
288 def remote_to_local(filename, source_dir, locales):
289 """Parse a remote filename and construct a local filesystem name.
290
291 Parameters
292 ----------
293 filename: str
294 The remote filename.
295 source_dir: str
296 The path to the source directory of the file.
297 locales: iterator
298 Of local language directories.
299
300 Returns
301 -------
302 str
303 The full path of the file.
304
305 """
306 path_elements = filename.split(const.FileNames.PATH_SEP_REP)
307 if path_elements[0] == '':
308 path_elements = path_elements[1:]
309 if path_elements[0] not in locales:
310 candidate_locale = path_elements[0].split('_')[0]
311 if candidate_locale not in locales:
312 raise Exception(
313 const.ErrorMessages.CANT_RESOLVE_REMOTE_LANG.format(
314 path_elements[0],
315 ),
316 )
317
318 path_elements[0] = ''.join([
319 candidate_locale,
320 path_elements[0].split('_')[1][len(candidate_locale):],
321 ])
322
323 return os.path.join(source_dir, *path_elements)
324
325
326 def run_and_wait(func, exc, err_msg, user_msg=None, retry_delay=1,
327 retries=10, **kw):
328 """Run a function and, if a specific exception occurs, try again.
329
330 Tries to run the function, and if an exception with a specific message
331 is raised, it sleeps and tries again.
332 Parameters
333 ----------
334 func: function
335 The function to be run.
336 retry_delay: int
337 The amount of time to wait on this specific run (in seconds).
338 exc: Exception
339 The exception we expect to be raised.
340 err_msg: str
341 The message we expect to be in the exception.
342 user_msg: str
343 Message to be displayed to the user if we waited for more than 3 steps.
344 retries: int
345 The number of retries left until actually raising the exception.
346 kw: dict
347 The keyword arguments for the function.
348
349 Returns
350 -------
351 The return result of the function.
352
353 """
354 if retries == 7 and user_msg:
355 logging.info(user_msg)
356 try:
357 result = func(**kw)
358 return result
359 except exc as err:
360 if retries == 0:
361 raise
362 if err_msg in str(err):
363 time.sleep(retry_delay)
364 return run_and_wait(
365 func, exc, err_msg, user_msg,
366 min(retry_delay * 2, const.MAX_WAIT_TIME), retries - 1, **kw
367 )
368 raise
369
370
371 def clear_files(dir_path, required_locales, extension='.json'):
372 """Delete translation files with a specific extension from dir_path.
373
374 Parameters
375 ----------
376 dir_path: str
377 Path to the root of the subtree we want to delete the files from.
378 required_locales: iterable
379 Only directories form required_locales will be included
380 extension: str
381 The extension of the files to delete.
382
383 """
384 for root, dirs, files in os.walk(dir_path, topdown=True):
385 if root == dir_path:
386 dirs[:] = [d for d in dirs if d in required_locales]
387 for f in files:
388 if f.lower().endswith(extension.lower()):
389 os.remove(os.path.join(root, f))
390
391
392 def get_locales(path, default):
393 """List the locales available in a website.
394
395 It will exclude the default language from the list.
396
397 Parameters
398 ----------
399 path: str
400 The path to the locales directory.
401 default: str
402 The default language for the website.
403
404 Returns
405 -------
406 iterable
407 Of the available locales.
408
409 """
410 full_contents = os.listdir(path)
411
412 return [
413 d for d in full_contents
414 if os.path.isdir(os.path.join(path, d)) and d != default
415 ]
416
417
418 def write_to_file(data, file_path):
419 """Write data to a given file path.
420
421 If the directory path does not exist, then it will be created by default.
422
423 Parameters
424 ----------
425 data: bytes
426 The data to be written to the file.
427 file_path: str
428 The path of the file we want to write.
429
430 """
431 dirs_path = os.path.join(*os.path.split(file_path)[:-1])
432 if not os.path.isdir(dirs_path):
433 os.makedirs(dirs_path)
434
435 with open(file_path, 'wb') as f:
436 f.write(data)
437
438
439 def input_fn(text):
440 try:
441 return raw_input(text)
442 except Exception:
443 return input(text)
LEFTRIGHT

Powered by Google App Engine
This is Rietveld