Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: cms/bin/translate.py

Issue 29328085: Issue 3076 - [CMS] Map locale names to match Crowdin's expectations (Closed)
Patch Set: Simplified local_locales assignment Created Sept. 16, 2015, 8:01 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # coding: utf-8 1 # coding: utf-8
2 2
3 # This file is part of the Adblock Plus web scripts, 3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2015 Eyeo GmbH 4 # Copyright (C) 2006-2015 Eyeo GmbH
5 # 5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify 6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as 7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation. 8 # published by the Free Software Foundation.
9 # 9 #
10 # Adblock Plus is distributed in the hope that it will be useful, 10 # Adblock Plus is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details. 13 # GNU General Public License for more details.
14 # 14 #
15 # You should have received a copy of the GNU General Public License 15 # You should have received a copy of the GNU General Public License
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
17 17
18 import collections 18 import collections
19 import io 19 import io
20 import itertools 20 import itertools
21 import json 21 import json
22 import logging 22 import logging
23 import os 23 import os
24 import posixpath 24 import posixpath
25 import shutil
25 import sys 26 import sys
26 import urllib 27 import urllib
27 import zipfile 28 import zipfile
28 29
29 import urllib3 30 import urllib3
30 31
31 import cms.utils 32 import cms.utils
32 from cms.sources import FileSource 33 from cms.sources import FileSource
33 34
34 logger = logging.getLogger("cms.bin.translate") 35 logger = logging.getLogger("cms.bin.translate")
(...skipping 75 matching lines...) Expand 10 before | Expand all | Expand 10 after
110 comment += ", ".join("{%d}: %s" % i_s 111 comment += ", ".join("{%d}: %s" % i_s
111 for i_s in enumerate(fixed_strings, 1)) 112 for i_s in enumerate(fixed_strings, 1))
112 if comment: 113 if comment:
113 store[name]["description"] = comment 114 store[name]["description"] = comment
114 115
115 for page, format in source.list_pages(): 116 for page, format in source.list_pages():
116 cms.utils.process_page(source, defaultlocale, page, 117 cms.utils.process_page(source, defaultlocale, page,
117 format=format, localized_string_callback=record_strin g) 118 format=format, localized_string_callback=record_strin g)
118 return page_strings 119 return page_strings
119 120
120 def configure_locales(crowdin_api, required_locales, enabled_locales, 121 def configure_locales(crowdin_api, local_locales, enabled_locales,
121 defaultlocale): 122 defaultlocale):
122 logger.info("Checking which locales are supported by Crowdin...") 123 logger.info("Checking which locales are supported by Crowdin...")
123 response = crowdin_api.request("GET", "supported-languages") 124 response = crowdin_api.request("GET", "supported-languages")
124 125
125 supported_locales = {l["crowdin_code"] for l in response} 126 supported_locales = {l["crowdin_code"] for l in response}
126 skipped_locales = required_locales - supported_locales 127
128 # We need to map the locale names we use to the ones that Crowdin is expecting
129 # and at the same time ensure that they are supported.
130 required_locales = {}
131 skipped_locales = []
132 for locale in local_locales:
133 if "_" in locale:
134 crowdin_locale = locale.replace("_", "-")
135 else:
136 if locale in supported_locales:
137 required_locales[locale] = locale
138 continue
139 crowdin_locale = "%s-%s" % (locale, locale.upper())
140
141 if crowdin_locale in supported_locales:
142 required_locales[locale] = crowdin_locale
143 else:
144 skipped_locales.append(locale)
127 145
128 if skipped_locales: 146 if skipped_locales:
129 logger.warning("Ignoring locales that Crowdin doesn't support: %s", 147 logger.warning("Ignoring locales that Crowdin doesn't support: %s",
130 ", ".join(skipped_locales)) 148 ", ".join(skipped_locales))
131 required_locales -= skipped_locales
132 149
133 if not required_locales.issubset(enabled_locales): 150 required_crowdin_locales = set(required_locales.values())
151 if not required_crowdin_locales.issubset(enabled_locales):
134 logger.info("Enabling the required locales for the Crowdin project...") 152 logger.info("Enabling the required locales for the Crowdin project...")
135 crowdin_api.request( 153 crowdin_api.request(
136 "POST", "edit-project", 154 "POST", "edit-project",
137 data={"languages": enabled_locales | required_locales} 155 data={"languages": enabled_locales | required_crowdin_locales}
138 ) 156 )
139 157
140 return required_locales 158 return required_locales
141 159
142 def list_remote_files(project_info): 160 def list_remote_files(project_info):
143 def parse_file_node(node, path=""): 161 def parse_file_node(node, path=""):
144 if node["node_type"] == "file": 162 if node["node_type"] == "file":
145 remote_files.add(path + node["name"]) 163 remote_files.add(path + node["name"])
146 elif node["node_type"] == "directory": 164 elif node["node_type"] == "directory":
147 dir_name = path + node["name"] 165 dir_name = path + node["name"]
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after
191 209
192 def upload_translations(crowdin_api, source_dir, new_files, required_locales): 210 def upload_translations(crowdin_api, source_dir, new_files, required_locales):
193 def open_locale_files(locale, files): 211 def open_locale_files(locale, files):
194 for file_name in files: 212 for file_name in files:
195 path = os.path.join(source_dir, "locales", locale, file_name) 213 path = os.path.join(source_dir, "locales", locale, file_name)
196 if os.path.isfile(path): 214 if os.path.isfile(path):
197 with open(path, "rb") as f: 215 with open(path, "rb") as f:
198 yield (file_name, f.read(), "application/json") 216 yield (file_name, f.read(), "application/json")
199 217
200 if new_files: 218 if new_files:
201 for locale in required_locales: 219 for locale, crowdin_locale in required_locales.iteritems():
202 for files in grouper(open_locale_files(locale, new_files), 220 for files in grouper(open_locale_files(locale, new_files),
203 crowdin_api.FILES_PER_REQUEST): 221 crowdin_api.FILES_PER_REQUEST):
204 logger.info("Uploading %d existing translation " 222 logger.info("Uploading %d existing translation "
205 "files for locale %s...", len(files), locale) 223 "files for locale %s...", len(files), locale)
206 crowdin_api.request("POST", "upload-translation", files=files, 224 crowdin_api.request("POST", "upload-translation", files=files,
207 data={"language": locale}) 225 data={"language": crowdin_locale})
208 226
209 def remove_old_files(crowdin_api, old_files): 227 def remove_old_files(crowdin_api, old_files):
210 for file_name in old_files: 228 for file_name in old_files:
211 logger.info("Removing old file %s", file_name) 229 logger.info("Removing old file %s", file_name)
212 crowdin_api.request("POST", "delete-file", data={"file": file_name}) 230 crowdin_api.request("POST", "delete-file", data={"file": file_name})
213 231
214 def remove_old_directories(crowdin_api, old_directories): 232 def remove_old_directories(crowdin_api, old_directories):
215 for directory in reversed(sorted(old_directories, key=len)): 233 for directory in reversed(sorted(old_directories, key=len)):
216 logger.info("Removing old directory %s", directory) 234 logger.info("Removing old directory %s", directory)
217 crowdin_api.request("POST", "delete-directory", data={"name": directory}) 235 crowdin_api.request("POST", "delete-directory", data={"name": directory})
218 236
219 def download_translations(crowdin_api, source_dir, required_locales): 237 def download_translations(crowdin_api, source_dir, required_locales):
220 logger.info("Requesting generation of fresh translations archive...") 238 logger.info("Requesting generation of fresh translations archive...")
221 result = crowdin_api.request("GET", "export") 239 result = crowdin_api.request("GET", "export")
222 if result.get("success", {}).get("status") == "skipped": 240 if result.get("success", {}).get("status") == "skipped":
223 logger.warning("Archive generation skipped, either " 241 logger.warning("Archive generation skipped, either "
224 "no changes or API usage excessive") 242 "no changes or API usage excessive")
225 243
226 logger.info("Downloading translations archive...") 244 logger.info("Downloading translations archive...")
227 response = crowdin_api.raw_request("GET", "download/all.zip") 245 response = crowdin_api.raw_request("GET", "download/all.zip")
228 246
247 inverted_required_locales = {crowdin: local for local, crowdin in
248 required_locales.iteritems()}
229 logger.info("Extracting translations archive...") 249 logger.info("Extracting translations archive...")
230 with zipfile.ZipFile(io.BytesIO(response.data), "r") as archive: 250 with zipfile.ZipFile(io.BytesIO(response.data), "r") as archive:
231 locale_path = os.path.join(source_dir, "locales") 251 locale_path = os.path.join(source_dir, "locales")
232 # First clear existing translation files 252 # First clear existing translation files
233 for root, dirs, files in os.walk(locale_path, topdown=True): 253 for root, dirs, files in os.walk(locale_path, topdown=True):
234 if root == locale_path: 254 if root == locale_path:
235 dirs[:] = [d for d in dirs if d in required_locales] 255 dirs[:] = [d for d in dirs if d in required_locales]
236 for f in files: 256 for f in files:
237 if f.lower().endswith(".json"): 257 if f.lower().endswith(".json"):
238 os.remove(os.path.join(root, f)) 258 os.remove(os.path.join(root, f))
239 # Then extract the new ones in place 259 # Then extract the new ones in place
240 for member in archive.namelist(): 260 for member in archive.namelist():
241 path, file_name = posixpath.split(member) 261 path, file_name = posixpath.split(member)
242 ext = posixpath.splitext(file_name)[1] 262 ext = posixpath.splitext(file_name)[1]
243 locale = path.split(posixpath.sep)[0] 263 path_parts = path.split(posixpath.sep)
244 if ext.lower() == ".json" and locale in required_locales: 264 locale, file_path = path_parts[0], path_parts[1:]
245 archive.extract(member, locale_path) 265 if ext.lower() == ".json" and locale in inverted_required_locales:
266 output_path = os.path.join(
267 locale_path, inverted_required_locales[locale],
268 *file_path + [file_name]
269 )
270 with archive.open(member) as f, open(output_path, "wb") as f_target:
271 shutil.copyfileobj(f, f_target)
246 272
247 def crowdin_sync(source_dir, crowdin_api_key): 273 def crowdin_sync(source_dir, crowdin_api_key):
248 with FileSource(source_dir) as source: 274 with FileSource(source_dir) as source:
249 config = source.read_config() 275 config = source.read_config()
250 defaultlocale = config.get("general", "defaultlocale") 276 defaultlocale = config.get("general", "defaultlocale")
251 crowdin_project_name = config.get("general", "crowdin-project-name") 277 crowdin_project_name = config.get("general", "crowdin-project-name")
252 278
253 crowdin_api = CrowdinAPI(crowdin_api_key, crowdin_project_name) 279 crowdin_api = CrowdinAPI(crowdin_api_key, crowdin_project_name)
254 280
255 logger.info("Requesting project information...") 281 logger.info("Requesting project information...")
256 project_info = crowdin_api.request("GET", "info") 282 project_info = crowdin_api.request("GET", "info")
257 page_strings = extract_strings(source, defaultlocale) 283 page_strings = extract_strings(source, defaultlocale)
258 284
259 required_locales = {l for l in source.list_locales() if l != defaultlocale} 285 local_locales = source.list_locales() - {defaultlocale}
260 enabled_locales = {l["code"] for l in project_info["languages"]} 286 enabled_locales = {l["code"] for l in project_info["languages"]}
261 287
262 required_locales = configure_locales(crowdin_api, required_locales, 288 required_locales = configure_locales(crowdin_api, local_locales,
263 enabled_locales, defaultlocale) 289 enabled_locales, defaultlocale)
264 290
265 remote_files, remote_directories = list_remote_files(project_info) 291 remote_files, remote_directories = list_remote_files(project_info)
266 local_files, local_directories = list_local_files(page_strings) 292 local_files, local_directories = list_local_files(page_strings)
267 293
268 # Avoid deleting all remote content if there was a problem listing local files 294 # Avoid deleting all remote content if there was a problem listing local files
269 if not local_files: 295 if not local_files:
270 logger.error("No existing strings found, maybe the project directory is " 296 logger.error("No existing strings found, maybe the project directory is "
271 "not set up correctly? Aborting!") 297 "not set up correctly? Aborting!")
272 sys.exit(1) 298 sys.exit(1)
(...skipping 18 matching lines...) Expand all
291 if __name__ == "__main__": 317 if __name__ == "__main__":
292 if len(sys.argv) < 3: 318 if len(sys.argv) < 3:
293 print >>sys.stderr, "Usage: python -m cms.bin.translate www_directory crowdi n_project_api_key [logging_level]" 319 print >>sys.stderr, "Usage: python -m cms.bin.translate www_directory crowdi n_project_api_key [logging_level]"
294 sys.exit(1) 320 sys.exit(1)
295 321
296 logging.basicConfig() 322 logging.basicConfig()
297 logger.setLevel(sys.argv[3] if len(sys.argv) > 3 else logging.INFO) 323 logger.setLevel(sys.argv[3] if len(sys.argv) > 3 else logging.INFO)
298 324
299 source_dir, crowdin_api_key = sys.argv[1:3] 325 source_dir, crowdin_api_key = sys.argv[1:3]
300 crowdin_sync(source_dir, crowdin_api_key) 326 crowdin_sync(source_dir, crowdin_api_key)
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld