Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: localeTools.py

Issue 8627097: Moved Chrome extension scripts to buildtools repository (Closed)
Patch Set: Created Oct. 19, 2012, 10:42 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « localeSyncChrome.py ('k') | packagerChrome.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # coding: utf-8 1 # coding: utf-8
2 2
3 # This Source Code is subject to the terms of the Mozilla Public License 3 # This Source Code is subject to the terms of the Mozilla Public License
4 # version 2.0 (the "License"). You can obtain a copy of the License at 4 # version 2.0 (the "License"). You can obtain a copy of the License at
5 # http://mozilla.org/MPL/2.0/. 5 # http://mozilla.org/MPL/2.0/.
6 6
7 import re, os, sys, codecs, json, urllib, urllib2 7 import re, os, sys, codecs, json, urllib, urllib2
8 from StringIO import StringIO 8 from StringIO import StringIO
9 from ConfigParser import SafeConfigParser 9 from ConfigParser import SafeConfigParser
10 from zipfile import ZipFile 10 from zipfile import ZipFile
11 from xml.parsers.expat import ParserCreate, XML_PARAM_ENTITY_PARSING_ALWAYS 11 from xml.parsers.expat import ParserCreate, XML_PARAM_ENTITY_PARSING_ALWAYS
12 12
13 langMapping = { 13 langMappingGecko = {
14 'dsb': 'dsb-DE', 14 'dsb': 'dsb-DE',
15 'hsb': 'hsb-DE', 15 'hsb': 'hsb-DE',
16 } 16 }
17 17
18 langMappingChrome = {
19 'es-419': 'es-AR',
20 'es': 'es-ES',
21 'sv': 'sv-SE',
22 'ml': 'ml-IN',
23 'nb': 'no',
24 }
25
26 chromeLocales = [
27 "am",
28 "ar",
29 "bg",
30 "bn",
31 "ca",
32 "cs",
33 "da",
34 "de",
35 "el",
36 "en-GB",
37 "en-US",
38 "es-419",
39 "es",
40 "et",
41 "fa",
42 "fi",
43 "fil",
44 "fr",
45 "gu",
46 "he",
47 "hi",
48 "hr",
49 "hu",
50 "id",
51 "it",
52 "ja",
53 "kn",
54 "ko",
55 "lt",
56 "lv",
57 "ml",
58 "mr",
59 "ms",
60 "nb",
61 "nl",
62 "pl",
63 "pt-BR",
64 "pt-PT",
65 "ro",
66 "ru",
67 "sk",
68 "sl",
69 "sr",
70 "sv",
71 "sw",
72 "ta",
73 "te",
74 "th",
75 "tr",
76 "uk",
77 "vi",
78 "zh-CN",
79 "zh-TW",
80 ]
81
18 class OrderedDict(dict): 82 class OrderedDict(dict):
19 def __init__(self): 83 def __init__(self):
20 self.__order = [] 84 self.__order = []
21 def __setitem__(self, key, value): 85 def __setitem__(self, key, value):
22 self.__order.append(key) 86 self.__order.append(key)
23 dict.__setitem__(self, key, value) 87 dict.__setitem__(self, key, value)
24 def iteritems(self): 88 def iteritems(self):
25 done = set() 89 done = set()
26 for key in self.__order: 90 for key in self.__order:
27 if not key in done and key in self: 91 if not key in done and key in self:
28 yield (key, self[key]) 92 yield (key, self[key])
29 done.add(key) 93 done.add(key)
30 94
31 def escapeEntity(value): 95 def escapeEntity(value):
32 return value.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').r eplace('"', '&quot;') 96 return value.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').r eplace('"', '&quot;')
33 97
34 def unescapeEntity(value): 98 def unescapeEntity(value):
35 return value.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').r eplace('&quot;', '"') 99 return value.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').r eplace('&quot;', '"')
36 100
101 def mapLocale(type, locale):
102 mapping = langMappingChrome if type == 'chrome' else langMappingGecko
103 return mapping.get(locale, locale)
104
37 def parseDTDString(data, path): 105 def parseDTDString(data, path):
38 result = [] 106 result = []
39 currentComment = [None] 107 currentComment = [None]
40 108
41 parser = ParserCreate() 109 parser = ParserCreate()
42 parser.UseForeignDTD(True) 110 parser.UseForeignDTD(True)
43 parser.SetParamEntityParsing(XML_PARAM_ENTITY_PARSING_ALWAYS) 111 parser.SetParamEntityParsing(XML_PARAM_ENTITY_PARSING_ALWAYS)
44 112
45 def ExternalEntityRefHandler(context, base, systemId, publicId): 113 def ExternalEntityRefHandler(context, base, systemId, publicId):
46 subparser = parser.ExternalEntityParserCreate(context, 'utf-8') 114 subparser = parser.ExternalEntityParserCreate(context, 'utf-8')
(...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after
138 return None 206 return None
139 207
140 result = OrderedDict() 208 result = OrderedDict()
141 for name, comment, value in it: 209 for name, comment, value in it:
142 obj = {'message': value} 210 obj = {'message': value}
143 if comment == None: 211 if comment == None:
144 obj['description'] = name 212 obj['description'] = name
145 else: 213 else:
146 obj['description'] = '%s: %s' % (name, comment) 214 obj['description'] = '%s: %s' % (name, comment)
147 result[name] = obj 215 result[name] = obj
148 return json.dumps(result, indent=2) 216 return json.dumps(result, ensure_ascii=False, indent=2)
149 217
150 def fromJSON(path, data): 218 def fromJSON(path, data):
151 data = json.loads(data) 219 data = json.loads(data)
152 if not data: 220 if not data:
153 if os.path.exists(path): 221 if os.path.exists(path):
154 os.remove(path) 222 os.remove(path)
155 return 223 return
156 224
157 dir = os.path.dirname(path) 225 dir = os.path.dirname(path)
158 if not os.path.exists(dir): 226 if not os.path.exists(dir):
159 os.makedirs(dir) 227 os.makedirs(dir)
160 file = codecs.open(path, 'wb', encoding='utf-8') 228 file = codecs.open(path, 'wb', encoding='utf-8')
161 for key, value in data.iteritems(): 229 for key, value in data.iteritems():
162 file.write(generateStringEntry(key, value['message'], path)) 230 file.write(generateStringEntry(key, value['message'], path))
163 file.close() 231 file.close()
164 232
165 def setupTranslations(locales, projectName, key): 233 def preprocessChromeLocale(path, metadata, isMaster):
234 fileHandle = codecs.open(path, 'rb', encoding='utf-8')
235 data = json.load(fileHandle)
236 fileHandle.close()
237
238 # Remove synced keys, these don't need to be translated
239 if metadata.has_section('locale_sync'):
240 for file, stringIDs in metadata.items('locale_sync'):
241 for stringID in re.split(r'\s+', stringIDs):
242 if file == 'remove':
243 key = stringID
244 else:
245 key = re.sub(r'\..*', '', file) + '_' + re.sub(r'\W', '_', stringID)
246 if key in data:
247 del data[key]
248
249 for key, value in data.iteritems():
250 if isMaster:
251 # Make sure the key name is listed in the description
252 if "description" in value:
253 value["description"] = "%s: %s" % (key, value["description"])
254 else:
255 value["description"] = key
256 else:
257 # Delete description from translations
258 if "description" in value:
259 del value["description"]
260
261 return json.dumps(data, ensure_ascii=False, sort_keys=True, indent=2)
262
263 def setupTranslations(type, locales, projectName, key):
264 # Copy locales list, we don't want to change the parameter
166 locales = set(locales) 265 locales = set(locales)
167 firefoxLocales = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.htm l').read() 266
168 for match in re.finditer(r'&amp;lang=([\w\-]+)"', firefoxLocales): 267 # Fill up with locales that we don't have but the browser supports
169 locales.add(langMapping.get(match.group(1), match.group(1))) 268 if type == 'chrome':
170 langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/language -tools/').read() 269 for locale in chromeLocales:
171 for match in re.finditer(r'<tr>.*?</tr>', langPacks, re.S): 270 locales.add(locale)
172 if match.group(0).find('Install Language Pack') >= 0: 271 else:
173 match2 = re.search(r'lang="([\w\-]+)"', match.group(0)) 272 firefoxLocales = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.h tml').read()
174 if match2: 273 for match in re.finditer(r'&amp;lang=([\w\-]+)"', firefoxLocales):
175 locales.add(langMapping.get(match2.group(1), match2.group(1))) 274 locales.add(mapLocale(type, match.group(1)))
275 langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/langua ge-tools/').read()
276 for match in re.finditer(r'<tr>.*?</tr>', langPacks, re.S):
277 if match.group(0).find('Install Language Pack') >= 0:
278 match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
279 if match2:
280 locales.add(mapLocale(type, match2.group(1)))
281
282 # Convert locale codes to the ones that Crowdin will understand
283 locales = set(map(lambda locale: mapLocale(type, locale), locales))
176 284
177 allowed = set() 285 allowed = set()
178 allowedLocales = urllib2.urlopen('http://crowdin.net/page/language-codes').rea d() 286 allowedLocales = urllib2.urlopen('http://crowdin.net/page/language-codes').rea d()
179 for match in re.finditer(r'<tr>\s*<td>([\w\-]+)</td>', allowedLocales, re.S): 287 for match in re.finditer(r'<tr>\s*<td>([\w\-]+)</td>', allowedLocales, re.S):
180 allowed.add(match.group(1)) 288 allowed.add(match.group(1))
181 if not allowed.issuperset(locales): 289 if not allowed.issuperset(locales):
182 print 'Warning, following locales aren\'t allowed by server: ' + ', '.join(l ocales - allowed) 290 print 'Warning, following locales aren\'t allowed by server: ' + ', '.join(l ocales - allowed)
183 291
184 locales = list(locales & allowed) 292 locales = list(locales & allowed)
185 locales.sort() 293 locales.sort()
186 params = urllib.urlencode([('languages[]', locale) for locale in locales]) 294 params = urllib.urlencode([('languages[]', locale) for locale in locales])
187 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/edit-project?k ey=%s&%s' % (projectName, key, params)).read() 295 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/edit-project?k ey=%s&%s' % (projectName, key, params)).read()
188 if result.find('<success') < 0: 296 if result.find('<success') < 0:
189 raise Exception('Server indicated that the operation was not successful\n' + result) 297 raise Exception('Server indicated that the operation was not successful\n' + result)
190 298
191 def updateTranslationMaster(dir, locale, projectName, key): 299 def postFiles(files, url):
300 boundary = '----------ThIs_Is_tHe_bouNdaRY_$'
301 body = ''
302 for file, data in files:
303 body += '--%s\r\n' % boundary
304 body += 'Content-Disposition: form-data; name="files[%s]"; filename="%s"\r\n ' % (file, file)
305 body += 'Content-Type: application/octet-stream\r\n'
306 body += 'Content-Transfer-Encoding: binary\r\n'
307 body += '\r\n' + data + '\r\n'
308 body += '--%s--\r\n' % boundary
309
310 body = body.encode('utf-8')
311 request = urllib2.Request(url, StringIO(body))
312 request.add_header('Content-Type', 'multipart/form-data; boundary=%s' % bounda ry)
313 request.add_header('Content-Length', len(body))
314 result = urllib2.urlopen(request).read()
315 if result.find('<success') < 0:
316 raise Exception('Server indicated that the operation was not successful\n' + result)
317
318 def updateTranslationMaster(type, metadata, dir, projectName, key):
192 result = json.load(urllib2.urlopen('http://api.crowdin.net/api/project/%s/info ?key=%s&json=1' % (projectName, key))) 319 result = json.load(urllib2.urlopen('http://api.crowdin.net/api/project/%s/info ?key=%s&json=1' % (projectName, key)))
193 320
194 existing = set(map(lambda f: f['name'], result['files'])) 321 existing = set(map(lambda f: f['name'], result['files']))
195 add = [] 322 add = []
196 update = [] 323 update = []
197 for file in os.listdir(dir): 324 for file in os.listdir(dir):
198 path = os.path.join(dir, file) 325 path = os.path.join(dir, file)
199 if os.path.isfile(path): 326 if os.path.isfile(path):
200 data = toJSON(path) 327 if type == 'chrome':
328 data = preprocessChromeLocale(path, metadata, True)
329 newName = file
330 else:
331 data = toJSON(path)
332 newName = file + '.json'
333
201 if data: 334 if data:
202 newName = file + '.json'
203 if newName in existing: 335 if newName in existing:
204 update.append((newName, data)) 336 update.append((newName, data))
205 existing.remove(newName) 337 existing.remove(newName)
206 else: 338 else:
207 add.append((newName, data)) 339 add.append((newName, data))
208 340
209 def postFiles(files, url):
210 boundary = '----------ThIs_Is_tHe_bouNdaRY_$'
211 body = ''
212 for file, data in files:
213 body += '--%s\r\n' % boundary
214 body += 'Content-Disposition: form-data; name="files[%s]"; filename="%s"\r \n' % (file, file)
215 body += 'Content-Type: application/octet-stream\r\n'
216 body += '\r\n' + data.encode('utf-8') + '\r\n'
217 body += '--%s--\r\n' % boundary
218
219 request = urllib2.Request(url, body)
220 request.add_header('Content-Type', 'multipart/form-data; boundary=%s' % boun dary)
221 request.add_header('Content-Length', len(body))
222 result = urllib2.urlopen(request).read()
223 if result.find('<success') < 0:
224 raise Exception('Server indicated that the operation was not successful\n' + result)
225
226 if len(add): 341 if len(add):
227 titles = urllib.urlencode([('titles[%s]' % name, re.sub(r'\.json', '', name) ) for name, data in add]) 342 titles = urllib.urlencode([('titles[%s]' % name, re.sub(r'\.json', '', name) ) for name, data in add])
228 postFiles(add, 'http://api.crowdin.net/api/project/%s/add-file?key=%s&type=c hrome&%s' % (projectName, key, titles)) 343 postFiles(add, 'http://api.crowdin.net/api/project/%s/add-file?key=%s&type=c hrome&%s' % (projectName, key, titles))
229 if len(update): 344 if len(update):
230 postFiles(update, 'http://api.crowdin.net/api/project/%s/update-file?key=%s' % (projectName, key)) 345 postFiles(update, 'http://api.crowdin.net/api/project/%s/update-file?key=%s' % (projectName, key))
231 for file in existing: 346 for file in existing:
232 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/delete-file? key=%s&file=%s' % (projectName, key, file)).read() 347 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/delete-file? key=%s&file=%s' % (projectName, key, file)).read()
233 if result.find('<success') < 0: 348 if result.find('<success') < 0:
234 raise Exception('Server indicated that the operation was not successful\n' + result) 349 raise Exception('Server indicated that the operation was not successful\n' + result)
235 350
351 def uploadTranslations(type, metadata, dir, locale, projectName, key):
352 files = []
353 for file in os.listdir(dir):
354 path = os.path.join(dir, file)
355 if os.path.isfile(path):
356 if type == 'chrome':
357 data = preprocessChromeLocale(path, metadata, False)
358 newName = file
359 else:
360 data = toJSON(path)
361 newName = file + '.json'
362
363 if data:
364 files.append((newName, data))
365 if len(files):
366 postFiles(files, 'http://api.crowdin.net/api/project/%s/upload-translation?k ey=%s&language=%s' % (projectName, key, mapLocale(type, locale)))
367
236 def getTranslations(localesDir, defaultLocale, projectName, key): 368 def getTranslations(localesDir, defaultLocale, projectName, key):
237 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/export?key=%s' % (projectName, key)).read() 369 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/export?key=%s' % (projectName, key)).read()
238 if result.find('<success') < 0: 370 if result.find('<success') < 0:
239 raise Exception('Server indicated that the operation was not successful\n' + result) 371 raise Exception('Server indicated that the operation was not successful\n' + result)
240 372
241 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/download/all.z ip?key=%s' % (projectName, key)).read() 373 result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/download/all.z ip?key=%s' % (projectName, key)).read()
242 zip = ZipFile(StringIO(result)) 374 zip = ZipFile(StringIO(result))
243 dirs = {} 375 dirs = {}
244 for info in zip.infolist(): 376 for info in zip.infolist():
245 if not info.filename.endswith('.dtd.json') and not info.filename.endswith('. properties.json'): 377 if not info.filename.endswith('.dtd.json') and not info.filename.endswith('. properties.json'):
246 continue 378 continue
247 379
248 dir, file = os.path.split(info.filename) 380 dir, file = os.path.split(info.filename)
249 origFile = re.sub(r'\.json$', '', file) 381 origFile = re.sub(r'\.json$', '', file)
250 if not re.match(r'^[\w\-]+$', dir) or dir == defaultLocale: 382 if not re.match(r'^[\w\-]+$', dir) or dir == defaultLocale:
251 continue 383 continue
252 384
253 for key, value in langMapping.iteritems(): 385 for key, value in langMappingGecko.iteritems():
254 if value == dir: 386 if value == dir:
255 dir = key 387 dir = key
256 388
257 if not dir in dirs: 389 if not dir in dirs:
258 dirs[dir] = set() 390 dirs[dir] = set()
259 dirs[dir].add(origFile) 391 dirs[dir].add(origFile)
260 392
261 data = zip.open(info.filename).read() 393 data = zip.open(info.filename).read()
262 fromJSON(os.path.join(localesDir, dir, origFile), data) 394 fromJSON(os.path.join(localesDir, dir, origFile), data)
263 395
264 # Remove any extra files 396 # Remove any extra files
265 for dir, files in dirs.iteritems(): 397 for dir, files in dirs.iteritems():
266 baseDir = os.path.join(localesDir, dir) 398 baseDir = os.path.join(localesDir, dir)
267 if not os.path.exists(baseDir): 399 if not os.path.exists(baseDir):
268 continue 400 continue
269 for file in os.listdir(baseDir): 401 for file in os.listdir(baseDir):
270 path = os.path.join(baseDir, file) 402 path = os.path.join(baseDir, file)
271 if os.path.isfile(path) and (file.endswith('.properties') or file.endswith ('.dtd')) and not file in files: 403 if os.path.isfile(path) and (file.endswith('.properties') or file.endswith ('.dtd')) and not file in files:
272 os.remove(path) 404 os.remove(path)
OLDNEW
« no previous file with comments | « localeSyncChrome.py ('k') | packagerChrome.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld