Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Unified Diff: localeTools.py

Issue 8627097: Moved Chrome extension scripts to buildtools repository (Closed)
Patch Set: Created Oct. 19, 2012, 10:42 a.m.
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « localeSyncChrome.py ('k') | packagerChrome.py » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: localeTools.py
===================================================================
--- a/localeTools.py
+++ b/localeTools.py
@@ -5,21 +5,85 @@
# http://mozilla.org/MPL/2.0/.
import re, os, sys, codecs, json, urllib, urllib2
from StringIO import StringIO
from ConfigParser import SafeConfigParser
from zipfile import ZipFile
from xml.parsers.expat import ParserCreate, XML_PARAM_ENTITY_PARSING_ALWAYS
-langMapping = {
+langMappingGecko = {
'dsb': 'dsb-DE',
'hsb': 'hsb-DE',
}
+langMappingChrome = {
+ 'es-419': 'es-AR',
+ 'es': 'es-ES',
+ 'sv': 'sv-SE',
+ 'ml': 'ml-IN',
+ 'nb': 'no',
+}
+
+chromeLocales = [
+ "am",
+ "ar",
+ "bg",
+ "bn",
+ "ca",
+ "cs",
+ "da",
+ "de",
+ "el",
+ "en-GB",
+ "en-US",
+ "es-419",
+ "es",
+ "et",
+ "fa",
+ "fi",
+ "fil",
+ "fr",
+ "gu",
+ "he",
+ "hi",
+ "hr",
+ "hu",
+ "id",
+ "it",
+ "ja",
+ "kn",
+ "ko",
+ "lt",
+ "lv",
+ "ml",
+ "mr",
+ "ms",
+ "nb",
+ "nl",
+ "pl",
+ "pt-BR",
+ "pt-PT",
+ "ro",
+ "ru",
+ "sk",
+ "sl",
+ "sr",
+ "sv",
+ "sw",
+ "ta",
+ "te",
+ "th",
+ "tr",
+ "uk",
+ "vi",
+ "zh-CN",
+ "zh-TW",
+]
+
class OrderedDict(dict):
def __init__(self):
self.__order = []
def __setitem__(self, key, value):
self.__order.append(key)
dict.__setitem__(self, key, value)
def iteritems(self):
done = set()
@@ -29,16 +93,20 @@ class OrderedDict(dict):
done.add(key)
def escapeEntity(value):
return value.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;')
def unescapeEntity(value):
return value.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').replace('&quot;', '"')
+def mapLocale(type, locale):
+ mapping = langMappingChrome if type == 'chrome' else langMappingGecko
+ return mapping.get(locale, locale)
+
def parseDTDString(data, path):
result = []
currentComment = [None]
parser = ParserCreate()
parser.UseForeignDTD(True)
parser.SetParamEntityParsing(XML_PARAM_ENTITY_PARSING_ALWAYS)
@@ -140,104 +208,168 @@ def toJSON(path):
result = OrderedDict()
for name, comment, value in it:
obj = {'message': value}
if comment == None:
obj['description'] = name
else:
obj['description'] = '%s: %s' % (name, comment)
result[name] = obj
- return json.dumps(result, indent=2)
+ return json.dumps(result, ensure_ascii=False, indent=2)
def fromJSON(path, data):
data = json.loads(data)
if not data:
if os.path.exists(path):
os.remove(path)
return
dir = os.path.dirname(path)
if not os.path.exists(dir):
os.makedirs(dir)
file = codecs.open(path, 'wb', encoding='utf-8')
for key, value in data.iteritems():
file.write(generateStringEntry(key, value['message'], path))
file.close()
-def setupTranslations(locales, projectName, key):
+def preprocessChromeLocale(path, metadata, isMaster):
+ fileHandle = codecs.open(path, 'rb', encoding='utf-8')
+ data = json.load(fileHandle)
+ fileHandle.close()
+
+ # Remove synced keys, these don't need to be translated
+ if metadata.has_section('locale_sync'):
+ for file, stringIDs in metadata.items('locale_sync'):
+ for stringID in re.split(r'\s+', stringIDs):
+ if file == 'remove':
+ key = stringID
+ else:
+ key = re.sub(r'\..*', '', file) + '_' + re.sub(r'\W', '_', stringID)
+ if key in data:
+ del data[key]
+
+ for key, value in data.iteritems():
+ if isMaster:
+ # Make sure the key name is listed in the description
+ if "description" in value:
+ value["description"] = "%s: %s" % (key, value["description"])
+ else:
+ value["description"] = key
+ else:
+ # Delete description from translations
+ if "description" in value:
+ del value["description"]
+
+ return json.dumps(data, ensure_ascii=False, sort_keys=True, indent=2)
+
+def setupTranslations(type, locales, projectName, key):
+ # Copy locales list, we don't want to change the parameter
locales = set(locales)
- firefoxLocales = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.html').read()
- for match in re.finditer(r'&amp;lang=([\w\-]+)"', firefoxLocales):
- locales.add(langMapping.get(match.group(1), match.group(1)))
- langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/language-tools/').read()
- for match in re.finditer(r'<tr>.*?</tr>', langPacks, re.S):
- if match.group(0).find('Install Language Pack') >= 0:
- match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
- if match2:
- locales.add(langMapping.get(match2.group(1), match2.group(1)))
+
+ # Fill up with locales that we don't have but the browser supports
+ if type == 'chrome':
+ for locale in chromeLocales:
+ locales.add(locale)
+ else:
+ firefoxLocales = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.html').read()
+ for match in re.finditer(r'&amp;lang=([\w\-]+)"', firefoxLocales):
+ locales.add(mapLocale(type, match.group(1)))
+ langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/language-tools/').read()
+ for match in re.finditer(r'<tr>.*?</tr>', langPacks, re.S):
+ if match.group(0).find('Install Language Pack') >= 0:
+ match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
+ if match2:
+ locales.add(mapLocale(type, match2.group(1)))
+
+ # Convert locale codes to the ones that Crowdin will understand
+ locales = set(map(lambda locale: mapLocale(type, locale), locales))
allowed = set()
allowedLocales = urllib2.urlopen('http://crowdin.net/page/language-codes').read()
for match in re.finditer(r'<tr>\s*<td>([\w\-]+)</td>', allowedLocales, re.S):
allowed.add(match.group(1))
if not allowed.issuperset(locales):
print 'Warning, following locales aren\'t allowed by server: ' + ', '.join(locales - allowed)
locales = list(locales & allowed)
locales.sort()
params = urllib.urlencode([('languages[]', locale) for locale in locales])
result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/edit-project?key=%s&%s' % (projectName, key, params)).read()
if result.find('<success') < 0:
raise Exception('Server indicated that the operation was not successful\n' + result)
-def updateTranslationMaster(dir, locale, projectName, key):
+def postFiles(files, url):
+ boundary = '----------ThIs_Is_tHe_bouNdaRY_$'
+ body = ''
+ for file, data in files:
+ body += '--%s\r\n' % boundary
+ body += 'Content-Disposition: form-data; name="files[%s]"; filename="%s"\r\n' % (file, file)
+ body += 'Content-Type: application/octet-stream\r\n'
+ body += 'Content-Transfer-Encoding: binary\r\n'
+ body += '\r\n' + data + '\r\n'
+ body += '--%s--\r\n' % boundary
+
+ body = body.encode('utf-8')
+ request = urllib2.Request(url, StringIO(body))
+ request.add_header('Content-Type', 'multipart/form-data; boundary=%s' % boundary)
+ request.add_header('Content-Length', len(body))
+ result = urllib2.urlopen(request).read()
+ if result.find('<success') < 0:
+ raise Exception('Server indicated that the operation was not successful\n' + result)
+
+def updateTranslationMaster(type, metadata, dir, projectName, key):
result = json.load(urllib2.urlopen('http://api.crowdin.net/api/project/%s/info?key=%s&json=1' % (projectName, key)))
existing = set(map(lambda f: f['name'], result['files']))
add = []
update = []
for file in os.listdir(dir):
path = os.path.join(dir, file)
if os.path.isfile(path):
- data = toJSON(path)
+ if type == 'chrome':
+ data = preprocessChromeLocale(path, metadata, True)
+ newName = file
+ else:
+ data = toJSON(path)
+ newName = file + '.json'
+
if data:
- newName = file + '.json'
if newName in existing:
update.append((newName, data))
existing.remove(newName)
else:
add.append((newName, data))
- def postFiles(files, url):
- boundary = '----------ThIs_Is_tHe_bouNdaRY_$'
- body = ''
- for file, data in files:
- body += '--%s\r\n' % boundary
- body += 'Content-Disposition: form-data; name="files[%s]"; filename="%s"\r\n' % (file, file)
- body += 'Content-Type: application/octet-stream\r\n'
- body += '\r\n' + data.encode('utf-8') + '\r\n'
- body += '--%s--\r\n' % boundary
-
- request = urllib2.Request(url, body)
- request.add_header('Content-Type', 'multipart/form-data; boundary=%s' % boundary)
- request.add_header('Content-Length', len(body))
- result = urllib2.urlopen(request).read()
- if result.find('<success') < 0:
- raise Exception('Server indicated that the operation was not successful\n' + result)
-
if len(add):
titles = urllib.urlencode([('titles[%s]' % name, re.sub(r'\.json', '', name)) for name, data in add])
postFiles(add, 'http://api.crowdin.net/api/project/%s/add-file?key=%s&type=chrome&%s' % (projectName, key, titles))
if len(update):
postFiles(update, 'http://api.crowdin.net/api/project/%s/update-file?key=%s' % (projectName, key))
for file in existing:
result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/delete-file?key=%s&file=%s' % (projectName, key, file)).read()
if result.find('<success') < 0:
raise Exception('Server indicated that the operation was not successful\n' + result)
+def uploadTranslations(type, metadata, dir, locale, projectName, key):
+ files = []
+ for file in os.listdir(dir):
+ path = os.path.join(dir, file)
+ if os.path.isfile(path):
+ if type == 'chrome':
+ data = preprocessChromeLocale(path, metadata, False)
+ newName = file
+ else:
+ data = toJSON(path)
+ newName = file + '.json'
+
+ if data:
+ files.append((newName, data))
+ if len(files):
+ postFiles(files, 'http://api.crowdin.net/api/project/%s/upload-translation?key=%s&language=%s' % (projectName, key, mapLocale(type, locale)))
+
def getTranslations(localesDir, defaultLocale, projectName, key):
result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/export?key=%s' % (projectName, key)).read()
if result.find('<success') < 0:
raise Exception('Server indicated that the operation was not successful\n' + result)
result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/download/all.zip?key=%s' % (projectName, key)).read()
zip = ZipFile(StringIO(result))
dirs = {}
@@ -245,17 +377,17 @@ def getTranslations(localesDir, defaultL
if not info.filename.endswith('.dtd.json') and not info.filename.endswith('.properties.json'):
continue
dir, file = os.path.split(info.filename)
origFile = re.sub(r'\.json$', '', file)
if not re.match(r'^[\w\-]+$', dir) or dir == defaultLocale:
continue
- for key, value in langMapping.iteritems():
+ for key, value in langMappingGecko.iteritems():
if value == dir:
dir = key
if not dir in dirs:
dirs[dir] = set()
dirs[dir].add(origFile)
data = zip.open(info.filename).read()
« no previous file with comments | « localeSyncChrome.py ('k') | packagerChrome.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld