Moved Chrome extension scripts to buildtools repository

localeTools.py

Index: localeTools.py
===================================================================
--- a/localeTools.py
+++ b/localeTools.py
@@ -5,21 +5,85 @@
 # http://mozilla.org/MPL/2.0/.
 
 import re, os, sys, codecs, json, urllib, urllib2
 from StringIO import StringIO
 from ConfigParser import SafeConfigParser
 from zipfile import ZipFile
 from xml.parsers.expat import ParserCreate, XML_PARAM_ENTITY_PARSING_ALWAYS
 
-langMapping = {
+langMappingGecko = {
   'dsb': 'dsb-DE',
   'hsb': 'hsb-DE',
 }
 
+langMappingChrome = {
+  'es-419': 'es-AR',
+  'es': 'es-ES',
+  'sv': 'sv-SE',
+  'ml': 'ml-IN',
+  'nb': 'no',
+}
+
+chromeLocales = [
+  "am",
+  "ar",
+  "bg",
+  "bn",
+  "ca",
+  "cs",
+  "da",
+  "de",
+  "el",
+  "en-GB",
+  "en-US",
+  "es-419",
+  "es",
+  "et",
+  "fa",
+  "fi",
+  "fil",
+  "fr",
+  "gu",
+  "he",
+  "hi",
+  "hr",
+  "hu",
+  "id",
+  "it",
+  "ja",
+  "kn",
+  "ko",
+  "lt",
+  "lv",
+  "ml",
+  "mr",
+  "ms",
+  "nb",
+  "nl",
+  "pl",
+  "pt-BR",
+  "pt-PT",
+  "ro",
+  "ru",
+  "sk",
+  "sl",
+  "sr",
+  "sv",
+  "sw",
+  "ta",
+  "te",
+  "th",
+  "tr",
+  "uk",
+  "vi",
+  "zh-CN",
+  "zh-TW",
+]
+
 class OrderedDict(dict):
   def __init__(self):
     self.__order = []
   def __setitem__(self, key, value):
     self.__order.append(key)
     dict.__setitem__(self, key, value)
   def iteritems(self):
     done = set()
@@ -29,16 +93,20 @@ class OrderedDict(dict):
         done.add(key)
 
 def escapeEntity(value):
   return value.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;')
 
 def unescapeEntity(value):
   return value.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').replace('&quot;', '"')
 
+def mapLocale(type, locale):
+  mapping = langMappingChrome if type == 'chrome' else langMappingGecko
+  return mapping.get(locale, locale)
+
 def parseDTDString(data, path):
   result = []
   currentComment = [None]
 
   parser = ParserCreate()
   parser.UseForeignDTD(True)
   parser.SetParamEntityParsing(XML_PARAM_ENTITY_PARSING_ALWAYS)
 
@@ -140,133 +208,226 @@ def toJSON(path):
   result = OrderedDict()
   for name, comment, value in it:
     obj = {'message': value}
     if comment == None:
       obj['description'] = name
     else:
       obj['description'] = '%s: %s' % (name, comment)
     result[name] = obj
-  return json.dumps(result, indent=2)
+  return json.dumps(result, ensure_ascii=False, indent=2)
 
 def fromJSON(path, data):
   data = json.loads(data)
   if not data:
     if os.path.exists(path):
       os.remove(path)
     return
 
   dir = os.path.dirname(path)
   if not os.path.exists(dir):
     os.makedirs(dir)
   file = codecs.open(path, 'wb', encoding='utf-8')
   for key, value in data.iteritems():
     file.write(generateStringEntry(key, value['message'], path))
   file.close()
 
-def setupTranslations(locales, projectName, key):
+def preprocessChromeLocale(path, metadata, isMaster):
+  fileHandle = codecs.open(path, 'rb', encoding='utf-8')
+  data = json.load(fileHandle)
+  fileHandle.close()
+
+  # Remove synced keys, these don't need to be translated
+  if metadata.has_section('locale_sync'):
+    for file, stringIDs in metadata.items('locale_sync'):
+      for stringID in re.split(r'\s+', stringIDs):
+        if file == 'remove':
+          key = stringID
+        else:
+          key = re.sub(r'\..*', '', file) + '_' + re.sub(r'\W', '_', stringID)

[Felix Dahlke, 2013/01/10 16:36:20]

We have pretty much the same line in localeSyncChrome.py:82, can we avoid that?
To me, it looks like this part would feel more at home in localeSyncChrome.py
anyway - I think it makes sense to have creation and removal close to each
other.

[Wladimir Palant, 2013/01/16 14:20:15]

Yes, the idea is to make locale syncing part of the build process rather than a
separate manual step. Then we wouldn't have to keep duplicates of Firefox
translations in the repository and this entire pre-processing step would become
unnecessary.

+        if key in data:
+          del data[key]
+
+  for key, value in data.iteritems():
+    if isMaster:
+      # Make sure the key name is listed in the description
+      if "description" in value:
+        value["description"] = "%s: %s" % (key, value["description"])
+      else:
+        value["description"] = key
+    else:
+      # Delete description from translations
+      if "description" in value:
+        del value["description"]
+
+  return json.dumps(data, ensure_ascii=False, sort_keys=True, indent=2)
+
+def postprocessChromeLocale(path, data):
+  parsed = json.loads(data)
+
+  # Delete description from translations
+  for key, value in parsed.iteritems():
+    if "description" in value:
+      del value["description"]
+
+  file = codecs.open(path, 'wb', encoding='utf-8')
+  json.dump(parsed, file, ensure_ascii=False, sort_keys=True, indent=2, separators=(',', ': '))
+  file.close()
+
+def setupTranslations(type, locales, projectName, key):
+  # Copy locales list, we don't want to change the parameter
   locales = set(locales)
-  firefoxLocales = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.html').read()
-  for match in re.finditer(r'&lang=([\w\-]+)"', firefoxLocales):
-    locales.add(langMapping.get(match.group(1), match.group(1)))
-  langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/language-tools/').read()
-  for match in re.finditer(r'<tr>.*?</tr>', langPacks, re.S):
-    if match.group(0).find('Install Language Pack') >= 0:
-      match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
-      if match2:
-        locales.add(langMapping.get(match2.group(1), match2.group(1)))
+
+  # Fill up with locales that we don't have but the browser supports
+  if type == 'chrome':
+    for locale in chromeLocales:
+      locales.add(locale)
+  else:
+    firefoxLocales = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.html').read()
+    for match in re.finditer(r'&amp;lang=([\w\-]+)"', firefoxLocales):
+      locales.add(mapLocale(type, match.group(1)))
+    langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/language-tools/').read()
+    for match in re.finditer(r'<tr>.*?</tr>', langPacks, re.S):
+      if match.group(0).find('Install Language Pack') >= 0:
+        match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
+        if match2:
+          locales.add(mapLocale(type, match2.group(1)))
+
+  # Convert locale codes to the ones that Crowdin will understand
+  locales = set(map(lambda locale: mapLocale(type, locale), locales))
 
   allowed = set()
   allowedLocales = urllib2.urlopen('http://crowdin.net/page/language-codes').read()
   for match in re.finditer(r'<tr>\s*<td>([\w\-]+)</td>', allowedLocales, re.S):
     allowed.add(match.group(1))
   if not allowed.issuperset(locales):
     print 'Warning, following locales aren\'t allowed by server: ' + ', '.join(locales - allowed)
 
   locales = list(locales & allowed)
   locales.sort()
   params = urllib.urlencode([('languages[]', locale) for locale in locales])
   result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/edit-project?key=%s&%s' % (projectName, key, params)).read()
   if result.find('<success') < 0:
     raise Exception('Server indicated that the operation was not successful\n' + result)
 
-def updateTranslationMaster(dir, locale, projectName, key):
+def postFiles(files, url):
+  boundary = '----------ThIs_Is_tHe_bouNdaRY_$'

[Felix Dahlke, 2013/01/10 16:36:20]

Would it make sense to generate this randomly?

[Wladimir Palant, 2013/01/16 14:20:15]

I'm not sure where this boundary string originates from but at the very least it
is the default boundary for Ruby's Net::HTTP class. Given that our payload is
known to be translations - it's a safe bet that we won't ever have issues using
that static boundary string.

If we want to implement this properly then we need to check the payload to
verify that it doesn't contain instances of our boundary string - and choose a
different boundary if it does. That's complicated enough that I didn't consider
it worth the effort in this case.

+  body = ''
+  for file, data in files:
+    body += '--%s\r\n' % boundary
+    body += 'Content-Disposition: form-data; name="files[%s]"; filename="%s"\r\n' % (file, file)
+    body += 'Content-Type: application/octet-stream\r\n'
+    body += 'Content-Transfer-Encoding: binary\r\n'
+    body += '\r\n' + data + '\r\n'
+    body += '--%s--\r\n' % boundary
+
+  body = body.encode('utf-8')
+  request = urllib2.Request(url, StringIO(body))
+  request.add_header('Content-Type', 'multipart/form-data; boundary=%s' % boundary)
+  request.add_header('Content-Length', len(body))
+  result = urllib2.urlopen(request).read()
+  if result.find('<success') < 0:
+    raise Exception('Server indicated that the operation was not successful\n' + result)
+
+def updateTranslationMaster(type, metadata, dir, projectName, key):
   result = json.load(urllib2.urlopen('http://api.crowdin.net/api/project/%s/info?key=%s&json=1' % (projectName, key)))
 
   existing = set(map(lambda f: f['name'], result['files']))
   add = []
   update = []
   for file in os.listdir(dir):
     path = os.path.join(dir, file)
     if os.path.isfile(path):
-      data = toJSON(path)
+      if type == 'chrome':
+        data = preprocessChromeLocale(path, metadata, True)
+        newName = file
+      else:
+        data = toJSON(path)
+        newName = file + '.json'
+
       if data:
-        newName = file + '.json'
         if newName in existing:
           update.append((newName, data))
           existing.remove(newName)
         else:
           add.append((newName, data))
 
-  def postFiles(files, url):
-    boundary = '----------ThIs_Is_tHe_bouNdaRY_$'
-    body = ''
-    for file, data in files:
-      body +=  '--%s\r\n' % boundary
-      body += 'Content-Disposition: form-data; name="files[%s]"; filename="%s"\r\n' % (file, file)
-      body += 'Content-Type: application/octet-stream\r\n'
-      body += '\r\n' + data.encode('utf-8') + '\r\n'
-      body += '--%s--\r\n' % boundary
-
-    request = urllib2.Request(url, body)
-    request.add_header('Content-Type', 'multipart/form-data; boundary=%s' % boundary)
-    request.add_header('Content-Length', len(body))
-    result = urllib2.urlopen(request).read()
-    if result.find('<success') < 0:
-      raise Exception('Server indicated that the operation was not successful\n' + result)
-
   if len(add):
     titles = urllib.urlencode([('titles[%s]' % name, re.sub(r'\.json', '', name)) for name, data in add])
     postFiles(add, 'http://api.crowdin.net/api/project/%s/add-file?key=%s&type=chrome&%s' % (projectName, key, titles))
   if len(update):
     postFiles(update, 'http://api.crowdin.net/api/project/%s/update-file?key=%s' % (projectName, key))
   for file in existing:
     result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/delete-file?key=%s&file=%s' % (projectName, key, file)).read()
     if result.find('<success') < 0:
       raise Exception('Server indicated that the operation was not successful\n' + result)
 
-def getTranslations(localesDir, defaultLocale, projectName, key):
+def uploadTranslations(type, metadata, dir, locale, projectName, key):
+  files = []
+  for file in os.listdir(dir):
+    path = os.path.join(dir, file)
+    if os.path.isfile(path):
+      if type == 'chrome':
+        data = preprocessChromeLocale(path, metadata, False)
+        newName = file
+      else:
+        data = toJSON(path)
+        newName = file + '.json'
+
+      if data:
+        files.append((newName, data))
+  if len(files):
+    postFiles(files, 'http://api.crowdin.net/api/project/%s/upload-translation?key=%s&language=%s' % (projectName, key, mapLocale(type, locale)))
+
+def getTranslations(type, localesDir, defaultLocale, projectName, key):
   result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/export?key=%s' % (projectName, key)).read()
   if result.find('<success') < 0:
     raise Exception('Server indicated that the operation was not successful\n' + result)
 
   result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/download/all.zip?key=%s' % (projectName, key)).read()
   zip = ZipFile(StringIO(result))
   dirs = {}
   for info in zip.infolist():
-    if not info.filename.endswith('.dtd.json') and not info.filename.endswith('.properties.json'):
+    if not info.filename.endswith('.json'):
       continue
 
     dir, file = os.path.split(info.filename)
-    origFile = re.sub(r'\.json$', '', file)
     if not re.match(r'^[\w\-]+$', dir) or dir == defaultLocale:
       continue
+    if type == 'chrome':
+      origFile = file
+    else:
+      origFile = re.sub(r'\.json$', '', file)
+      if not origFile.endswith('.dtd') and not origFile.endswith('.properties'):
+        continue
 
-    for key, value in langMapping.iteritems():
+    mapping = langMappingChrome if type == 'chrome' else langMappingGecko
+    for key, value in mapping.iteritems():
       if value == dir:
         dir = key
+    if type == 'chrome':
+      dir = dir.replace('-', '_')
+
+    data = zip.open(info.filename).read()
+    if data == '[]':
+      continue
 
     if not dir in dirs:
       dirs[dir] = set()
     dirs[dir].add(origFile)
 
-    data = zip.open(info.filename).read()
-    fromJSON(os.path.join(localesDir, dir, origFile), data)
+    path = os.path.join(localesDir, dir, origFile)
+    if not os.path.exists(os.path.dirname(path)):
+      os.makedirs(os.path.dirname(path))
+    if type == 'chrome':
+      postprocessChromeLocale(path, data)
+    else:
+      fromJSON(path, data)
 
   # Remove any extra files
   for dir, files in dirs.iteritems():
     baseDir = os.path.join(localesDir, dir)
     if not os.path.exists(baseDir):
       continue
     for file in os.listdir(baseDir):
       path = os.path.join(baseDir, file)
-      if os.path.isfile(path) and (file.endswith('.properties') or file.endswith('.dtd')) and not file in files:
+      if os.path.isfile(path) and (file.endswith('.json') or file.endswith('.properties') or file.endswith('.dtd')) and not file in files:
         os.remove(path)