Moved Chrome extension scripts to buildtools repository

localeTools.py

 # coding: utf-8
 
 # This Source Code is subject to the terms of the Mozilla Public License
 # version 2.0 (the "License"). You can obtain a copy of the License at
 # http://mozilla.org/MPL/2.0/.
 
 import re, os, sys, codecs, json, urllib, urllib2
 from StringIO import StringIO
 from ConfigParser import SafeConfigParser
 from zipfile import ZipFile
@@ skipping 224 matching lines @@
   data = json.load(fileHandle)
   fileHandle.close()
 
   # Remove synced keys, these don't need to be translated
   if metadata.has_section('locale_sync'):
     for file, stringIDs in metadata.items('locale_sync'):
       for stringID in re.split(r'\s+', stringIDs):
         if file == 'remove':
           key = stringID
         else:
           key = re.sub(r'\..*', '', file) + '_' + re.sub(r'\W', '_', stringID)

[Felix Dahlke, 2013/01/10 16:36:20]

We have pretty much the same line in localeSyncChrome.py:82, can we avoid that?
To me, it looks like this part would feel more at home in localeSyncChrome.py
anyway - I think it makes sense to have creation and removal close to each
other.

[Wladimir Palant, 2013/01/16 14:20:15]

Yes, the idea is to make locale syncing part of the build process rather than a
separate manual step. Then we wouldn't have to keep duplicates of Firefox
translations in the repository and this entire pre-processing step would become
unnecessary.

         if key in data:
           del data[key]
 
   for key, value in data.iteritems():
     if isMaster:
       # Make sure the key name is listed in the description
       if "description" in value:
         value["description"] = "%s: %s" % (key, value["description"])
       else:
         value["description"] = key
     else:
       # Delete description from translations
       if "description" in value:
         del value["description"]
 
   return json.dumps(data, ensure_ascii=False, sort_keys=True, indent=2)
 
+def postprocessChromeLocale(path, data):
+  parsed = json.loads(data)
+
+  # Delete description from translations
+  for key, value in parsed.iteritems():
+    if "description" in value:
+      del value["description"]
+
+  file = codecs.open(path, 'wb', encoding='utf-8')
+  json.dump(parsed, file, ensure_ascii=False, sort_keys=True, indent=2, separators=(',', ': '))
+  file.close()
+
 def setupTranslations(type, locales, projectName, key):
   # Copy locales list, we don't want to change the parameter
   locales = set(locales)
 
   # Fill up with locales that we don't have but the browser supports
   if type == 'chrome':
     for locale in chromeLocales:
       locales.add(locale)
   else:
     firefoxLocales = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.html').read()
@@ skipping 17 matching lines @@
     print 'Warning, following locales aren\'t allowed by server: ' + ', '.join(locales - allowed)
 
   locales = list(locales & allowed)
   locales.sort()
   params = urllib.urlencode([('languages[]', locale) for locale in locales])
   result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/edit-project?key=%s&%s' % (projectName, key, params)).read()
   if result.find('<success') < 0:
     raise Exception('Server indicated that the operation was not successful\n' + result)
 
 def postFiles(files, url):
   boundary = '----------ThIs_Is_tHe_bouNdaRY_$'

[Felix Dahlke, 2013/01/10 16:36:20]

Would it make sense to generate this randomly?

[Wladimir Palant, 2013/01/16 14:20:15]

I'm not sure where this boundary string originates from but at the very least it
is the default boundary for Ruby's Net::HTTP class. Given that our payload is
known to be translations - it's a safe bet that we won't ever have issues using
that static boundary string.

If we want to implement this properly then we need to check the payload to
verify that it doesn't contain instances of our boundary string - and choose a
different boundary if it does. That's complicated enough that I didn't consider
it worth the effort in this case.

   body = ''
   for file, data in files:
     body += '--%s\r\n' % boundary
     body += 'Content-Disposition: form-data; name="files[%s]"; filename="%s"\r\n' % (file, file)
     body += 'Content-Type: application/octet-stream\r\n'
     body += 'Content-Transfer-Encoding: binary\r\n'
     body += '\r\n' + data + '\r\n'
     body += '--%s--\r\n' % boundary
 
   body = body.encode('utf-8')
@@ skipping 47 matching lines @@
         newName = file
       else:
         data = toJSON(path)
         newName = file + '.json'
 
       if data:
         files.append((newName, data))
   if len(files):
     postFiles(files, 'http://api.crowdin.net/api/project/%s/upload-translation?key=%s&language=%s' % (projectName, key, mapLocale(type, locale)))
 
-def getTranslations(localesDir, defaultLocale, projectName, key):
+def getTranslations(type, localesDir, defaultLocale, projectName, key):
   result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/export?key=%s' % (projectName, key)).read()
   if result.find('<success') < 0:
     raise Exception('Server indicated that the operation was not successful\n' + result)
 
   result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/download/all.zip?key=%s' % (projectName, key)).read()
   zip = ZipFile(StringIO(result))
   dirs = {}
   for info in zip.infolist():
-    if not info.filename.endswith('.dtd.json') and not info.filename.endswith('.properties.json'):
+    if not info.filename.endswith('.json'):
       continue
 
     dir, file = os.path.split(info.filename)
-    origFile = re.sub(r'\.json$', '', file)
     if not re.match(r'^[\w\-]+$', dir) or dir == defaultLocale:
       continue
-
-    for key, value in langMappingGecko.iteritems():
+    if type == 'chrome':
+      origFile = file
+    else:
+      origFile = re.sub(r'\.json$', '', file)
+      if not origFile.endswith('.dtd') and not origFile.endswith('.properties'):
+        continue
+
+    mapping = langMappingChrome if type == 'chrome' else langMappingGecko
+    for key, value in mapping.iteritems():
       if value == dir:
         dir = key
+    if type == 'chrome':
+      dir = dir.replace('-', '_')
+
+    data = zip.open(info.filename).read()
+    if data == '[]':
+      continue
 
     if not dir in dirs:
       dirs[dir] = set()
     dirs[dir].add(origFile)
 
-    data = zip.open(info.filename).read()
-    fromJSON(os.path.join(localesDir, dir, origFile), data)
+    path = os.path.join(localesDir, dir, origFile)
+    if not os.path.exists(os.path.dirname(path)):
+      os.makedirs(os.path.dirname(path))
+    if type == 'chrome':
+      postprocessChromeLocale(path, data)
+    else:
+      fromJSON(path, data)
 
   # Remove any extra files
   for dir, files in dirs.iteritems():
     baseDir = os.path.join(localesDir, dir)
     if not os.path.exists(baseDir):
       continue
     for file in os.listdir(baseDir):
       path = os.path.join(baseDir, file)
-      if os.path.isfile(path) and (file.endswith('.properties') or file.endswith('.dtd')) and not file in files:
+      if os.path.isfile(path) and (file.endswith('.json') or file.endswith('.properties') or file.endswith('.dtd')) and not file in files:
         os.remove(path)