Issue 1709 - Allow importing Chrome-style locales in Chrome extensions

packagerChrome.py

Index: packagerChrome.py
===================================================================
--- a/packagerChrome.py
+++ b/packagerChrome.py
@@ -1,15 +1,20 @@
 # coding: utf-8
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import sys, os, re, json, struct
+import sys
+import os
+import re
+import json
+import struct
+import io
 from StringIO import StringIO
 
 import packager
 from packager import readMetadata, getMetadataPath, getDefaultFileName, getBuildVersion, getTemplate, Files
 
 defaultLocale = 'en_US'
 
 def getIgnoredFiles(params):
@@ -157,78 +162,60 @@ def toJson(data):
   return json.dumps(
     data, ensure_ascii=False, sort_keys=True,
     indent=2, separators=(',', ': ')
   ).encode('utf-8') + '\n'
 
 def importGeckoLocales(params, files):
   import localeTools
 
-  localeCodeMapping = {
-    'ar': 'ar',
-    'bg': 'bg',
-    'ca': 'ca',
-    'cs': 'cs',
-    'da': 'da',
-    'de': 'de',
-    'el': 'el',
-    'en-US': 'en_US',
-    'en-GB': 'en_GB',
-    'es-ES': 'es',
-    'es-AR': 'es_419',
-    'et': 'et',
-    'fi': 'fi',
-  #   '': 'fil', ???
-    'fr': 'fr',
-    'he': 'he',
-    'hi-IN': 'hi',
-    'hr': 'hr',
-    'hu': 'hu',
-    'id': 'id',
-    'it': 'it',
-    'ja': 'ja',
-    'ko': 'ko',
-    'lt': 'lt',
-    'lv': 'lv',
-    'nl': 'nl',
-  #    'nb-NO': 'no', ???
-    'pl': 'pl',
-    'pt-BR': 'pt_BR',
-    'pt-PT': 'pt_PT',
-    'ro': 'ro',
-    'ru': 'ru',
-    'sk': 'sk',
-    'sl': 'sl',
-    'sr': 'sr',
-    'sv-SE': 'sv',
-    'th': 'th',
-    'tr': 'tr',
-    'uk': 'uk',
-    'vi': 'vi',
-    'zh-CN': 'zh_CN',
-    'zh-TW': 'zh_TW',
-  }
+  # We need to map Gecko locales to Chrome locales. Start by mapping Chrome
+  # locales to themselves.
+  locale_mapping = {l: l for l in localeTools.chromeLocales}
 
-  for source, target in localeCodeMapping.iteritems():
+  # Convert values to Crowdin locales first (use Chrome => Crowdin mapping).
+  for chrome_locale, crowdin_locale in localeTools.langMappingChrome.iteritems():
+    locale_mapping[chrome_locale] = crowdin_locale
+
+  # Now convert values to Gecko locales (use Gecko => Crowdin mapping).
+  def get_key(dict, value):

[Sebastian Noack, 2014/12/17 09:04:06]

Maybe I overlook something here? But it seems that this code can be simplified
by generating the mapping the other way around:

 locale_mapping = {l: l for l in localeTools.chromeLocales} 

 for chrome_locale, crowdin_locale in localeTools.langMappingChrome.iteritems():
     locale_mapping[crowdin_locale] = chrome_locale

 for gecko_locale, crowdin_locale in localeTools.langMappingGecko.iteritems():
     chrome_locale = locale_mapping.get(crowdin_locale)
     if chrome_locale is not None:
         locale_mapping[gecko_locale] = chrome_locale

[Wladimir Palant, 2014/12/17 10:09:54]

This will produce lots of duplicates - we need each Chrome locale exactly once
as value. Frankly, reversing the mapping isn't really necessary and we could
keep keys as keys - I only did it to keep the original structure.

[Sebastian Noack, 2014/12/17 10:18:36]

Well, I were hoping to get rid of get_key() by doing proper dict-lookups
instead.

[Wladimir Palant, 2014/12/17 10:25:49]

I think we can only do this by reversing the mapping in langMappingGecko - we
can have a temporary dict instead of get_key.

[Sebastian Noack, 2014/12/17 10:31:49]

Both sounds good to me.

[Wladimir Palant, 2014/12/17 13:21:12]

Ok, the current approach removes unnecessary steps.

+    for k, v in dict.iteritems():
+      if v == value:
+        return k
+    return None
+
+  for gecko_locale, crowdin_locale in localeTools.langMappingGecko.iteritems():
+    chrome_locale = get_key(locale_mapping, crowdin_locale)
+    if chrome_locale is not None:
+      locale_mapping[chrome_locale] = gecko_locale
+
+  # Reverse mapping and make sure keys use underscores as separator
+  locale_mapping = {v: k.replace('-', '_') for k, v in locale_mapping.iteritems()}
+
+  for source, target in locale_mapping.iteritems():
     targetFile = '_locales/%s/messages.json' % target
+    if not targetFile in files:
+      continue
 
     for item in params['metadata'].items('import_locales'):
       fileName, keys = item
       parts = map(lambda n: source if n == '*' else n, fileName.split('/'))
       sourceFile = os.path.join(os.path.dirname(item.source), *parts)
       incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incomplete')
       if not os.path.exists(sourceFile) or os.path.exists(incompleteMarker):
         continue
 
-      data = {}
-      if targetFile in files:
-        data = json.loads(files[targetFile].decode('utf-8'))
+      data = json.loads(files[targetFile].decode('utf-8'))
 
       try:
-        sourceData = localeTools.readFile(sourceFile)
+        if sourceFile.endswith('.json'):
+          with io.open(sourceFile, 'r', encoding='utf-8') as handle:
+            sourceData = {k: v['message'] for k, v in json.load(handle).iteritems()}
+        else:
+          sourceData = localeTools.readFile(sourceFile)
 
         # Resolve wildcard imports
         if keys == '*' or keys == '=*':
           importList = sourceData.keys()
           importList = filter(lambda k: not k.startswith('_'), importList)
           if keys == '=*':
             importList = map(lambda k: '=' + k, importList)
           keys = ' '.join(importList)