Issue 1707 - Allow importing Chrome-style locales in Firefox extensions

packagerGecko.py

 # coding: utf-8
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import os, sys, re, hashlib, base64, urllib, json
+import os
+import sys
+import re
+import hashlib
+import base64
+import urllib
+import json
+import io
 from ConfigParser import SafeConfigParser
 from StringIO import StringIO
 import xml.dom.minidom as minidom
 import buildtools.localeTools as localeTools
 
 import packager
 from packager import readMetadata, getMetadataPath, getDefaultFileName, getBuildVersion, getTemplate, Files
 
 KNOWN_APPS = {
   'conkeror':   '{a79fe89b-6662-4ff4-8e88-09950ad4dfde}',
@@ skipping 31 matching lines @@
 
   baseDir = params['baseDir']
   for file in os.listdir(baseDir):
     if file.endswith('.js') or file.endswith('.xml'):
       result.add(file)
   return result
 
 def getIgnoredFiles(params):
   return {'.incomplete', 'meta.properties'}
 
+def archive_path(path, baseDir):
+  return '/'.join(os.path.relpath(path, baseDir).split(os.sep))

[Sebastian Noack, 2014/12/17 08:18:12]

Nit: x.split(os.sep) -> os.path.split(x)

[Wladimir Palant, 2014/12/17 13:26:04]

Done.

+
 def isValidLocale(localesDir, dir, includeIncomplete=False):
   if re.search(r'[^\w\-]', dir):
     return False
   curLocaleDir = os.path.join(localesDir, dir)
   if not os.path.isdir(curLocaleDir):
     return False
   if len(os.listdir(curLocaleDir)) == 0:
     return False
   if not includeIncomplete and os.path.exists(os.path.join(localesDir, dir, '.incomplete')):
     return False
@@ skipping 67 matching lines @@
 def createManifest(params):
   global KNOWN_APPS, defaultLocale
   template = getTemplate('install.rdf.tmpl', autoEscape=True)
   templateData = dict(params)
   templateData['localeMetadata'] = readLocaleMetadata(params['baseDir'], params['locales'])
   initTranslators(templateData['localeMetadata'])
   templateData['KNOWN_APPS'] = KNOWN_APPS
   templateData['defaultLocale'] = defaultLocale
   return template.render(templateData).encode('utf-8')
 
+def importLocales(params, files):
+  SECTION = 'import_locales'
+  if not params['metadata'].has_section(SECTION):
+    return
+
+  import localeTools
+
+  for locale in params['locales']:
+    for item in params['metadata'].items(SECTION):
+      path, keys = item
+      parts = [locale if p == '*' else p for p in path.split('/')]
+      source = os.path.join(os.path.dirname(item.source), *parts)
+      if not os.path.exists(source):

[Sebastian Noack, 2014/12/17 08:18:12]

Nit: I'd prefer to catch the IOError when openening the file, instead checking
whether the file exists before, to avoid race conditions.

[Wladimir Palant, 2014/12/17 13:26:04]

Sure, but that's not a scenario where race conditions can happen - and a
try/catch block is pretty verbose. So I'd rather not change this.

[Sebastian Noack, 2014/12/17 13:59:07]

Theoretically, the contents of the file system can still change between the
os.path.exists() and io.open() call. Sure, this is a rather theoretical case,
and I'm not going to mention that FS operations are kinda expensive, and
therefore performance critical code should do as few round-trips as possible. ;)

But still, catching the IOError would be the correct thing to do here IMO.
However, I guess it's not important. So LGTM if you prefer to don't do so.

+        continue
+
+      with io.open(source, 'r', encoding='utf-8') as handle:
+        data = json.load(handle)
+
+      target_name = os.path.splitext(os.path.basename(source))[0] + '.properties'
+      target = archive_path(os.path.join(getLocalesDir(params['baseDir']), locale, target_name), params['baseDir'])
+
+      files[target] = ''
+      for key, value in sorted(data.items()):
+        message = value['message']
+        files[target] += localeTools.generateStringEntry(key, message, target).encode('utf-8')
+
 def fixupLocales(params, files):
   global defaultLocale
 
-  # Read in default locale data, it might not be included in files
+  # Read in default locale data, it might not be included in package files
   defaultLocaleDir = os.path.join(getLocalesDir(params['baseDir']), defaultLocale)
+  reference_files = Files(getPackageFiles(params), getIgnoredFiles(params))
+  reference_files.read(defaultLocaleDir, archive_path(defaultLocaleDir, params['baseDir']))
+  reference_params = dict(params)
+  reference_params['locales'] = [defaultLocale]
+  importLocales(reference_params, reference_files)
+
   reference = {}
-  ignoredFiles = getIgnoredFiles(params)
-  for file in os.listdir(defaultLocaleDir):
-    path = os.path.join(defaultLocaleDir, file)
-    if file in ignoredFiles or not os.path.isfile(path):
-      continue
-    data = localeTools.readFile(path)
+  for path, data in reference_files.iteritems():
+    filename = path.split('/')[-1]
+    data = localeTools.parseString(data.decode('utf-8'), filename)
     if data:
-      reference[file] = data
+      reference[filename] = data
 
   for locale in params['locales']:
     for file in reference.iterkeys():
       path = 'chrome/locale/%s/%s' % (locale, file)
       if path in files:
         data = localeTools.parseString(files[path].decode('utf-8'), path)
         for key, value in reference[file].iteritems():
           if not key in data:
             files[path] += localeTools.generateStringEntry(key, value, path).encode('utf-8')
       else:
@@ skipping 131 matching lines @@
 
   files = Files(getPackageFiles(params), getIgnoredFiles(params),
                 process=lambda path, data: processFile(path, data, params))
   files['install.rdf'] = createManifest(params)
   if metadata.has_section('mapping'):
     files.readMappedFiles(metadata.items('mapping'))
   files.read(baseDir, skip=('chrome'))
   for name, path in getChromeSubdirs(baseDir, params['locales']).iteritems():
     if os.path.isdir(path):
       files.read(path, 'chrome/%s' % name)
+  importLocales(params, files)
   fixupLocales(params, files)
   if not 'bootstrap.js' in files:
     addMissingFiles(params, files)
   if metadata.has_section('preprocess'):
     files.preprocess([f for f, _ in metadata.items('preprocess')])
   if keyFile:
     signFiles(files, keyFile)
   files.zip(outFile, sortKey=lambda x: '!' if x == 'META-INF/zigbert.rsa' else x)
 
 def autoInstall(baseDir, type, host, port, multicompartment=False):
   fileBuffer = StringIO()
   createBuild(baseDir, type=type, outFile=fileBuffer, multicompartment=multicompartment)
   urllib.urlopen('http://%s:%s/' % (host, port), data=fileBuffer.getvalue())