Issue 2109 - Allow for translation of app independent repositories

build.py

 # coding: utf-8
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os, sys, re, subprocess, shutil, buildtools
 from getopt import getopt, GetoptError
 from StringIO import StringIO
 from zipfile import ZipFile
 
-knownTypes = ('gecko', 'chrome', 'opera', 'safari')
+knownTypes = ('gecko', 'chrome', 'opera', 'safari', 'generic')
 
 class Command(object):
   name = property(lambda self: self._name)
   shortDescription = property(lambda self: self._shortDescription,
       lambda self, value: self.__dict__.update({'_shortDescription': value}))
   description = property(lambda self: self._description,
       lambda self, value: self.__dict__.update({'_description': value}))
   params = property(lambda self: self._params,
       lambda self, value: self.__dict__.update({'_params': value}))
   supportedTypes = property(lambda self: self._supportedTypes,
@@ skipping 206 matching lines @@
   from buildtools.packager import getDevEnvPath
   devenv_dir = getDevEnvPath(baseDir, type)
 
   shutil.rmtree(devenv_dir, ignore_errors=True)
 
   file.seek(0)
   with ZipFile(file, 'r') as zip_file:
     zip_file.extractall(devenv_dir)
 
 
+def readLocaleConfig(baseDir, type, metadata, includeIncomplete=False):
+  if type == 'gecko':
+    import buildtools.packagerGecko as packager
+    basePath = packager.getLocalesDir(baseDir)
+    return {
+      'base_path': basePath,
+      'name_format': 'BCP-47',
+      'file_format': 'gecko-dtd',
+      'target_platforms': {'gecko'},
+      'default_locale': packager.defaultLocale,
+      'locales': {locale: os.path.join(basePath, locale)
+                  for locale in packager.getLocales(baseDir, includeIncomplete)}
+    }

[Wladimir Palant, 2016/02/12 19:05:43]

Gecko is still being treated as a special case here, for no good reason IMHO. I
think that it should be something like this:

  if type == 'gecko':
    localeDir = packager.getLocalesDir(baseDir)
    ...
  elif type == 'chrome' or type == 'opera':
    localeDir = os.path.join(baseDir, '_locales')
    ...
  else:
    localeDir = os.path.join(baseDir, *metadata.get('locales',
'base_path').split('/'))
    ...

  localeConfig['base_path'] = localeDir
  locales = packager.getLocales(baseDir, includeIncomplete) if type == 'gecko'
else os.listdir(localeDir)
  ...

Note that localeDir is always absolute, not relative. Also, I think that
explicitly splitting base_path setting from metadata by forward slash is a good
idea - the paths specified there aren't supposed to be OS-dependent. I presume
that os.path.join(baseDir, 'foo/bar') will still do the right thing on Windows
but I'd rather not rely on this unnecessarily. Sebastian has the last word here
of course but so far we've been using this approach consistently throughout
buildtools.

Note for future: we should seriously consider implementing the same kind of
locale validation that packager.getLocales() performs for other platforms as
well.

[kzar, 2016/02/12 19:40:26]

Done.

+
+  if type == 'chrome' or type == 'opera':
+    import buildtools.packagerChrome as packager
+    localeDir = '_locales'
+    localeConfig = {
+      'name_format': 'ISO-15897',
+      'file_format': 'chrome-json',
+      'target_platforms': {'chrome'},
+      'default_locale': packager.defaultLocale,
+    }
+  else:
+    localeDir = metadata.get('locales', 'base_path')
+    localeConfig = {
+      'name_format': metadata.get('locales', 'name_format'),
+      'file_format': metadata.get('locales', 'file_format'),
+      'target_platforms': set(metadata.get('locales',
+                                           'target_platforms').split()),
+      'default_locale': metadata.get('locales', 'default_locale')
+    }
+
+  localeConfig['base_path'] = fullBasePath = os.path.join(baseDir, localeDir)
+
+  locales = [(locale, os.path.join(fullBasePath, locale))
+             for locale in os.listdir(fullBasePath)]
+  if localeConfig['name_format'] == 'ISO-15897':
+    locales = [(locale.replace('_', '-'), localePath)
+               for locale, localePath in locales]
+  localeConfig['locales'] = dict(locales)
+
+  return localeConfig
+
 def setupTranslations(baseDir, scriptName, opts, args, type):
   if len(args) < 1:
     print 'Project key is required to update translation master files.'
     usage(scriptName, type, 'setuptrans')
     return
 
   key = args[0]
 
   from buildtools.packager import readMetadata
   metadata = readMetadata(baseDir, type)
+
   basename = metadata.get('general', 'basename')
-
-  if type == 'chrome' or type == 'opera':
-    import buildtools.packagerChrome as packager
-    locales = os.listdir(os.path.join(baseDir, '_locales'))
-    locales = map(lambda locale: locale.replace('_', '-'), locales)
-  else:
-    import buildtools.packagerGecko as packager
-    locales = packager.getLocales(baseDir, True)
+  localeConfig = readLocaleConfig(baseDir, type, metadata, True)
 
   import buildtools.localeTools as localeTools
-  localeTools.setupTranslations(type, locales, basename, key)
+  localeTools.setupTranslations(localeConfig, basename, key)
 
 
 def updateTranslationMaster(baseDir, scriptName, opts, args, type):
   if len(args) < 1:
     print 'Project key is required to update translation master files.'
     usage(scriptName, type, 'translate')
     return
 
   key = args[0]
 
   from buildtools.packager import readMetadata
   metadata = readMetadata(baseDir, type)
+
   basename = metadata.get('general', 'basename')
+  localeConfig = readLocaleConfig(baseDir, type, metadata)
 
-  if type == 'chrome' or type == 'opera':
-    import buildtools.packagerChrome as packager
-    defaultLocaleDir = os.path.join(baseDir, '_locales', packager.defaultLocale)
-  else:
-    import buildtools.packagerGecko as packager
-    defaultLocaleDir = os.path.join(packager.getLocalesDir(baseDir), packager.defaultLocale)
+  defaultLocaleDir = os.path.join(localeConfig['base_path'],
+                                  localeConfig['default_locale'])
 
   import buildtools.localeTools as localeTools
-  localeTools.updateTranslationMaster(type, metadata, defaultLocaleDir, basename, key)
+  localeTools.updateTranslationMaster(localeConfig, metadata, defaultLocaleDir,
+                                      basename, key)
 
 
 def uploadTranslations(baseDir, scriptName, opts, args, type):
   if len(args) < 1:
     print 'Project key is required to upload existing translations.'
     usage(scriptName, type, 'uploadtrans')
     return
 
   key = args[0]
 
   from buildtools.packager import readMetadata
   metadata = readMetadata(baseDir, type)
+
   basename = metadata.get('general', 'basename')
-
-  if type == 'chrome' or type == 'opera':
-    import buildtools.packagerChrome as packager
-    localesDir = os.path.join(baseDir, '_locales')
-    locales = os.listdir(localesDir)
-    locales = map(lambda locale: (locale.replace('_', '-'), os.path.join(localesDir, locale)), locales)
-  else:
-    import buildtools.packagerGecko as packager
-    localesDir = packager.getLocalesDir(baseDir)
-    locales = packager.getLocales(baseDir, True)
-    locales = map(lambda locale: (locale, os.path.join(localesDir, locale)), locales)
+  localeConfig = readLocaleConfig(baseDir, type, metadata, True)
 
   import buildtools.localeTools as localeTools
-  for locale, localeDir in locales:
-    if locale != packager.defaultLocale:
-      localeTools.uploadTranslations(type, metadata, localeDir, locale, basename, key)
+  for locale, localeDir in localeConfig['locales'].iteritems():
+    if locale != localeConfig['default_locale']:
+      localeTools.uploadTranslations(localeConfig, metadata, localeDir, locale,
+                                     basename, key)
 
 
 def getTranslations(baseDir, scriptName, opts, args, type):
   if len(args) < 1:
     print 'Project key is required to update translation master files.'
     usage(scriptName, type, 'translate')
     return
 
+  key = args[0]
+
   from buildtools.packager import readMetadata
   metadata = readMetadata(baseDir, type)
+
   basename = metadata.get('general', 'basename')
-
-  key = args[0]
-  if type == 'chrome' or type == 'opera':
-    import buildtools.packagerChrome as packager
-    localesDir = os.path.join(baseDir, '_locales')
-  else:
-    import buildtools.packagerGecko as packager
-    localesDir = packager.getLocalesDir(baseDir)
+  localeConfig = readLocaleConfig(baseDir, type, metadata)
 
   import buildtools.localeTools as localeTools
-  localeTools.getTranslations(type, localesDir, packager.defaultLocale.replace('_', '-'), basename, key)
+  localeTools.getTranslations(localeConfig, basename, key)
 
 
 def showDescriptions(baseDir, scriptName, opts, args, type):
   locales = None
   for option, value in opts:
     if option in ('-l', '--locales'):
       locales = value.split(',')
 
   import buildtools.packagerGecko as packager
   if locales == None:
@@ skipping 107 matching lines @@
 
 with addCommand(createDevEnv, 'devenv') as command:
   command.shortDescription = 'Set up a development environment'
   command.description = 'Will set up or update the devenv folder as an unpacked extension folder for development.'
   command.supportedTypes = ('chrome', 'opera', 'safari')
 
 with addCommand(setupTranslations, 'setuptrans') as command:
   command.shortDescription = 'Sets up translation languages'
   command.description = 'Sets up translation languages for the project on crowdin.net.'
   command.params = '[options] project-key'
-  command.supportedTypes = ('gecko', 'chrome', 'opera')
+  command.supportedTypes = ('gecko', 'chrome', 'opera', 'generic')
 
 with addCommand(updateTranslationMaster, 'translate') as command:
   command.shortDescription = 'Updates translation master files'
   command.description = 'Updates the translation master files in the project on crowdin.net.'
   command.params = '[options] project-key'
-  command.supportedTypes = ('gecko', 'chrome', 'opera')
+  command.supportedTypes = ('gecko', 'chrome', 'opera', 'generic')
 
 with addCommand(uploadTranslations, 'uploadtrans') as command:
   command.shortDescription = 'Uploads existing translations'
   command.description = 'Uploads already existing translations to the project on crowdin.net.'
   command.params = '[options] project-key'
-  command.supportedTypes = ('gecko', 'chrome', 'opera')
+  command.supportedTypes = ('gecko', 'chrome', 'opera', 'generic')
 
 with addCommand(getTranslations, 'gettranslations') as command:
   command.shortDescription = 'Downloads translation updates'
   command.description = 'Downloads updated translations from crowdin.net.'
   command.params = '[options] project-key'
-  command.supportedTypes = ('gecko', 'chrome', 'opera')
+  command.supportedTypes = ('gecko', 'chrome', 'opera', 'generic')
 
 with addCommand(showDescriptions, 'showdesc') as command:
   command.shortDescription = 'Print description strings for all locales'
   command.description = 'Display description strings for all locales as specified in the corresponding meta.properties files.'
   command.addOption('Only include the given locales', short='l', long='locales', value='l1,l2,l3')
   command.params = '[options]'
   command.supportedTypes = ('gecko')
 
 with addCommand(generateDocs, 'docs') as command:
   command.shortDescription = 'Generate documentation (requires node.js)'
@@ skipping 82 matching lines @@
         if option in ('-h', '--help'):
           usage(scriptName, type, command)
           sys.exit()
       commands[command](baseDir, scriptName, opts, args, type)
     else:
       print 'Command %s is not supported for this application type' % command
       usage(scriptName, type)
   else:
     print 'Command %s is unrecognized' % command
     usage(scriptName, type)