More build tools improvements

packagerChrome.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus build tools,
 # Copyright (C) 2006-2012 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
@@ skipping 94 matching lines @@
   manifest = json.dumps(data, sort_keys=True, indent=2)
 
   return manifest.encode('utf-8')
 
 def createPoller(params):
   template = getTemplate('chromeDevenvPoller__.js.tmpl')
   return template.render(params).encode('utf-8');
 
 def convertJS(params, files):
   from jshydra.abp_rewrite import doRewrite
-  baseDir = params['baseDir']
 
-  for file, sources in params['metadata'].items('convert_js'):
+  for item in params['metadata'].items('convert_js'):
+    file, sources = item
+    baseDir = os.path.dirname(item.source)
+
     # Make sure the file is inside an included directory
     if '/' in file and not files.isIncluded(file):
       continue
 
     sourceFiles = re.split(r'\s+', sources)
     args = []
     try:
       argsStart = sourceFiles.index('--arg')
       args = sourceFiles[argsStart + 1:]
       sourceFiles = sourceFiles[0:argsStart]
     except ValueError:
       pass
 
     sourceFiles = map(lambda f: os.path.abspath(os.path.join(baseDir, f)), sourceFiles)
     files[file] = doRewrite(sourceFiles, args)
 
+def importGeckoLocales(params, files):
+  import localeTools
+
+  localeCodeMapping = {
+    'ar': 'ar',
+    'bg': 'bg',
+    'ca': 'ca',
+    'cs': 'cs',
+    'da': 'da',
+    'de': 'de',
+    'el': 'el',
+    'en-US': 'en_US',
+    'en-GB': 'en_GB',
+    'es-ES': 'es',
+    'es-AR': 'es_419',
+    'et': 'et',
+    'fi': 'fi',
+  #   '': 'fil', ???
+    'fr': 'fr',
+    'he': 'he',
+    'hi-IN': 'hi',
+    'hr': 'hr',
+    'hu': 'hu',
+    'id': 'id',
+    'it': 'it',
+    'ja': 'ja',
+    'ko': 'ko',
+    'lt': 'lt',
+    'lv': 'lv',
+    'nl': 'nl',
+  #    'nb-NO': 'no', ???
+    'pl': 'pl',
+    'pt-BR': 'pt_BR',
+    'pt-PT': 'pt_PT',
+    'ro': 'ro',
+    'ru': 'ru',
+    'sk': 'sk',
+    'sl': 'sl',
+    'sr': 'sr',
+    'sv-SE': 'sv',
+    'th': 'th',
+    'tr': 'tr',
+    'uk': 'uk',
+    'vi': 'vi',
+    'zh-CN': 'zh_CN',
+    'zh-TW': 'zh_TW',
+  }
+
+  for source, target in localeCodeMapping.iteritems():
+    targetFile = '_locales/%s/messages.json' % target
+
+    for item in params['metadata'].items('import_locales'):
+      fileName, keys = item
+      parts = map(lambda n: source if n == '*' else n, fileName.split('/'))
+      sourceFile = os.path.join(os.path.dirname(item.source), *parts)
+      incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incomplete')
+      if not os.path.exists(sourceFile) or os.path.exists(incompleteMarker):
+        continue
+
+      data = {}
+      if targetFile in files:
+        data = json.loads(files[targetFile].decode('utf-8'))
+
+      try:
+        sourceData = localeTools.readFile(sourceFile)
+        for stringID in re.split(r'\s+', keys):
+          noMangling = False
+          if stringID.startswith('='):
+            stringID = stringID[1:]
+            noMangling = True
+
+          if stringID in sourceData:
+            if noMangling:
+              key = stringID
+            else:
+              key = re.sub(r'\..*', '', parts[-1]) + '_' + re.sub(r'\W', '_', stringID)
+            if key in data:
+              print 'Warning: locale string %s defined multiple times' % key
+            data[key] = {'message': sourceData[stringID]}
+      except Exception, e:
+        print 'Warning: error importing locale data from %s: %s' % (sourceFile, e)
+
+      files[targetFile] = json.dumps(data, ensure_ascii=False, sort_keys=True,
+                            indent=2, separators=(',', ': ')).encode('utf-8') + '\n'
+
 def signBinary(zipdata, keyFile):
   import M2Crypto
   if not os.path.exists(keyFile):
     M2Crypto.RSA.gen_key(1024, 65537, callback=lambda x: None).save_key(keyFile, cipher=None)
   key = M2Crypto.EVP.load_key(keyFile)
   key.sign_init()
   key.sign_update(zipdata)
   return key.final()
 
 def getPublicKey(keyFile):
@@ skipping 23 matching lines @@
     'releaseBuild': releaseBuild,
     'version': version,
     'experimentalAPI': experimentalAPI,
     'devenv': devenv,
     'metadata': metadata,
   }
 
   files = Files(getPackageFiles(params), getIgnoredFiles(params))
   files['manifest.json'] = createManifest(params)
   if metadata.has_section('mapping'):
-    files.readMappedFiles(baseDir, metadata.items('mapping'))
+    files.readMappedFiles(metadata.items('mapping'))
   files.read(baseDir)
 
   if metadata.has_section('convert_js'):
     convertJS(params, files)
 
+  if metadata.has_section('import_locales'):
+    importGeckoLocales(params, files)
+
   if devenv:
     files['devenvPoller__.js'] = createPoller(params)
 
   zipdata = files.zipToString()
   signature = None
   pubkey = None
   if keyFile != None:
     signature = signBinary(zipdata, keyFile)
     pubkey = getPublicKey(keyFile)
   writePackage(outFile, pubkey, signature, zipdata)
@@ skipping 23 matching lines @@
   def shutdown_server(server):
     time.sleep(10)
     server.shutdown()
   thread.start_new_thread(shutdown_server, (server,))
   server.serve_forever()
 
   if connections[0] == 0:
     print 'Warning: No incoming connections, extension probably not active in the browser yet'
   else:
     print 'Handled %i connection(s)' % connections[0]