More build tools improvements

packagerChrome.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus build tools,
 # Copyright (C) 2006-2012 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
@@ skipping 26 matching lines @@
 
   if params['devenv']:
     result.add('qunit')
 
   baseDir = params['baseDir']
   for file in os.listdir(baseDir):
     if file.endswith('.js') or file.endswith('.html') or file.endswith('.xml'):
       result.add(file)
   return result
 
+def processFile(path, data, params):
+  # We don't change anything yet, this function currently only exists here so
+  # that it can be overridden if necessary.
+  return data
+
 def createManifest(params):
   template = getTemplate('manifest.json.tmpl')
   templateData = dict(params)
 
   baseDir = templateData['baseDir']
   metadata = templateData['metadata']
 
-  if metadata.has_option('general', 'pageAction'):
+  if metadata.has_option('general', 'pageAction') and metadata.get('general', 'pageAction') != '':
     icon, popup = re.split(r'\s+', metadata.get('general', 'pageAction'), 1)
     templateData['pageAction'] = {'icon': icon, 'popup': popup}
 
   if metadata.has_option('general', 'icons'):
     icons = {}
     iconsDir = baseDir
     for dir in metadata.get('general', 'icons').split('/')[0:-1]:
       iconsDir = os.path.join(iconsDir, dir)
 
     prefix, suffix = metadata.get('general', 'icons').split('/')[-1].split('?', 1)
     for file in os.listdir(iconsDir):
       path = os.path.join(iconsDir, file)
       if os.path.isfile(path) and file.startswith(prefix) and file.endswith(suffix):
         size = file[len(prefix):-len(suffix)]
         if not re.search(r'\D', size):
           icons[size] = os.path.relpath(path, baseDir).replace('\\', '/')
 
     templateData['icons'] = icons
 
   if metadata.has_option('general', 'permissions'):
     templateData['permissions'] = re.split(r'\s+', metadata.get('general', 'permissions'))
     if params['experimentalAPI']:
       templateData['permissions'].append('experimental')
 
   if metadata.has_option('general', 'backgroundScripts'):
     templateData['backgroundScripts'] = re.split(r'\s+', metadata.get('general', 'backgroundScripts'))
     if params['devenv']:
       templateData['backgroundScripts'].append('devenvPoller__.js')
 
-  if metadata.has_option('general', 'webAccessible'):
+  if metadata.has_option('general', 'webAccessible') and metadata.get('general', 'webAccessible') != '':
     templateData['webAccessible'] = re.split(r'\s+', metadata.get('general', 'webAccessible'))
 
   if metadata.has_section('contentScripts'):
     contentScripts = []
     for run_at, scripts in metadata.items('contentScripts'):
+      if scripts == '':
+        continue
       contentScripts.append({
         'matches': ['http://*/*', 'https://*/*'],
         'js': re.split(r'\s+', scripts),
         'run_at': run_at,
         'all_frames': True,
       })
     templateData['contentScripts'] = contentScripts
 
   manifest = template.render(templateData)
 
   # Normalize JSON structure
   licenseComment = re.compile(r'/\*.*?\*/', re.S)
   data = json.loads(re.sub(licenseComment, '', manifest, 1))
   if '_dummy' in data:
     del data['_dummy']
   manifest = json.dumps(data, sort_keys=True, indent=2)
 
   return manifest.encode('utf-8')
 
 def createPoller(params):
   template = getTemplate('chromeDevenvPoller__.js.tmpl')
   return template.render(params).encode('utf-8');
 
 def convertJS(params, files):
   from jshydra.abp_rewrite import doRewrite
-  baseDir = params['baseDir']
 
-  for file, sources in params['metadata'].items('convert_js'):
+  for item in params['metadata'].items('convert_js'):
+    file, sources = item
+    baseDir = os.path.dirname(item.source)
+
     # Make sure the file is inside an included directory
     if '/' in file and not files.isIncluded(file):
       continue
 
     sourceFiles = re.split(r'\s+', sources)
     args = []
     try:
       argsStart = sourceFiles.index('--arg')
       args = sourceFiles[argsStart + 1:]
       sourceFiles = sourceFiles[0:argsStart]
     except ValueError:
       pass
 
     sourceFiles = map(lambda f: os.path.abspath(os.path.join(baseDir, f)), sourceFiles)
     files[file] = doRewrite(sourceFiles, args)
 
+def importGeckoLocales(params, files):
+  import localeTools
+
+  localeCodeMapping = {
+    'ar': 'ar',
+    'bg': 'bg',
+    'ca': 'ca',
+    'cs': 'cs',
+    'da': 'da',
+    'de': 'de',
+    'el': 'el',
+    'en-US': 'en_US',
+    'en-GB': 'en_GB',
+    'es-ES': 'es',
+    'es-AR': 'es_419',
+    'et': 'et',
+    'fi': 'fi',
+  #   '': 'fil', ???
+    'fr': 'fr',
+    'he': 'he',
+    'hi-IN': 'hi',
+    'hr': 'hr',
+    'hu': 'hu',
+    'id': 'id',
+    'it': 'it',
+    'ja': 'ja',
+    'ko': 'ko',
+    'lt': 'lt',
+    'lv': 'lv',
+    'nl': 'nl',
+  #    'nb-NO': 'no', ???
+    'pl': 'pl',
+    'pt-BR': 'pt_BR',
+    'pt-PT': 'pt_PT',
+    'ro': 'ro',
+    'ru': 'ru',
+    'sk': 'sk',
+    'sl': 'sl',
+    'sr': 'sr',
+    'sv-SE': 'sv',
+    'th': 'th',
+    'tr': 'tr',
+    'uk': 'uk',
+    'vi': 'vi',
+    'zh-CN': 'zh_CN',
+    'zh-TW': 'zh_TW',
+  }
+
+  for source, target in localeCodeMapping.iteritems():
+    targetFile = '_locales/%s/messages.json' % target
+
+    for item in params['metadata'].items('import_locales'):
+      fileName, keys = item
+      parts = map(lambda n: source if n == '*' else n, fileName.split('/'))
+      sourceFile = os.path.join(os.path.dirname(item.source), *parts)
+      incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incomplete')
+      if not os.path.exists(sourceFile) or os.path.exists(incompleteMarker):
+        continue
+
+      data = {}
+      if targetFile in files:
+        data = json.loads(files[targetFile].decode('utf-8'))
+
+      try:
+        sourceData = localeTools.readFile(sourceFile)
+
+        # Resolve wildcard imports
+        if keys == '*' or keys == '=*':
+          importList = sourceData.keys()
+          importList = filter(lambda k: not k.startswith('_'), importList)
+          if keys == '=*':
+            importList = map(lambda k: '=' + k, importList)
+          keys = ' '.join(importList)
+
+        for stringID in re.split(r'\s+', keys):
+          noMangling = False
+          if stringID.startswith('='):
+            stringID = stringID[1:]
+            noMangling = True
+
+          if stringID in sourceData:
+            if noMangling:
+              key = re.sub(r'\W', '_', stringID)
+            else:
+              key = re.sub(r'\..*', '', parts[-1]) + '_' + re.sub(r'\W', '_', stringID)
+            if key in data:
+              print 'Warning: locale string %s defined multiple times' % key
+            data[key] = {'message': sourceData[stringID]}
+      except Exception, e:
+        print 'Warning: error importing locale data from %s: %s' % (sourceFile, e)
+
+      files[targetFile] = json.dumps(data, ensure_ascii=False, sort_keys=True,
+                            indent=2, separators=(',', ': ')).encode('utf-8') + '\n'
+
 def signBinary(zipdata, keyFile):
   import M2Crypto
   if not os.path.exists(keyFile):
     M2Crypto.RSA.gen_key(1024, 65537, callback=lambda x: None).save_key(keyFile, cipher=None)
   key = M2Crypto.EVP.load_key(keyFile)
   key.sign_init()
   key.sign_update(zipdata)
   return key.final()
 
 def getPublicKey(keyFile):
@@ skipping 20 matching lines @@
 
   params = {
     'baseDir': baseDir,
     'releaseBuild': releaseBuild,
     'version': version,
     'experimentalAPI': experimentalAPI,
     'devenv': devenv,
     'metadata': metadata,
   }
 
-  files = Files(getPackageFiles(params), getIgnoredFiles(params))
+  files = Files(getPackageFiles(params), getIgnoredFiles(params),
+                process=lambda path, data: processFile(path, data, params))
   files['manifest.json'] = createManifest(params)
   if metadata.has_section('mapping'):
-    files.readMappedFiles(baseDir, metadata.items('mapping'))
+    files.readMappedFiles(metadata.items('mapping'))
   files.read(baseDir)
 
   if metadata.has_section('convert_js'):
     convertJS(params, files)
 
+  if metadata.has_section('import_locales'):
+    importGeckoLocales(params, files)
+
   if devenv:
     files['devenvPoller__.js'] = createPoller(params)
 
   zipdata = files.zipToString()
   signature = None
   pubkey = None
   if keyFile != None:
     signature = signBinary(zipdata, keyFile)
     pubkey = getPublicKey(keyFile)
   writePackage(outFile, pubkey, signature, zipdata)
@@ skipping 23 matching lines @@
   def shutdown_server(server):
     time.sleep(10)
     server.shutdown()
   thread.start_new_thread(shutdown_server, (server,))
   server.serve_forever()
 
   if connections[0] == 0:
     print 'Warning: No incoming connections, extension probably not active in the browser yet'
   else:
     print 'Handled %i connection(s)' % connections[0]