Prepared buildtools for Safari

packagerSafari.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus build tools,
 # Copyright (C) 2006-2013 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import re
 import json
 import ConfigParser
 from urlparse import urlparse
-from collections import OrderedDict
 
 from packager import readMetadata, getDefaultFileName, getBuildVersion, getTemplate, Files
-from buildtools.packagerChrome import convertJS, importGeckoLocales, getIgnoredFiles, getPackageFiles, ImageConverter
-
-def createPlist(params, files):
-  template = getTemplate('Info.plist.tmpl')
+from packagerChrome import convertJS, importGeckoLocales, getIgnoredFiles, getPackageFiles, defaultLocale
+
+def processFile(path, data, params):
+  return data
+
+def createManifest(params, files):
+  template = getTemplate('Info.plist.tmpl', autoEscape=True)
   metadata = params['metadata']
-  catalog = json.loads(files['_locales/en_US/messages.json'])
+  catalog = json.loads(files['_locales/%s/messages.json' % defaultLocale])
 
   def parse_section(section, depth=1):
-    rv = {}
+    result = {}
 
     if not metadata.has_section(section):
-      return rv
+      return result
 
     for opt in metadata.options(section):
       bits = opt.split('_', depth)
-      key = bits.pop(-1).replace('_', ' ').title()
-      d = rv
-
-      for x in bits:
-        d = d.setdefault(x, {})
-
+      key = bits.pop().replace('_', ' ').title()
       val = metadata.get(section, opt)
 
       try:
-        float(val)
+        val = int(val)
       except ValueError:
-        type = 'string'
-      else:
-        type = 'real'
-
-      d[key] = '<%s>%s</%s>' % (type, val, type)
-
-    return rv
+        try:
+          val = float(val)
+        except ValueError:
+          pass
+
+      reduce(lambda d, x: d.setdefault(x, {}), bits, result)[key] = val
+
+    return result
 
   def get_optional(*args):
     try:
       return metadata.get(*args)
     except ConfigParser.Error:
       return None
 
   allowedDomains = set()
   allowAllDomains = False
   allowSecurePages = False
 
-  for perm in re.split(r'\s+', metadata.get('general', 'permissions')):
+  for perm in metadata.get('general', 'permissions').split():
     if perm == '<all_urls>':
       allowAllDomains = True
       allowSecurePages = True
       continue
 
     url = urlparse(perm)
 
     if url.scheme == 'https':
       allowSecurePages = True
     elif url.scheme != 'http':
       continue
 
     if '*' in url.hostname:
       allowAllDomains = True
       continue
 
     allowedDomains.add(url.hostname)
 
   return template.render(
+    basename=metadata.get('general', 'basename'),
     version=params['version'],
+    shortVersion=metadata.get('general', 'version'),
+    releaseBuild=params['releaseBuild'],
     name=catalog['name']['message'],
-    description=catalog['description']['message'],
+    description=catalog['description_safari']['message'],
     author=get_optional('general', 'author'),
-    website=get_optional('general', 'website'),
+    homepage=get_optional('general', 'homepage'),
     updateURL=get_optional('general', 'updateURL'),
-    identifier=metadata.get('general', 'identifier'),
     allowedDomains=allowedDomains,
     allowAllDomains=allowAllDomains,
     allowSecurePages=allowSecurePages,
-    contentScripts={
-      'start': metadata.get('contentScripts', 'document_start').split(),
-      'end':   metadata.get('contentScripts', 'document_end'  ).split(),
-    },
+    startScripts=(get_optional('contentScripts', 'document_start') or '').split(),
+    endScripts=(get_optional('contentScripts', 'document_end') or '').split(),
     menus=parse_section('menus', 2),
     toolbarItems=parse_section('toolbar_items'),
     popovers=parse_section('popovers')
   ).encode('utf-8')
 
 def createBackgroundPage(params):
-  template = getTemplate('background.html.tmpl')
+  template = getTemplate('background.html.tmpl', autoEscape=True)
   return template.render(
-    backgroundScripts=re.split(r'\s+', params['metadata'].get(
+    backgroundScripts=params['metadata'].get(
       'general', 'backgroundScripts'
-    ))
+    ).split()
   ).encode('utf-8')
 
 def createInfoModule(params):
   template = getTemplate('safariInfo.js.tmpl')
-  return template.render(params).encode('utf-8');
-
-def createSignedXarArchive(outFile, files, keyFile, certs):
+  return template.render(params).encode('utf-8')
+
+def fixAbsoluteUrls(files):
+  for filename, content in files.iteritems():
+    if os.path.splitext(filename)[1].lower() == '.html':
+      files[filename] = re.sub(
+        r'(<[^<>]*?\b(?:href|src)\s*=\s*["\']?)\/+',
+        r'\1' + '/'.join(['..'] * filename.count('/') + ['']),
+        content, re.S | re.I
+      )
+
+def createSignedXarArchive(outFile, files, keyFile):
   import subprocess
   import tempfile
   import shutil
-  import errno
   import M2Crypto
 
   # write files to temporary directory and create a xar archive
   dirname = tempfile.mkdtemp()
   try:
     for filename, contents in files.iteritems():
       path = os.path.join(dirname, filename)
 
-      while True:
-        try:
-          file = open(path, 'wb')
-          break
-        except IOError, e:
-          if e.errno != errno.ENOENT:
-            raise
-          os.makedirs(os.path.dirname(path))

[Sebastian Noack, 2013/09/10 12:40:43]

This uses exactly the same amount of lines of code, but it duplicates logic,
which I really want to avoid. Note that in order to avoid an endless loop I
re-raise the IOError in any other case than the directory doesn't exist.

-
-      with file:
+      try:
+        os.makedirs(os.path.dirname(path))
+      except OSError:
+        pass
+
+      with open(path, 'wb') as file:
         file.write(contents)
 
     subprocess.check_output(
       ['xar', '-czf', os.path.abspath(outFile), '--distribution'] + os.listdir(dirname),
       cwd=dirname
     )
   finally:
     shutil.rmtree(dirname)
 
-  # add placeholder signature and certificates to the xar archive, and get data to sign
-  key = M2Crypto.RSA.load_key(keyFile)
-  digestinfo_filename = tempfile.mktemp()
+  certificate_filenames = []
   try:
-    subprocess.check_call(
-      [
-        'xar', '--sign', '-f', outFile,
-        '--digestinfo-to-sign', digestinfo_filename,
-        '--sig-size', str(len(key.sign('')))
-      ] + [
-        arg for cert in certs for arg in ('--cert-loc', cert)
-      ]
-    )
-
-    with open(digestinfo_filename, 'rb') as file:
-      digestinfo = file.read()
+    # load key and certificates from the all-in-one key file
+    # and write each certificate in DER format to a seperate
+    # temporary file, that they can be passed to xar
+    bio = M2Crypto.BIO.openfile(keyFile)
+    try:
+      key = M2Crypto.RSA.load_key_bio(bio)
+
+      bio.reset()
+      while True:
+        try:
+          cert = M2Crypto.X509.load_cert_bio(bio)
+        except M2Crypto.X509.X509Error:
+          break
+
+        fd, filename = tempfile.mkstemp()
+        try:
+          certificate_filenames.append(filename)
+          os.write(fd, cert.as_der())
+        finally:
+          os.close(fd)
+    finally:
+      bio.close()
+
+    # add certificates and placeholder signature
+    # to the xar archive, and get data to sign
+    fd, digestinfo_filename = tempfile.mkstemp()
+    os.close(fd)
+    try:
+      subprocess.check_call(
+        [
+          'xar', '--sign', '-f', outFile,
+          '--digestinfo-to-sign', digestinfo_filename,
+          '--sig-size', str(len(key.private_encrypt('', M2Crypto.RSA.pkcs1_padding)))
+        ] + [
+          arg for cert in certificate_filenames for arg in ('--cert-loc', cert)
+        ]
+      )
+
+      with open(digestinfo_filename, 'rb') as file:
+        digestinfo = file.read()
+    finally:
+      os.unlink(digestinfo_filename)
   finally:
-    try:
-      os.unlink(digestinfo_filename)
-    except OSError, e:
-      if e.errno != errno.ENOENT:
-        raise
+    for filename in certificate_filenames:
+      os.unlink(filename)
 
   # sign data and inject signature into xar archive
   fd, signature_filename = tempfile.mkstemp()

[Wladimir Palant, 2013/09/10 14:02:12]

You are right, my comment was wishful thinking, not how NamedTemporaryFile
actually works. Too bad that deleting was tied to closing here, I don't really
get the logic behind that decision. My assumption was that delete=False would
decouple them again but looking at the source code I see that I was wrong.

   try:
     try:
       os.write(fd, key.private_encrypt(
         digestinfo,
         M2Crypto.RSA.pkcs1_padding
       ))
     finally:
       os.close(fd)
 
     subprocess.check_call(['xar', '--inject-sig', signature_filename, '-f', outFile])
   finally:
     os.unlink(signature_filename)
 
-def createBuild(baseDir, type, outFile=None, buildNum=None, releaseBuild=False, keyFile=None, certs=()):
+def createBuild(baseDir, type, outFile=None, buildNum=None, releaseBuild=False, keyFile=None):
   metadata = readMetadata(baseDir, type)
   version = getBuildVersion(baseDir, metadata, releaseBuild, buildNum)
 
   if not outFile:
     outFile = getDefaultFileName(baseDir, metadata, version, 'safariextz' if keyFile else 'zip')
 
   params = {
     'type': type,
     'baseDir': baseDir,
     'releaseBuild': releaseBuild,
     'version': version,
     'devenv': False,
     'metadata': metadata,
   }
 
   files = Files(getPackageFiles(params), getIgnoredFiles(params),
-                process=lambda path, data: data)
+                process=lambda path, data: processFile(path, data, params))
   if metadata.has_section('mapping'):
     files.readMappedFiles(metadata.items('mapping'))
   files.read(baseDir)
 
   if metadata.has_section('convert_js'):
     convertJS(params, files)
 
   if metadata.has_section('convert_img'):
-    ImageConverter().convert(params, files)
+    from imageConversion import convertImages
+    convertImages(params, files)
+
+  if metadata.has_section('preprocess'):
+    files.preprocess(
+      [f for f, _ in metadata.items('preprocess')],
+      {'needsExt': True}
+    )
 
   if metadata.has_section('import_locales'):
     importGeckoLocales(params, files)
 
   files['lib/info.js'] = createInfoModule(params)
   files['background.html'] = createBackgroundPage(params)
-  files['Info.plist'] = createPlist(params, files)
-
-  dirname = getDefaultFileName('', metadata, version, 'safariextension')
+  files['Info.plist'] = createManifest(params, files)
+
+  fixAbsoluteUrls(files)
+
+  dirname = metadata.get('general', 'basename') + '.safariextension'
   for filename in files.keys():
     files[os.path.join(dirname, filename)] = files.pop(filename)
 
   if keyFile:
-    createSignedXarArchive(outFile, files, keyFile, certs)
+    createSignedXarArchive(outFile, files, keyFile)
   else:
     files.zip(outFile)