Changes to Chrome build process

packagerChrome.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus build tools,
 # Copyright (C) 2006-2012 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
-import sys, os, subprocess, re, json, codecs, struct
+import sys, os, subprocess, re, json, codecs, struct, jinja2, buildtools
 from ConfigParser import SafeConfigParser
 from StringIO import StringIO
 from zipfile import ZipFile, ZIP_DEFLATED
 
 defaultLocale = 'en_US'
 
-def getDefaultFileName(baseDir, metadata, version, ext):
-  return os.path.join(baseDir, '%s-%s.%s' % (metadata.get('general', 'basename'), version, ext))
+def getDefaultFileName(baseDir, metadata, ext):
+  return os.path.join(baseDir, '%s-%s.%s' % (metadata.get('general', 'basename'), metadata.get('general', 'version'), ext))
 
 def getMetadataPath(baseDir):
   return os.path.join(baseDir, 'metadata')
 
 def getBuildNum(baseDir):
   try:
     (result, dummy) = subprocess.Popen(['hg', 'id', '-n'], stdout=subprocess.PIPE).communicate()
     return re.sub(r'\D', '', result)
   except Exception:
     return '0'
 
+def getIgnoredFiles(params):
+  return ['store.description']
+
 def readMetadata(baseDir):
   metadata = SafeConfigParser()
   metadata.optionxform = str
   file = codecs.open(getMetadataPath(baseDir), 'rb', encoding='utf-8')
   metadata.readfp(file)
   file.close()
   return metadata
 
-def readVersion(baseDir):
-  file = open(os.path.join(baseDir, 'manifest.json'))
-  data = json.load(file)
-  file.close()
-  return data['version']
+def getPackageFiles(params):
+  baseDir = params['baseDir']
+  for file in ('_locales', 'icons', 'jquery-ui', 'lib', 'skin', 'ui'):
+    yield os.path.join(baseDir, file)
+  if params['devenv']:
+    yield os.path.join(baseDir, 'qunit')
+  for file in os.listdir(baseDir):
+    if file.endswith('.js') or file.endswith('.html') or file.endswith('.xml'):
+      yield os.path.join(baseDir, file)
 
-def setUpdateURL(updateName, zip, dir, fileName, fileData):
-  if fileName == 'manifest.json':
-    data = json.loads(fileData)
-    data['update_url'] = 'https://adblockplus.org/devbuilds/%s/updates.xml' % updateName
-    return json.dumps(data, sort_keys=True, indent=2)
-  return fileData
+def createManifest(params):
+  env = jinja2.Environment(loader=jinja2.FileSystemLoader(buildtools.__path__[0]))
+  env.filters.update({'json': json.dumps})
+  template = env.get_template('manifest.json.tmpl')
+  templateData = dict(params)
 
-def setExperimentalSettings(zip, dir, fileName, fileData):
-  if fileName == 'manifest.json':
-    data = json.loads(fileData)
-    data['permissions'] += ['experimental']
-    data['name'] += ' experimental build'
-    return json.dumps(data, sort_keys=True, indent=2)
-  return fileData
+  baseDir = templateData['baseDir']
+  metadata = templateData['metadata']
 
-def addBuildNumber(revision, zip, dir, fileName, fileData):
-  if fileName == 'manifest.json':
-    if len(revision) > 0:
-      data = json.loads(fileData)
-      while data['version'].count('.') < 2:
-        data['version'] += '.0'
-      data['version'] += '.' + revision
-      return json.dumps(data, sort_keys=True, indent=2)
-  return fileData
+  if metadata.has_option('general', 'pageAction'):
+    icon, popup = re.split(r'\s+', metadata.get('general', 'pageAction'), 1)
+    templateData['pageAction'] = {'icon': icon, 'popup': popup}
 
-def mergeContentScripts(zip, dir, fileName, fileData):
-  if fileName == 'manifest.json':
-    data = json.loads(fileData)
-    if 'content_scripts' in data:
-      scriptIndex = 1
-      for contentScript in data['content_scripts']:
-        if 'js' in contentScript:
-          scriptData = ''
-          for scriptFile in contentScript['js']:
-            parts = [dir] + scriptFile.split('/')
-            scriptPath = os.path.join(*parts)
-            handle = open(scriptPath, 'rb')
-            scriptData += handle.read()
-            handle.close()
-          contentScript['js'] = ['contentScript' + str(scriptIndex) + '.js']
-          zip.writestr('contentScript' + str(scriptIndex) + '.js', scriptData)
-          scriptIndex += 1
-    return json.dumps(data, sort_keys=True, indent=2)
-  return fileData
+  if metadata.has_option('general', 'icons'):
+    icons = {}
+    iconsDir = baseDir
+    for dir in metadata.get('general', 'icons').split('/')[0:-1]:
+      iconsDir = os.path.join(iconsDir, dir)
 
-def addToZip(zip, filters, dir, baseName):
-  for file in os.listdir(dir):
-    filelc = file.lower()
-    if (file.startswith('.') or
-        file == 'buildtools' or file == 'qunit' or file == 'metadata' or
-        file == 'store.description' or
-        filelc.endswith('.py') or filelc.endswith('.pyc') or
-        filelc.endswith('.crx') or filelc.endswith('.zip') or
-        filelc.endswith('.sh') or filelc.endswith('.bat') or
-        filelc.endswith('.txt')):
-      # skip special files, scripts, existing archives
-      continue
-    if file.startswith('include.'):
-      # skip includes, they will be added by other means
-      continue
+    prefix, suffix = metadata.get('general', 'icons').split('/')[-1].split('?', 1)
+    for file in os.listdir(iconsDir):
+      path = os.path.join(iconsDir, file)
+      if os.path.isfile(path) and file.startswith(prefix) and file.endswith(suffix):
+        size = file[len(prefix):-len(suffix)]

[Felix Dahlke, 2013/01/11 12:24:16]

How about using a regex here?

if os.path.isfile(path):
  match = re.match(prefix + r'(.*)' + suffix, file)
  if match:
    size = match.group(0)

It's a bit longer vertically, but I bet easier to maintain. Your call.

[Wladimir Palant, 2013/01/11 14:45:39]

As soon as we add the required escaping (prefix and suffix can be anything,
could include special characters like e.g. ".") and the missing ^ and $ things
won't be simple at all any more. No, regexps are a pretty bad choice if you need
to do string comparison.

[Felix Dahlke, 2013/01/11 14:47:55]

Right, forgot about the escaping. I suppose it's better this way.

+        if not re.search(r'\D', size):
+          icons[size] = os.path.relpath(path, baseDir).replace('\\', '/')
 
-    filePath = os.path.join(dir, file)
-    if os.path.isdir(filePath):
-      addToZip(zip, filters, filePath, baseName + file + '/')
-    else:
-      handle = open(filePath, 'rb')
-      fileData = handle.read()
-      handle.close()
+    templateData['icons'] = icons
 
-      for filter in filters:
-        fileData = filter(zip, dir, baseName + file, fileData)
-      zip.writestr(baseName + file, fileData)
+  if metadata.has_option('general', 'permissions'):
+    templateData['permissions'] = re.split(r'\s+', metadata.get('general', 'permissions'))
+    if params['experimentalAPI']:
+      templateData['permissions'].append('experimental')
 
-def packDirectory(dir, filters):
+  if metadata.has_option('general', 'backgroundScripts'):
+    templateData['backgroundScripts'] = re.split(r'\s+', metadata.get('general', 'backgroundScripts'))
+    if params['devenv']:
+      templateData['backgroundScripts'].append('devenvPoller__.js')
+
+  if metadata.has_option('general', 'webAccessible'):
+    templateData['webAccessible'] = re.split(r'\s+', metadata.get('general', 'webAccessible'))
+
+  if metadata.has_section('contentScripts'):
+    contentScripts = []
+    for run_at, scripts in metadata.items('contentScripts'):
+      contentScripts.append({
+        'matches': ['http://*/*', 'https://*/*'],
+        'js': re.split(r'\s+', scripts),
+        'run_at': run_at,
+        'all_frames': True,
+      })
+    templateData['contentScripts'] = contentScripts
+
+  manifest = template.render(templateData)
+
+  # Normalize JSON structure
+  licenseComment = re.compile(r'/\*.*?\*/', re.S)
+  data = json.loads(re.sub(licenseComment, '', manifest, 1))
+  if '_dummy' in data:
+    del data['_dummy']
+  manifest = json.dumps(data, sort_keys=True, indent=2)
+
+  return manifest.encode('utf-8')
+
+def createPoller(params):
+  env = jinja2.Environment(loader=jinja2.FileSystemLoader(buildtools.__path__[0]))
+  env.filters.update({'json': json.dumps})
+  template = env.get_template('chromeDevenvPoller__.js.tmpl')
+  return template.render(params).encode('utf-8');
+
+def readFile(params, files, path):
+  ignoredFiles = getIgnoredFiles(params)
+  if os.path.isdir(path):
+    for file in os.listdir(path):
+      if file in ignoredFiles:
+        continue
+      readFile(params, files, os.path.join(path, file))
+  else:
+    file = open(path, 'rb')
+    data = file.read()
+    file.close()
+
+    name = os.path.relpath(path, params['baseDir']).replace('\\', '/')
+    files[name] = data
+
+def convertJS(params, files):
+  from jshydra.abp_rewrite import doRewrite
+  baseDir = params['baseDir']
+
+  for file, sources in params['metadata'].items('convert_js'):
+    dirsep = file.find('/')
+    if dirsep >= 0:
+      # Not a top-level file, make sure it is inside an included directory
+      dirname = file[0:dirsep]
+      if os.path.join(baseDir, dirname) not in getPackageFiles(params):
+        continue
+
+    sourceFiles = re.split(r'\s+', sources)
+    args = []
+    try:
+      argsStart = sourceFiles.index('--arg')
+      args = sourceFiles[argsStart + 1:]
+      sourceFiles = sourceFiles[0:argsStart]
+    except ValueError:
+      pass
+
+    sourceFiles = map(lambda f: os.path.abspath(os.path.join(baseDir, f)), sourceFiles)
+    files[file] = doRewrite(sourceFiles, args)
+
+def packFiles(files):
   buffer = StringIO()
   zip = ZipFile(buffer, 'w', ZIP_DEFLATED)
-  addToZip(zip, filters, dir, '')
+  for file, data in files.iteritems():
+    zip.writestr(file, data)
   zip.close()
   return buffer.getvalue()
 
 def signBinary(zipdata, keyFile):
   import M2Crypto
   if not os.path.exists(keyFile):
     M2Crypto.RSA.gen_key(1024, 65537, callback=lambda x: None).save_key(keyFile, cipher=None)
   key = M2Crypto.EVP.load_key(keyFile)
   key.sign_init()
   key.sign_update(zipdata)
   return key.final()
 
 def getPublicKey(keyFile):
   import M2Crypto
   return M2Crypto.EVP.load_key(keyFile).as_der()
 
 def writePackage(outputFile, pubkey, signature, zipdata):
-  file = open(outputFile, 'wb')
+  if isinstance(outputFile, basestring):
+    file = open(outputFile, 'wb')
+  else:
+    file = outputFile
   if pubkey != None and signature != None:
     file.write(struct.pack('<4sIII', 'Cr24', 2, len(pubkey), len(signature)))
     file.write(pubkey)
     file.write(signature)
   file.write(zipdata)
-  file.close()
 
-def createBuild(baseDir, outFile=None, buildNum=None, releaseBuild=False, keyFile=None, experimentalAPI=False):
+def createBuild(baseDir, outFile=None, buildNum=None, releaseBuild=False, keyFile=None, experimentalAPI=False, devenv=False):
   metadata = readMetadata(baseDir)
-  version = readVersion(baseDir)
   if outFile == None:
-    outFile = getDefaultFileName(baseDir, metadata, version, 'crx' if keyFile else 'zip')
+    outFile = getDefaultFileName(baseDir, metadata, 'crx' if keyFile else 'zip')
 
-  filters = []
+  version = metadata.get('general', 'version')
   if not releaseBuild:
     if buildNum == None:
       buildNum = getBuildNum(baseDir)
-    filters.append(lambda zip, dir, fileName, fileData: addBuildNumber(buildNum, zip, dir, fileName, fileData))
+    if len(buildNum) > 0:
+      while version.count('.') < 2:
+        version += '.0'
+      version += '.' + buildNum
 
-    baseName = metadata.get('general', 'basename')
-    updateName = baseName + '-experimental' if experimentalAPI else baseName
-    filters.append(lambda zip, dir, fileName, fileData: setUpdateURL(updateName, zip, dir, fileName, fileData))
-    if experimentalAPI:
-      filters.append(setExperimentalSettings)
-  filters.append(mergeContentScripts)
+  params = {
+    'baseDir': baseDir,
+    'releaseBuild': releaseBuild,
+    'version': version,
+    'experimentalAPI': experimentalAPI,
+    'devenv': devenv,
+    'metadata': metadata,
+  }
 
-  zipdata = packDirectory(baseDir, filters)
+  files = {}
+  files['manifest.json'] = createManifest(params)
+  for path in getPackageFiles(params):
+    if os.path.exists(path):
+      readFile(params, files, path)
+
+  if metadata.has_section('convert_js'):
+    convertJS(params, files)
+
+  if devenv:
+    files['devenvPoller__.js'] = createPoller(params)
+
+  zipdata = packFiles(files)
   signature = None
   pubkey = None
   if keyFile != None:
     signature = signBinary(zipdata, keyFile)
     pubkey = getPublicKey(keyFile)
   writePackage(outFile, pubkey, signature, zipdata)
+
+def createDevEnv(baseDir):
+  fileBuffer = StringIO()
+  createBuild(baseDir, outFile=fileBuffer, devenv=True, releaseBuild=True)
+  zip = ZipFile(StringIO(fileBuffer.getvalue()), 'r')
+  zip.extractall(os.path.join(baseDir, 'devenv'))
+  zip.close()
+
+  print 'Development environment created, waiting for connections from active extensions...'
+  metadata = readMetadata(baseDir)
+  connections = [0]
+
+  import SocketServer, time, thread
+
+  class ConnectionHandler(SocketServer.BaseRequestHandler):
+    def handle(self):
+      connections[0] += 1
+      self.request.sendall('HTTP/1.0 OK\nConnection: close\n\n%s' % metadata.get('general', 'basename'))
+
+  server = SocketServer.TCPServer(('localhost', 43816), ConnectionHandler)
+
+  def shutdown_server(server):
+    time.sleep(10)
+    server.shutdown()
+  thread.start_new_thread(shutdown_server, (server,))
+  server.serve_forever()
+
+  if connections[0] == 0:
+    print 'Warning: No incoming connections, extension probably not active in the browser yet'
+  else:
+    print 'Handled %i connection(s)' % connections[0]