Unified in-memory file processing between Gecko and Chrome

packagerGecko.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus build tools,
 # Copyright (C) 2006-2012 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 import os, sys, re, hashlib, base64, urllib, json
 from ConfigParser import SafeConfigParser
 from StringIO import StringIO
-from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
 import xml.dom.minidom as minidom
 import buildtools.localeTools as localeTools
 
-from packager import getDefaultFileName, readMetadata, getBuildVersion, getTemplate
+from packager import getDefaultFileName, readMetadata, getBuildVersion, getTemplate, Files
 
 KNOWN_APPS = {
   'conkeror':   '{a79fe89b-6662-4ff4-8e88-09950ad4dfde}',
   'emusic':     'dlm@emusic.com',
   'fennec':     '{a23983c0-fd0e-11dc-95ff-0800200c9a66}',
   'fennec2':    '{aa3c5121-dab2-40e2-81ca-7ea25febc110}',
   'firefox':    '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}',
   'midbrowser': '{aa5ca914-c309-495d-91cf-3141bbb04115}',
   'prism':      'prism@developer.mozilla.org',
   'seamonkey':  '{92650c4d-4b8e-4d2a-b7eb-24ecf4f6b63a}',
@@ skipping 12 matching lines @@
 
 def getChromeSubdirs(baseDir, locales):
   result = {}
   chromeDir = getChromeDir(baseDir)
   for subdir in ('content', 'skin'):
     result[subdir] = os.path.join(chromeDir, subdir)
   for locale in locales:
     result['locale/%s' % locale] = os.path.join(chromeDir, 'locale', locale)
   return result
 
-def getXPIFiles(baseDir):
-  for file in ('components', 'modules', 'lib', 'resources', 'defaults', 'chrome.manifest', 'icon.png', 'icon64.png'):
-    yield os.path.join(baseDir, file)
+def getPackageFiles(params):
+  result = set(('chrome', 'components', 'modules', 'lib', 'resources', 'defaults', 'chrome.manifest', 'icon.png', 'icon64.png',))
+
+  baseDir = params['baseDir']
   for file in os.listdir(baseDir):
     if file.endswith('.js') or file.endswith('.xml'):
-      yield os.path.join(baseDir, file)
+      result.add(file)
+  return result
 
 def getIgnoredFiles(params):
-  result = ['.incomplete', 'meta.properties']
+  result = set(('.incomplete', 'meta.properties',))
   if params['releaseBuild']:
-    result.append('timeline.js')
+    result.add('timeline.js')
   return result
 
 def isValidLocale(localesDir, dir, includeIncomplete=False):
   if re.search(r'[^\w\-]', dir):
     return False
   if not os.path.isdir(os.path.join(localesDir, dir)):
     return False
   if not includeIncomplete and os.path.exists(os.path.join(localesDir, dir, '.incomplete')):
     return False
   return True
 
 def getLocales(baseDir, includeIncomplete=False):
   global defaultLocale
   localesDir = getLocalesDir(baseDir)
   locales = filter(lambda dir:  isValidLocale(localesDir, dir, includeIncomplete), os.listdir(localesDir))
   locales.sort(key=lambda x: '!' if x == defaultLocale else x)
   return locales
 
 def processFile(path, data, params):
-  if not re.search(r'\.(manifest|xul|jsm?|xml|xhtml|rdf|dtd|properties|css)$', path):
-    return data
-
-  data = re.sub(r'\r', '', data)
-  data = data.replace('{{VERSION}}', params['version'])
-
-  whitespaceRegExp = re.compile(r'^(  )+', re.M)
-  data = re.sub(whitespaceRegExp, lambda match: '\t' * (len(match.group(0)) / 2), data)
-
   if path.endswith('.manifest') and data.find('{{LOCALE}}') >= 0:
     localesRegExp = re.compile(r'^(.*?){{LOCALE}}(.*?){{LOCALE}}(.*)$', re.M)
     replacement = '\n'.join(map(lambda locale: r'\1%s\2%s\3' % (locale, locale), params['locales']))
     data = re.sub(localesRegExp, replacement, data)
 
   if params['releaseBuild'] and path.endswith('.js'):
     # Remove timeline calls from release builds
     timelineRegExp1 = re.compile(r'^.*\b[tT]imeLine\.(\w+)\(.*', re.M)
     timelineRegExp2 = re.compile(r'^.*\brequire\(\"timeline\"\).*', re.M)
     data = re.sub(timelineRegExp1, '', data)
@@ skipping 42 matching lines @@
         main.append(value)
   return main + sorted(additional, key=unicode.lower)
 
 def initTranslators(localeMetadata):
   for locale in localeMetadata.itervalues():
     if 'translator' in locale:
       locale['translators'] = sorted(map(lambda t: t.strip(), locale['translator'].split(',')), key=unicode.lower)
     else:
       locale['translators'] = []
 
-def createManifest(baseDir, params):
+def createManifest(params):
   global KNOWN_APPS, defaultLocale
   template = getTemplate('install.rdf.tmpl', autoEscape=True)
   templateData = dict(params)
-  templateData['localeMetadata'] = readLocaleMetadata(baseDir, params['locales'])
+  templateData['localeMetadata'] = readLocaleMetadata(params['baseDir'], params['locales'])
   initTranslators(templateData['localeMetadata'])
   templateData['KNOWN_APPS'] = KNOWN_APPS
   templateData['defaultLocale'] = defaultLocale
   return template.render(templateData).encode('utf-8')
 
-def readFile(files, params, path, name):
-  ignoredFiles = getIgnoredFiles(params)
-  if os.path.isdir(path):
-    for file in os.listdir(path):
-      if file in ignoredFiles:
-        continue
-      readFile(files, params, os.path.join(path, file), '%s/%s' % (name, file))
-  else:
-    file = open(path, 'rb')
-    data = processFile(path, file.read(), params)
-    file.close()
-    files[name] = data
-
-def fixupLocales(baseDir, files, params):
+def fixupLocales(params, files):
   global defaultLocale
 
   # Read in default locale data, it might not be included in files
-  defaultLocaleDir = os.path.join(getLocalesDir(baseDir), defaultLocale)
+  defaultLocaleDir = os.path.join(getLocalesDir(params['baseDir']), defaultLocale)
   reference = {}
   ignoredFiles = getIgnoredFiles(params)
   for file in os.listdir(defaultLocaleDir):
     path = os.path.join(defaultLocaleDir, file)
     if file in ignoredFiles or not os.path.isfile(path):
       continue
     data = localeTools.readFile(path)
     if data:
       reference[file] = data
 
   for locale in params['locales']:
     for file in reference.iterkeys():
       path = 'chrome/locale/%s/%s' % (locale, file)
       if path in files:
         data = localeTools.parseString(files[path].decode('utf-8'), path)
         for key, value in reference[file].iteritems():
           if not key in data:
             files[path] += localeTools.generateStringEntry(key, value, path).encode('utf-8')
       else:
         files[path] = reference[file]['_origData'].encode('utf-8')
 
-def readXPIFiles(baseDir, params, files):
-  for path in getXPIFiles(baseDir):
-    if os.path.exists(path):
-      readFile(files, params, path, os.path.basename(path))
-
-def addMissingFiles(baseDir, params, files):
+def addMissingFiles(params, files):
   templateData = {
     'hasChrome': False,
     'hasChromeRequires': False,
     'hasShutdownHandlers': False,
     'hasVersionPref': False,
     'chromeWindows': [],
     'requires': {},
     'metadata': params['metadata'],
     'multicompartment': params['multicompartment'],
     'applications': dict((v, k) for k, v in KNOWN_APPS.iteritems()),
@@ skipping 27 matching lines @@
     for module in templateData['requires']:
       moduleFile = 'lib/' + module + '.js'
       if not moduleFile in files:
         import buildtools
         path = os.path.join(buildtools.__path__[0], moduleFile)
         if os.path.exists(path):
           missing.append((path, moduleFile))
     if not len(missing):
       break
     for path, moduleFile in missing:
-      readFile(files, params, path, moduleFile)
+      files.read(path, moduleFile)
       checkScript(moduleFile)
 
   template = getTemplate('bootstrap.js.tmpl')
-  files['bootstrap.js'] = processFile('bootstrap.js', template.render(templateData).encode('utf-8'), params)
+  files['bootstrap.js'] = template.render(templateData).encode('utf-8')
 
 def signFiles(files, keyFile):
   import M2Crypto
   manifest = []
   signature = []
 
   def getDigest(data):
     md5 = hashlib.md5()
     md5.update(data)
     sha1 = hashlib.sha1()
@@ skipping 30 matching lines @@
 
   mime = M2Crypto.SMIME.SMIME()
   mime.load_key(keyFile)
   mime.set_x509_stack(stack)
   signature = mime.sign(M2Crypto.BIO.MemoryBuffer(files['META-INF/zigbert.sf'].encode('utf-8')), M2Crypto.SMIME.PKCS7_DETACHED | M2Crypto.SMIME.PKCS7_BINARY)
 
   buffer = M2Crypto.BIO.MemoryBuffer()
   signature.write_der(buffer)
   files['META-INF/zigbert.rsa'] = buffer.read()
 
-def writeXPI(files, outFile):
-  zip = ZipFile(outFile, 'w', ZIP_DEFLATED)
-  names = files.keys()
-  names.sort(key=lambda x: '!' if x == 'META-INF/zigbert.rsa' else x)
-  for name in names:
-    zip.writestr(name, files[name])
-  zip.close()
-
 def createBuild(baseDir, outFile=None, locales=None, buildNum=None, releaseBuild=False, keyFile=None, multicompartment=False):
   if locales == None:
     locales = getLocales(baseDir)
   elif locales == 'all':
     locales = getLocales(baseDir, True)
 
   metadata = readMetadata(baseDir)
   version = getBuildVersion(baseDir, metadata, releaseBuild, buildNum)
 
   if outFile == None:
     outFile = getDefaultFileName(baseDir, metadata, version, 'xpi')
 
   contributors = getContributors(baseDir, metadata)
 
   params = {
+    'baseDir': baseDir,
     'locales': locales,
     'releaseBuild': releaseBuild,
     'version': version.encode('utf-8'),
     'metadata': metadata,
     'contributors': contributors,
     'multicompartment': multicompartment,
   }
-  files = {}
-  files['install.rdf'] = createManifest(baseDir, params)
+
+  files = Files(getPackageFiles(params), getIgnoredFiles(params),
+                process=lambda path, data: processFile(path, data, params))
+  files['install.rdf'] = createManifest(params)
+  files.read(baseDir, skip=('chrome'))
   for name, path in getChromeSubdirs(baseDir, params['locales']).iteritems():
     if os.path.isdir(path):
-      readFile(files, params, path, 'chrome/%s' % name)
-  fixupLocales(baseDir, files, params)
-  readXPIFiles(baseDir, params, files)
+      files.read(path, 'chrome/%s' % name)
+  fixupLocales(params, files)
   if not 'bootstrap.js' in files:
-    addMissingFiles(baseDir, params, files)
+    addMissingFiles(params, files)
   if keyFile:
     signFiles(files, keyFile)
-  writeXPI(files, outFile)
+  files.zip(outFile, sortKey=lambda x: '!' if x == 'META-INF/zigbert.rsa' else x)
 
 def autoInstall(baseDir, host, port, multicompartment=False):
   fileBuffer = StringIO()
   createBuild(baseDir, outFile=fileBuffer, multicompartment=multicompartment)
   urllib.urlopen('http://%s:%s/' % (host, port), data=fileBuffer.getvalue())