Packager refactoring, moved common packager code into separate module

packagerGecko.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus build tools,
 # Copyright (C) 2006-2012 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
-import os, sys, re, subprocess, jinja2, buildtools, codecs, hashlib, base64, shutil, urllib, json
+import os, sys, re, hashlib, base64, urllib, json
 from ConfigParser import SafeConfigParser
 from StringIO import StringIO
 from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
 import xml.dom.minidom as minidom
 import buildtools.localeTools as localeTools
 
+from packager import getDefaultFileName, readMetadata, getBuildVersion, getTemplate
+
 KNOWN_APPS = {
   'conkeror':   '{a79fe89b-6662-4ff4-8e88-09950ad4dfde}',
   'emusic':     'dlm@emusic.com',
   'fennec':     '{a23983c0-fd0e-11dc-95ff-0800200c9a66}',
   'fennec2':    '{aa3c5121-dab2-40e2-81ca-7ea25febc110}',
   'firefox':    '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}',
   'midbrowser': '{aa5ca914-c309-495d-91cf-3141bbb04115}',
   'prism':      'prism@developer.mozilla.org',
   'seamonkey':  '{92650c4d-4b8e-4d2a-b7eb-24ecf4f6b63a}',
   'songbird':   'songbird@songbirdnest.com',
   'thunderbird':  '{3550f703-e582-4d05-9a08-453d09bdfdc6}',
   'toolkit':    'toolkit@mozilla.org',
 }
 
 defaultLocale = 'en-US'
 
-def getDefaultFileName(baseDir, metadata, version, ext='xpi'):
-  return os.path.join(baseDir, '%s-%s.%s' % (metadata.get('general', 'basename'), version, ext))
-
-def getMetadataPath(baseDir):
-  return os.path.join(baseDir, 'metadata')
-
 def getChromeDir(baseDir):
   return os.path.join(baseDir, 'chrome')
 
 def getLocalesDir(baseDir):
   return os.path.join(getChromeDir(baseDir), 'locale')
 
 def getChromeSubdirs(baseDir, locales):
   result = {}
   chromeDir = getChromeDir(baseDir)
   for subdir in ('content', 'skin'):
@@ skipping 26 matching lines @@
     return False
   return True
 
 def getLocales(baseDir, includeIncomplete=False):
   global defaultLocale
   localesDir = getLocalesDir(baseDir)
   locales = filter(lambda dir:  isValidLocale(localesDir, dir, includeIncomplete), os.listdir(localesDir))
   locales.sort(key=lambda x: '!' if x == defaultLocale else x)
   return locales
 
-def getBuildNum(baseDir):
-  try:
-    (result, dummy) = subprocess.Popen(['hg', 'id', '-n'], stdout=subprocess.PIPE).communicate()
-    return re.sub(r'\W', '', result)
-  except Exception:
-    return '0'
-
-def readMetadata(baseDir):
-  metadata = SafeConfigParser()
-  metadata.optionxform = str
-  file = codecs.open(getMetadataPath(baseDir), 'rb', encoding='utf-8')
-  metadata.readfp(file)
-  file.close()
-  return metadata
-
 def processFile(path, data, params):
   if not re.search(r'\.(manifest|xul|jsm?|xml|xhtml|rdf|dtd|properties|css)$', path):
     return data
 
   data = re.sub(r'\r', '', data)
-  data = data.replace('{{BUILD}}', params['buildNum'])
   data = data.replace('{{VERSION}}', params['version'])
 
   whitespaceRegExp = re.compile(r'^(  )+', re.M)
   data = re.sub(whitespaceRegExp, lambda match: '\t' * (len(match.group(0)) / 2), data)
 
   if path.endswith('.manifest') and data.find('{{LOCALE}}') >= 0:
     localesRegExp = re.compile(r'^(.*?){{LOCALE}}(.*?){{LOCALE}}(.*)$', re.M)
     replacement = '\n'.join(map(lambda locale: r'\1%s\2%s\3' % (locale, locale), params['locales']))
     data = re.sub(localesRegExp, replacement, data)
 
@@ skipping 49 matching lines @@
 
 def initTranslators(localeMetadata):
   for locale in localeMetadata.itervalues():
     if 'translator' in locale:
       locale['translators'] = sorted(map(lambda t: t.strip(), locale['translator'].split(',')), key=unicode.lower)
     else:
       locale['translators'] = []
 
 def createManifest(baseDir, params):
   global KNOWN_APPS, defaultLocale
-  env = jinja2.Environment(loader=jinja2.FileSystemLoader(buildtools.__path__[0]), autoescape=True, extensions=['jinja2.ext.autoescape'])
-  template = env.get_template('install.rdf.tmpl')
+  template = getTemplate('install.rdf.tmpl', autoEscape=True)
   templateData = dict(params)
   templateData['localeMetadata'] = readLocaleMetadata(baseDir, params['locales'])
   initTranslators(templateData['localeMetadata'])
   templateData['KNOWN_APPS'] = KNOWN_APPS
   templateData['defaultLocale'] = defaultLocale
   return template.render(templateData).encode('utf-8')
 
 def readFile(files, params, path, name):
   ignoredFiles = getIgnoredFiles(params)
   if os.path.isdir(path):
@@ skipping 72 matching lines @@
     elif name.endswith('.xul'):
       match = re.search(r'<(?:window|dialog)\s[^>]*\bwindowtype="([^">]+)"', content)
       if match:
         templateData['chromeWindows'].append(match.group(1))
 
   while True:
     missing = []
     for module in templateData['requires']:
       moduleFile = 'lib/' + module + '.js'
       if not moduleFile in files:
+        import buildtools
         path = os.path.join(buildtools.__path__[0], moduleFile)
         if os.path.exists(path):
           missing.append((path, moduleFile))
     if not len(missing):
       break
     for path, moduleFile in missing:
       readFile(files, params, path, moduleFile)
       checkScript(moduleFile)
 
-  env = jinja2.Environment(loader=jinja2.FileSystemLoader(buildtools.__path__[0]))
-  env.filters['json'] = json.dumps
-  template = env.get_template('bootstrap.js.tmpl')
+  template = getTemplate('bootstrap.js.tmpl')
   files['bootstrap.js'] = processFile('bootstrap.js', template.render(templateData).encode('utf-8'), params)
 
 def signFiles(files, keyFile):
   import M2Crypto
   manifest = []
   signature = []
 
   def getDigest(data):
     md5 = hashlib.md5()
     md5.update(data)
@@ skipping 40 matching lines @@
 
 def writeXPI(files, outFile):
   zip = ZipFile(outFile, 'w', ZIP_DEFLATED)
   names = files.keys()
   names.sort(key=lambda x: '!' if x == 'META-INF/zigbert.rsa' else x)
   for name in names:
     zip.writestr(name, files[name])
   zip.close()
 
 def createBuild(baseDir, outFile=None, locales=None, buildNum=None, releaseBuild=False, keyFile=None, limitMetadata=False, multicompartment=False):
-  if buildNum == None:
-    buildNum = getBuildNum(baseDir)
   if locales == None:
     locales = getLocales(baseDir)
   elif locales == 'all':
     locales = getLocales(baseDir, True)
 
   metadata = readMetadata(baseDir)
-  version = metadata.get('general', 'version')
-  if not releaseBuild:
-    version += '.' + buildNum
+  version = getBuildVersion(baseDir, metadata, releaseBuild, buildNum)
 
   if limitMetadata:
     for option in metadata.options('compat'):
       if not option in ('firefox', 'thunderbird', 'seamonkey'):
         metadata.remove_option('compat', option)
 
   if outFile == None:
-    outFile = getDefaultFileName(baseDir, metadata, version)
+    outFile = getDefaultFileName(baseDir, metadata, version, 'xpi')
 
   contributors = getContributors(baseDir, metadata)
 
   params = {
     'locales': locales,
     'releaseBuild': releaseBuild,
-    'buildNum': buildNum,
     'version': version.encode('utf-8'),
     'metadata': metadata,
     'limitMetadata': limitMetadata,
     'contributors': contributors,
     'multicompartment': multicompartment,
   }
   files = {}
   files['install.rdf'] = createManifest(baseDir, params)
   for name, path in getChromeSubdirs(baseDir, params['locales']).iteritems():
     if os.path.isdir(path):
       readFile(files, params, path, 'chrome/%s' % name)
   if not params['limitMetadata']:
     fixupLocales(baseDir, files, params)
   readXPIFiles(baseDir, params, files)
   if not 'bootstrap.js' in files:
     addMissingFiles(baseDir, params, files)
   if keyFile:
     signFiles(files, keyFile)
   writeXPI(files, outFile)
 
 def autoInstall(baseDir, host, port, multicompartment=False):
   fileBuffer = StringIO()
   createBuild(baseDir, outFile=fileBuffer, multicompartment=multicompartment)
   urllib.urlopen('http://%s:%s/' % (host, port), data=fileBuffer.getvalue())