Index: packagerGecko.py diff --git a/packagerGecko.py b/packagerGecko.py deleted file mode 100644 index 0b038d2c05de350381bb4bf132cf598b14f1e094..0000000000000000000000000000000000000000 --- a/packagerGecko.py +++ /dev/null @@ -1,340 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import os -import sys -import re -import hashlib -import base64 -import urllib -import json -import io -from ConfigParser import SafeConfigParser -from StringIO import StringIO -import xml.dom.minidom as minidom -import buildtools.localeTools as localeTools - -import packager -from packager import readMetadata, getMetadataPath, getDefaultFileName, getBuildVersion, getTemplate, Files - -KNOWN_APPS = { - 'conkeror': '{a79fe89b-6662-4ff4-8e88-09950ad4dfde}', - 'emusic': 'dlm@emusic.com', - 'fennec': '{a23983c0-fd0e-11dc-95ff-0800200c9a66}', - 'fennec2': '{aa3c5121-dab2-40e2-81ca-7ea25febc110}', - 'firefox': '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}', - 'midbrowser': '{aa5ca914-c309-495d-91cf-3141bbb04115}', - 'prism': 'prism@developer.mozilla.org', - 'seamonkey': '{92650c4d-4b8e-4d2a-b7eb-24ecf4f6b63a}', - 'songbird': 'songbird@songbirdnest.com', - 'thunderbird': '{3550f703-e582-4d05-9a08-453d09bdfdc6}', - 'toolkit': 'toolkit@mozilla.org', - 'adblockbrowser': '{55aba3ac-94d3-41a8-9e25-5c21fe874539}', -} - -defaultLocale = 'en-US' - - -def getChromeDir(baseDir): - return os.path.join(baseDir, 'chrome') - - -def getLocalesDir(baseDir): - return os.path.join(getChromeDir(baseDir), 'locale') - - -def getChromeSubdirs(baseDir, locales): - result = {} - chromeDir = getChromeDir(baseDir) - for subdir in ('content', 'skin'): - result[subdir] = os.path.join(chromeDir, subdir) - for locale in locales: - result['locale/%s' % locale] = os.path.join(chromeDir, 'locale', locale) - return result - - -def getPackageFiles(params): - result = { - 'chrome', 'components', 'modules', 'lib', 'resources', 'webextension', - 'chrome.manifest', 'icon.png', 'icon64.png' - } - - baseDir = params['baseDir'] - for file in os.listdir(baseDir): - if file.endswith('.js') or file.endswith('.xml'): - result.add(file) - return result - - -def getIgnoredFiles(params): - return {'.incomplete', 'meta.properties'} - - -def archive_path(path, baseDir): - return '/'.join(os.path.split(os.path.relpath(path, baseDir))) - - -def isValidLocale(localesDir, dir, includeIncomplete=False): - if re.search(r'[^\w\-]', dir): - return False - curLocaleDir = os.path.join(localesDir, dir) - if not os.path.isdir(curLocaleDir): - return False - if len(os.listdir(curLocaleDir)) == 0: - return False - if not includeIncomplete and os.path.exists(os.path.join(localesDir, dir, '.incomplete')): - return False - return True - - -def getLocales(baseDir, includeIncomplete=False): - global defaultLocale - localesDir = getLocalesDir(baseDir) - locales = filter(lambda dir: isValidLocale(localesDir, dir, includeIncomplete), os.listdir(localesDir)) - locales.sort(key=lambda x: '!' if x == defaultLocale else x) - return locales - - -def processFile(path, data, params): - if path.endswith('.manifest') and data.find('{{LOCALE}}') >= 0: - localesRegExp = re.compile(r'^(.*?){{LOCALE}}(.*?){{LOCALE}}(.*)$', re.M) - replacement = '\n'.join(map(lambda locale: r'\1%s\2%s\3' % (locale, locale), params['locales'])) - data = re.sub(localesRegExp, replacement, data) - - return data - - -def readLocaleMetadata(baseDir, locales): - result = {} - - # Make sure we always have fallback data even if the default locale isn't part - # of the build - locales = list(locales) - if not defaultLocale in locales: - locales.append(defaultLocale) - - for locale in locales: - data = SafeConfigParser() - data.optionxform = str - try: - result[locale] = localeTools.readFile(os.path.join(getLocalesDir(baseDir), locale, 'meta.properties')) - except: - result[locale] = {} - return result - - -def getContributors(metadata): - main = [] - additional = set() - if metadata.has_section('contributors'): - options = metadata.options('contributors') - options.sort() - for option in options: - value = metadata.get('contributors', option) - if re.search(r'\D', option): - match = re.search(r'^\s*(\S+)\s+//([^/\s]+)/@(\S+)\s*$', value) - if not match: - print >>sys.stderr, 'Warning: unrecognized contributor location "%s"\n' % value - continue - baseDir = os.path.dirname(metadata.option_source('contributors', option)) - parts = match.group(1).split('/') - dom = minidom.parse(os.path.join(baseDir, *parts)) - tags = dom.getElementsByTagName(match.group(2)) - for tag in tags: - if tag.hasAttribute(match.group(3)): - for name in re.split(r'\s*,\s*', tag.getAttribute(match.group(3))): - additional.add(name) - else: - main.append(value) - return main + sorted(additional, key=unicode.lower) - - -def initTranslators(localeMetadata): - for locale in localeMetadata.itervalues(): - if 'translator' in locale: - locale['translators'] = sorted(map(lambda t: t.strip(), locale['translator'].split(',')), key=unicode.lower) - else: - locale['translators'] = [] - - -def createManifest(params): - global KNOWN_APPS, defaultLocale - template = getTemplate('install.rdf.tmpl', autoEscape=True) - templateData = dict(params) - templateData['localeMetadata'] = readLocaleMetadata(params['baseDir'], params['locales']) - initTranslators(templateData['localeMetadata']) - templateData['KNOWN_APPS'] = KNOWN_APPS - templateData['defaultLocale'] = defaultLocale - return template.render(templateData).encode('utf-8') - - -def importLocales(params, files): - SECTION = 'import_locales' - if not params['metadata'].has_section(SECTION): - return - - import localeTools - - for locale in params['locales']: - for item in params['metadata'].items(SECTION): - path, keys = item - parts = [locale if p == '*' else p for p in path.split('/')] - source = os.path.join(os.path.dirname(item.source), *parts) - if not os.path.exists(source): - continue - - with io.open(source, 'r', encoding='utf-8') as handle: - data = json.load(handle) - - target_name = os.path.splitext(os.path.basename(source))[0] + '.properties' - target = archive_path(os.path.join(getLocalesDir(params['baseDir']), locale, target_name), params['baseDir']) - - files[target] = '' - for key, value in sorted(data.items()): - message = value['message'] - files[target] += localeTools.generateStringEntry(key, message, target).encode('utf-8') - - -def fixupLocales(params, files): - global defaultLocale - - # Read in default locale data, it might not be included in package files - defaultLocaleDir = os.path.join(getLocalesDir(params['baseDir']), defaultLocale) - reference_files = Files(getPackageFiles(params), getIgnoredFiles(params)) - reference_files.read(defaultLocaleDir, archive_path(defaultLocaleDir, params['baseDir'])) - reference_params = dict(params) - reference_params['locales'] = [defaultLocale] - importLocales(reference_params, reference_files) - - reference = {} - for path, data in reference_files.iteritems(): - filename = path.split('/')[-1] - data = localeTools.parseString(data.decode('utf-8'), filename) - if data: - reference[filename] = data - - for locale in params['locales']: - for file in reference.iterkeys(): - path = 'chrome/locale/%s/%s' % (locale, file) - if path in files: - data = localeTools.parseString(files[path].decode('utf-8'), path) - for key, value in reference[file].iteritems(): - if not key in data: - files[path] += localeTools.generateStringEntry(key, value, path).encode('utf-8') - else: - files[path] = reference[file]['_origData'].encode('utf-8') - - -def processJSONFiles(params, files): - prefix = 'lib/' - for name, content in files.iteritems(): - if name.startswith(prefix) and name.endswith('.json'): - params['jsonRequires'][name[len(prefix):]] = json.loads(content) - for name in params['jsonRequires'].iterkeys(): - del files[prefix + name] - - -def addMissingFiles(params, files): - templateData = { - 'hasChrome': False, - 'hasChromeRequires': False, - 'hasShutdownHandlers': False, - 'chromeWindows': [], - 'requires': set(), - 'jsonRequires': params['jsonRequires'], - 'metadata': params['metadata'], - 'hasWebExtension': params['hasWebExtension'], - 'multicompartment': params['multicompartment'], - 'applications': dict((v, k) for k, v in KNOWN_APPS.iteritems()), - } - - def checkScript(name): - content = files[name] - for match in re.finditer(r'(?:^|\s)require\(\s*"([\w\-]+)"\s*\)', content): - templateData['requires'].add(match.group(1)) - if name.startswith('chrome/content/'): - templateData['hasChromeRequires'] = True - if not '/' in name or name.startswith('lib/'): - if re.search(r'(?:^|\s)onShutdown\.', content): - templateData['hasShutdownHandlers'] = True - - for name, content in files.iteritems(): - if name == 'chrome.manifest': - templateData['hasChrome'] = True - elif name.endswith('.js'): - checkScript(name) - elif name.endswith('.xul'): - match = re.search(r'<(?:window|dialog)\s[^>]*\bwindowtype="([^">]+)"', content) - if match: - templateData['chromeWindows'].append(match.group(1)) - - while True: - missing = [] - for module in templateData['requires']: - moduleFile = 'lib/' + module + '.js' - if not moduleFile in files: - import buildtools - path = os.path.join(buildtools.__path__[0], moduleFile) - if os.path.exists(path): - missing.append((path, moduleFile)) - if not len(missing): - break - for path, moduleFile in missing: - files.read(path, moduleFile) - checkScript(moduleFile) - - template = getTemplate('bootstrap.js.tmpl') - files['bootstrap.js'] = template.render(templateData).encode('utf-8') - - -def createBuild(baseDir, type='gecko', outFile=None, locales=None, buildNum=None, releaseBuild=False, multicompartment=False): - if locales == None: - locales = getLocales(baseDir) - elif locales == 'all': - locales = getLocales(baseDir, True) - - metadata = readMetadata(baseDir, type) - version = getBuildVersion(baseDir, metadata, releaseBuild, buildNum) - - if outFile == None: - outFile = getDefaultFileName(metadata, version, 'xpi') - - contributors = getContributors(metadata) - - params = { - 'baseDir': baseDir, - 'locales': locales, - 'releaseBuild': releaseBuild, - 'version': version.encode('utf-8'), - 'metadata': metadata, - 'contributors': contributors, - 'multicompartment': multicompartment, - 'hasWebExtension': os.path.isdir(os.path.join(baseDir, 'webextension')), - 'jsonRequires': {}, - } - - mapped = metadata.items('mapping') if metadata.has_section('mapping') else [] - skip = [opt for opt, _ in mapped] + ['chrome'] - files = Files(getPackageFiles(params), getIgnoredFiles(params), - process=lambda path, data: processFile(path, data, params)) - files['install.rdf'] = createManifest(params) - files.readMappedFiles(mapped) - files.read(baseDir, skip=skip) - for name, path in getChromeSubdirs(baseDir, params['locales']).iteritems(): - if os.path.isdir(path): - files.read(path, 'chrome/%s' % name, skip=skip) - importLocales(params, files) - fixupLocales(params, files) - processJSONFiles(params, files) - if not 'bootstrap.js' in files: - addMissingFiles(params, files) - if metadata.has_section('preprocess'): - files.preprocess([f for f, _ in metadata.items('preprocess')]) - files.zip(outFile, sortKey=lambda x: '!' if x == 'META-INF/zigbert.rsa' else x) - - -def autoInstall(baseDir, type, host, port, multicompartment=False): - fileBuffer = StringIO() - createBuild(baseDir, type=type, outFile=fileBuffer, multicompartment=multicompartment) - urllib.urlopen('http://%s:%s/' % (host, port), data=fileBuffer.getvalue())