Index: packagerSafari.py diff --git a/packagerSafari.py b/packagerSafari.py deleted file mode 100644 index 288ba8d467e81ecbadd6146285e8d35f0f6d09f7..0000000000000000000000000000000000000000 --- a/packagerSafari.py +++ /dev/null @@ -1,181 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import base64 -import ConfigParser -import json -import os -import re -from urlparse import urlparse - -from packager import readMetadata, getDefaultFileName, getBuildVersion, getTemplate, Files -from packagerChrome import convertJS, import_locales, getIgnoredFiles, getPackageFiles, defaultLocale, createScriptPage - - -def processFile(path, data, params): - return data - - -def createManifest(params, files): - template = getTemplate('Info.plist.tmpl', autoEscape=True) - metadata = params['metadata'] - catalog = json.loads(files['_locales/%s/messages.json' % defaultLocale]) - - def parse_section(section, depth=1): - result = {} - - if not metadata.has_section(section): - return result - - for opt in metadata.options(section): - bits = opt.split('_', depth) - key = bits.pop().replace('_', ' ').title() - val = metadata.get(section, opt) - - try: - val = int(val) - except ValueError: - try: - val = float(val) - except ValueError: - pass - - reduce(lambda d, x: d.setdefault(x, {}), bits, result)[key] = val - - return result - - def get_optional(*args): - try: - return metadata.get(*args) - except ConfigParser.Error: - return None - - allowedDomains = set() - allowAllDomains = False - allowSecurePages = False - - for perm in metadata.get('general', 'permissions').split(): - if perm == '': - allowAllDomains = True - allowSecurePages = True - continue - - url = urlparse(perm) - - if url.scheme == 'https': - allowSecurePages = True - elif url.scheme != 'http': - continue - - if '*' in url.hostname: - allowAllDomains = True - continue - - allowedDomains.add(url.hostname) - - return template.render( - basename=metadata.get('general', 'basename'), - version=params['version'], - releaseBuild=params['releaseBuild'], - name=catalog['name']['message'], - description=catalog['description']['message'], - author=get_optional('general', 'author'), - homepage=get_optional('general', 'homepage'), - updateURL=get_optional('general', 'updateURL'), - allowedDomains=allowedDomains, - allowAllDomains=allowAllDomains, - allowSecurePages=allowSecurePages, - startScripts=(get_optional('contentScripts', 'document_start') or '').split(), - endScripts=(get_optional('contentScripts', 'document_end') or '').split(), - menus=parse_section('menus', 2), - toolbarItems=parse_section('toolbar_items'), - popovers=parse_section('popovers'), - developerIdentifier=params.get('developerIdentifier') - ).encode('utf-8') - - -def createInfoModule(params): - template = getTemplate('safariInfo.js.tmpl') - return template.render(params).encode('utf-8') - - -def _get_sequence(data): - from Crypto.Util import asn1 - sequence = asn1.DerSequence() - sequence.decode(data) - return sequence - - -def get_developer_identifier(certs): - for cert in certs: - # See https://tools.ietf.org/html/rfc5280#section-4 - tbscertificate = _get_sequence(base64.b64decode(cert))[0] - subject = _get_sequence(tbscertificate)[5] - - # We could decode the subject but since we have to apply a regular - # expression on CN entry anyway we can just skip that. - m = re.search(r'Safari Developer: \((\S*?)\)', subject) - if m: - return m.group(1) - - raise Exception('No Safari developer certificate found in chain') - - -def createBuild(baseDir, type, outFile=None, buildNum=None, releaseBuild=False, keyFile=None, devenv=False): - metadata = readMetadata(baseDir, type) - version = getBuildVersion(baseDir, metadata, releaseBuild, buildNum) - - if not outFile: - outFile = getDefaultFileName(metadata, version, 'safariextz' if keyFile else 'zip') - - params = { - 'type': type, - 'baseDir': baseDir, - 'releaseBuild': releaseBuild, - 'version': version, - 'devenv': devenv, - 'metadata': metadata, - } - - mapped = metadata.items('mapping') if metadata.has_section('mapping') else [] - files = Files(getPackageFiles(params), getIgnoredFiles(params), - process=lambda path, data: processFile(path, data, params)) - files.readMappedFiles(mapped) - files.read(baseDir, skip=[opt for opt, _ in mapped]) - - if metadata.has_section('convert_js'): - convertJS(params, files) - - if metadata.has_section('preprocess'): - files.preprocess( - [f for f, _ in metadata.items('preprocess')], - {'needsExt': True} - ) - - if metadata.has_section('import_locales'): - import_locales(params, files) - - if metadata.has_option('general', 'testScripts'): - files['qunit/index.html'] = createScriptPage(params, 'testIndex.html.tmpl', - ('general', 'testScripts')) - - if keyFile: - from buildtools import xarfile - certs, key = xarfile.read_certificates_and_key(keyFile) - params['developerIdentifier'] = get_developer_identifier(certs) - - files['lib/info.js'] = createInfoModule(params) - files['background.html'] = createScriptPage(params, 'background.html.tmpl', - ('general', 'backgroundScripts')) - files['Info.plist'] = createManifest(params, files) - - dirname = metadata.get('general', 'basename') + '.safariextension' - for filename in files.keys(): - files[os.path.join(dirname, filename)] = files.pop(filename) - - if not devenv and keyFile: - from buildtools import xarfile - xarfile.create(outFile, files, keyFile) - else: - files.zip(outFile) Index: templates/Info.plist.tmpl diff --git a/templates/Info.plist.tmpl b/templates/Info.plist.tmpl deleted file mode 100644 index 141f897d55774bf254bc0573260316cda6497f4b..0000000000000000000000000000000000000000 --- a/templates/Info.plist.tmpl +++ /dev/null @@ -1,165 +0,0 @@ -{%- macro str_or_real(value) -%} -{%- if value is number -%} - {{ value }} -{%- else -%} - {{ value }} -{%- endif -%} -{%- endmacro -%} - - - - - - CFBundleDisplayName - {{ name }} - CFBundleIdentifier - org.adblockplus.{{ basename }} - CFBundleInfoDictionaryVersion - 6.0 - CFBundleShortVersionString - {{ version }} - CFBundleVersion - {{ version }} - {%- if developerIdentifier %} - DeveloperIdentifier - {{ developerIdentifier }} - {%- endif %} - Chrome - - Database Quota - 104857600 - Global Page - background.html - {%- if menus %} - Menus - - {%- for identifier, items in menus.iteritems() %} - - Identifier - {{ identifier }} - Menu Items - - {%- for identifier, props in items.iteritems() %} - - Identifier - {{ identifier }} - {%- for key, value in props.iteritems() %} - {{ key }} - {{ str_or_real(value) }} - {%- endfor %} - - {%- endfor %} - - - {%- endfor %} - - {%- endif %} - {%- if popovers %} - Popovers - - {%- for identifier, props in popovers.iteritems() %} - - Identifier - {{ identifier }} - {%- for key, value in props.iteritems() %} - {{ key }} - {{ str_or_real(value) }} - {%- endfor %} - - {%- endfor %} - - {%- endif %} - {%- if toolbarItems %} - Toolbar Items - - {%- for identifier, props in toolbarItems.iteritems() %} - - Identifier - {{ identifier }} - Label - {{ name }} - {%- for key, value in props.iteritems() %} - {{ key }} - {{ str_or_real(value) }} - {%- endfor %} - - {%- endfor %} - - {%- endif %} - - {%- if startScripts or endScripts %} - Content - - Scripts - - {%- if startScripts %} - Start - - {%- for script in startScripts %} - {{ script }} - {%- endfor %} - - {%- endif %} - {%- if endScripts %} - End - - {%- for script in endScripts %} - {{ script }} - {%- endfor %} - - {%- endif %} - - - {%- endif %} - ExtensionInfoDictionaryVersion - 1.0 - Permissions - - Website Access - - {%- if allowedDomains and not allowAllDomains %} - Allowed Domains - - {%- for domain in allowedDomains %} - {{ domain }} - {%- endfor %} - - {%- endif %} - Include Secure Pages - {%- if allowSecurePages %} - - {%- else %} - - {%- endif %} - Level - {%- if allowAllDomains %} - All - {%- elif allowedDomains %} - Some - {%- else %} - None - {%- endif %} - - - Description - {{ description }} - {%- if author %} - Author - {{ author }} - {%- endif %} - {%- if homepage %} - Website - {{ homepage }} - {%- endif %} - {%- if not releaseBuild or updateURL %} - Update Manifest URL - - {%- if not releaseBuild -%} - https://adblockplus.org/devbuilds/{{ basename }}/updates.plist - {%- else -%} - {{ updateURL }} - {%- endif -%} - - {%- endif %} - - Index: templates/background.html.tmpl diff --git a/templates/background.html.tmpl b/templates/background.html.tmpl deleted file mode 100644 index e8febc9c779179354fe76d15ae0dfc098916fccb..0000000000000000000000000000000000000000 --- a/templates/background.html.tmpl +++ /dev/null @@ -1,9 +0,0 @@ - - -{%- for script in scripts %} - -{%- endfor %} - - - - Index: templates/safariInfo.js.tmpl diff --git a/templates/safariInfo.js.tmpl b/templates/safariInfo.js.tmpl deleted file mode 100644 index 59b4b4d096c462b6fd24d791e542683ad0804b1e..0000000000000000000000000000000000000000 --- a/templates/safariInfo.js.tmpl +++ /dev/null @@ -1,18 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -require.scopes.info = { - addonName: {{ metadata.get('general', 'basename')|json }}, - addonVersion: {{ version|json }}, - - application: 'safari', - get applicationVersion() { - return navigator.userAgent.match(/Version\/([\d.]+)/)[1]; - }, - - platform: 'safari', - get platformVersion() { - return this.applicationVersion; - } -}; Index: templates/xartoc.xml.tmpl diff --git a/templates/xartoc.xml.tmpl b/templates/xartoc.xml.tmpl deleted file mode 100644 index bc8bada013705feea83ad59a7fb820e35c250b58..0000000000000000000000000000000000000000 --- a/templates/xartoc.xml.tmpl +++ /dev/null @@ -1,54 +0,0 @@ - - - - -{% macro file(data) %} - - {{data.name}} - {{data.type}} - {{data.mode}} - {% if data.type == 'directory' %} - {% for f in data.children %} - {{file(f)}} - {% endfor %} - {% else %} - - {{data.checksum_compressed}} - {{data.checksum_uncompressed}} - - {{data.size_uncompressed}} - {{data.offset}} - {{data.size_compressed}} - - {% endif %} - -{% endmacro %} - - - - {{timestamp_numerical}} - - {{checksum.offset}} - {{checksum.size}} - - {{timestamp_iso}} - - {{signature.offset}} - {{signature.size}} - - - {% for certificate in certificates %} - {{certificate}} - {% endfor %} - - - - {% for f in files %} - {{file(f)}} - {% endfor %} - - Index: xarfile.py diff --git a/xarfile.py b/xarfile.py deleted file mode 100644 index 448b7a6d1fc3fe1f7c6f63d70983c53ffecaecd2..0000000000000000000000000000000000000000 --- a/xarfile.py +++ /dev/null @@ -1,140 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import re -import struct -import time -import zlib - -from Crypto.Hash import SHA -from Crypto.PublicKey import RSA -from Crypto.Signature import PKCS1_v1_5 - -from buildtools.packager import getTemplate - -XAR_HEADER = struct.Struct('>IHHQQI') -XAR_HEADER_MAGIC = 0x78617221 -XAR_VERSION = 1 -XAR_CKSUM_SHA1 = 1 - - -def read_certificates_and_key(keyfile): - with open(keyfile, 'r') as file: - data = file.read() - - certificates = [] - key = None - for match in re.finditer(r'-+BEGIN (.*?)-+(.*?)-+END \1-+', data, re.S): - section = match.group(1) - if section == 'CERTIFICATE': - certificates.append(re.sub(r'\s+', '', match.group(2))) - elif section == 'PRIVATE KEY': - key = RSA.importKey(match.group(0)) - if not key: - raise Exception('Could not find private key in file') - - return certificates, key - - -def get_checksum(data): - return SHA.new(data).digest() - - -def get_hexchecksum(data): - return SHA.new(data).hexdigest() - - -def get_signature(key, data): - return PKCS1_v1_5.new(key).sign(SHA.new(data)) - - -def compress_files(filedata, root, offset): - compressed_data = [] - filedata = sorted(filedata.iteritems()) - directory_stack = [('', root)] - file_id = 1 - for path, data in filedata: - # Remove directories that are done - while not path.startswith(directory_stack[-1][0]): - directory_stack.pop() - - # Add new directories - directory_path = directory_stack[-1][0] - relpath = path[len(directory_path):] - while '/' in relpath: - name, relpath = relpath.split('/', 1) - directory_path += name + '/' - directory = { - 'id': file_id, - 'name': name, - 'type': 'directory', - 'mode': '0755', - 'children': [], - } - file_id += 1 - directory_stack[-1][1].append(directory) - directory_stack.append((directory_path, directory['children'])) - - # Add the actual file - compressed = zlib.compress(data, 9) - file = { - 'id': file_id, - 'name': relpath, - 'type': 'file', - 'mode': '0644', - 'checksum_uncompressed': get_hexchecksum(data), - 'size_uncompressed': len(data), - 'checksum_compressed': get_hexchecksum(compressed), - 'size_compressed': len(compressed), - 'offset': offset, - } - file_id += 1 - offset += len(compressed) - directory_stack[-1][1].append(file) - compressed_data.append(compressed) - return compressed_data - - -def create(archivepath, contents, keyfile): - certificates, key = read_certificates_and_key(keyfile) - checksum_length = len(get_checksum('')) - params = { - 'certificates': certificates, - - # Timestamp epoch starts at 2001-01-01T00:00:00.000Z - 'timestamp_numerical': time.time() - 978307200, - 'timestamp_iso': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), - - 'checksum': { - 'offset': 0, - 'size': checksum_length, - }, - 'signature': { - 'offset': checksum_length, - 'size': len(get_signature(key, '')), - }, - 'files': [], - } - - offset = params['signature']['offset'] + params['signature']['size'] - compressed_data = compress_files(contents, params['files'], offset) - - template = getTemplate('xartoc.xml.tmpl', autoEscape=True) - toc_uncompressed = template.render(params).encode('utf-8') - toc_compressed = zlib.compress(toc_uncompressed, 9) - - with open(archivepath, 'wb') as file: - # The file starts with a minimalistic header - file.write(XAR_HEADER.pack(XAR_HEADER_MAGIC, XAR_HEADER.size, - XAR_VERSION, len(toc_compressed), - len(toc_uncompressed), XAR_CKSUM_SHA1)) - - # It's followed up with a compressed XML table of contents - file.write(toc_compressed) - - # Now the actual data, all the offsets are in the table of contents - file.write(get_checksum(toc_compressed)) - file.write(get_signature(key, toc_compressed)) - for blob in compressed_data: - file.write(blob) Index: build.py diff --git a/build.py b/build.py index cc2d1091b31951380e5477826d1e8f8501ec72cc..0ecab321b8b9f423f0b69d1cb8606a81b587203c 100644 --- a/build.py +++ b/build.py @@ -11,7 +11,7 @@ from getopt import getopt, GetoptError from StringIO import StringIO from zipfile import ZipFile -knownTypes = ('gecko-webext', 'chrome', 'safari', 'generic', 'edge') +knownTypes = ('gecko-webext', 'chrome', 'generic', 'edge') class Command(object): @@ -190,8 +190,6 @@ def runBuild(baseDir, scriptName, opts, args, type): if type in {'chrome', 'gecko-webext'}: import buildtools.packagerChrome as packager - elif type == 'safari': - import buildtools.packagerSafari as packager elif type == 'edge': import buildtools.packagerEdge as packager @@ -199,10 +197,7 @@ def runBuild(baseDir, scriptName, opts, args, type): def createDevEnv(baseDir, scriptName, opts, args, type): - if type == 'safari': - import buildtools.packagerSafari as packager - else: - import buildtools.packagerChrome as packager + import buildtools.packagerChrome as packager file = StringIO() packager.createBuild(baseDir, type=type, outFile=file, devenv=True, releaseBuild=True) @@ -370,7 +365,7 @@ def runReleaseAutomation(baseDir, scriptName, opts, args, type): usage(scriptName, type, 'release') return - if type in {'chrome', 'safari'} and keyFile is None: + if type == 'chrome' and keyFile is None: print >>sys.stderr, 'Error: you must specify a key file for this release' usage(scriptName, type, 'release') return @@ -392,14 +387,14 @@ with addCommand(runBuild, 'build') as command: command.description = 'Creates an extension build with given file name. If output_file is missing a default name will be chosen.' command.params = '[options] [output_file]' command.addOption('Use given build number (if omitted the build number will be retrieved from Mercurial)', short='b', long='build', value='num') - command.addOption('File containing private key and certificates required to sign the package', short='k', long='key', value='file', types=('chrome', 'safari')) + command.addOption('File containing private key and certificates required to sign the package', short='k', long='key', value='file', types=('chrome',)) command.addOption('Create a release build', short='r', long='release') - command.supportedTypes = ('gecko-webext', 'chrome', 'safari', 'edge') + command.supportedTypes = ('gecko-webext', 'chrome', 'edge') with addCommand(createDevEnv, 'devenv') as command: command.shortDescription = 'Set up a development environment' command.description = 'Will set up or update the devenv folder as an unpacked extension folder for development.' - command.supportedTypes = ('gecko-webext', 'chrome', 'safari') + command.supportedTypes = ('gecko-webext', 'chrome') with addCommand(setupTranslations, 'setuptrans') as command: command.shortDescription = 'Sets up translation languages' @@ -432,10 +427,10 @@ with addCommand(generateDocs, 'docs') as command: with addCommand(runReleaseAutomation, 'release') as command: command.shortDescription = 'Run release automation' command.description = 'Note: If you are not the project owner then you ' "probably don't want to run this!\n\n" 'Runs release automation: creates downloads for the new version, tags ' 'source code repository as well as downloads and buildtools repository.' - command.addOption('File containing private key and certificates required to sign the release.', short='k', long='key', value='file', types=('chrome', 'safari', 'edge')) + command.addOption('File containing private key and certificates required to sign the release.', short='k', long='key', value='file', types=('chrome', 'edge')) command.addOption('Directory containing downloads repository (if omitted ../downloads is assumed)', short='d', long='downloads', value='dir') command.params = '[options] ' - command.supportedTypes = ('chrome', 'safari', 'edge') + command.supportedTypes = ('chrome', 'edge') with addCommand(updatePSL, 'updatepsl') as command: command.shortDescription = 'Updates Public Suffix List' Index: releaseAutomation.py diff --git a/releaseAutomation.py b/releaseAutomation.py index a7c09aa7782cd20d3139a678ffc971cc0ed00c8e..805fa257c92f49f52317a82b4725c8a3205e2632 100644 --- a/releaseAutomation.py +++ b/releaseAutomation.py @@ -118,9 +118,7 @@ def run(baseDir, type, version, keyFile, downloadsRepo): print('Aborting release.') return 1 - if type == 'safari': - import buildtools.packagerSafari as packager - elif type == 'edge': + if type == 'edge': import buildtools.packagerEdge as packager elif type == 'chrome': import buildtools.packagerChrome as packager @@ -148,7 +146,7 @@ def run(baseDir, type, version, keyFile, downloadsRepo): # Now commit the change and tag it subprocess.check_call(['hg', 'commit', '-R', baseDir, '-m', 'Releasing %s %s' % (extensionName, version)]) tag_name = version - if type in {'safari', 'edge'}: + if type == 'edge': tag_name = '{}-{}'.format(tag_name, type) subprocess.check_call(['hg', 'tag', '-R', baseDir, '-f', tag_name]) @@ -162,10 +160,6 @@ def run(baseDir, type, version, keyFile, downloadsRepo): buildPathUnsigned = os.path.join(baseDir, getDefaultFileName(metadata, version, 'zip')) packager.createBuild(baseDir, type=type, outFile=buildPathUnsigned, releaseBuild=True, keyFile=None) - elif type == 'safari': - buildPath = os.path.join(downloadsRepo, getDefaultFileName(metadata, version, 'safariextz')) - packager.createBuild(baseDir, type='safari', outFile=buildPath, releaseBuild=True, keyFile=keyFile) - downloads.append(buildPath) elif type == 'edge': # We only offer the Edge extension for use through the Windows Store buildPath = os.path.join(downloadsRepo, getDefaultFileName(metadata, version, 'appx')) Index: tox.ini diff --git a/tox.ini b/tox.ini index 5a2d5daed6124cfbdf9a6c37ec6b496ea2db6868..f79f0494d5b73efcff10242414862a6f75352f6c 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,6 @@ per-file-ignores = localeTools.py : A103,A104,A107,A301,A302,E501,E713,F401,N802,N803,N806 packager.py : A102,A107,A206,A302,E501,E711,N802,N803,N806 packagerChrome.py : A101,A104,A107,A111,A112,A302,E501,E711,F841,N802,N803,N806 - packagerSafari.py : A107,A302,E501,N802,N803,N806 publicSuffixListUpdater.py : A108,D200,D202,D205,D400,D401,D403,E501,F821,N802,N803 releaseAutomation.py : A102,A107,A108,A302,D202,D400,D401,E501,F401,N803,N806 tests/test_packagerEdge.py : D401