Index: .sitescripts.example
===================================================================
--- a/.sitescripts.example
+++ b/.sitescripts.example
@@ -44,8 +44,9 @@
[extensions]
abp_repository=%(root)s/hg/adblockplus
-abp_name=Adblock Plus
+abp_name=Adblock Plus for Mozilla Firefox
abp_galleryID=adblock-plus
+abp_pad=true
abpchrome_repository=%(root)s/hg/adblockpluschrome
abpchrome_type=chrome
abpchrome_name=Adblock Plus for Google Chrome
@@ -55,19 +56,23 @@
abpchrome_clientID=...apps.googleusercontent.com
abpchrome_clientSecret=secret
abpchrome_refreshToken=1/rwn...
+abpchrome_pad=true
abpopera_repository=%(root)s/hg/adblockpluschrome
abpopera_type=opera
abpopera_name=Adblock Plus for Opera
abpopera_galleryID=opera-adblock
abpopera_key=%(root)s/data/adblockpluschrome.pem
+abpopera_pad=true
abpsafari_repository=%(root)s/subdomains/hg/adblockpluschrome/
abpsafari_type=safari
abpsafari_name=Adblock Plus for Safari
abpsafari_key=%(root)s/files/adblockplussafari.pem
+abpsafari_pad=true
abpandroid_repository=%(root)s/hg/adblockplusandroid
abpandroid_type=android
abpandroid_name=Adblock Plus for Android
abpandroid_downloadPage=/en/android
+abpandroid_pad=true
ehh_repository=%(root)s/hg/elemhidehelper
ehh_name=Element Hiding Helper
ehh_galleryID=elemhidehelper
@@ -77,6 +82,10 @@
deobfuscator_repository=%(root)s/hg/jsdeobfuscator
deobfuscator_name=JavaScript Deobfuscator
deobfuscator_galleryID=javascript-deobfuscator
+urlfixer_repository=%(root)s/hg/urlfixer
+urlfixer_name=URL fixer
+urlfixer_galleryID=url-fixer
+urlfixer_pad=true
buildRepository=%(root)s/hg/buildtools
jsdocRepository=%(root)s/hg/jsdoc-toolkit
nightliesData=%(root)s/data/nightlies
@@ -89,6 +98,8 @@
androidUpdateManifestPath=%(root)s/www/androidupdates.xml
safariUpdateManifestPath=%(root)s/www/updates.plist
docsDirectory=%(root)s/www/jsdoc
+padDirectory=%(root)s/www/pad
+padURL=https://adblockplus.org/pad/
errorTemplate=extensions/template/error.html
languageCheckTemplate=extensions/template/languageCheckResult.html
languageFormTemplate=extensions/template/languageForm.html
@@ -98,6 +109,7 @@
safariUpdateManifest=extensions/template/updates.plist
androidUpdateManifest=extensions/template/androidupdates.xml
nightlyIndexPage=extensions/template/nightlies.html
+padTemplate=extensions/template/pad.xml
[stats]
geoip_db=%(root)s/data/GeoIP.dat
Index: sitescripts/extensions/android.py
===================================================================
new file mode 100644
--- /dev/null
+++ b/sitescripts/extensions/android.py
@@ -0,0 +1,31 @@
+# This file is part of the Adblock Plus web scripts,
+# Copyright (C) 2006-2014 Eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus. If not, see .
+
+import subprocess
+import xml.dom.minidom as dom
+
+ANDROID_VERSIONS = ['1.0', '1.1', '1.5', '1.6', '2.0', '2.0.1', '2.1',
+ '2.2', '2.3', '2.3.3', '3.0', '3.1', '3.2', '4.0',
+ '4.0.3', '4.1', '4.2', '4.3', '4.4']
+
+def get_min_sdk_version(repo, version):
+ command = ['hg', 'cat', '-r', version, 'AndroidManifest.xml']
+ result = subprocess.check_output(command, cwd=repo.repository)
+
+ uses_sdk = dom.parseString(result).getElementsByTagName('uses-sdk')[0]
+ return uses_sdk.attributes["android:minSdkVersion"].value
+
+def get_min_android_version(repo, version):
+ return ANDROID_VERSIONS[int(get_min_sdk_version(repo, version)) - 1]
Index: sitescripts/extensions/bin/updateDownloadLinks.py
===================================================================
--- a/sitescripts/extensions/bin/updateDownloadLinks.py
+++ b/sitescripts/extensions/bin/updateDownloadLinks.py
@@ -29,6 +29,8 @@
from StringIO import StringIO
from sitescripts.utils import get_config, get_template
from sitescripts.extensions.utils import compareVersions, Configuration, getSafariCertificateID
+from sitescripts.extensions.android import get_min_sdk_version
+from sitescripts.extensions.pad import PadFile
from buildtools.packagerGecko import KNOWN_APPS
def urlencode(value):
@@ -66,84 +68,77 @@
"""
gets download link for a Chrome add-on from the Chrome Gallery site
"""
- param = 'id=%s&uc' % urlencode(galleryID)
- url = 'https://clients2.google.com/service/update2/crx?x=%s' % urlencode(param)
- contents = urlopen(url).read()
- document = dom.parseString(contents)
+ galleryID = urlencode(galleryID)
+
+ url = 'https://clients2.google.com/service/update2/crx?x=%s' % urlencode('id=%s&uc' % galleryID)
+ document = dom.parse(urlopen(url))
updateTags = document.getElementsByTagName('updatecheck')
- updateTag = updateTags[0] if len(updateTags) > 0 else None
- if updateTag and updateTag.hasAttribute('codebase') and updateTag.hasAttribute('version'):
- return (updateTag.getAttribute('codebase'), updateTag.getAttribute('version'))
- else:
+ version = updateTags and updateTags[0].getAttribute('version')
+
+ if not version:
return (None, None)
+ request = urllib2.Request('https://chrome.google.com/webstore/detail/_/' + galleryID)
+ request.get_method = lambda : 'HEAD'
+ url = urllib2.urlopen(request).geturl()
+
+ return (url, version)
+
def getOperaDownloadLink(galleryID):
"""
gets download link for an Opera add-on from the Opera Addons site
"""
- class HeadRequest(urllib2.Request):
- def get_method(self):
- return "HEAD"
+ galleryID = urlencode(galleryID)
- url = 'https://addons.opera.com/extensions/download/%s/' % urlencode(galleryID)
- response = urllib2.urlopen(HeadRequest(url))
- content_disposition = response.info().dict.get('content-disposition', None)
- if content_disposition != None:
- match = re.search(r'filename=\S+-([\d.]+)-\d+\.oex$', content_disposition)
- else:
- match = None;
- if match:
- return (url, match.group(1))
- else:
- return (None, None)
+ request = urllib2.Request('https://addons.opera.com/extensions/download/%s/' % galleryID)
+ request.get_method = lambda : 'HEAD'
+ response = urllib2.urlopen(request)
+
+ content_disposition = response.info().getheader('Content-Disposition')
+ if content_disposition:
+ match = re.search(r'filename=\S+-([\d.]+)-\d+\.crx$', content_disposition)
+ if match:
+ return ('https://addons.opera.com/extensions/details/%s/' % galleryID , match.group(1))
+
+ return (None, None)
def getLocalLink(repo):
"""
gets the link for the newest download of an add-on in the local downloads
repository
"""
- url = repo.downloadsURL
-
highestURL = None
highestVersion = None
- if repo.type in ('gecko', 'chrome', 'opera', 'safari'):
- prefix = readRawMetadata(repo).get('general', 'basename')
- else:
- prefix = os.path.basename(repo.repository)
- prefix += '-'
- suffix = repo.packageSuffix
+ for filename, version in repo.getDownloads():
+ if not highestVersion or compareVersions(version, highestVersion) > 0:
+ highestURL = urlparse.urljoin(repo.downloadsURL, filename)
+ highestVersion = version
- # go through the downloads repository looking for downloads matching this extension
- command = ['hg', 'locate', '-R', repo.downloadsRepo, '-r', 'default']
- result = subprocess.check_output(command)
- for fileName in result.splitlines():
- if fileName.startswith(prefix) and fileName.endswith(suffix):
- version = fileName[len(prefix):len(fileName) - len(suffix)]
- if highestVersion == None or compareVersions(version, highestVersion) > 0:
- highestURL = urlparse.urljoin(url, fileName)
- highestVersion = version
return (highestURL, highestVersion)
def getDownloadLink(repo):
"""
gets the download link to the most current version of an extension
"""
- galleryURL = None
- galleryVersion = None
- if repo.type == "gecko" and repo.galleryID:
+ # you can't easily install extensions from third-party sources on Chrome
+ # and Opera. So always get the link for the version on the Web Store.
+ if repo.galleryID:
+ if repo.type == "chrome":
+ return getGoogleDownloadLink(repo.galleryID)
+ if repo.type == "opera":
+ return getOperaDownloadLink(repo.galleryID)
+
+ (localURL, localVersion) = getLocalLink(repo)
+
+ # get a link to Firefox Add-Ons, if the latest version has been published there
+ if repo.type == 'gecko' and repo.galleryID:
(galleryURL, galleryVersion) = getMozillaDownloadLink(repo.galleryID)
- elif repo.type == "chrome" and repo.galleryID:
- (galleryURL, galleryVersion) = getGoogleDownloadLink(repo.galleryID)
- elif repo.type == "opera" and repo.galleryID:
- (galleryURL, galleryVersion) = getOperaDownloadLink(repo.galleryID)
+ if not localVersion or (galleryVersion and
+ compareVersions(galleryVersion, localVersion) >= 0):
+ return (galleryURL, galleryVersion)
- (downloadsURL, downloadsVersion) = getLocalLink(repo)
- if galleryVersion == None or (downloadsVersion != None and
- compareVersions(galleryVersion, downloadsVersion) < 0):
- return (downloadsURL, downloadsVersion)
- else:
- return (galleryURL, galleryVersion)
+ return (localURL, localVersion)
def getQRCode(text):
try:
@@ -175,23 +170,6 @@
if qrcode != None:
result.set(repo.repositoryName, "qrcode", qrcode)
-def readRawMetadata(repo, version='tip'):
- files = subprocess.check_output(['hg', '-R', repo.repository, 'locate', '-r', version]).splitlines()
- genericFilename = 'metadata'
- filename = '%s.%s' % (genericFilename, repo.type)
-
- # Fall back to platform-independent metadata file
- if filename not in files:
- filename = genericFilename
-
- command = ['hg', '-R', repo.repository, 'cat', '-r', version, os.path.join(repo.repository, filename)]
- result = subprocess.check_output(command)
-
- parser = SafeConfigParser()
- parser.readfp(StringIO(result))
-
- return parser
-
def readMetadata(repo, version):
"""
reads extension ID and compatibility information from metadata file in the
@@ -202,17 +180,12 @@
result = subprocess.check_output(command)
revision = re.sub(r'\D', '', result)
- command = ['hg', '-R', repo.repository, 'cat', '-r', version, os.path.join(repo.repository, 'AndroidManifest.xml')]
- result = subprocess.check_output(command)
- manifest = dom.parseString(result)
- usesSdk = manifest.getElementsByTagName('uses-sdk')[0]
-
return {
'revision': revision,
- 'minSdkVersion': usesSdk.attributes["android:minSdkVersion"].value,
+ 'minSdkVersion': get_min_sdk_version(repo, version),
}
elif repo.type == 'safari':
- metadata = readRawMetadata(repo, version)
+ metadata = repo.readMetadata(version)
return {
'certificateID': getSafariCertificateID(repo.keyFile),
'version': version,
@@ -220,7 +193,7 @@
'basename': metadata.get('general', 'basename'),
}
elif repo.type == 'gecko':
- metadata = readRawMetadata(repo, version)
+ metadata = repo.readMetadata(version)
result = {
'extensionID': metadata.get('general', 'id'),
'version': version,
@@ -257,6 +230,12 @@
template = get_template(get_config().get('extensions', '%sUpdateManifest' % repoType))
template.stream({'extensions': extensions[repoType]}).dump(manifestPath)
+def writePadFile(links):
+ for repo in Configuration.getRepositoryConfigurations():
+ if repo.pad and links.has_section(repo.repositoryName):
+ PadFile.forRepository(repo, links.get(repo.repositoryName, 'version'),
+ links.get(repo.repositoryName, 'downloadURL')).write()
+
def updateLinks():
"""
writes the current extension download links to a file
@@ -270,6 +249,7 @@
file.close()
writeUpdateManifest(result)
+ writePadFile(result)
if __name__ == "__main__":
updateLinks()
Index: sitescripts/extensions/pad/__init__.py
===================================================================
new file mode 100644
--- /dev/null
+++ b/sitescripts/extensions/pad/__init__.py
@@ -0,0 +1,187 @@
+# This file is part of the Adblock Plus web scripts,
+# Copyright (C) 2006-2014 Eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus. If not, see .
+
+import subprocess
+import os
+import re
+import time
+from datetime import datetime
+from urlparse import urljoin
+
+from sitescripts.utils import get_template
+from sitescripts.extensions.android import get_min_android_version
+from sitescripts.extensions.pad.language import iso2pad
+from sitescripts.extensions.pad.validation import validate_pad
+
+OS_WINDOWS = ['Windows 8', 'Win7 x32', 'Win7 x64', 'WinVista', 'WinVista x64', 'WinXP']
+OS_LINUX = ['Linux']
+OS_MAC = ['Mac OS X']
+OS_ANDROID = ['Android']
+
+class PadFile:
+ def __init__(self, repo, version, download_url):
+ self.repo = repo
+ self.version = version
+ self.download_url = download_url
+
+ self.first_release = True
+ for i, (filename, version) in enumerate(repo.getDownloads()):
+ if i > 0:
+ self.first_release = False
+
+ if version == self.version:
+ self.download_filename = filename
+
+ @property
+ def release_status(self):
+ if self.first_release:
+ return 'New Release'
+
+ if self.version.rstrip('.0').count('.') < 2:
+ return 'Major Update'
+
+ return 'Minor Update'
+
+ @property
+ def release_date(self):
+ command = ['hg', 'log', '-l', '1', self.download_filename, '--template', '{date}']
+ result = subprocess.check_output(command, cwd=self.repo.downloadsRepo)
+ timestamp, offset = re.match(r'(.*)([-+].*)', result).groups()
+ return datetime(*time.gmtime(float(timestamp) + float(offset))[:6])
+
+ @property
+ def download_size(self):
+ return len(subprocess.check_output(
+ ['hg', 'cat', '-r', 'tip', self.download_filename],
+ cwd=self.repo.downloadsRepo
+ ))
+
+ @property
+ def browser_min_version(self):
+ metadata = self.repo.readMetadata(self.version)
+ compat_option = getattr(self, 'compat_option', self.repo.type)
+ return metadata.get('compat', compat_option).split('/')[0].rstrip('.0')
+
+ @property
+ def languages(self):
+ files = self.repo.listContents(self.version)
+ languages = set()
+ skipped = set()
+
+ for filename in files:
+ match = re.match(self.translation_files_regex, filename)
+
+ if match:
+ groups = match.groupdict()
+
+ # support translation files that vary from the naming
+ # scheme like the default translation for Android
+ for k, v in groups.iteritems():
+ if k.startswith('is_') and v is not None:
+ code = k[3:]
+ break
+ else:
+ code = groups['code']
+
+ # support .incomplete files like on Firefox
+ if groups.get('skip') is not None:
+ skipped.add(code)
+ continue
+
+ languages.add(code)
+
+ return iso2pad(languages.difference(skipped))
+
+ def write(self):
+ template = get_template(self.repo.padTemplate)
+ basename = self.repo.basename
+ filename = basename + '.xml'
+
+ pad = template.render({
+ 'name': self.repo.name,
+ 'type': self.repo.type,
+ 'basename': basename,
+ 'browser_name': self.browser_name,
+ 'browser_min_version': self.browser_min_version,
+ 'version': self.version,
+ 'release_date': self.release_date,
+ 'release_status': self.release_status,
+ 'os_support': ','.join(self.os_support),
+ 'language': ','.join(self.languages),
+ 'download_size': self.download_size,
+ 'download_url': self.download_url,
+ 'pad_url': urljoin(self.repo.padURL, filename),
+ }).encode('utf-8')
+
+ path = os.path.join(self.repo.padDirectory, filename)
+ validate_pad(pad, path)
+
+ with open(path, 'wb') as file:
+ file.write(pad)
+
+ @staticmethod
+ def forRepository(repo, *args, **kwargs):
+ if repo.type == 'gecko':
+ return FirefoxPadFile(repo, *args, **kwargs)
+ if repo.type == 'chrome':
+ return ChromePadFile(repo, *args, **kwargs)
+ if repo.type == 'opera':
+ return OperaPadFile(repo, *args, **kwargs)
+ if repo.type == 'safari':
+ return SafariPadFile(repo, *args, **kwargs)
+ if repo.type == 'ie':
+ return InternetExplorerPadFile(repo, *args, **kwargs)
+ if repo.type == 'android':
+ return AndroidPadFile(repo, *args, **kwargs)
+
+ raise Exception('unknown repository type %r' % repo.type)
+
+class FirefoxPadFile(PadFile):
+ browser_name = 'Mozilla Firefox'
+ os_support = OS_WINDOWS + OS_MAC + OS_LINUX + OS_ANDROID
+ translation_files_regex = r'chrome\/locale\/(?P.+?)\/(?P\.incomplete$)?'
+ compat_option = 'firefox'
+
+class ChromePadFile(PadFile):
+ browser_name = 'Google Chrome'
+ os_support = OS_WINDOWS + OS_MAC + OS_LINUX
+ translation_files_regex = r'_locales\/(?P.+?)\/'
+
+class OperaPadFile(PadFile):
+ browser_name = 'Opera'
+ browser_min_version = '17'
+ os_support = OS_WINDOWS + OS_MAC + OS_LINUX
+ translation_files_regex = ChromePadFile.translation_files_regex
+
+class SafariPadFile(PadFile):
+ browser_name = 'Safari'
+ browser_min_version = '6'
+ os_support = OS_MAC
+ translation_files_regex = ChromePadFile.translation_files_regex
+
+class InternetExplorerPadFile(PadFile):
+ browser_name = 'Internet Explorer'
+ browser_min_version = '8'
+ os_support = OS_WINDOWS
+ translation_files_regex = r'locales\/(?P.+)\.ini$'
+
+class AndroidPadFile(PadFile):
+ browser_name = 'Android'
+ os_support = OS_ANDROID
+ translation_files_regex = r'res\/(?:raw|values)(?:-(?P.+?)|(?P))\/'
+
+ @property
+ def browser_min_version(self):
+ return get_min_android_version(self.repo, self.version)
Index: sitescripts/extensions/pad/language.py
===================================================================
new file mode 100644
--- /dev/null
+++ b/sitescripts/extensions/pad/language.py
@@ -0,0 +1,175 @@
+# This file is part of the Adblock Plus web scripts,
+# Copyright (C) 2006-2014 Eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus. If not, see .
+
+ISO2PAD = {
+ 'ab': 'Abkhazian',
+ 'aa': 'Afar',
+ 'af': 'Afrikaans',
+ 'sq': 'Albanian',
+ 'am': 'Amharic',
+ 'ar': 'Arabic',
+ 'hy': 'Armenian',
+ 'as': 'Assamese',
+ 'ay': 'Aymara',
+ 'az': 'Azerbaijani',
+ 'ba': 'Bashkir',
+ 'eu': 'Basque',
+ 'be': 'Byelorussian',
+ 'bn': 'Bengali',
+ 'bh': 'Bihari',
+ 'bi': 'Bislama',
+ 'br': 'Breton',
+ 'bg': 'Bulgarian',
+ 'my': 'Burmese',
+ 'ca': 'Catalan',
+ 'zh': 'Chinese',
+ 'zh-cn': 'ChineseSimplified',
+ 'zh-tw': 'ChineseTraditional',
+ 'co': 'Corsican',
+ 'hr': 'Croatian',
+ 'cs': 'Czech',
+ 'da': 'Danish',
+ 'nl': 'Dutch',
+ 'en': 'English',
+ 'eo': 'Esperanto',
+ 'et': 'Estonian',
+ 'fo': 'Faeroese',
+ 'fj': 'Fiji',
+ 'fi': 'Finnish',
+ 'fr': 'French',
+ 'gl': 'Galician',
+ 'ka': 'Georgian',
+ 'de': 'German',
+ 'el': 'Greek',
+ 'gn': 'Guarani',
+ 'gu': 'Gujarati',
+ 'ha': 'Hausa',
+ 'he': 'Hebrew',
+ 'hi': 'Hindi',
+ 'hu': 'Hungarian',
+ 'ia': 'Interlingua',
+ 'id': 'Indonesian',
+ 'ie': 'Interlingue',
+ 'ga': 'Irish',
+ 'ik': 'Inupiak',
+ 'is': 'Icelandic',
+ 'it': 'Italian',
+ 'ja': 'Japanese',
+ 'jv': 'Javanese',
+ 'kl': 'Greenlandic',
+ 'kn': 'Kannada',
+ 'ks': 'Kashmiri',
+ 'kk': 'Kazakh',
+ 'km': 'Cambodian',
+ 'rw': 'Kinyarwanda',
+ 'ky': 'Kirghiz',
+ 'ko': 'Korean',
+ 'ku': 'Kurdish',
+ 'la': 'Latin',
+ 'ln': 'Lingala',
+ 'lo': 'Laothian',
+ 'lt': 'Lithuanian',
+ 'lv': 'Latvian',
+ 'mk': 'Macedonian',
+ 'mg': 'Malagasy',
+ 'ms': 'Malay',
+ 'ml': 'Malayalam',
+ 'mt': 'Maltese',
+ 'mi': 'Maori',
+ 'mr': 'Marathi',
+ 'mn': 'Mongolian',
+ 'na': 'Nauru',
+ 'ne': 'Nepali',
+ 'no': 'Norwegian',
+ 'oc': 'Occitan',
+ 'om': 'Oromo',
+ 'or': 'Oriya',
+ 'pa': 'Punjabi',
+ 'fa': 'Persian',
+ 'pl': 'Polish',
+ 'ps': 'Pashto',
+ 'pt': 'Portuguese',
+ 'qu': 'Quechua',
+ 'rm': 'Rhaeto-Romance',
+ 'rn': 'Kirundi',
+ 'ro': 'Romanian',
+ 'ru': 'Russian',
+ 'sa': 'Sanskrit',
+ 'sd': 'Sindhi',
+ 'sm': 'Samoan',
+ 'sg': 'Sangro',
+ 'sr': 'Serbian',
+ 'gd': 'Gaelic',
+ 'sn': 'Shona',
+ 'si': 'Singhalese',
+ 'sk': 'Slovak',
+ 'sl': 'Slovenian',
+ 'so': 'Somali',
+ 'st': 'Sesotho',
+ 'es': 'Spanish',
+ 'sw': 'Swahili',
+ 'ss': 'Siswati',
+ 'sv': 'Swedish',
+ 'sh': 'Serbo-Croatian',
+ 'ta': 'Tamil',
+ 'te': 'Telugu',
+ 'tg': 'Tajik',
+ 'th': 'Thai',
+ 'ti': 'Tigrinya',
+ 'bo': 'Tibetan',
+ 'tk': 'Turkmen',
+ 'tl': 'Tagalog',
+ 'tn': 'Setswana',
+ 'to': 'Tonga',
+ 'tr': 'Turkish',
+ 'ts': 'Tsonga',
+ 'tt': 'Tatar',
+ 'tw': 'Twi',
+ 'uk': 'Ukrainian',
+ 'ur': 'Urdu',
+ 'uz': 'Uzbek',
+ 'vi': 'Vietnamese',
+ 'vo': 'Volapuk',
+ 'cy': 'Welsh',
+ 'wo': 'Wolof',
+ 'fy': 'Frisian',
+ 'xh': 'Xhosa',
+ 'yi': 'Yiddish',
+ 'yo': 'Yoruba',
+ 'zu': 'Zulu',
+}
+
+def iso2pad(iso_languages):
+ pad_languages = []
+ has_other = False
+
+ for iso in iso_languages:
+ iso = iso.replace('_', '-').lower()
+
+ pad = ISO2PAD.get(iso)
+ if not pad:
+ pad = ISO2PAD.get(iso.split('-')[0])
+ if not pad:
+ has_other = True
+ continue
+
+ if pad not in pad_languages:
+ pad_languages.append(pad)
+
+ pad_languages.sort()
+ if has_other:
+ pad_languages.append('Other')
+
+ return pad_languages
Index: sitescripts/extensions/pad/validation.py
===================================================================
new file mode 100644
--- /dev/null
+++ b/sitescripts/extensions/pad/validation.py
@@ -0,0 +1,204 @@
+# This file is part of the Adblock Plus web scripts,
+# Copyright (C) 2006-2014 Eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus. If not, see .
+
+import itertools
+import warnings
+import re
+import urllib2
+from xml.dom import minidom
+
+FIELDS = [
+ (['XML_DIZ_INFO', 'MASTER_PAD_VERSION_INFO', 'MASTER_PAD_VERSION'], r'^4\.0\Z'),
+ (['XML_DIZ_INFO', 'MASTER_PAD_VERSION_INFO', 'MASTER_PAD_EDITOR'], r'^[^<\x09]{0,100}\Z'),
+ (['XML_DIZ_INFO', 'MASTER_PAD_VERSION_INFO', 'MASTER_PAD_INFO'], r'^[^<\x09]{0,1000}\Z'),
+ (['XML_DIZ_INFO', 'MASTER_PAD_VERSION_INFO', 'CERTIFIED'], r'^[YyNn]\Z'),
+ (['XML_DIZ_INFO', 'MASTER_PAD_VERSION_INFO', 'CERTIFICATE_ID'], r'^(crt\-[0-9A-Z]{12}|)\Z'),
+ (['XML_DIZ_INFO', 'MASTER_PAD_VERSION_INFO', 'CERTIFICATE_LICENSE'], r'^(http\:\/\/repository\.appvisor\.com\/crt\-[0-9a-z]{12}|)\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'PublisherID'], r'^(pid-[0-9a-z]{12}|)\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Company_Name'], r'^[^<\x09]{2,40}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Address_1'], r'^[a-zA-Z0-9\xbc-\xff .\-,#\/\x27]{0,40}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Address_2'], r'^[a-zA-Z\xbc-\xff0-9 .\-,#\/\x27]{0,40}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'City_Town'], r'^[a-zA-Z\xbc-\xff0-9 .\-,#\/\x27]{2,40}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'State_Province'], r'^[a-zA-Z\xbc-\xff0-9 .\-,\/]{0,30}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Zip_Postal_Code'], r'^[^<\x09]{0,20}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Country'], r'^[a-z A-Z\xbc-\xff\x27-]{2,40}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Company_WebSite_URL'], r'^(http|https):\/\/.{2,120}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Contact_Info', 'Author_First_Name'], r'^[^<\x09]{2,30}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Contact_Info', 'Author_Last_Name'], r'^[^<\x09]{2,30}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Contact_Info', 'Author_Email'], r'^.{2,30}\@.{2,63}\..{2,20}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Contact_Info', 'Contact_First_Name'], r'^[^<\x09]{2,30}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Contact_Info', 'Contact_Last_Name'], r'^[^<\x09]{2,30}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Contact_Info', 'Contact_Email'], r'^.{2,30}\@.{2,63}\..{2,20}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Support_Info', 'Sales_Email'], r'^.{2,30}\@.{2,63}\..{2,20}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Support_Info', 'Support_Email'], r'^.{2,30}\@.{2,63}\..{2,20}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Support_Info', 'General_Email'], r'^.{2,30}\@.{2,63}\..{2,20}\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Support_Info', 'Sales_Phone'], r'^\+{0,2}(([0-9#*()-\/_] *){7,40})?\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Support_Info', 'Support_Phone'], r'^\+{0,2}(([0-9#*()-\/_] *){7,40})?\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Support_Info', 'General_Phone'], r'^\+{0,2}(([0-9#*()-\/_] *){7,40})?\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'Support_Info', 'Fax_Phone'], r'^\+{0,2}(([0-9#*()-\/_] *){7,40})?\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'GooglePlusPage'], r'^((http|https):\/\/.{4,120}|)\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'LinkedinPage'], r'^((http|https):\/\/.{4,120}|)\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'TwitterCompanyPage'], r'^((http|https):\/\/.{4,120}|)\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'FacebookCompanyPage'], r'^((http|https):\/\/.{4,120}|)\Z'),
+ (['XML_DIZ_INFO', 'Company_Info', 'CompanyStorePage'], r'^((http|https):\/\/.{4,120}|)\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'AppID'], r'^(app-[0-9a-z]{12}|)\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_Name'], r'^[^<\x09]{1,40}\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_Version'], r'^[a-zA-Z0-9_.\-]{1,15}\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_Release_Month'], r'^(0[1-9]|1[0-2])\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_Release_Day'], r'^(0[1-9]|[12][0-9]|3[01])\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_Release_Year'], r'^(19|20|21)[0-9]{2}\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_Cost_Dollars'], r'^([0-9]+(\.[0-9]{2})?)?\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_Cost_Other_Code'], r'^(AED|AFN|ALL|AMD|ANG|AOA|ARS|AUD|AWG|AZM|BAM|BBD|BDT|BGN|BHD|BIF|BMD|BND|BOB|BRL|BSD|BTN|BWP|BYR|BZD|CAD|CDF|CHF|CLP|CNY|COP|COU|CRC|CSD|CZK|CUP|CVE|CYP|DJF|DKK|DOP|DZD|EEK|EGP|ERN|ETB|EUR|FJD|FKP|GBP|GEL|GHC|GIP|GMD|GNF|GTQ|GYD|HKD|HNL|HRK|HTG|HUF|IDR|ILS|INR|IQD|IRR|ISK|JMD|JOD|JPY|KES|KGS|KHR|KMF|KPW|KRW|KWD|KYD|KZT|LAK|LBP|LKR|LRD|LSL|LTL|LVL|LYD|MAD|MDL|MGA|MKD|MMK|MNT|MOP|MRO|MTL|MUR|MVR|MWK|MXN|MYR|MZN|NAD|NGN|NIO|NOK|NPR|NZD|OMR|PAB|PEN|PGK|PHP|PKR|PLN|PYG|QAR|RON|RUB|RWF|SAR|SBD|SCR|SDD|SEK|SGD|SHP|SIT|SKK|SLL|SOS|SRD|STD|SYP|SZL|THB|TJS|TMM|TND|TOP|TRY|TTD|TWD|TZS|UAH|UGX|USD|UYU|UZS|VEB|VND|VUV|WST|XAF|XCD|XOF|XPF|YER|ZAR|ZMK|ZWD)?\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_Cost_Other'], r'^([0-9]+(\.[0-9]{2})?)?\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_Type'], r'^(Shareware|Freeware|Adware|Demo|Commercial|Data Only)\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_Release_Status'], r'^(Major Update|Minor Update|New Release|Beta|Alpha|Media Only)\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_Install_Support'], r'^(Install and Uninstall|Install Only|No Install Support|Uninstall Only)\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_OS_Support'], r'^((Android|BlackBerry|Handheld\/Mobile Other|iPhone|iPad|iPod|iTouch|Java|Linux|Linux Console|Linux Gnome|Linux GPL|Linux Open Source|Mac OS X|Mac Other|MS-DOS|Netware|OpenVMS|Palm|Pocket PC|Symbian|Unix|Win2000|Win7 x32|Win7 x64|Win98|WinMobile|WinOther|WinServer|WinVista|WinVista x64|WinXP|Windows 8|Windows Phone 7|Windows Phone 8|Windows RT|Other|Not Applicable)[, ]*)+\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_Language'], r'^(Abkhazian|Afar|Afrikaans|Albanian|Amharic|Arabic|Armenian|Assamese|Aymara|Azerbaijani|Bashkir|Basque|Bengali|Bhutani|Bihari|Bislama|Breton|Bulgarian|Burmese|Byelorussian|Cambodian|Catalan|Chinese|ChineseSimplified|ChineseTraditional|Corsican|Croatian|Czech|Danish|Dutch|English|Esperanto|Estonian|Faeroese|Fiji|Finnish|French|Frisian|Gaelic|Galician|Georgian|German|Greek|Greenlandic|Guarani|Gujarati|Hausa|Hebrew|Hindi|Hungarian|Icelandic|Indonesian|Interlingua|Interlingue|Inupiak|Irish|Italian|Japanese|Javanese|Kannada|Kashmiri|Kazakh|Kinyarwanda|Kirghiz|Kirundi|Korean|Kurdish|Laothian|Latin|Latvian|Lingala|Lithuanian|Macedonian|Malagasy|Malay|Malayalam|Maltese|Maori|Marathi|Moldavian|Mongolian|Nauru|Nepali|Norwegian|Occitan|Oriya|Oromo|Other|Pashto|Persian|Polish|Portuguese|Punjabi|Quechua|Rhaeto-Romance|Romanian|Russian|Samoan|Sangro|Sanskrit|Serbian|Serbo-Croatian|Sesotho|Setswana|Shona|Sindhi|Singhalese|Siswati|Slovak|Slovenian|Somali|Spanish|Sudanese|Swahili|Swedish|Tagalog|Tajik|Tamil|Tatar|Telugu|Thai|Tibetan|Tigrinya|Tonga|Tsonga|Turkish|Turkmen|Twi|Ukrainian|Urdu|Uzbek|Vietnamese|Volapuk|Welsh|Wolof|Xhosa|Yiddish|Yoruba|Zulu|,)+\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'File_Info', 'File_Size_Bytes'], r'^[0-9]{3,16}\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'File_Info', 'File_Size_K'], r'^[0-9.]{1,12}\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'File_Info', 'File_Size_MB'], r'^[0-9.]{1,8}\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Expire_Info', 'Has_Expire_Info'], r'^[YyNn]\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Expire_Info', 'Expire_Count'], r'^[0-9]{0,15}\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Expire_Info', 'Expire_Based_On'], r'^(Days|Uses|Either\/Or)?\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Expire_Info', 'Expire_Other_Info'], r'^[^<\x09]{0,100}\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Expire_Info', 'Expire_Month'], r'^(0[1-9]|1[0-2])?\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Expire_Info', 'Expire_Day'], r'^(0[1-9]|[12][0-9]|3[01])?\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Expire_Info', 'Expire_Year'], r'^((19|20|21)[0-9]{2})?\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_Change_Info'], r'^[^<\x09]{0,300}\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_Category_Class'], r'^(Audio & Multimedia::Audio Encoders\/Decoders|Audio & Multimedia::Audio File Players|Audio & Multimedia::Audio File Recorders|Audio & Multimedia::CD Burners|Audio & Multimedia::CD Players|Audio & Multimedia::Multimedia Creation Tools|Audio & Multimedia::Music Composers|Audio & Multimedia::Other|Audio & Multimedia::Presentation Tools|Audio & Multimedia::Rippers & Converters|Audio & Multimedia::Speech|Audio & Multimedia::Video Tools|Business::Accounting & Finance|Business::Calculators & Converters|Business::Databases & Tools|Business::Helpdesk & Remote PC|Business::Inventory & Barcoding|Business::Investment Tools|Business::Math & Scientific Tools|Business::Office Suites & Tools|Business::Other|Business::PIMS & Calendars|Business::Project Management|Business::Vertical Market Apps|Communications::Chat & Instant Messaging|Communications::Dial Up & Connection Tools|Communications::E-Mail Clients|Communications::E-Mail List Management|Communications::Fax Tools|Communications::Newsgroup Clients|Communications::Other Comms Tools|Communications::Other E-Mail Tools|Communications::Pager Tools|Communications::Telephony|Communications::Web\/Video Cams|Desktop::Clocks & Alarms|Desktop::Cursors & Fonts|Desktop::Icons|Desktop::Other|Desktop::Screen Savers: Art|Desktop::Screen Savers: Cartoons|Desktop::Screen Savers: Nature|Desktop::Screen Savers: Other|Desktop::Screen Savers: People|Desktop::Screen Savers: Science|Desktop::Screen Savers: Seasonal|Desktop::Screen Savers: Vehicles|Desktop::Themes & Wallpaper|Development::Active X|Development::Basic, VB, VB DotNet|Development::C \/ C\+\+ \/ C\#|Development::Compilers & Interpreters|Development::Components & Libraries|Development::Debugging|Development::Delphi|Development::Help Tools|Development::Install & Setup|Development::Management & Distribution|Development::Other|Development::Source Editors|Education::Computer|Education::Dictionaries|Education::Geography|Education::Kids|Education::Languages|Education::Mathematics|Education::Other|Education::Reference Tools|Education::Science|Education::Teaching & Training Tools|Games & Entertainment::Action|Games & Entertainment::Adventure & Roleplay|Games & Entertainment::Arcade|Games & Entertainment::Board|Games & Entertainment::Card|Games & Entertainment::Casino & Gambling|Games & Entertainment::Kids|Games & Entertainment::Online Gaming|Games & Entertainment::Other|Games & Entertainment::Puzzle & Word Games|Games & Entertainment::Simulation|Games & Entertainment::Sports|Games & Entertainment::Strategy & War Games|Games & Entertainment::Tools & Editors|Graphic Apps::Animation Tools|Graphic Apps::CAD|Graphic Apps::Converters & Optimizers|Graphic Apps::Editors|Graphic Apps::Font Tools|Graphic Apps::Gallery & Cataloging Tools|Graphic Apps::Icon Tools|Graphic Apps::Other|Graphic Apps::Screen Capture|Graphic Apps::Viewers|Home & Hobby::Astrology\/Biorhythms\/Mystic|Home & Hobby::Astronomy|Home & Hobby::Cataloging|Home & Hobby::Food & Drink|Home & Hobby::Genealogy|Home & Hobby::Health & Nutrition|Home & Hobby::Other|Home & Hobby::Personal Finance|Home & Hobby::Personal Interest|Home & Hobby::Recreation|Home & Hobby::Religion|Network & Internet::Ad Blockers|Network & Internet::Browser Tools|Network & Internet::Browsers|Network & Internet::Download Managers|Network & Internet::File Sharing\/Peer to Peer|Network & Internet::FTP Clients|Network & Internet::Network Monitoring|Network & Internet::Other|Network & Internet::Remote Computing|Network & Internet::Search\/Lookup Tools|Network & Internet::Terminal & Telnet Clients|Network & Internet::Timers & Time Synch|Network & Internet::Trace & Ping Tools|Security & Privacy::Access Control|Security & Privacy::Anti-Spam & Anti-Spy Tools|Security & Privacy::Anti-Virus Tools|Security & Privacy::Covert Surveillance|Security & Privacy::Encryption Tools|Security & Privacy::Other|Security & Privacy::Password Managers|Servers::Firewall & Proxy Servers|Servers::FTP Servers|Servers::Mail Servers|Servers::News Servers|Servers::Other Server Applications|Servers::Telnet Servers|Servers::Web Servers|System Utilities::Automation Tools|System Utilities::Backup & Restore|System Utilities::Benchmarking|System Utilities::Clipboard Tools|System Utilities::File & Disk Management|System Utilities::File Compression|System Utilities::Launchers & Task Managers|System Utilities::Other|System Utilities::Printer|System Utilities::Registry Tools|System Utilities::Shell Tools|System Utilities::System Maintenance|System Utilities::Text\/Document Editors|Web Development::ASP & PHP|Web Development::E-Commerce|Web Development::Flash Tools|Web Development::HTML Tools|Web Development::Java & JavaScript|Web Development::Log Analysers|Web Development::Other|Web Development::Site Administration|Web Development::Wizards & Components|Web Development::XML\/CSS Tools)\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_Specific_Category'], r'^(Audio|Business|Development Tools|Education|Games|Graphics|Home\/Hobby|Internet|Miscellaneous|Screen Savers|Utilities)?\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'Program_System_Requirements'], r'^[^<\x09]{0,100}\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'FacebookProductPage'], r'^((http|https):\/\/.{4,120}|)\Z'),
+ (['XML_DIZ_INFO', 'Program_Info', 'GooglePlusProductPage'], r'^((http|https):\/\/.{4,120}|)\Z'),
+ (['XML_DIZ_INFO', 'Program_Descriptions', 'English', 'Keywords'], r'^[^<\x09]{0,250}\Z'),
+ (['XML_DIZ_INFO', 'Program_Descriptions', 'English', 'Char_Desc_45'], r'^[^<\x09\x0a\x0d]{0,45}\Z'),
+ (['XML_DIZ_INFO', 'Program_Descriptions', 'English', 'Char_Desc_80'], r'^[^<\x09\x0a\x0d]{0,80}\Z'),
+ (['XML_DIZ_INFO', 'Program_Descriptions', 'English', 'Char_Desc_250'], r'^[^<\x09\x0a\x0d]{0,250}\Z'),
+ (['XML_DIZ_INFO', 'Program_Descriptions', 'English', 'Char_Desc_450'], r'^[^<\x09\x0a\x0d]{0,450}\Z'),
+ (['XML_DIZ_INFO', 'Program_Descriptions', 'English', 'Char_Desc_2000'], r'^[^<]{0,2000}\Z'),
+ (['XML_DIZ_INFO', 'Web_Info', 'Application_URLs', 'Application_Info_URL'], r'^(http|https):\/\/.{2,120}\Z'),
+ (['XML_DIZ_INFO', 'Web_Info', 'Application_URLs', 'Application_Order_URL'], r'^((http|https):\/\/.{2,120})?\Z'),
+ (['XML_DIZ_INFO', 'Web_Info', 'Application_URLs', 'Application_Screenshot_URL'], r'^(http|https):\/\/.{2,120}\.(gif|jpg|png)\Z'),
+ (['XML_DIZ_INFO', 'Web_Info', 'Application_URLs', 'Application_Icon_URL'], r'^(http|https):\/\/.{2,120}\.(gif|jpg|png)\Z'),
+ (['XML_DIZ_INFO', 'Web_Info', 'Application_URLs', 'Application_XML_File_URL'], r'^(http|https):\/\/.{2,120}\.(xml|cgi|php|asp)\Z'),
+ (['XML_DIZ_INFO', 'Web_Info', 'Application_URLs', 'Video_Link_1_URL'], r'^((http|https):\/\/.{4,120}|)\Z'),
+ (['XML_DIZ_INFO', 'Web_Info', 'Application_URLs', 'Video_Link_2_URL'], r'^((http|https):\/\/.{4,120}|)\Z'),
+ (['XML_DIZ_INFO', 'Web_Info', 'Download_URLs', 'Primary_Download_URL'], r'^(http|https|ftp):\/\/.{2,120}\Z'),
+ (['XML_DIZ_INFO', 'Web_Info', 'Download_URLs', 'Secondary_Download_URL'], r'^((http|https|ftp):\/\/.{2,120})?\Z'),
+ (['XML_DIZ_INFO', 'Web_Info', 'Download_URLs', 'Additional_Download_URL_1'], r'^((http|https|ftp):\/\/.{2,120})?\Z'),
+ (['XML_DIZ_INFO', 'Web_Info', 'Download_URLs', 'Additional_Download_URL_2'], r'^((http|https|ftp):\/\/.{2,120})?\Z'),
+ (['XML_DIZ_INFO', 'Permissions', 'Distribution_Permissions'], r'^[^<]{0,2000}\Z'),
+ (['XML_DIZ_INFO', 'Permissions', 'EULA'], r'^[^<]{0,20000}\Z'),
+ (['XML_DIZ_INFO', 'Press_Release', 'Keywords'], r'^[^<\x09\x0a\x0d]{0,250}\Z'),
+ (['XML_DIZ_INFO', 'Press_Release', 'Headline'], r'^([^<\x09\x0a\x0d]{20,100}|)\Z'),
+ (['XML_DIZ_INFO', 'Press_Release', 'Summary'], r'^([^<\x09\x0a\x0d]{20,250}|)\Z'),
+ (['XML_DIZ_INFO', 'Press_Release', 'Press_Release'], r'^[^<\x09\x0a\x0d]{0,3000}\Z'),
+ (['XML_DIZ_INFO', 'Press_Release', 'Press_Release_Plain'], r'^[^<\x09\x0a\x0d]{0,3000}\Z'),
+ (['XML_DIZ_INFO', 'Press_Release', 'Related_URL'], r'^((http|https):\/\/.{0,100}|)\Z'),
+ (['XML_DIZ_INFO', 'NewsFeed', 'NewsFeed_Feed_URL'], r'^((http|https):\/\/.{0,100}|)\Z'),
+ (['XML_DIZ_INFO', 'NewsFeed', 'NewsFeed_Type'], r'^(RSS 0\.90|RSS 0\.91|RSS 0\.92|RSS 0\.93|RSS 0\.94|RSS 1\.0|RSS 2\.0|Atom 0\.3|Atom 1\.0|)\Z'),
+ (['XML_DIZ_INFO', 'NewsFeed', 'NewsFeed_Title'], r'^[^<\x09\x0a\x0d]{0,60}\Z'),
+ (['XML_DIZ_INFO', 'NewsFeed', 'NewsFeed_Keywords'], r'^[^<\x09]{0,250}\Z'),
+ (['XML_DIZ_INFO', 'NewsFeed', 'NewsFeed_Description_70'], r'^[^<\x09\x0a\x0d]{0,70}\Z'),
+ (['XML_DIZ_INFO', 'NewsFeed', 'NewsFeed_Description_250'], r'^[^<\x09\x0a\x0d]{0,250}\Z'),
+ (['XML_DIZ_INFO', 'Affiliates', 'Affiliates_ShareIt_Order_Page'], None),
+ (['XML_DIZ_INFO', 'Affiliates', 'Affiliates_ShareIt_Vendor_ID'], None),
+ (['XML_DIZ_INFO', 'Affiliates', 'Affiliates_ShareIt_Product_ID'], None),
+ (['XML_DIZ_INFO', 'Affiliates', 'Affiliates_ShareIt_Maximum_Commission_Rate'], None),
+ (['XML_DIZ_INFO', 'Affiliates', 'Affiliates_PayPro_Order_Page'], None),
+ (['XML_DIZ_INFO', 'Affiliates', 'Affiliates_PayPro_Vendor_ID'], None),
+ (['XML_DIZ_INFO', 'Affiliates', 'Affiliates_PayPro_Product_ID'], None),
+ (['XML_DIZ_INFO', 'Affiliates', 'Affiliates_PayPro_Maximum_Commission_Rate'], None),
+ (['XML_DIZ_INFO', 'Affiliates', 'Affiliates_Avangate_Order_Page'], None),
+ (['XML_DIZ_INFO', 'Affiliates', 'Affiliates_Avangate_Vendor_ID'], None),
+ (['XML_DIZ_INFO', 'Affiliates', 'Affiliates_Avangate_Product_ID'], None),
+ (['XML_DIZ_INFO', 'Affiliates', 'Affiliates_Avangate_Maximum_Commission_Rate'], None),
+ (['XML_DIZ_INFO', 'ASP', 'ASP_Member'], r'^[YyNn]\Z'),
+ (['XML_DIZ_INFO', 'ASP', 'ASP_Member_Number'], None),
+]
+
+def validate_fields(fields, nodes, filename):
+ expected_nodes = set()
+
+ for node_name, fields in itertools.groupby(fields, lambda (path, regex): path[0]):
+ expected_nodes.add(node_name)
+
+ regex = None
+ nested_fields = []
+ for path, regex_ in fields:
+ if path == [node_name]:
+ regex = regex_
+ else:
+ nested_fields.append((path[1:], regex_))
+
+ found = False
+ for node in nodes:
+ if node.nodeName == node_name:
+ if found:
+ warnings.warn('invalid PAD file (duplicate node)\n'
+ 'filename: %s\n'
+ 'node: %s' % (filename, node_name))
+
+ if regex:
+ value = ''.join(child.toxml() for child in node.childNodes)
+ if not re.match(regex, value):
+ warnings.warn('invalid PAD file (invalid value)\n'
+ 'filename: %s\n'
+ 'node: %s\n'
+ 'value: %s\n'
+ 'regex: %s' % (filename, node_name, value, regex))
+
+ if nested_fields:
+ validate_fields(nested_fields, node.childNodes, filename)
+
+ found = True
+
+ if not found:
+ if regex and not re.match(regex, ''):
+ warnings.warn('invalid PAD file (missing node)\n'
+ 'filename: %s\n'
+ 'node: %s' % (filename, node_name))
+
+ validate_fields(nested_fields, [], filename)
+
+ for node in nodes:
+ if node.nodeType == node.COMMENT_NODE:
+ continue
+ if node.nodeType == node.TEXT_NODE and node.nodeValue.strip() == '':
+ continue
+
+ if node.nodeName not in expected_nodes:
+ warnings.warn('invalid PAD file (unexpected node)\n'
+ 'filename: %s\n'
+ 'node: %s' % (filename, node.nodeName))
+
+def validate_pad(pad, filename):
+ validate_fields(FIELDS, minidom.parseString(pad).childNodes, filename)
+
+def print_fields():
+ doc = minidom.parse(urllib2.urlopen('http://repository.appvisor.com/padspec/files/padspec40.xml'))
+
+ print '['
+ for field in doc.getElementsByTagName('Field'):
+ path, regex = [
+ ''.join(node.nodeValue for node in field.getElementsByTagName(name)[0].childNodes)
+ for name in ('Path', 'RegEx')
+ ]
+ print ' (%r, %s),' % (str(path).split('/'), "r'%s'" % regex if regex else 'None')
+ print ']'
+
+if __name__ == '__main__':
+ print_fields()
Index: sitescripts/extensions/template/pad.xml
===================================================================
new file mode 100644
--- /dev/null
+++ b/sitescripts/extensions/template/pad.xml
@@ -0,0 +1,182 @@
+
+
+
+ 4.0
+
+ Portable Application Description, or PAD for short, is a data set that is used by shareware authors to disseminate information to anyone interested in their software products. To find out more go to http://www.asp-shareware.org/pad
+ N
+
+
+ N
+
+
+ Eyeo GmbH
+ Im Klapperhof 7-23
+ Cologne
+ NRW
+ 50670
+ Germany
+ https://eyeo.com
+
+ Wladimir
+ Palant
+ trev@adblockplus.org
+ Christian
+ Dommers
+ christian@adblockplus.org
+
+
+ christian@adblockplus.org
+ info@adblockplus.org
+ info@adblockplus.org
+ +4922165028598
+ +4922165028598
+ +4922165028598
+ +4922165028599
+
+
+
+ {{ name }}
+ {{ version }}
+ {{ '%.2i'|format(release_date.month) }}
+ {{ '%.2i'|format(release_date.day) }}
+ {{ release_date.year }}
+ {{ release_status }}
+ 0
+ Freeware
+ Install and Uninstall
+ {{ os_support }}
+ {{ language }}
+ Utilities
+ {{ browser_name }} {{ browser_min_version }} or higher
+
+ {{ download_size }}
+ {{ (download_size / (2 ** 10))|round(2, 'ceil') }}
+ {{ (download_size / (2 ** 20))|round(2, 'ceil') }}
+
+
+ N
+
+{%- if basename == 'url-fixer' %}
+ Network & Internet::Browser Tools
+
+
+
+ urlfixer, typosquatting, url, security, privacy, {{ browser_name }}
+ Typo correction for URLs
+ URL Fixer corrects typos in URLs that you enter in the address bar.
+ URL Fixer corrects typos in URLs that you enter in the address bar. For example, if you type google.con, it will correct it to google.com (asking first, if you enable confirmation).
+ URL Fixer corrects typos in URLs that you enter in the address bar. For example, if you type google.con, it will correct it to google.com (if you enable confirmation). This version will correct common misspellings of .com, .net, .org, .edu, .gov, .mil, and other mainstream TLDs, as well as the protocol. By right-clicking on the address bar, you can set it to auto-correct your errors, or you can have it ask you before making any corrections.
+ URL Fixer corrects typos in URLs that you enter in the address bar. For example, if you type google.con, it will correct it to google.com (asking first, if you enable confirmation).
+
+This version will correct common misspellings of .com, .net, .org, .edu, .gov, .mil, and all other mainstream TLDs, as well as the protocol (http:, https:). By right-clicking on the address bar, you can set it to auto-correct your errors, or you can have it ask you before making any corrections.
+
+You can specify your own set of custom corrections. Add or remove your own corrections in URL Fixer's options dialog, accessible via Tools » Add-ons » URL Fixer » Preferences.
+
+You can also correct errors as they happen; if Firefox can't find the website you typed in, URL Fixer lets you mark that URL as a typo, and you'll never see that error page again.
+
+
+
+
+ http://urlfixer.org/
+ http://urlfixer.org/images/screenshot2-big.png
+ http://urlfixer.org/images/logo.png
+{%- else %}
+ Network & Internet::Ad Blockers
+ https://www.facebook.com/adblockplus
+ https://plus.google.com/+AdblockPlus
+
+
+
+ adblock, adblocker, security, privacy, {{ browser_name }}
+ {%- if basename == 'adblockplusandroid' %}
+ Adblock Plus blocks all annoying ads.
+ Adblock Plus blocks all annoying ads on your Android device. NO ROOT REQUIRED!
+ ABP blocks all advertising when browsing, and when using your favorite apps. It is free and makes your Android device much more enjoyable. Adblock Plus blocks: mobile ads, video ads, banners, push notifications, display ads, HTML5 ads, and more.
+ Adblock Plus blocks all annoying ads on your Android device. NO ROOT REQUIRED! It blocks all online advertising when browsing, and when using your favorite apps like Angry Birds. It is 100% free and makes your Android device much more enjoyable. Adblock Plus blocks: mobile ads, video advertising, banners, push notifications, display advertising, HTML5 advertising, and much more.
+ Adblock Plus blocks all annoying ads on your Android device. NO ROOT REQUIRED!
+It blocks all online advertising when browsing, and when using your favorite apps like Angry Birds. It is 100% free and makes your Android device much more enjoyable.
+Adblock Plus blocks: mobile ads, video advertising, banners, push notifications, display advertising, HTML5 advertising, and much more.
+PLEASE NOTE: Adblock Plus might have limited functionality, depending on the Android version on your device and whether your device has been rooted:
+Rooted: Blocks ads over Wi-Fi and 3G
+Non-rooted with Android 3.1 or later: Blocks ads over Wi-Fi
+Non-rooted with Android 3.0 or earlier: Some manual configuration is required. Please visit our website to learn how to setup your own proxy: https://adblockplus.org/en/android-config#proxy
+Also, Android does not allow ads to be blocked on SSL encrypted websites.
+Annoyed by the icon? Please read this: https://adblockplus.org/en/android-faq#icon
+Adblock Plus is an open-source, community driven project. Many volunteers are contributing to the development and maintenance of Adblock Plus. Currently, there are over forty continuously updating filter lists in over a dozen different languages, ensuring that all obtrusive ads remain blocked.
+Please note that Adblock Plus will probably become the highest data-using application on your Android device. However, this is only because Adblock Plus needs to filter the data to ensure that the ads can be blocked. Adblock Plus is actually only using a fraction of the data shown on your device.
+Adblock Plus is also available for Mozilla Firefox, Google Chrome, Opera, Safari and Internet Explorer.
+ {%- else %}
+ Blocks all annoying ads on the web.
+ Blocks all annoying ads on the web: YouTube ads, Facebook ads, banners and more.
+ {%- if basename == 'adblockplussafari' %}
+ Browse the Internet without annoying ads with the best adblocker for Safari! Adblock Plus for Safari blocks all YouTube ads, pop-ups, banners, Facebook and Twitter ads.
+ Adblock Plus for Safari automatically removes: YouTube ads, including the 30-second preroll ads, Facebook ads, Banners, Pop-ups, All other annoying ads.The most popular adblocker is highly customizable and includes an option to block tracking and malware. Unlike other adblocking solutions for Mac OS X, Adblock Plus for Safari blocks all Youtube ads. It is also the only adblocker for Safari that shows how many ads you block in each site.
+ Browse the Internet without annoying ads with the best adblocker for Safari!
+
+Our Safari adblocker blocks all YouTube ads, pop-ups, banners and even the ads in your Facebook and Twitter feeds. Adblock Plus is the world's most popular browser extension, used by tens of millions of users all around the world.
+
+Adblock Plus blocks all intrusive ads. Under default settings it supports websites and encourages better advertising by allowing non-annoying ads (configurable).
+
+Adblock Plus for Safari automatically removes:
+* YouTube ads, including the 30-second preroll ads
+* Facebook ads
+* Banners
+* Pop-ups
+* All other annoying ads
+
+The most popular adblocker is highly customizable and includes an option to block tracking and malware. Unlike other adblocking solutions for Mac OS X, Adblock Plus for Safari is the only tool that blocks all YouTube ads. It is also the only adblocker for Safari that shows how many ads you block in each site.
+
+Adblock Plus is also available for Mozilla Firefox, Google Chrome, Opera, Internet Explorer and Android.
+
+By supporting our Acceptable Ads initiative, you allow ads through which meet our criteria. By doing so, you support websites that rely on advertising but choose to do it in a non-intrusive way. This feature can be disabled at any time. Go to http://adblockplus.org/en/acceptable-ads to find out more.
+
+Download Adblock Plus for Safari for free and browse the Internet without annoying ads!
+ {%- else %}
+ Enjoy surfing the web without obtrusive ads cluttering your screen! {{ name }} blocks: Banners, YouTube video ads, Facebook advertisements, Pop-ups, All other obtrusive ads.
+ {{ name }} blocks: Banners, YouTube video ads, Facebook advertisements, Pop-ups, All other obtrusive ads. Adblock Plus is the world’s most popular browser extension, and is used by millions of users worldwide. It is a community-driven open source project, and hundreds of volunteers are contributing to the success of Adblock Plus to make sure that all annoying ads are automatically blocked.
+ Adblock Plus blocks all annoying ads, and supports websites by not blocking unobtrusive ads by default (configurable).
+
+Enjoy surfing the web without obtrusive ads cluttering your screen!
+
+{{ name }} blocks:
+
+* Banners
+* YouTube video ads
+* Facebook advertisements
+* Pop-ups
+* All other obtrusive ads
+
+Adblock Plus is the world’s most popular browser extension, and is used by millions of users worldwide. It is a community-driven open source project, and hundreds of volunteers are contributing to the success of Adblock Plus to make sure that all annoying ads are automatically blocked.
+
+Please note: When installing {{ name }}, your browser shows a warning that Adblock Plus has access to your browsing history and data. This is a standard message, we NEVER collect any information whatsoever!
+
+Recently, the Adblock Plus community introduced the Acceptable Ads initiative. By allowing some small and static ads, you support websites that rely on advertising but choose to do it in a non-intrusive way. This feature can be disabled at any time. Go to https://adblockplus.org/en/acceptable-ads to find out more.
+
+***********
+Release announcements and changelogs can be found here: https://adblockplus.org/releases/
+
+Problems? Try restarting {{ browser_name }} and/or clicking "Update now" in the Adblock Plus Options.
+
+Found a bug or need help? Visit our forum: https://adblockplus.org/redirect?link={{ type }}_support
+
+*************
+Feeling adventurous? Try out a development build to always get the latest features of Adblock Plus: https://adblockplus.org/en/development-builds (updated separately, independent settings).
+ {%- endif %}
+ {%- endif %}
+
+
+
+
+ https://adblockplus.org
+ https://eyeo.com/pad/abp_screenshot.jpg
+ https://eyeo.com/pad/abp_icon.png
+ http://www.youtube.com/watch?v=pVYtzF5SemU
+{%- endif %}
+ {{ pad_url }}
+
+
+ {{ download_url }}
+
+
+
Index: sitescripts/extensions/utils.py
===================================================================
--- a/sitescripts/extensions/utils.py
+++ b/sitescripts/extensions/utils.py
@@ -16,7 +16,10 @@
# along with Adblock Plus. If not, see .
import re
-from ConfigParser import NoOptionError
+import os
+import subprocess
+from ConfigParser import SafeConfigParser, NoOptionError
+from StringIO import StringIO
from sitescripts.utils import get_config
def compareVersionParts(part1, part2):
@@ -89,26 +92,19 @@
changed (latestRevision), others come from the global config and are
read-only (repository, repositoryName, nightliesDirectory).
"""
+ def _defineProperty(name, local=False, type='', default=None):
+ def getter(self):
+ method = getattr(self.config, 'get' + type)
+ key = '%s_%s' % (self.repositoryName, name) if local else name
- def _defineGlobalProperty(key):
- """
- Creates a property corresponding with a key in the config file
- """
- return property(lambda self: self.config.get('extensions', key))
+ try:
+ return method('extensions', key)
+ except NoOptionError:
+ if default is None:
+ raise
+ return default
- def _defineLocalProperty(key, default = None):
- """
- Creates a property corresponding with a repository-specific key in the config file
- """
- def getLocalProperty(self):
- try:
- return self.config.get('extensions', self.repositoryName + '_' + key)
- except NoOptionError, e:
- if default != None:
- return default
- else:
- raise e
- return property(getLocalProperty)
+ return property(getter)
def _defineNightlyProperty(key):
"""
@@ -122,26 +118,30 @@
repositoryName = None
repository = None
- buildRepository = _defineGlobalProperty('buildRepository')
- nightliesDirectory = _defineGlobalProperty('nightliesDirectory')
- nightliesURL = _defineGlobalProperty('nightliesURL')
- downloadsRepo = _defineGlobalProperty('downloadsRepo')
- downloadsURL = _defineGlobalProperty('downloadsURL')
- docsDirectory = _defineGlobalProperty('docsDirectory')
- signtool = _defineGlobalProperty('signtool')
- certname = _defineGlobalProperty('signtool_certname')
- dbdir = _defineGlobalProperty('signtool_dbdir')
- dbpass = _defineGlobalProperty('signtool_dbpass')
+ buildRepository = _defineProperty('buildRepository')
+ nightliesDirectory = _defineProperty('nightliesDirectory')
+ nightliesURL = _defineProperty('nightliesURL')
+ downloadsRepo = _defineProperty('downloadsRepo')
+ downloadsURL = _defineProperty('downloadsURL')
+ docsDirectory = _defineProperty('docsDirectory')
+ signtool = _defineProperty('signtool')
+ certname = _defineProperty('signtool_certname')
+ dbdir = _defineProperty('signtool_dbdir')
+ dbpass = _defineProperty('signtool_dbpass')
+ padDirectory = _defineProperty('padDirectory')
+ padURL = _defineProperty('padURL')
+ padTemplate = _defineProperty('padTemplate')
- keyFile = _defineLocalProperty('key', '')
- name = _defineLocalProperty('name')
- galleryID = _defineLocalProperty('galleryID', '')
- devbuildGalleryID = _defineLocalProperty('devbuildGalleryID', '')
- downloadPage = _defineLocalProperty('downloadPage', '')
- experimental = _defineLocalProperty('experimental', '')
- clientID = _defineLocalProperty('clientID', '')
- clientSecret = _defineLocalProperty('clientSecret', '')
- refreshToken = _defineLocalProperty('refreshToken', '')
+ keyFile = _defineProperty('key', local=True, default='')
+ name = _defineProperty('name', local=True)
+ galleryID = _defineProperty('galleryID', local=True, default='')
+ devbuildGalleryID = _defineProperty('devbuildGalleryID', local=True, default='')
+ downloadPage = _defineProperty('downloadPage', local=True, default='')
+ experimental = _defineProperty('experimental', local=True, default='')
+ clientID = _defineProperty('clientID', local=True, default='')
+ clientSecret = _defineProperty('clientSecret', local=True, default='')
+ refreshToken = _defineProperty('refreshToken', local=True, default='')
+ pad = _defineProperty('pad', local=True, type='boolean', default=False)
latestRevision = _defineNightlyProperty('latestRevision')
@@ -181,6 +181,46 @@
"""
return self.repositoryName
+ def listContents(self, version='tip'):
+ return subprocess.check_output(['hg', '-R', self.repository, 'locate', '-r', version]).splitlines()
+
+ def readMetadata(self, version='tip'):
+ genericFilename = 'metadata'
+ filename = '%s.%s' % (genericFilename, self.type)
+ files = self.listContents(version)
+
+ if filename not in files:
+ # some repositories like those for Android and
+ # Internet Explorer don't have metadata files
+ if genericFilename not in files:
+ return None
+
+ # Fall back to platform-independent metadata file
+ filename = genericFilename
+
+ command = ['hg', '-R', self.repository, 'cat', '-r', version, os.path.join(self.repository, filename)]
+ result = subprocess.check_output(command)
+
+ parser = SafeConfigParser()
+ parser.readfp(StringIO(result))
+
+ return parser
+
+ @property
+ def basename(self):
+ metadata = self.readMetadata()
+ if metadata:
+ return metadata.get('general', 'basename')
+ return os.path.basename(self.repository)
+
+ def getDownloads(self):
+ prefix = self.basename + '-'
+ command = ['hg', 'locate', '-R', self.downloadsRepo, '-r', 'default']
+
+ for filename in subprocess.check_output(command).splitlines():
+ if filename.startswith(prefix) and filename.endswith(self.packageSuffix):
+ yield (filename, filename[len(prefix):len(filename) - len(self.packageSuffix)])
+
@staticmethod
def getRepositoryConfigurations(nightlyConfig = None):
"""