Issue 520 - Generate PAD files for download portals when updating download links

sitescripts/extensions/bin/updateDownloadLinks.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2014 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
@@ skipping 11 matching lines @@
   This script generates a list of extensions and saves these with download links
   and version information
 """
 
 import sys, os, re, urllib, urllib2, urlparse, subprocess, time
 import xml.dom.minidom as dom
 from ConfigParser import SafeConfigParser
 from StringIO import StringIO
 from sitescripts.utils import get_config, get_template
 from sitescripts.extensions.utils import compareVersions, Configuration, getSafariCertificateID
+from sitescripts.extensions.android import get_min_sdk_version
+from sitescripts.extensions.pad import PadFile
 from buildtools.packagerGecko import KNOWN_APPS
 
 def urlencode(value):
   return urllib.quote(value.encode('utf-8'), '')
 
 def urlopen(url, attempts=3):
   """
   Tries to open a particular URL, retries on failure.
   """
   for i in range(attempts):
@@ skipping 17 matching lines @@
   versionTag = versionTags[0] if len(versionTags) > 0 else None
   if linkTag and versionTag and linkTag.firstChild and versionTag.firstChild:
     return (linkTag.firstChild.data, versionTag.firstChild.data)
   else:
     return (None, None)
 
 def getGoogleDownloadLink(galleryID):
   """
   gets download link for a Chrome add-on from the Chrome Gallery site
   """
-  param = 'id=%s&uc' % urlencode(galleryID)
-  url = 'https://clients2.google.com/service/update2/crx?x=%s' % urlencode(param)
-  contents = urlopen(url).read()
-  document = dom.parseString(contents)
+  galleryID = urlencode(galleryID)
+
+  response = urlopen('https://clients2.google.com/service/update2/crx?x=' + urlencode('id=%s&uc' % galleryID))

[Wladimir Palant, 2014/05/26 10:54:57]

a) Why use an overly long line instead of a temporary param variable?

b) Why use string concatenation rather than substitution here? I seem to
remember that you were generally opposing string concatenation in Python.

[Sebastian Noack, 2014/05/26 12:04:53]

Agreed.
When only concatenating two strings, string concatenation by plus operator feels
more intuitive and less complex. However when concatenating multiple strings,
string formatting is more efficient and IMO easier to read.

[Wladimir Palant, 2014/05/26 13:26:53]

What we have here is a URL template, substitution seems to be a better choice
then - the string to be replaced doesn't necessarily need to be at the end of
the URL.

[Sebastian Noack, 2014/05/27 12:01:23]

Done.

+  document = dom.parse(response)
   updateTags = document.getElementsByTagName('updatecheck')
-  updateTag = updateTags[0] if len(updateTags) > 0 else None
-  if updateTag and updateTag.hasAttribute('codebase') and updateTag.hasAttribute('version'):
-    return (updateTag.getAttribute('codebase'), updateTag.getAttribute('version'))
-  else:
+  version = updateTags and updateTags[0].getAttribute('version')
+
+  if not version:
     return (None, None)
 
+  request = urllib2.Request('https://chrome.google.com/webstore/detail/_/' + galleryID)
+  request.get_method = lambda : 'HEAD'
+  url = urllib2.urlopen(request).geturl()
+
+  return (url, version)
+
 def getOperaDownloadLink(galleryID):
   """
   gets download link for an Opera add-on from the Opera Addons site
   """
-  class HeadRequest(urllib2.Request):
-    def get_method(self):
-      return "HEAD"
+  galleryID = urlencode(galleryID)
 
-  url = 'https://addons.opera.com/extensions/download/%s/' % urlencode(galleryID)
-  response = urllib2.urlopen(HeadRequest(url))
-  content_disposition = response.info().dict.get('content-disposition', None)
-  if content_disposition != None:
-    match = re.search(r'filename=\S+-([\d.]+)-\d+\.oex$', content_disposition)
-  else:
-    match = None;
-  if match:
-    return (url, match.group(1))
-  else:
-    return (None, None)
+  request = urllib2.Request('https://addons.opera.com/extensions/download/%s/' % galleryID)
+  request.get_method = lambda : 'HEAD'
+  response = urllib2.urlopen(request)
+
+  content_disposition = response.info().getheader('Content-Disposition')
+  if content_disposition:
+    match = re.search(r'filename=\S+-([\d.]+)-\d+\.crx$', content_disposition)
+    if match:
+      return ('https://addons.opera.com/extensions/details/%s/' % galleryID , match.group(1))
+
+  return (None, None)
 
 def getLocalLink(repo):
   """
   gets the link for the newest download of an add-on in the local downloads
   repository
   """
-  url = repo.downloadsURL
-
   highestURL = None
   highestVersion = None
 
-  if repo.type in ('gecko', 'chrome', 'opera', 'safari'):
-    prefix = readRawMetadata(repo).get('general', 'basename')
-  else:
-    prefix = os.path.basename(repo.repository)
-  prefix += '-'
-  suffix = repo.packageSuffix
+  for filename, version in repo.getDownloads():
+    if not highestVersion or compareVersions(version, highestVersion) > 0:
+      highestURL = urlparse.urljoin(repo.downloadsURL, filename)
+      highestVersion = version
 
-  # go through the downloads repository looking for downloads matching this extension
-  command = ['hg', 'locate', '-R', repo.downloadsRepo, '-r', 'default']
-  result = subprocess.check_output(command)
-  for fileName in result.splitlines():
-    if fileName.startswith(prefix) and fileName.endswith(suffix):
-      version = fileName[len(prefix):len(fileName) - len(suffix)]
-      if highestVersion == None or compareVersions(version, highestVersion) > 0:
-        highestURL = urlparse.urljoin(url, fileName)
-        highestVersion = version
   return (highestURL, highestVersion)
 
 def getDownloadLink(repo):
   """
   gets the download link to the most current version of an extension
   """
-  galleryURL = None
-  galleryVersion = None
-  if repo.type == "gecko" and repo.galleryID:
+  # you can't easily install extensions from third-party sources on Chrome
+  # and Opera. So always get the link for the version on the Web Store.
+  if repo.type in ("chrome", "opera") and repo.galleryID:

[Wladimir Palant, 2014/05/26 10:54:57]

Isn't the repo.type check redundant here? You check the type below anyway.

[Sebastian Noack, 2014/05/26 12:04:53]

You are right.

+    if repo.type == "chrome":
+      return getGoogleDownloadLink(repo.galleryID)
+
+    if repo.type == "opera":
+      return getOperaDownloadLink(repo.galleryID)
+
+  (localURL, localVersion) = getLocalLink(repo)
+
+  # get a link to Firefox Add-Ons, if the latest version has been published there
+  if repo.type == 'gecko' and repo.galleryID:
     (galleryURL, galleryVersion) = getMozillaDownloadLink(repo.galleryID)
-  elif repo.type == "chrome" and repo.galleryID:
-    (galleryURL, galleryVersion) = getGoogleDownloadLink(repo.galleryID)
-  elif repo.type == "opera" and repo.galleryID:
-    (galleryURL, galleryVersion) = getOperaDownloadLink(repo.galleryID)
+    if not localVersion or (galleryVersion and
+                            compareVersions(galleryVersion, localVersion) >= 0):
+      return (galleryURL, galleryVersion)
 
-  (downloadsURL, downloadsVersion) = getLocalLink(repo)
-  if galleryVersion == None or (downloadsVersion != None and
-                                compareVersions(galleryVersion, downloadsVersion) < 0):
-    return (downloadsURL, downloadsVersion)
-  else:
-    return (galleryURL, galleryVersion)
+  return (localURL, localVersion)
 
 def getQRCode(text):
   try:
     import qrcode
     import base64
     import Image    # required by qrcode but not formally a dependency
   except:
     return None
 
   data = StringIO()
@@ skipping 11 matching lines @@
       continue
     if not result.has_section(repo.repositoryName):
       result.add_section(repo.repositoryName)
     result.set(repo.repositoryName, "downloadURL", downloadURL)
     result.set(repo.repositoryName, "version", version)
 
     qrcode = getQRCode(downloadURL)
     if qrcode != None:
       result.set(repo.repositoryName, "qrcode", qrcode)
 
-def readRawMetadata(repo, version='tip'):
-  files = subprocess.check_output(['hg', '-R', repo.repository, 'locate', '-r', version]).splitlines()
-  genericFilename = 'metadata'
-  filename = '%s.%s' % (genericFilename, repo.type)
-
-  # Fall back to platform-independent metadata file
-  if filename not in files:
-    filename = genericFilename
-
-  command = ['hg', '-R', repo.repository, 'cat', '-r', version, os.path.join(repo.repository, filename)]
-  result = subprocess.check_output(command)
-
-  parser = SafeConfigParser()
-  parser.readfp(StringIO(result))
-
-  return parser
-
 def readMetadata(repo, version):
   """
   reads extension ID and compatibility information from metadata file in the
   extension's repository
   """
   if repo.type == 'android':
     command = ['hg', '-R', repo.repository, 'id', '-r', version, '-n']
     result = subprocess.check_output(command)
     revision = re.sub(r'\D', '', result)
 
-    command = ['hg', '-R', repo.repository, 'cat', '-r', version, os.path.join(repo.repository, 'AndroidManifest.xml')]
-    result = subprocess.check_output(command)
-    manifest = dom.parseString(result)
-    usesSdk = manifest.getElementsByTagName('uses-sdk')[0]
-
     return {
       'revision': revision,
-      'minSdkVersion': usesSdk.attributes["android:minSdkVersion"].value,
+      'minSdkVersion': get_min_sdk_version(repo, version),
     }
   elif repo.type == 'safari':
-    metadata = readRawMetadata(repo, version)
+    metadata = repo.readMetadata(version)
     return {
       'certificateID': getSafariCertificateID(repo.keyFile),
       'version': version,
       'shortVersion': version,
       'basename': metadata.get('general', 'basename'),
     }
   elif repo.type == 'gecko':
-    metadata = readRawMetadata(repo, version)
+    metadata = repo.readMetadata(version)
     result = {
       'extensionID': metadata.get('general', 'id'),
       'version': version,
       'compat': []
     }
     for key, value in KNOWN_APPS.iteritems():
       if metadata.has_option('compat', key):
         minVersion, maxVersion = metadata.get('compat', key).split('/')
         result['compat'].append({'id': value, 'minVersion': minVersion, 'maxVersion': maxVersion})
     return result
@@ skipping 16 matching lines @@
     extensions[repo.type].append(data)
 
   if len(extensions['android']) > 1:
     print >>sys.stderr, 'Warning: more than one Android app defined, update manifest only works for one'
 
   for repoType in extensions.iterkeys():
     manifestPath = get_config().get('extensions', '%sUpdateManifestPath' % repoType)
     template = get_template(get_config().get('extensions', '%sUpdateManifest' % repoType))
     template.stream({'extensions': extensions[repoType]}).dump(manifestPath)
 
+def writePadFile(links):
+  for repo in Configuration.getRepositoryConfigurations():
+    if repo.padTemplate and links.has_section(repo.repositoryName):
+      PadFile.forRepository(repo, links.get(repo.repositoryName, 'version'),
+                                  links.get(repo.repositoryName, 'downloadURL')).write()
+
 def updateLinks():
   """
   writes the current extension download links to a file
   """
 
   # Now get download links and save them to file
   result = SafeConfigParser()
   getDownloadLinks(result)
   file = open(get_config().get('extensions', 'downloadLinksFile'), 'wb')
   result.write(file)
   file.close()
 
   writeUpdateManifest(result)
+  writePadFile(result)
 
 if __name__ == "__main__":
   updateLinks()