Issue 520 - Generate PAD files for download portals when updating download links

sitescripts/extensions/bin/updateDownloadLinks.py

Index: sitescripts/extensions/bin/updateDownloadLinks.py
===================================================================
--- a/sitescripts/extensions/bin/updateDownloadLinks.py
+++ b/sitescripts/extensions/bin/updateDownloadLinks.py
@@ -29,6 +29,8 @@
 from StringIO import StringIO
 from sitescripts.utils import get_config, get_template
 from sitescripts.extensions.utils import compareVersions, Configuration, getSafariCertificateID
+from sitescripts.extensions.android import get_min_sdk_version
+from sitescripts.extensions.pad import PadFile
 from buildtools.packagerGecko import KNOWN_APPS
 
 def urlencode(value):
@@ -66,84 +68,77 @@
   """
   gets download link for a Chrome add-on from the Chrome Gallery site
   """
-  param = 'id=%s&uc' % urlencode(galleryID)
-  url = 'https://clients2.google.com/service/update2/crx?x=%s' % urlencode(param)
-  contents = urlopen(url).read()
-  document = dom.parseString(contents)
+  galleryID = urlencode(galleryID)
+
+  url = 'https://clients2.google.com/service/update2/crx?x=%s' % urlencode('id=%s&uc' % galleryID)
+  document = dom.parse(urlopen(url))
   updateTags = document.getElementsByTagName('updatecheck')
-  updateTag = updateTags[0] if len(updateTags) > 0 else None
-  if updateTag and updateTag.hasAttribute('codebase') and updateTag.hasAttribute('version'):
-    return (updateTag.getAttribute('codebase'), updateTag.getAttribute('version'))
-  else:
+  version = updateTags and updateTags[0].getAttribute('version')
+
+  if not version:
     return (None, None)
 
+  request = urllib2.Request('https://chrome.google.com/webstore/detail/_/' + galleryID)
+  request.get_method = lambda : 'HEAD'
+  url = urllib2.urlopen(request).geturl()
+
+  return (url, version)
+
 def getOperaDownloadLink(galleryID):
   """
   gets download link for an Opera add-on from the Opera Addons site
   """
-  class HeadRequest(urllib2.Request):
-    def get_method(self):
-      return "HEAD"
+  galleryID = urlencode(galleryID)
 
-  url = 'https://addons.opera.com/extensions/download/%s/' % urlencode(galleryID)
-  response = urllib2.urlopen(HeadRequest(url))
-  content_disposition = response.info().dict.get('content-disposition', None)
-  if content_disposition != None:
-    match = re.search(r'filename=\S+-([\d.]+)-\d+\.oex$', content_disposition)
-  else:
-    match = None;
-  if match:
-    return (url, match.group(1))
-  else:
-    return (None, None)
+  request = urllib2.Request('https://addons.opera.com/extensions/download/%s/' % galleryID)
+  request.get_method = lambda : 'HEAD'
+  response = urllib2.urlopen(request)
+
+  content_disposition = response.info().getheader('Content-Disposition')
+  if content_disposition:
+    match = re.search(r'filename=\S+-([\d.]+)-\d+\.crx$', content_disposition)
+    if match:
+      return ('https://addons.opera.com/extensions/details/%s/' % galleryID , match.group(1))
+
+  return (None, None)
 
 def getLocalLink(repo):
   """
   gets the link for the newest download of an add-on in the local downloads
   repository
   """
-  url = repo.downloadsURL
-
   highestURL = None
   highestVersion = None
 
-  if repo.type in ('gecko', 'chrome', 'opera', 'safari'):
-    prefix = readRawMetadata(repo).get('general', 'basename')
-  else:
-    prefix = os.path.basename(repo.repository)
-  prefix += '-'
-  suffix = repo.packageSuffix
+  for filename, version in repo.getDownloads():
+    if not highestVersion or compareVersions(version, highestVersion) > 0:
+      highestURL = urlparse.urljoin(repo.downloadsURL, filename)
+      highestVersion = version
 
-  # go through the downloads repository looking for downloads matching this extension
-  command = ['hg', 'locate', '-R', repo.downloadsRepo, '-r', 'default']
-  result = subprocess.check_output(command)
-  for fileName in result.splitlines():
-    if fileName.startswith(prefix) and fileName.endswith(suffix):
-      version = fileName[len(prefix):len(fileName) - len(suffix)]
-      if highestVersion == None or compareVersions(version, highestVersion) > 0:
-        highestURL = urlparse.urljoin(url, fileName)
-        highestVersion = version
   return (highestURL, highestVersion)
 
 def getDownloadLink(repo):
   """
   gets the download link to the most current version of an extension
   """
-  galleryURL = None
-  galleryVersion = None
-  if repo.type == "gecko" and repo.galleryID:
+  # you can't easily install extensions from third-party sources on Chrome
+  # and Opera. So always get the link for the version on the Web Store.
+  if repo.galleryID:
+    if repo.type == "chrome":
+      return getGoogleDownloadLink(repo.galleryID)
+    if repo.type == "opera":
+      return getOperaDownloadLink(repo.galleryID)
+
+  (localURL, localVersion) = getLocalLink(repo)
+
+  # get a link to Firefox Add-Ons, if the latest version has been published there
+  if repo.type == 'gecko' and repo.galleryID:
     (galleryURL, galleryVersion) = getMozillaDownloadLink(repo.galleryID)
-  elif repo.type == "chrome" and repo.galleryID:
-    (galleryURL, galleryVersion) = getGoogleDownloadLink(repo.galleryID)
-  elif repo.type == "opera" and repo.galleryID:
-    (galleryURL, galleryVersion) = getOperaDownloadLink(repo.galleryID)
+    if not localVersion or (galleryVersion and
+                            compareVersions(galleryVersion, localVersion) >= 0):
+      return (galleryURL, galleryVersion)
 
-  (downloadsURL, downloadsVersion) = getLocalLink(repo)
-  if galleryVersion == None or (downloadsVersion != None and
-                                compareVersions(galleryVersion, downloadsVersion) < 0):
-    return (downloadsURL, downloadsVersion)
-  else:
-    return (galleryURL, galleryVersion)
+  return (localURL, localVersion)
 
 def getQRCode(text):
   try:
@@ -175,23 +170,6 @@
     if qrcode != None:
       result.set(repo.repositoryName, "qrcode", qrcode)
 
-def readRawMetadata(repo, version='tip'):
-  files = subprocess.check_output(['hg', '-R', repo.repository, 'locate', '-r', version]).splitlines()
-  genericFilename = 'metadata'
-  filename = '%s.%s' % (genericFilename, repo.type)
-
-  # Fall back to platform-independent metadata file
-  if filename not in files:
-    filename = genericFilename
-
-  command = ['hg', '-R', repo.repository, 'cat', '-r', version, os.path.join(repo.repository, filename)]
-  result = subprocess.check_output(command)
-
-  parser = SafeConfigParser()
-  parser.readfp(StringIO(result))
-
-  return parser
-
 def readMetadata(repo, version):
   """
   reads extension ID and compatibility information from metadata file in the
@@ -202,17 +180,12 @@
     result = subprocess.check_output(command)
     revision = re.sub(r'\D', '', result)
 
-    command = ['hg', '-R', repo.repository, 'cat', '-r', version, os.path.join(repo.repository, 'AndroidManifest.xml')]
-    result = subprocess.check_output(command)
-    manifest = dom.parseString(result)
-    usesSdk = manifest.getElementsByTagName('uses-sdk')[0]
-
     return {
       'revision': revision,
-      'minSdkVersion': usesSdk.attributes["android:minSdkVersion"].value,
+      'minSdkVersion': get_min_sdk_version(repo, version),
     }
   elif repo.type == 'safari':
-    metadata = readRawMetadata(repo, version)
+    metadata = repo.readMetadata(version)
     return {
       'certificateID': getSafariCertificateID(repo.keyFile),
       'version': version,
@@ -220,7 +193,7 @@
       'basename': metadata.get('general', 'basename'),
     }
   elif repo.type == 'gecko':
-    metadata = readRawMetadata(repo, version)
+    metadata = repo.readMetadata(version)
     result = {
       'extensionID': metadata.get('general', 'id'),
       'version': version,
@@ -257,6 +230,12 @@
     template = get_template(get_config().get('extensions', '%sUpdateManifest' % repoType))
     template.stream({'extensions': extensions[repoType]}).dump(manifestPath)
 
+def writePadFile(links):
+  for repo in Configuration.getRepositoryConfigurations():
+    if repo.pad and links.has_section(repo.repositoryName):
+      PadFile.forRepository(repo, links.get(repo.repositoryName, 'version'),
+                                  links.get(repo.repositoryName, 'downloadURL')).write()
+
 def updateLinks():
   """
   writes the current extension download links to a file
@@ -270,6 +249,7 @@
   file.close()
 
   writeUpdateManifest(result)
+  writePadFile(result)
 
 if __name__ == "__main__":
   updateLinks()