Issue 2432 - Auto-generate browser versions on requirements page

globals/get_browser_versions.py

 import re
 import os
 import sys
 import json
 import urllib2
 import errno
+import logging
 from xml.dom import minidom
 
 from jinja2 import contextfunction
 
 BROWSERS = {}
 
 CHROME_UPDATE_XML = '''\
 <?xml version="1.0" encoding="UTF-8"?>
 <request protocol="3.0" ismachine="0">
   <os platform="win" version="99" arch="x64"/>
   <app appid="{4DC8B4CA-1BDA-483E-B5FA-D3C12E15B62D}">
     <updatecheck/>
   </app>
   <app appid="{4DC8B4CA-1BDA-483E-B5FA-D3C12E15B62D}" ap="x64-beta-multi-chrome">
     <updatecheck/>
   </app>
   <app appid="{4DC8B4CA-1BDA-483E-B5FA-D3C12E15B62D}" ap="x64-dev-multi-chrome">
     <updatecheck/>
   </app>
 </request>'''
 
-def get_mozilla_version(product, origin_version, channel, minor=False,
-                        subdomain='aus4', origin_build='-', attribute='appVersion'):
-  response = urllib2.urlopen('https://%s.mozilla.org/update/3/%s/%s/%s/WINNT_x86-msvc/en-US/%s/-/default/default/update.xml?force=1' % (subdomain, product, origin_version, origin_build, channel))
+def get_mozilla_version(product, origin_version, channel,
+                        minor=False, subdomain='aus4', origin_build='-',
+                        attribute='appVersion', platform='WINNT_x86-msvc'):
+  response = urllib2.urlopen('https://%s.mozilla.org/update/3/%s/%s/%s/%s/en-US/%s/-/default/default/update.xml?force=1' % (
+    subdomain,
+    product,
+    origin_version,
+    origin_build,
+    platform,
+    channel
+  ))
   try:
     doc = minidom.parse(response)
   finally:
     response.close()
 
   update = doc.getElementsByTagName('update')[0]
   full_version = update.getAttribute(attribute)
 
   match = re.search(r'^(\d+)(?:\.\d+)?', full_version)
   if minor:
     return match.group(0)
   return match.group(1)
 
 def get_mozilla_versions(product, origin_version, release_minor=False):
   return {
     'current': get_mozilla_version(product, origin_version, 'release', release_minor),
     'unreleased': [
       get_mozilla_version(product, origin_version, 'beta'),
       get_mozilla_version(product, origin_version, 'aurora'),
       get_mozilla_version(product, origin_version, 'nightly'),
     ]
   }
 
-BROWSERS['firefox']     = lambda: get_mozilla_versions('Firefox',     '37.0')
+BROWSERS['firefox'] = lambda: get_mozilla_versions('Firefox', '37.0')
 BROWSERS['thunderbird'] = lambda: get_mozilla_versions('Thunderbird', '31.0', True)
 
-def get_seamonkey_version(origin_version, origin_build, channel):
+def get_seamonkey_version(origin_version, origin_build, channel, **kw):
   return get_mozilla_version('SeaMonkey', origin_version, channel, True,
-                             'aus2-community', origin_build, 'version')
+                             'aus2-community', origin_build, 'version', **kw)
 
 def get_seamonkey_versions():
   return {
-    'current':     get_seamonkey_version('2.32', '20150112201917', 'release'),
-    'unreleased': [get_seamonkey_version('2.32', '20150101215737', 'beta')]
+    'current': get_seamonkey_version('2.32', '20150112201917', 'release'),
+    'unreleased': [
+      get_seamonkey_version('2.32', '20150101215737', 'beta'),
+
+      # Aurora and Nightly builds for Windows are currently broken.
+      # https://bugzilla.mozilla.org/show_bug.cgi?id=1086553
+      get_seamonkey_version('2.32', '-', 'aurora', platform='Linux_x86-gcc3'),
+      get_seamonkey_version('2.32', '-', 'nightly', platform='Linux_x86-gcc3')
+    ]
   }
 
 BROWSERS['seamonkey'] = get_seamonkey_versions
 
 def get_chrome_version(manifest):
   return manifest.getAttribute('version').split('.')[0]
 
 def get_chrome_versions():
   response = urllib2.urlopen(urllib2.Request('https://tools.google.com/service/update2', CHROME_UPDATE_XML))
   try:
@@ skipping 41 matching lines @@
   return re.search(r'\d+\.\d+', description.firstChild.nodeValue).group(0)
 
 def get_yandex_versions():
   return {
     'current': get_yandex_version(''),
     'unreleased': [get_yandex_version('-beta')]
   }
 
 BROWSERS['yandex'] = get_yandex_versions
 
-def open_cache_file(source):
-  cache_dir = source.get_cache_dir()
-  filename = os.path.join(cache_dir, 'browsers.json')
+def open_cache_file(filename):
   flags = os.O_RDWR | os.O_CREAT
   try:
     fd = os.open(filename, flags)
   except OSError as e:
     if e.errno != errno.ENOENT:
       raise
-    os.makedirs(cache_dir)
+    os.makedirs(os.path.dirname(filename))
     fd = os.open(filename, flags)
   return os.fdopen(fd, 'w+')
 
 @contextfunction
 def get_browser_versions(context, browser):
   func = BROWSERS[browser]
-  exception = None
+  exc_info = None
   try:
     versions = func()
-  except Exception as e:
-    exception = e
-
-  with open_cache_file(context['source']) as file:
+  except Exception:
+    exc_info = sys.exc_info()
+
+  filename = os.path.join(context['source'].get_cache_dir(), 'browsers.json')
+  with open_cache_file(filename) as file:
     try:
       cache = json.load(file)
     except ValueError:
       if file.tell() > 0:
         raise
       cache = {}
 
     cached_versions = cache.get(browser)
-    if exception:
+    if exc_info:
       if not cached_versions:
-        raise exception
-
-      print >>sys.stderr, "Warning: Failed to get %s versions, falling back to cached versions" % browser
-      return cached_versions
-
-    # Determine previous version: If we recorded the version before and it
-    # changed since then, the old current version becomes the new previous
-    # version. If the version didn't change, use the cached previous version.
-    current = versions['current']
-    if cached_versions:
-      cached_current = cached_versions['current']
-      if cached_current != current:
-        versions['previous'] = cached_current
-      else:
-        cached_previous = cached_versions.get('previous')
-        if cached_previous:
-          versions['previous'] = cached_previous
-
-    # Remove duplicated from unreleased versions. Occasionally,
-    # different channels are on the same version, but we want
-    # to list each version only once.
-    unreleased = versions['unreleased']
-    previous = versions.get('previous')
-    for i, version in list(enumerate(unreleased))[::-1]:
-      if version == current or previous and version == previous:
-        del unreleased[i]
-
-    cache[browser] = versions
-    file.seek(0)
-    json.dump(cache, file)
-    file.truncate()
+        raise exc_info[0], exc_info[1], exc_info[2]
+
+      versions = cached_versions
+      logging.warning('Failed to get %s versions, falling back to '
+                      'cached versions', browser, exc_info=exc_info)
+    else:
+      # Determine previous version: If we recorded the version before and it
+      # changed since then, the old current version becomes the new previous
+      # version. If the version didn't change, use the cached previous version.
+      current = versions['current']
+      previous = None
+      if cached_versions:
+        cached_current = cached_versions['current']
+        if cached_current != current:
+          previous = cached_current
+        else:
+          previous = cached_versions['previous']
+      versions['previous'] = previous
+
+      # Remove duplicates from unreleased versions. Occasionally,
+      # different channels are on the same version, but we want
+      # to list each version only once.
+      versions['unreleased'] = sorted(
+        set(versions['unreleased']) - {current, previous},
+        key=lambda ver: map(int, ver.split('.'))
+      )
+
+      cache[browser] = versions
+      file.seek(0)
+      json.dump(cache, file)
+      file.truncate()
+
+  if not versions['previous']:
+    logging.warning("Couldn't determine previous browser version, "
+                    'please set %s.previous in %s', browser, filename)
 
   return versions