issue 4087 - Use Mozilla's product-details data to determine current product versions

globals/get_browser_versions.py

 import re
 import os
 import sys
 import json
 import urllib
 import errno
 import logging
 import time
 from xml.dom import minidom
 
 from jinja2 import contextfunction
 
 BROWSERS = {}
+BASE_URL = 'https://product-details.mozilla.org/1.0'
+FIREFOX_URL = BASE_URL + '/firefox_versions.json'
+THUNDERBIRD_URL = BASE_URL + '/thunderbird_versions.json'
+SEAMONKEY_URL = 'http://www.seamonkey-project.org/seamonkey_versions.json'
 
 CHROME_UPDATE_XML = '''\
 <?xml version="1.0" encoding="UTF-8"?>
 <request protocol="3.0" ismachine="0">
   <os platform="win" version="99" arch="x64"/>
   <app appid="{4DC8B4CA-1BDA-483E-B5FA-D3C12E15B62D}">
     <updatecheck/>
   </app>
   <app appid="{4DC8B4CA-1BDA-483E-B5FA-D3C12E15B62D}" ap="x64-beta-multi-chrome">
     <updatecheck/>
   </app>
   <app appid="{4DC8B4CA-1BDA-483E-B5FA-D3C12E15B62D}" ap="x64-dev-multi-chrome">
     <updatecheck/>
   </app>
 </request>'''
 
 cache = {}
 
 
-def get_json_versions(product):
-    urls = {
-            'Firefox':
-            'https://product-details.mozilla.org/firefox_versions.json',

[Wladimir Palant, 2016/08/12 11:40:55]

Gotta love it - the URL changed into
https://product-details.mozilla.org/1.0/firefox_versions.json without a redirect
(same with the Thunderbird versions) ;)

[Jon Sonesen, 2016/08/16 02:27:54]

will update :D

-            'Thunderbird':
-            'https://product-details.mozilla.org/thunderbird_versions.json',
-            'Seamonkey':
-            'http://www.seamonkey-project.org/seamonkey_versions.json'
-    }
-    response = urllib.urlopen(urls[product])
-    try:
-        doc = json.load(response.read())
-    finally:
-        response.close()
+def get_json_versions(product_url):
+    response = urllib.urlopen(product_url)
+    try:
+        doc = json.load(response)
+    except json.ValueError:
+        print 'URL: %s not returning json object'.format(product_url)
+    finally:
+        response.close()
+
+    for key, value in doc.iteritems():
+        if value:
+            match = re.search(r'^(\d+)(?:\.\d+)?', value)
+            if match:
+                doc[key] = match.group(0)
     return doc
 
 
 def get_firefox_versions():
-    versions = get_json_versions('Firefox')
+    versions = get_json_versions(FIREFOX_URL)
     return {
             'current': versions['LATEST_FIREFOX_VERSION'],
             'unreleased': [
                 versions['LATEST_FIREFOX_DEVEL_VERSION'],
                 versions['FIREFOX_AURORA'],
                 versions['FIREFOX_NIGHTLY'],

[Wladimir Palant, 2016/08/12 11:40:55]

We need to remove the "a1" and similar suffixes from the version numbers - we
only want the actual milestone.

[Jon Sonesen, 2016/08/16 02:27:54]

Just to clarify; do you mean that I should remove any alpha numeric leading
suffixes? Or rather that anything past the second decimal point should be
removed?

[Wladimir Palant, 2016/08/16 10:40:17]

The code you are replacing here was using r'^(\d+)(?:\.\d+)?' regular expression
to extract the version number. It's probably best if you do the same.

[Jon Sonesen, 2016/08/16 18:46:09]

Thank you, also I added a log statement in case they change the url with no
redirect, additionally added a value check for the regex in case they add keys
that have null strings.

             ]
     }
 
 
 def get_thunderbird_versions():
-    versions = get_json_versions('Thunderbird')
-    firefox_versions = get_json_versions('Firefox')
-    return {
-        'current': versions['LATEST_THUNDERBIRD_VERSION'],
+    tbird_versions = get_json_versions(THUNDERBIRD_URL)
+    firefox_versions = get_json_versions(FIREFOX_URL)
+    return {
+        'current': tbird_versions['LATEST_THUNDERBIRD_VERSION'],
         'unreleased': [
-            versions['LATEST_THUNDERBIRD_DEVEL_VERSION'],
-            firefox_versions['FIREFOX_AURORA'],
+            tbird_versions['LATEST_THUNDERBIRD_DEVEL_VERSION'],
+            tbird_versions['LATEST_THUNDERBIRD_ALPHA_VERSION'],
             firefox_versions['FIREFOX_NIGHTLY'],

[Wladimir Palant, 2016/08/12 11:40:55]

There is LATEST_THUNDERBIRD_ALPHA_VERSION in the data, we should use that rather
than FIREFOX_AURORA even though it seems to be outdated right now.

[Jon Sonesen, 2016/08/16 02:27:54]

Will use that instead.

         ]
     }
 
 BROWSERS['firefox'] = lambda: get_firefox_versions()
 BROWSERS['thunderbird'] = lambda: get_thunderbird_versions()
 
 
 def get_seamonkey_versions():
-    # Aurora and Nightly builds for Windows are permantently broken.
-    # Occasionally, builds for other platforms are broken as well.
-    # https://bugzilla.mozilla.org/show_bug.cgi?id=1086553

[Wladimir Palant, 2016/08/12 11:40:55]

This comment no longer makes sense, it doesn't matter whether their update
server is working.

-    seamonkey_versions = get_json_versions('Seamonkey')
+    seamonkey_versions = get_json_versions(SEAMONKEY_URL)
     versions = {
         'current': seamonkey_versions['LATEST_SEAMONKEY_VERSION'],
         'unreleased': [
-            seamonkey_versions['LATEST_SEAMONKEY_DEVEL_VERSION'],

[Wladimir Palant, 2016/08/12 11:40:55]

This should be the last entry in the list since it seems to designate SeaMonkey
nightlies (yes, they aren't being consistent with Firefox).

             seamonkey_versions['LATEST_SEAMONKEY_MILESTONE_VERSION'],
             seamonkey_versions['LATEST_SEAMONKEY_TESTING_VERSION'],
+            seamonkey_versions['LATEST_SEAMONKEY_DEVEL_VERSION'],
         ]
     }
 
     return versions
 
 BROWSERS['seamonkey'] = get_seamonkey_versions
 
 
 def get_chrome_version(manifest):
     return manifest.getAttribute('version').split('.')[0]
@@ skipping 143 matching lines @@
         file.seek(0)
         json.dump(persistent_cache, file)
         file.truncate()
 
     if not versions['previous']:
         logging.warning("Couldn't determine previous browser version, "
                         'please set %s.previous in %s', browser, filename)
 
     cache[browser] = versions
     return versions