issue 4087 - Use Mozilla's product-details data to determine current product versions

globals/get_browser_versions.py

 import re
 import os
 import sys
 import json
 import urllib
 import errno
 import logging
 import time
 from xml.dom import minidom
 
@@ skipping 12 matching lines @@
     <updatecheck/>
   </app>
   <app appid="{4DC8B4CA-1BDA-483E-B5FA-D3C12E15B62D}" ap="x64-dev-multi-chrome">
     <updatecheck/>
   </app>
 </request>'''
 
 cache = {}
 
 
-def get_mozilla_version(product, origin_version, channel,
-                        minor=False, subdomain='aus4', origin_build='-',
-                        attribute='appVersion', platform='WINNT_x86-msvc'):
-    response = urllib.urlopen('https://%s.mozilla.org/update/3/%s/%s/%s/%s/en-US/%s/-/default/default/update.xml?force=1' % (
-        subdomain,
-        product,
-        origin_version,
-        origin_build,
-        platform,
-        channel
-    ))
+def get_json_versions(product):
+    urls = {

[Vasily Kuznetsov, 2016/08/22 17:51:55]

This dict seems somewhat redundant here. Wouldn't it be better to move these
urls into constants at the top level of the file and then use them directly in
`get_xxx_versions` functions? Kind of like this:

def get_json_versions(url):
    ...

FIREFOX_URL = '....'
def get_firefox_versions():
    versions = get_json_versions(FIREFOX_URL)

Then `get_json_versions` becomes more generic and the hardcoded URLs end up in
constants instead of the dict. I know that in other functions the URLs are
hardcoded directly into the calls, but here we'd be using FIREFOX_URL twice and,
well, having the URLs in the constants seems cleaner anyway.

[Jon Sonesen, 2016/08/29 13:15:39]

So rather than a constant dict just have the function call using the constant
url as the product? I am happy to do this :D

+            'Firefox':
+            'https://product-details.mozilla.org/1.0/firefox_versions.json',
+            'Thunderbird':
+            'https://product-details.mozilla.org/1.0/thunderbird_versions.json',
+            'Seamonkey':
+            'http://www.seamonkey-project.org/seamonkey_versions.json'
+    }
+    response = urllib.urlopen(urls[product])
     try:
-        doc = minidom.parse(response)
+        doc = json.load(response)
+    except json.ValueError:
+        print 'URL: %s not returning json object'.format(urls[product])
     finally:
         response.close()
 
-    updates = doc.getElementsByTagName('update')
-    if not updates:
-        raise Exception('No updates for %s in %s channel' % (product, channel))
-    full_version = updates[0].getAttribute(attribute)
-
-    match = re.search(r'^(\d+)(?:\.\d+)?', full_version)
-    if minor:
-        return match.group(0)
-    return match.group(1)
+    for key, value in doc.iteritems():
+        if value:
+            doc[key] = re.search(r'^(\d+)(?:\.\d+)?', value).group(0)

[Vasily Kuznetsov, 2016/08/22 17:51:55]

Isn't this a bit fragile? What if the regex doesn't match?

[Jon Sonesen, 2016/08/29 13:15:39]

I agree that it is fragile...but also I thought we wanted this to fail 'loudly'
which this would be doing. Also, this is how the code was originally, also, 

+    return doc
 
 
-def get_mozilla_versions(product, origin_version, release_minor=False):
+def get_firefox_versions():
+    versions = get_json_versions('Firefox')
     return {
-        'current': get_mozilla_version(product, origin_version, 'release', release_minor),
+            'current': versions['LATEST_FIREFOX_VERSION'],
+            'unreleased': [
+                versions['LATEST_FIREFOX_DEVEL_VERSION'],

[Vasily Kuznetsov, 2016/08/22 17:51:55]

It would be nice to give more friendly error messages in case the key is not
there, but all other browser version getters don't really take care of this.
Probably makes sense to follow suit here for consistency.

[Jon Sonesen, 2016/08/29 13:15:39]

I thought we decided not to so the build 'fails loudly' basically meaning we
need to go investigate why there is not the expected key in case they change
something.

+                versions['FIREFOX_AURORA'],
+                versions['FIREFOX_NIGHTLY'],
+            ]
+    }
+
+
+def get_thunderbird_versions():
+    versions = get_json_versions('Thunderbird')
+    firefox_versions = get_json_versions('Firefox')
+    return {
+        'current': versions['LATEST_THUNDERBIRD_VERSION'],
         'unreleased': [
-            get_mozilla_version(product, origin_version, 'beta'),
-            get_mozilla_version(product, origin_version, 'aurora'),
-            get_mozilla_version(product, origin_version, 'nightly'),
+            versions['LATEST_THUNDERBIRD_DEVEL_VERSION'],
+            versions['LATEST_THUNDERBIRD_ALPHA'],

[Vasily Kuznetsov, 2016/08/22 17:51:55]

I think this should be LATEST_THUNDERBIRD_ALPHA_VERSION

[Jon Sonesen, 2016/08/29 13:15:39]

you're right....dang haha

+            firefox_versions['FIREFOX_NIGHTLY'],
         ]
     }
 
-BROWSERS['firefox'] = lambda: get_mozilla_versions('Firefox', '37.0')
-BROWSERS['thunderbird'] = lambda: get_mozilla_versions('Thunderbird', '31.0', True)
-
-
-def get_seamonkey_version(origin_version, origin_build, channel, **kw):
-    return get_mozilla_version('SeaMonkey', origin_version, channel, True,
-                               'aus2-community', origin_build, 'version', **kw)
+BROWSERS['firefox'] = lambda: get_firefox_versions()
+BROWSERS['thunderbird'] = lambda: get_thunderbird_versions()
 
 
 def get_seamonkey_versions():
+    seamonkey_versions = get_json_versions('Seamonkey')
     versions = {
-        'current': get_seamonkey_version('2.32', '20150112201917', 'release'),
-        'unreleased': [get_seamonkey_version('2.32', '20150101215737', 'beta')]
+        'current': seamonkey_versions['LATEST_SEAMONKEY_VERSION'],
+        'unreleased': [
+            seamonkey_versions['LATEST_SEAMONKEY_MILESTONE_VERSION'],
+            seamonkey_versions['LATEST_SEAMONKEY_TESTING_VERSION'],
+            seamonkey_versions['LATEST_SEAMONKEY_DEVEL_VERSION'],
+        ]
     }
 
-    # Aurora and Nightly builds for Windows are permantently broken.
-    # Occasionally, builds for other platforms are broken as well.
-    # https://bugzilla.mozilla.org/show_bug.cgi?id=1086553
-    for channel in ('aurora', 'nightly'):
-        try:
-            version = get_seamonkey_version('2.32', '-', channel, platform='Linux_x86-gcc3')
-        except Exception:
-            continue
-        versions['unreleased'].append(version)
-
     return versions
 
 BROWSERS['seamonkey'] = get_seamonkey_versions
 
 
 def get_chrome_version(manifest):
     return manifest.getAttribute('version').split('.')[0]
 
 
 def get_chrome_versions():
@@ skipping 140 matching lines @@
         file.seek(0)
         json.dump(persistent_cache, file)
         file.truncate()
 
     if not versions['previous']:
         logging.warning("Couldn't determine previous browser version, "
                         'please set %s.previous in %s', browser, filename)
 
     cache[browser] = versions
     return versions