Issue 2432 - Auto-generate browser versions on requirements page

globals/get_browser_versions.py

Index: globals/get_browser_versions.py
===================================================================
new file mode 100644
--- /dev/null
+++ b/globals/get_browser_versions.py
@@ -0,0 +1,180 @@
+import re
+import os
+import sys
+import json
+import urllib2
+import errno
+from xml.dom import minidom
+
+CACHE_FILENAME = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'cache', 'browsers.json')
+
+CHROME_UPDATE_XML = '''\
+<?xml version="1.0" encoding="UTF-8"?>
+<request protocol="3.0" ismachine="0">
+  <os platform="win" version="99" arch="x64"/>
+  <app appid="{4DC8B4CA-1BDA-483E-B5FA-D3C12E15B62D}">
+    <updatecheck/>
+  </app>
+  <app appid="{4DC8B4CA-1BDA-483E-B5FA-D3C12E15B62D}" ap="x64-beta-multi-chrome">
+    <updatecheck/>
+  </app>
+  <app appid="{4DC8B4CA-1BDA-483E-B5FA-D3C12E15B62D}" ap="x64-dev-multi-chrome">
+    <updatecheck/>
+  </app>
+</request>'''
+
+def get_mozilla_update(subdomain, product, version, build, channel):
+  response = urllib2.urlopen('https://%s.mozilla.org/update/3/%s/%s/%s/WINNT_x86-msvc/en-US/%s/-/default/default/update.xml?force=1' % (subdomain, product, version, build, channel))
+  try:
+    doc = minidom.parse(response)
+  finally:
+    response.close()
+
+  return doc.getElementsByTagName('update')[0]
+
+def get_mozilla_version(product, version, channel):
+  update = get_mozilla_update('aus4', product, version, '-', channel)
+  return update.getAttribute('appVersion').split('.')[0]
+
+def get_mozilla_versions(product, version):
+  return {
+    'current': get_mozilla_version(product, version, 'release'),
+    'unreleased': [
+      get_mozilla_version(product, version, 'beta'),
+      get_mozilla_version(product, version, 'aurora'),
+      get_mozilla_version(product, version, 'nightly'),
+    ]
+  }
+
+def get_seamonkey_version(channel, build):
+  update = get_mozilla_update('aus2-community', 'SeaMonkey', '2.32', build, channel)
+  return re.search(r'^^\d+\.\d+', update.getAttribute('version')).group(0)
+
+def get_chrome_version(manifest):
+  return manifest.getAttribute('version').split('.')[0]
+
+def get_opera_version(channel):
+  response = urllib2.urlopen('https://autoupdate.geo.opera.com/netinstaller/' + channel)
+  try:
+    spec = json.load(response)
+  finally:
+    response.close()
+
+  return re.search(r'\d+', spec['installer_filename']).group(0)
+
+def get_yandex_version(suffix):
+  response = urllib2.urlopen('https://api.browser.yandex.ru/update-info/browser/yandex%s/win-yandex.xml' % suffix)
+  try:
+    doc = minidom.parse(response)
+  finally:
+    response.close()
+
+  item = doc.getElementsByTagName('item')[0]
+  description = item.getElementsByTagName('description')[0]
+  return re.search(r'\d+\.\d+', description.firstChild.nodeValue).group(0)
+
+def open_cache_file():
+  flags = os.O_RDWR | os.O_CREAT
+  try:
+    fd = os.open(CACHE_FILENAME, flags)
+  except OSError as e:
+    if e.errno != errno.ENOENT:
+      raise
+    os.makedirs(os.path.dirname(CACHE_FILENAME))
+    fd = os.open(CACHE_FILENAME, flags)
+  return os.fdopen(fd, 'w+')
+
+class BrowserVersions:
+  def get_firefox_versions(self):
+    return get_mozilla_versions('Firefox', '37.0')
+
+  def get_thunderbird_versions(self):
+    return get_mozilla_versions('Thunderbird', '31.0')
+
+  def get_seamonkey_versions(self):
+    return {
+      'current': get_seamonkey_version('release', '20150112201917'),
+      'unreleased': [get_seamonkey_version('beta', '20150101215737')]
+    }
+
+  def get_chrome_versions(self):
+    response = urllib2.urlopen(urllib2.Request('https://tools.google.com/service/update2', CHROME_UPDATE_XML))
+    try:
+      doc = minidom.parse(response)
+    finally:
+      response.close()
+
+    manifests = doc.getElementsByTagName('manifest')
+    return {
+      'current': get_chrome_version(manifests[0]),
+      'unreleased': map(get_chrome_version, manifests[1:])
+    }
+
+  def get_opera_versions(self):
+    return {
+      'current': get_opera_version('Stable'),
+      'unreleased': [
+        get_opera_version('Beta'),
+        get_opera_version('Developer')
+      ]
+    }
+
+  def get_yandex_versions(self):
+    return {
+      'current': get_yandex_version(''),
+      'unreleased': [get_yandex_version('-beta')]
+    }
+
+  def get_versions(self, browser):
+    method = getattr(self, 'get_%s_versions' % browser)
+    exception = None
+    try:
+      versions = method()
+    except Exception as e:
+      exception = e
+
+    with open_cache_file() as file:
+      try:
+        cache = json.load(file)
+      except ValueError:
+        if file.tell() > 0:
+          raise
+        cache = {}
+
+      cached_versions = cache.get(browser)
+      if exception:
+        if not cached_versions:
+          raise exception
+
+        print >>sys.stderr, "Warning: Failed to get %s versions, falling back to cached versions" % browser
+        return cached_versions
+
+      # Determine previous version: If we recorded the version before and it
+      # changed since then, the old current version becomes the new previous
+      # version. If the version didn't change, use the cached previous version.
+      current = versions['current']
+      if cached_versions:
+        cached_current = cached_versions['current']
+        if cached_current != current:
+          versions['previous'] = cached_current
+        else:
+          cached_previous = cached_versions.get('previous')
+          if cached_previous:
+            versions['previous'] = cached_previous
+
+      # Remove duplicated from unreleased versions. Occasionally,
+      # different channels are on the same version, but we want
+      # to list each version only once.
+      unreleased = versions['unreleased']
+      previous = versions.get('previous')
+      for i, version in list(enumerate(unreleased))[::-1]:
+        if version == current or previous and version == previous:
+          del unreleased[i]
+
+      cache[browser] = versions
+      file.seek(0)
+      json.dump(cache, file)
+
+    return versions
+
+get_browser_versions = BrowserVersions().get_versions