Issue 5763 - Target languages supported by Firefox

localeTools.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import re
 import os
 import sys
 import codecs
 import json
 import urlparse
 import urllib
 import urllib2
 import mimetypes
 from StringIO import StringIO
 from ConfigParser import SafeConfigParser
 from zipfile import ZipFile
 from xml.parsers.expat import ParserCreate, XML_PARAM_ENTITY_PARSING_ALWAYS
-
-CROWDIN_AP_URL = 'https://api.crowdin.com/api/project'
 
 CROWDIN_LANG_MAPPING = {
     'br': 'br-FR',
     'dsb': 'dsb-DE',
     'es': 'es-ES',
     'fur': 'fur-IT',
     'fy': 'fy-NL',
     'ga': 'ga-IE',
     'gu': 'gu-IN',
     'hsb': 'hsb-DE',
     'hy': 'hy-AM',
     'ml': 'ml-IN',
     'nn': 'nn-NO',
     'pa': 'pa-IN',
     'rm': 'rm-CH',
     'si': 'si-LK',
     'sv': 'sv-SE',
     'ur': 'ur-PK',
 }
 
+CROWDIN_AP_URL = 'https://api.crowdin.com/api/project'
+FIREFOX_RELEASES_URL = 'http://www.mozilla.org/en-US/firefox/all.html'
+FIREFOX_LP_URL = 'https://addons.mozilla.org/en-US/firefox/language-tools/'
+CHROMIUM_DEB_URL = 'https://packages.debian.org/sid/all/chromium-l10n/filelist'
+
 
 def crowdin_request(project_name, action, key, get={}, post_data=None,
                     headers={}, raw=False):
     """Perform a call to crowdin and raise an Exception on failure."""
     request = urllib2.Request(
         '{}/{}/{}?{}'.format(CROWDIN_AP_URL,
                              urllib.quote(project_name),
                              urllib.quote(action),
                              urllib.urlencode(dict(get, key=key, json=1))),
         post_data,
@@ skipping 186 matching lines @@
 
     file = codecs.open(path, 'wb', encoding='utf-8')
     json.dump(parsed, file, ensure_ascii=False, sort_keys=True, indent=2, separators=(',', ': '))
     file.close()
 
 
 def setupTranslations(localeConfig, projectName, key):
     locales = set()
 
     # Languages supported by Firefox
-    data = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.html').read()

[Vasily Kuznetsov, 2017/10/05 20:26:03]

What do you think about moving all these long URLs into constants at the
beginning of the file? This way we'd get rid of the hardcoded strings and of the
lines that are too long in this function at the same time.

[Sebastian Noack, 2017/10/05 20:58:38]

Done.

+    data = urllib2.urlopen(FIREFOX_RELEASES_URL).read()
     for match in re.finditer(r'&lang=([\w\-]+)"', data):
         locales.add(match.group(1))
 
     # Languages supported by Firefox Language Packs
-    data = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/language-tools/').read()
+    data = urllib2.urlopen(FIREFOX_LP_URL).read()
     for match in re.finditer(r'<tr>.*?</tr>', data, re.S):
         if match.group(0).find('Install Language Pack') >= 0:
             match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
             if match2:
                 locales.add(match2.group(1))
 
     # Languages supported by Chrome (excluding es-419)
-    data = urllib2.urlopen('https://packages.debian.org/sid/all/chromium-l10n/filelist').read()
+    data = urllib2.urlopen(CHROMIUM_DEB_URL).read()
     for match in re.finditer(r'locales/(?!es-419)([\w\-]+)\.pak', data):
         locales.add(match.group(1))
 
     # We don't translate indvidual dialects of languages
     # other than English, Spanish, Portuguese and Chinese.
     for locale in list(locales):
         prefix = locale.split('-')[0]
         if prefix not in {'en', 'es', 'pt', 'zh'}:
             locales.remove(locale)
             locales.add(prefix)
@@ skipping 177 matching lines @@
 
     # Remove any extra files
     for dir, files in dirs.iteritems():
         baseDir = os.path.join(localeConfig['base_path'], dir)
         if not os.path.exists(baseDir):
             continue
         for file in os.listdir(baseDir):
             path = os.path.join(baseDir, file)
             if os.path.isfile(path) and (file.endswith('.json') or file.endswith('.properties') or file.endswith('.dtd')) and not file in files:
                 os.remove(path)