Issue 5751 - Removing legacy gecko support

localeTools.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import re
 import os
 import sys
 import codecs
 import json
 import urlparse
 import urllib
 import urllib2
 import mimetypes
 from StringIO import StringIO
 from ConfigParser import SafeConfigParser
 from zipfile import ZipFile
 from xml.parsers.expat import ParserCreate, XML_PARAM_ENTITY_PARSING_ALWAYS
-
-CROWDIN_AP_URL = 'https://api.crowdin.com/api/project'
 
 CROWDIN_LANG_MAPPING = {

[tlucas, 2017/10/04 11:48:39]

Note: result of rebasing

     'br': 'br-FR',
     'dsb': 'dsb-DE',
     'es': 'es-ES',
     'fur': 'fur-IT',
     'fy': 'fy-NL',
     'ga': 'ga-IE',
     'gu': 'gu-IN',
     'hsb': 'hsb-DE',
     'hy': 'hy-AM',
     'ml': 'ml-IN',
     'nn': 'nn-NO',
     'pa': 'pa-IN',
     'rm': 'rm-CH',
     'si': 'si-LK',
     'sv': 'sv-SE',
     'ur': 'ur-PK',
 }
 
+CROWDIN_AP_URL = 'https://api.crowdin.com/api/project'
+FIREFOX_RELEASES_URL = 'http://www.mozilla.org/en-US/firefox/all.html'
+FIREFOX_LP_URL = 'https://addons.mozilla.org/en-US/firefox/language-tools/'
+CHROMIUM_DEB_URL = 'https://packages.debian.org/sid/all/chromium-l10n/filelist'
+
 
 def crowdin_request(project_name, action, key, get={}, post_data=None,
                     headers={}, raw=False):
     """Perform a call to crowdin and raise an Exception on failure."""
     request = urllib2.Request(
         '{}/{}/{}?{}'.format(CROWDIN_AP_URL,
                              urllib.quote(project_name),
                              urllib.quote(action),
                              urllib.urlencode(dict(get, key=key, json=1))),
         post_data,
@@ skipping 44 matching lines @@
 
     file = codecs.open(path, 'wb', encoding='utf-8')
     json.dump(parsed, file, ensure_ascii=False, sort_keys=True, indent=2, separators=(',', ': '))
     file.close()
 
 
 def setupTranslations(localeConfig, projectName, key):
     locales = set()
 
     # Languages supported by Firefox
-    data = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.html').read()
+    data = urllib2.urlopen(FIREFOX_RELEASES_URL).read()
     for match in re.finditer(r'&lang=([\w\-]+)"', data):
         locales.add(match.group(1))
 
     # Languages supported by Firefox Language Packs
-    data = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/language-tools/').read()
+    data = urllib2.urlopen(FIREFOX_LP_URL).read()
     for match in re.finditer(r'<tr>.*?</tr>', data, re.S):
         if match.group(0).find('Install Language Pack') >= 0:
             match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
             if match2:
                 locales.add(match2.group(1))
 
     # Languages supported by Chrome (excluding es-419)
-    data = urllib2.urlopen('https://packages.debian.org/sid/all/chromium-l10n/filelist').read()
+    data = urllib2.urlopen(CHROMIUM_DEB_URL).read()
     for match in re.finditer(r'locales/(?!es-419)([\w\-]+)\.pak', data):
         locales.add(match.group(1))
 
     # We don't translate indvidual dialects of languages
     # other than English, Spanish, Portuguese and Chinese.
     for locale in list(locales):
         prefix = locale.split('-')[0]
         if prefix not in {'en', 'es', 'pt', 'zh'}:
             locales.remove(locale)
             locales.add(prefix)
 
     # Add languages with existing translations.
     locales.update(localeConfig['locales'])
 
     # Don't add the language we translate from as target translation.
     locales.remove(localeConfig['default_locale'].replace('_', '-'))
 
     # Convert to locales understood by Crowdin.
     locales = {CROWDIN_LANG_MAPPING.get(locale, locale) for locale in locales}
     allowed = {locale['crowdin_code'] for locale in
                crowdin_request(projectName, 'supported-languages', key)}

[tlucas, 2017/10/04 11:48:39]

Note: result of rebasing

     if not allowed.issuperset(locales):
         print "Warning, following locales aren't allowed by server: " + ', '.join(locales - allowed)
 
     locales = sorted(locales & allowed)

[tlucas, 2017/10/04 11:48:40]

Note: result of rebasing

     params = urllib.urlencode([('languages[]', locale) for locale in locales])
     crowdin_request(projectName, 'edit-project', key, post_data=params)
 
 
 def crowdin_prepare_upload(files):
     """Create a post body and matching headers, which Crowdin can handle."""
     boundary = '----------ThIs_Is_tHe_bouNdaRY_$'
     body = ''
     for name, data in files:
         body += (
@@ skipping 68 matching lines @@
                 newName = file
             else:
                 fileHandle = codecs.open(path, 'rb', encoding='utf-8')
                 data = json.dumps({file: {'message': fileHandle.read()}})
                 fileHandle.close()
                 newName = file + '.json'
 
             if data:
                 files.append((newName, data))
     if len(files):
         language = CROWDIN_LANG_MAPPING.get(locale, locale)

[tlucas, 2017/10/04 11:48:39]

Note: result of rebasing

         data, headers = crowdin_prepare_upload(files)
         crowdin_request(projectName, 'upload-translation', key,
                         {'language': language}, post_data=data,
                         headers=headers)
 
 
 def getTranslations(localeConfig, projectName, key):
     """Download all available translations from crowdin.
 
     Trigger crowdin to build the available export, wait for crowdin to
@@ skipping 14 matching lines @@
             continue
 
         dir, file = os.path.split(info.filename)
         if not re.match(r'^[\w\-]+$', dir) or dir == normalizedDefaultLocale:
             continue
         if file.count('.') == 1:
             origFile = file
         else:
             origFile = os.path.splitext(file)[0]
 
         for key, value in CROWDIN_LANG_MAPPING.iteritems():

[tlucas, 2017/10/04 11:48:39]

Note: result of rebasing

             if value == dir:
                 dir = key
 
         data = zip.open(info.filename).read()
         if data == '[]':
             continue
 
         if not dir in dirs:
             dirs[dir] = set()
         dirs[dir].add(origFile)
@@ skipping 13 matching lines @@
     # Remove any extra files
     for dir, files in dirs.iteritems():
         baseDir = os.path.join(localeConfig['base_path'], dir)
         if not os.path.exists(baseDir):
             continue
         for file in os.listdir(baseDir):
             path = os.path.join(baseDir, file)
             valid_extension = file.endswith('.json')
             if os.path.isfile(path) and valid_extension and not file in files:
                 os.remove(path)