Issue 4836 - Have the setuptrans command use the current Crowdin API

localeTools.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import re
 import os
 import sys
 import codecs
 import json
 import urllib
@@ skipping 300 matching lines @@
         for match in re.finditer(r'&lang=([\w\-]+)"', firefoxLocales):
             locales.add(mapLocale('BCP-47', match.group(1)))
         langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/language-tools/').read()
         for match in re.finditer(r'<tr>.*?</tr>', langPacks, re.S):
             if match.group(0).find('Install Language Pack') >= 0:
                 match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
                 if match2:
                     locales.add(mapLocale('BCP-47', match2.group(1)))
 
     allowed = set()
-    allowedLocales = urllib2.urlopen('http://crowdin.net/page/language-codes').read()
-    for match in re.finditer(r'<tr>\s*<td\b[^<>]*>([\w\-]+)</td>', allowedLocales, re.S):
+    allowedLocales = urllib2.urlopen(
+        'https://crowdin.com/languages/languages_list?callback=foo'

[Wladimir Palant, 2017/01/24 11:36:23]

Please use https://crowdin.com/languages/languages_list?callback=, the URL will
give you proper JSON if the callback parameter is empty.

[kzar, 2017/01/25 03:45:04]

Ah cool, didn't think to try that. Done.

+    ).read()
+    for match in re.finditer(r'"code"\s*:\s*"([\w\-]+)"', allowedLocales, re.S):

[Wladimir Palant, 2017/01/24 11:36:23]

Please don't use regular expressions to parse JSON, it can be parsed properly.

[kzar, 2017/01/25 03:45:04]

(I would have already if I had figured out the way to avoid it being wrapped in
the callback.) Done.

         allowed.add(match.group(1))
     if not allowed.issuperset(locales):
         print "Warning, following locales aren't allowed by server: " + ', '.join(locales - allowed)
 
     locales = list(locales & allowed)
     locales.sort()
     params = urllib.urlencode([('languages[]', locale) for locale in locales])
     result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/edit-project?key=%s' % (projectName, key), params).read()
     if result.find('<success') < 0:
         raise Exception('Server indicated that the operation was not successful\n' + result)
@@ skipping 149 matching lines @@
 
     # Remove any extra files
     for dir, files in dirs.iteritems():
         baseDir = os.path.join(localeConfig['base_path'], dir)
         if not os.path.exists(baseDir):
             continue
         for file in os.listdir(baseDir):
             path = os.path.join(baseDir, file)
             if os.path.isfile(path) and (file.endswith('.json') or file.endswith('.properties') or file.endswith('.dtd')) and not file in files:
                 os.remove(path)