Issue 5777 - Update crowdin interface

localeTools.py

Index: localeTools.py
diff --git a/localeTools.py b/localeTools.py
index 140f22c1f7bb8298ce12bf7c1d4547a47de346a4..5f9191812cce89449bd0a813e220ea92ab9736e2 100644
--- a/localeTools.py
+++ b/localeTools.py
@@ -7,6 +7,7 @@ import os
 import sys
 import codecs
 import json
+import urlparse
 import urllib
 import urllib2
 from StringIO import StringIO
@@ -92,6 +93,41 @@ chromeLocales = [
     'zh-TW',
 ]
 
+CROWDIN_AP_URL = 'https://api.crowdin.com/api/project/{}/{}'
+
+
+def crowdin_url(project_name, action, key, get={}):
+    """Create a valid url for a crowdin endpoint."""
+    url = CROWDIN_AP_URL.format(project_name, action)
+    get['key'] = key
+
+    scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
+
+    query = dict(urlparse.parse_qsl(query))
+    query.update(get)
+
+    return urlparse.urlunparse((
+        scheme, netloc, path, params, urllib.urlencode(query), fragment
+    ))
+
+
+def crowdin_post(post_data, project_name, action, key, get={}, raises=False):
+    """Perform a call to crowdin and raise an Exception on failure."""
+
+    result = urllib2.urlopen(crowdin_url(project_name, action, key, get),
+                             data=post_data).read()
+
+    if raises and result.find('<success') < 0:
+        raise Exception(
+            'Server indicated that the operation was not successful\n' + result
+        )
+
+    return result
+
+
+def crowdin_get(*args, **kwargs):

[Vasily Kuznetsov, 2017/09/26 13:11:36]

This get via post with None as a first argument looks kind of sloppy. I mildly
prefer renaming `crowdin_post` to `crowdin_request` and then creating an
additional `crowdin_post` that delegates to it (or is an alias for it). It would
be functionally equivalent to the current code as far as interpreter is
concerned, but would express the intention better for the human readers of the
code.

[tlucas, 2017/09/26 14:40:29]

Done.

+    return crowdin_post(None, *args, **kwargs)
+
 
 class OrderedDict(dict):
     def __init__(self):
@@ -297,20 +333,19 @@ def setupTranslations(localeConfig, projectName, key):
                     locales.add(mapLocale('BCP-47', match2.group(1)))
 
     allowed = set()
-    allowedLocales = json.load(urllib2.urlopen(
-        'https://crowdin.com/languages/languages_list?callback='
-    ))
+    allowedLocales = json.loads(crowdin_get(projectName, 'supported-languages',

[Vasily Kuznetsov, 2017/09/26 13:11:36]

For extra style points you could check if there's `{'json': 1}` in parameters in
`crowdin_post` and automatically `json.loads` if there is. This would make this
code even leaner. Totally up to you if you want to do it? :D

[tlucas, 2017/09/26 14:40:29]

Nice idea, done.

+                                            key, {'json': 1}))
+
     for locale in allowedLocales:
-        allowed.add(locale['code'])
+        allowed.add(locale['crowdin_code'])
     if not allowed.issuperset(locales):
         print "Warning, following locales aren't allowed by server: " + ', '.join(locales - allowed)
 
     locales = list(locales & allowed)
     locales.sort()
     params = urllib.urlencode([('languages[]', locale) for locale in locales])
-    result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/edit-project?key=%s' % (projectName, key), params).read()
-    if result.find('<success') < 0:
-        raise Exception('Server indicated that the operation was not successful\n' + result)
+
+    crowdin_post(params, projectName, 'edit-project', key, raises=True)
 
 
 def postFiles(files, url):
@@ -334,7 +369,7 @@ def postFiles(files, url):
 
 
 def updateTranslationMaster(localeConfig, metadata, dir, projectName, key):
-    result = json.load(urllib2.urlopen('http://api.crowdin.net/api/project/%s/info?key=%s&json=1' % (projectName, key)))
+    result = json.loads(crowdin_get(projectName, 'info', key, {'json': 1}))
 
     existing = set(map(lambda f: f['name'], result['files']))
     add = []
@@ -362,12 +397,16 @@ def updateTranslationMaster(localeConfig, metadata, dir, projectName, key):
                     add.append((newName, data))
 
     if len(add):
-        titles = urllib.urlencode([('titles[%s]' % name, re.sub(r'\.json', '', name)) for name, data in add])
-        postFiles(add, 'http://api.crowdin.net/api/project/%s/add-file?key=%s&type=chrome&%s' % (projectName, key, titles))
+        data = {'titles[{}]'.format(name): re.sub(r'\.json', '', name)
+                for name, data in add}
+        data['type'] = 'chrome'
+        postFiles(add, crowdin_url(projectName, 'add-file', key, data))
     if len(update):
-        postFiles(update, 'http://api.crowdin.net/api/project/%s/update-file?key=%s' % (projectName, key))
+        postFiles(update, crowdin_url(projectName, 'update-file', key))
     for file in existing:
-        result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/delete-file?key=%s&file=%s' % (projectName, key, file)).read()
+        result = urllib2.urlopen(

[Vasily Kuznetsov, 2017/09/26 13:11:36]

Isn't this `crowdin_get`?

[tlucas, 2017/09/26 14:40:29]

It is, missed that :) Done.

+            crowdin_url(projectName, 'delete-file', key, {'file': file})
+        ).read()
         if result.find('<success') < 0:
             raise Exception('Server indicated that the operation was not successful\n' + result)
 
@@ -392,17 +431,17 @@ def uploadTranslations(localeConfig, metadata, dir, locale, projectName, key):
             if data:
                 files.append((newName, data))
     if len(files):
-        postFiles(files, 'http://api.crowdin.net/api/project/%s/upload-translation?key=%s&language=%s' % (
-            projectName, key, mapLocale(localeConfig['name_format'], locale))
-        )
+        language = mapLocale(localeConfig['name_format'], locale)
+        url = crowdin_url(projectName, 'upload-translation', key,
+                          {'language': language})
+        postFiles(files, url)
 
 
 def getTranslations(localeConfig, projectName, key):
-    result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/export?key=%s' % (projectName, key)).read()
-    if result.find('<success') < 0:
-        raise Exception('Server indicated that the operation was not successful\n' + result)
+    # let crowdin build the project
+    crowdin_get(projectName, 'export', key, raises=True)
 
-    result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/download/all.zip?key=%s' % (projectName, key)).read()
+    result = crowdin_get(projectName, 'download/all.zip', key)
     zip = ZipFile(StringIO(result))
     dirs = {}