Issue 5507 - Download only translated files from Crowdin

translations.py

 #!/usr/bin/env python
 
 import filecmp
 import os
 import re
 import shutil
 import sys
 import urllib2
+from lxml import etree
 from StringIO import StringIO
 from zipfile import ZipFile
 
 import buildtools.localeTools
 
 BASE_PATH = os.path.dirname(os.path.abspath(__file__))
 LOCALES_PATH = os.path.join(BASE_PATH, "mobile", "android", "base", "locales",
                             "adblockbrowser")
 PROJECT_URL = "http://api.crowdin.net/api/project/adblockbrowserandroid"
 DEFAULT_LOCALE = "en-US"
 
 
 def print_usage():
     print >>sys.stderr, "Usage: %s get API_KEY" % os.path.basename(sys.argv[0])
 
 
 def get_translations(api_key):
     # We could use buildtools.localeTools.getTranslations here if we were
     # submitting JSON wrappers for DTDs, the same way ABP for Firefox does it.
     # See https://issues.adblockplus.org/2911.
 
     result = urllib2.urlopen("%s/export?key=%s" % (PROJECT_URL, api_key))
     if result.read().find("<success") < 0:
         raise Exception("Export failed: " + result)
 
     result = urllib2.urlopen("%s/download/all.zip?key=%s" %
                              (PROJECT_URL, api_key))
     zip = ZipFile(StringIO(result.read()))
+    progress_for_translation = get_progress_for_translation_dictionary(api_key)
     for info in zip.infolist():
+        # skip languages without translations
+        index = info.filename.index('/')
+        if progress_for_translation[info.filename[:index]] == '0':
+            continue
+            
         if not info.filename.endswith(".dtd"):
             continue
 
         dir, file = os.path.split(info.filename)
         if not re.match(r"^[\w\-]+$", dir) or dir == DEFAULT_LOCALE:
             continue
 
         mapping = buildtools.localeTools.langMappingGecko
         for key, value in mapping.iteritems():
             if value == dir:
                 dir = key
 
         data = zip.open(info.filename).read()
         if not data:
             continue
 
         path = os.path.join(LOCALES_PATH, dir, file)
         if not os.path.exists(os.path.dirname(path)):
             os.makedirs(os.path.dirname(path))
         with open(path, "w") as file:
             file.write(data)
 
+def get_progress_for_translation_dictionary(api_key):
+    parser = etree.HTMLParser()
+    f = urllib2.urlopen('{}/status?key={}'.format(PROJECT_URL, api_key))
+    tree = etree.parse(f, parser)
+    languages = {}
+    for language in tree.iter():
+        languages.update({
+            language.findtext('code'): language.findtext('translated_progress')
+        })
+    return languages
 
 def remove_redundant_translations():
     default_locale_path = os.path.join(LOCALES_PATH, DEFAULT_LOCALE)
     for locale in os.listdir(LOCALES_PATH):
         if locale == DEFAULT_LOCALE:
             continue
         locale_path = os.path.join(LOCALES_PATH, locale)
         if not filecmp.dircmp(locale_path, default_locale_path).diff_files:
             shutil.rmtree(locale_path)
 
 if __name__ == "__main__":
     if len(sys.argv) < 3:
         print_usage()
         sys.exit(1)
 
     command, api_key = sys.argv[1:]
     if command != "get":
         print_usage()
         sys.exit(2)
 
     get_translations(api_key)
     remove_redundant_translations()