Issue 3246 - Strip empty locale files provided by Crowdin

cms/bin/translate.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2015 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
+import codecs
 import collections
 import io
 import itertools
 import json
 import logging
 import os
 import posixpath
-import shutil
 import sys
 import urllib
 import zipfile
 
 import urllib3
 
 import cms.utils
 from cms.sources import FileSource
 
 logger = logging.getLogger("cms.bin.translate")
@@ skipping 219 matching lines @@
     for member in archive.namelist():
       path, file_name = posixpath.split(member)
       ext = posixpath.splitext(file_name)[1]
       path_parts = path.split(posixpath.sep)
       locale, file_path = path_parts[0], path_parts[1:]
       if ext.lower() == ".json" and locale in inverted_required_locales:
         output_path = os.path.join(
           locale_path, inverted_required_locales[locale],
           *file_path + [file_name]
         )
-        with archive.open(member) as source_file, \
-             open(output_path, "wb") as target_file:
-          shutil.copyfileobj(source_file, target_file)
+        with archive.open(member) as source_file:
+          try:
+            locale_file_contents = json.load(source_file)
+          except ValueError:
+            continue

[Wladimir Palant, 2015/10/30 18:02:40]

I don't think that sweeping exceptions under the carpet is a good idea here - we
should at least warn if a file wasn't valid JSON.

[kzar, 2015/10/30 18:17:29]

Fair point, this might hide an unrelated issue. (I was trying to handle the case
of them returning a completely empty file, but I guess we can cross that bridge
when we come to it.) I think it's better to not not catch the exception rather
than log a warning, Done.

+          if len(locale_file_contents):
+            with codecs.open(output_path, "wb", "utf-8") as target_file:
+              json.dump(locale_file_contents, target_file, ensure_ascii=False,
+                        sort_keys=True, indent=2, separators=(",", ": "))
 
 def crowdin_sync(source_dir, crowdin_api_key):
   with FileSource(source_dir) as source:
     config = source.read_config()
     defaultlocale = config.get("general", "defaultlocale")
     crowdin_project_name = config.get("general", "crowdin-project-name")
 
     crowdin_api = CrowdinAPI(crowdin_api_key, crowdin_project_name)
 
     logger.info("Requesting project information...")
@@ skipping 35 matching lines @@
 if __name__ == "__main__":
   if len(sys.argv) < 3:
     print >>sys.stderr, "Usage: python -m cms.bin.translate www_directory crowdin_project_api_key [logging_level]"
     sys.exit(1)
 
   logging.basicConfig()
   logger.setLevel(sys.argv[3] if len(sys.argv) > 3 else logging.INFO)
 
   source_dir, crowdin_api_key = sys.argv[1:3]
   crowdin_sync(source_dir, crowdin_api_key)