Issue 3246 - Strip empty locale files provided by Crowdin

cms/bin/translate.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2015 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
+import codecs
 import collections
 import io
 import itertools
 import json
 import logging
 import os
 import posixpath
-import re
 import sys
 import urllib
 import zipfile
 
 import urllib3
 
 import cms.utils
 from cms.sources import FileSource
 
 logger = logging.getLogger("cms.bin.translate")
@@ skipping 209 matching lines @@
   with zipfile.ZipFile(io.BytesIO(response.data), "r") as archive:
     locale_path = os.path.join(source_dir, "locales")
     # First clear existing translation files
     for root, dirs, files in os.walk(locale_path, topdown=True):
       if root == locale_path:
         dirs[:] = [d for d in dirs if d in required_locales]
       for f in files:
         if f.lower().endswith(".json"):
           os.remove(os.path.join(root, f))
     # Then extract the new ones in place
-    non_empty_file_regexp = re.compile(r"[^\s{}[\]]")
     for member in archive.namelist():
       path, file_name = posixpath.split(member)
       ext = posixpath.splitext(file_name)[1]
       path_parts = path.split(posixpath.sep)
       locale, file_path = path_parts[0], path_parts[1:]
       if ext.lower() == ".json" and locale in inverted_required_locales:
         output_path = os.path.join(
           locale_path, inverted_required_locales[locale],
           *file_path + [file_name]
         )
         with archive.open(member) as source_file:
-          locale_file_contents = source_file.read()
-          if re.search(non_empty_file_regexp, locale_file_contents):

[Wladimir Palant, 2015/10/29 19:12:35]

I don't think that using regular expressions on JSON data is a good idea. It
makes more sense to check len(json.load(source_file)), this will also ensure
validity of the data. Positive side-effect: we can dump the JSON data formatted
the way we like it (usually means these parameters: ensure_ascii=False,
sort_keys=True, indent=2, separators=(',', ': ')).

Having to handle UTF-8 here makes things a bit more complicated unfortunately,
the simplest solution should be using the codecs module (io module is sadly
incompatible to zipfile in Python 2).

[kzar, 2015/10/30 12:03:03]

Done.

-            with open(output_path, "wb") as target_file:
-              target_file.write(locale_file_contents)
+          locale_file_contents = json.load(source_file)
+          if len(locale_file_contents):
+            with codecs.open(output_path, "wb", "utf-8") as target_file:
+              json.dump(locale_file_contents, target_file, ensure_ascii=False,
+                        sort_keys=True, indent=2, separators=(",", ": "))
 
 def crowdin_sync(source_dir, crowdin_api_key):
   with FileSource(source_dir) as source:
     config = source.read_config()
     defaultlocale = config.get("general", "defaultlocale")
     crowdin_project_name = config.get("general", "crowdin-project-name")
 
     crowdin_api = CrowdinAPI(crowdin_api_key, crowdin_project_name)
 
     logger.info("Requesting project information...")
@@ skipping 35 matching lines @@
 if __name__ == "__main__":
   if len(sys.argv) < 3:
     print >>sys.stderr, "Usage: python -m cms.bin.translate www_directory crowdin_project_api_key [logging_level]"
     sys.exit(1)
 
   logging.basicConfig()
   logger.setLevel(sys.argv[3] if len(sys.argv) > 3 else logging.INFO)
 
   source_dir, crowdin_api_key = sys.argv[1:3]
   crowdin_sync(source_dir, crowdin_api_key)