Issue 7042 - [XTM Integration] Solve UnicodeDecodeErrors occuring

tests/test_xtm_translations_utils.py

 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-present eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
+from __future__ import unicode_literals
+
 import json
 import os
 
 import pytest
 
 from cms.translations.xtm import utils
 import cms.translations.xtm.constants as const
 from cms.sources import FileSource
 from cms.translations.xtm.xtm_api import XTMCloudAPI
 from tests.utils import exception_test
@@ skipping 186 matching lines @@
 
 
 @pytest.mark.parametrize('path', ['de/test.json', 'de/dir1/dir2/test.json'])
 def test_write_data(toydir, path):
     """Test if writing data to files works as expected."""
     data = bytes(json.dumps({'a': 'b'}))
 
     utils.write_to_file(data, str(toydir.join(path)))
 
     assert toydir.join(path).read('rb') == data
+
+
+def test_extract_unicode_strings(temp_site):
+    """Test correct extraction of unicode strings for translation.
+
+    Testing for presence of "Ͷ" - \u0376.

[Vasily Kuznetsov, 2018/10/12 09:39:51]

Doesn't this line cause an error for you? I get:

.tox/py27/lib/python2.7/site-packages/_pytest/python.py:482: in
_importtestmodule
    mod = self.fspath.pyimport(ensuresyspath=importmode)
.tox/py27/lib/python2.7/site-packages/py/_path/local.py:668: in pyimport
    __import__(modname)
E     File
"/Users/vkuznetsov/Documents/prog/hg.adblockplus.org/cms/tests/test_xtm_translations_utils.py",
line 229
E   SyntaxError: Non-ASCII character '\xcd' in file
/Users/vkuznetsov/Documents/prog/hg.adblockplus.org/cms/tests/test_xtm_translations_utils.py
on line 230, but no encoding declared; see http://python.org/dev/peps/pep-0263/
for details

I think the right approach here would be to remove the "Ͷ" from the docstring
instead of declaring encoding -- it's better to stick to ASCII in source code
unless you really have to.

[Tudor Avram, 2018/10/12 10:37:02]

Huh, ok. It didn't give me an error, but well,  I guess it makes sense to have
ASCII only characters in the codebase. Changing it now.

+    """
+    with FileSource(temp_site) as fs:
+        strings = utils.extract_strings(fs)
+
+    assert '\u0376' in strings['translate-unicode']['simple']['message']