Issue 7238 - Fix multi-locale builds

abb-build/transform_locales.py

+# This file is part of Adblock Plus
+# Copyright (C) 2006-present eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import os
+import re
+import shutil
+import sys
+
+_LOCALE_RE = re.compile("^([a-z]{2,3}(?:-[A-Z]{2})?)$")
+_VALUES_LOCALE_RE = re.compile("^values-([a-z]{2,3}(?:-r[A-Z]{2})?)$")
+
+_SEARCH_PROPS_RE = re.compile("^browser\.search\."
+                              "(defaultenginename|order\.).*$")
+_SHORTNAME_RE = re.compile("^<ShortName>(.*)</ShortName>$")
+
+_PROPERTY_FORMAT_RE = re.compile("^(([^=]*)=)(.*)$")
+_ENTITY_FORMAT_RE = re.compile("^(\s*<!ENTITY\s*([^\"\s]*)\s*\")(.*)(\">)$")
+_STRING_FORMAT_RE = re.compile(
+    "^(\s*<string name=\"([^\"]*)\">)(.*)(</string>)$")
+
+_MOZBUILD_PATH = os.path.join("python", "mozbuild")
+
+_CHROME_PATH = os.path.join("dist", "bin", "chrome")
+_RES_PATH = os.path.join("mobile", "android", "base", "res")
+_L10N_PATH = os.path.join("abb-build", "l10n")
+_LOCALES_PATH = os.path.join("mobile", "locales")
+_LISTJSON_PATH = os.path.join(_LOCALES_PATH, "search", "list.json")
+_GENERAL_SEARCHPLUGINS_PATH = os.path.join(_LOCALES_PATH, "searchplugins")
+
+_BROWSER_DIR = "browser"
+_REGION_PROPS_PATH = os.path.join(_BROWSER_DIR, "region.properties")
+_LOCALE_SEARCHPLUGINS_PATH = os.path.join(_BROWSER_DIR, "searchplugins")
+
+_APPSTRINGS_PROPS_PATH = os.path.join(_BROWSER_DIR, "appstrings.properties")
+_STRINGS_XML_PATH = "strings.xml"
+
+_DEFAULT_LOCALE = "en-US"
+_DEF_ENGINES = "visibleDefaultEngines"
+
+# Add Ecosia as secondary search engine.
+# See https://issues.adblockplus.org/ticket/5518
+_ECOSIA_ID = "ecosia"
+_ECOSIA_PATH = os.path.join(_GENERAL_SEARCHPLUGINS_PATH, "ecosia.xml")
+
+_SEARCH_ENGINE_ORDER_DEFAULT = [
+    "duckduckgo",
+    "yahoo",
+    "google",
+    "wikipedia",
+    "amazondotcom"]
+
+_SEARCH_ENGINE_ORDER_ECOSIA = [
+    "duckduckgo",
+    "yahoo",
+    "google",
+    _ECOSIA_ID,
+    "wikipedia",
+    "amazon"]
+
+_SEARCH_ENGINE_ORDER = {
+    "de": _SEARCH_ENGINE_ORDER_ECOSIA,
+    "en-GB": _SEARCH_ENGINE_ORDER_ECOSIA,
+    "en-US": _SEARCH_ENGINE_ORDER_ECOSIA,
+    "fr": _SEARCH_ENGINE_ORDER_ECOSIA,
+    "nl": _SEARCH_ENGINE_ORDER_ECOSIA,
+    "zh-CN": ["baidu",
+              "duckduckgo",
+              "yahoo",
+              "google",
+              "wikipedia",
+              "amazon"
+              ]
+}
+
+_FIREFOX_REPLACE_STR = "Firefox"
+_ABB_REPLACEMENT_STR = "Adblock Browser"
+
+# Some string values that contain Firefox such as 'Firefox Sync' shouldn't be
+# replaced, so we keep a list of ids that are exceptions
+_ENTITY_EXCEPTIONS = [
+    "overlay_no_synced_devices",
+    "home_remote_tabs_need_to_sign_in",
+    "home_remote_tabs_need_to_finish_migrating",
+    "home_remote_tabs_need_to_verify",
+    "syncBrand.fullName.label",
+    "sync.subtitle.connectlocation2.label",
+    "sync.subtitle.failmultiple.label",
+    "fxaccount_full_label",
+    "fxaccount_create_account_header2",
+    "fxaccount_create_account_policy_text2",
+    "fxaccount_status_header2",
+    "fxaccount_status_needs_finish_migrating",
+    "fxaccount_remove_account_dialog_title",
+    "fxaccount_remove_account_toast",
+    "fxaccount_account_type_label",
+]
+
+
+def _check_path_exists(path, logger):
+    if not os.path.exists(path):
+        logger.fatal("'%s' does not exist" % path)
+
+
+def _get_locales_from_path(path, locale_re):
+    locales = []
+    for dir_name in next(os.walk(path))[1]:
+        match = locale_re.match(dir_name)
+        if match:
+            locales.append(match.group(1))
+    locales.sort
+    return locales
+
+
+def _get_shortname_from_id(needle, engine_ids, engine_names):
+    """Fuzzy finds needle in engine_ids and returns ShortName"""
+    for engine in engine_ids:
+        if engine.startswith(needle):
+            return engine_names[engine]
+    return None
+
+
+def _replace_in_value(format_re, str, old, new, exceptions=[]):
+    match = format_re.match(str)
+    if match and match.lastindex > 2:
+        # The prefix contains all characters that precedes the value, including
+        # the id/key
+        str_value_prefix = match.group(1)
+        str_id = match.group(2)
+        str_value = match.group(3)
+        if str_id not in exceptions and old in str_value:
+            new_str = str_value_prefix + str_value.replace(old, new)
+            if match.lastindex == 4:
+                # The suffix contains all characters that succeeds the value
+                str_value_suffix = match.group(4)
+                new_str = new_str + str_value_suffix
+            return new_str
+    return None
+
+
+def _write_lines(filename, lines):
+    """Writes lines into file appending \\n"""
+    with open(filename, "w") as fd:
+        for l in lines:
+            fd.write("%s\n" % l)
+
+
+def _transform_locale(data, locale, project_dir, locale_path, logger):
+    logger.info("Processing locale '%s'..." % locale)
+
+    # Check for region.properties existence
+    region_file_path = os.path.join(locale_path, _REGION_PROPS_PATH)
+    _check_path_exists(region_file_path, logger)
+
+    # Check for appstrings.properties existence
+    appstrings_file_path = os.path.join(locale_path, _APPSTRINGS_PROPS_PATH)
+    _check_path_exists(appstrings_file_path, logger)
+
+    ecosia_dst = os.path.join(locale_path,
+                              _LOCALE_SEARCHPLUGINS_PATH, "ecosia.xml")
+
+    # Get whitelist and build regex
+    whitelist = _SEARCH_ENGINE_ORDER.get(locale,
+                                         _SEARCH_ENGINE_ORDER_DEFAULT)
+    white_re = re.compile("^(%s).*$" % "|".join(whitelist))
+
+    if _ECOSIA_ID in whitelist and not os.path.exists(ecosia_dst):
+        shutil.copyfile(os.path.join(project_dir, _ECOSIA_PATH), ecosia_dst)
+
+    all_engine_ids = []
+    engine_ids = []
+    removed_engine_ids = []
+
+    for item in data['locales'][locale]['default'][_DEF_ENGINES]:
+        all_engine_ids.append(item)
+        if len(item) > 0:
+            if white_re.match(item):
+                engine_ids.append(item)
+            else:
+                removed_engine_ids.append(item)
+
+    if _ECOSIA_ID in whitelist and _ECOSIA_ID not in all_engine_ids:
+        all_engine_ids.append(_ECOSIA_ID)
+        engine_ids.append(_ECOSIA_ID)
+
+    # Make sure we still have search engines left
+    if len(engine_ids) == 0:
+        logger.fatal("No search engines left over for '%s'" % locale)
+
+    data['locales'][locale]['default'][_DEF_ENGINES] = all_engine_ids
+
+    # 'Parse' XML to get matching 'ShortName' for all engine IDs
+    engine_names = {}
+    search_plugins_path = os.path.join(project_dir,
+                                       _GENERAL_SEARCHPLUGINS_PATH)
+    for eid in engine_ids[:]:
+        xml_file_path = os.path.join(search_plugins_path, "%s.xml" % eid)
+        if not os.path.exists(xml_file_path):
+            logger.info("Missing xml file for plugin %s. Searched in path %s" %
+                        (eid, xml_file_path))
+            engine_ids.remove(eid)
+            continue
+        short_name = None
+        with open(xml_file_path, "r") as fd:
+            for line in fd:
+                line = line.strip()
+                match = _SHORTNAME_RE.match(line)
+                if match:
+                    short_name = match.group(1).strip()
+
+        if not short_name:
+            logger.fatal("No ShortName defined for '%s' in '%s" %
+                         (eid, locale))
+        engine_names[eid] = short_name
+
+    logger.info("Removed search engine IDs: %s" %
+                ", ".join(removed_engine_ids))
+    logger.info("Remaining search engine IDs: %s" % ", ".join(engine_ids))
+
+    # Create search engine order with real engine names
+    engine_order = []
+    for eid in whitelist:
+        sn = _get_shortname_from_id(eid, engine_ids, engine_names)
+        if sn:
+            engine_order.append(sn)
+
+    logger.info("Resulting search engine ordered list: %s" %
+                (", ".join(engine_order)))
+
+    # Read region.properties and remove browser.search.* lines
+    props = []
+    with open(region_file_path, "r") as fd:
+        for line in fd:
+            line = line.rstrip("\r\n")
+            if not _SEARCH_PROPS_RE.match(line.strip()):
+                props.append(line)
+
+    # Append default search engine name
+    props.append("browser.search.defaultenginename=%s" % engine_order[0])
+
+    # Append search engine order
+    for i in range(0, len(engine_order)):
+        props.append("browser.search.order.%d=%s" % (i + 1, engine_order[i]))
+
+    # Write back region.properties
+    _write_lines(region_file_path, props)
+
+    # Replaces ocurrences of 'Firefox' by 'Adblock Browser' in
+    # 'appstrings.properties'
+    lines = []
+    replacement_count = 0
+
+    with open(appstrings_file_path, "r") as fd:
+        for line in fd:
+            line = line.rstrip("\r\n")
+            replacement = _replace_in_value(_PROPERTY_FORMAT_RE, line,
+                                            _FIREFOX_REPLACE_STR,
+                                            _ABB_REPLACEMENT_STR)
+            if replacement:
+                line = replacement
+                replacement_count += 1
+            lines.append(line)
+
+    # Apply changes to appstrings.properties
+    _write_lines(appstrings_file_path, lines)
+    logger.info("Replaced %d ocurrences of %s in 'appstrings.properties'" %
+                (replacement_count, _FIREFOX_REPLACE_STR))
+
+
+def _generate_browser_search(locale, locale_path, res_path, project_dir):
+    raw_dir = "raw" if locale == _DEFAULT_LOCALE else (
+        "raw-%s" % locale.replace("-", "-r"))
+
+    browser_path = os.path.join(locale_path, _BROWSER_DIR)
+    browsersearch_file_path = os.path.join(res_path, raw_dir,
+                                           "browsersearch.json")
+
+    sys.path.append(os.path.join(project_dir, _MOZBUILD_PATH))
+    import mozbuild.action.generate_browsersearch as generate_browsersearch
+
+    # Call generate_browsersearch.py script to regenerate
+    # res/raw-LOCALE/browsersearch.json with the updated search engines
+    generate_browsersearch.main(["--verbose", "--srcdir", browser_path,
+                                 browsersearch_file_path])
+
+
+def _generate_search_json(locale, locale_path, project_dir):
+    script_path = os.path.join(project_dir, "python", "mozbuild",
+                               "mozbuild", "action", "generate_searchjson.py")
+    list_json_path = os.path.join(project_dir, _LISTJSON_PATH)
+    searchplugins_path = os.path.join(locale_path, _BROWSER_DIR,
+                                      "searchplugins", "list.json")
+
+    import subprocess as s
+    # Call generate_searchjson.py script
+    s.check_call(['python', script_path, list_json_path,
+                 locale, searchplugins_path])
+
+
+def _transform_values_locale(locale, path, logger):
+    logger.info("Processing values-%s..." % locale)
+
+    # Check for strings.xml existence
+    strings_file_path = os.path.join(path, _STRINGS_XML_PATH)
+    _check_path_exists(strings_file_path, logger)
+
+    # Replaces ocurrences of 'Firefox' by 'Adblock Browser' in 'strings.xml'
+    lines = []
+    replacement_count = 0
+
+    with open(strings_file_path, "r") as fd:
+        for line in fd:
+            line = line.rstrip("\r\n")
+            replacement = _replace_in_value(_ENTITY_FORMAT_RE, line,
+                                            _FIREFOX_REPLACE_STR,
+                                            _ABB_REPLACEMENT_STR,
+                                            _ENTITY_EXCEPTIONS)
+            if replacement:
+                line = replacement
+                replacement_count += 1
+            else:
+                replacement = _replace_in_value(_STRING_FORMAT_RE, line,
+                                                _FIREFOX_REPLACE_STR,
+                                                _ABB_REPLACEMENT_STR)
+                if replacement:
+                    line = replacement
+                    replacement_count += 1
+            lines.append(line)
+
+    # Apply changes to strings.xml
+    _write_lines(strings_file_path, lines)
+    logger.info("Replaced %d ocurrences of %s in 'strings.xml'" %
+                (replacement_count, _FIREFOX_REPLACE_STR))
+
+
+class MinimalLogger:
+    def info(self, s):
+        print "INFO: %s" % s
+
+    def error(self, s):
+        print "ERROR: %s" % s
+
+    def fatal(self, s):
+        print "FATAL: %s" % s
+        exit(1)
+
+
+def transform_locales(project_dir, obj_dir, logger=MinimalLogger()):
+    chrome_path = os.path.join(obj_dir, _CHROME_PATH)
+    _check_path_exists(chrome_path, logger)
+
+    res_path = os.path.join(obj_dir, _RES_PATH)
+    _check_path_exists(res_path, logger)
+
+    list_json_path = os.path.join(project_dir, _LISTJSON_PATH)
+    _check_path_exists(list_json_path, logger)
+
+    locales = _get_locales_from_path(chrome_path, _LOCALE_RE)
+    values_locales = _get_locales_from_path(res_path, _VALUES_LOCALE_RE)
+
+    locales_found_msg = "Found %d locales in %s"
+    logger.info(locales_found_msg % (len(locales), chrome_path))
+    logger.info(locales_found_msg % (len(values_locales), res_path))
+
+    # open the Mozilla list of search engines, put it into a buffer and
+    # close the JSON file after reading
+    with open(list_json_path, 'r') as json_file:
+        data = json.load(json_file)
+
+        # set default search engine order
+        data['default'][_DEF_ENGINES] = _SEARCH_ENGINE_ORDER_DEFAULT
+
+        for locale in locales:
+            locale_path = os.path.join(chrome_path, locale, "locale", locale)
+            if os.path.exists(locale_path):
+
+                # Mozilla default list does not contain locale bn-BD,
+                # so we create it and use the values from locale bn-IN
+                if locale == 'bn-BD':
+                    data['locales'].update({locale: {'default':
+                                           {_DEF_ENGINES:
+                        data['locales']['bn-IN']['default'][_DEF_ENGINES]}}})
+                # Mozilla default list does not contain locale wo, so we use
+                # the default order. In case they will not support any other
+                # locales in the future, we want the build to fail, to decide
+                #  which order to use
+                elif locale == 'wo':
+                    data['locales'].update({locale: {'default':
+                                           {_DEF_ENGINES:
+                                            _SEARCH_ENGINE_ORDER_DEFAULT}}})
+
+                _transform_locale(data, locale, project_dir, locale_path,
+                                  logger)
+                _generate_browser_search(locale, locale_path, res_path,
+                                         project_dir)
+            else:
+                logger.error("Missing folder for locale '%s' in path: %s" %
+                             (locale, locale_path))
+
+        # Save changes to list.json
+        with open(list_json_path, 'w') as outfile:
+            json.dump(data, outfile, indent=4, sort_keys=True)
+
+        # Generate search.json for each locale
+        for locale in locales:
+            locale_path = os.path.join(chrome_path, locale, "locale", locale)
+            _generate_search_json(locale, locale_path, project_dir)
+
+        for locale in values_locales:
+            locale_path = os.path.join(res_path, "values-" + locale)
+            _transform_values_locale(locale, locale_path, logger)