Issue 7386 - Default search engine is not preserved during upgrade

abb-build/transform_locales.py

 # This file is part of Adblock Plus
 # Copyright (C) 2006-present eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
@@ skipping 14 matching lines @@
 _SEARCH_PROPS_RE = re.compile("^browser\.search\."
                               "(defaultenginename|order\.).*$")
 _SHORTNAME_RE = re.compile("^<ShortName>(.*)</ShortName>$")
 
 _PROPERTY_FORMAT_RE = re.compile("^(([^=]*)=)(.*)$")
 _ENTITY_FORMAT_RE = re.compile("^(\s*<!ENTITY\s*([^\"\s]*)\s*\")(.*)(\">)$")
 _STRING_FORMAT_RE = re.compile(
     "^(\s*<string name=\"([^\"]*)\">)(.*)(</string>)$")
 
 _MOZBUILD_PATH = os.path.join("python", "mozbuild")
-
+_GENERATED_PATH = os.path.join("abb-build", "generated")
 _CHROME_PATH = os.path.join("dist", "bin", "chrome")
 _RES_PATH = os.path.join("mobile", "android", "base", "res")
-_L10N_PATH = os.path.join("abb-build", "l10n")
 _LOCALES_PATH = os.path.join("mobile", "locales")
-_LISTJSON_PATH = os.path.join(_LOCALES_PATH, "search", "list.json")
+_LIST_JSON_FILE = "list.json"
+_LIST_JSON_PATH = os.path.join(_LOCALES_PATH, "search", _LIST_JSON_FILE)
+_GENERATED_LIST_JSON_PATH = os.path.join(_GENERATED_PATH, _LIST_JSON_FILE)
 _GENERAL_SEARCHPLUGINS_PATH = os.path.join(_LOCALES_PATH, "searchplugins")
-
 _BROWSER_DIR = "browser"
 _REGION_PROPS_PATH = os.path.join(_BROWSER_DIR, "region.properties")
 _LOCALE_SEARCHPLUGINS_PATH = os.path.join(_BROWSER_DIR, "searchplugins")
-
 _APPSTRINGS_PROPS_PATH = os.path.join(_BROWSER_DIR, "appstrings.properties")
-_STRINGS_XML_PATH = "strings.xml"
+_STRINGS_XML_FILE = "strings.xml"
 
 _DEFAULT_LOCALE = "en-US"
 _DEF_ENGINES = "visibleDefaultEngines"
 
 # Add Ecosia as secondary search engine.
 # See https://issues.adblockplus.org/ticket/5518
 _ECOSIA_ID = "ecosia"
 _ECOSIA_PATH = os.path.join(_GENERAL_SEARCHPLUGINS_PATH, "ecosia.xml")
 
 _SEARCH_ENGINE_ORDER_DEFAULT = [
-    "duckduckgo",
+    "ddg",

[diegocarloslima, 2019/03/29 12:51:01]

The id changed from duckduckgo to ddg

     "yahoo",
     "google",
     "wikipedia",
-    "amazondotcom"]
+    "amazon"]

[diegocarloslima, 2019/03/29 12:51:01]

Took the opportunity to also fix this, otherwise amazon wouldn't be included in
some locales that don't match amazondotcom (e.g. for 'ja' locale is 'amazon-jp')

 
 _SEARCH_ENGINE_ORDER_ECOSIA = [
-    "duckduckgo",
+    "ddg",
     "yahoo",
     "google",
     _ECOSIA_ID,
     "wikipedia",
     "amazon"]
 
 _SEARCH_ENGINE_ORDER = {
     "de": _SEARCH_ENGINE_ORDER_ECOSIA,
     "en-GB": _SEARCH_ENGINE_ORDER_ECOSIA,
     "en-US": _SEARCH_ENGINE_ORDER_ECOSIA,
@@ skipping 73 matching lines @@
     return None
 
 
 def _write_lines(filename, lines):
     """Writes lines into file appending \\n"""
     with open(filename, "w") as fd:
         for l in lines:
             fd.write("%s\n" % l)
 
 
-def _transform_locale(data, locale, project_dir, locale_path, logger):
-    logger.info("Processing locale '%s'..." % locale)
-
+def _transform_region_props(locale, locale_path, engines_ordered, logger):
     # Check for region.properties existence
     region_file_path = os.path.join(locale_path, _REGION_PROPS_PATH)
     _check_path_exists(region_file_path, logger)
 
-    # Check for appstrings.properties existence
-    appstrings_file_path = os.path.join(locale_path, _APPSTRINGS_PROPS_PATH)
-    _check_path_exists(appstrings_file_path, logger)
-
-    ecosia_dst = os.path.join(locale_path,
-                              _LOCALE_SEARCHPLUGINS_PATH, "ecosia.xml")
-
+    # Read region.properties and remove browser.search.* lines
+    lines = []
+    with open(region_file_path, "r") as fd:
+        for line in fd:
+            line = line.rstrip("\r\n")
+            if not _SEARCH_PROPS_RE.match(line.strip()):
+                lines.append(line)
+
+    # Append default search engine name
+    lines.append("browser.search.defaultenginename=%s" % engines_ordered[0])
+
+    # Append search engine order
+    for i in range(0, len(engines_ordered)):
+        lines.append("browser.search.order.%d=%s" % (i + 1,
+                                                     engines_ordered[i]))
+    # Write back region.properties
+    _write_lines(region_file_path, lines)
+    logger.info("Resulting ordered search engines for %s: %s" %
+                (locale, ", ".join(engines_ordered)))
+
+
+def _generate_browser_search(project_dir, res_path, locale, locale_path,
+                             logger):
+

[anton, 2019/04/05 12:42:24]

unrequired line?

+    raw_dir = "raw" if locale == _DEFAULT_LOCALE else (
+        "raw-%s" % locale.replace("-", "-r"))
+
+    browser_path = os.path.join(locale_path, _BROWSER_DIR)
+    browsersearch_file_path = os.path.join(res_path, raw_dir,
+                                           "browsersearch.json")
+
+    sys.path.append(os.path.join(project_dir, _MOZBUILD_PATH))
+    import mozbuild.action.generate_browsersearch as generate_browsersearch
+
+    # Call generate_browsersearch.py script to regenerate
+    # res/raw-LOCALE/browsersearch.json with the updated search engines
+    generate_browsersearch.main(["--verbose", "--srcdir", browser_path,
+                                 browsersearch_file_path])
+
+    logger.info("Generated browsersearch.json for %s" % locale)
+
+
+def _transform_search_locale(project_dir, data, locale, locale_path, logger):
     # Get whitelist and build regex
-    whitelist = _SEARCH_ENGINE_ORDER.get(locale,
-                                         _SEARCH_ENGINE_ORDER_DEFAULT)
+    whitelist = _SEARCH_ENGINE_ORDER.get(locale, _SEARCH_ENGINE_ORDER_DEFAULT)
     white_re = re.compile("^(%s).*$" % "|".join(whitelist))
 
-    if _ECOSIA_ID in whitelist and not os.path.exists(ecosia_dst):
+    all_engine_ids = data['locales'][locale]['default'][_DEF_ENGINES]
+    engine_ids = []
+    for item in all_engine_ids:
+        if white_re.match(item):
+            engine_ids.append(item)
+
+    # We include Ecosia for some locales if its not there already
+    if _ECOSIA_ID in whitelist and _ECOSIA_ID not in all_engine_ids:
+        ecosia_dst = os.path.join(locale_path, _LOCALE_SEARCHPLUGINS_PATH,
+                                  "ecosia.xml")
+
         shutil.copyfile(os.path.join(project_dir, _ECOSIA_PATH), ecosia_dst)
-
-    all_engine_ids = []
-    engine_ids = []
-    removed_engine_ids = []
-
-    for item in data['locales'][locale]['default'][_DEF_ENGINES]:
-        all_engine_ids.append(item)
-        if len(item) > 0:
-            if white_re.match(item):
-                engine_ids.append(item)
-            else:
-                removed_engine_ids.append(item)
-
-    if _ECOSIA_ID in whitelist and _ECOSIA_ID not in all_engine_ids:
         all_engine_ids.append(_ECOSIA_ID)
         engine_ids.append(_ECOSIA_ID)
 
-    # Make sure we still have search engines left
-    if len(engine_ids) == 0:
-        logger.fatal("No search engines left over for '%s'" % locale)
-
-    data['locales'][locale]['default'][_DEF_ENGINES] = all_engine_ids
-
-    # 'Parse' XML to get matching 'ShortName' for all engine IDs
+    # 'Parse' XML to get matching ShortName for all engine IDs
     engine_names = {}
-    search_plugins_path = os.path.join(project_dir,
-                                       _GENERAL_SEARCHPLUGINS_PATH)
-    for eid in engine_ids[:]:
-        xml_file_path = os.path.join(search_plugins_path, "%s.xml" % eid)
-        if not os.path.exists(xml_file_path):
-            logger.info("Missing xml file for plugin %s. Searched in path %s" %
-                        (eid, xml_file_path))
+    plugins_path = os.path.join(project_dir, _GENERAL_SEARCHPLUGINS_PATH)
+    for eid in engine_ids:
+        plugin_path = os.path.join(plugins_path, "%s.xml" % eid)
+        if not os.path.exists(plugin_path):
+            logger.error("Missing file for plugin %s. Searched in path %s" %
+                         (eid, plugin_path))
             engine_ids.remove(eid)
             continue
+
         short_name = None
-        with open(xml_file_path, "r") as fd:
+        with open(plugin_path, "r") as fd:
             for line in fd:
                 line = line.strip()
                 match = _SHORTNAME_RE.match(line)
                 if match:
                     short_name = match.group(1).strip()
 
         if not short_name:
             logger.fatal("No ShortName defined for '%s' in '%s" %
                          (eid, locale))
+
         engine_names[eid] = short_name
 
-    logger.info("Removed search engine IDs: %s" %
-                ", ".join(removed_engine_ids))
-    logger.info("Remaining search engine IDs: %s" % ", ".join(engine_ids))
-
     # Create search engine order with real engine names
-    engine_order = []
+    engines_ordered = []
     for eid in whitelist:
-        sn = _get_shortname_from_id(eid, engine_ids, engine_names)
-        if sn:
-            engine_order.append(sn)
-
-    logger.info("Resulting search engine ordered list: %s" %
-                (", ".join(engine_order)))
-
-    # Read region.properties and remove browser.search.* lines
-    props = []
-    with open(region_file_path, "r") as fd:
-        for line in fd:
-            line = line.rstrip("\r\n")
-            if not _SEARCH_PROPS_RE.match(line.strip()):
-                props.append(line)
-
-    # Append default search engine name
-    props.append("browser.search.defaultenginename=%s" % engine_order[0])
-
-    # Append search engine order
-    for i in range(0, len(engine_order)):
-        props.append("browser.search.order.%d=%s" % (i + 1, engine_order[i]))
-
-    # Write back region.properties
-    _write_lines(region_file_path, props)
+        shortname = _get_shortname_from_id(eid, engine_ids, engine_names)
+        if shortname:
+            engines_ordered.append(shortname)
+
+    # Make sure we still have search engines left
+    if len(engines_ordered) == 0:
+        logger.fatal("No search engines left over for '%s'" % locale)
+
+    _transform_region_props(locale, locale_path, engines_ordered, logger)
+
+
+def _transform_search_engines(project_dir, chrome_path, res_path, locales,
+                              logger):
+

[anton, 2019/04/05 12:42:24]

unrequired line?

+    list_json_path = os.path.join(project_dir, _LIST_JSON_PATH)
+    _check_path_exists(list_json_path, logger)
+
+    # Open the Mozilla list of search engines, put it into a buffer and close
+    # the JSON file after reading
+    with open(list_json_path, 'r') as json_file:
+        data = json.load(json_file)
+
+        # Set default search engine order
+        data['default'][_DEF_ENGINES] = _SEARCH_ENGINE_ORDER_DEFAULT
+
+        for locale in locales:
+            locale_path = os.path.join(chrome_path, locale, "locale", locale)
+            if os.path.exists(locale_path):
+
+                # Mozilla default list does not contain locale bn-BD,
+                # so we create it and use the values from locale bn-IN
+                if locale == 'bn-BD':
+                    engines = data['locales']['bn-IN']['default'][_DEF_ENGINES]
+                    data['locales'].update({locale: {'default':
+                                           {_DEF_ENGINES: engines}}})
+
+                # Mozilla default list does not contain locale wo, so we use
+                # the default order. In case they will not support any other
+                # locales in the future, we want the build to fail, to decide
+                #  which order to use
+                elif locale == 'wo':
+                    engines = _SEARCH_ENGINE_ORDER_DEFAULT
+                    data['locales'].update({locale: {'default':
+                                           {_DEF_ENGINES: engines}}})
+
+                _transform_search_locale(project_dir, data, locale,
+                                         locale_path, logger)
+
+                _generate_browser_search(project_dir, res_path, locale,
+                                         locale_path, logger)
+            else:
+                logger.error("Missing folder for locale '%s' in path: %s" %
+                             (locale, locale_path))
+
+        # Save modified list.json in generated folder, so we dont change the
+        # original file
+        generated_list_json = os.path.join(project_dir,

[diegocarloslima, 2019/03/29 12:51:01]

Saving the modified list.json into the generated folder, so we don't have file
modifications kept into the mercurial tracked files. That way, we avoid
accidentally committing the changes in the original list.json

+                                           _GENERATED_LIST_JSON_PATH)
+
+        with open(generated_list_json, 'w') as outfile:
+            json.dump(data, outfile, indent=4, sort_keys=True)
+
+        logger.info("Generated list.json in  %s" % generated_list_json)
+
+
+def _generate_search_json(project_dir, locale, locale_path, logger):
+    script_path = os.path.join(project_dir, _MOZBUILD_PATH,
+                               "mozbuild", "action", "generate_searchjson.py")
+
+    # We read from the list.json in generated folder, that contains our own
+    # modifications
+    list_json_src = os.path.join(project_dir, _GENERATED_LIST_JSON_PATH)
+    list_json_dst = os.path.join(locale_path, _BROWSER_DIR, "searchplugins",
+                                 "list.json")
+
+    import subprocess as sub
+    # Call generate_searchjson.py script
+    sub.check_call(['python', script_path, list_json_src, locale,
+                   list_json_dst])
+
+    logger.info("Generated search list.json for %s" % locale)
+
+
+def _transform_appstrings(locale, locale_path, logger):
+    logger.info("Processing appstrings for %s..." % locale)
+    # Check for appstrings.properties existence
+    appstrings_file_path = os.path.join(locale_path, _APPSTRINGS_PROPS_PATH)
+    _check_path_exists(appstrings_file_path, logger)
 
     # Replaces ocurrences of 'Firefox' by 'Adblock Browser' in
-    # 'appstrings.properties'
+    # appstrings.properties
     lines = []
     replacement_count = 0
 
     with open(appstrings_file_path, "r") as fd:
         for line in fd:
             line = line.rstrip("\r\n")
             replacement = _replace_in_value(_PROPERTY_FORMAT_RE, line,
                                             _FIREFOX_REPLACE_STR,
                                             _ABB_REPLACEMENT_STR)
             if replacement:
                 line = replacement
                 replacement_count += 1
             lines.append(line)
 
     # Apply changes to appstrings.properties
     _write_lines(appstrings_file_path, lines)
     logger.info("Replaced %d ocurrences of %s in 'appstrings.properties'" %
                 (replacement_count, _FIREFOX_REPLACE_STR))
 
 
-def _generate_browser_search(locale, locale_path, res_path, project_dir):
-    raw_dir = "raw" if locale == _DEFAULT_LOCALE else (
-        "raw-%s" % locale.replace("-", "-r"))
-
-    browser_path = os.path.join(locale_path, _BROWSER_DIR)
-    browsersearch_file_path = os.path.join(res_path, raw_dir,
-                                           "browsersearch.json")
-
-    sys.path.append(os.path.join(project_dir, _MOZBUILD_PATH))
-    import mozbuild.action.generate_browsersearch as generate_browsersearch
-
-    # Call generate_browsersearch.py script to regenerate
-    # res/raw-LOCALE/browsersearch.json with the updated search engines
-    generate_browsersearch.main(["--verbose", "--srcdir", browser_path,
-                                 browsersearch_file_path])
-
-
-def _generate_search_json(locale, locale_path, project_dir):
-    script_path = os.path.join(project_dir, "python", "mozbuild",
-                               "mozbuild", "action", "generate_searchjson.py")
-    list_json_path = os.path.join(project_dir, _LISTJSON_PATH)
-    searchplugins_path = os.path.join(locale_path, _BROWSER_DIR,
-                                      "searchplugins", "list.json")
-
-    import subprocess as s
-    # Call generate_searchjson.py script
-    s.check_call(['python', script_path, list_json_path,
-                 locale, searchplugins_path])
-
-
-def _transform_values_locale(locale, path, logger):
+def _transform_values_locale(locale, locale_path, logger):
     logger.info("Processing values-%s..." % locale)
-
     # Check for strings.xml existence
-    strings_file_path = os.path.join(path, _STRINGS_XML_PATH)
+    strings_file_path = os.path.join(locale_path, _STRINGS_XML_FILE)
     _check_path_exists(strings_file_path, logger)
 
     # Replaces ocurrences of 'Firefox' by 'Adblock Browser' in 'strings.xml'
     lines = []
     replacement_count = 0
 
     with open(strings_file_path, "r") as fd:
         for line in fd:
             line = line.rstrip("\r\n")
             replacement = _replace_in_value(_ENTITY_FORMAT_RE, line,
@@ skipping 30 matching lines @@
         exit(1)
 
 
 def transform_locales(project_dir, obj_dir, logger=MinimalLogger()):
     chrome_path = os.path.join(obj_dir, _CHROME_PATH)
     _check_path_exists(chrome_path, logger)
 
     res_path = os.path.join(obj_dir, _RES_PATH)
     _check_path_exists(res_path, logger)
 
-    list_json_path = os.path.join(project_dir, _LISTJSON_PATH)
-    _check_path_exists(list_json_path, logger)
-
     locales = _get_locales_from_path(chrome_path, _LOCALE_RE)
     values_locales = _get_locales_from_path(res_path, _VALUES_LOCALE_RE)
 
     locales_found_msg = "Found %d locales in %s"
     logger.info(locales_found_msg % (len(locales), chrome_path))
     logger.info(locales_found_msg % (len(values_locales), res_path))
 
-    # open the Mozilla list of search engines, put it into a buffer and
-    # close the JSON file after reading
-    with open(list_json_path, 'r') as json_file:
-        data = json.load(json_file)
+    _transform_search_engines(project_dir, chrome_path, res_path, locales,
+                              logger)
 
-        # set default search engine order
-        data['default'][_DEF_ENGINES] = _SEARCH_ENGINE_ORDER_DEFAULT
+    for locale in locales:
+        locale_path = os.path.join(chrome_path, locale, "locale", locale)
+        _generate_search_json(project_dir, locale, locale_path, logger)
+        _transform_appstrings(locale, locale_path, logger)
 
-        for locale in locales:
-            locale_path = os.path.join(chrome_path, locale, "locale", locale)
-            if os.path.exists(locale_path):
-
-                # Mozilla default list does not contain locale bn-BD,
-                # so we create it and use the values from locale bn-IN
-                if locale == 'bn-BD':
-                    data['locales'].update({locale: {'default':
-                                           {_DEF_ENGINES:
-                        data['locales']['bn-IN']['default'][_DEF_ENGINES]}}})
-                # Mozilla default list does not contain locale wo, so we use
-                # the default order. In case they will not support any other
-                # locales in the future, we want the build to fail, to decide
-                #  which order to use
-                elif locale == 'wo':
-                    data['locales'].update({locale: {'default':
-                                           {_DEF_ENGINES:
-                                            _SEARCH_ENGINE_ORDER_DEFAULT}}})
-
-                _transform_locale(data, locale, project_dir, locale_path,
-                                  logger)
-                _generate_browser_search(locale, locale_path, res_path,
-                                         project_dir)
-            else:
-                logger.error("Missing folder for locale '%s' in path: %s" %
-                             (locale, locale_path))
-
-        # Save changes to list.json
-        with open(list_json_path, 'w') as outfile:
-            json.dump(data, outfile, indent=4, sort_keys=True)
-
-        # Generate search.json for each locale
-        for locale in locales:
-            locale_path = os.path.join(chrome_path, locale, "locale", locale)
-            _generate_search_json(locale, locale_path, project_dir)
-
-        for locale in values_locales:
-            locale_path = os.path.join(res_path, "values-" + locale)
-            _transform_values_locale(locale, locale_path, logger)
+    for locale in values_locales:
+        locale_path = os.path.join(res_path, "values-" + locale)
+        _transform_values_locale(locale, locale_path, logger)