Issue 6077 - Create script to modify the list of search engines dynamically

mozharness/abb/transform_locales.py

 # This file is part of Adblock Plus
 # Copyright (C) 2006-present eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 import json
 import os
 import re
+import shutil
 import sys
 
 _LOCALE_RE = re.compile("^([a-z]{2,3}(?:-[A-Z]{2})?)$")
 _VALUES_LOCALE_RE = re.compile("^values-([a-z]{2,3}(?:-r[A-Z]{2})?)$")
 
 _SEARCH_PROPS_RE = re.compile("^browser\.search\."
                               "(defaultenginename|order\.).*$")
 _SHORTNAME_RE = re.compile("^<ShortName>(.*)</ShortName>$")
 
 _PROPERTY_FORMAT_RE = re.compile("^(([^=]*)=)(.*)$")
 _ENTITY_FORMAT_RE = re.compile("^(\s*<!ENTITY\s*([^\"\s]*)\s*\")(.*)(\">)$")
 _STRING_FORMAT_RE = re.compile(
     "^(\s*<string name=\"([^\"]*)\">)(.*)(</string>)$")
 
 _MOZBUILD_PATH = os.path.join("python", "mozbuild")
 
 _CHROME_PATH = os.path.join("dist", "bin", "chrome")
 _RES_PATH = os.path.join("mobile", "android", "base", "res")
 _L10N_PATH = os.path.join("abb-build", "l10n")
+_LOCALES_PATH = os.path.join("mobile", "locales")
 _LISTJSON_PATH = os.path.join(_LOCALES_PATH, "search", "list.json")

[diegocarloslima, 2018/01/03 14:25:07]

This seems to cause an error, since _LOCALES_PATH wasn't defined yet

-_LOCALES_PATH = os.path.join("mobile", "locales")
-_SEARCHPLUGINS_PATH = os.path.join(_LOCALES_PATH, "searchplugins")
+_GENERAL_SEARCHPLUGINS_PATH = os.path.join(_LOCALES_PATH, "searchplugins")
 
 _BROWSER_DIR = "browser"
 _REGION_PROPS_PATH = os.path.join(_BROWSER_DIR, "region.properties")
+_LOCALE_SEARCHPLUGINS_PATH = os.path.join(_BROWSER_DIR, "searchplugins")
 
 _APPSTRINGS_PROPS_PATH = os.path.join(_BROWSER_DIR, "appstrings.properties")
 _STRINGS_XML_PATH = "strings.xml"
 
 _DEFAULT_LOCALE = "en-US"
+_DEF_ENGINES = "visibleDefaultEngines"
 
 # Add Ecosia as secondary search engine.
 # See https://issues.adblockplus.org/ticket/5518
 _ECOSIA_ID = "ecosia"
+_ECOSIA_PATH = os.path.join(_GENERAL_SEARCHPLUGINS_PATH, "ecosia.xml")
 
 _SEARCH_ENGINE_ORDER_DEFAULT = [
     "duckduckgo",
     "yahoo",
     "google",
     "wikipedia",
     "amazondotcom"]
 
 _SEARCH_ENGINE_ORDER_ECOSIA = [
     "duckduckgo",
     "yahoo",
     "google",
-    "ecosia",
+    _ECOSIA_ID,
     "wikipedia",
     "amazon"]
 
 _SEARCH_ENGINE_ORDER = {
     "de": _SEARCH_ENGINE_ORDER_ECOSIA,
     "en-GB": _SEARCH_ENGINE_ORDER_ECOSIA,
     "en-US": _SEARCH_ENGINE_ORDER_ECOSIA,
     "fr": _SEARCH_ENGINE_ORDER_ECOSIA,
     "nl": _SEARCH_ENGINE_ORDER_ECOSIA,
     "zh-CN": ["baidu",
@@ skipping 81 matching lines @@
     logger.info("Processing locale '%s'..." % locale)
 
     # Check for region.properties existence
     region_file_path = os.path.join(locale_path, _REGION_PROPS_PATH)
     _check_path_exists(region_file_path, logger)
 
     # Check for appstrings.properties existence
     appstrings_file_path = os.path.join(locale_path, _APPSTRINGS_PROPS_PATH)
     _check_path_exists(appstrings_file_path, logger)
 
+    ecosia_dst = os.path.join(locale_path,
+                              _LOCALE_SEARCHPLUGINS_PATH, "ecosia.xml")
+
     # Get whitelist and build regex
     whitelist = _SEARCH_ENGINE_ORDER.get(locale,
                                          _SEARCH_ENGINE_ORDER_DEFAULT)
     white_re = re.compile("^(%s).*$" % "|".join(whitelist))
 
+    if _ECOSIA_ID in whitelist and not os.path.exists(ecosia_dst):
+        shutil.copyfile(os.path.join(project_dir, _ECOSIA_PATH), ecosia_dst)
+
     all_engine_ids = []
     engine_ids = []
     removed_engine_ids = []
 
-

[diegocarloslima, 2018/01/03 14:25:07]

2 new lines here... seems that it should be just one

-    for item in data['locales'][locale]['default']['visibleDefaultEngines']:
+    for item in data['locales'][locale]['default'][_DEF_ENGINES]:
         all_engine_ids.append(item)
         if len(item) > 0:
             if white_re.match(item):
                 engine_ids.append(item)
             else:
                 removed_engine_ids.append(item)
 
     if _ECOSIA_ID in whitelist and _ECOSIA_ID not in all_engine_ids:
         all_engine_ids.append(_ECOSIA_ID)
         engine_ids.append(_ECOSIA_ID)
 
     # Make sure we still have search engines left
     if len(engine_ids) == 0:
         logger.fatal("No search engines left over for '%s'" % locale)
 
-    data['locales'][locale]['default']['visibleDefaultEngines'] = all_engine_ids
-
-
+    data['locales'][locale]['default'][_DEF_ENGINES] = all_engine_ids
 

[diegocarloslima, 2018/01/03 14:25:07]

3 new lines here... seems that it should be just one

     # 'Parse' XML to get matching 'ShortName' for all engine IDs
     engine_names = {}
-    search_plugins_path = os.path.join(project_dir, _SEARCHPLUGINS_PATH)
+    search_plugins_path = os.path.join(project_dir,
+                                       _GENERAL_SEARCHPLUGINS_PATH)
     for eid in engine_ids[:]:
         xml_file_path = os.path.join(search_plugins_path, "%s.xml" % eid)
         if not os.path.exists(xml_file_path):
             logger.info("Missing xml file for plugin %s. Searched in path %s" %
                         (eid, xml_file_path))
             engine_ids.remove(eid)
             continue
         short_name = None
         with open(xml_file_path, "r") as fd:
             for line in fd:
@@ skipping 69 matching lines @@
     browsersearch_file_path = os.path.join(res_path, raw_dir,
                                            "browsersearch.json")
 
     sys.path.append(os.path.join(project_dir, _MOZBUILD_PATH))
     import mozbuild.action.generate_browsersearch as generate_browsersearch
 
     # Call generate_browsersearch.py script to regenerate
     # res/raw-LOCALE/browsersearch.json with the updated search engines
     generate_browsersearch.main(["--verbose", "--srcdir", browser_path,
                                  browsersearch_file_path])
+
+
+def _generate_search_json(locale, locale_path, project_dir):
+    script_path = os.path.join(project_dir, "python", "mozbuild",
+                               "mozbuild", "action", "generate_searchjson.py")
+    list_json_path = os.path.join(project_dir, _LISTJSON_PATH)
+    searchplugins_path = os.path.join(locale_path, _BROWSER_DIR,
+                                      "searchplugins", "list.json")
+
+    import subprocess as s
+    # Call generate_searchjson.py script
+    s.check_call(['python', script_path, list_json_path,
+                 locale, searchplugins_path])
 
 
 def _transform_values_locale(locale, path, logger):
     logger.info("Processing values-%s..." % locale)
 
     # Check for strings.xml existence
     strings_file_path = os.path.join(path, _STRINGS_XML_PATH)
     _check_path_exists(strings_file_path, logger)
 
     # Replaces ocurrences of 'Firefox' by 'Adblock Browser' in 'strings.xml'
@@ skipping 50 matching lines @@
     locales = _get_locales_from_path(chrome_path, _LOCALE_RE)
     values_locales = _get_locales_from_path(res_path, _VALUES_LOCALE_RE)
 
     locales_found_msg = "Found %d locales in %s"
     logger.info(locales_found_msg % (len(locales), chrome_path))
     logger.info(locales_found_msg % (len(values_locales), res_path))
 
     # open the Mozilla list of search engines, put it into a buffer and
     # close the JSON file after reading
     with open(list_json_path, 'r') as json_file:
-        data = json.load(jsonFile)
+        data = json.load(json_file)
 
         # set default search engine order
-        data['default']['visibleDefaultEngines'] = _SEARCH_ENGINE_ORDER_DEFAULT
+        data['default'][_DEF_ENGINES] = _SEARCH_ENGINE_ORDER_DEFAULT
 
         for locale in locales:
             locale_path = os.path.join(chrome_path, locale, "locale", locale)
             if os.path.exists(locale_path):
 
-                # Mozilla default list does not contain locale bn-BD, so we create it
-                # and use the values from locale bn-IN
+                # Mozilla default list does not contain locale bn-BD,
+                # so we create it and use the values from locale bn-IN
                 if locale == 'bn-BD':
-                    data['locales'].update({locale: {'default': {'visibleDefaultEngines': data['locales']

[diegocarloslima, 2018/01/03 14:25:07]

This line is too big to be in compliance with PEP8. I suggest you to put the
whole code into pep8online.com to check if it meets with PEP8 requirements

-                                           ['bn-IN']['default']['visibleDefaultEngines']}}})
-                # Mozilla default list does not contain locale wo, so we use the
-                # default order. In case they will not support any other locales in
-                # the future, we want the build to fail, to decide which order to use
+                    data['locales'].update({locale: {'default':
+                                           {_DEF_ENGINES:
+                        data['locales']['bn-IN']['default'][_DEF_ENGINES]}}})
+                # Mozilla default list does not contain locale wo, so we use
+                # the default order. In case they will not support any other
+                # locales in the future, we want the build to fail, to decide
+                #  which order to use
                 elif locale == 'wo':
                     data['locales'].update({locale: {'default':
-                                           {'visibleDefaultEngines': _SEARCH_ENGINE_ORDER_DEFAULT}}})
-
-                _transform_locale(data, locale, project_dir, locale_path, logger)
-                _generate_browser_search(locale, locale_path, res_path, project_dir)
+                                           {_DEF_ENGINES:
+                                            _SEARCH_ENGINE_ORDER_DEFAULT}}})
+
+                _transform_locale(data, locale, project_dir, locale_path,
+                                  logger)
+                _generate_browser_search(locale, locale_path, res_path,
+                                         project_dir)
             else:
                 logger.error("Missing folder for locale '%s' in path: %s" %
                              (locale, locale_path))
 
-        # Save our changes to list.json
+        # Save changes to list.json
         with open(list_json_path, 'w') as outfile:
             json.dump(data, outfile, indent=4, sort_keys=True)
+
+        # Generate search.json for each locale
+        for locale in locales:
+            locale_path = os.path.join(chrome_path, locale, "locale", locale)
+            _generate_search_json(locale, locale_path, project_dir)
 
         for locale in values_locales:
             locale_path = os.path.join(res_path, "values-" + locale)
             _transform_values_locale(locale, locale_path, logger)
-