Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Unified Diff: mozharness/abb/transform_locales.py

Issue 29633713: Issue 6077 - Create script to modify the list of search engines dynamically (Closed)
Patch Set: refactored _transform_locales() Created Dec. 19, 2017, 4:49 p.m.
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: mozharness/abb/transform_locales.py
===================================================================
--- a/mozharness/abb/transform_locales.py
+++ b/mozharness/abb/transform_locales.py
@@ -8,40 +8,84 @@
# Adblock Plus is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
+import json
import os
import re
import sys
_LOCALE_RE = re.compile("^([a-z]{2,3}(?:-[A-Z]{2})?)$")
_VALUES_LOCALE_RE = re.compile("^values-([a-z]{2,3}(?:-r[A-Z]{2})?)$")
+_SEARCH_PROPS_RE = re.compile("^browser\.search\."
+ "(defaultenginename|order\.).*$")
+_SHORTNAME_RE = re.compile("^<ShortName>(.*)</ShortName>$")
+
_PROPERTY_FORMAT_RE = re.compile("^(([^=]*)=)(.*)$")
_ENTITY_FORMAT_RE = re.compile("^(\s*<!ENTITY\s*([^\"\s]*)\s*\")(.*)(\">)$")
_STRING_FORMAT_RE = re.compile(
"^(\s*<string name=\"([^\"]*)\">)(.*)(</string>)$")
_MOZBUILD_PATH = os.path.join("python", "mozbuild")
_CHROME_PATH = os.path.join("dist", "bin", "chrome")
_RES_PATH = os.path.join("mobile", "android", "base", "res")
+_L10N_PATH = os.path.join("abb-build", "l10n")
+_LISTJSON_PATH = os.path.join(_LOCALES_PATH, "search", "list.json")
diegocarloslima 2018/01/03 14:25:07 This seems to cause an error, since _LOCALES_PATH
+_LOCALES_PATH = os.path.join("mobile", "locales")
+_SEARCHPLUGINS_PATH = os.path.join(_LOCALES_PATH, "searchplugins")
_BROWSER_DIR = "browser"
+_REGION_PROPS_PATH = os.path.join(_BROWSER_DIR, "region.properties")
_APPSTRINGS_PROPS_PATH = os.path.join(_BROWSER_DIR, "appstrings.properties")
_STRINGS_XML_PATH = "strings.xml"
_DEFAULT_LOCALE = "en-US"
+# Add Ecosia as secondary search engine.
+# See https://issues.adblockplus.org/ticket/5518
+_ECOSIA_ID = "ecosia"
+
+_SEARCH_ENGINE_ORDER_DEFAULT = [
+ "duckduckgo",
+ "yahoo",
+ "google",
+ "wikipedia",
+ "amazondotcom"]
+
+_SEARCH_ENGINE_ORDER_ECOSIA = [
+ "duckduckgo",
+ "yahoo",
+ "google",
+ "ecosia",
+ "wikipedia",
+ "amazon"]
+
+_SEARCH_ENGINE_ORDER = {
+ "de": _SEARCH_ENGINE_ORDER_ECOSIA,
+ "en-GB": _SEARCH_ENGINE_ORDER_ECOSIA,
+ "en-US": _SEARCH_ENGINE_ORDER_ECOSIA,
+ "fr": _SEARCH_ENGINE_ORDER_ECOSIA,
+ "nl": _SEARCH_ENGINE_ORDER_ECOSIA,
+ "zh-CN": ["baidu",
+ "duckduckgo",
+ "yahoo",
+ "google",
+ "wikipedia",
+ "amazon"
+ ]
+}
+
_FIREFOX_REPLACE_STR = "Firefox"
_ABB_REPLACEMENT_STR = "Adblock Browser"
# Some string values that contain Firefox such as 'Firefox Sync' shouldn't be
# replaced, so we keep a list of ids that are exceptions
_ENTITY_EXCEPTIONS = [
"overlay_no_synced_devices",
"home_remote_tabs_need_to_sign_in",
@@ -104,23 +148,112 @@ def _replace_in_value(format_re, str, ol
def _write_lines(filename, lines):
"""Writes lines into file appending \\n"""
with open(filename, "w") as fd:
for l in lines:
fd.write("%s\n" % l)
-def _transform_locale(locale, path, logger):
+def _transform_locale(data, locale, project_dir, locale_path, logger):
logger.info("Processing locale '%s'..." % locale)
+ # Check for region.properties existence
+ region_file_path = os.path.join(locale_path, _REGION_PROPS_PATH)
+ _check_path_exists(region_file_path, logger)
+
# Check for appstrings.properties existence
- appstrings_file_path = os.path.join(path, _APPSTRINGS_PROPS_PATH)
+ appstrings_file_path = os.path.join(locale_path, _APPSTRINGS_PROPS_PATH)
_check_path_exists(appstrings_file_path, logger)
+ # Get whitelist and build regex
+ whitelist = _SEARCH_ENGINE_ORDER.get(locale,
+ _SEARCH_ENGINE_ORDER_DEFAULT)
+ white_re = re.compile("^(%s).*$" % "|".join(whitelist))
+
+ all_engine_ids = []
+ engine_ids = []
+ removed_engine_ids = []
+
+
diegocarloslima 2018/01/03 14:25:07 2 new lines here... seems that it should be just o
+ for item in data['locales'][locale]['default']['visibleDefaultEngines']:
+ all_engine_ids.append(item)
+ if len(item) > 0:
+ if white_re.match(item):
+ engine_ids.append(item)
+ else:
+ removed_engine_ids.append(item)
+
+ if _ECOSIA_ID in whitelist and _ECOSIA_ID not in all_engine_ids:
+ all_engine_ids.append(_ECOSIA_ID)
+ engine_ids.append(_ECOSIA_ID)
+
+ # Make sure we still have search engines left
+ if len(engine_ids) == 0:
+ logger.fatal("No search engines left over for '%s'" % locale)
+
+ data['locales'][locale]['default']['visibleDefaultEngines'] = all_engine_ids
+
+
+
diegocarloslima 2018/01/03 14:25:07 3 new lines here... seems that it should be just o
+ # 'Parse' XML to get matching 'ShortName' for all engine IDs
+ engine_names = {}
+ search_plugins_path = os.path.join(project_dir, _SEARCHPLUGINS_PATH)
+ for eid in engine_ids[:]:
+ xml_file_path = os.path.join(search_plugins_path, "%s.xml" % eid)
+ if not os.path.exists(xml_file_path):
+ logger.info("Missing xml file for plugin %s. Searched in path %s" %
+ (eid, xml_file_path))
+ engine_ids.remove(eid)
+ continue
+ short_name = None
+ with open(xml_file_path, "r") as fd:
+ for line in fd:
+ line = line.strip()
+ match = _SHORTNAME_RE.match(line)
+ if match:
+ short_name = match.group(1).strip()
+
+ if not short_name:
+ logger.fatal("No ShortName defined for '%s' in '%s" %
+ (eid, locale))
+ engine_names[eid] = short_name
+
+ logger.info("Removed search engine IDs: %s" %
+ ", ".join(removed_engine_ids))
+ logger.info("Remaining search engine IDs: %s" % ", ".join(engine_ids))
+
+ # Create search engine order with real engine names
+ engine_order = []
+ for eid in whitelist:
+ sn = _get_shortname_from_id(eid, engine_ids, engine_names)
+ if sn:
+ engine_order.append(sn)
+
+ logger.info("Resulting search engine ordered list: %s" %
+ (", ".join(engine_order)))
+
+ # Read region.properties and remove browser.search.* lines
+ props = []
+ with open(region_file_path, "r") as fd:
+ for line in fd:
+ line = line.rstrip("\r\n")
+ if not _SEARCH_PROPS_RE.match(line.strip()):
+ props.append(line)
+
+ # Append default search engine name
+ props.append("browser.search.defaultenginename=%s" % engine_order[0])
+
+ # Append search engine order
+ for i in range(0, len(engine_order)):
+ props.append("browser.search.order.%d=%s" % (i + 1, engine_order[i]))
+
+ # Write back region.properties
+ _write_lines(region_file_path, props)
+
# Replaces ocurrences of 'Firefox' by 'Adblock Browser' in
# 'appstrings.properties'
lines = []
replacement_count = 0
with open(appstrings_file_path, "r") as fd:
for line in fd:
line = line.rstrip("\r\n")
@@ -133,25 +266,25 @@ def _transform_locale(locale, path, logg
lines.append(line)
# Apply changes to appstrings.properties
_write_lines(appstrings_file_path, lines)
logger.info("Replaced %d ocurrences of %s in 'appstrings.properties'" %
(replacement_count, _FIREFOX_REPLACE_STR))
-def _generate_browser_search(locale, locale_path, res_path, build_dir):
+def _generate_browser_search(locale, locale_path, res_path, project_dir):
raw_dir = "raw" if locale == _DEFAULT_LOCALE else (
"raw-%s" % locale.replace("-", "-r"))
browser_path = os.path.join(locale_path, _BROWSER_DIR)
browsersearch_file_path = os.path.join(res_path, raw_dir,
"browsersearch.json")
- sys.path.append(os.path.join(build_dir, _MOZBUILD_PATH))
+ sys.path.append(os.path.join(project_dir, _MOZBUILD_PATH))
import mozbuild.action.generate_browsersearch as generate_browsersearch
# Call generate_browsersearch.py script to regenerate
# res/raw-LOCALE/browsersearch.json with the updated search engines
generate_browsersearch.main(["--verbose", "--srcdir", browser_path,
browsersearch_file_path])
@@ -198,34 +331,63 @@ class MinimalLogger:
def error(self, s):
print "ERROR: %s" % s
def fatal(self, s):
print "FATAL: %s" % s
exit(1)
-def transform_locales(build_dir, obj_dir, logger=MinimalLogger()):
+def transform_locales(project_dir, obj_dir, logger=MinimalLogger()):
chrome_path = os.path.join(obj_dir, _CHROME_PATH)
_check_path_exists(chrome_path, logger)
res_path = os.path.join(obj_dir, _RES_PATH)
_check_path_exists(res_path, logger)
+ list_json_path = os.path.join(project_dir, _LISTJSON_PATH)
+ _check_path_exists(list_json_path, logger)
+
locales = _get_locales_from_path(chrome_path, _LOCALE_RE)
values_locales = _get_locales_from_path(res_path, _VALUES_LOCALE_RE)
locales_found_msg = "Found %d locales in %s"
logger.info(locales_found_msg % (len(locales), chrome_path))
logger.info(locales_found_msg % (len(values_locales), res_path))
- for locale in locales:
- locale_path = os.path.join(chrome_path, locale, "locale", locale)
- if os.path.exists(locale_path):
- _transform_locale(locale, locale_path, logger)
- _generate_browser_search(locale, locale_path, res_path, build_dir)
- else:
- logger.error("Missing folder for locale '%s' in path: %s" %
- (locale, locale_path))
+ # open the Mozilla list of search engines, put it into a buffer and
+ # close the JSON file after reading
+ with open(list_json_path, 'r') as json_file:
+ data = json.load(jsonFile)
+
+ # set default search engine order
+ data['default']['visibleDefaultEngines'] = _SEARCH_ENGINE_ORDER_DEFAULT
+
+ for locale in locales:
+ locale_path = os.path.join(chrome_path, locale, "locale", locale)
+ if os.path.exists(locale_path):
- for locale in values_locales:
- locale_path = os.path.join(res_path, "values-" + locale)
- _transform_values_locale(locale, locale_path, logger)
+ # Mozilla default list does not contain locale bn-BD, so we create it
+ # and use the values from locale bn-IN
+ if locale == 'bn-BD':
+ data['locales'].update({locale: {'default': {'visibleDefaultEngines': data['locales']
diegocarloslima 2018/01/03 14:25:07 This line is too big to be in compliance with PEP8
+ ['bn-IN']['default']['visibleDefaultEngines']}}})
+ # Mozilla default list does not contain locale wo, so we use the
+ # default order. In case they will not support any other locales in
+ # the future, we want the build to fail, to decide which order to use
+ elif locale == 'wo':
+ data['locales'].update({locale: {'default':
+ {'visibleDefaultEngines': _SEARCH_ENGINE_ORDER_DEFAULT}}})
+
+ _transform_locale(data, locale, project_dir, locale_path, logger)
+ _generate_browser_search(locale, locale_path, res_path, project_dir)
+ else:
+ logger.error("Missing folder for locale '%s' in path: %s" %
+ (locale, locale_path))
+
+ # Save our changes to list.json
+ with open(list_json_path, 'w') as outfile:
+ json.dump(data, outfile, indent=4, sort_keys=True)
+
+ for locale in values_locales:
+ locale_path = os.path.join(res_path, "values-" + locale)
+ _transform_values_locale(locale, locale_path, logger)
+
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld