Index: abb-build/transform_locales.py =================================================================== --- a/abb-build/transform_locales.py +++ b/abb-build/transform_locales.py @@ -27,48 +27,47 @@ _SHORTNAME_RE = re.compile("^(.*)$") _PROPERTY_FORMAT_RE = re.compile("^(([^=]*)=)(.*)$") _ENTITY_FORMAT_RE = re.compile("^(\s*)$") _STRING_FORMAT_RE = re.compile( "^(\s*)(.*)()$") _MOZBUILD_PATH = os.path.join("python", "mozbuild") - +_GENERATED_PATH = os.path.join("abb-build", "generated") _CHROME_PATH = os.path.join("dist", "bin", "chrome") _RES_PATH = os.path.join("mobile", "android", "base", "res") -_L10N_PATH = os.path.join("abb-build", "l10n") _LOCALES_PATH = os.path.join("mobile", "locales") -_LISTJSON_PATH = os.path.join(_LOCALES_PATH, "search", "list.json") +_LIST_JSON_FILE = "list.json" +_LIST_JSON_PATH = os.path.join(_LOCALES_PATH, "search", _LIST_JSON_FILE) +_GENERATED_LIST_JSON_PATH = os.path.join(_GENERATED_PATH, _LIST_JSON_FILE) _GENERAL_SEARCHPLUGINS_PATH = os.path.join(_LOCALES_PATH, "searchplugins") - _BROWSER_DIR = "browser" _REGION_PROPS_PATH = os.path.join(_BROWSER_DIR, "region.properties") _LOCALE_SEARCHPLUGINS_PATH = os.path.join(_BROWSER_DIR, "searchplugins") - _APPSTRINGS_PROPS_PATH = os.path.join(_BROWSER_DIR, "appstrings.properties") -_STRINGS_XML_PATH = "strings.xml" +_STRINGS_XML_FILE = "strings.xml" _DEFAULT_LOCALE = "en-US" _DEF_ENGINES = "visibleDefaultEngines" # Add Ecosia as secondary search engine. # See https://issues.adblockplus.org/ticket/5518 _ECOSIA_ID = "ecosia" _ECOSIA_PATH = os.path.join(_GENERAL_SEARCHPLUGINS_PATH, "ecosia.xml") _SEARCH_ENGINE_ORDER_DEFAULT = [ - "duckduckgo", + "ddg", "yahoo", "google", "wikipedia", - "amazondotcom"] + "amazon"] _SEARCH_ENGINE_ORDER_ECOSIA = [ - "duckduckgo", + "ddg", "yahoo", "google", _ECOSIA_ID, "wikipedia", "amazon"] _SEARCH_ENGINE_ORDER = { "de": _SEARCH_ENGINE_ORDER_ECOSIA, @@ -152,118 +151,202 @@ def _write_lines(filename, lines): """Writes lines into file appending \\n""" with open(filename, "w") as fd: for l in lines: fd.write("%s\n" % l) -def _transform_locale(data, locale, project_dir, locale_path, logger): - logger.info("Processing locale '%s'..." % locale) - +def _transform_region_props(locale, locale_path, engines_ordered, logger): # Check for region.properties existence region_file_path = os.path.join(locale_path, _REGION_PROPS_PATH) _check_path_exists(region_file_path, logger) - # Check for appstrings.properties existence - appstrings_file_path = os.path.join(locale_path, _APPSTRINGS_PROPS_PATH) - _check_path_exists(appstrings_file_path, logger) + # Read region.properties and remove browser.search.* lines + lines = [] + with open(region_file_path, "r") as fd: + for line in fd: + line = line.rstrip("\r\n") + if not _SEARCH_PROPS_RE.match(line.strip()): + lines.append(line) - ecosia_dst = os.path.join(locale_path, - _LOCALE_SEARCHPLUGINS_PATH, "ecosia.xml") + # Append default search engine name + lines.append("browser.search.defaultenginename=%s" % engines_ordered[0]) + # Append search engine order + for i in range(0, len(engines_ordered)): + lines.append("browser.search.order.%d=%s" % (i + 1, + engines_ordered[i])) + # Write back region.properties + _write_lines(region_file_path, lines) + logger.info("Resulting ordered search engines for %s: %s" % + (locale, ", ".join(engines_ordered))) + + +def _generate_browser_search(project_dir, res_path, locale, locale_path, + logger): + + raw_dir = "raw" if locale == _DEFAULT_LOCALE else ( + "raw-%s" % locale.replace("-", "-r")) + + browser_path = os.path.join(locale_path, _BROWSER_DIR) + browsersearch_file_path = os.path.join(res_path, raw_dir, + "browsersearch.json") + + sys.path.append(os.path.join(project_dir, _MOZBUILD_PATH)) + import mozbuild.action.generate_browsersearch as generate_browsersearch + + # Call generate_browsersearch.py script to regenerate + # res/raw-LOCALE/browsersearch.json with the updated search engines + generate_browsersearch.main(["--verbose", "--srcdir", browser_path, + browsersearch_file_path]) + + logger.info("Generated browsersearch.json for %s" % locale) + + +def _transform_search_locale(project_dir, data, locale, locale_path, logger): # Get whitelist and build regex - whitelist = _SEARCH_ENGINE_ORDER.get(locale, - _SEARCH_ENGINE_ORDER_DEFAULT) + whitelist = _SEARCH_ENGINE_ORDER.get(locale, _SEARCH_ENGINE_ORDER_DEFAULT) white_re = re.compile("^(%s).*$" % "|".join(whitelist)) - if _ECOSIA_ID in whitelist and not os.path.exists(ecosia_dst): + all_engine_ids = data['locales'][locale]['default'][_DEF_ENGINES] + engine_ids = [] + for item in all_engine_ids: + if white_re.match(item): + engine_ids.append(item) + + # We include Ecosia for some locales if its not there already + if _ECOSIA_ID in whitelist and _ECOSIA_ID not in all_engine_ids: + ecosia_dst = os.path.join(locale_path, _LOCALE_SEARCHPLUGINS_PATH, + "ecosia.xml") + shutil.copyfile(os.path.join(project_dir, _ECOSIA_PATH), ecosia_dst) - - all_engine_ids = [] - engine_ids = [] - removed_engine_ids = [] - - for item in data['locales'][locale]['default'][_DEF_ENGINES]: - all_engine_ids.append(item) - if len(item) > 0: - if white_re.match(item): - engine_ids.append(item) - else: - removed_engine_ids.append(item) - - if _ECOSIA_ID in whitelist and _ECOSIA_ID not in all_engine_ids: all_engine_ids.append(_ECOSIA_ID) engine_ids.append(_ECOSIA_ID) - # Make sure we still have search engines left - if len(engine_ids) == 0: - logger.fatal("No search engines left over for '%s'" % locale) - - data['locales'][locale]['default'][_DEF_ENGINES] = all_engine_ids - - # 'Parse' XML to get matching 'ShortName' for all engine IDs + # 'Parse' XML to get matching ShortName for all engine IDs engine_names = {} - search_plugins_path = os.path.join(project_dir, - _GENERAL_SEARCHPLUGINS_PATH) - for eid in engine_ids[:]: - xml_file_path = os.path.join(search_plugins_path, "%s.xml" % eid) - if not os.path.exists(xml_file_path): - logger.info("Missing xml file for plugin %s. Searched in path %s" % - (eid, xml_file_path)) + plugins_path = os.path.join(project_dir, _GENERAL_SEARCHPLUGINS_PATH) + for eid in engine_ids: + plugin_path = os.path.join(plugins_path, "%s.xml" % eid) + if not os.path.exists(plugin_path): + logger.error("Missing file for plugin %s. Searched in path %s" % + (eid, plugin_path)) engine_ids.remove(eid) continue + short_name = None - with open(xml_file_path, "r") as fd: + with open(plugin_path, "r") as fd: for line in fd: line = line.strip() match = _SHORTNAME_RE.match(line) if match: short_name = match.group(1).strip() if not short_name: logger.fatal("No ShortName defined for '%s' in '%s" % (eid, locale)) + engine_names[eid] = short_name - logger.info("Removed search engine IDs: %s" % - ", ".join(removed_engine_ids)) - logger.info("Remaining search engine IDs: %s" % ", ".join(engine_ids)) + # Create search engine order with real engine names + engines_ordered = [] + for eid in whitelist: + shortname = _get_shortname_from_id(eid, engine_ids, engine_names) + if shortname: + engines_ordered.append(shortname) - # Create search engine order with real engine names - engine_order = [] - for eid in whitelist: - sn = _get_shortname_from_id(eid, engine_ids, engine_names) - if sn: - engine_order.append(sn) + # Make sure we still have search engines left + if len(engines_ordered) == 0: + logger.fatal("No search engines left over for '%s'" % locale) - logger.info("Resulting search engine ordered list: %s" % - (", ".join(engine_order))) + _transform_region_props(locale, locale_path, engines_ordered, logger) - # Read region.properties and remove browser.search.* lines - props = [] - with open(region_file_path, "r") as fd: - for line in fd: - line = line.rstrip("\r\n") - if not _SEARCH_PROPS_RE.match(line.strip()): - props.append(line) - # Append default search engine name - props.append("browser.search.defaultenginename=%s" % engine_order[0]) +def _transform_search_engines(project_dir, chrome_path, res_path, locales, + logger): - # Append search engine order - for i in range(0, len(engine_order)): - props.append("browser.search.order.%d=%s" % (i + 1, engine_order[i])) + list_json_path = os.path.join(project_dir, _LIST_JSON_PATH) + _check_path_exists(list_json_path, logger) - # Write back region.properties - _write_lines(region_file_path, props) + # Open the Mozilla list of search engines, put it into a buffer and close + # the JSON file after reading + with open(list_json_path, 'r') as json_file: + data = json.load(json_file) + + # Set default search engine order + data['default'][_DEF_ENGINES] = _SEARCH_ENGINE_ORDER_DEFAULT + + for locale in locales: + locale_path = os.path.join(chrome_path, locale, "locale", locale) + if os.path.exists(locale_path): + + # Mozilla default list does not contain locale bn-BD, + # so we create it and use the values from locale bn-IN + if locale == 'bn-BD': + engines = data['locales']['bn-IN']['default'][_DEF_ENGINES] + data['locales'].update({locale: {'default': + {_DEF_ENGINES: engines}}}) + + # Mozilla default list does not contain locale wo, so we use + # the default order. In case they will not support any other + # locales in the future, we want the build to fail, to decide + # which order to use + elif locale == 'wo': + engines = _SEARCH_ENGINE_ORDER_DEFAULT + data['locales'].update({locale: {'default': + {_DEF_ENGINES: engines}}}) + + _transform_search_locale(project_dir, data, locale, + locale_path, logger) + + _generate_browser_search(project_dir, res_path, locale, + locale_path, logger) + else: + logger.error("Missing folder for locale '%s' in path: %s" % + (locale, locale_path)) + + # Save modified list.json in generated folder, so we dont change the + # original file + generated_list_json = os.path.join(project_dir, + _GENERATED_LIST_JSON_PATH) + + with open(generated_list_json, 'w') as outfile: + json.dump(data, outfile, indent=4, sort_keys=True) + + logger.info("Generated list.json in %s" % generated_list_json) + + +def _generate_search_json(project_dir, locale, locale_path, logger): + script_path = os.path.join(project_dir, _MOZBUILD_PATH, + "mozbuild", "action", "generate_searchjson.py") + + # We read from the list.json in generated folder, that contains our own + # modifications + list_json_src = os.path.join(project_dir, _GENERATED_LIST_JSON_PATH) + list_json_dst = os.path.join(locale_path, _BROWSER_DIR, "searchplugins", + "list.json") + + import subprocess as sub + # Call generate_searchjson.py script + sub.check_call(['python', script_path, list_json_src, locale, + list_json_dst]) + + logger.info("Generated search list.json for %s" % locale) + + +def _transform_appstrings(locale, locale_path, logger): + logger.info("Processing appstrings for %s..." % locale) + # Check for appstrings.properties existence + appstrings_file_path = os.path.join(locale_path, _APPSTRINGS_PROPS_PATH) + _check_path_exists(appstrings_file_path, logger) # Replaces ocurrences of 'Firefox' by 'Adblock Browser' in - # 'appstrings.properties' + # appstrings.properties lines = [] replacement_count = 0 with open(appstrings_file_path, "r") as fd: for line in fd: line = line.rstrip("\r\n") replacement = _replace_in_value(_PROPERTY_FORMAT_RE, line, _FIREFOX_REPLACE_STR, @@ -274,51 +357,20 @@ lines.append(line) # Apply changes to appstrings.properties _write_lines(appstrings_file_path, lines) logger.info("Replaced %d ocurrences of %s in 'appstrings.properties'" % (replacement_count, _FIREFOX_REPLACE_STR)) -def _generate_browser_search(locale, locale_path, res_path, project_dir): - raw_dir = "raw" if locale == _DEFAULT_LOCALE else ( - "raw-%s" % locale.replace("-", "-r")) - - browser_path = os.path.join(locale_path, _BROWSER_DIR) - browsersearch_file_path = os.path.join(res_path, raw_dir, - "browsersearch.json") - - sys.path.append(os.path.join(project_dir, _MOZBUILD_PATH)) - import mozbuild.action.generate_browsersearch as generate_browsersearch - - # Call generate_browsersearch.py script to regenerate - # res/raw-LOCALE/browsersearch.json with the updated search engines - generate_browsersearch.main(["--verbose", "--srcdir", browser_path, - browsersearch_file_path]) - - -def _generate_search_json(locale, locale_path, project_dir): - script_path = os.path.join(project_dir, "python", "mozbuild", - "mozbuild", "action", "generate_searchjson.py") - list_json_path = os.path.join(project_dir, _LISTJSON_PATH) - searchplugins_path = os.path.join(locale_path, _BROWSER_DIR, - "searchplugins", "list.json") - - import subprocess as s - # Call generate_searchjson.py script - s.check_call(['python', script_path, list_json_path, - locale, searchplugins_path]) - - -def _transform_values_locale(locale, path, logger): +def _transform_values_locale(locale, locale_path, logger): logger.info("Processing values-%s..." % locale) - # Check for strings.xml existence - strings_file_path = os.path.join(path, _STRINGS_XML_PATH) + strings_file_path = os.path.join(locale_path, _STRINGS_XML_FILE) _check_path_exists(strings_file_path, logger) # Replaces ocurrences of 'Firefox' by 'Adblock Browser' in 'strings.xml' lines = [] replacement_count = 0 with open(strings_file_path, "r") as fd: for line in fd: @@ -359,65 +411,26 @@ def transform_locales(project_dir, obj_dir, logger=MinimalLogger()): chrome_path = os.path.join(obj_dir, _CHROME_PATH) _check_path_exists(chrome_path, logger) res_path = os.path.join(obj_dir, _RES_PATH) _check_path_exists(res_path, logger) - list_json_path = os.path.join(project_dir, _LISTJSON_PATH) - _check_path_exists(list_json_path, logger) - locales = _get_locales_from_path(chrome_path, _LOCALE_RE) values_locales = _get_locales_from_path(res_path, _VALUES_LOCALE_RE) locales_found_msg = "Found %d locales in %s" logger.info(locales_found_msg % (len(locales), chrome_path)) logger.info(locales_found_msg % (len(values_locales), res_path)) - # open the Mozilla list of search engines, put it into a buffer and - # close the JSON file after reading - with open(list_json_path, 'r') as json_file: - data = json.load(json_file) + _transform_search_engines(project_dir, chrome_path, res_path, locales, + logger) - # set default search engine order - data['default'][_DEF_ENGINES] = _SEARCH_ENGINE_ORDER_DEFAULT + for locale in locales: + locale_path = os.path.join(chrome_path, locale, "locale", locale) + _generate_search_json(project_dir, locale, locale_path, logger) + _transform_appstrings(locale, locale_path, logger) - for locale in locales: - locale_path = os.path.join(chrome_path, locale, "locale", locale) - if os.path.exists(locale_path): - - # Mozilla default list does not contain locale bn-BD, - # so we create it and use the values from locale bn-IN - if locale == 'bn-BD': - data['locales'].update({locale: {'default': - {_DEF_ENGINES: - data['locales']['bn-IN']['default'][_DEF_ENGINES]}}}) - # Mozilla default list does not contain locale wo, so we use - # the default order. In case they will not support any other - # locales in the future, we want the build to fail, to decide - # which order to use - elif locale == 'wo': - data['locales'].update({locale: {'default': - {_DEF_ENGINES: - _SEARCH_ENGINE_ORDER_DEFAULT}}}) - - _transform_locale(data, locale, project_dir, locale_path, - logger) - _generate_browser_search(locale, locale_path, res_path, - project_dir) - else: - logger.error("Missing folder for locale '%s' in path: %s" % - (locale, locale_path)) - - # Save changes to list.json - with open(list_json_path, 'w') as outfile: - json.dump(data, outfile, indent=4, sort_keys=True) - - # Generate search.json for each locale - for locale in locales: - locale_path = os.path.join(chrome_path, locale, "locale", locale) - _generate_search_json(locale, locale_path, project_dir) - - for locale in values_locales: - locale_path = os.path.join(res_path, "values-" + locale) - _transform_values_locale(locale, locale_path, logger) + for locale in values_locales: + locale_path = os.path.join(res_path, "values-" + locale) + _transform_values_locale(locale, locale_path, logger)