Issue 3047 - Change default search engines

mozharness/abb/abb_transform_locales.py

Index: mozharness/abb/abb_transform_locales.py
diff --git a/mozharness/abb/abb_transform_locales.py b/mozharness/abb/abb_transform_locales.py
new file mode 100644
index 0000000000000000000000000000000000000000..12215fad9a94d7fd07f8e013e166e82d444ed18f
--- /dev/null
+++ b/mozharness/abb/abb_transform_locales.py
@@ -0,0 +1,202 @@
+# vim:fileencoding=utf-8:et:ts=4:sts=4

[Felix Dahlke, 2015/09/17 09:52:34]

We generally don't add mode line headers:
https://adblockplus.org/en/coding-style#general

[René Jeschke, 2015/09/17 10:39:49]

Done.

+#
+# This file is part of Adblock Plus
+# Copyright (C) 2006-2015 Eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import re
+
+ABB_LOCALE_RE = re.compile("^[a-z]{2}(-[A-Z]{2})?$")

[Felix Dahlke, 2015/09/17 09:52:34]

There is no need to prefix anything with "abb" here, every module has its own
namespace.

What Python doesn't have, however, is a notion of visibility - that's why
private symbols should be prefixed with a single underscore, e.g. `_LOCALE_RE`.

[René Jeschke, 2015/09/17 10:39:49]

Done.

+ABB_SEARCH_PROPS_RE = re.compile("^browser\.search\."
+                                 "(defaultenginename|order\.).*$")
+ABB_SHORTNAME_RE = re.compile("^<ShortName>.*</ShortName>$")
+
+ABB_SEARCHPLUGINS_PATH = os.path.join("browser", "searchplugins")
+ABB_LIST_TXT_PATH = os.path.join(ABB_SEARCHPLUGINS_PATH, "list.txt")
+ABB_REGION_PROPS_PATH = os.path.join("browser", "region.properties")
+
+ABB_DEFAULT_LOCALE = "en-US"
+
+ABB_CFG = {

[Felix Dahlke, 2015/09/17 09:52:35]

Looks like this could be simplified:

_SEARCH_ENGINE_ORDER = {
  "en-US": [
    "duckduckgo",
    ...
   ],
   "zh-CN": [
     "baidu",
     ...
   ]
}

[René Jeschke, 2015/09/17 10:39:49]

Done.

+    "en-US": {"ordered-whitelist": ["duckduckgo",
+                                    "yahoo",
+                                    "google",
+                                    "wikipedia",
+                                    "amazon"]},
+    "zh-CN": {"ordered-whitelist": ["baidu",
+                                    "duckduckgo",
+                                    "yahoo",
+                                    "google",
+                                    "wikipedia",
+                                    "amazon"]}
+}
+
+
+def abb_get_shortname_from_id(needle, engine_ids, engine_map):
+    """Fuzzy finds needle in engine_ids and returns ShortName"""
+    regex = re.compile("^%s.*$" % needle)
+    for engine in engine_ids:
+        if regex.match(engine.lower()):
+            return engine_map[engine]
+    return None
+
+
+def abb_write_lines(filename, lines):
+    """Writes lines into file appending \\n"""
+    with open(filename, "w") as fd:
+        for l in lines:
+            fd.write("%s\n" % l)
+
+
+def abb_transform_locale(locale, path, fns):
+    fns["info"]("Processing locale '%s'..." % locale)
+
+    # Get configuration for current locale
+    cfg = ABB_CFG.get(locale, ABB_CFG[ABB_DEFAULT_LOCALE])
+
+    # Check for list.txt existence
+    list_file = os.path.join(path, ABB_LIST_TXT_PATH)
+    if not os.path.exists(list_file):
+        fns["fatal"]("Missing 'list.txt' for locale '%s'" % locale)
+
+    # Check for region.properties existence
+    region_file = os.path.join(path, ABB_REGION_PROPS_PATH)
+    if not os.path.exists(region_file):
+        fns["fatal"]("Missing 'region.properties' for locale '%s'" % locale)
+
+    # Get whitelist and build regex
+    whitelist = cfg["ordered-whitelist"]
+    white_re = re.compile("^(%s).*$" % ("|".join(whitelist)))
+
+    # Read engine IDs from list.txt, discard engines not on whitelist
+    engine_ids = []
+    for line in open(list_file, "r"):
+        line = line.strip()
+        if len(line) > 0:
+            if white_re.match(line.lower()):
+                engine_ids.append(line)
+            else:
+                fns["info"]("Removing '%s'" % line)
+
+    # Make sure we still have search engines left
+    if len(engine_ids) == 0:
+        fns["fatal"]("No search engines left over for '%s'" % locale)
+
+    # 'Parse' XML to get matching 'ShortName' for all engine IDs
+    engine_names = {}
+    for eid in engine_ids:
+        xml_file = os.path.join(path, ABB_SEARCHPLUGINS_PATH, "%s.xml" % eid)
+        if os.path.exists(xml_file):
+            short_name = None
+            for line in open(xml_file, "r"):
+                line = line.strip()
+                if ABB_SHORTNAME_RE.match(line):
+                    short_name = line[11:-12].trim()
+
+            if not short_name:
+                fns["fatal"]("No ShortName defined for '%s' in '%s" %
+                             (eid, locale))
+            engine_names[eid] = short_name
+        else:
+            fns["fatal"]("XML definiton for '%s' in '%s' missing" %
+                         (eid, locale))
+
+    fns["info"]("Remaining engine IDs: %s" % ", ".join(engine_ids))
+
+    # Create search engine order with real engine names
+    engine_order = []
+    for eid in whitelist:
+        sn = abb_get_shortname_from_id(eid, engine_ids, engine_names)
+        if sn:
+            engine_order.append(sn)
+
+    fns["info"]("Resulting ordered list: %s" % (", ".join(engine_order)))
+
+    # Read region.properties and remove browser.search.* lines
+    props = []
+    for line in open(region_file, "r"):
+        line = line.rstrip("\r\n")
+        if not ABB_SEARCH_PROPS_RE.match(line.strip()):
+            props.append(line)
+
+    # Append default search engine name
+    props.append("browser.search.defaultenginename=%s" % engine_order[0])
+
+    # Append search engine order
+    for i in range(0, min(3, len(engine_order))):
+        props.append("browser.search.order.%d=%s" % (i + 1, engine_order[i]))
+
+    # Write back list.txt
+    abb_write_lines(list_file, engine_ids)
+
+    # Write back region.properties
+    abb_write_lines(region_file, props)
+
+
+def abb_print_info(obj):
+    """ Wrapper for 'self.info' (to be self-contained) """
+    def fn(s):
+        if obj:
+            obj.info(s)
+        else:
+            print "INFO: %s" % s
+    return fn
+
+
+def abb_print_error(obj):
+    """ Wrapper for 'self.error' (to be self-contained) """
+    def fn(s):
+        if obj:
+            obj.error(s)
+        else:
+            print "ERROR: %s" % s
+    return fn
+
+
+def abb_exit_fatal(obj):
+    """ Wrapper for 'self.fatal' (to be self-contained) """
+    def fn(s):
+        if obj:
+            obj.fatal(s)
+        else:
+            print "FATAL: %s" % s
+            exit(1)
+    return fn
+
+
+def abb_transform_locales_impl(obj_dir, obj):
+    fns = {"info": abb_print_info(obj),
+           "error": abb_print_error(obj),
+           "fatal": abb_exit_fatal(obj)}
+
+    chrome_path = os.path.join(obj_dir, "dist", "bin", "chrome")
+    if not os.path.exists(chrome_path):
+        fns["fatal"]("'dist/bin/chrome' not existent in '%s'" % obj_dir)
+
+    locales = []
+    for p in next(os.walk(chrome_path))[1]:
+        if ABB_LOCALE_RE.match(p):
+            locales.append(p)
+    locales.sort()
+
+    fns["info"]("Found %d locales" % len(locales))
+
+    for locale in locales:
+        locale_path = os.path.join(chrome_path, locale, "locale", locale)
+        if os.path.exists(locale_path):
+            abb_transform_locale(locale, locale_path, fns)
+        else:
+            fns["error"]("Missing 'locale' folder for '%s'" % locale)
+