Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: mozharness/abb/transform_locales.py

Issue 29628702: Issue 6076 - Remove old search engine ordering logic (Closed)
Patch Set: Removed obsolete shutil import Created Dec. 5, 2017, 10:47 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « mozharness/abb/ecosia.xml ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # This file is part of Adblock Plus 1 # This file is part of Adblock Plus
2 # Copyright (C) 2006-present eyeo GmbH 2 # Copyright (C) 2006-present eyeo GmbH
3 # 3 #
4 # Adblock Plus is free software: you can redistribute it and/or modify 4 # Adblock Plus is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License version 3 as 5 # it under the terms of the GNU General Public License version 3 as
6 # published by the Free Software Foundation. 6 # published by the Free Software Foundation.
7 # 7 #
8 # Adblock Plus is distributed in the hope that it will be useful, 8 # Adblock Plus is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details. 11 # GNU General Public License for more details.
12 # 12 #
13 # You should have received a copy of the GNU General Public License 13 # You should have received a copy of the GNU General Public License
14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
15 15
16 import os 16 import os
17 import re 17 import re
18 import shutil
19 import sys 18 import sys
20 19
21 _LOCALE_RE = re.compile("^([a-z]{2,3}(?:-[A-Z]{2})?)$") 20 _LOCALE_RE = re.compile("^([a-z]{2,3}(?:-[A-Z]{2})?)$")
22 _VALUES_LOCALE_RE = re.compile("^values-([a-z]{2,3}(?:-r[A-Z]{2})?)$") 21 _VALUES_LOCALE_RE = re.compile("^values-([a-z]{2,3}(?:-r[A-Z]{2})?)$")
23 22
24 _SEARCH_PROPS_RE = re.compile("^browser\.search\."
25 "(defaultenginename|order\.).*$")
26 _SHORTNAME_RE = re.compile("^<ShortName>(.*)</ShortName>$")
27
28 _PROPERTY_FORMAT_RE = re.compile("^(([^=]*)=)(.*)$") 23 _PROPERTY_FORMAT_RE = re.compile("^(([^=]*)=)(.*)$")
29 _ENTITY_FORMAT_RE = re.compile("^(\s*<!ENTITY\s*([^\"\s]*)\s*\")(.*)(\">)$") 24 _ENTITY_FORMAT_RE = re.compile("^(\s*<!ENTITY\s*([^\"\s]*)\s*\")(.*)(\">)$")
30 _STRING_FORMAT_RE = re.compile( 25 _STRING_FORMAT_RE = re.compile(
31 "^(\s*<string name=\"([^\"]*)\">)(.*)(</string>)$") 26 "^(\s*<string name=\"([^\"]*)\">)(.*)(</string>)$")
32 27
33 _MOZBUILD_PATH = os.path.join("python", "mozbuild") 28 _MOZBUILD_PATH = os.path.join("python", "mozbuild")
34 29
35 _CHROME_PATH = os.path.join("dist", "bin", "chrome") 30 _CHROME_PATH = os.path.join("dist", "bin", "chrome")
36 _RES_PATH = os.path.join("mobile", "android", "base", "res") 31 _RES_PATH = os.path.join("mobile", "android", "base", "res")
37 32
38 _BROWSER_DIR = "browser" 33 _BROWSER_DIR = "browser"
39 _SEARCHPLUGINS_PATH = os.path.join(_BROWSER_DIR, "searchplugins")
40 _LIST_TXT_PATH = os.path.join(_SEARCHPLUGINS_PATH, "list.txt")
41 _REGION_PROPS_PATH = os.path.join(_BROWSER_DIR, "region.properties")
42 34
43 _APPSTRINGS_PROPS_PATH = os.path.join(_BROWSER_DIR, "appstrings.properties") 35 _APPSTRINGS_PROPS_PATH = os.path.join(_BROWSER_DIR, "appstrings.properties")
44 _STRINGS_XML_PATH = "strings.xml" 36 _STRINGS_XML_PATH = "strings.xml"
45 37
46 _DEFAULT_LOCALE = "en-US" 38 _DEFAULT_LOCALE = "en-US"
47 39
48 # Add Ecosia as secondary search engine.
49 # See https://issues.adblockplus.org/ticket/5518
50 _ECOSIA_ID = "ecosia"
51 _ECOSIA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
52 "ecosia.xml")
53
54 _SEARCH_ENGINE_ORDER_DEFAULT = [
55 "duckduckgo",
56 "yahoo",
57 "google",
58 "wikipedia",
59 "amazon"]
60
61 _SEARCH_ENGINE_ORDER_ECOSIA = [
62 "duckduckgo",
63 "yahoo",
64 "google",
65 _ECOSIA_ID,
66 "wikipedia",
67 "amazon"]
68
69 _SEARCH_ENGINE_ORDER = {
70 "de": _SEARCH_ENGINE_ORDER_ECOSIA,
71 "en-GB": _SEARCH_ENGINE_ORDER_ECOSIA,
72 "en-US": _SEARCH_ENGINE_ORDER_ECOSIA,
73 "fr": _SEARCH_ENGINE_ORDER_ECOSIA,
74 "nl": _SEARCH_ENGINE_ORDER_ECOSIA,
75 "zh-CN": ["baidu",
76 "duckduckgo",
77 "yahoo",
78 "google",
79 "wikipedia",
80 "amazon"
81 ]
82 }
83
84 _FIREFOX_REPLACE_STR = "Firefox" 40 _FIREFOX_REPLACE_STR = "Firefox"
85 _ABB_REPLACEMENT_STR = "Adblock Browser" 41 _ABB_REPLACEMENT_STR = "Adblock Browser"
86 42
87 # Some string values that contain Firefox such as 'Firefox Sync' shouldn't be 43 # Some string values that contain Firefox such as 'Firefox Sync' shouldn't be
88 # replaced, so we keep a list of ids that are exceptions 44 # replaced, so we keep a list of ids that are exceptions
89 _ENTITY_EXCEPTIONS = [ 45 _ENTITY_EXCEPTIONS = [
90 "overlay_no_synced_devices", 46 "overlay_no_synced_devices",
91 "home_remote_tabs_need_to_sign_in", 47 "home_remote_tabs_need_to_sign_in",
92 "home_remote_tabs_need_to_finish_migrating", 48 "home_remote_tabs_need_to_finish_migrating",
93 "home_remote_tabs_need_to_verify", 49 "home_remote_tabs_need_to_verify",
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after
149 def _write_lines(filename, lines): 105 def _write_lines(filename, lines):
150 """Writes lines into file appending \\n""" 106 """Writes lines into file appending \\n"""
151 with open(filename, "w") as fd: 107 with open(filename, "w") as fd:
152 for l in lines: 108 for l in lines:
153 fd.write("%s\n" % l) 109 fd.write("%s\n" % l)
154 110
155 111
156 def _transform_locale(locale, path, logger): 112 def _transform_locale(locale, path, logger):
157 logger.info("Processing locale '%s'..." % locale) 113 logger.info("Processing locale '%s'..." % locale)
158 114
159 # Check for list.txt existence
160 list_file_path = os.path.join(path, _LIST_TXT_PATH)
161 _check_path_exists(list_file_path, logger)
162
163 # Check for region.properties existence
164 region_file_path = os.path.join(path, _REGION_PROPS_PATH)
165 _check_path_exists(region_file_path, logger)
166
167 # Check for appstrings.properties existence 115 # Check for appstrings.properties existence
168 appstrings_file_path = os.path.join(path, _APPSTRINGS_PROPS_PATH) 116 appstrings_file_path = os.path.join(path, _APPSTRINGS_PROPS_PATH)
169 _check_path_exists(appstrings_file_path, logger) 117 _check_path_exists(appstrings_file_path, logger)
170 118
171 ecosia_dst = os.path.join(path, _SEARCHPLUGINS_PATH,
172 os.path.basename(_ECOSIA_PATH))
173
174 # Get whitelist and build regex
175 whitelist = _SEARCH_ENGINE_ORDER.get(locale,
176 _SEARCH_ENGINE_ORDER_DEFAULT)
177 white_re = re.compile("^(%s).*$" % "|".join(whitelist))
178
179 if _ECOSIA_ID in whitelist and not os.path.exists(ecosia_dst):
180 shutil.copyfile(_ECOSIA_PATH, ecosia_dst)
181
182 # Read engine IDs from list.txt, discard engines not on whitelist
183 all_engine_ids = []
184 engine_ids = []
185 removed_engine_ids = []
186 with open(list_file_path, "r") as fd:
187 for line in fd:
188 line = line.strip()
189 all_engine_ids.append(line)
190 if len(line) > 0:
191 if white_re.match(line):
192 engine_ids.append(line)
193 else:
194 removed_engine_ids.append(line)
195
196 if _ECOSIA_ID in whitelist and _ECOSIA_ID not in all_engine_ids:
197 all_engine_ids.append(_ECOSIA_ID)
198 engine_ids.append(_ECOSIA_ID)
199
200 # Make sure we still have search engines left
201 if len(engine_ids) == 0:
202 logger.fatal("No search engines left over for '%s'" % locale)
203
204 # 'Parse' XML to get matching 'ShortName' for all engine IDs
205 engine_names = {}
206 for eid in engine_ids[:]:
207 xml_file_path = os.path.join(path, _SEARCHPLUGINS_PATH, "%s.xml" % eid)
208 if not os.path.exists(xml_file_path):
209 logger.info("Missing xml file for plugin %s. Searched in path %s" %
210 (eid, xml_file_path))
211 engine_ids.remove(eid)
212 continue
213 short_name = None
214 with open(xml_file_path, "r") as fd:
215 for line in fd:
216 line = line.strip()
217 match = _SHORTNAME_RE.match(line)
218 if match:
219 short_name = match.group(1).strip()
220
221 if not short_name:
222 logger.fatal("No ShortName defined for '%s' in '%s" %
223 (eid, locale))
224 engine_names[eid] = short_name
225
226 logger.info("Removed search engine IDs: %s" %
227 ", ".join(removed_engine_ids))
228 logger.info("Remaining search engine IDs: %s" % ", ".join(engine_ids))
229
230 # Create search engine order with real engine names
231 engine_order = []
232 for eid in whitelist:
233 sn = _get_shortname_from_id(eid, engine_ids, engine_names)
234 if sn:
235 engine_order.append(sn)
236
237 logger.info("Resulting search engine ordered list: %s" %
238 (", ".join(engine_order)))
239
240 # Read region.properties and remove browser.search.* lines
241 props = []
242 with open(region_file_path, "r") as fd:
243 for line in fd:
244 line = line.rstrip("\r\n")
245 if not _SEARCH_PROPS_RE.match(line.strip()):
246 props.append(line)
247
248 # Append default search engine name
249 props.append("browser.search.defaultenginename=%s" % engine_order[0])
250
251 # Append search engine order
252 for i in range(0, min(5, len(engine_order))):
253 props.append("browser.search.order.%d=%s" % (i + 1, engine_order[i]))
254
255 # Write back list.txt
256 _write_lines(list_file_path, all_engine_ids)
257
258 # Write back region.properties
259 _write_lines(region_file_path, props)
260
261 # Replaces ocurrences of 'Firefox' by 'Adblock Browser' in 119 # Replaces ocurrences of 'Firefox' by 'Adblock Browser' in
262 # 'appstrings.properties' 120 # 'appstrings.properties'
263 lines = [] 121 lines = []
264 replacement_count = 0 122 replacement_count = 0
265 123
266 with open(appstrings_file_path, "r") as fd: 124 with open(appstrings_file_path, "r") as fd:
267 for line in fd: 125 for line in fd:
268 line = line.rstrip("\r\n") 126 line = line.rstrip("\r\n")
269 replacement = _replace_in_value(_PROPERTY_FORMAT_RE, line, 127 replacement = _replace_in_value(_PROPERTY_FORMAT_RE, line,
270 _FIREFOX_REPLACE_STR, 128 _FIREFOX_REPLACE_STR,
(...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after
364 if os.path.exists(locale_path): 222 if os.path.exists(locale_path):
365 _transform_locale(locale, locale_path, logger) 223 _transform_locale(locale, locale_path, logger)
366 _generate_browser_search(locale, locale_path, res_path, build_dir) 224 _generate_browser_search(locale, locale_path, res_path, build_dir)
367 else: 225 else:
368 logger.error("Missing folder for locale '%s' in path: %s" % 226 logger.error("Missing folder for locale '%s' in path: %s" %
369 (locale, locale_path)) 227 (locale, locale_path))
370 228
371 for locale in values_locales: 229 for locale in values_locales:
372 locale_path = os.path.join(res_path, "values-" + locale) 230 locale_path = os.path.join(res_path, "values-" + locale)
373 _transform_values_locale(locale, locale_path, logger) 231 _transform_values_locale(locale, locale_path, logger)
OLDNEW
« no previous file with comments | « mozharness/abb/ecosia.xml ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld