OLD | NEW |
1 # coding: utf-8 | 1 # coding: utf-8 |
2 | 2 |
3 # This file is part of the Adblock Plus web scripts, | 3 # This file is part of the Adblock Plus web scripts, |
4 # Copyright (C) 2006-2016 Eyeo GmbH | 4 # Copyright (C) 2006-2016 Eyeo GmbH |
5 # | 5 # |
6 # Adblock Plus is free software: you can redistribute it and/or modify | 6 # Adblock Plus is free software: you can redistribute it and/or modify |
7 # it under the terms of the GNU General Public License version 3 as | 7 # it under the terms of the GNU General Public License version 3 as |
8 # published by the Free Software Foundation. | 8 # published by the Free Software Foundation. |
9 # | 9 # |
10 # Adblock Plus is distributed in the hope that it will be useful, | 10 # Adblock Plus is distributed in the hope that it will be useful, |
(...skipping 24 matching lines...) Expand all Loading... |
35 | 35 |
36 def getTopDomains(count=5000): | 36 def getTopDomains(count=5000): |
37 db = _get_db() | 37 db = _get_db() |
38 cursor = db.cursor(MySQLdb.cursors.DictCursor) | 38 cursor = db.cursor(MySQLdb.cursors.DictCursor) |
39 cursor.execute("SELECT id, domain, forceinclusion FROM domains") | 39 cursor.execute("SELECT id, domain, forceinclusion FROM domains") |
40 | 40 |
41 domains = {} | 41 domains = {} |
42 mandatory = [] | 42 mandatory = [] |
43 for result in cursor: | 43 for result in cursor: |
44 domain = result["domain"] | 44 domain = result["domain"] |
45 if ("." not in domain or not re.search(r"[a-zA-Z]", domain) or | 45 if "." not in domain or not re.search(r"[a-zA-Z]", domain): |
46 re.search(r"['\"_,<>:;!$%&/()*+#~]|^\.|\.$|\.\.", domain)): | 46 continue |
| 47 if re.search(r"['\"_,<>:;!$%&/()*+#~]|^\.|\.$|\.\.", domain): |
47 continue | 48 continue |
48 | 49 |
49 typed = _get_weighted_count(db, result["id"], STATUS_TYPED) | 50 typed = _get_weighted_count(db, result["id"], STATUS_TYPED) |
50 correction = _get_weighted_count(db, result["id"], STATUS_CORRECTION) | 51 correction = _get_weighted_count(db, result["id"], STATUS_CORRECTION) |
51 typo = _get_weighted_count(db, result["id"], STATUS_TYPO) | 52 typo = _get_weighted_count(db, result["id"], STATUS_TYPO) |
52 fp = _get_weighted_count(db, result["id"], STATUS_FALSE_POSITIVE) | 53 fp = _get_weighted_count(db, result["id"], STATUS_FALSE_POSITIVE) |
53 correctness = _calculate_correctness(typed + correction, typo + fp) | 54 correctness = _calculate_correctness(typed + correction, typo + fp) |
54 | 55 |
55 domains[domain] = correctness | 56 domains[domain] = correctness |
56 if result["forceinclusion"]: | 57 if result["forceinclusion"]: |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
94 else: | 95 else: |
95 return MySQLdb.connect(user=dbuser, passwd=dbpasswd, db=database, | 96 return MySQLdb.connect(user=dbuser, passwd=dbpasswd, db=database, |
96 use_unicode=True, charset="utf8") | 97 use_unicode=True, charset="utf8") |
97 | 98 |
98 if __name__ == '__main__': | 99 if __name__ == '__main__': |
99 setupStderr() | 100 setupStderr() |
100 | 101 |
101 domains = getTopDomains() | 102 domains = getTopDomains() |
102 for domain in domains: | 103 for domain in domains: |
103 print domain.encode("utf-8") | 104 print domain.encode("utf-8") |
OLD | NEW |