Implemented extraction of URL Fixer data

sitescripts/urlfixer/bin/topDomains.py

Index: sitescripts/urlfixer/bin/topDomains.py
===================================================================
new file mode 100644
--- /dev/null
+++ b/sitescripts/urlfixer/bin/topDomains.py
@@ -0,0 +1,95 @@
+# coding: utf-8
+
+# This file is part of the Adblock Plus web scripts,
+# Copyright (C) 2006-2012 Eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys, os, re, math, MySQLdb
+from sitescripts.utils import get_config, setupStderr
+
+"""
+This script produces the list of top correct domain names currently in the
+database.
+"""
+
+STATUS_TYPED = 1
+STATUS_TYPO = 2
+STATUS_CORRECTION = 3
+STATUS_FALSE_POSITIVE = 4
+
+def getTopDomains(count=1000):

[Wladimir Palant, 2012/11/28 10:49:24]

Note that this is count=5000 on the server currently. Ideally we will soon
increase it to 10000, provided our data sample is good enough.

[Thomas Greiner, 2012/11/28 11:17:39]

Ok.

+  db = _get_db()
+  cursor = db.cursor(MySQLdb.cursors.DictCursor)
+  cursor.execute("SELECT id, domain, forceinclusion FROM domains")
+
+  domains = {}
+  mandatory = []
+  for result in cursor:
+    domain = result["domain"]
+    if ("." not in domain or not re.search(r"[a-zA-Z]", domain) or
+        re.search(r"['\"_,<>;]|^\.|\.$|\.\.", domain)):

[Thomas Greiner, 2012/11/28 10:05:42]

That's a small selection of special characters. It will still accept URLs such
as "www.a$%&§b.c0m". A whitelist should be able to do a better job at that.

My recommendation:
not re.search(r"^([\da-z\.-]+)\.([a-z\.]{2,6})$", domain)

[Wladimir Palant, 2012/11/28 10:49:24]

What about international domain names? I explicitly didn't want to reject
something that might be valid, that's why I limited it to a few things I noticed
in our data.

[Thomas Greiner, 2012/11/28 11:17:39]

Ok. For now I think it would be sufficient to check for special characters from
the ASCII character set so that we at least detect $, %, & and such as well.

We should resume that discussion when we include the filters in URL Fixer which
might result in sending a different version number to the server.

+      continue
+
+    typed = _get_weighted_count(db, result["id"], STATUS_TYPED)
+    correction = _get_weighted_count(db, result["id"], STATUS_CORRECTION)
+    typo = _get_weighted_count(db, result["id"], STATUS_TYPO)
+    fp = _get_weighted_count(db, result["id"], STATUS_FALSE_POSITIVE)
+    correctness = _calculate_correctness(typed + correction, typo + fp)
+
+    domains[domain] = correctness
+    if result["forceinclusion"]:
+      mandatory.append(domain)
+  return sorted(domains.iterkeys(), key=lambda d: domains[d], reverse=True)[:count] + mandatory
+
+def _get_weighted_count(db, domain, status):
+  cursor = db.cursor(MySQLdb.cursors.DictCursor)
+  cursor.execute("""SELECT curr_month * 0.4 + prev_month * 0.3 +
+                      curr_year * 0.2 + prev_year * 0.1 AS result

[Thomas Greiner, 2012/11/28 10:05:42]

|result| is not very descriptive here. Please rename. (e.g. |weighted_count|)

+                    FROM corrections WHERE domain = %s AND status = %s""",
+                 (domain, status))
+  result = cursor.fetchone()
+  if result == None:
+    return 0
+  else:
+    return result["result"]

[Thomas Greiner, 2012/11/28 10:05:42]

|result["result"]| is not very descriptive here. Please rename. (e.g.
|result["weighted_score"]|)

+
+def _calculate_correctness(positive, negative):
+  if positive + negative > 0:
+    # Determine the correctness score with a confidence level of 0.95
+    # (see http://www.evanmiller.org/how-not-to-sort-by-average-rating.html)
+    fp = float(positive)
+    fn = float(negative)
+    total = fp + fn
+    return ((fp + 1.9208) / total - 1.96 * math.sqrt((fp * fn) / total +
+        0.9604) / total) / (1 + 3.8416 / total)
+  else:
+    return 0
+
+def _get_db():
+  database = get_config().get("urlfixer", "database")
+  dbuser = get_config().get("urlfixer", "dbuser")
+  dbpasswd = get_config().get("urlfixer", "dbpassword")
+  if os.name == "nt":
+    return MySQLdb.connect(user=dbuser, passwd=dbpasswd, db=database,
+                           use_unicode=True, charset="utf8", named_pipe=True)
+  else:
+    return MySQLdb.connect(user=dbuser, passwd=dbpasswd, db=database,
+                           use_unicode=True, charset="utf8")
+
+if __name__ == '__main__':
+  setupStderr()
+
+  domains = getTopDomains()
+  for domain in domains:
+    print domain.encode("utf-8")