Implemented extraction of URL Fixer data

sitescripts/urlfixer/bin/topDomains.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2012 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
@@ skipping 10 matching lines @@
 """
 This script produces the list of top correct domain names currently in the
 database.
 """
 
 STATUS_TYPED = 1
 STATUS_TYPO = 2
 STATUS_CORRECTION = 3
 STATUS_FALSE_POSITIVE = 4
 
-def getTopDomains(count=1000):

[Wladimir Palant, 2012/11/28 10:49:24]

Note that this is count=5000 on the server currently. Ideally we will soon
increase it to 10000, provided our data sample is good enough.

[Thomas Greiner, 2012/11/28 11:17:39]

Ok.

+def getTopDomains(count=5000):
   db = _get_db()
   cursor = db.cursor(MySQLdb.cursors.DictCursor)
   cursor.execute("SELECT id, domain, forceinclusion FROM domains")
 
   domains = {}
   mandatory = []
   for result in cursor:
     domain = result["domain"]
     if ("." not in domain or not re.search(r"[a-zA-Z]", domain) or
-        re.search(r"['\"_,<>;]|^\.|\.$|\.\.", domain)):

[Thomas Greiner, 2012/11/28 10:05:42]

That's a small selection of special characters. It will still accept URLs such
as "www.a$%&§b.c0m". A whitelist should be able to do a better job at that.

My recommendation:
not re.search(r"^([\da-z\.-]+)\.([a-z\.]{2,6})$", domain)

[Wladimir Palant, 2012/11/28 10:49:24]

What about international domain names? I explicitly didn't want to reject
something that might be valid, that's why I limited it to a few things I noticed
in our data.

[Thomas Greiner, 2012/11/28 11:17:39]

Ok. For now I think it would be sufficient to check for special characters from
the ASCII character set so that we at least detect $, %, & and such as well.

We should resume that discussion when we include the filters in URL Fixer which
might result in sending a different version number to the server.

+        re.search(r"['\"_,<>:;!$%&/()*+#~]|^\.|\.$|\.\.", domain)):
       continue
 
     typed = _get_weighted_count(db, result["id"], STATUS_TYPED)
     correction = _get_weighted_count(db, result["id"], STATUS_CORRECTION)
     typo = _get_weighted_count(db, result["id"], STATUS_TYPO)
     fp = _get_weighted_count(db, result["id"], STATUS_FALSE_POSITIVE)
     correctness = _calculate_correctness(typed + correction, typo + fp)
 
     domains[domain] = correctness
     if result["forceinclusion"]:
       mandatory.append(domain)
   return sorted(domains.iterkeys(), key=lambda d: domains[d], reverse=True)[:count] + mandatory
 
 def _get_weighted_count(db, domain, status):
   cursor = db.cursor(MySQLdb.cursors.DictCursor)
   cursor.execute("""SELECT curr_month * 0.4 + prev_month * 0.3 +
-                      curr_year * 0.2 + prev_year * 0.1 AS result

[Thomas Greiner, 2012/11/28 10:05:42]

|result| is not very descriptive here. Please rename. (e.g. |weighted_count|)

+                      curr_year * 0.2 + prev_year * 0.1 AS weighted_count
                     FROM corrections WHERE domain = %s AND status = %s""",
                  (domain, status))
   result = cursor.fetchone()
   if result == None:
     return 0
   else:
-    return result["result"]

[Thomas Greiner, 2012/11/28 10:05:42]

|result["result"]| is not very descriptive here. Please rename. (e.g.
|result["weighted_score"]|)

+    return result["weighted_count"]
 
 def _calculate_correctness(positive, negative):
   if positive + negative > 0:
     # Determine the correctness score with a confidence level of 0.95
     # (see http://www.evanmiller.org/how-not-to-sort-by-average-rating.html)
     fp = float(positive)
     fn = float(negative)
     total = fp + fn
     return ((fp + 1.9208) / total - 1.96 * math.sqrt((fp * fn) / total +
         0.9604) / total) / (1 + 3.8416 / total)
@@ skipping 10 matching lines @@
   else:
     return MySQLdb.connect(user=dbuser, passwd=dbpasswd, db=database,
                            use_unicode=True, charset="utf8")
 
 if __name__ == '__main__':
   setupStderr()
 
   domains = getTopDomains()
   for domain in domains:
     print domain.encode("utf-8")