Implemented extraction of URL Fixer data

sitescripts/urlfixer/bin/topDomains.py

 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2012 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
@@ skipping 20 matching lines @@
 def getTopDomains(count=5000):
   db = _get_db()
   cursor = db.cursor(MySQLdb.cursors.DictCursor)
   cursor.execute("SELECT id, domain, forceinclusion FROM domains")
 
   domains = {}
   mandatory = []
   for result in cursor:
     domain = result["domain"]
     if ("." not in domain or not re.search(r"[a-zA-Z]", domain) or
-        re.search(r"['\"_,<>;]|^\.|\.$|\.\.", domain)):
+        re.search(r"['\"_,<>:;!$%&/()*+#~]|^\.|\.$|\.\.", domain)):
       continue
 
     typed = _get_weighted_count(db, result["id"], STATUS_TYPED)
     correction = _get_weighted_count(db, result["id"], STATUS_CORRECTION)
     typo = _get_weighted_count(db, result["id"], STATUS_TYPO)
     fp = _get_weighted_count(db, result["id"], STATUS_FALSE_POSITIVE)
     correctness = _calculate_correctness(typed + correction, typo + fp)
 
     domains[domain] = correctness
     if result["forceinclusion"]:
@@ skipping 34 matching lines @@
   else:
     return MySQLdb.connect(user=dbuser, passwd=dbpasswd, db=database,
                            use_unicode=True, charset="utf8")
 
 if __name__ == '__main__':
   setupStderr()
 
   domains = getTopDomains()
   for domain in domains:
     print domain.encode("utf-8")