| OLD | NEW |
| 1 # coding: utf-8 | 1 # coding: utf-8 |
| 2 | 2 |
| 3 # This file is part of the Adblock Plus web scripts, | 3 # This file is part of the Adblock Plus web scripts, |
| 4 # Copyright (C) 2006-2013 Eyeo GmbH | 4 # Copyright (C) 2006-2013 Eyeo GmbH |
| 5 # | 5 # |
| 6 # Adblock Plus is free software: you can redistribute it and/or modify | 6 # Adblock Plus is free software: you can redistribute it and/or modify |
| 7 # it under the terms of the GNU General Public License version 3 as | 7 # it under the terms of the GNU General Public License version 3 as |
| 8 # published by the Free Software Foundation. | 8 # published by the Free Software Foundation. |
| 9 # | 9 # |
| 10 # Adblock Plus is distributed in the hope that it will be useful, | 10 # Adblock Plus is distributed in the hope that it will be useful, |
| (...skipping 16 matching lines...) Expand all Loading... |
| 27 return MySQLdb.connect(user=dbuser, passwd=dbpasswd, db=database, | 27 return MySQLdb.connect(user=dbuser, passwd=dbpasswd, db=database, |
| 28 use_unicode=True, charset="utf8", named_pipe=True) | 28 use_unicode=True, charset="utf8", named_pipe=True) |
| 29 else: | 29 else: |
| 30 return MySQLdb.connect(user=dbuser, passwd=dbpasswd, db=database, | 30 return MySQLdb.connect(user=dbuser, passwd=dbpasswd, db=database, |
| 31 use_unicode=True, charset="utf8") | 31 use_unicode=True, charset="utf8") |
| 32 | 32 |
| 33 def _get_cursor(): | 33 def _get_cursor(): |
| 34 return _get_db().cursor(MySQLdb.cursors.DictCursor) | 34 return _get_db().cursor(MySQLdb.cursors.DictCursor) |
| 35 | 35 |
| 36 def _hg(args): | 36 def _hg(args): |
| 37 return subprocess.Popen(["hg"] + args, stdout = subprocess.PIPE) | 37 return subprocess.check_output(["hg"] + args) |
| 38 | 38 |
| 39 def _extract_sites(easylist_dir): | 39 def _extract_sites(easylist_dir): |
| 40 os.chdir(easylist_dir) | 40 os.chdir(easylist_dir) |
| 41 process = _hg(["log", "--template", "{desc}\n"]) | 41 process = _hg(["log", "--template", "{desc}\n"]) |
| 42 urls = set([]) | 42 urls = set([]) |
| 43 | 43 |
| 44 for line in process.stdout: | 44 for line in process.stdout: |
| 45 match = re.search(r"\b(https?://\S*)", line) | 45 match = re.search(r"\b(https?://\S*)", line) |
| 46 if not match: | 46 if not match: |
| 47 continue | 47 continue |
| 48 | 48 |
| 49 url = match.group(1).strip() | 49 url = match.group(1).strip() |
| 50 urls.add(url) | 50 urls.add(url) |
| 51 | 51 |
| 52 return urls | 52 return urls |
| 53 | 53 |
| 54 def _insert_sites(site_urls): | 54 def _insert_sites(site_urls): |
| 55 cursor = _get_cursor() | 55 cursor = _get_cursor() |
| 56 for url in site_urls: | 56 for url in site_urls: |
| 57 cursor.execute("INSERT IGNORE INTO crawler_sites (url) VALUES (%s)", url) | 57 cursor.execute("INSERT IGNORE INTO crawler_sites (url) VALUES (%s)", url) |
| 58 _get_db().commit(); | 58 _get_db().commit(); |
| 59 | 59 |
| 60 if __name__ == "__main__": | 60 if __name__ == "__main__": |
| 61 easylist_dir = get_config().get("crawler", "easylist_repository") | 61 easylist_dir = get_config().get("crawler", "easylist_repository") |
| 62 site_urls = _extract_sites(easylist_dir) | 62 site_urls = _extract_sites(easylist_dir) |
| 63 _insert_sites(site_urls) | 63 _insert_sites(site_urls) |
| OLD | NEW |