OLD | NEW |
1 # coding: utf-8 | |
2 | |
3 # This file is part of the Adblock Plus web scripts, | 1 # This file is part of the Adblock Plus web scripts, |
4 # Copyright (C) 2006-2016 Eyeo GmbH | 2 # Copyright (C) 2006-2016 Eyeo GmbH |
5 # | 3 # |
6 # Adblock Plus is free software: you can redistribute it and/or modify | 4 # Adblock Plus is free software: you can redistribute it and/or modify |
7 # it under the terms of the GNU General Public License version 3 as | 5 # it under the terms of the GNU General Public License version 3 as |
8 # published by the Free Software Foundation. | 6 # published by the Free Software Foundation. |
9 # | 7 # |
10 # Adblock Plus is distributed in the hope that it will be useful, | 8 # Adblock Plus is distributed in the hope that it will be useful, |
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of | 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
(...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
62 def _insert_sites(site_urls): | 60 def _insert_sites(site_urls): |
63 cursor = _get_cursor() | 61 cursor = _get_cursor() |
64 for url in site_urls: | 62 for url in site_urls: |
65 cursor.execute("INSERT IGNORE INTO crawler_sites (url) VALUES (%s)", url
) | 63 cursor.execute("INSERT IGNORE INTO crawler_sites (url) VALUES (%s)", url
) |
66 _get_db().commit() | 64 _get_db().commit() |
67 | 65 |
68 if __name__ == "__main__": | 66 if __name__ == "__main__": |
69 easylist_dir = get_config().get("crawler", "easylist_repository") | 67 easylist_dir = get_config().get("crawler", "easylist_repository") |
70 site_urls = _extract_sites(easylist_dir) | 68 site_urls = _extract_sites(easylist_dir) |
71 _insert_sites(site_urls) | 69 _insert_sites(site_urls) |
OLD | NEW |