Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/crawler/bin/extract_crawler_sites.py

Issue 8327353: Crawler backend (Closed)
Patch Set: Created Sept. 27, 2012, 2:15 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 # coding: utf-8
2
3 # This Source Code is subject to the terms of the Mozilla Public License
4 # version 2.0 (the "License"). You can obtain a copy of the License at
5 # http://mozilla.org/MPL/2.0/.
6
7 import MySQLdb, os, re, subprocess
8 from sitescripts.utils import get_config
9
10 def hg(args):
11 return subprocess.Popen(["hg"] + args, stdout = subprocess.PIPE)
12
13 def extract_urls(filter_list_dir):
14 os.chdir(filter_list_dir)
15 process = hg(["log", "--template", "{desc}\n"])
16 urls = set([])
17
18 for line in process.stdout:
19 match = re.search(r"\b(https?://\S*)", line)
20 if not match:
21 continue
22
23 url = match.group(1).strip()
24 urls.add(url)
25
26 return urls
27
28 def print_statements(urls):
29 for url in urls:
30 escaped_url = MySQLdb.escape_string(url)
31 print "INSERT INTO crawler_sites (url) VALUES ('" + escaped_url + "');"
32
33 if __name__ == "__main__":
34 filter_list_dir = get_config().get("crawler", "filter_list_repository")
35 urls = extract_urls(filter_list_dir)
36 print_statements(urls)
OLDNEW

Powered by Google App Engine
This is Rietveld