Crawler backend

sitescripts/crawler/web/crawler.py

 import MySQLdb, os, re, simplejson, sys
 from sitescripts.utils import cached, get_config
 from sitescripts.web import url_handler, basic_auth
 
 @cached(600)
 def _get_db():
   database = get_config().get("crawler", "database")
   dbuser = get_config().get("crawler", "dbuser")
   dbpasswd = get_config().get("crawler", "dbpassword")
   if os.name == "nt":
     return MySQLdb.connect(user=dbuser, passwd=dbpasswd, db=database,
                            use_unicode=True, charset="utf8", named_pipe=True)
   else:
     return MySQLdb.connect(user=dbuser, passwd=dbpasswd, db=database,
                            use_unicode=True, charset="utf8")
 
 def get_cursor():
   return _get_db().cursor(MySQLdb.cursors.DictCursor)
 
 def _fetch_crawlable_sites():
   cursor = get_cursor()
   cursor.execute("SELECT url from crawler_sites")
   results = cursor.fetchall()
   sites = [result["url"] for result in results]
   return sites
 
 @url_handler("/crawlableSites")
-@basic_auth
+@basic_auth("crawler")
 def crawlable_sites(environ, start_response):
   urls = _fetch_crawlable_sites()
   start_response("200 OK", [("Content-Type", "text/plain")])
   return "\n".join(urls)
 
 def _find_site_id(site_url):
   cursor = get_cursor()
   cursor.execute("SELECT id FROM crawler_sites WHERE url = %s", site_url)
   result = cursor.fetchone()
   return result["id"] if result else None
 
 def _read_multipart_lines(environ, line_callback):
   data_file = environ["wsgi.input"]
-  boundary = re.search(r"boundary=(.*)", environ["CONTENT_TYPE"]).group(1)

[Wladimir Palant, 2012/09/27 07:34:17]

Assumptions here:

1) There is a CONTENT_TYPE header.
2) It starts with "multipart/form-data".
3) It also contains "boundary=" substring.

These are probably valid assumptions to make but I would normally make sure to
produce a proper error message if any of these is violated.

[Felix Dahlke, 2012/09/27 09:26:24]

Done. You're finding a lot of those because my mindset has always been to write
the simplest code possible, see what exceptions occur during testing and handle
errors properly where necessary. But that was in environments with lots of
testing before deployment, so I guess I'll better get used to handling things
like that properly form the start.

+  content_type = environ.get("CONTENT_TYPE")
+  if not content_type:
+    raise ValueError("Content-Type missing from header")
+
+  match = re.search(r"boundary=(.*)", content_type)
+  if not match:
+    raise ValueError("Multipart form data or boundary declaration missing")
+
+  boundary = match.group(1)
   boundary_passed = False
   header_passed = False
 
   for line in data_file:
     line = line.strip()
 
     if not boundary_passed:
       if line == "--" + boundary:
         boundary_passed = True
       continue
@@ skipping 12 matching lines @@
 def _create_run():
   cursor = get_cursor()
   cursor.execute("INSERT INTO crawler_runs () VALUES ()")
   return cursor.lastrowid
 
 def _insert_data(run_id, site, url, filtered):
   site_id = _find_site_id(site)
   if site_id is None:
     print >>sys.stderr, "Unable to find site '%s' in the database" % site
     return
 

[Wladimir Palant, 2012/09/27 07:34:17]

Thinking about that... So we give the client a list of URLs from crawler_sites
table but ignore IDs, then we get the URL back from the client and have to find
the corresponding ID? Seems not very logical. Maybe send both the URL and the ID
to the client and have it use the site ID in the data it sends back?

[Felix Dahlke, 2012/09/27 09:26:24]

I did that for two reasons:
1. The site urls are definitely unique (now even guaranteed by the database)
2. The ID is a database detail that could actually change for some reason, e.g.
denormalization

The problem is that this is blowing the size of the crawlerData request up
somewhat. I was planning to wait and see if that's going to be a problem.

   cursor = get_cursor()
   cursor.execute("""
 INSERT INTO crawler_data (run, site, url, filtered)
 VALUES (%s, %s, %s, %s)""",
                  (run_id, site_id, url, filtered))
 
 @url_handler("/crawlerData")
-@basic_auth
+@basic_auth("crawler")
 def crawler_data(environ, start_response):
   def line_callback(line):
     try:
       url, site, filtered = simplejson.loads(line)
       _insert_data(run_id, site, url, filtered)
     except simplejson.JSONDecodeError:
       print >>sys.stderr, "Unable to parse JSON from '%s'" % line
 
   run_id = _create_run()
-  _read_multipart_lines(environ, line_callback)
-  start_response("200 OK", [("Content-Type", "text/plain")])
-  return ""
+  try:
+    _read_multipart_lines(environ, line_callback)
+    start_response("200 OK", [("Content-Type", "text/plain")])
+    return ""
+  except ValueError as e:
+    start_response("401 Bad Request", [("Content-Type", "text/plain")])
+    return e