Issue 2951 - Fix handling of URLs without a scheme

run.py

 #!/usr/bin/env python
 # coding: utf-8
 
 import argparse
 import datetime
 import errno
 import hashlib
 import io
 import json
 import os
@@ skipping 28 matching lines @@
     elif path == '/save':
       try:
         request_body_size = int(environ.get('CONTENT_LENGTH', 0))
       except (ValueError):
         start_response('400 Bad Request', [])
         return ''
 
       data = json.loads(environ['wsgi.input'].read(request_body_size))
       self.urls.remove(data['url'])
 
-      fullurl = data['url'] if ':' in data['url'] else 'http://' + data['url']

[Sebastian Noack, 2015/08/26 13:11:52]

Please don't repeat yourself:

  fullurl = data['url']
  if ':' not in fullurl:
    fullurl = 'http://' + fullurl

Also what's about example.com:8080 for example? If you just check for ":" it
will give you an unexpected result if the there is no protocol but a port given.
Probably the safest and simplest way would be to parse the URL:

  fullurl = data['url']
  if not urlparse(fullurl).scheme:
    fullurl = 'http://' + fullurl

[Wladimir Palant, 2015/10/09 10:27:34]

Done.

+      fullurl = data['url']
+      if not urlparse.urlparse(fullurl).scheme:
+        fullurl = 'http://' + fullurl
       parsedurl = urlparse.urlparse(fullurl)
       urlhash = hashlib.new('md5', data['url']).hexdigest()
       timestamp = datetime.datetime.fromtimestamp(data['startTime'] / 1000.0).strftime('%Y-%m-%dT%H%M%S.%f')
       basename = "%s-%s-%s" % (parsedurl.hostname, timestamp, urlhash)
       datapath = os.path.join(self.parameters.outdir, basename + ".json")
       screenshotpath = os.path.join(self.parameters.outdir, basename + ".jpg")
       sourcepath = os.path.join(self.parameters.outdir, basename + ".xml")
 
       try:
         os.makedirs(self.parameters.outdir)
@@ skipping 127 matching lines @@
   BASE_DIR = os.path.dirname(os.path.abspath(__file__))
   DEPENDENCY_SCRIPT = os.path.join(BASE_DIR, "ensure_dependencies.py")
 
   try:
     subprocess.check_call([sys.executable, DEPENDENCY_SCRIPT, BASE_DIR])
   except subprocess.CalledProcessError as e:
     print >>sys.stderr, e
     print >>sys.stderr, "Failed to ensure dependencies being up-to-date!"
 
   run()