Subscription downloads: Ignore stated charset for remote downloads, always assume UTF-8

sitescripts/subscriptions/combineSubscriptions.py

 #!/usr/bin/env python
 # coding: utf-8
 
 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-2012 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
@@ skipping 139 matching lines @@
           try:
             request = urllib2.urlopen(file, None, timeout)
             error = None
             break
           except urllib2.URLError, e:
             error = e
             time.sleep(5)
         if error:
           raise error
 
-        charset = 'utf-8'
-        contentType = request.headers.get('content-type', '')
-        if contentType.find('charset=') >= 0:
-          charset = contentType.split('charset=', 1)[1]
-        newLines = unicode(request.read(), charset).split('\n')
+        # We should really get the charset from the headers rather than assuming
+        # that it is UTF-8. However, some of the Google Code mirrors are
+        # misconfigured and will return ISO-8859-1 as charset instead of UTF-8.
+        newLines = unicode(request.read(), 'utf-8').split('\n')
         newLines = map(lambda l: re.sub(r'[\r\n]', '', l), newLines)
         newLines = filter(lambda l: not re.search(r'^\s*!.*?\bExpires\s*(?::|after)\s*(\d+)\s*(h)?', l, re.M | re.I), newLines)
         newLines = filter(lambda l: not re.search(r'^\s*!\s*(Redirect|Homepage|Title)\s*:', l, re.M | re.I), newLines)
       else:
         result.append('! *** %s ***' % file)
 
         includeSource = sourceName
         if file.find(':') >= 0:
           includeSource, file = file.split(':', 1)
         if not includeSource in sourceDirs:
@@ skipping 151 matching lines @@
       usage()
       sys.exit()
     elif option in ('-t', '--timeout'):
       timeout = int(value)
 
   if os.path.exists(os.path.join(sourceDir, '.hg')):
     # Our source is a Mercurial repository, try updating
     subprocess.Popen(['hg', '-R', sourceDir, 'pull', '--update']).communicate()
 
   combineSubscriptions(sourceDir, targetDir, timeout)