Issue 4814 - Adds csv log to formmail2

sitescripts/formmail/web/formmail2.py

 # This file is part of the Adblock Plus web scripts,
-# Copyright (C) 2006-2016 Eyeo GmbH
+# Copyright (C) 2006-2017 eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import print_function
+
 import os
+import sys
 import datetime
+import traceback
 import collections
 from csv import DictWriter, DictReader
+
+import jinja2
 
 from sitescripts.utils import (get_config, sendMail, encode_email_address,
                                get_template)
 from sitescripts.web import registerUrlHandler, form_handler
 
 
 def get_config_items():
     config = get_config()
     default_keys = set(config.defaults())
     for name, value in config.items('formmail2'):
@@ skipping 40 matching lines @@
         writer.writerow(parameters)
     raise Exception('Field names have changed, error log '
                     'written to ' + err_path)
 
 
 def log_formdata(params, path):
     if os.path.isfile(path):
         with open(path, 'ab+') as formlog:
             formlog.seek(0)
             reader = DictReader(formlog)
             if reader.fieldnames != params.keys():

[Jon Sonesen, 2017/03/22 09:23:30]

I think this is incorrect, after some local testing I found that if the fields
submitted by the user do not match what has been configured (ie the user is
crafting their own post requests) sendMail raises a 500, but the error log is
written with the additional field since only validates the user submited params.
So I think I need to also add logic to check that the fieldnames differ from the
configured fields. 

Does that make sense?

[Jon Sonesen, 2017/03/22 14:28:21]

Done.

                 log_formfield_error(params, path)
             formlog.seek(os.SEEK_END)
             writer = DictWriter(formlog, fieldnames=params.keys())
             writer.writerow(params)
         return
     with open(path, 'w') as new_formlog:
         writer = DictWriter(new_formlog, fieldnames=params.keys())
         writer.writeheader()
         writer.writerow(params)
     return
 
 
 def validate_fields(fields, params):
     errors = []
     for field, spec in fields.items():
-        if 'mandatory' in spec.value:
-            if field not in params.keys():
+        if 'mandatory' in spec.value and field not in params:
                 errors.append(make_error(spec, 'mandatory',
                                          'No {} entered'.format(field)))
-        if 'email' in spec.value and field in params.keys():
+        if 'email' in spec.value and field in params:
             try:
                 params[field] = encode_email_address(params[field])
             except ValueError:
                 errors.append(make_error(spec, 'email', 'Invalid email'))
+
+    unexpected_fields = ' '.join(set(params.keys()) - set(fields.keys()))
+    if unexpected_fields:
+        errors.append('Unexpected field/fields: ' + str(unexpected_fields))
     return errors
 
 
 def make_handler(name, config):
     try:
         url = config['url'].value
     except (KeyError, AttributeError):
         raise Exception('No URL configured for form handler: ' + name)
     try:
         template = config['template'].value
         get_template(template, autoescape=False)
     except (KeyError, AttributeError):
         raise Exception('No template configured for form handler: ' + name)
+    except (jinja2.TemplateNotFound):
+        raise Exception('Template not found at: ' + template)
     try:
         fields = config['fields']
         for field, spec in fields.items():
             spec.value = {s.strip() for s in spec.value.split(',')}
     except KeyError:
         raise Exception('No fields configured for form handler: ' + name)
     if len(fields) == 0:
         raise Exception('No fields configured for form handler: ' + name)
 
     @form_handler
     def handler(environ, start_response, params):
         response_headers = [('Content-Type', 'text/plain; charset=utf-8')]
         errors = validate_fields(fields, params)
         if errors:
             start_response('400 Bad Request', response_headers)
             return '\n'.join(errors)
         time = datetime.datetime.now()
         template_args = {
             'time': time,
             'fields': {field: params.get(field, '') for field in fields}
         }
         try:
             sendMail(template, template_args)
         except:
+            print(traceback.print_exc(), file=sys.stderr)
             start_response('500 Server Error', response_headers)

[Vasily Kuznetsov, 2017/03/21 14:00:35]

I wonder if we could log the stacktrace here somehow. Perhaps `stderr` would go
to the server log. Let's ask Matze what happens when multiplexer scripts print
to `stderr`.

[Jon Sonesen, 2017/03/22 14:28:21]

Done.

             return ''
         finally:
             if 'csv_log' in config:
                 params = {field: params.get(field, '').encode('utf8')
                           for field in fields}
                 params['time'] = time
                 log_formdata(params, config['csv_log'].value)
         start_response('200 OK', response_headers)
         return ''
 
     return url, handler
 
 
 conf_dict = conf_parse(get_config_items())
 for name, config in conf_dict.items():
     url, handler = make_handler(name, config)
     registerUrlHandler(url, handler)