Issue 3943 - add support of configuration file

run.py

 #!/usr/bin/env python
 # coding: utf-8
 
 import argparse
 import datetime
 import errno
 import hashlib
 import io
 import json
 import os
@@ skipping 79 matching lines @@
 
 
 class Parameters:
     """This class loads config file and parses command line parameters.
        Values are stored in attibutes of this class instance.
     """
     def __init__(self):
         cli_parameters = vars(Parameters._parse_command_line())
         config_parameters = Parameters._load_config(cli_parameters["config"])
         for field in cli_parameters.keys():
-            config_value = config_parameters[field] if field in config_parameters else None
-            value = cli_parameters[field] if cli_parameters[field] is not None else config_value

[tschuster, 2016/11/21 18:07:55]

Can't you just write:
value = cli_parameters[field] if cli_parameters[field] is not None else
config_parameters[field]

We should document that CLI parameters take precedence.

[sergei, 2016/11/22 09:23:31]

Changed. Now lines are not so long and you are right, config_parameters.get is
better than our implementation. config_parameters[field] throws exception if
there is no such key.
It's already documented in README.md.

+            value = cli_parameters[field]
+            if value is None:
+                value = config_parameters.get(field)
             setattr(self, field, value)
 
     @staticmethod
     def _parse_command_line():
         parser = argparse.ArgumentParser(description='Run crawler')
         parser.add_argument(
             '-c', '--config', type=str,
             help='path to config file, example is config.json.example'
         )
         parser.add_argument(
@@ skipping 124 matching lines @@
     BASE_DIR = os.path.dirname(os.path.abspath(__file__))
     DEPENDENCY_SCRIPT = os.path.join(BASE_DIR, "ensure_dependencies.py")
 
     try:
         subprocess.check_call([sys.executable, DEPENDENCY_SCRIPT, BASE_DIR])
     except subprocess.CalledProcessError as e:
         print >>sys.stderr, e
         print >>sys.stderr, "Failed to ensure dependencies being up-to-date!"
 
     run()