Issue 3943 - add support of configuration file

run.py

 #!/usr/bin/env python
 # coding: utf-8
 
 import argparse
 import datetime
 import errno
 import hashlib
 import io
 import json
 import os
@@ skipping 66 matching lines @@
 
             with io.open(datapath, 'w', encoding='utf-8') as handle:
                 handle.write(unicode(json.dumps(data, indent=2, ensure_ascii=False, sort_keys=True)) + u'\n')
             start_response('204 No Content', [])
             return ''
 
         start_response('404 Not Found', [])
         return ''
 
 
+def read_as_json(file_path):
+    with open(file_path, mode='r') as json_file:
+        return json.load(json_file, encoding='UTF-8')
+
+
+class Parameters:
+    """This class loads config file and parses command line parameters.
+       Values are stored in attibutes of this class instance.
+    """
+    def __init__(self):
+        cli_parameters = vars(Parameters._parse_command_line())
+        config_parameters = Parameters._load_config(cli_parameters["config"])
+        for field in cli_parameters.keys():
+            config_value = config_parameters[field] if field in config_parameters else None
+            value = cli_parameters[field] if cli_parameters[field] is not None else config_value

[tschuster, 2016/11/21 18:07:55]

Can't you just write:
value = cli_parameters[field] if cli_parameters[field] is not None else
config_parameters[field]

We should document that CLI parameters take precedence.

[sergei, 2016/11/22 09:23:31]

Changed. Now lines are not so long and you are right, config_parameters.get is
better than our implementation. config_parameters[field] throws exception if
there is no such key.
It's already documented in README.md.

+            setattr(self, field, value)
+
+    @staticmethod
+    def _parse_command_line():
+        parser = argparse.ArgumentParser(description='Run crawler')
+        parser.add_argument(
+            '-c', '--config', type=str,
+            help='path to config file, example is config.json.example'
+        )
+        parser.add_argument(
+            '-b', '--binary', type=str,
+            help='path to the Firefox binary'
+        )
+        parser.add_argument(
+            '-a', '--abpdir', type=str,
+            help='path to the Adblock Plus repository'
+        )
+        parser.add_argument(
+            '-f', '--filters', metavar='url', type=str, nargs='+',
+            help='filter lists to install in Adblock Plus. The arguments can also have the format path=url, the data will be read from the specified path then.'
+        )
+        parser.add_argument(
+            '-t', '--timeout', type=int,
+            help='Load timeout (seconds)'
+        )
+        parser.add_argument(
+            '-x', '--maxtabs', type=int,
+            help='Maximal number of tabs to open in parallel'
+        )
+        parser.add_argument(
+            '-l', '--list', type=str,
+            help='URL list to process',
+        )
+        parser.add_argument(
+            '-o', '--outdir', type=str,
+            help='directory to write data into',
+        )
+        parameters = parser.parse_args()
+        return parameters
+
+    @staticmethod
+    def _load_config(config_file_path):
+        config = {
+          "filters": [
+            "https://easylist-downloads.adblockplus.org/easylist.txt",
+            "https://easylist-downloads.adblockplus.org/exceptionrules.txt"
+          ],
+          "timeout": 300,
+          "maxtabs": 15
+        }
+        if config_file_path is not None:
+            config.update(read_as_json(config_file_path))
+        return config
+
+
 def run():
-    parser = argparse.ArgumentParser(description='Run crawler')
-    parser.add_argument(
-        '-b', '--binary', type=str,
-        help='path to the Firefox binary'
-    )
-    parser.add_argument(
-        '-a', '--abpdir', type=str,
-        help='path to the Adblock Plus repository'
-    )
-    parser.add_argument(
-        '-f', '--filters', metavar='url', type=str, nargs='+',
-        default=["https://easylist-downloads.adblockplus.org/easylist.txt", "https://easylist-downloads.adblockplus.org/exceptionrules.txt"],
-        help='filter lists to install in Adblock Plus. The arguments can also have the format path=url, the data will be read from the specified path then.'
-    )
-    parser.add_argument(
-        '-t', '--timeout', type=int, default=300,
-        help='Load timeout (seconds)'
-    )
-    parser.add_argument(
-        '-x', '--maxtabs', type=int, default=15,
-        help='Maximal number of tabs to open in parallel'
-    )
-    parser.add_argument(
-        'list', type=str,
-        help='URL list to process'
-    )
-    parser.add_argument(
-        'outdir', type=str,
-        help='directory to write data into'
-    )
-    parameters = parser.parse_args()
-
+    parameters = Parameters()
     import buildtools.packagerGecko as packager
     cleanup = []
     try:
         base_dir = os.path.dirname(os.path.abspath(__file__))
         handle, crawlerxpi = tempfile.mkstemp(suffix='.xpi')
         os.close(handle)
         cleanup.append(crawlerxpi)
         packager.createBuild(base_dir, outFile=crawlerxpi, releaseBuild=True)
 
         abpxpi = 'https://addons.mozilla.org/firefox/downloads/latest/1865/addon-1865-latest.xpi'
@@ skipping 67 matching lines @@
     BASE_DIR = os.path.dirname(os.path.abspath(__file__))
     DEPENDENCY_SCRIPT = os.path.join(BASE_DIR, "ensure_dependencies.py")
 
     try:
         subprocess.check_call([sys.executable, DEPENDENCY_SCRIPT, BASE_DIR])
     except subprocess.CalledProcessError as e:
         print >>sys.stderr, e
         print >>sys.stderr, "Failed to ensure dependencies being up-to-date!"
 
     run()