From d59c8e5390044e4e24014f2cd6697e98e637072f Mon Sep 17 00:00:00 2001
From: Stefano Oldeman <stefano.oldeman@gmail.com>
Date: Sun, 14 Jun 2020 15:28:00 +0200
Subject: [PATCH 01/16] fix: version fix requirements

---
 setup.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/setup.py b/setup.py
index 12aee0d..12bdcd4 100644
--- a/setup.py
+++ b/setup.py
@@ -13,15 +13,15 @@
     url="https://github.com/kpn/pyrandall",
     entry_points={"console_scripts": ["pyrandall = pyrandall.cli:main"]},
     install_requires=[
-        "jsonschema",
-        "jsondiff",
+        "jsonschema~=3.2.0",
+        "jsondiff~=1.2.0",
         "requests~=2.20",
-        "PyYAML==5.1",
-        "Mako",
-        "curlify",
+        "PyYAML~=5.1",
+        "Mako~=1.1.3",
+        "curlify~=2.2.0",
         "confluent-kafka~=1.0",
         "deepdiff==3.3",
-        "pluggy",
+        "pluggy~=0.13.0"
     ],
     tests_require=["pytest", "pytest_httpserver", "responses", "vcrpy", "freezegun"],
 )

From df24d42ce257d6c8cd4791ccf37353efa29f8183 Mon Sep 17 00:00:00 2001
From: Stefano Oldeman <stefano.oldeman@gmail.com>
Date: Sun, 14 Jun 2020 15:28:12 +0200
Subject: [PATCH 02/16] feat: add click requirement

---
 setup.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 12bdcd4..c0bbda2 100644
--- a/setup.py
+++ b/setup.py
@@ -21,7 +21,8 @@
         "curlify~=2.2.0",
         "confluent-kafka~=1.0",
         "deepdiff==3.3",
-        "pluggy~=0.13.0"
+        "pluggy~=0.13.0",
+        "Click~=7.0"
     ],
     tests_require=["pytest", "pytest_httpserver", "responses", "vcrpy", "freezegun"],
 )

From 0f30200d410821c4c1367398b5c18d3053442ae8 Mon Sep 17 00:00:00 2001
From: Stefano Oldeman <stefano.oldeman@gmail.com>
Date: Tue, 16 Jun 2020 13:46:23 +0200
Subject: [PATCH 03/16] fix: deepdiff fixes and enhancements

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index c0bbda2..5888ce5 100644
--- a/setup.py
+++ b/setup.py
@@ -20,7 +20,7 @@
         "Mako~=1.1.3",
         "curlify~=2.2.0",
         "confluent-kafka~=1.0",
-        "deepdiff==3.3",
+        "deepdiff==4.3",
         "pluggy~=0.13.0",
         "Click~=7.0"
     ],

From 770fd8a19410a31b0d9a4c3db35aa891e80ace7d Mon Sep 17 00:00:00 2001
From: Stefano Oldeman <stefano.oldeman@gmail.com>
Date: Thu, 18 Jun 2020 09:17:38 +0200
Subject: [PATCH 04/16] BREAKING CHANGE: cli commands accept filepaths

* Refactored cli, using Click now.
* Depricated cli arg --dataflow
* Adds new features:
 - pyrandall simulate scenarios/file1 scenarios/file2.
   Also accepting linux pipes eg. `cat file1 | pyrandall simulate`
 - event and result files will be looked up not based on dataflow dir.
   But by convention relatively to the scenarios dir.
---
 pyrandall/cli.py                       | 162 ++++++++++---------------
 pyrandall/const.py                     |   3 +
 pyrandall/spec.py                      |  27 ++---
 tests/functional/test_http_simulate.py |  85 ++++++-------
 tests/functional/test_http_validate.py |  52 ++++----
 5 files changed, 135 insertions(+), 194 deletions(-)
 create mode 100644 pyrandall/const.py

diff --git a/pyrandall/cli.py b/pyrandall/cli.py
index d2cf647..b6e86ed 100644
--- a/pyrandall/cli.py
+++ b/pyrandall/cli.py
@@ -1,126 +1,88 @@
 import json
 import sys
+import os
+import itertools
 from argparse import ArgumentParser
 
+import click
 import jsonschema
 
+from pyrandall import const
 from pyrandall import commander
 from pyrandall.hookspecs import get_plugin_manager
 from pyrandall.spec import SpecBuilder
 from pyrandall.types import Flags
 
-FLAGS_MAP = {
-    "simulate": Flags.SIMULATE,
-    "validate": Flags.VALIDATE,
-    "sanitytest": Flags.VALIDATE,  # legacy command
-}
 
+@click.group()
+@click.pass_context
+@click.option("--config", type=click.File('r'), default="pyrandall_config.json", help="path to json file for pyrandall config.")
+def main(ctx, config):
+    """
+    pyrandall a test framework oriented around data validation instead of code
+    """
+    ctx.obj = {}
+    if config:
+        ctx.obj.update(json.load(config))
+
+
+@main.command()
+@click.pass_context
+@click.argument("specfiles", type=click.File('r'), nargs=-1)
+def simulate(ctx, specfiles):
+    """
+    run Simulator for specfiles
+    """
+    ctx.obj['flags'] = Flags.SIMULATE
+    start(specfiles, ctx.obj)
+
+
+@main.command()
+@click.pass_context
+@click.argument("specfiles", type=click.File('r'), nargs=-1)
+def validate(ctx, specfiles):
+    """
+    run Validate for specfiles
+    """
+    ctx.obj['flags'] = Flags.VALIDATE
+    start(specfiles, ctx.obj)
+
+
+def start(specfiles: tuple, config: dict):
+    if len(specfiles) > 1:
+        raise NotImplemented("Passing multiple paths is not supported yet")
+    config['specfile'] = specfiles[0]
+    try:
+        run_command(config)
+    except jsonschema.exceptions.ValidationError:
+        print("Failed validating input yaml")
+        exit(4)
 
-def run_command(config):
-    specfile = config.pop("specfile")
-    config["default_request_url"] = config["requests"].pop("url")
-
-    # register plugins and call their initialize
-    plugin_manager = get_plugin_manager()
-    plugin_manager.hook.pyrandall_initialize(config=config)
-
-    spec = SpecBuilder(specfile, hook=plugin_manager.hook, **config).feature()
-    flags = FLAGS_MAP[config["command"]]
-    # commander handles execution flow with specified data and config
-    commander.Commander(spec, flags).invoke()
-
-
-def add_common_args(parser):
-    parser.add_argument("specfile", type=str, help="name of yaml file in scenario/")
-
-
-def setup_args():
-    parser = ArgumentParser(
-        description="pyrandall a test framework oriented around data validation instead of code"
-    )
-    parser.add_argument(
-        "--config",
-        type=str,
-        default="pyrandall_config.json",
-        dest="config_path",
-        help="path to json file for pyrandall config.",
-    )
-    parser.add_argument(
-        "--dataflow",
-        type=str,
-        required=True,
-        dest="dataflow_path",
-        help="path to dataflow root directory",
-    )
-
-    subparsers = parser.add_subparsers(dest="command")
-    # add simulate subcommand
-    sim_parser = subparsers.add_parser("simulate", help="run Simulator for specfile")
-    add_common_args(sim_parser)
-    # add sanitycheck (legacy name)
-    san_parser = subparsers.add_parser(
-        "sanitytest", help="run Validate for specfile (Use validate command)"
-    )
-    add_common_args(san_parser)
-    # add validate subcommand
-    val_parser = subparsers.add_parser("validate", help="run Validate for specfile")
-    add_common_args(val_parser)
-
-    return parser
-
-
-def argparse_error(args_data):
-    msg = """
-#######
-Exit code was 2! Its assumed mocked arguments are wrong, see argparse usage below:
-
-\t%s
-
-actual arguments passed to mock:
-\t%s
-
-######
-""" % (
-        setup_args().format_help().replace("\n", "\n\t"),
-        args_data,
-    )
-    return msg
-
-
-def load_config(fpath):
-    with open(fpath, "r") as f:
-        return json.load(f)
-
-
-def start(argv, config=None):
-    parser = setup_args()
-    args_config = parser.parse_args(argv)
 
+def run_command(config):
     # TODO: add logging options
     # with open("logging.yaml") as log_conf_file:
     #     log_conf = yaml.safe_load(log_conf_file)
     #     dictConfig(log_conf)
 
-    if args_config.command is None:
-        parser.error("not a valid pyrandall command")
-        exit(1)
-
-    if config is None:
-        config = load_config(args_config.config_path)
+    config["default_request_url"] = config["requests"].pop("url")
+    config['dataflow_path'] = build_basedir(config['specfile'])
 
-    # overwrite with cli options
-    config.update(args_config.__dict__)
+    # register plugins and call their initialize
+    plugin_manager = get_plugin_manager()
+    plugin_manager.hook.pyrandall_initialize(config=config)
 
-    try:
-        run_command(config)
-    except jsonschema.exceptions.ValidationError:
-        print("Failed validating input yaml")
-        exit(4)
-    exit(0)
+    spec = SpecBuilder(hook=plugin_manager.hook, **config).feature()
+    # commander handles execution flow with specified data and config
+    commander.Commander(spec, config["flags"]).invoke()
 
 
-def main():
-    start(sys.argv[1:])
+def build_basedir(specfile):
+    parts = specfile.name.split('/')
+    out = []
+    for x in itertools.takewhile(lambda x: x != const.DIRNAME_SCENARIOS, parts):
+        out.append(x)
+    return os.path.abspath('/'.join(out))
 
 
 if __name__ == "__main__":
diff --git a/pyrandall/const.py b/pyrandall/const.py
new file mode 100644
index 0000000..9b5d704
--- /dev/null
+++ b/pyrandall/const.py
@@ -0,0 +1,3 @@
+DIRNAME_SCENARIOS = "scenarios"
+DIRNAME_EVENTS = "events"
+DIRNAME_RESULTS = "results"
diff --git a/pyrandall/spec.py b/pyrandall/spec.py
index ec89332..29a689b 100644
--- a/pyrandall/spec.py
+++ b/pyrandall/spec.py
@@ -16,11 +16,10 @@
 
 from .network import join_urlpath
 
-DIR_PATH_DEFAULT = "."
 
+# Constants to resolve private schema files
 DIR_PYRANDALL_HOME = os.path.dirname(os.path.abspath(__file__))
 SCHEMAS_SCENARIO_V2 = os.path.join(DIR_PYRANDALL_HOME, "schemas/scenario/v2.yaml")
-
 VERSION_SCENARIO_V2 = "scenario/v2"
 VERSIONS = [VERSION_SCENARIO_V2]
 
@@ -37,10 +36,9 @@ def scenario_group(self, nr, data):
 
 
 class SpecBuilder:
-    def __init__(self, specfile, scenarios_dirname="scenarios", **kwargs):
+    def __init__(self, specfile, **kwargs):
         self.factory = V2Factory(**kwargs)
-        dataflow_path = kwargs.get("dataflow_path", DIR_PATH_DEFAULT)
-        self.scenario_file = os.path.join(dataflow_path, scenarios_dirname, specfile)
+        self.specfile = specfile
 
     def feature(self):
         # creating Feature object will marshall everything below it
@@ -48,15 +46,14 @@ def feature(self):
 
     def load_spec(self):
         # TODO: prevent reading sensitive files from filesystem
-        with open(self.scenario_file, "r") as f:
-            data = yaml.load(f, Loader=yaml.FullLoader)
-            # implicitly assume scenario v2 schema
-            version = data.get("version", VERSION_SCENARIO_V2)
-            if version not in VERSIONS:
-                raise InvalidSchenarioVersion(VERSIONS)
-            # raises errors if unvalid to jsonschema
-            jsonschema.validate(data, self.scenario_v2_schema())
-            return data
+        data = yaml.load(self.specfile, Loader=yaml.FullLoader)
+        # implicitly assume scenario v2 schema
+        version = data.get("version", VERSION_SCENARIO_V2)
+        if version not in VERSIONS:
+            raise InvalidSchenarioVersion(VERSIONS)
+        # raises errors if unvalid to jsonschema
+        jsonschema.validate(data, self.scenario_v2_schema())
+        return data
 
     def scenario_v2_schema(self):
         with open(SCHEMAS_SCENARIO_V2) as f:
@@ -89,7 +86,7 @@ def __init__(
         self,
         nr,
         data,
-        dataflow_path=DIR_PATH_DEFAULT,
+        dataflow_path,
         events_dirname="events",
         results_dirname="results",
         default_request_url=None,
diff --git a/tests/functional/test_http_simulate.py b/tests/functional/test_http_simulate.py
index c592b82..6bc3984 100644
--- a/tests/functional/test_http_simulate.py
+++ b/tests/functional/test_http_simulate.py
@@ -1,5 +1,4 @@
-import unittest
-from unittest.mock import patch
+from click.testing import CliRunner
 
 from pyrandall import cli
 from tests.conftest import vcr
@@ -7,60 +6,46 @@
 ARGV_RESPONSE_200 = [
     "--config",
     "examples/config/v1.json",
-    "--dataflow",
-    "examples/",
     "simulate",
-    "http/simulate_200.yaml",
+    "examples/scenarios/http/simulate_200.yaml",
 ]
 ARGV_RESPONSE_400 = [
     "--config",
     "examples/config/v1.json",
-    "--dataflow",
-    "examples/",
     "simulate",
-    "http/simulate_400.yaml",
+    "examples/scenarios/http/simulate_400.yaml",
 ]
 
 
-class SimulatorTest(unittest.TestCase):
-    @patch("argparse.ArgumentParser._print_message")
-    def test_execute_a_simulation_fails(self, print_message):
-        with self.assertRaises(SystemExit) as context:
-            cli.start([])
-            self.assertEqual(context.exception.code, 2)
-            print_message.assert_called_once()
-
-    def test_simulate_json_response_200(self):
-        with vcr.use_cassette("test_simulate_json_response_200") as cassette:
-            with self.assertRaises(SystemExit) as context:
-                cli.start(ARGV_RESPONSE_200)
-
-            assert len(cassette) == 1
-            r1 = cassette.requests[0]
-            assert r1.path == "/v1/actions/produce-event"
-            assert cassette.responses_of(r1)[0]["status"]["code"] == 204
-            assert cassette.all_played
-            if context.exception.code == 2:
-                self.fail(cli.argparse_error(ARGV_RESPONSE_200))
-
-            # not all request had the expected status code (see assertions)
-            assert context.exception.code == 0
-
-    def test_simulate_json_response_400(self):
-        with vcr.use_cassette("test_simulate_json_response_400") as cassette:
-            with self.assertRaises(SystemExit) as context:
-                cli.start(ARGV_RESPONSE_400)
-
-            assert len(cassette) == 1
-            r0 = cassette.requests[0]
-            assert r0.path == "/cant_find_this"
-            assert cassette.responses_of(r0)[0]["status"]["code"] == 404
-            assert cassette.all_played
-            if context.exception.code == 2:
-                self.fail(cli.argparse_error(ARGV_RESPONSE_400))
-
-            assert context.exception.code == 1
-
-
-if __name__ == "__main__":
-    unittest.main()
+def test_execute_a_simulation_fails():
+    runner = CliRunner()
+    result = runner.invoke(cli.main, [], catch_exceptions=False)
+    assert 'Usage: main' in result.output
+    assert result.exit_code == 0
+
+def test_simulate_json_response_200():
+    with vcr.use_cassette("test_simulate_json_response_200") as cassette:
+        runner = CliRunner()
+        result = runner.invoke(cli.main, ARGV_RESPONSE_200, catch_exceptions=False)
+        assert 'Usage: main' not in result.output
+        assert result.exit_code == 0
+
+        assert len(cassette) == 1
+        r1 = cassette.requests[0]
+        assert r1.path == "/v1/actions/produce-event"
+        assert cassette.responses_of(r1)[0]["status"]["code"] == 204
+        assert cassette.all_played
+        # not all request had the expected status code (see assertions)
+
+def test_simulate_json_response_400():
+    with vcr.use_cassette("test_simulate_json_response_400") as cassette:
+        runner = CliRunner()
+        result = runner.invoke(cli.main, ARGV_RESPONSE_400, catch_exceptions=False)
+        assert 'Usage: main' not in result.output
+        assert result.exit_code == 1
+
+        assert len(cassette) == 1
+        r0 = cassette.requests[0]
+        assert r0.path == "/cant_find_this"
+        assert cassette.responses_of(r0)[0]["status"]["code"] == 404
+        assert cassette.all_played
diff --git a/tests/functional/test_http_validate.py b/tests/functional/test_http_validate.py
index 30a9a81..4ac89da 100644
--- a/tests/functional/test_http_validate.py
+++ b/tests/functional/test_http_validate.py
@@ -1,6 +1,5 @@
-from unittest.mock import patch
-
 import pytest
+from click.testing import CliRunner
 
 from pyrandall import cli
 from tests.conftest import vcr
@@ -9,44 +8,39 @@
 ARGV_HTTP_VALIDATE_1_OK = [
     "--config",
     CONFIG,
-    "--dataflow",
-    "examples/",
-    "sanitytest",
-    "http/validate_ok_status_code.yaml",
+    "validate",
+    "examples/scenarios/http/validate_ok_status_code.yaml",
 ]
 ARGV_HTTP_VALIDATE_STAUTS_CODE_FAIL = [
     "--config",
     CONFIG,
-    "--dataflow",
-    "examples/",
-    "sanitytest",
-    "http/validate_bad_status_code.yaml",
+    "validate",
+    "examples/scenarios/http/validate_bad_status_code.yaml",
 ]
 
 
-@patch("pyrandall.cli.ArgumentParser._print_message")
-def test_execute_a_sanitytest_fails(print_message):
-    with pytest.raises(SystemExit) as context:
-        cli.start([])
-    assert context.value.code == 2
-    print_message.assert_called()
+def test_execute_a_sanitytest_fails():
+    runner = CliRunner()
+    result = runner.invoke(cli.main, [], catch_exceptions=False)
+    assert 'Usage: main' in result.output
+    assert result.exit_code == 0
 
 
-@vcr.use_cassette("test_validate_assertions_pass")
 def test_validate_assertions_pass():
-    with pytest.raises(SystemExit) as context:
-        cli.start(ARGV_HTTP_VALIDATE_1_OK)
-    if context.value.code == 2:
-        pytest.fail(cli.argparse_error(ARGV_HTTP_VALIDATE_1_OK))
-
-    assert context.value.code == 0
+    with vcr.use_cassette("test_validate_assertions_pass") as cassette:
+        runner = CliRunner()
+        result = runner.invoke(cli.main, ARGV_HTTP_VALIDATE_1_OK, catch_exceptions=False)
+        assert 'Usage: main' not in result.output
+        assert result.exit_code == 0
 
+        assert cassette.all_played
 
-@vcr.use_cassette("test_validate_fail_status_code")
 def test_validate_fail_status_code():
-    with pytest.raises(SystemExit) as context:
-        cli.start(ARGV_HTTP_VALIDATE_STAUTS_CODE_FAIL)
-    if context.value.code == 2:
-        pytest.fail(cli.argparse_error(ARGV_HTTP_VALIDATE_STAUTS_CODE_FAIL))
+    with vcr.use_cassette("test_validate_fail_status_code") as cassette:
+        runner = CliRunner()
+        result = runner.invoke(cli.main, ARGV_HTTP_VALIDATE_STAUTS_CODE_FAIL, catch_exceptions=False)
+
+        assert 'Usage: main' not in result.output
+        assert result.exit_code == 1
 
-    assert context.value.code == 1
+        assert cassette.all_played

From 92ee8434928fcb1c501aeb485d2ddae36598b6a0 Mon Sep 17 00:00:00 2001
From: Stefano Oldeman <stefano.oldeman@gmail.com>
Date: Mon, 22 Jun 2020 09:14:53 +0200
Subject: [PATCH 05/16] feat: single command with flags in cli

---
 pyrandall/cli.py                        | 62 ++++++++----------
 pyrandall/commander.py                  |  4 +-
 pyrandall/spec.py                       |  1 +
 pyrandall/types.py                      | 12 ++--
 tests/conftest.py                       | 14 ++++
 tests/functional/test_http_simulate.py  | 27 ++++----
 tests/functional/test_http_validate.py  | 86 ++++++++++++++-----------
 tests/functional/test_kafka_simulate.py | 34 +++++-----
 tests/functional/test_kafka_validate.py | 69 +++++++++-----------
 tests/unit/test_commander.py            |  4 +-
 tests/unit/test_spec.py                 | 11 +---
 11 files changed, 159 insertions(+), 165 deletions(-)

diff --git a/pyrandall/cli.py b/pyrandall/cli.py
index b6e86ed..7486807 100644
--- a/pyrandall/cli.py
+++ b/pyrandall/cli.py
@@ -14,59 +14,53 @@
 from pyrandall.types import Flags
 
 
-@click.group()
-@click.pass_context
-@click.option("--config", type=click.File('r'), default="pyrandall_config.json", help="path to json file for pyrandall config.")
-def main(ctx, config):
-    """
-    pyrandall a test framework oriented around data validation instead of code
-    """
-    ctx.obj = {}
-    if config:
-        ctx.obj.update(json.load(config))
-
+def validate_specfiles(ctx, param, value):
+    if len(value) == 0:
+        raise click.BadParameter('expecting at least one argument')
 
-@main.command()
-@click.pass_context
-@click.argument("specfiles", type=click.File('r'), nargs=-1)
-def simulate(ctx, specfiles):
-    """
-    run Simulator for specfiles
-    """
-    ctx.obj['flags'] = Flags.SIMULATE
-    start(specfiles, ctx.obj)
+    if len(value) > 1:
+        raise click.UsageError("passing multiple arguments is not supported yet")
 
 
-@main.command()
+@click.command()
 @click.pass_context
-@click.argument("specfiles", type=click.File('r'), nargs=-1)
-def validate(ctx, specfiles):
+@click.option("-c", "--config", 'config_file', type=click.File('r'), default="pyrandall_config.json", help="path to json file for pyrandall config.")
+@click.option("-s", "--only-simulate", 'command_flag', flag_value=Flags.SIMULATE)
+@click.option("-V", "--only-validate", 'command_flag', flag_value=Flags.VALIDATE)
+@click.option("-e", "--everything", 'command_flag', flag_value=Flags.E2E, default=True)
+@click.option("-d", "--dry-run", 'filter_flag', flag_value=Flags.DESCRIBE)
+@click.argument("specfiles", type=click.File('r'), nargs=-1, callback=validate_specfiles)
+def main(ctx, config_file, command_flag, filter_flag, specfiles):
     """
-    run Validate for specfiles
+    pyrandall a test framework oriented around data validation instead of code
     """
-    ctx.obj['flags'] = Flags.VALIDATE
-    start(specfiles, ctx.obj)
+    config = {}
+    if config_file:
+        config = json.load(config_file)
+
 
+    # translate None to NO OP Flag
+    if filter_flag is None:
+        filter_flag = Flags.NOOP
+    flags = command_flag | filter_flag
 
-def start(specfiles: tuple, config: dict):
-    if len(specfiles) > 1:
-        raise NotImplemented("Passing multiple paths is not supported yet")
-    config['specfile'] = specfiles[0]
     try:
-        run_command(config)
+        run_command(config, flags, specfiles[0])
     except jsonschema.exceptions.ValidationError:
         print("Failed validating input yaml")
         exit(4)
 
 
-def run_command(config):
+def run_command(config, flags, specfile):
     # TODO: add logging options
     # with open("logging.yaml") as log_conf_file:
     #     log_conf = yaml.safe_load(log_conf_file)
     #     dictConfig(log_conf)
 
     config["default_request_url"] = config["requests"].pop("url")
-    config['dataflow_path'] = build_basedir(config['specfile'])
+    config['dataflow_path'] = build_basedir(specfile)
+    config['specfile'] = specfile
+    config['flags'] = flags
 
     # register plugins and call their initialize
     plugin_manager = get_plugin_manager()
@@ -74,7 +68,7 @@ def run_command(config):
 
     spec = SpecBuilder(hook=plugin_manager.hook, **config).feature()
     # commander handles execution flow with specified data and config
-    commander.Commander(spec, config["flags"]).invoke()
+    commander.Commander(spec, flags).invoke()
 
 
 def build_basedir(specfile):
diff --git a/pyrandall/commander.py b/pyrandall/commander.py
index 0146b5c..becb855 100644
--- a/pyrandall/commander.py
+++ b/pyrandall/commander.py
@@ -37,7 +37,7 @@ def run_scenarios(self, scenario_items, reporter):
             # 2. call output interface
             reporter.scenario(scenario.description)
 
-            if self.flags & Flags.SIMULATE:
+            if self.flags.has_simulate():
                 reporter.simulate()
                 resultset = reporter.create_and_track_resultset()
                 for spec in scenario.simulate_tasks:
@@ -45,7 +45,7 @@ def run_scenarios(self, scenario_items, reporter):
                     reporter.run_task(e.represent())
                     e.execute(resultset)
 
-            if self.flags & Flags.VALIDATE:
+            if self.flags.has_validate():
                 reporter.validate()
                 resultset = reporter.create_and_track_resultset()
                 for spec in scenario.validate_tasks:
diff --git a/pyrandall/spec.py b/pyrandall/spec.py
index 29a689b..ea28d6e 100644
--- a/pyrandall/spec.py
+++ b/pyrandall/spec.py
@@ -36,6 +36,7 @@ def scenario_group(self, nr, data):
 
 
 class SpecBuilder:
+
     def __init__(self, specfile, **kwargs):
         self.factory = V2Factory(**kwargs)
         self.specfile = specfile
diff --git a/pyrandall/types.py b/pyrandall/types.py
index ecca49d..b8eb08b 100644
--- a/pyrandall/types.py
+++ b/pyrandall/types.py
@@ -23,6 +23,8 @@ def __str__(self):
 
 
 class Flags(Flag):
+    NOOP = 0  # No Operation
+    
     DESCRIBE = auto()
 
     BLOCKING = auto()
@@ -31,20 +33,16 @@ class Flags(Flag):
     SIMULATE = auto()
     VALIDATE = auto()
 
-    BLOCKING_E2E = BLOCKING | SIMULATE | VALIDATE
-    # REALTIME_E2E = REALTIME | SIMULATE | VALIDATE
+    E2E = BLOCKING | SIMULATE | VALIDATE
 
     # def run_realtime(self):
     #     return self & Flags.REALTIME
 
-    def run_blocking(self):
-        return self & Flags.BLOCKING
-
     def has_validate(self):
-        return self & Flags.VALIDATE
+        return Flags.VALIDATE in self
 
     def has_simulate(self):
-        return self & Flags.SIMULATE
+        return Flags.SIMULATE in self
 
 
 class Assertion:
diff --git a/tests/conftest.py b/tests/conftest.py
index 8a2ce34..9187c48 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,8 +1,11 @@
 import os
 
+import pyrandall.cli
+
 import vcr
 import pytest
 
+from click.testing import CliRunner
 from confluent_kafka.admin import AdminClient, ClusterMetadata
 
 defaults = ["method", "scheme", "host", "port", "path", "query"]
@@ -25,3 +28,14 @@ def kafka_cluster_info() -> ClusterMetadata:
     cluster = admin.list_topics()
     assert ('{1: BrokerMetadata(1, %s)}' % kafka) == str(cluster.brokers)
     return cluster
+
+@pytest.fixture
+def pyrandall_cli():
+    return PyrandallCli()
+
+
+class PyrandallCli():
+
+    def invoke(self, command):
+        runner = CliRunner()
+        return runner.invoke(pyrandall.cli.main, command, catch_exceptions=False)
diff --git a/tests/functional/test_http_simulate.py b/tests/functional/test_http_simulate.py
index 6bc3984..462cc0b 100644
--- a/tests/functional/test_http_simulate.py
+++ b/tests/functional/test_http_simulate.py
@@ -1,32 +1,30 @@
-from click.testing import CliRunner
-
-from pyrandall import cli
 from tests.conftest import vcr
 
 ARGV_RESPONSE_200 = [
     "--config",
     "examples/config/v1.json",
-    "simulate",
+    "-s",
     "examples/scenarios/http/simulate_200.yaml",
 ]
 ARGV_RESPONSE_400 = [
     "--config",
     "examples/config/v1.json",
-    "simulate",
+    "-s",
     "examples/scenarios/http/simulate_400.yaml",
 ]
 
 
-def test_execute_a_simulation_fails():
-    runner = CliRunner()
-    result = runner.invoke(cli.main, [], catch_exceptions=False)
+def test_execute_a_simulation_fails(pyrandall_cli):
+    result = pyrandall_cli.invoke([
+        "--config",
+        "examples/config/v1.json"
+    ])
     assert 'Usage: main' in result.output
-    assert result.exit_code == 0
+    assert result.exit_code == 2
 
-def test_simulate_json_response_200():
+def test_simulate_json_response_200(pyrandall_cli):
     with vcr.use_cassette("test_simulate_json_response_200") as cassette:
-        runner = CliRunner()
-        result = runner.invoke(cli.main, ARGV_RESPONSE_200, catch_exceptions=False)
+        result = pyrandall_cli.invoke(ARGV_RESPONSE_200)
         assert 'Usage: main' not in result.output
         assert result.exit_code == 0
 
@@ -37,10 +35,9 @@ def test_simulate_json_response_200():
         assert cassette.all_played
         # not all request had the expected status code (see assertions)
 
-def test_simulate_json_response_400():
+def test_simulate_json_response_400(pyrandall_cli):
     with vcr.use_cassette("test_simulate_json_response_400") as cassette:
-        runner = CliRunner()
-        result = runner.invoke(cli.main, ARGV_RESPONSE_400, catch_exceptions=False)
+        result = pyrandall_cli.invoke(ARGV_RESPONSE_400)
         assert 'Usage: main' not in result.output
         assert result.exit_code == 1
 
diff --git a/tests/functional/test_http_validate.py b/tests/functional/test_http_validate.py
index 4ac89da..68434c7 100644
--- a/tests/functional/test_http_validate.py
+++ b/tests/functional/test_http_validate.py
@@ -1,46 +1,54 @@
+import os
 import pytest
-from click.testing import CliRunner
+from freezegun import freeze_time
 
-from pyrandall import cli
+import threading
 from tests.conftest import vcr
+from tests.helper import KafkaProducer
+from pyrandall.kafka import KafkaSetupError
 
-CONFIG = "examples/config/v1.json"
-ARGV_HTTP_VALIDATE_1_OK = [
-    "--config",
-    CONFIG,
-    "validate",
-    "examples/scenarios/http/validate_ok_status_code.yaml",
-]
-ARGV_HTTP_VALIDATE_STAUTS_CODE_FAIL = [
+TOPIC_1 = "pyrandall-tests-validate-1"
+TOPIC_2 = "pyrandall-tests-validate-2"
+
+MOCK_ARGV = [
     "--config",
-    CONFIG,
-    "validate",
-    "examples/scenarios/http/validate_bad_status_code.yaml",
+    "examples/config/v1.json",
+    "-V"
 ]
-
-
-def test_execute_a_sanitytest_fails():
-    runner = CliRunner()
-    result = runner.invoke(cli.main, [], catch_exceptions=False)
-    assert 'Usage: main' in result.output
+ARGV_SMALL = MOCK_ARGV + ["examples/scenarios/v2_ingest_kafka_small.yaml"]
+
+
+def test_error_on_connection_timeout(monkeypatch, pyrandall_cli):
+    monkeypatch.setenv("KAFKA_BOOTSTRAP_SERVERS", "localhost:3330")
+    with pytest.raises(KafkaSetupError) as e:
+        pyrandall_cli.invoke(ARGV_SMALL)
+
+
+# freeze time in order to hardcode timestamps
+@freeze_time("2012-01-14 14:33:12")
+@vcr.use_cassette("test_ingest_to_kafka")
+def test_received_no_events(monkeypatch, kafka_cluster_info, pyrandall_cli):
+    result = pyrandall_cli.invoke(ARGV_SMALL)
+    # exit code should be 1 (error)
+    assert 'Usage: main' not in result.output
+    print(result.output)
+    assert result.exit_code == 1
+
+def produce_events():
+    # produce the events
+    producer = KafkaProducer(TOPIC_1)
+    producer.send(b'{"click": "three"}')
+    producer.send(b'{"click": "one"}')
+    producer.send(b'{"click": "two"}')
+
+    producer = KafkaProducer(TOPIC_2)
+    producer.send(b'{"click": "three"}')
+
+@freeze_time("2012-01-14 14:33:12")
+@vcr.use_cassette("test_ingest_to_kafka")
+def test_validate_unordered_passed(kafka_cluster_info, pyrandall_cli):
+    produce_events()
+    result = pyrandall_cli.invoke(ARGV_SMALL)
+    # exit code should be 1 (error)
+    assert 'Usage: main' not in result.output
     assert result.exit_code == 0
-
-
-def test_validate_assertions_pass():
-    with vcr.use_cassette("test_validate_assertions_pass") as cassette:
-        runner = CliRunner()
-        result = runner.invoke(cli.main, ARGV_HTTP_VALIDATE_1_OK, catch_exceptions=False)
-        assert 'Usage: main' not in result.output
-        assert result.exit_code == 0
-
-        assert cassette.all_played
-
-def test_validate_fail_status_code():
-    with vcr.use_cassette("test_validate_fail_status_code") as cassette:
-        runner = CliRunner()
-        result = runner.invoke(cli.main, ARGV_HTTP_VALIDATE_STAUTS_CODE_FAIL, catch_exceptions=False)
-
-        assert 'Usage: main' not in result.output
-        assert result.exit_code == 1
-
-        assert cassette.all_played
diff --git a/tests/functional/test_kafka_simulate.py b/tests/functional/test_kafka_simulate.py
index 7384ecc..6bee6e4 100644
--- a/tests/functional/test_kafka_simulate.py
+++ b/tests/functional/test_kafka_simulate.py
@@ -2,44 +2,40 @@
 import pytest
 from freezegun import freeze_time
 
-from pyrandall import cli
 from tests.conftest import vcr
 from tests.helper import KafkaConsumer
 
 
 TEST_TOPIC = "pyrandall-tests-e2e"
 
+MOCK_ARGV = [
+    "--config",
+    "examples/config/v1.json",
+    "-s"
+]
+ARGV_INVALID = MOCK_ARGV + ["examples/scenarios/v2_ingest_kafka_invalid.yaml"]
+ARGV_SMALL = MOCK_ARGV + ["examples/scenarios/v2_ingest_kafka_small.yaml"]
 
-config = "examples/config/v1.json"
-MOCK_ARGV = ["--config", config, "--dataflow", "examples/", "simulate"]
 
-
-def test_invalid_kafka_scenario():
-    argv = MOCK_ARGV + ["v2_ingest_kafka_invalid.yaml"]
-    with pytest.raises(SystemExit) as context:
-        cli.start(argv)
-    if context.value.code == 2:
-        pytest.fail(cli.argparse_error(argv))
-    assert context.value.code == 4
+def test_fail_on_invaild_schema(pyrandall_cli):
+    result = pyrandall_cli.invoke(ARGV_INVALID)
+    assert 'Failed validating' in result.output
+    assert result.exit_code == 4
 
 
 # freeze time in order to hardcode timestamps
 @freeze_time("2012-01-14 14:33:12")
 @vcr.use_cassette("test_ingest_to_kafka")
-def test_simulate_produces_event(kafka_cluster_info):
+def test_simulate_produces_event(kafka_cluster_info, pyrandall_cli):
     consumer = KafkaConsumer(TEST_TOPIC)
     try:
         highwater = consumer.get_high_watermark()
 
         # run simulate to create a message to kafka
         # running following command:
-        argv = MOCK_ARGV + ["v2_ingest_kafka_small.yaml"]
-        print(f"running {argv}")
-        with pytest.raises(SystemExit) as context:
-            cli.start(argv)
-        if context.value.code == 2:
-            pytest.fail(cli.argparse_error(argv))
-        assert context.value.code == 0
+        result = pyrandall_cli.invoke(ARGV_SMALL)
+        assert 'Usage: main' not in result.output
+        assert result.exit_code == 0
 
         messages = consumer.get_messages(expecting=2)
         assert len(messages) == 2
diff --git a/tests/functional/test_kafka_validate.py b/tests/functional/test_kafka_validate.py
index 69695ff..c248691 100644
--- a/tests/functional/test_kafka_validate.py
+++ b/tests/functional/test_kafka_validate.py
@@ -3,7 +3,6 @@
 from freezegun import freeze_time
 
 import threading
-from pyrandall import cli
 from tests.conftest import vcr
 from tests.helper import KafkaProducer
 from pyrandall.kafka import KafkaSetupError
@@ -11,33 +10,13 @@
 TOPIC_1 = "pyrandall-tests-validate-1"
 TOPIC_2 = "pyrandall-tests-validate-2"
 
+MOCK_ARGV = [
+    "--config",
+    "examples/config/v1.json",
+    "-V"
+]
+ARGV_SMALL = MOCK_ARGV + ["examples/scenarios/v2_ingest_kafka_small.yaml"]
 
-config = "examples/config/v1.json"
-MOCK_ARGV = ["--config", config, "--dataflow", "examples/", "validate"]
-
-
-def test_error_on_connection_timeout(monkeypatch):
-    monkeypatch.setenv("KAFKA_BOOTSTRAP_SERVERS", "localhost:3330")
-    argv = MOCK_ARGV + ["v2_ingest_kafka_small.yaml"]
-    with pytest.raises(KafkaSetupError) as e:
-        cli.start(argv)
-
-
-# freeze time in order to hardcode timestamps
-@freeze_time("2012-01-14 14:33:12")
-@vcr.use_cassette("test_ingest_to_kafka")
-def test_received_no_events(monkeypatch, kafka_cluster_info):
-    # run validate to consume a message from kafka
-    # running following command:
-    argv = MOCK_ARGV + ["v2_ingest_kafka_small.yaml"]
-    print(f"running {argv}")
-    with pytest.raises(SystemExit) as context:
-        cli.start(argv)
-    if context.value.code == 2:
-        pytest.fail(cli.argparse_error(argv))
-
-    # exit code should be 1 (error)
-    assert context.value.code == 1
 
 def produce_events():
     # produce the events
@@ -49,21 +28,33 @@ def produce_events():
     producer = KafkaProducer(TOPIC_2)
     producer.send(b'{"click": "three"}')
 
+
+def test_error_on_connection_timeout(monkeypatch, pyrandall_cli):
+    monkeypatch.setenv("KAFKA_BOOTSTRAP_SERVERS", "localhost:3330")
+    with pytest.raises(KafkaSetupError) as e:
+        pyrandall_cli.invoke(ARGV_SMALL)
+
+
+# freeze time in order to hardcode timestamps
 @freeze_time("2012-01-14 14:33:12")
 @vcr.use_cassette("test_ingest_to_kafka")
-def test_validate_unordered_passed(kafka_cluster_info):
-    # run validate to consume a message from kafka
-    # running following command:
-    argv = MOCK_ARGV + ["v2_ingest_kafka_small.yaml"]
+def test_received_no_events(monkeypatch, kafka_cluster_info, pyrandall_cli):
+    """
+    run validate to consume a message from kafka
+    """
+    result = pyrandall_cli.invoke(ARGV_SMALL)
+    # exit code should be 1 (error)
+    assert 'Usage: main' not in result.output
+    print(result.output)
+    assert result.exit_code == 1
 
-    print(f"running {argv}")
-    with pytest.raises(SystemExit) as context:
-        produce_events()
-        cli.start(argv)
-    if context.value.code == 2:
-        pytest.fail(cli.argparse_error(argv))
 
-    # exit code should be 1 (error)
-    assert context.value.code == 0
+@freeze_time("2012-01-14 14:33:12")
+@vcr.use_cassette("test_ingest_to_kafka")
+def test_validate_unordered_passed(kafka_cluster_info, pyrandall_cli):
+    produce_events()
+    result = pyrandall_cli.invoke(ARGV_SMALL)
+    assert 'Usage: main' not in result.output
+    assert result.exit_code == 0
 
 
diff --git a/tests/unit/test_commander.py b/tests/unit/test_commander.py
index 9ce1c02..f64e569 100644
--- a/tests/unit/test_commander.py
+++ b/tests/unit/test_commander.py
@@ -11,7 +11,7 @@
 @pytest.fixture
 def spec():
     builder = SpecBuilder(
-        specfile="one_event.yaml",
+        specfile=open("examples/scenarios/one_event.yaml"),
         dataflow_path="examples/",
         default_request_url="http://localhost:5000",
         schemas_url="http://localhost:8899/schemas/",
@@ -24,7 +24,7 @@ def test_commander_run_one_for_one(spec):
     reporter = MagicMock(Reporter(), unsafe=True)
     reporter.create_and_track_resultset.return_value = MagicMock(ResultSet, unsafe=True)
 
-    c = Commander(spec, Flags.BLOCKING_E2E)
+    c = Commander(spec, Flags.E2E)
     c.run(reporter)
 
     reporter.feature.assert_called_once_with("One event"),
diff --git a/tests/unit/test_spec.py b/tests/unit/test_spec.py
index 20cc844..76232d0 100644
--- a/tests/unit/test_spec.py
+++ b/tests/unit/test_spec.py
@@ -7,7 +7,7 @@
 @pytest.fixture
 def feature():
     builder = SpecBuilder(
-        specfile="v2.yaml",
+        specfile=open("examples/scenarios/v2.yaml"),
         dataflow_path="examples/",
         default_request_url="http://localhost:5000",
         schemas_url="http://localhost:8899/schemas/",
@@ -15,14 +15,9 @@ def feature():
     return builder.feature()
 
 
-def test_loads_valid_scenario_yaml():
-    with pytest.raises(FileNotFoundError):
-        SpecBuilder("spec.yaml", dataflow_path="random_dir").feature()
-
-
 def test_request_url_missing():
     with pytest.raises(ValueError) as e:
-        SpecBuilder("v2.yaml", dataflow_path="examples/").feature()
+        SpecBuilder(open("examples/scenarios/v2.yaml"), dataflow_path="examples/").feature()
         expected = (
             "self.default_request_url is None. "
             "See README.md on how to configure a request URL."
@@ -73,7 +68,7 @@ def test_creates_executors_for_validators_only(feature):
 
 def test_parse_events_plugins():
     SpecBuilder(
-        specfile="v2.yaml",
+        specfile=open("examples/scenarios/v2.yaml"),
         dataflow_path="examples/",
         default_request_url="http://localhost:5000",
         schemas_url="http://localhost:8899/schemas/",

From 3abe50c3db6ded32a5b4b8b4759192683351440d Mon Sep 17 00:00:00 2001
From: Stefano Oldeman <stefano.oldeman@gmail.com>
Date: Wed, 24 Jun 2020 09:07:41 +0200
Subject: [PATCH 06/16] fix: move custom validation

---
 pyrandall/cli.py                       | 25 ++++++++++++-------------
 tests/functional/test_http_simulate.py |  6 +++---
 2 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/pyrandall/cli.py b/pyrandall/cli.py
index 7486807..e63c1b5 100644
--- a/pyrandall/cli.py
+++ b/pyrandall/cli.py
@@ -14,26 +14,26 @@
 from pyrandall.types import Flags
 
 
-def validate_specfiles(ctx, param, value):
-    if len(value) == 0:
-        raise click.BadParameter('expecting at least one argument')
-
-    if len(value) > 1:
-        raise click.UsageError("passing multiple arguments is not supported yet")
-
-
-@click.command()
-@click.pass_context
+@click.command(name="pyrandall")
+@click.argument("specfiles", type=click.File('r'), nargs=-1)
 @click.option("-c", "--config", 'config_file', type=click.File('r'), default="pyrandall_config.json", help="path to json file for pyrandall config.")
 @click.option("-s", "--only-simulate", 'command_flag', flag_value=Flags.SIMULATE)
 @click.option("-V", "--only-validate", 'command_flag', flag_value=Flags.VALIDATE)
 @click.option("-e", "--everything", 'command_flag', flag_value=Flags.E2E, default=True)
 @click.option("-d", "--dry-run", 'filter_flag', flag_value=Flags.DESCRIBE)
-@click.argument("specfiles", type=click.File('r'), nargs=-1, callback=validate_specfiles)
-def main(ctx, config_file, command_flag, filter_flag, specfiles):
+@click.help_option()
+def main(config_file, command_flag, filter_flag, specfiles):
     """
     pyrandall a test framework oriented around data validation instead of code
     """
+    # quickfix: Click will bypass argument callback when nargs=-1
+    # raising these click exceptions will translate to exit(2)
+    if not specfiles:
+        raise click.BadParameter('expecting at least one argument for specfiles')
+
+    if len(specfiles) > 1:
+        raise click.UsageError("passing multiple specfiles is not supported yet")
+
     config = {}
     if config_file:
         config = json.load(config_file)
@@ -78,6 +78,5 @@ def build_basedir(specfile):
         out.append(x)
     return os.path.abspath('/'.join(out))
 
-
 if __name__ == "__main__":
     main()
diff --git a/tests/functional/test_http_simulate.py b/tests/functional/test_http_simulate.py
index 462cc0b..b36a2c1 100644
--- a/tests/functional/test_http_simulate.py
+++ b/tests/functional/test_http_simulate.py
@@ -19,13 +19,13 @@ def test_execute_a_simulation_fails(pyrandall_cli):
         "--config",
         "examples/config/v1.json"
     ])
-    assert 'Usage: main' in result.output
+    assert 'Usage: pyrandall' in result.output
     assert result.exit_code == 2
 
 def test_simulate_json_response_200(pyrandall_cli):
     with vcr.use_cassette("test_simulate_json_response_200") as cassette:
         result = pyrandall_cli.invoke(ARGV_RESPONSE_200)
-        assert 'Usage: main' not in result.output
+        assert 'Usage: pyrandall' not in result.output
         assert result.exit_code == 0
 
         assert len(cassette) == 1
@@ -38,7 +38,7 @@ def test_simulate_json_response_200(pyrandall_cli):
 def test_simulate_json_response_400(pyrandall_cli):
     with vcr.use_cassette("test_simulate_json_response_400") as cassette:
         result = pyrandall_cli.invoke(ARGV_RESPONSE_400)
-        assert 'Usage: main' not in result.output
+        assert 'Usage: pyrandall' not in result.output
         assert result.exit_code == 1
 
         assert len(cassette) == 1

From 026770b7435a5bcecccdab711b18f852d074441d Mon Sep 17 00:00:00 2001
From: Stefano Oldeman <stefano.oldeman@gmail.com>
Date: Wed, 24 Jun 2020 09:09:50 +0200
Subject: [PATCH 07/16] feat: add version to cli

VERSION file is now added to package resources
---
 MANIFEST.in                                    |  3 ++-
 pyrandall/cli.py                               |  5 +++++
 pyrandall/const.py                             | 14 ++++++++++++++
 pyrandall/files/VERSION                        |  1 +
 pyrandall/{ => files}/schemas/__init__.py      |  0
 pyrandall/{ => files}/schemas/scenario/v1.yaml |  0
 pyrandall/{ => files}/schemas/scenario/v2.yaml |  0
 pyrandall/schemas/scenario/__init__.py         |  0
 pyrandall/spec.py                              | 16 +++++-----------
 setup.py                                       |  4 ++++
 10 files changed, 31 insertions(+), 12 deletions(-)
 create mode 100644 pyrandall/files/VERSION
 rename pyrandall/{ => files}/schemas/__init__.py (100%)
 rename pyrandall/{ => files}/schemas/scenario/v1.yaml (100%)
 rename pyrandall/{ => files}/schemas/scenario/v2.yaml (100%)
 delete mode 100644 pyrandall/schemas/scenario/__init__.py

diff --git a/MANIFEST.in b/MANIFEST.in
index 415c9c6..1c7ab0e 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,7 +1,8 @@
-include pyrandall/schemas/scenario/v2.yaml
 include README.md
 include CHANGELOG
 include VERSION
+include pyrandall/files/VERSION
+include pyrandall/files/schemas/scenario/v2.yaml
 
 exclude Makefile
 exclude dev_requirements.txt
diff --git a/pyrandall/cli.py b/pyrandall/cli.py
index e63c1b5..d498f28 100644
--- a/pyrandall/cli.py
+++ b/pyrandall/cli.py
@@ -14,6 +14,10 @@
 from pyrandall.types import Flags
 
 
+# if this fails you have not installed the package (see setup.py)
+VERSION = open(const.VERSION_PATH).read().strip()
+
+
 @click.command(name="pyrandall")
 @click.argument("specfiles", type=click.File('r'), nargs=-1)
 @click.option("-c", "--config", 'config_file', type=click.File('r'), default="pyrandall_config.json", help="path to json file for pyrandall config.")
@@ -22,6 +26,7 @@
 @click.option("-e", "--everything", 'command_flag', flag_value=Flags.E2E, default=True)
 @click.option("-d", "--dry-run", 'filter_flag', flag_value=Flags.DESCRIBE)
 @click.help_option()
+@click.version_option(version=VERSION)
 def main(config_file, command_flag, filter_flag, specfiles):
     """
     pyrandall a test framework oriented around data validation instead of code
diff --git a/pyrandall/const.py b/pyrandall/const.py
index 9b5d704..29c3230 100644
--- a/pyrandall/const.py
+++ b/pyrandall/const.py
@@ -1,3 +1,17 @@
+from os import path
+
 DIRNAME_SCENARIOS = "scenarios"
 DIRNAME_EVENTS = "events"
 DIRNAME_RESULTS = "results"
+
+DIR_PYRANDALL_HOME = path.dirname(path.abspath(__file__))
+
+# Constants to resolve private schema files
+_v2_path = path.join("files", "schemas", "scenario", "v2.yaml")
+SCHEMA_V2_PATH = path.join(DIR_PYRANDALL_HOME,  _v2_path)
+VERSION_SCENARIO_V2 = "scenario/v2"
+SCHEMA_VERSIONS = [VERSION_SCENARIO_V2]
+
+# Package version
+_version_path = path.join("files", "VERSION")
+VERSION_PATH = path.join(DIR_PYRANDALL_HOME, _version_path)
diff --git a/pyrandall/files/VERSION b/pyrandall/files/VERSION
new file mode 100644
index 0000000..0ea3a94
--- /dev/null
+++ b/pyrandall/files/VERSION
@@ -0,0 +1 @@
+0.2.0
diff --git a/pyrandall/schemas/__init__.py b/pyrandall/files/schemas/__init__.py
similarity index 100%
rename from pyrandall/schemas/__init__.py
rename to pyrandall/files/schemas/__init__.py
diff --git a/pyrandall/schemas/scenario/v1.yaml b/pyrandall/files/schemas/scenario/v1.yaml
similarity index 100%
rename from pyrandall/schemas/scenario/v1.yaml
rename to pyrandall/files/schemas/scenario/v1.yaml
diff --git a/pyrandall/schemas/scenario/v2.yaml b/pyrandall/files/schemas/scenario/v2.yaml
similarity index 100%
rename from pyrandall/schemas/scenario/v2.yaml
rename to pyrandall/files/schemas/scenario/v2.yaml
diff --git a/pyrandall/schemas/scenario/__init__.py b/pyrandall/schemas/scenario/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/pyrandall/spec.py b/pyrandall/spec.py
index ea28d6e..d0fd2fd 100644
--- a/pyrandall/spec.py
+++ b/pyrandall/spec.py
@@ -5,6 +5,7 @@
 import yaml
 
 import pyrandall.behaviors
+from pyrandall import const
 from pyrandall.exceptions import InvalidSchenarioVersion
 from pyrandall.types import (
     Adapter,
@@ -17,13 +18,6 @@
 from .network import join_urlpath
 
 
-# Constants to resolve private schema files
-DIR_PYRANDALL_HOME = os.path.dirname(os.path.abspath(__file__))
-SCHEMAS_SCENARIO_V2 = os.path.join(DIR_PYRANDALL_HOME, "schemas/scenario/v2.yaml")
-VERSION_SCENARIO_V2 = "scenario/v2"
-VERSIONS = [VERSION_SCENARIO_V2]
-
-
 class V2Factory(object):
     def __init__(self, **kwargs):
         self.kwargs = kwargs
@@ -49,15 +43,15 @@ def load_spec(self):
         # TODO: prevent reading sensitive files from filesystem
         data = yaml.load(self.specfile, Loader=yaml.FullLoader)
         # implicitly assume scenario v2 schema
-        version = data.get("version", VERSION_SCENARIO_V2)
-        if version not in VERSIONS:
-            raise InvalidSchenarioVersion(VERSIONS)
+        version = data.get("version", const.VERSION_SCENARIO_V2)
+        if version not in const.SCHEMA_VERSIONS:
+            raise InvalidSchenarioVersion(const.SCHEMA_VERSIONS)
         # raises errors if unvalid to jsonschema
         jsonschema.validate(data, self.scenario_v2_schema())
         return data
 
     def scenario_v2_schema(self):
-        with open(SCHEMAS_SCENARIO_V2) as f:
+        with open(const.SCHEMA_V2_PATH) as f:
             return yaml.load(f.read(), Loader=yaml.FullLoader)
 
 
diff --git a/setup.py b/setup.py
index 5888ce5..082f6a3 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,10 @@
+import shutil
+from os import path
 from setuptools import find_packages, setup
 
 __version__ = open("VERSION").read().strip()
+# Also see MANIFEST.ini when changing paths
+shutil.copy('VERSION', path.join('pyrandall', 'files', 'VERSION'))
 
 
 setup(

From 15052613f4afc7a2cfef6451d34f3b4b52da71e4 Mon Sep 17 00:00:00 2001
From: Stefano Oldeman <stefano.oldeman@gmail.com>
Date: Wed, 24 Jun 2020 10:53:55 +0200
Subject: [PATCH 08/16] doc: update examples in README

---
 README.md          | 20 +++++++++++++++++---
 examples/pyrandall |  8 +++-----
 2 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 2947bb9..1173e8b 100644
--- a/README.md
+++ b/README.md
@@ -14,19 +14,33 @@ pip install flask
 python stubserver.py
 ```
 
+pyrandall follows conventions when looking up files defined in the spec.
+This is an example directory structure:
+
+* `results/` is used for lookups by `assertions` Example:
+  `- { equals_to_event: word_count.json }`
+* `events/` is used for lookups when files are referenced in `simulate`
+
+* `scenarios/` can only contain yaml specfiles of the supported json schema.
+
+```
+# Run both simulate and validate
+./examples/pyrandall scenarios/v2.yaml
+```
+
 ```
 # Simulate events from examples/scenarios/v2.yaml and files found in examples/events
-./examples/pyrandall simulate v2.yaml
+./examples/pyrandall --only-simulate scenarios/v2.yaml
 ```
 
 ```
 # Validate results from examples/scenarios/v2.yaml and files found in examples/results
-./examples/pyrandall validate v2.yaml
+./examples/pyrandall --only-validate scenarios/v2.yaml
 ```
 
 # Example of scenario/v2 schema
 
-The input yaml is validated with jsonschema, the schema can be found here [pyrandall/schemas/scenario/v2.yaml](https://github.com/kpn/pyrandall/tree/master/pyrandall/schemas/scenario/v2.yaml).
+The input yaml is validated with jsonschema, the schema can be found here [pyrandall/files/schemas/scenario/v2.yaml](https://github.com/kpn/pyrandall/tree/master/pyrandall/files/schemas/scenario/v2.yaml).
 
 ```
 ---
diff --git a/examples/pyrandall b/examples/pyrandall
index 25f6230..3812c57 100755
--- a/examples/pyrandall
+++ b/examples/pyrandall
@@ -1,15 +1,13 @@
 #!/usr/bin/env sh
 
+set -euf -o pipefail
+
 SCRIPT=`realpath $0`
 SCRIPTPATH=`dirname $SCRIPT`
 
 KAFKA_BOOTSTRAP_SERVERS=${KAFKA_BOOTSTRAP_SERVERS:-"localhost:9092"}
-SCHEMAS_SERVER_URL=${SCHEMAS_SERVER_URL:-"http://localhost:8080/"}
-
-echo $SCRIPTPATH
 
 docker run --rm -t --network host --volume "${SCRIPTPATH}:/cwd" \
   -e KAFKA_BOOTSTRAP_SERVERS="$KAFKA_BOOTSTRAP_SERVERS" \
-  -e SCHEMAS="$SCHEMAS_SERVER_URL" \
   pyrandall:latest \
-  --config /cwd/config/test_env.json --dataflow /cwd $@
+  --config /cwd/config/test_env.json $@

From e5b11da25307d7576e3dac17312ad4c6c023ed4a Mon Sep 17 00:00:00 2001
From: Stefano Oldeman <stefano.oldeman@gmail.com>
Date: Sat, 27 Jun 2020 12:33:50 +0200
Subject: [PATCH 09/16] fix: set custom user-agent

motivation: requests user-agent keeps braking vcrpy fixtures
Added tests/unit/test_requests_http_useragent.py
this introduced side-effects in vcr config.
Thus refactored vcr import to a vcr fixture.
I evaluated github.com/kiwicom/pytest-recording
but the tests need cassette in as test argument
---
 pyrandall/cli.py                              |  6 +--
 pyrandall/const.py                            | 10 +++-
 pyrandall/executors/requests_http.py          |  8 +++-
 pyrandall/files/VERSION                       |  2 +-
 tests/conftest.py                             | 44 +++++++++++++----
 .../vcr/test_commander_run_one_for_one.yaml   | 12 ++---
 .../test_http_executor_simulate_post_201.yaml | 10 +++-
 ...tp_executor_simulate_post_201_and_400.yaml | 20 ++++++--
 ...p_executor_simulate_post_400_response.yaml | 10 +++-
 ...tor_validate__matches_body_and_status.yaml | 12 +++--
 ...validate_body_and_status_do_not_match.yaml | 12 +++--
 ...test_http_executor_validate_makes_get.yaml | 14 ++++--
 ...r_validate_makes_get_and_matches_body.yaml | 14 ++++--
 ...ns_pass.yaml => test_http_user_agent.yaml} | 22 +++++----
 .../vcr/test_simulate_json_response_200.yaml  |  6 +--
 .../vcr/test_simulate_json_response_400.yaml  |  6 +--
 .../vcr/test_validate_fail_status_code.yaml   | 39 ---------------
 tests/functional/test_http_simulate.py        |  6 +--
 tests/functional/test_http_validate.py        | 35 ++++++--------
 tests/functional/test_kafka_simulate.py       |  3 --
 tests/functional/test_kafka_validate.py       | 11 ++---
 tests/unit/test_commander.py                  | 30 ++++++------
 tests/unit/test_requests_http_simulate.py     | 25 +++-------
 tests/unit/test_requests_http_useragent.py    | 47 +++++++++++++++++++
 tests/unit/test_requests_http_validate.py     | 17 ++++---
 25 files changed, 241 insertions(+), 180 deletions(-)
 rename tests/fixtures/vcr/{test_validate_assertions_pass.yaml => test_http_user_agent.yaml} (55%)
 delete mode 100644 tests/fixtures/vcr/test_validate_fail_status_code.yaml
 create mode 100644 tests/unit/test_requests_http_useragent.py

diff --git a/pyrandall/cli.py b/pyrandall/cli.py
index d498f28..c8fb03a 100644
--- a/pyrandall/cli.py
+++ b/pyrandall/cli.py
@@ -14,10 +14,6 @@
 from pyrandall.types import Flags
 
 
-# if this fails you have not installed the package (see setup.py)
-VERSION = open(const.VERSION_PATH).read().strip()
-
-
 @click.command(name="pyrandall")
 @click.argument("specfiles", type=click.File('r'), nargs=-1)
 @click.option("-c", "--config", 'config_file', type=click.File('r'), default="pyrandall_config.json", help="path to json file for pyrandall config.")
@@ -26,7 +22,7 @@
 @click.option("-e", "--everything", 'command_flag', flag_value=Flags.E2E, default=True)
 @click.option("-d", "--dry-run", 'filter_flag', flag_value=Flags.DESCRIBE)
 @click.help_option()
-@click.version_option(version=VERSION)
+@click.version_option(version=const.get_version())
 def main(config_file, command_flag, filter_flag, specfiles):
     """
     pyrandall a test framework oriented around data validation instead of code
diff --git a/pyrandall/const.py b/pyrandall/const.py
index 29c3230..ff69a89 100644
--- a/pyrandall/const.py
+++ b/pyrandall/const.py
@@ -13,5 +13,11 @@
 SCHEMA_VERSIONS = [VERSION_SCENARIO_V2]
 
 # Package version
-_version_path = path.join("files", "VERSION")
-VERSION_PATH = path.join(DIR_PYRANDALL_HOME, _version_path)
+VERSION_PATH = path.join("files", "VERSION")
+
+PYRANDALL_USER_AGENT = "pyrandall"
+
+def get_version():
+    version_path = path.join(DIR_PYRANDALL_HOME, VERSION_PATH)
+    # if this fails you have not installed the package (see setup.py)
+    return open(version_path).read().strip()
diff --git a/pyrandall/executors/requests_http.py b/pyrandall/executors/requests_http.py
index b21d015..07e54a2 100644
--- a/pyrandall/executors/requests_http.py
+++ b/pyrandall/executors/requests_http.py
@@ -1,6 +1,7 @@
 import requests
 
 from pyrandall.types import Assertion
+from pyrandall import const
 
 from .common import Executor
 
@@ -9,7 +10,7 @@ class RequestHttp(Executor):
     def __init__(self, spec, *args, **kwargs):
         super().__init__()
         self.execution_mode = spec.execution_mode
-        self.spec = spec
+        self.spec = self.add_custom_headers(spec)
 
     def execute(self, reporter):
         spec = self.spec
@@ -42,6 +43,11 @@ def execute(self, reporter):
         # TODO: depricate this, not functioally needed anymore
         return all([a.passed() for a in assertions])
 
+    def add_custom_headers(self, spec):
+        version = const.get_version()
+        spec.headers['User-Agent'] = f"{const.PYRANDALL_USER_AGENT}/{version}"
+        return spec
+
     # TODO: move this to reporter
     def create_jsondiff(self, expected, actual):
         print("Output data different")
diff --git a/pyrandall/files/VERSION b/pyrandall/files/VERSION
index 0ea3a94..3eefcb9 100644
--- a/pyrandall/files/VERSION
+++ b/pyrandall/files/VERSION
@@ -1 +1 @@
-0.2.0
+1.0.0
diff --git a/tests/conftest.py b/tests/conftest.py
index 9187c48..a3c96d0 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,23 +1,47 @@
 import os
+from unittest.mock import MagicMock
 
 import pyrandall.cli
 
-import vcr
+from vcr import VCR
 import pytest
 
 from click.testing import CliRunner
 from confluent_kafka.admin import AdminClient, ClusterMetadata
 
-defaults = ["method", "scheme", "host", "port", "path", "query"]
-vcr = vcr.VCR(
-    cassette_library_dir="tests/fixtures/vcr",
-    # record_mode =  [once, new_episodes, none, all]
+
+@pytest.fixture
+def vcr_headers_filter():
+    return [("User-Agent", None)]
+
+@pytest.fixture
+def vcr_match_on():
+    return ["body", "headers"]
+
+@pytest.fixture
+def vcr(request):
+    defaults = ["method", "scheme", "host", "port", "path", "query"]
+    defaults += request.getfixturevalue("vcr_match_on")
+
+    # record_mode = {once, new_episodes, none, all}
     # https://vcrpy.readthedocs.io/en/latest/usage.html#record-modes
-    record_mode=os.environ.get("VCR_MODE", "once"),
-    match_on=(defaults + ["body", "headers"]),
-    path_transformer=vcr.VCR.ensure_suffix(".yaml"),
-)
+    return VCR(
+        filter_headers=request.getfixturevalue("vcr_headers_filter"),
+        record_mode=os.environ.get("VCR_MODE", "once"),
+        cassette_library_dir="tests/fixtures/vcr",
+        match_on=defaults,
+        path_transformer=VCR.ensure_suffix(".yaml"),
+    )
 
+# Internal Class Mocks, Stubs etc.
+
+@pytest.fixture
+def reporter():
+    return MagicMock(unsafe=True)
+
+
+# Helper functions
+# for example: file, http, kafka utilities or Cli Runner
 
 @pytest.fixture
 def kafka_cluster_info() -> ClusterMetadata:
@@ -39,3 +63,5 @@ class PyrandallCli():
     def invoke(self, command):
         runner = CliRunner()
         return runner.invoke(pyrandall.cli.main, command, catch_exceptions=False)
+
+
diff --git a/tests/fixtures/vcr/test_commander_run_one_for_one.yaml b/tests/fixtures/vcr/test_commander_run_one_for_one.yaml
index 5608330..6d21422 100644
--- a/tests/fixtures/vcr/test_commander_run_one_for_one.yaml
+++ b/tests/fixtures/vcr/test_commander_run_one_for_one.yaml
@@ -10,8 +10,6 @@ interactions:
       - keep-alive
       Content-Length:
       - '14'
-      User-Agent:
-      - python-requests/2.23.0
       content-type:
       - application/json
     method: POST
@@ -34,9 +32,9 @@ interactions:
       Content-Type:
       - text/html
       Date:
-      - Mon, 16 Sep 2019 09:54:45 GMT
+      - Sat, 27 Jun 2020 09:31:48 GMT
       Server:
-      - Werkzeug/0.15.6 Python/3.7.4
+      - Werkzeug/0.16.0 Python/3.7.5
     status:
       code: 404
       message: NOT FOUND
@@ -49,8 +47,6 @@ interactions:
       - gzip, deflate
       Connection:
       - keep-alive
-      User-Agent:
-      - python-requests/2.23.0
     method: GET
     uri: http://localhost:5000/foo/bar/123
   response:
@@ -62,9 +58,9 @@ interactions:
       Content-Type:
       - text/html; charset=utf-8
       Date:
-      - Mon, 16 Sep 2019 09:54:45 GMT
+      - Sat, 27 Jun 2020 09:31:48 GMT
       Server:
-      - Werkzeug/0.15.6 Python/3.7.4
+      - Werkzeug/0.16.0 Python/3.7.5
     status:
       code: 200
       message: OK
diff --git a/tests/fixtures/vcr/test_http_executor_simulate_post_201.yaml b/tests/fixtures/vcr/test_http_executor_simulate_post_201.yaml
index 5c6e72b..3ade710 100644
--- a/tests/fixtures/vcr/test_http_executor_simulate_post_201.yaml
+++ b/tests/fixtures/vcr/test_http_executor_simulate_post_201.yaml
@@ -2,6 +2,12 @@ interactions:
 - request:
     body: '{"foo": "bar"}'
     headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
       Content-Length:
       - '14'
     method: POST
@@ -15,9 +21,9 @@ interactions:
       Content-Type:
       - text/html; charset=utf-8
       Date:
-      - Mon, 16 Sep 2019 09:54:45 GMT
+      - Sat, 27 Jun 2020 09:31:48 GMT
       Server:
-      - Werkzeug/0.15.6 Python/3.7.4
+      - Werkzeug/0.16.0 Python/3.7.5
     status:
       code: 201
       message: CREATED
diff --git a/tests/fixtures/vcr/test_http_executor_simulate_post_201_and_400.yaml b/tests/fixtures/vcr/test_http_executor_simulate_post_201_and_400.yaml
index 48ea1bf..79a767e 100644
--- a/tests/fixtures/vcr/test_http_executor_simulate_post_201_and_400.yaml
+++ b/tests/fixtures/vcr/test_http_executor_simulate_post_201_and_400.yaml
@@ -2,6 +2,12 @@ interactions:
 - request:
     body: null
     headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
       Content-Length:
       - '0'
     method: POST
@@ -23,15 +29,21 @@ interactions:
       Content-Type:
       - text/html
       Date:
-      - Mon, 16 Sep 2019 09:54:45 GMT
+      - Sat, 27 Jun 2020 09:31:48 GMT
       Server:
-      - Werkzeug/0.15.6 Python/3.7.4
+      - Werkzeug/0.16.0 Python/3.7.5
     status:
       code: 400
       message: BAD REQUEST
 - request:
     body: '{"foo": "bar"}'
     headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
       Content-Length:
       - '14'
     method: POST
@@ -45,9 +57,9 @@ interactions:
       Content-Type:
       - text/html; charset=utf-8
       Date:
-      - Mon, 16 Sep 2019 09:54:45 GMT
+      - Sat, 27 Jun 2020 09:31:48 GMT
       Server:
-      - Werkzeug/0.15.6 Python/3.7.4
+      - Werkzeug/0.16.0 Python/3.7.5
     status:
       code: 201
       message: CREATED
diff --git a/tests/fixtures/vcr/test_http_executor_simulate_post_400_response.yaml b/tests/fixtures/vcr/test_http_executor_simulate_post_400_response.yaml
index a3997ee..ab1705a 100644
--- a/tests/fixtures/vcr/test_http_executor_simulate_post_400_response.yaml
+++ b/tests/fixtures/vcr/test_http_executor_simulate_post_400_response.yaml
@@ -2,6 +2,12 @@ interactions:
 - request:
     body: null
     headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
       Content-Length:
       - '0'
     method: POST
@@ -23,9 +29,9 @@ interactions:
       Content-Type:
       - text/html
       Date:
-      - Mon, 16 Sep 2019 09:54:45 GMT
+      - Sat, 27 Jun 2020 09:31:48 GMT
       Server:
-      - Werkzeug/0.15.6 Python/3.7.4
+      - Werkzeug/0.16.0 Python/3.7.5
     status:
       code: 400
       message: BAD REQUEST
diff --git a/tests/fixtures/vcr/test_http_executor_validate__matches_body_and_status.yaml b/tests/fixtures/vcr/test_http_executor_validate__matches_body_and_status.yaml
index 6cda7b6..e2f4435 100644
--- a/tests/fixtures/vcr/test_http_executor_validate__matches_body_and_status.yaml
+++ b/tests/fixtures/vcr/test_http_executor_validate__matches_body_and_status.yaml
@@ -1,7 +1,13 @@
 interactions:
 - request:
     body: null
-    headers: {}
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
     method: GET
     uri: http://localhost:5000/foo/2
   response:
@@ -13,9 +19,9 @@ interactions:
       Content-Type:
       - text/html; charset=utf-8
       Date:
-      - Mon, 16 Sep 2019 09:54:45 GMT
+      - Sat, 27 Jun 2020 09:31:48 GMT
       Server:
-      - Werkzeug/0.15.6 Python/3.7.4
+      - Werkzeug/0.16.0 Python/3.7.5
     status:
       code: 200
       message: OK
diff --git a/tests/fixtures/vcr/test_http_executor_validate_body_and_status_do_not_match.yaml b/tests/fixtures/vcr/test_http_executor_validate_body_and_status_do_not_match.yaml
index 6cda7b6..b4e3f0c 100644
--- a/tests/fixtures/vcr/test_http_executor_validate_body_and_status_do_not_match.yaml
+++ b/tests/fixtures/vcr/test_http_executor_validate_body_and_status_do_not_match.yaml
@@ -1,7 +1,13 @@
 interactions:
 - request:
     body: null
-    headers: {}
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
     method: GET
     uri: http://localhost:5000/foo/2
   response:
@@ -13,9 +19,9 @@ interactions:
       Content-Type:
       - text/html; charset=utf-8
       Date:
-      - Mon, 16 Sep 2019 09:54:45 GMT
+      - Sat, 27 Jun 2020 09:46:28 GMT
       Server:
-      - Werkzeug/0.15.6 Python/3.7.4
+      - Werkzeug/0.16.0 Python/3.7.5
     status:
       code: 200
       message: OK
diff --git a/tests/fixtures/vcr/test_http_executor_validate_makes_get.yaml b/tests/fixtures/vcr/test_http_executor_validate_makes_get.yaml
index 933e42d..0f17db9 100644
--- a/tests/fixtures/vcr/test_http_executor_validate_makes_get.yaml
+++ b/tests/fixtures/vcr/test_http_executor_validate_makes_get.yaml
@@ -1,7 +1,15 @@
 interactions:
 - request:
     body: null
-    headers: {}
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+      User-Agent:
+      - pyrandall
     method: GET
     uri: http://localhost:5000/foo/1
   response:
@@ -22,9 +30,9 @@ interactions:
       Content-Type:
       - text/html
       Date:
-      - Mon, 16 Sep 2019 09:54:45 GMT
+      - Sat, 27 Jun 2020 09:31:48 GMT
       Server:
-      - Werkzeug/0.15.6 Python/3.7.4
+      - Werkzeug/0.16.0 Python/3.7.5
     status:
       code: 404
       message: NOT FOUND
diff --git a/tests/fixtures/vcr/test_http_executor_validate_makes_get_and_matches_body.yaml b/tests/fixtures/vcr/test_http_executor_validate_makes_get_and_matches_body.yaml
index 6cda7b6..f7d7a7a 100644
--- a/tests/fixtures/vcr/test_http_executor_validate_makes_get_and_matches_body.yaml
+++ b/tests/fixtures/vcr/test_http_executor_validate_makes_get_and_matches_body.yaml
@@ -1,7 +1,15 @@
 interactions:
 - request:
     body: null
-    headers: {}
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+      User-Agent:
+      - pyrandall
     method: GET
     uri: http://localhost:5000/foo/2
   response:
@@ -13,9 +21,9 @@ interactions:
       Content-Type:
       - text/html; charset=utf-8
       Date:
-      - Mon, 16 Sep 2019 09:54:45 GMT
+      - Sat, 27 Jun 2020 09:31:48 GMT
       Server:
-      - Werkzeug/0.15.6 Python/3.7.4
+      - Werkzeug/0.16.0 Python/3.7.5
     status:
       code: 200
       message: OK
diff --git a/tests/fixtures/vcr/test_validate_assertions_pass.yaml b/tests/fixtures/vcr/test_http_user_agent.yaml
similarity index 55%
rename from tests/fixtures/vcr/test_validate_assertions_pass.yaml
rename to tests/fixtures/vcr/test_http_user_agent.yaml
index 08126dc..7b63b93 100644
--- a/tests/fixtures/vcr/test_validate_assertions_pass.yaml
+++ b/tests/fixtures/vcr/test_http_user_agent.yaml
@@ -1,6 +1,6 @@
 interactions:
 - request:
-    body: null
+    body: '{"foo": "bar"}'
     headers:
       Accept:
       - '*/*'
@@ -8,23 +8,25 @@ interactions:
       - gzip, deflate
       Connection:
       - keep-alive
+      Content-Length:
+      - '14'
       User-Agent:
-      - python-requests/2.23.0
-    method: GET
-    uri: http://localhost:5000/kv/pyrandall/avro_ok
+      - pyrandall/1.0.0
+    method: POST
+    uri: http://localhost:5000/users
   response:
     body:
-      string: '{"Avro": "ok"}'
+      string: ''
     headers:
       Content-Length:
-      - '14'
+      - '0'
       Content-Type:
       - text/html; charset=utf-8
       Date:
-      - Mon, 16 Sep 2019 09:54:44 GMT
+      - Sat, 27 Jun 2020 10:26:54 GMT
       Server:
-      - Werkzeug/0.15.6 Python/3.7.4
+      - Werkzeug/0.16.0 Python/3.7.5
     status:
-      code: 200
-      message: OK
+      code: 201
+      message: CREATED
 version: 1
diff --git a/tests/fixtures/vcr/test_simulate_json_response_200.yaml b/tests/fixtures/vcr/test_simulate_json_response_200.yaml
index 6ea0780..6aff903 100644
--- a/tests/fixtures/vcr/test_simulate_json_response_200.yaml
+++ b/tests/fixtures/vcr/test_simulate_json_response_200.yaml
@@ -10,8 +10,6 @@ interactions:
       - keep-alive
       Content-Length:
       - '13'
-      User-Agent:
-      - python-requests/2.23.0
       content-type:
       - application/json
     method: POST
@@ -23,9 +21,9 @@ interactions:
       Content-Type:
       - text/html; charset=utf-8
       Date:
-      - Mon, 16 Sep 2019 09:54:43 GMT
+      - Sat, 27 Jun 2020 09:55:09 GMT
       Server:
-      - Werkzeug/0.15.6 Python/3.7.4
+      - Werkzeug/0.16.0 Python/3.7.5
     status:
       code: 204
       message: NO CONTENT
diff --git a/tests/fixtures/vcr/test_simulate_json_response_400.yaml b/tests/fixtures/vcr/test_simulate_json_response_400.yaml
index 2b2886d..9b5eda9 100644
--- a/tests/fixtures/vcr/test_simulate_json_response_400.yaml
+++ b/tests/fixtures/vcr/test_simulate_json_response_400.yaml
@@ -10,8 +10,6 @@ interactions:
       - keep-alive
       Content-Length:
       - '13'
-      User-Agent:
-      - python-requests/2.23.0
       content-type:
       - application/json
     method: POST
@@ -34,9 +32,9 @@ interactions:
       Content-Type:
       - text/html
       Date:
-      - Mon, 16 Sep 2019 09:54:43 GMT
+      - Sat, 27 Jun 2020 09:55:09 GMT
       Server:
-      - Werkzeug/0.15.6 Python/3.7.4
+      - Werkzeug/0.16.0 Python/3.7.5
     status:
       code: 404
       message: NOT FOUND
diff --git a/tests/fixtures/vcr/test_validate_fail_status_code.yaml b/tests/fixtures/vcr/test_validate_fail_status_code.yaml
deleted file mode 100644
index c9d71bc..0000000
--- a/tests/fixtures/vcr/test_validate_fail_status_code.yaml
+++ /dev/null
@@ -1,39 +0,0 @@
-interactions:
-- request:
-    body: null
-    headers:
-      Accept:
-      - '*/*'
-      Accept-Encoding:
-      - gzip, deflate
-      Connection:
-      - keep-alive
-      User-Agent:
-      - python-requests/2.23.0
-    method: GET
-    uri: http://localhost:5000/cant_find_this
-  response:
-    body:
-      string: '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
-
-        <title>404 Not Found</title>
-
-        <h1>Not Found</h1>
-
-        <p>The requested URL was not found on the server. If you entered the URL manually
-        please check your spelling and try again.</p>
-
-        '
-    headers:
-      Content-Length:
-      - '232'
-      Content-Type:
-      - text/html
-      Date:
-      - Mon, 16 Sep 2019 09:54:44 GMT
-      Server:
-      - Werkzeug/0.15.6 Python/3.7.4
-    status:
-      code: 404
-      message: NOT FOUND
-version: 1
diff --git a/tests/functional/test_http_simulate.py b/tests/functional/test_http_simulate.py
index b36a2c1..dc65dca 100644
--- a/tests/functional/test_http_simulate.py
+++ b/tests/functional/test_http_simulate.py
@@ -1,5 +1,3 @@
-from tests.conftest import vcr
-
 ARGV_RESPONSE_200 = [
     "--config",
     "examples/config/v1.json",
@@ -22,7 +20,7 @@ def test_execute_a_simulation_fails(pyrandall_cli):
     assert 'Usage: pyrandall' in result.output
     assert result.exit_code == 2
 
-def test_simulate_json_response_200(pyrandall_cli):
+def test_simulate_json_response_200(pyrandall_cli, vcr):
     with vcr.use_cassette("test_simulate_json_response_200") as cassette:
         result = pyrandall_cli.invoke(ARGV_RESPONSE_200)
         assert 'Usage: pyrandall' not in result.output
@@ -35,7 +33,7 @@ def test_simulate_json_response_200(pyrandall_cli):
         assert cassette.all_played
         # not all request had the expected status code (see assertions)
 
-def test_simulate_json_response_400(pyrandall_cli):
+def test_simulate_json_response_400(pyrandall_cli, vcr):
     with vcr.use_cassette("test_simulate_json_response_400") as cassette:
         result = pyrandall_cli.invoke(ARGV_RESPONSE_400)
         assert 'Usage: pyrandall' not in result.output
diff --git a/tests/functional/test_http_validate.py b/tests/functional/test_http_validate.py
index 68434c7..2fdf1c1 100644
--- a/tests/functional/test_http_validate.py
+++ b/tests/functional/test_http_validate.py
@@ -1,21 +1,30 @@
 import os
 import pytest
-from freezegun import freeze_time
-
 import threading
-from tests.conftest import vcr
+
+from freezegun import freeze_time
 from tests.helper import KafkaProducer
 from pyrandall.kafka import KafkaSetupError
 
 TOPIC_1 = "pyrandall-tests-validate-1"
 TOPIC_2 = "pyrandall-tests-validate-2"
 
-MOCK_ARGV = [
+ARGV_SMALL = [
     "--config",
     "examples/config/v1.json",
-    "-V"
+    "-V",
+    "examples/scenarios/v2_ingest_kafka_small.yaml"
 ]
-ARGV_SMALL = MOCK_ARGV + ["examples/scenarios/v2_ingest_kafka_small.yaml"]
+
+
+def produce_events():
+    producer = KafkaProducer(TOPIC_1)
+    producer.send(b'{"click": "three"}')
+    producer.send(b'{"click": "one"}')
+    producer.send(b'{"click": "two"}')
+
+    producer = KafkaProducer(TOPIC_2)
+    producer.send(b'{"click": "three"}')
 
 
 def test_error_on_connection_timeout(monkeypatch, pyrandall_cli):
@@ -23,10 +32,7 @@ def test_error_on_connection_timeout(monkeypatch, pyrandall_cli):
     with pytest.raises(KafkaSetupError) as e:
         pyrandall_cli.invoke(ARGV_SMALL)
 
-
-# freeze time in order to hardcode timestamps
 @freeze_time("2012-01-14 14:33:12")
-@vcr.use_cassette("test_ingest_to_kafka")
 def test_received_no_events(monkeypatch, kafka_cluster_info, pyrandall_cli):
     result = pyrandall_cli.invoke(ARGV_SMALL)
     # exit code should be 1 (error)
@@ -34,18 +40,7 @@ def test_received_no_events(monkeypatch, kafka_cluster_info, pyrandall_cli):
     print(result.output)
     assert result.exit_code == 1
 
-def produce_events():
-    # produce the events
-    producer = KafkaProducer(TOPIC_1)
-    producer.send(b'{"click": "three"}')
-    producer.send(b'{"click": "one"}')
-    producer.send(b'{"click": "two"}')
-
-    producer = KafkaProducer(TOPIC_2)
-    producer.send(b'{"click": "three"}')
-
 @freeze_time("2012-01-14 14:33:12")
-@vcr.use_cassette("test_ingest_to_kafka")
 def test_validate_unordered_passed(kafka_cluster_info, pyrandall_cli):
     produce_events()
     result = pyrandall_cli.invoke(ARGV_SMALL)
diff --git a/tests/functional/test_kafka_simulate.py b/tests/functional/test_kafka_simulate.py
index 6bee6e4..0bdfd96 100644
--- a/tests/functional/test_kafka_simulate.py
+++ b/tests/functional/test_kafka_simulate.py
@@ -2,7 +2,6 @@
 import pytest
 from freezegun import freeze_time
 
-from tests.conftest import vcr
 from tests.helper import KafkaConsumer
 
 
@@ -23,9 +22,7 @@ def test_fail_on_invaild_schema(pyrandall_cli):
     assert result.exit_code == 4
 
 
-# freeze time in order to hardcode timestamps
 @freeze_time("2012-01-14 14:33:12")
-@vcr.use_cassette("test_ingest_to_kafka")
 def test_simulate_produces_event(kafka_cluster_info, pyrandall_cli):
     consumer = KafkaConsumer(TEST_TOPIC)
     try:
diff --git a/tests/functional/test_kafka_validate.py b/tests/functional/test_kafka_validate.py
index c248691..dfe6165 100644
--- a/tests/functional/test_kafka_validate.py
+++ b/tests/functional/test_kafka_validate.py
@@ -3,19 +3,18 @@
 from freezegun import freeze_time
 
 import threading
-from tests.conftest import vcr
 from tests.helper import KafkaProducer
 from pyrandall.kafka import KafkaSetupError
 
 TOPIC_1 = "pyrandall-tests-validate-1"
 TOPIC_2 = "pyrandall-tests-validate-2"
 
-MOCK_ARGV = [
+ARGV_SMALL = [
     "--config",
     "examples/config/v1.json",
-    "-V"
+    "-V",
+    "examples/scenarios/v2_ingest_kafka_small.yaml"
 ]
-ARGV_SMALL = MOCK_ARGV + ["examples/scenarios/v2_ingest_kafka_small.yaml"]
 
 
 def produce_events():
@@ -37,7 +36,6 @@ def test_error_on_connection_timeout(monkeypatch, pyrandall_cli):
 
 # freeze time in order to hardcode timestamps
 @freeze_time("2012-01-14 14:33:12")
-@vcr.use_cassette("test_ingest_to_kafka")
 def test_received_no_events(monkeypatch, kafka_cluster_info, pyrandall_cli):
     """
     run validate to consume a message from kafka
@@ -50,11 +48,8 @@ def test_received_no_events(monkeypatch, kafka_cluster_info, pyrandall_cli):
 
 
 @freeze_time("2012-01-14 14:33:12")
-@vcr.use_cassette("test_ingest_to_kafka")
 def test_validate_unordered_passed(kafka_cluster_info, pyrandall_cli):
     produce_events()
     result = pyrandall_cli.invoke(ARGV_SMALL)
     assert 'Usage: main' not in result.output
     assert result.exit_code == 0
-
-
diff --git a/tests/unit/test_commander.py b/tests/unit/test_commander.py
index f64e569..7b590ba 100644
--- a/tests/unit/test_commander.py
+++ b/tests/unit/test_commander.py
@@ -5,7 +5,6 @@
 from pyrandall.commander import Commander, Flags
 from pyrandall.reporter import Reporter, ResultSet
 from pyrandall.spec import SpecBuilder
-from tests.conftest import vcr
 
 
 @pytest.fixture
@@ -19,19 +18,20 @@ def spec():
     return builder.feature()
 
 
-@vcr.use_cassette("test_commander_run_one_for_one")
-def test_commander_run_one_for_one(spec):
-    reporter = MagicMock(Reporter(), unsafe=True)
-    reporter.create_and_track_resultset.return_value = MagicMock(ResultSet, unsafe=True)
+def test_commander_run_one_for_one(spec, vcr):
+    with vcr.use_cassette("test_commander_run_one_for_one") as cassette:
+        reporter = MagicMock(Reporter(), unsafe=True)
+        reporter.create_and_track_resultset.return_value = MagicMock(ResultSet, unsafe=True)
 
-    c = Commander(spec, Flags.E2E)
-    c.run(reporter)
+        c = Commander(spec, Flags.E2E)
+        c.run(reporter)
 
-    reporter.feature.assert_called_once_with("One event"),
-    reporter.scenario.assert_any_call("Send words1 event")
-    # at least once called
-    reporter.simulate.assert_called()
-    reporter.validate.assert_called()
-    reporter.run_task.assert_called()
-    reporter.print_failures.assert_called_once_with()
-    reporter.passed.assert_called_once()
+        reporter.feature.assert_called_once_with("One event"),
+        reporter.scenario.assert_any_call("Send words1 event")
+        # at least once called
+        reporter.simulate.assert_called()
+        reporter.validate.assert_called()
+        reporter.run_task.assert_called()
+        reporter.print_failures.assert_called_once_with()
+        reporter.passed.assert_called_once()
+        assert len(cassette) == 2
diff --git a/tests/unit/test_requests_http_simulate.py b/tests/unit/test_requests_http_simulate.py
index 17578b1..814a766 100644
--- a/tests/unit/test_requests_http_simulate.py
+++ b/tests/unit/test_requests_http_simulate.py
@@ -5,17 +5,6 @@
 from pyrandall.executors import RequestHttp, RequestHttpEvents
 from pyrandall.spec import RequestEventsSpec, RequestHttpSpec
 from pyrandall.types import Assertion, ExecutionMode
-from tests.conftest import vcr
-
-
-@pytest.fixture
-def reporter():
-    return MagicMock(unsafe=True)
-
-
-@pytest.fixture
-def reporter_1():
-    return MagicMock(assertion=MagicMock(spec_set=Assertion), unsafe=True)
 
 
 STATUS_CODE_ASSERTION = {"status_code": 201}
@@ -29,7 +18,7 @@ def simulator_1():
         url="http://localhost:5000/users",
         body=b'{"foo": "bar"}',
         method="POST",
-        headers=[],
+        headers={},
     )
     return RequestHttp(spec)
 
@@ -42,7 +31,7 @@ def simulator_2():
         url="http://localhost:5000/users",
         body=b"",
         method="POST",
-        headers=[],
+        headers={},
     )
     return RequestHttp(spec)
 
@@ -55,7 +44,7 @@ def simulator_3():
         url="http://localhost:5000/users",
         body=b"",
         method="POST",
-        headers=[],
+        headers={},
     )
     r2 = RequestHttpSpec(
         execution_mode=ExecutionMode.SIMULATING,
@@ -63,13 +52,13 @@ def simulator_3():
         url="http://localhost:5000/users",
         body=b'{"foo": "bar"}',
         method="POST",
-        headers=[],
+        headers={},
     )
     spec = RequestEventsSpec(requests=[r1, r2])
     return RequestHttpEvents(spec)
 
 
-def test_simulate__post_201_repsonse(simulator_1, reporter):
+def test_simulate__post_201_repsonse(simulator_1, reporter, vcr):
     with vcr.use_cassette("test_http_executor_simulate_post_201") as cassette:
         result = simulator_1.execute(reporter)
 
@@ -85,7 +74,7 @@ def test_simulate__post_201_repsonse(simulator_1, reporter):
         assert result
 
 
-def test_simulate_post_400_response(simulator_2, reporter):
+def test_simulate_post_400_response(simulator_2, reporter, vcr):
     with vcr.use_cassette("test_http_executor_simulate_post_400_response") as cassette:
         result = simulator_2.execute(reporter)
 
@@ -108,7 +97,7 @@ def test_simulate_fails_zero_requests(reporter):
     assert not result
 
 
-def test_simulate_post_200_and_400(simulator_3, reporter):
+def test_simulate_post_200_and_400(simulator_3, reporter, vcr):
     with vcr.use_cassette("test_http_executor_simulate_post_201_and_400") as cassette:
         result = simulator_3.execute(reporter)
 
diff --git a/tests/unit/test_requests_http_useragent.py b/tests/unit/test_requests_http_useragent.py
new file mode 100644
index 0000000..b5de6ce
--- /dev/null
+++ b/tests/unit/test_requests_http_useragent.py
@@ -0,0 +1,47 @@
+import pytest
+
+from pyrandall import const
+from pyrandall.executors import RequestHttp
+from pyrandall.spec import RequestHttpSpec
+from pyrandall.types import ExecutionMode
+from pytest_httpserver import HeaderValueMatcher
+
+STATUS_CODE_ASSERTION = {"status_code": 201}
+
+
+@pytest.fixture
+def request_http_spec():
+    return RequestHttpSpec(
+        execution_mode=ExecutionMode.SIMULATING,
+        assertions=STATUS_CODE_ASSERTION,
+        url="http://localhost:5000/users",
+        body=b'{"foo": "bar"}',
+        method="POST",
+        headers={},
+    )
+
+@pytest.fixture
+def executor_http(request_http_spec):
+    return RequestHttp(request_http_spec)
+
+@pytest.fixture
+def vcr_headers_filter():
+    return []
+
+@pytest.fixture
+def vcr_match_on():
+    return []
+
+def test_header_present(httpserver, executor_http, reporter, vcr):
+    with vcr.use_cassette("test_http_user_agent") as cassette:
+        result = executor_http.execute(reporter)
+
+        assert len(cassette) == 1
+        r0 = cassette.requests[0]
+        assert r0.url == "http://localhost:5000/users"
+        assert r0.method == 'POST'
+        assert len(r0.headers) == 5
+        assert 'User-Agent' in r0.headers
+        agent = r0.headers['User-Agent']
+        assert agent.startswith("pyrandall/")
+        assert agent.split('/')[1] == const.get_version()
diff --git a/tests/unit/test_requests_http_validate.py b/tests/unit/test_requests_http_validate.py
index 2762393..e8146b2 100644
--- a/tests/unit/test_requests_http_validate.py
+++ b/tests/unit/test_requests_http_validate.py
@@ -6,7 +6,6 @@
 from pyrandall.reporter import Reporter
 from pyrandall.spec import RequestHttpSpec
 from pyrandall.types import Assertion, ExecutionMode
-from tests.conftest import vcr
 
 
 @pytest.fixture
@@ -26,7 +25,7 @@ def validator_1():
         url="http://localhost:5000/foo/1",
         events=[],
         method="GET",
-        headers=[],
+        headers={},
         assertions={"status_code": 404},
     )
     return RequestHttp(spec)
@@ -38,7 +37,7 @@ def validator_3():
         execution_mode=ExecutionMode.VALIDATING,
         events=[],
         method="GET",
-        headers=[],
+        headers={},
         url="http://localhost:5000/foo/2",
         assertions={"body": b'{"foo": 2, "bar": false}'},
     )
@@ -51,7 +50,7 @@ def validator_4():
         execution_mode=ExecutionMode.VALIDATING,
         events=[],
         method="GET",
-        headers=[],
+        headers={},
         url="http://localhost:5000/foo/2",
         assertions={"status_code": 201, "body": b'{"foo": 2, "bar": false}'},
     )
@@ -64,14 +63,14 @@ def validator_5():
         execution_mode=ExecutionMode.VALIDATING,
         events=[],
         method="GET",
-        headers=[],
+        headers={},
         url="http://localhost:5000/foo/2",
         assertions={"status_code": 201, "body": b'{"foo": 2'},
     )
     return RequestHttp(spec)
 
 
-def test_validate_makes_get(validator_1, reporter):
+def test_validate_makes_get(validator_1, reporter, vcr):
     with vcr.use_cassette("test_http_executor_validate_makes_get") as cassette:
         result = validator_1.execute(reporter)
 
@@ -96,7 +95,7 @@ def test_executor_fails_zero_assertions(reporter):
     assert result is False
 
 
-def test_validate_makes_get_and_matches_body(validator_3, reporter):
+def test_validate_makes_get_and_matches_body(validator_3, reporter, vcr):
     with vcr.use_cassette(
         "test_http_executor_validate_makes_get_and_matches_body"
     ) as cassette:
@@ -114,7 +113,7 @@ def test_validate_makes_get_and_matches_body(validator_3, reporter):
         assert result
 
 
-def test_validate__matches_body_not_status(validator_4, reporter):
+def test_validate__matches_body_not_status(validator_4, reporter, vcr):
     with vcr.use_cassette(
         "test_http_executor_validate__matches_body_and_status"
     ) as cassette:
@@ -133,7 +132,7 @@ def test_validate__matches_body_not_status(validator_4, reporter):
 
 
 @patch("pyrandall.executors.requests_http.Assertion")
-def test_validate_body_and_status_do_not_match(assertion, validator_5, reporter_1):
+def test_validate_body_and_status_do_not_match(assertion, validator_5, reporter_1, vcr):
     with vcr.use_cassette(
         "test_http_executor_validate_body_and_status_do_not_match"
     ) as cassette:

From a2638cb77d239ebb531541f7faf6c87da33ae722 Mon Sep 17 00:00:00 2001
From: Stefano Oldeman <stefano.oldeman@gmail.com>
Date: Wed, 24 Jun 2020 10:55:02 +0200
Subject: [PATCH 10/16] doc: cli options

---
 CHANGELOG.md     | 21 +++++++++++++++++++--
 VERSION          |  2 +-
 pyrandall/cli.py |  8 +++++---
 3 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8683c21..55669c5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,11 +4,28 @@ All notable changes to pyrandall will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.0] - 2020-06-24
+### Changed
+- *BREAKING CHNAGES*:
+Pyrandall is moving to single command cli. Similar to pytest and rspec.
+### Removed
+- the cli sub-commands `simulate`, `sanitycheck` and `validate`
+in favor of a single command with options flags.
+- the option for `--dataflow` in favor of absolute paths to a scenario file.
+the event/result files mentioned in a specfile are resolved by relative lookup
+still trying to adhere to "convention over configuration".
+In future release support for directory wildcards can be added without breaking the api.
+### Added
+- added option `--everything` (to run e2e) that is the default. Meaning will run both simulate and validate steps from the spec.
+The execution order (sync or async) is open for extension.
+This should be treated as an alpha feature followed by fixes and enhancements.
+
+
 ## [0.2.0] - 2020-05-13
 ### Fixed
 - Implemented `assert_that_received` in kafka validate spec.
-  And `assert_that_empty` in kafka validate spec.
-  See `examples/scenarios/v2_ingest_kafka_small.yaml`
+And `assert_that_empty` in kafka validate spec.
+See `examples/scenarios/v2_ingest_kafka_small.yaml`
 
 
 ## [0.1.0] - 2019-09-20
diff --git a/VERSION b/VERSION
index 0ea3a94..3eefcb9 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.2.0
+1.0.0
diff --git a/pyrandall/cli.py b/pyrandall/cli.py
index c8fb03a..152f68d 100644
--- a/pyrandall/cli.py
+++ b/pyrandall/cli.py
@@ -17,15 +17,17 @@
 @click.command(name="pyrandall")
 @click.argument("specfiles", type=click.File('r'), nargs=-1)
 @click.option("-c", "--config", 'config_file', type=click.File('r'), default="pyrandall_config.json", help="path to json file for pyrandall config.")
-@click.option("-s", "--only-simulate", 'command_flag', flag_value=Flags.SIMULATE)
-@click.option("-V", "--only-validate", 'command_flag', flag_value=Flags.VALIDATE)
-@click.option("-e", "--everything", 'command_flag', flag_value=Flags.E2E, default=True)
+@click.option("-s", "--only-simulate", 'command_flag', flag_value=Flags.SIMULATE, help="filters the spec and runs simulate steps")
+@click.option("-V", "--only-validate", 'command_flag', flag_value=Flags.VALIDATE, help="filters the spec and runs simulate steps")
+@click.option("-e", "--everything", 'command_flag', flag_value=Flags.E2E, default=True, help="(default) run simulate, then validate synchronously")
 @click.option("-d", "--dry-run", 'filter_flag', flag_value=Flags.DESCRIBE)
 @click.help_option()
 @click.version_option(version=const.get_version())
 def main(config_file, command_flag, filter_flag, specfiles):
     """
     pyrandall a test framework oriented around data validation instead of code
+
+    Example: pyrandall scenarios/foobar.yaml
     """
     # quickfix: Click will bypass argument callback when nargs=-1
     # raising these click exceptions will translate to exit(2)

From 0a6f83adbb725762689e89a1b0574cba84574f4e Mon Sep 17 00:00:00 2001
From: Stefano Oldeman <stefano.oldeman@gmail.com>
Date: Sat, 27 Jun 2020 13:39:25 +0200
Subject: [PATCH 11/16] fix: rerecord without default vcr config

---
 tests/fixtures/vcr/test_http_executor_validate_makes_get.yaml | 4 +---
 ...est_http_executor_validate_makes_get_and_matches_body.yaml | 4 +---
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/tests/fixtures/vcr/test_http_executor_validate_makes_get.yaml b/tests/fixtures/vcr/test_http_executor_validate_makes_get.yaml
index 0f17db9..94e0f24 100644
--- a/tests/fixtures/vcr/test_http_executor_validate_makes_get.yaml
+++ b/tests/fixtures/vcr/test_http_executor_validate_makes_get.yaml
@@ -8,8 +8,6 @@ interactions:
       - gzip, deflate
       Connection:
       - keep-alive
-      User-Agent:
-      - pyrandall
     method: GET
     uri: http://localhost:5000/foo/1
   response:
@@ -30,7 +28,7 @@ interactions:
       Content-Type:
       - text/html
       Date:
-      - Sat, 27 Jun 2020 09:31:48 GMT
+      - Sat, 27 Jun 2020 11:38:40 GMT
       Server:
       - Werkzeug/0.16.0 Python/3.7.5
     status:
diff --git a/tests/fixtures/vcr/test_http_executor_validate_makes_get_and_matches_body.yaml b/tests/fixtures/vcr/test_http_executor_validate_makes_get_and_matches_body.yaml
index f7d7a7a..97db9cc 100644
--- a/tests/fixtures/vcr/test_http_executor_validate_makes_get_and_matches_body.yaml
+++ b/tests/fixtures/vcr/test_http_executor_validate_makes_get_and_matches_body.yaml
@@ -8,8 +8,6 @@ interactions:
       - gzip, deflate
       Connection:
       - keep-alive
-      User-Agent:
-      - pyrandall
     method: GET
     uri: http://localhost:5000/foo/2
   response:
@@ -21,7 +19,7 @@ interactions:
       Content-Type:
       - text/html; charset=utf-8
       Date:
-      - Sat, 27 Jun 2020 09:31:48 GMT
+      - Sat, 27 Jun 2020 11:39:54 GMT
       Server:
       - Werkzeug/0.16.0 Python/3.7.5
     status:

From 3adf0b53a62f217ef39662b31f85764caf5bd072 Mon Sep 17 00:00:00 2001
From: Stefano Oldeman <stefano.oldeman@gmail.com>
Date: Sun, 28 Jun 2020 14:40:42 +0200
Subject: [PATCH 12/16] fix: restore functional test

this file was overwritten accidentally
---
 .../vcr/test_validate_assertions_pass.yaml    | 28 +++++++
 .../vcr/test_validate_fail_status_code.yaml   | 37 ++++++++++
 tests/functional/test_http_validate.py        | 74 +++++++++----------
 3 files changed, 99 insertions(+), 40 deletions(-)
 create mode 100644 tests/fixtures/vcr/test_validate_assertions_pass.yaml
 create mode 100644 tests/fixtures/vcr/test_validate_fail_status_code.yaml

diff --git a/tests/fixtures/vcr/test_validate_assertions_pass.yaml b/tests/fixtures/vcr/test_validate_assertions_pass.yaml
new file mode 100644
index 0000000..69ca0f1
--- /dev/null
+++ b/tests/fixtures/vcr/test_validate_assertions_pass.yaml
@@ -0,0 +1,28 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+    method: GET
+    uri: http://localhost:5000/kv/pyrandall/avro_ok
+  response:
+    body:
+      string: '{"Avro": "ok"}'
+    headers:
+      Content-Length:
+      - '14'
+      Content-Type:
+      - text/html; charset=utf-8
+      Date:
+      - Sun, 28 Jun 2020 12:36:04 GMT
+      Server:
+      - Werkzeug/0.16.0 Python/3.7.5
+    status:
+      code: 200
+      message: OK
+version: 1
diff --git a/tests/fixtures/vcr/test_validate_fail_status_code.yaml b/tests/fixtures/vcr/test_validate_fail_status_code.yaml
new file mode 100644
index 0000000..8433a3b
--- /dev/null
+++ b/tests/fixtures/vcr/test_validate_fail_status_code.yaml
@@ -0,0 +1,37 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+    method: GET
+    uri: http://localhost:5000/cant_find_this
+  response:
+    body:
+      string: '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+
+        <title>404 Not Found</title>
+
+        <h1>Not Found</h1>
+
+        <p>The requested URL was not found on the server. If you entered the URL manually
+        please check your spelling and try again.</p>
+
+        '
+    headers:
+      Content-Length:
+      - '232'
+      Content-Type:
+      - text/html
+      Date:
+      - Sun, 28 Jun 2020 12:40:06 GMT
+      Server:
+      - Werkzeug/0.16.0 Python/3.7.5
+    status:
+      code: 404
+      message: NOT FOUND
+version: 1
diff --git a/tests/functional/test_http_validate.py b/tests/functional/test_http_validate.py
index 2fdf1c1..f37dde3 100644
--- a/tests/functional/test_http_validate.py
+++ b/tests/functional/test_http_validate.py
@@ -1,49 +1,43 @@
-import os
 import pytest
-import threading
+from click.testing import CliRunner
 
-from freezegun import freeze_time
-from tests.helper import KafkaProducer
-from pyrandall.kafka import KafkaSetupError
+from pyrandall import cli
 
-TOPIC_1 = "pyrandall-tests-validate-1"
-TOPIC_2 = "pyrandall-tests-validate-2"
-
-ARGV_SMALL = [
+ARGV_HTTP_VALIDATE_1_OK = [
+    "--config",
+    "examples/config/v1.json",
+    "-V",
+    "examples/scenarios/http/validate_ok_status_code.yaml",
+]
+ARGV_HTTP_VALIDATE_STAUTS_CODE_FAIL = [
     "--config",
     "examples/config/v1.json",
     "-V",
-    "examples/scenarios/v2_ingest_kafka_small.yaml"
+    "examples/scenarios/http/validate_bad_status_code.yaml",
 ]
 
 
-def produce_events():
-    producer = KafkaProducer(TOPIC_1)
-    producer.send(b'{"click": "three"}')
-    producer.send(b'{"click": "one"}')
-    producer.send(b'{"click": "two"}')
-
-    producer = KafkaProducer(TOPIC_2)
-    producer.send(b'{"click": "three"}')
-
-
-def test_error_on_connection_timeout(monkeypatch, pyrandall_cli):
-    monkeypatch.setenv("KAFKA_BOOTSTRAP_SERVERS", "localhost:3330")
-    with pytest.raises(KafkaSetupError) as e:
-        pyrandall_cli.invoke(ARGV_SMALL)
-
-@freeze_time("2012-01-14 14:33:12")
-def test_received_no_events(monkeypatch, kafka_cluster_info, pyrandall_cli):
-    result = pyrandall_cli.invoke(ARGV_SMALL)
-    # exit code should be 1 (error)
-    assert 'Usage: main' not in result.output
-    print(result.output)
-    assert result.exit_code == 1
-
-@freeze_time("2012-01-14 14:33:12")
-def test_validate_unordered_passed(kafka_cluster_info, pyrandall_cli):
-    produce_events()
-    result = pyrandall_cli.invoke(ARGV_SMALL)
-    # exit code should be 1 (error)
-    assert 'Usage: main' not in result.output
-    assert result.exit_code == 0
+def test_execute_a_sanitytest_fails():
+    runner = CliRunner()
+    result = runner.invoke(cli.main, [], catch_exceptions=False)
+    assert 'Usage: pyrandall' in result.output
+    assert result.exit_code == 2
+
+
+def test_validate_assertions_pass(vcr):
+    with vcr.use_cassette("test_validate_assertions_pass") as cassette:
+        runner = CliRunner()
+        result = runner.invoke(cli.main, ARGV_HTTP_VALIDATE_1_OK, catch_exceptions=False)
+        assert 'Usage: pyrandall' not in result.output
+        assert result.exit_code == 0
+        assert cassette.all_played
+
+
+def test_validate_fail_status_code(vcr):
+    with vcr.use_cassette("test_validate_fail_status_code") as cassette:
+        runner = CliRunner()
+        result = runner.invoke(cli.main, ARGV_HTTP_VALIDATE_STAUTS_CODE_FAIL, catch_exceptions=False)
+
+        assert 'Usage: pyrandall' not in result.output
+        assert result.exit_code == 1
+        assert cassette.all_played

From 51d31e4ad0eb2818bad2bb1fbcac76c86d4fd4d0 Mon Sep 17 00:00:00 2001
From: Stefano Oldeman <stefano.oldeman@gmail.com>
Date: Sun, 28 Jun 2020 16:14:54 +0200
Subject: [PATCH 13/16] fix: speed up kafka unittest

---
 tests/unit/test_broker_kafka_validate.py | 28 +++++++-----------------
 1 file changed, 8 insertions(+), 20 deletions(-)

diff --git a/tests/unit/test_broker_kafka_validate.py b/tests/unit/test_broker_kafka_validate.py
index 07832db..e657a4a 100644
--- a/tests/unit/test_broker_kafka_validate.py
+++ b/tests/unit/test_broker_kafka_validate.py
@@ -9,11 +9,6 @@
 from pyrandall.types import Assertion, ExecutionMode
 
 
-@pytest.fixture
-def reporter():
-    return Reporter().scenario("pytest example scenario")
-
-
 @pytest.fixture
 def reporter_1():
     return MagicMock(assertion=MagicMock(spec_set=Assertion), unsafe=True)
@@ -32,19 +27,10 @@ def new_executor(assertions):
 MESSAGE_JSON = b'{\n  "uri": "iphone://settings/updates",\n  "session": "111",\n  "timestamp": 2\n}\n'
 
 
-# @mock.patch("pyrandall.executors.broker_kafka.KafkaConn.consume")
-# def test_executor_fails_zero_assertions(kafka_mock, reporter_1):
-#     spec = MagicMock(
-#         unsafe=True, execution_mode=ExecutionMode.VALIDATING, assertions={}
-#     )
-#     executor = BrokerKafka(spec)
-#     result = executor.execute(reporter_1)
-#     reporter_1.assertion_failed.assert_called_with(mock.ANY)
-
-
 # given consumer returns a list with 0 messages
+@mock.patch("pyrandall.executors.broker_kafka.KafkaConn.check_connection", return_value=True)
 @mock.patch("pyrandall.executors.broker_kafka.KafkaConn.consume", return_value=[])
-def test_validate_fail_zero_messages(kafka_mock, reporter_1):
+def test_validate_fail_zero_messages(_consume, _check, reporter_1):
     # when the expected value is 1
     validator = new_executor({"total_events": 1})
     # and it is executed
@@ -55,10 +41,11 @@ def test_validate_fail_zero_messages(kafka_mock, reporter_1):
     )
 
 
+@mock.patch("pyrandall.executors.broker_kafka.KafkaConn.check_connection", return_value=True)
 @mock.patch("pyrandall.executors.broker_kafka.KafkaConn.consume")
-def test_validate_fail_one_messages_body(kafka_mock, reporter_1):
+def test_validate_fail_one_messages_body(consume, _check, reporter_1):
     # given a value that is empty json
-    kafka_mock.return_value = [{"value": b"{}"}]
+    consume.return_value = [{"value": b"{}"}]
     # and a assertion on a full example json
     validator = new_executor(
         {"total_events": 1, "unordered": [{"value": MESSAGE_JSON}]}
@@ -69,10 +56,11 @@ def test_validate_fail_one_messages_body(kafka_mock, reporter_1):
     reporter_1.assertion_failed.assert_called_with(mock.ANY, "unordered events")
 
 
+@mock.patch("pyrandall.executors.broker_kafka.KafkaConn.check_connection", return_value=True)
 @mock.patch("pyrandall.executors.broker_kafka.KafkaConn.consume")
-def test_validate_matches_all(kafka_mock, reporter_1):
+def test_validate_matches_all(consume, _check, reporter_1):
     # given a message with bytes json
-    kafka_mock.return_value = [MESSAGE_JSON]
+    consume.return_value = [MESSAGE_JSON]
     # and validators that asserts 1 message and 1 message value
     validator = new_executor(
         {"total_events": 1, "unordered": [MESSAGE_JSON]}

From 1d3a0604c862ffbbcabe8827e8b872012608a8a1 Mon Sep 17 00:00:00 2001
From: Stefano Oldeman <stefano.oldeman@gmail.com>
Date: Sun, 28 Jun 2020 16:41:20 +0200
Subject: [PATCH 14/16] fix: do not accept stdin

---
 CHANGELOG.md     | 15 +++++++++++----
 pyrandall/cli.py |  9 ++++-----
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 55669c5..78050f8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,14 +9,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - *BREAKING CHNAGES*:
 Pyrandall is moving to single command cli. Similar to pytest and rspec.
 ### Removed
-- the cli sub-commands `simulate`, `sanitycheck` and `validate`
+- Removed commands `simulate`, `sanitycheck` and `validate`
 in favor of a single command with options flags.
-- the option for `--dataflow` in favor of absolute paths to a scenario file.
+- Dropped arg option `--dataflow`. Please rewrite this
+```
+pyrandall --dataflow examples/ simulate http/simulate_200.yaml
+```
+
+into this:
+```
+pyrandall -S examples/scenarios/http/simulate_200.yaml
+```
 the event/result files mentioned in a specfile are resolved by relative lookup
 still trying to adhere to "convention over configuration".
-In future release support for directory wildcards can be added without breaking the api.
 ### Added
-- added option `--everything` (to run e2e) that is the default. Meaning will run both simulate and validate steps from the spec.
+- added option `--everything` (to run e2e) that is the default. Meaning pyrandall executes the steps simulate and validate in sequence.
 The execution order (sync or async) is open for extension.
 This should be treated as an alpha feature followed by fixes and enhancements.
 
diff --git a/pyrandall/cli.py b/pyrandall/cli.py
index 152f68d..b1bdd3e 100644
--- a/pyrandall/cli.py
+++ b/pyrandall/cli.py
@@ -15,7 +15,7 @@
 
 
 @click.command(name="pyrandall")
-@click.argument("specfiles", type=click.File('r'), nargs=-1)
+@click.argument("specfiles", type=click.Path(exists=True), nargs=-1)
 @click.option("-c", "--config", 'config_file', type=click.File('r'), default="pyrandall_config.json", help="path to json file for pyrandall config.")
 @click.option("-s", "--only-simulate", 'command_flag', flag_value=Flags.SIMULATE, help="filters the spec and runs simulate steps")
 @click.option("-V", "--only-validate", 'command_flag', flag_value=Flags.VALIDATE, help="filters the spec and runs simulate steps")
@@ -41,12 +41,10 @@ def main(config_file, command_flag, filter_flag, specfiles):
     if config_file:
         config = json.load(config_file)
 
-
     # translate None to NO OP Flag
     if filter_flag is None:
         filter_flag = Flags.NOOP
     flags = command_flag | filter_flag
-
     try:
         run_command(config, flags, specfiles[0])
     except jsonschema.exceptions.ValidationError:
@@ -62,7 +60,7 @@ def run_command(config, flags, specfile):
 
     config["default_request_url"] = config["requests"].pop("url")
     config['dataflow_path'] = build_basedir(specfile)
-    config['specfile'] = specfile
+    config['specfile'] = click.open_file(specfile, 'r')
     config['flags'] = flags
 
     # register plugins and call their initialize
@@ -75,11 +73,12 @@ def run_command(config, flags, specfile):
 
 
 def build_basedir(specfile):
-    parts = specfile.name.split('/')
+    parts = specfile.split('/')
     out = []
     for x in itertools.takewhile(lambda x: x != const.DIRNAME_SCENARIOS, parts):
         out.append(x)
     return os.path.abspath('/'.join(out))
 
+
 if __name__ == "__main__":
     main()

From 9b40a3dc6d7e14b61f0e23d4fd944943e3395fcf Mon Sep 17 00:00:00 2001
From: Stefano Oldeman <stefano.oldeman@gmail.com>
Date: Sun, 28 Jun 2020 16:42:46 +0200
Subject: [PATCH 15/16] chore: refactor cli unit tests

---
 pyrandall/cli.py                        |  9 ++--
 tests/functional/test_http_simulate.py  |  8 ----
 tests/functional/test_http_validate.py  |  7 ---
 tests/functional/test_kafka_simulate.py | 14 ++----
 tests/unit/test_cli.py                  | 64 +++++++++++++++++++++++++
 5 files changed, 73 insertions(+), 29 deletions(-)
 create mode 100644 tests/unit/test_cli.py

diff --git a/pyrandall/cli.py b/pyrandall/cli.py
index b1bdd3e..865881a 100644
--- a/pyrandall/cli.py
+++ b/pyrandall/cli.py
@@ -44,11 +44,14 @@ def main(config_file, command_flag, filter_flag, specfiles):
     # translate None to NO OP Flag
     if filter_flag is None:
         filter_flag = Flags.NOOP
+
     flags = command_flag | filter_flag
+    specfile = specfiles[0]
     try:
-        run_command(config, flags, specfiles[0])
-    except jsonschema.exceptions.ValidationError:
-        print("Failed validating input yaml")
+        run_command(config, flags, specfile)
+    except jsonschema.exceptions.ValidationError as e:
+        click.echo(f"Error on validating specfile {specfile} with jsonschema, given error:", err=True)
+        click.echo(e, err=True)
         exit(4)
 
 
diff --git a/tests/functional/test_http_simulate.py b/tests/functional/test_http_simulate.py
index dc65dca..ba52eda 100644
--- a/tests/functional/test_http_simulate.py
+++ b/tests/functional/test_http_simulate.py
@@ -12,14 +12,6 @@
 ]
 
 
-def test_execute_a_simulation_fails(pyrandall_cli):
-    result = pyrandall_cli.invoke([
-        "--config",
-        "examples/config/v1.json"
-    ])
-    assert 'Usage: pyrandall' in result.output
-    assert result.exit_code == 2
-
 def test_simulate_json_response_200(pyrandall_cli, vcr):
     with vcr.use_cassette("test_simulate_json_response_200") as cassette:
         result = pyrandall_cli.invoke(ARGV_RESPONSE_200)
diff --git a/tests/functional/test_http_validate.py b/tests/functional/test_http_validate.py
index f37dde3..4442b8a 100644
--- a/tests/functional/test_http_validate.py
+++ b/tests/functional/test_http_validate.py
@@ -17,13 +17,6 @@
 ]
 
 
-def test_execute_a_sanitytest_fails():
-    runner = CliRunner()
-    result = runner.invoke(cli.main, [], catch_exceptions=False)
-    assert 'Usage: pyrandall' in result.output
-    assert result.exit_code == 2
-
-
 def test_validate_assertions_pass(vcr):
     with vcr.use_cassette("test_validate_assertions_pass") as cassette:
         runner = CliRunner()
diff --git a/tests/functional/test_kafka_simulate.py b/tests/functional/test_kafka_simulate.py
index 0bdfd96..ff1b2b3 100644
--- a/tests/functional/test_kafka_simulate.py
+++ b/tests/functional/test_kafka_simulate.py
@@ -6,20 +6,12 @@
 
 
 TEST_TOPIC = "pyrandall-tests-e2e"
-
-MOCK_ARGV = [
+ARGV_SMALL = [
     "--config",
     "examples/config/v1.json",
-    "-s"
+    "-s",
+    "examples/scenarios/v2_ingest_kafka_small.yaml"
 ]
-ARGV_INVALID = MOCK_ARGV + ["examples/scenarios/v2_ingest_kafka_invalid.yaml"]
-ARGV_SMALL = MOCK_ARGV + ["examples/scenarios/v2_ingest_kafka_small.yaml"]
-
-
-def test_fail_on_invaild_schema(pyrandall_cli):
-    result = pyrandall_cli.invoke(ARGV_INVALID)
-    assert 'Failed validating' in result.output
-    assert result.exit_code == 4
 
 
 @freeze_time("2012-01-14 14:33:12")
diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py
new file mode 100644
index 0000000..ef4a8f7
--- /dev/null
+++ b/tests/unit/test_cli.py
@@ -0,0 +1,64 @@
+import re
+
+
+def test_empty_args_fails(pyrandall_cli):
+    result = pyrandall_cli.invoke([])
+    assert 'Usage: pyrandall' in result.output
+    assert result.exit_code == 2
+
+def test_cli_shows_version(pyrandall_cli):
+    result = pyrandall_cli.invoke([
+        "--version"
+    ])
+    assert result.exit_code == 0
+    regex = r"pyrandall, version (\d+\.\d+\.\d+)\n$"
+    matched = re.search(regex, result.output)
+    assert matched
+
+def test_missing_specfiles_arg(pyrandall_cli):
+    result = pyrandall_cli.invoke([
+        "--config",
+        "examples/config/v1.json"
+    ])
+    assert 'Usage: pyrandall' in result.output
+    assert "expecting at least one argument for specfiles" in result.output
+    assert result.exit_code == 2
+
+def test_too_much_specfiles_arg(pyrandall_cli):
+    result = pyrandall_cli.invoke([
+        "--config", "examples/config/v1.json",
+        "examples/scenarios/one_event.yaml", "examples/scenarios/v2.yaml"
+    ])
+    assert 'Usage: pyrandall' in result.output
+    assert 'passing multiple specfiles is not supported yet' in result.output
+    assert result.exit_code == 2
+
+def test_too_much_specfiles_arg(pyrandall_cli):
+    result = pyrandall_cli.invoke([
+        "--config", "examples/config/v1.json",
+        "examples/scenarios/one_event.yaml", "examples/scenarios/v2.yaml"
+    ])
+    assert 'Usage: pyrandall' in result.output
+    assert 'passing multiple specfiles is not supported yet' in result.output
+    assert result.exit_code == 2
+
+def test_fail_on_invaild_specfile_jsonschema(pyrandall_cli):
+    result = pyrandall_cli.invoke([
+        "--config",
+        "examples/config/v1.json",
+        "examples/scenarios/v2_ingest_kafka_invalid.yaml"
+    ])
+    expected_output = """Error on validating specfile examples/scenarios/v2_ingest_kafka_invalid.yaml with jsonschema, given error:
+'messages' is a required property
+
+Failed validating 'required' in schema['properties']['feature']['properties']['scenarios']['items']['properties']['simulate']['allOf'][0]['if']['then']:
+    {'$id': '#/feature/properties/scenarios/items/properties/simulate/allOf/items/0/messages',
+     'properties': {'messages': {'$ref': '#/definitions/simulateMessages'}},
+     'required': ['messages']}
+
+On instance['feature']['scenarios'][0]['simulate']:
+    {'adapter': 'broker/kafka'}
+"""
+
+    assert expected_output == result.output
+    assert result.exit_code == 4

From 30152bb4a2c6165cd7bf98e488934dc8437535b5 Mon Sep 17 00:00:00 2001
From: Stefano Oldeman <stefano.oldeman@gmail.com>
Date: Sun, 28 Jun 2020 16:49:16 +0200
Subject: [PATCH 16/16] fix: human error check on isdir

---
 pyrandall/cli.py       | 5 ++++-
 tests/unit/test_cli.py | 9 +++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/pyrandall/cli.py b/pyrandall/cli.py
index 865881a..61828c1 100644
--- a/pyrandall/cli.py
+++ b/pyrandall/cli.py
@@ -37,6 +37,10 @@ def main(config_file, command_flag, filter_flag, specfiles):
     if len(specfiles) > 1:
         raise click.UsageError("passing multiple specfiles is not supported yet")
 
+    specfile = specfiles[0]
+    if os.path.isdir(specfile):
+        raise click.UsageError("passing directory path is not supported yet")
+
     config = {}
     if config_file:
         config = json.load(config_file)
@@ -46,7 +50,6 @@ def main(config_file, command_flag, filter_flag, specfiles):
         filter_flag = Flags.NOOP
 
     flags = command_flag | filter_flag
-    specfile = specfiles[0]
     try:
         run_command(config, flags, specfile)
     except jsonschema.exceptions.ValidationError as e:
diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py
index ef4a8f7..0c3bb80 100644
--- a/tests/unit/test_cli.py
+++ b/tests/unit/test_cli.py
@@ -42,6 +42,15 @@ def test_too_much_specfiles_arg(pyrandall_cli):
     assert 'passing multiple specfiles is not supported yet' in result.output
     assert result.exit_code == 2
 
+def test_fail_directory_specfile_arg(pyrandall_cli):
+    result = pyrandall_cli.invoke([
+        "--config", "examples/config/v1.json",
+        "examples/scenarios"
+    ])
+    assert 'Usage: pyrandall' in result.output
+    assert 'passing directory path is not supported yet' in result.output
+    assert result.exit_code == 2
+
 def test_fail_on_invaild_specfile_jsonschema(pyrandall_cli):
     result = pyrandall_cli.invoke([
         "--config",