Single command with feature flags

CHANGELOG.md

@@ -4,11 +4,35 @@ All notable changes to pyrandall will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.0] - 2020-06-24
+### Changed
+- *BREAKING CHNAGES*:
+Pyrandall is moving to single command cli. Similar to pytest and rspec.
+### Removed
+- Removed commands `simulate`, `sanitycheck` and `validate`
+in favor of a single command with options flags.
+- Dropped arg option `--dataflow`. Please rewrite this
+```
+pyrandall --dataflow examples/ simulate http/simulate_200.yaml
+```
+
+into this:
+```
+pyrandall -S examples/scenarios/http/simulate_200.yaml
+```
+the event/result files mentioned in a specfile are resolved by relative lookup
+still trying to adhere to "convention over configuration".
+### Added
+- added option `--everything` (to run e2e) that is the default. Meaning pyrandall executes the steps simulate and validate in sequence.
+The execution order (sync or async) is open for extension.
+This should be treated as an alpha feature followed by fixes and enhancements.
+
+
 ## [0.2.0] - 2020-05-13
 ### Fixed
 - Implemented `assert_that_received` in kafka validate spec.
-  And `assert_that_empty` in kafka validate spec.
-  See `examples/scenarios/v2_ingest_kafka_small.yaml`
+And `assert_that_empty` in kafka validate spec.
+See `examples/scenarios/v2_ingest_kafka_small.yaml`
 
 
 ## [0.1.0] - 2019-09-20

MANIFEST.in

@@ -1,7 +1,8 @@
-include pyrandall/schemas/scenario/v2.yaml
 include README.md
 include CHANGELOG
 include VERSION
+include pyrandall/files/VERSION
+include pyrandall/files/schemas/scenario/v2.yaml
 
 exclude Makefile
 exclude dev_requirements.txt

README.md

@@ -14,19 +14,33 @@ pip install flask
 python stubserver.py
 ```
 
+pyrandall follows conventions when looking up files defined in the spec.
+This is an example directory structure:
+
+* `results/` is used for lookups by `assertions` Example:
+  `- { equals_to_event: word_count.json }`
+* `events/` is used for lookups when files are referenced in `simulate`
+
+* `scenarios/` can only contain yaml specfiles of the supported json schema.
+
+```
+# Run both simulate and validate
+./examples/pyrandall scenarios/v2.yaml
+```
+
 ```
 # Simulate events from examples/scenarios/v2.yaml and files found in examples/events
-./examples/pyrandall simulate v2.yaml
+./examples/pyrandall --only-simulate scenarios/v2.yaml
 ```
 
 ```
 # Validate results from examples/scenarios/v2.yaml and files found in examples/results
-./examples/pyrandall validate v2.yaml
+./examples/pyrandall --only-validate scenarios/v2.yaml
 ```
 
 # Example of scenario/v2 schema
 
-The input yaml is validated with jsonschema, the schema can be found here [pyrandall/schemas/scenario/v2.yaml](https://github.com/kpn/pyrandall/tree/master/pyrandall/schemas/scenario/v2.yaml).
+The input yaml is validated with jsonschema, the schema can be found here [pyrandall/files/schemas/scenario/v2.yaml](https://github.com/kpn/pyrandall/tree/master/pyrandall/files/schemas/scenario/v2.yaml).
 
 ```
 ---

VERSION

@@ -1 +1 @@
-0.2.0
+1.0.0

examples/pyrandall

@@ -1,15 +1,13 @@
 #!/usr/bin/env sh
 
+set -euf -o pipefail
+
 SCRIPT=`realpath $0`
 SCRIPTPATH=`dirname $SCRIPT`
 
 KAFKA_BOOTSTRAP_SERVERS=${KAFKA_BOOTSTRAP_SERVERS:-"localhost:9092"}
-SCHEMAS_SERVER_URL=${SCHEMAS_SERVER_URL:-"http://localhost:8080/"}
-
-echo $SCRIPTPATH
 
 docker run --rm -t --network host --volume "${SCRIPTPATH}:/cwd" \
   -e KAFKA_BOOTSTRAP_SERVERS="$KAFKA_BOOTSTRAP_SERVERS" \
-  -e SCHEMAS="$SCHEMAS_SERVER_URL" \
   pyrandall:latest \
-  --config /cwd/config/test_env.json --dataflow /cwd $@
+  --config /cwd/config/test_env.json $@

pyrandall/cli.py

@@ -1,126 +1,89 @@
 import json
 import sys
+import os
+import itertools
 from argparse import ArgumentParser
 
+import click
 import jsonschema
 
+from pyrandall import const
 from pyrandall import commander
 from pyrandall.hookspecs import get_plugin_manager
 from pyrandall.spec import SpecBuilder
 from pyrandall.types import Flags
 
-FLAGS_MAP = {
-    "simulate": Flags.SIMULATE,
-    "validate": Flags.VALIDATE,
-    "sanitytest": Flags.VALIDATE,  # legacy command
-}
-
-
-def run_command(config):
-    specfile = config.pop("specfile")
-    config["default_request_url"] = config["requests"].pop("url")
-
-    # register plugins and call their initialize
-    plugin_manager = get_plugin_manager()
-    plugin_manager.hook.pyrandall_initialize(config=config)
-
-    spec = SpecBuilder(specfile, hook=plugin_manager.hook, **config).feature()
-    flags = FLAGS_MAP[config["command"]]
-    # commander handles execution flow with specified data and config
-    commander.Commander(spec, flags).invoke()
 
+@click.command(name="pyrandall")
+@click.argument("specfiles", type=click.Path(exists=True), nargs=-1)
+@click.option("-c", "--config", 'config_file', type=click.File('r'), default="pyrandall_config.json", help="path to json file for pyrandall config.")
+@click.option("-s", "--only-simulate", 'command_flag', flag_value=Flags.SIMULATE, help="filters the spec and runs simulate steps")
+@click.option("-V", "--only-validate", 'command_flag', flag_value=Flags.VALIDATE, help="filters the spec and runs simulate steps")
+@click.option("-e", "--everything", 'command_flag', flag_value=Flags.E2E, default=True, help="(default) run simulate, then validate synchronously")
+@click.option("-d", "--dry-run", 'filter_flag', flag_value=Flags.DESCRIBE)
+@click.help_option()
+@click.version_option(version=const.get_version())
+def main(config_file, command_flag, filter_flag, specfiles):
+    """
+    pyrandall a test framework oriented around data validation instead of code
+
+    Example: pyrandall scenarios/foobar.yaml
+    """
+    # quickfix: Click will bypass argument callback when nargs=-1
+    # raising these click exceptions will translate to exit(2)
+    if not specfiles:
+        raise click.BadParameter('expecting at least one argument for specfiles')
+
+    if len(specfiles) > 1:
+        raise click.UsageError("passing multiple specfiles is not supported yet")
+
+    specfile = specfiles[0]
+    if os.path.isdir(specfile):
+        raise click.UsageError("passing directory path is not supported yet")
+
+    config = {}
+    if config_file:
+        config = json.load(config_file)
+
+    # translate None to NO OP Flag
+    if filter_flag is None:
+        filter_flag = Flags.NOOP
+
+    flags = command_flag | filter_flag
+    try:
+        run_command(config, flags, specfile)
+    except jsonschema.exceptions.ValidationError as e:
+        click.echo(f"Error on validating specfile {specfile} with jsonschema, given error:", err=True)
+        click.echo(e, err=True)
+        exit(4)
 
-def add_common_args(parser):
-    parser.add_argument("specfile", type=str, help="name of yaml file in scenario/")
-
-
-def setup_args():
-    parser = ArgumentParser(
-        description="pyrandall a test framework oriented around data validation instead of code"
-    )
-    parser.add_argument(
-        "--config",
-        type=str,
-        default="pyrandall_config.json",
-        dest="config_path",
-        help="path to json file for pyrandall config.",
-    )
-    parser.add_argument(
-        "--dataflow",
-        type=str,
-        required=True,
-        dest="dataflow_path",
-        help="path to dataflow root directory",
-    )
-
-    subparsers = parser.add_subparsers(dest="command")
-    # add simulate subcommand
-    sim_parser = subparsers.add_parser("simulate", help="run Simulator for specfile")
-    add_common_args(sim_parser)
-    # add sanitycheck (legacy name)
-    san_parser = subparsers.add_parser(
-        "sanitytest", help="run Validate for specfile (Use validate command)"
-    )
-    add_common_args(san_parser)
-    # add validate subcommand
-    val_parser = subparsers.add_parser("validate", help="run Validate for specfile")
-    add_common_args(val_parser)
-
-    return parser
-
-
-def argparse_error(args_data):
-    msg = """
-#######
-Exit code was 2! Its assumed mocked arguments are wrong, see argparse usage below:
-
-\t%s
-
-actual arguments passed to mock:
-\t%s
-
-######
-""" % (
-        setup_args().format_help().replace("\n", "\n\t"),
-        args_data,
-    )
-    return msg
-
-
-def load_config(fpath):
-    with open(fpath, "r") as f:
-        return json.load(f)
-
-
-def start(argv, config=None):
-    parser = setup_args()
-    args_config = parser.parse_args(argv)
 
+def run_command(config, flags, specfile):
     # TODO: add logging options
     # with open("logging.yaml") as log_conf_file:
     #     log_conf = yaml.safe_load(log_conf_file)
     #     dictConfig(log_conf)
 
-    if args_config.command is None:
-        parser.error("not a valid pyrandall command")
-        exit(1)
-
-    if config is None:
-        config = load_config(args_config.config_path)
+    config["default_request_url"] = config["requests"].pop("url")
+    config['dataflow_path'] = build_basedir(specfile)
+    config['specfile'] = click.open_file(specfile, 'r')
+    config['flags'] = flags
 
-    # overwrite with cli options
-    config.update(args_config.__dict__)
+    # register plugins and call their initialize
+    plugin_manager = get_plugin_manager()
+    plugin_manager.hook.pyrandall_initialize(config=config)
 
-    try:
-        run_command(config)
-    except jsonschema.exceptions.ValidationError:
-        print("Failed validating input yaml")
-        exit(4)
-    exit(0)
+    spec = SpecBuilder(hook=plugin_manager.hook, **config).feature()
+    # commander handles execution flow with specified data and config
+    commander.Commander(spec, flags).invoke()
 
 
-def main():
-    start(sys.argv[1:])
+def build_basedir(specfile):
+    parts = specfile.split('/')
+    out = []
+    for x in itertools.takewhile(lambda x: x != const.DIRNAME_SCENARIOS, parts):
+        out.append(x)
+    return os.path.abspath('/'.join(out))
 
 
 if __name__ == "__main__":

pyrandall/commander.py

@@ -37,15 +37,15 @@ def run_scenarios(self, scenario_items, reporter):
             # 2. call output interface
             reporter.scenario(scenario.description)
 
-            if self.flags & Flags.SIMULATE:
+            if self.flags.has_simulate():
                 reporter.simulate()
                 resultset = reporter.create_and_track_resultset()
                 for spec in scenario.simulate_tasks:
                     e = self.executor_factory(spec)
                     reporter.run_task(e.represent())
                     e.execute(resultset)
 
-            if self.flags & Flags.VALIDATE:
+            if self.flags.has_validate():
                 reporter.validate()
                 resultset = reporter.create_and_track_resultset()
                 for spec in scenario.validate_tasks:

pyrandall/const.py

@@ -0,0 +1,23 @@
+from os import path
+
+DIRNAME_SCENARIOS = "scenarios"
+DIRNAME_EVENTS = "events"
+DIRNAME_RESULTS = "results"
+
+DIR_PYRANDALL_HOME = path.dirname(path.abspath(__file__))
+
+# Constants to resolve private schema files
+_v2_path = path.join("files", "schemas", "scenario", "v2.yaml")
+SCHEMA_V2_PATH = path.join(DIR_PYRANDALL_HOME,  _v2_path)
+VERSION_SCENARIO_V2 = "scenario/v2"
+SCHEMA_VERSIONS = [VERSION_SCENARIO_V2]
+
+# Package version
+VERSION_PATH = path.join("files", "VERSION")
+
+PYRANDALL_USER_AGENT = "pyrandall"
+
+def get_version():
+    version_path = path.join(DIR_PYRANDALL_HOME, VERSION_PATH)
+    # if this fails you have not installed the package (see setup.py)
+    return open(version_path).read().strip()

pyrandall/executors/requests_http.py

@@ -1,6 +1,7 @@
 import requests
 
 from pyrandall.types import Assertion
+from pyrandall import const
 
 from .common import Executor
 
@@ -9,7 +10,7 @@ class RequestHttp(Executor):
     def __init__(self, spec, *args, **kwargs):
         super().__init__()
         self.execution_mode = spec.execution_mode
-        self.spec = spec
+        self.spec = self.add_custom_headers(spec)
 
     def execute(self, reporter):
         spec = self.spec
@@ -42,6 +43,11 @@ def execute(self, reporter):
         # TODO: depricate this, not functioally needed anymore
         return all([a.passed() for a in assertions])
 
+    def add_custom_headers(self, spec):
+        version = const.get_version()
+        spec.headers['User-Agent'] = f"{const.PYRANDALL_USER_AGENT}/{version}"
+        return spec
+
     # TODO: move this to reporter
     def create_jsondiff(self, expected, actual):
         print("Output data different")

pyrandall/files/VERSION

@@ -0,0 +1 @@
+1.0.0