diff --git a/CHANGELOG.md b/CHANGELOG.md index 8683c21..78050f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,35 @@ All notable changes to pyrandall will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.0.0] - 2020-06-24 +### Changed +- *BREAKING CHNAGES*: +Pyrandall is moving to single command cli. Similar to pytest and rspec. +### Removed +- Removed commands `simulate`, `sanitycheck` and `validate` +in favor of a single command with options flags. +- Dropped arg option `--dataflow`. Please rewrite this +``` +pyrandall --dataflow examples/ simulate http/simulate_200.yaml +``` + +into this: +``` +pyrandall -S examples/scenarios/http/simulate_200.yaml +``` +the event/result files mentioned in a specfile are resolved by relative lookup +still trying to adhere to "convention over configuration". +### Added +- added option `--everything` (to run e2e) that is the default. Meaning pyrandall executes the steps simulate and validate in sequence. +The execution order (sync or async) is open for extension. +This should be treated as an alpha feature followed by fixes and enhancements. + + ## [0.2.0] - 2020-05-13 ### Fixed - Implemented `assert_that_received` in kafka validate spec. - And `assert_that_empty` in kafka validate spec. - See `examples/scenarios/v2_ingest_kafka_small.yaml` +And `assert_that_empty` in kafka validate spec. +See `examples/scenarios/v2_ingest_kafka_small.yaml` ## [0.1.0] - 2019-09-20 diff --git a/MANIFEST.in b/MANIFEST.in index 415c9c6..1c7ab0e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,8 @@ -include pyrandall/schemas/scenario/v2.yaml include README.md include CHANGELOG include VERSION +include pyrandall/files/VERSION +include pyrandall/files/schemas/scenario/v2.yaml exclude Makefile exclude dev_requirements.txt diff --git a/README.md b/README.md index 2947bb9..1173e8b 100644 --- a/README.md +++ b/README.md @@ -14,19 +14,33 @@ pip install flask python stubserver.py ``` +pyrandall follows conventions when looking up files defined in the spec. +This is an example directory structure: + +* `results/` is used for lookups by `assertions` Example: + `- { equals_to_event: word_count.json }` +* `events/` is used for lookups when files are referenced in `simulate` + +* `scenarios/` can only contain yaml specfiles of the supported json schema. + +``` +# Run both simulate and validate +./examples/pyrandall scenarios/v2.yaml +``` + ``` # Simulate events from examples/scenarios/v2.yaml and files found in examples/events -./examples/pyrandall simulate v2.yaml +./examples/pyrandall --only-simulate scenarios/v2.yaml ``` ``` # Validate results from examples/scenarios/v2.yaml and files found in examples/results -./examples/pyrandall validate v2.yaml +./examples/pyrandall --only-validate scenarios/v2.yaml ``` # Example of scenario/v2 schema -The input yaml is validated with jsonschema, the schema can be found here [pyrandall/schemas/scenario/v2.yaml](https://github.com/kpn/pyrandall/tree/master/pyrandall/schemas/scenario/v2.yaml). +The input yaml is validated with jsonschema, the schema can be found here [pyrandall/files/schemas/scenario/v2.yaml](https://github.com/kpn/pyrandall/tree/master/pyrandall/files/schemas/scenario/v2.yaml). ``` --- diff --git a/VERSION b/VERSION index 0ea3a94..3eefcb9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.0 +1.0.0 diff --git a/examples/pyrandall b/examples/pyrandall index 25f6230..3812c57 100755 --- a/examples/pyrandall +++ b/examples/pyrandall @@ -1,15 +1,13 @@ #!/usr/bin/env sh +set -euf -o pipefail + SCRIPT=`realpath $0` SCRIPTPATH=`dirname $SCRIPT` KAFKA_BOOTSTRAP_SERVERS=${KAFKA_BOOTSTRAP_SERVERS:-"localhost:9092"} -SCHEMAS_SERVER_URL=${SCHEMAS_SERVER_URL:-"http://localhost:8080/"} - -echo $SCRIPTPATH docker run --rm -t --network host --volume "${SCRIPTPATH}:/cwd" \ -e KAFKA_BOOTSTRAP_SERVERS="$KAFKA_BOOTSTRAP_SERVERS" \ - -e SCHEMAS="$SCHEMAS_SERVER_URL" \ pyrandall:latest \ - --config /cwd/config/test_env.json --dataflow /cwd $@ + --config /cwd/config/test_env.json $@ diff --git a/pyrandall/cli.py b/pyrandall/cli.py index d2cf647..61828c1 100644 --- a/pyrandall/cli.py +++ b/pyrandall/cli.py @@ -1,126 +1,89 @@ import json import sys +import os +import itertools from argparse import ArgumentParser +import click import jsonschema +from pyrandall import const from pyrandall import commander from pyrandall.hookspecs import get_plugin_manager from pyrandall.spec import SpecBuilder from pyrandall.types import Flags -FLAGS_MAP = { - "simulate": Flags.SIMULATE, - "validate": Flags.VALIDATE, - "sanitytest": Flags.VALIDATE, # legacy command -} - - -def run_command(config): - specfile = config.pop("specfile") - config["default_request_url"] = config["requests"].pop("url") - - # register plugins and call their initialize - plugin_manager = get_plugin_manager() - plugin_manager.hook.pyrandall_initialize(config=config) - - spec = SpecBuilder(specfile, hook=plugin_manager.hook, **config).feature() - flags = FLAGS_MAP[config["command"]] - # commander handles execution flow with specified data and config - commander.Commander(spec, flags).invoke() +@click.command(name="pyrandall") +@click.argument("specfiles", type=click.Path(exists=True), nargs=-1) +@click.option("-c", "--config", 'config_file', type=click.File('r'), default="pyrandall_config.json", help="path to json file for pyrandall config.") +@click.option("-s", "--only-simulate", 'command_flag', flag_value=Flags.SIMULATE, help="filters the spec and runs simulate steps") +@click.option("-V", "--only-validate", 'command_flag', flag_value=Flags.VALIDATE, help="filters the spec and runs simulate steps") +@click.option("-e", "--everything", 'command_flag', flag_value=Flags.E2E, default=True, help="(default) run simulate, then validate synchronously") +@click.option("-d", "--dry-run", 'filter_flag', flag_value=Flags.DESCRIBE) +@click.help_option() +@click.version_option(version=const.get_version()) +def main(config_file, command_flag, filter_flag, specfiles): + """ + pyrandall a test framework oriented around data validation instead of code + + Example: pyrandall scenarios/foobar.yaml + """ + # quickfix: Click will bypass argument callback when nargs=-1 + # raising these click exceptions will translate to exit(2) + if not specfiles: + raise click.BadParameter('expecting at least one argument for specfiles') + + if len(specfiles) > 1: + raise click.UsageError("passing multiple specfiles is not supported yet") + + specfile = specfiles[0] + if os.path.isdir(specfile): + raise click.UsageError("passing directory path is not supported yet") + + config = {} + if config_file: + config = json.load(config_file) + + # translate None to NO OP Flag + if filter_flag is None: + filter_flag = Flags.NOOP + + flags = command_flag | filter_flag + try: + run_command(config, flags, specfile) + except jsonschema.exceptions.ValidationError as e: + click.echo(f"Error on validating specfile {specfile} with jsonschema, given error:", err=True) + click.echo(e, err=True) + exit(4) -def add_common_args(parser): - parser.add_argument("specfile", type=str, help="name of yaml file in scenario/") - - -def setup_args(): - parser = ArgumentParser( - description="pyrandall a test framework oriented around data validation instead of code" - ) - parser.add_argument( - "--config", - type=str, - default="pyrandall_config.json", - dest="config_path", - help="path to json file for pyrandall config.", - ) - parser.add_argument( - "--dataflow", - type=str, - required=True, - dest="dataflow_path", - help="path to dataflow root directory", - ) - - subparsers = parser.add_subparsers(dest="command") - # add simulate subcommand - sim_parser = subparsers.add_parser("simulate", help="run Simulator for specfile") - add_common_args(sim_parser) - # add sanitycheck (legacy name) - san_parser = subparsers.add_parser( - "sanitytest", help="run Validate for specfile (Use validate command)" - ) - add_common_args(san_parser) - # add validate subcommand - val_parser = subparsers.add_parser("validate", help="run Validate for specfile") - add_common_args(val_parser) - - return parser - - -def argparse_error(args_data): - msg = """ -####### -Exit code was 2! Its assumed mocked arguments are wrong, see argparse usage below: - -\t%s - -actual arguments passed to mock: -\t%s - -###### -""" % ( - setup_args().format_help().replace("\n", "\n\t"), - args_data, - ) - return msg - - -def load_config(fpath): - with open(fpath, "r") as f: - return json.load(f) - - -def start(argv, config=None): - parser = setup_args() - args_config = parser.parse_args(argv) +def run_command(config, flags, specfile): # TODO: add logging options # with open("logging.yaml") as log_conf_file: # log_conf = yaml.safe_load(log_conf_file) # dictConfig(log_conf) - if args_config.command is None: - parser.error("not a valid pyrandall command") - exit(1) - - if config is None: - config = load_config(args_config.config_path) + config["default_request_url"] = config["requests"].pop("url") + config['dataflow_path'] = build_basedir(specfile) + config['specfile'] = click.open_file(specfile, 'r') + config['flags'] = flags - # overwrite with cli options - config.update(args_config.__dict__) + # register plugins and call their initialize + plugin_manager = get_plugin_manager() + plugin_manager.hook.pyrandall_initialize(config=config) - try: - run_command(config) - except jsonschema.exceptions.ValidationError: - print("Failed validating input yaml") - exit(4) - exit(0) + spec = SpecBuilder(hook=plugin_manager.hook, **config).feature() + # commander handles execution flow with specified data and config + commander.Commander(spec, flags).invoke() -def main(): - start(sys.argv[1:]) +def build_basedir(specfile): + parts = specfile.split('/') + out = [] + for x in itertools.takewhile(lambda x: x != const.DIRNAME_SCENARIOS, parts): + out.append(x) + return os.path.abspath('/'.join(out)) if __name__ == "__main__": diff --git a/pyrandall/commander.py b/pyrandall/commander.py index 0146b5c..becb855 100644 --- a/pyrandall/commander.py +++ b/pyrandall/commander.py @@ -37,7 +37,7 @@ def run_scenarios(self, scenario_items, reporter): # 2. call output interface reporter.scenario(scenario.description) - if self.flags & Flags.SIMULATE: + if self.flags.has_simulate(): reporter.simulate() resultset = reporter.create_and_track_resultset() for spec in scenario.simulate_tasks: @@ -45,7 +45,7 @@ def run_scenarios(self, scenario_items, reporter): reporter.run_task(e.represent()) e.execute(resultset) - if self.flags & Flags.VALIDATE: + if self.flags.has_validate(): reporter.validate() resultset = reporter.create_and_track_resultset() for spec in scenario.validate_tasks: diff --git a/pyrandall/const.py b/pyrandall/const.py new file mode 100644 index 0000000..ff69a89 --- /dev/null +++ b/pyrandall/const.py @@ -0,0 +1,23 @@ +from os import path + +DIRNAME_SCENARIOS = "scenarios" +DIRNAME_EVENTS = "events" +DIRNAME_RESULTS = "results" + +DIR_PYRANDALL_HOME = path.dirname(path.abspath(__file__)) + +# Constants to resolve private schema files +_v2_path = path.join("files", "schemas", "scenario", "v2.yaml") +SCHEMA_V2_PATH = path.join(DIR_PYRANDALL_HOME, _v2_path) +VERSION_SCENARIO_V2 = "scenario/v2" +SCHEMA_VERSIONS = [VERSION_SCENARIO_V2] + +# Package version +VERSION_PATH = path.join("files", "VERSION") + +PYRANDALL_USER_AGENT = "pyrandall" + +def get_version(): + version_path = path.join(DIR_PYRANDALL_HOME, VERSION_PATH) + # if this fails you have not installed the package (see setup.py) + return open(version_path).read().strip() diff --git a/pyrandall/executors/requests_http.py b/pyrandall/executors/requests_http.py index b21d015..07e54a2 100644 --- a/pyrandall/executors/requests_http.py +++ b/pyrandall/executors/requests_http.py @@ -1,6 +1,7 @@ import requests from pyrandall.types import Assertion +from pyrandall import const from .common import Executor @@ -9,7 +10,7 @@ class RequestHttp(Executor): def __init__(self, spec, *args, **kwargs): super().__init__() self.execution_mode = spec.execution_mode - self.spec = spec + self.spec = self.add_custom_headers(spec) def execute(self, reporter): spec = self.spec @@ -42,6 +43,11 @@ def execute(self, reporter): # TODO: depricate this, not functioally needed anymore return all([a.passed() for a in assertions]) + def add_custom_headers(self, spec): + version = const.get_version() + spec.headers['User-Agent'] = f"{const.PYRANDALL_USER_AGENT}/{version}" + return spec + # TODO: move this to reporter def create_jsondiff(self, expected, actual): print("Output data different") diff --git a/pyrandall/files/VERSION b/pyrandall/files/VERSION new file mode 100644 index 0000000..3eefcb9 --- /dev/null +++ b/pyrandall/files/VERSION @@ -0,0 +1 @@ +1.0.0 diff --git a/pyrandall/schemas/__init__.py b/pyrandall/files/schemas/__init__.py similarity index 100% rename from pyrandall/schemas/__init__.py rename to pyrandall/files/schemas/__init__.py diff --git a/pyrandall/schemas/scenario/v1.yaml b/pyrandall/files/schemas/scenario/v1.yaml similarity index 100% rename from pyrandall/schemas/scenario/v1.yaml rename to pyrandall/files/schemas/scenario/v1.yaml diff --git a/pyrandall/schemas/scenario/v2.yaml b/pyrandall/files/schemas/scenario/v2.yaml similarity index 100% rename from pyrandall/schemas/scenario/v2.yaml rename to pyrandall/files/schemas/scenario/v2.yaml diff --git a/pyrandall/schemas/scenario/__init__.py b/pyrandall/schemas/scenario/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pyrandall/spec.py b/pyrandall/spec.py index ec89332..d0fd2fd 100644 --- a/pyrandall/spec.py +++ b/pyrandall/spec.py @@ -5,6 +5,7 @@ import yaml import pyrandall.behaviors +from pyrandall import const from pyrandall.exceptions import InvalidSchenarioVersion from pyrandall.types import ( Adapter, @@ -16,14 +17,6 @@ from .network import join_urlpath -DIR_PATH_DEFAULT = "." - -DIR_PYRANDALL_HOME = os.path.dirname(os.path.abspath(__file__)) -SCHEMAS_SCENARIO_V2 = os.path.join(DIR_PYRANDALL_HOME, "schemas/scenario/v2.yaml") - -VERSION_SCENARIO_V2 = "scenario/v2" -VERSIONS = [VERSION_SCENARIO_V2] - class V2Factory(object): def __init__(self, **kwargs): @@ -37,10 +30,10 @@ def scenario_group(self, nr, data): class SpecBuilder: - def __init__(self, specfile, scenarios_dirname="scenarios", **kwargs): + + def __init__(self, specfile, **kwargs): self.factory = V2Factory(**kwargs) - dataflow_path = kwargs.get("dataflow_path", DIR_PATH_DEFAULT) - self.scenario_file = os.path.join(dataflow_path, scenarios_dirname, specfile) + self.specfile = specfile def feature(self): # creating Feature object will marshall everything below it @@ -48,18 +41,17 @@ def feature(self): def load_spec(self): # TODO: prevent reading sensitive files from filesystem - with open(self.scenario_file, "r") as f: - data = yaml.load(f, Loader=yaml.FullLoader) - # implicitly assume scenario v2 schema - version = data.get("version", VERSION_SCENARIO_V2) - if version not in VERSIONS: - raise InvalidSchenarioVersion(VERSIONS) - # raises errors if unvalid to jsonschema - jsonschema.validate(data, self.scenario_v2_schema()) - return data + data = yaml.load(self.specfile, Loader=yaml.FullLoader) + # implicitly assume scenario v2 schema + version = data.get("version", const.VERSION_SCENARIO_V2) + if version not in const.SCHEMA_VERSIONS: + raise InvalidSchenarioVersion(const.SCHEMA_VERSIONS) + # raises errors if unvalid to jsonschema + jsonschema.validate(data, self.scenario_v2_schema()) + return data def scenario_v2_schema(self): - with open(SCHEMAS_SCENARIO_V2) as f: + with open(const.SCHEMA_V2_PATH) as f: return yaml.load(f.read(), Loader=yaml.FullLoader) @@ -89,7 +81,7 @@ def __init__( self, nr, data, - dataflow_path=DIR_PATH_DEFAULT, + dataflow_path, events_dirname="events", results_dirname="results", default_request_url=None, diff --git a/pyrandall/types.py b/pyrandall/types.py index ecca49d..b8eb08b 100644 --- a/pyrandall/types.py +++ b/pyrandall/types.py @@ -23,6 +23,8 @@ def __str__(self): class Flags(Flag): + NOOP = 0 # No Operation + DESCRIBE = auto() BLOCKING = auto() @@ -31,20 +33,16 @@ class Flags(Flag): SIMULATE = auto() VALIDATE = auto() - BLOCKING_E2E = BLOCKING | SIMULATE | VALIDATE - # REALTIME_E2E = REALTIME | SIMULATE | VALIDATE + E2E = BLOCKING | SIMULATE | VALIDATE # def run_realtime(self): # return self & Flags.REALTIME - def run_blocking(self): - return self & Flags.BLOCKING - def has_validate(self): - return self & Flags.VALIDATE + return Flags.VALIDATE in self def has_simulate(self): - return self & Flags.SIMULATE + return Flags.SIMULATE in self class Assertion: diff --git a/setup.py b/setup.py index 12aee0d..082f6a3 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,10 @@ +import shutil +from os import path from setuptools import find_packages, setup __version__ = open("VERSION").read().strip() +# Also see MANIFEST.ini when changing paths +shutil.copy('VERSION', path.join('pyrandall', 'files', 'VERSION')) setup( @@ -13,15 +17,16 @@ url="https://github.com/kpn/pyrandall", entry_points={"console_scripts": ["pyrandall = pyrandall.cli:main"]}, install_requires=[ - "jsonschema", - "jsondiff", + "jsonschema~=3.2.0", + "jsondiff~=1.2.0", "requests~=2.20", - "PyYAML==5.1", - "Mako", - "curlify", + "PyYAML~=5.1", + "Mako~=1.1.3", + "curlify~=2.2.0", "confluent-kafka~=1.0", - "deepdiff==3.3", - "pluggy", + "deepdiff==4.3", + "pluggy~=0.13.0", + "Click~=7.0" ], tests_require=["pytest", "pytest_httpserver", "responses", "vcrpy", "freezegun"], ) diff --git a/tests/conftest.py b/tests/conftest.py index 8a2ce34..a3c96d0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,20 +1,47 @@ import os +from unittest.mock import MagicMock -import vcr +import pyrandall.cli + +from vcr import VCR import pytest +from click.testing import CliRunner from confluent_kafka.admin import AdminClient, ClusterMetadata -defaults = ["method", "scheme", "host", "port", "path", "query"] -vcr = vcr.VCR( - cassette_library_dir="tests/fixtures/vcr", - # record_mode = [once, new_episodes, none, all] + +@pytest.fixture +def vcr_headers_filter(): + return [("User-Agent", None)] + +@pytest.fixture +def vcr_match_on(): + return ["body", "headers"] + +@pytest.fixture +def vcr(request): + defaults = ["method", "scheme", "host", "port", "path", "query"] + defaults += request.getfixturevalue("vcr_match_on") + + # record_mode = {once, new_episodes, none, all} # https://vcrpy.readthedocs.io/en/latest/usage.html#record-modes - record_mode=os.environ.get("VCR_MODE", "once"), - match_on=(defaults + ["body", "headers"]), - path_transformer=vcr.VCR.ensure_suffix(".yaml"), -) + return VCR( + filter_headers=request.getfixturevalue("vcr_headers_filter"), + record_mode=os.environ.get("VCR_MODE", "once"), + cassette_library_dir="tests/fixtures/vcr", + match_on=defaults, + path_transformer=VCR.ensure_suffix(".yaml"), + ) +# Internal Class Mocks, Stubs etc. + +@pytest.fixture +def reporter(): + return MagicMock(unsafe=True) + + +# Helper functions +# for example: file, http, kafka utilities or Cli Runner @pytest.fixture def kafka_cluster_info() -> ClusterMetadata: @@ -25,3 +52,16 @@ def kafka_cluster_info() -> ClusterMetadata: cluster = admin.list_topics() assert ('{1: BrokerMetadata(1, %s)}' % kafka) == str(cluster.brokers) return cluster + +@pytest.fixture +def pyrandall_cli(): + return PyrandallCli() + + +class PyrandallCli(): + + def invoke(self, command): + runner = CliRunner() + return runner.invoke(pyrandall.cli.main, command, catch_exceptions=False) + + diff --git a/tests/fixtures/vcr/test_commander_run_one_for_one.yaml b/tests/fixtures/vcr/test_commander_run_one_for_one.yaml index 5608330..6d21422 100644 --- a/tests/fixtures/vcr/test_commander_run_one_for_one.yaml +++ b/tests/fixtures/vcr/test_commander_run_one_for_one.yaml @@ -10,8 +10,6 @@ interactions: - keep-alive Content-Length: - '14' - User-Agent: - - python-requests/2.23.0 content-type: - application/json method: POST @@ -34,9 +32,9 @@ interactions: Content-Type: - text/html Date: - - Mon, 16 Sep 2019 09:54:45 GMT + - Sat, 27 Jun 2020 09:31:48 GMT Server: - - Werkzeug/0.15.6 Python/3.7.4 + - Werkzeug/0.16.0 Python/3.7.5 status: code: 404 message: NOT FOUND @@ -49,8 +47,6 @@ interactions: - gzip, deflate Connection: - keep-alive - User-Agent: - - python-requests/2.23.0 method: GET uri: http://localhost:5000/foo/bar/123 response: @@ -62,9 +58,9 @@ interactions: Content-Type: - text/html; charset=utf-8 Date: - - Mon, 16 Sep 2019 09:54:45 GMT + - Sat, 27 Jun 2020 09:31:48 GMT Server: - - Werkzeug/0.15.6 Python/3.7.4 + - Werkzeug/0.16.0 Python/3.7.5 status: code: 200 message: OK diff --git a/tests/fixtures/vcr/test_http_executor_simulate_post_201.yaml b/tests/fixtures/vcr/test_http_executor_simulate_post_201.yaml index 5c6e72b..3ade710 100644 --- a/tests/fixtures/vcr/test_http_executor_simulate_post_201.yaml +++ b/tests/fixtures/vcr/test_http_executor_simulate_post_201.yaml @@ -2,6 +2,12 @@ interactions: - request: body: '{"foo": "bar"}' headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive Content-Length: - '14' method: POST @@ -15,9 +21,9 @@ interactions: Content-Type: - text/html; charset=utf-8 Date: - - Mon, 16 Sep 2019 09:54:45 GMT + - Sat, 27 Jun 2020 09:31:48 GMT Server: - - Werkzeug/0.15.6 Python/3.7.4 + - Werkzeug/0.16.0 Python/3.7.5 status: code: 201 message: CREATED diff --git a/tests/fixtures/vcr/test_http_executor_simulate_post_201_and_400.yaml b/tests/fixtures/vcr/test_http_executor_simulate_post_201_and_400.yaml index 48ea1bf..79a767e 100644 --- a/tests/fixtures/vcr/test_http_executor_simulate_post_201_and_400.yaml +++ b/tests/fixtures/vcr/test_http_executor_simulate_post_201_and_400.yaml @@ -2,6 +2,12 @@ interactions: - request: body: null headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive Content-Length: - '0' method: POST @@ -23,15 +29,21 @@ interactions: Content-Type: - text/html Date: - - Mon, 16 Sep 2019 09:54:45 GMT + - Sat, 27 Jun 2020 09:31:48 GMT Server: - - Werkzeug/0.15.6 Python/3.7.4 + - Werkzeug/0.16.0 Python/3.7.5 status: code: 400 message: BAD REQUEST - request: body: '{"foo": "bar"}' headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive Content-Length: - '14' method: POST @@ -45,9 +57,9 @@ interactions: Content-Type: - text/html; charset=utf-8 Date: - - Mon, 16 Sep 2019 09:54:45 GMT + - Sat, 27 Jun 2020 09:31:48 GMT Server: - - Werkzeug/0.15.6 Python/3.7.4 + - Werkzeug/0.16.0 Python/3.7.5 status: code: 201 message: CREATED diff --git a/tests/fixtures/vcr/test_http_executor_simulate_post_400_response.yaml b/tests/fixtures/vcr/test_http_executor_simulate_post_400_response.yaml index a3997ee..ab1705a 100644 --- a/tests/fixtures/vcr/test_http_executor_simulate_post_400_response.yaml +++ b/tests/fixtures/vcr/test_http_executor_simulate_post_400_response.yaml @@ -2,6 +2,12 @@ interactions: - request: body: null headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive Content-Length: - '0' method: POST @@ -23,9 +29,9 @@ interactions: Content-Type: - text/html Date: - - Mon, 16 Sep 2019 09:54:45 GMT + - Sat, 27 Jun 2020 09:31:48 GMT Server: - - Werkzeug/0.15.6 Python/3.7.4 + - Werkzeug/0.16.0 Python/3.7.5 status: code: 400 message: BAD REQUEST diff --git a/tests/fixtures/vcr/test_http_executor_validate__matches_body_and_status.yaml b/tests/fixtures/vcr/test_http_executor_validate__matches_body_and_status.yaml index 6cda7b6..e2f4435 100644 --- a/tests/fixtures/vcr/test_http_executor_validate__matches_body_and_status.yaml +++ b/tests/fixtures/vcr/test_http_executor_validate__matches_body_and_status.yaml @@ -1,7 +1,13 @@ interactions: - request: body: null - headers: {} + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive method: GET uri: http://localhost:5000/foo/2 response: @@ -13,9 +19,9 @@ interactions: Content-Type: - text/html; charset=utf-8 Date: - - Mon, 16 Sep 2019 09:54:45 GMT + - Sat, 27 Jun 2020 09:31:48 GMT Server: - - Werkzeug/0.15.6 Python/3.7.4 + - Werkzeug/0.16.0 Python/3.7.5 status: code: 200 message: OK diff --git a/tests/fixtures/vcr/test_http_executor_validate_body_and_status_do_not_match.yaml b/tests/fixtures/vcr/test_http_executor_validate_body_and_status_do_not_match.yaml index 6cda7b6..b4e3f0c 100644 --- a/tests/fixtures/vcr/test_http_executor_validate_body_and_status_do_not_match.yaml +++ b/tests/fixtures/vcr/test_http_executor_validate_body_and_status_do_not_match.yaml @@ -1,7 +1,13 @@ interactions: - request: body: null - headers: {} + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive method: GET uri: http://localhost:5000/foo/2 response: @@ -13,9 +19,9 @@ interactions: Content-Type: - text/html; charset=utf-8 Date: - - Mon, 16 Sep 2019 09:54:45 GMT + - Sat, 27 Jun 2020 09:46:28 GMT Server: - - Werkzeug/0.15.6 Python/3.7.4 + - Werkzeug/0.16.0 Python/3.7.5 status: code: 200 message: OK diff --git a/tests/fixtures/vcr/test_http_executor_validate_makes_get.yaml b/tests/fixtures/vcr/test_http_executor_validate_makes_get.yaml index 933e42d..94e0f24 100644 --- a/tests/fixtures/vcr/test_http_executor_validate_makes_get.yaml +++ b/tests/fixtures/vcr/test_http_executor_validate_makes_get.yaml @@ -1,7 +1,13 @@ interactions: - request: body: null - headers: {} + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive method: GET uri: http://localhost:5000/foo/1 response: @@ -22,9 +28,9 @@ interactions: Content-Type: - text/html Date: - - Mon, 16 Sep 2019 09:54:45 GMT + - Sat, 27 Jun 2020 11:38:40 GMT Server: - - Werkzeug/0.15.6 Python/3.7.4 + - Werkzeug/0.16.0 Python/3.7.5 status: code: 404 message: NOT FOUND diff --git a/tests/fixtures/vcr/test_http_executor_validate_makes_get_and_matches_body.yaml b/tests/fixtures/vcr/test_http_executor_validate_makes_get_and_matches_body.yaml index 6cda7b6..97db9cc 100644 --- a/tests/fixtures/vcr/test_http_executor_validate_makes_get_and_matches_body.yaml +++ b/tests/fixtures/vcr/test_http_executor_validate_makes_get_and_matches_body.yaml @@ -1,7 +1,13 @@ interactions: - request: body: null - headers: {} + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive method: GET uri: http://localhost:5000/foo/2 response: @@ -13,9 +19,9 @@ interactions: Content-Type: - text/html; charset=utf-8 Date: - - Mon, 16 Sep 2019 09:54:45 GMT + - Sat, 27 Jun 2020 11:39:54 GMT Server: - - Werkzeug/0.15.6 Python/3.7.4 + - Werkzeug/0.16.0 Python/3.7.5 status: code: 200 message: OK diff --git a/tests/fixtures/vcr/test_http_user_agent.yaml b/tests/fixtures/vcr/test_http_user_agent.yaml new file mode 100644 index 0000000..7b63b93 --- /dev/null +++ b/tests/fixtures/vcr/test_http_user_agent.yaml @@ -0,0 +1,32 @@ +interactions: +- request: + body: '{"foo": "bar"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '14' + User-Agent: + - pyrandall/1.0.0 + method: POST + uri: http://localhost:5000/users + response: + body: + string: '' + headers: + Content-Length: + - '0' + Content-Type: + - text/html; charset=utf-8 + Date: + - Sat, 27 Jun 2020 10:26:54 GMT + Server: + - Werkzeug/0.16.0 Python/3.7.5 + status: + code: 201 + message: CREATED +version: 1 diff --git a/tests/fixtures/vcr/test_simulate_json_response_200.yaml b/tests/fixtures/vcr/test_simulate_json_response_200.yaml index 6ea0780..6aff903 100644 --- a/tests/fixtures/vcr/test_simulate_json_response_200.yaml +++ b/tests/fixtures/vcr/test_simulate_json_response_200.yaml @@ -10,8 +10,6 @@ interactions: - keep-alive Content-Length: - '13' - User-Agent: - - python-requests/2.23.0 content-type: - application/json method: POST @@ -23,9 +21,9 @@ interactions: Content-Type: - text/html; charset=utf-8 Date: - - Mon, 16 Sep 2019 09:54:43 GMT + - Sat, 27 Jun 2020 09:55:09 GMT Server: - - Werkzeug/0.15.6 Python/3.7.4 + - Werkzeug/0.16.0 Python/3.7.5 status: code: 204 message: NO CONTENT diff --git a/tests/fixtures/vcr/test_simulate_json_response_400.yaml b/tests/fixtures/vcr/test_simulate_json_response_400.yaml index 2b2886d..9b5eda9 100644 --- a/tests/fixtures/vcr/test_simulate_json_response_400.yaml +++ b/tests/fixtures/vcr/test_simulate_json_response_400.yaml @@ -10,8 +10,6 @@ interactions: - keep-alive Content-Length: - '13' - User-Agent: - - python-requests/2.23.0 content-type: - application/json method: POST @@ -34,9 +32,9 @@ interactions: Content-Type: - text/html Date: - - Mon, 16 Sep 2019 09:54:43 GMT + - Sat, 27 Jun 2020 09:55:09 GMT Server: - - Werkzeug/0.15.6 Python/3.7.4 + - Werkzeug/0.16.0 Python/3.7.5 status: code: 404 message: NOT FOUND diff --git a/tests/fixtures/vcr/test_validate_assertions_pass.yaml b/tests/fixtures/vcr/test_validate_assertions_pass.yaml index 08126dc..69ca0f1 100644 --- a/tests/fixtures/vcr/test_validate_assertions_pass.yaml +++ b/tests/fixtures/vcr/test_validate_assertions_pass.yaml @@ -8,8 +8,6 @@ interactions: - gzip, deflate Connection: - keep-alive - User-Agent: - - python-requests/2.23.0 method: GET uri: http://localhost:5000/kv/pyrandall/avro_ok response: @@ -21,9 +19,9 @@ interactions: Content-Type: - text/html; charset=utf-8 Date: - - Mon, 16 Sep 2019 09:54:44 GMT + - Sun, 28 Jun 2020 12:36:04 GMT Server: - - Werkzeug/0.15.6 Python/3.7.4 + - Werkzeug/0.16.0 Python/3.7.5 status: code: 200 message: OK diff --git a/tests/fixtures/vcr/test_validate_fail_status_code.yaml b/tests/fixtures/vcr/test_validate_fail_status_code.yaml index c9d71bc..8433a3b 100644 --- a/tests/fixtures/vcr/test_validate_fail_status_code.yaml +++ b/tests/fixtures/vcr/test_validate_fail_status_code.yaml @@ -8,8 +8,6 @@ interactions: - gzip, deflate Connection: - keep-alive - User-Agent: - - python-requests/2.23.0 method: GET uri: http://localhost:5000/cant_find_this response: @@ -30,9 +28,9 @@ interactions: Content-Type: - text/html Date: - - Mon, 16 Sep 2019 09:54:44 GMT + - Sun, 28 Jun 2020 12:40:06 GMT Server: - - Werkzeug/0.15.6 Python/3.7.4 + - Werkzeug/0.16.0 Python/3.7.5 status: code: 404 message: NOT FOUND diff --git a/tests/functional/test_http_simulate.py b/tests/functional/test_http_simulate.py index c592b82..ba52eda 100644 --- a/tests/functional/test_http_simulate.py +++ b/tests/functional/test_http_simulate.py @@ -1,66 +1,38 @@ -import unittest -from unittest.mock import patch - -from pyrandall import cli -from tests.conftest import vcr - ARGV_RESPONSE_200 = [ "--config", "examples/config/v1.json", - "--dataflow", - "examples/", - "simulate", - "http/simulate_200.yaml", + "-s", + "examples/scenarios/http/simulate_200.yaml", ] ARGV_RESPONSE_400 = [ "--config", "examples/config/v1.json", - "--dataflow", - "examples/", - "simulate", - "http/simulate_400.yaml", + "-s", + "examples/scenarios/http/simulate_400.yaml", ] -class SimulatorTest(unittest.TestCase): - @patch("argparse.ArgumentParser._print_message") - def test_execute_a_simulation_fails(self, print_message): - with self.assertRaises(SystemExit) as context: - cli.start([]) - self.assertEqual(context.exception.code, 2) - print_message.assert_called_once() - - def test_simulate_json_response_200(self): - with vcr.use_cassette("test_simulate_json_response_200") as cassette: - with self.assertRaises(SystemExit) as context: - cli.start(ARGV_RESPONSE_200) - - assert len(cassette) == 1 - r1 = cassette.requests[0] - assert r1.path == "/v1/actions/produce-event" - assert cassette.responses_of(r1)[0]["status"]["code"] == 204 - assert cassette.all_played - if context.exception.code == 2: - self.fail(cli.argparse_error(ARGV_RESPONSE_200)) - - # not all request had the expected status code (see assertions) - assert context.exception.code == 0 - - def test_simulate_json_response_400(self): - with vcr.use_cassette("test_simulate_json_response_400") as cassette: - with self.assertRaises(SystemExit) as context: - cli.start(ARGV_RESPONSE_400) - - assert len(cassette) == 1 - r0 = cassette.requests[0] - assert r0.path == "/cant_find_this" - assert cassette.responses_of(r0)[0]["status"]["code"] == 404 - assert cassette.all_played - if context.exception.code == 2: - self.fail(cli.argparse_error(ARGV_RESPONSE_400)) - - assert context.exception.code == 1 - - -if __name__ == "__main__": - unittest.main() +def test_simulate_json_response_200(pyrandall_cli, vcr): + with vcr.use_cassette("test_simulate_json_response_200") as cassette: + result = pyrandall_cli.invoke(ARGV_RESPONSE_200) + assert 'Usage: pyrandall' not in result.output + assert result.exit_code == 0 + + assert len(cassette) == 1 + r1 = cassette.requests[0] + assert r1.path == "/v1/actions/produce-event" + assert cassette.responses_of(r1)[0]["status"]["code"] == 204 + assert cassette.all_played + # not all request had the expected status code (see assertions) + +def test_simulate_json_response_400(pyrandall_cli, vcr): + with vcr.use_cassette("test_simulate_json_response_400") as cassette: + result = pyrandall_cli.invoke(ARGV_RESPONSE_400) + assert 'Usage: pyrandall' not in result.output + assert result.exit_code == 1 + + assert len(cassette) == 1 + r0 = cassette.requests[0] + assert r0.path == "/cant_find_this" + assert cassette.responses_of(r0)[0]["status"]["code"] == 404 + assert cassette.all_played diff --git a/tests/functional/test_http_validate.py b/tests/functional/test_http_validate.py index 30a9a81..4442b8a 100644 --- a/tests/functional/test_http_validate.py +++ b/tests/functional/test_http_validate.py @@ -1,52 +1,36 @@ -from unittest.mock import patch - import pytest +from click.testing import CliRunner from pyrandall import cli -from tests.conftest import vcr -CONFIG = "examples/config/v1.json" ARGV_HTTP_VALIDATE_1_OK = [ "--config", - CONFIG, - "--dataflow", - "examples/", - "sanitytest", - "http/validate_ok_status_code.yaml", + "examples/config/v1.json", + "-V", + "examples/scenarios/http/validate_ok_status_code.yaml", ] ARGV_HTTP_VALIDATE_STAUTS_CODE_FAIL = [ "--config", - CONFIG, - "--dataflow", - "examples/", - "sanitytest", - "http/validate_bad_status_code.yaml", + "examples/config/v1.json", + "-V", + "examples/scenarios/http/validate_bad_status_code.yaml", ] -@patch("pyrandall.cli.ArgumentParser._print_message") -def test_execute_a_sanitytest_fails(print_message): - with pytest.raises(SystemExit) as context: - cli.start([]) - assert context.value.code == 2 - print_message.assert_called() - - -@vcr.use_cassette("test_validate_assertions_pass") -def test_validate_assertions_pass(): - with pytest.raises(SystemExit) as context: - cli.start(ARGV_HTTP_VALIDATE_1_OK) - if context.value.code == 2: - pytest.fail(cli.argparse_error(ARGV_HTTP_VALIDATE_1_OK)) - - assert context.value.code == 0 +def test_validate_assertions_pass(vcr): + with vcr.use_cassette("test_validate_assertions_pass") as cassette: + runner = CliRunner() + result = runner.invoke(cli.main, ARGV_HTTP_VALIDATE_1_OK, catch_exceptions=False) + assert 'Usage: pyrandall' not in result.output + assert result.exit_code == 0 + assert cassette.all_played -@vcr.use_cassette("test_validate_fail_status_code") -def test_validate_fail_status_code(): - with pytest.raises(SystemExit) as context: - cli.start(ARGV_HTTP_VALIDATE_STAUTS_CODE_FAIL) - if context.value.code == 2: - pytest.fail(cli.argparse_error(ARGV_HTTP_VALIDATE_STAUTS_CODE_FAIL)) +def test_validate_fail_status_code(vcr): + with vcr.use_cassette("test_validate_fail_status_code") as cassette: + runner = CliRunner() + result = runner.invoke(cli.main, ARGV_HTTP_VALIDATE_STAUTS_CODE_FAIL, catch_exceptions=False) - assert context.value.code == 1 + assert 'Usage: pyrandall' not in result.output + assert result.exit_code == 1 + assert cassette.all_played diff --git a/tests/functional/test_kafka_simulate.py b/tests/functional/test_kafka_simulate.py index 7384ecc..ff1b2b3 100644 --- a/tests/functional/test_kafka_simulate.py +++ b/tests/functional/test_kafka_simulate.py @@ -2,44 +2,29 @@ import pytest from freezegun import freeze_time -from pyrandall import cli -from tests.conftest import vcr from tests.helper import KafkaConsumer TEST_TOPIC = "pyrandall-tests-e2e" +ARGV_SMALL = [ + "--config", + "examples/config/v1.json", + "-s", + "examples/scenarios/v2_ingest_kafka_small.yaml" +] -config = "examples/config/v1.json" -MOCK_ARGV = ["--config", config, "--dataflow", "examples/", "simulate"] - - -def test_invalid_kafka_scenario(): - argv = MOCK_ARGV + ["v2_ingest_kafka_invalid.yaml"] - with pytest.raises(SystemExit) as context: - cli.start(argv) - if context.value.code == 2: - pytest.fail(cli.argparse_error(argv)) - assert context.value.code == 4 - - -# freeze time in order to hardcode timestamps @freeze_time("2012-01-14 14:33:12") -@vcr.use_cassette("test_ingest_to_kafka") -def test_simulate_produces_event(kafka_cluster_info): +def test_simulate_produces_event(kafka_cluster_info, pyrandall_cli): consumer = KafkaConsumer(TEST_TOPIC) try: highwater = consumer.get_high_watermark() # run simulate to create a message to kafka # running following command: - argv = MOCK_ARGV + ["v2_ingest_kafka_small.yaml"] - print(f"running {argv}") - with pytest.raises(SystemExit) as context: - cli.start(argv) - if context.value.code == 2: - pytest.fail(cli.argparse_error(argv)) - assert context.value.code == 0 + result = pyrandall_cli.invoke(ARGV_SMALL) + assert 'Usage: main' not in result.output + assert result.exit_code == 0 messages = consumer.get_messages(expecting=2) assert len(messages) == 2 diff --git a/tests/functional/test_kafka_validate.py b/tests/functional/test_kafka_validate.py index 69695ff..dfe6165 100644 --- a/tests/functional/test_kafka_validate.py +++ b/tests/functional/test_kafka_validate.py @@ -3,41 +3,19 @@ from freezegun import freeze_time import threading -from pyrandall import cli -from tests.conftest import vcr from tests.helper import KafkaProducer from pyrandall.kafka import KafkaSetupError TOPIC_1 = "pyrandall-tests-validate-1" TOPIC_2 = "pyrandall-tests-validate-2" +ARGV_SMALL = [ + "--config", + "examples/config/v1.json", + "-V", + "examples/scenarios/v2_ingest_kafka_small.yaml" +] -config = "examples/config/v1.json" -MOCK_ARGV = ["--config", config, "--dataflow", "examples/", "validate"] - - -def test_error_on_connection_timeout(monkeypatch): - monkeypatch.setenv("KAFKA_BOOTSTRAP_SERVERS", "localhost:3330") - argv = MOCK_ARGV + ["v2_ingest_kafka_small.yaml"] - with pytest.raises(KafkaSetupError) as e: - cli.start(argv) - - -# freeze time in order to hardcode timestamps -@freeze_time("2012-01-14 14:33:12") -@vcr.use_cassette("test_ingest_to_kafka") -def test_received_no_events(monkeypatch, kafka_cluster_info): - # run validate to consume a message from kafka - # running following command: - argv = MOCK_ARGV + ["v2_ingest_kafka_small.yaml"] - print(f"running {argv}") - with pytest.raises(SystemExit) as context: - cli.start(argv) - if context.value.code == 2: - pytest.fail(cli.argparse_error(argv)) - - # exit code should be 1 (error) - assert context.value.code == 1 def produce_events(): # produce the events @@ -49,21 +27,29 @@ def produce_events(): producer = KafkaProducer(TOPIC_2) producer.send(b'{"click": "three"}') -@freeze_time("2012-01-14 14:33:12") -@vcr.use_cassette("test_ingest_to_kafka") -def test_validate_unordered_passed(kafka_cluster_info): - # run validate to consume a message from kafka - # running following command: - argv = MOCK_ARGV + ["v2_ingest_kafka_small.yaml"] - print(f"running {argv}") - with pytest.raises(SystemExit) as context: - produce_events() - cli.start(argv) - if context.value.code == 2: - pytest.fail(cli.argparse_error(argv)) +def test_error_on_connection_timeout(monkeypatch, pyrandall_cli): + monkeypatch.setenv("KAFKA_BOOTSTRAP_SERVERS", "localhost:3330") + with pytest.raises(KafkaSetupError) as e: + pyrandall_cli.invoke(ARGV_SMALL) + +# freeze time in order to hardcode timestamps +@freeze_time("2012-01-14 14:33:12") +def test_received_no_events(monkeypatch, kafka_cluster_info, pyrandall_cli): + """ + run validate to consume a message from kafka + """ + result = pyrandall_cli.invoke(ARGV_SMALL) # exit code should be 1 (error) - assert context.value.code == 0 + assert 'Usage: main' not in result.output + print(result.output) + assert result.exit_code == 1 +@freeze_time("2012-01-14 14:33:12") +def test_validate_unordered_passed(kafka_cluster_info, pyrandall_cli): + produce_events() + result = pyrandall_cli.invoke(ARGV_SMALL) + assert 'Usage: main' not in result.output + assert result.exit_code == 0 diff --git a/tests/unit/test_broker_kafka_validate.py b/tests/unit/test_broker_kafka_validate.py index 07832db..e657a4a 100644 --- a/tests/unit/test_broker_kafka_validate.py +++ b/tests/unit/test_broker_kafka_validate.py @@ -9,11 +9,6 @@ from pyrandall.types import Assertion, ExecutionMode -@pytest.fixture -def reporter(): - return Reporter().scenario("pytest example scenario") - - @pytest.fixture def reporter_1(): return MagicMock(assertion=MagicMock(spec_set=Assertion), unsafe=True) @@ -32,19 +27,10 @@ def new_executor(assertions): MESSAGE_JSON = b'{\n "uri": "iphone://settings/updates",\n "session": "111",\n "timestamp": 2\n}\n' -# @mock.patch("pyrandall.executors.broker_kafka.KafkaConn.consume") -# def test_executor_fails_zero_assertions(kafka_mock, reporter_1): -# spec = MagicMock( -# unsafe=True, execution_mode=ExecutionMode.VALIDATING, assertions={} -# ) -# executor = BrokerKafka(spec) -# result = executor.execute(reporter_1) -# reporter_1.assertion_failed.assert_called_with(mock.ANY) - - # given consumer returns a list with 0 messages +@mock.patch("pyrandall.executors.broker_kafka.KafkaConn.check_connection", return_value=True) @mock.patch("pyrandall.executors.broker_kafka.KafkaConn.consume", return_value=[]) -def test_validate_fail_zero_messages(kafka_mock, reporter_1): +def test_validate_fail_zero_messages(_consume, _check, reporter_1): # when the expected value is 1 validator = new_executor({"total_events": 1}) # and it is executed @@ -55,10 +41,11 @@ def test_validate_fail_zero_messages(kafka_mock, reporter_1): ) +@mock.patch("pyrandall.executors.broker_kafka.KafkaConn.check_connection", return_value=True) @mock.patch("pyrandall.executors.broker_kafka.KafkaConn.consume") -def test_validate_fail_one_messages_body(kafka_mock, reporter_1): +def test_validate_fail_one_messages_body(consume, _check, reporter_1): # given a value that is empty json - kafka_mock.return_value = [{"value": b"{}"}] + consume.return_value = [{"value": b"{}"}] # and a assertion on a full example json validator = new_executor( {"total_events": 1, "unordered": [{"value": MESSAGE_JSON}]} @@ -69,10 +56,11 @@ def test_validate_fail_one_messages_body(kafka_mock, reporter_1): reporter_1.assertion_failed.assert_called_with(mock.ANY, "unordered events") +@mock.patch("pyrandall.executors.broker_kafka.KafkaConn.check_connection", return_value=True) @mock.patch("pyrandall.executors.broker_kafka.KafkaConn.consume") -def test_validate_matches_all(kafka_mock, reporter_1): +def test_validate_matches_all(consume, _check, reporter_1): # given a message with bytes json - kafka_mock.return_value = [MESSAGE_JSON] + consume.return_value = [MESSAGE_JSON] # and validators that asserts 1 message and 1 message value validator = new_executor( {"total_events": 1, "unordered": [MESSAGE_JSON]} diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py new file mode 100644 index 0000000..0c3bb80 --- /dev/null +++ b/tests/unit/test_cli.py @@ -0,0 +1,73 @@ +import re + + +def test_empty_args_fails(pyrandall_cli): + result = pyrandall_cli.invoke([]) + assert 'Usage: pyrandall' in result.output + assert result.exit_code == 2 + +def test_cli_shows_version(pyrandall_cli): + result = pyrandall_cli.invoke([ + "--version" + ]) + assert result.exit_code == 0 + regex = r"pyrandall, version (\d+\.\d+\.\d+)\n$" + matched = re.search(regex, result.output) + assert matched + +def test_missing_specfiles_arg(pyrandall_cli): + result = pyrandall_cli.invoke([ + "--config", + "examples/config/v1.json" + ]) + assert 'Usage: pyrandall' in result.output + assert "expecting at least one argument for specfiles" in result.output + assert result.exit_code == 2 + +def test_too_much_specfiles_arg(pyrandall_cli): + result = pyrandall_cli.invoke([ + "--config", "examples/config/v1.json", + "examples/scenarios/one_event.yaml", "examples/scenarios/v2.yaml" + ]) + assert 'Usage: pyrandall' in result.output + assert 'passing multiple specfiles is not supported yet' in result.output + assert result.exit_code == 2 + +def test_too_much_specfiles_arg(pyrandall_cli): + result = pyrandall_cli.invoke([ + "--config", "examples/config/v1.json", + "examples/scenarios/one_event.yaml", "examples/scenarios/v2.yaml" + ]) + assert 'Usage: pyrandall' in result.output + assert 'passing multiple specfiles is not supported yet' in result.output + assert result.exit_code == 2 + +def test_fail_directory_specfile_arg(pyrandall_cli): + result = pyrandall_cli.invoke([ + "--config", "examples/config/v1.json", + "examples/scenarios" + ]) + assert 'Usage: pyrandall' in result.output + assert 'passing directory path is not supported yet' in result.output + assert result.exit_code == 2 + +def test_fail_on_invaild_specfile_jsonschema(pyrandall_cli): + result = pyrandall_cli.invoke([ + "--config", + "examples/config/v1.json", + "examples/scenarios/v2_ingest_kafka_invalid.yaml" + ]) + expected_output = """Error on validating specfile examples/scenarios/v2_ingest_kafka_invalid.yaml with jsonschema, given error: +'messages' is a required property + +Failed validating 'required' in schema['properties']['feature']['properties']['scenarios']['items']['properties']['simulate']['allOf'][0]['if']['then']: + {'$id': '#/feature/properties/scenarios/items/properties/simulate/allOf/items/0/messages', + 'properties': {'messages': {'$ref': '#/definitions/simulateMessages'}}, + 'required': ['messages']} + +On instance['feature']['scenarios'][0]['simulate']: + {'adapter': 'broker/kafka'} +""" + + assert expected_output == result.output + assert result.exit_code == 4 diff --git a/tests/unit/test_commander.py b/tests/unit/test_commander.py index 9ce1c02..7b590ba 100644 --- a/tests/unit/test_commander.py +++ b/tests/unit/test_commander.py @@ -5,13 +5,12 @@ from pyrandall.commander import Commander, Flags from pyrandall.reporter import Reporter, ResultSet from pyrandall.spec import SpecBuilder -from tests.conftest import vcr @pytest.fixture def spec(): builder = SpecBuilder( - specfile="one_event.yaml", + specfile=open("examples/scenarios/one_event.yaml"), dataflow_path="examples/", default_request_url="http://localhost:5000", schemas_url="http://localhost:8899/schemas/", @@ -19,19 +18,20 @@ def spec(): return builder.feature() -@vcr.use_cassette("test_commander_run_one_for_one") -def test_commander_run_one_for_one(spec): - reporter = MagicMock(Reporter(), unsafe=True) - reporter.create_and_track_resultset.return_value = MagicMock(ResultSet, unsafe=True) +def test_commander_run_one_for_one(spec, vcr): + with vcr.use_cassette("test_commander_run_one_for_one") as cassette: + reporter = MagicMock(Reporter(), unsafe=True) + reporter.create_and_track_resultset.return_value = MagicMock(ResultSet, unsafe=True) - c = Commander(spec, Flags.BLOCKING_E2E) - c.run(reporter) + c = Commander(spec, Flags.E2E) + c.run(reporter) - reporter.feature.assert_called_once_with("One event"), - reporter.scenario.assert_any_call("Send words1 event") - # at least once called - reporter.simulate.assert_called() - reporter.validate.assert_called() - reporter.run_task.assert_called() - reporter.print_failures.assert_called_once_with() - reporter.passed.assert_called_once() + reporter.feature.assert_called_once_with("One event"), + reporter.scenario.assert_any_call("Send words1 event") + # at least once called + reporter.simulate.assert_called() + reporter.validate.assert_called() + reporter.run_task.assert_called() + reporter.print_failures.assert_called_once_with() + reporter.passed.assert_called_once() + assert len(cassette) == 2 diff --git a/tests/unit/test_requests_http_simulate.py b/tests/unit/test_requests_http_simulate.py index 17578b1..814a766 100644 --- a/tests/unit/test_requests_http_simulate.py +++ b/tests/unit/test_requests_http_simulate.py @@ -5,17 +5,6 @@ from pyrandall.executors import RequestHttp, RequestHttpEvents from pyrandall.spec import RequestEventsSpec, RequestHttpSpec from pyrandall.types import Assertion, ExecutionMode -from tests.conftest import vcr - - -@pytest.fixture -def reporter(): - return MagicMock(unsafe=True) - - -@pytest.fixture -def reporter_1(): - return MagicMock(assertion=MagicMock(spec_set=Assertion), unsafe=True) STATUS_CODE_ASSERTION = {"status_code": 201} @@ -29,7 +18,7 @@ def simulator_1(): url="http://localhost:5000/users", body=b'{"foo": "bar"}', method="POST", - headers=[], + headers={}, ) return RequestHttp(spec) @@ -42,7 +31,7 @@ def simulator_2(): url="http://localhost:5000/users", body=b"", method="POST", - headers=[], + headers={}, ) return RequestHttp(spec) @@ -55,7 +44,7 @@ def simulator_3(): url="http://localhost:5000/users", body=b"", method="POST", - headers=[], + headers={}, ) r2 = RequestHttpSpec( execution_mode=ExecutionMode.SIMULATING, @@ -63,13 +52,13 @@ def simulator_3(): url="http://localhost:5000/users", body=b'{"foo": "bar"}', method="POST", - headers=[], + headers={}, ) spec = RequestEventsSpec(requests=[r1, r2]) return RequestHttpEvents(spec) -def test_simulate__post_201_repsonse(simulator_1, reporter): +def test_simulate__post_201_repsonse(simulator_1, reporter, vcr): with vcr.use_cassette("test_http_executor_simulate_post_201") as cassette: result = simulator_1.execute(reporter) @@ -85,7 +74,7 @@ def test_simulate__post_201_repsonse(simulator_1, reporter): assert result -def test_simulate_post_400_response(simulator_2, reporter): +def test_simulate_post_400_response(simulator_2, reporter, vcr): with vcr.use_cassette("test_http_executor_simulate_post_400_response") as cassette: result = simulator_2.execute(reporter) @@ -108,7 +97,7 @@ def test_simulate_fails_zero_requests(reporter): assert not result -def test_simulate_post_200_and_400(simulator_3, reporter): +def test_simulate_post_200_and_400(simulator_3, reporter, vcr): with vcr.use_cassette("test_http_executor_simulate_post_201_and_400") as cassette: result = simulator_3.execute(reporter) diff --git a/tests/unit/test_requests_http_useragent.py b/tests/unit/test_requests_http_useragent.py new file mode 100644 index 0000000..b5de6ce --- /dev/null +++ b/tests/unit/test_requests_http_useragent.py @@ -0,0 +1,47 @@ +import pytest + +from pyrandall import const +from pyrandall.executors import RequestHttp +from pyrandall.spec import RequestHttpSpec +from pyrandall.types import ExecutionMode +from pytest_httpserver import HeaderValueMatcher + +STATUS_CODE_ASSERTION = {"status_code": 201} + + +@pytest.fixture +def request_http_spec(): + return RequestHttpSpec( + execution_mode=ExecutionMode.SIMULATING, + assertions=STATUS_CODE_ASSERTION, + url="http://localhost:5000/users", + body=b'{"foo": "bar"}', + method="POST", + headers={}, + ) + +@pytest.fixture +def executor_http(request_http_spec): + return RequestHttp(request_http_spec) + +@pytest.fixture +def vcr_headers_filter(): + return [] + +@pytest.fixture +def vcr_match_on(): + return [] + +def test_header_present(httpserver, executor_http, reporter, vcr): + with vcr.use_cassette("test_http_user_agent") as cassette: + result = executor_http.execute(reporter) + + assert len(cassette) == 1 + r0 = cassette.requests[0] + assert r0.url == "http://localhost:5000/users" + assert r0.method == 'POST' + assert len(r0.headers) == 5 + assert 'User-Agent' in r0.headers + agent = r0.headers['User-Agent'] + assert agent.startswith("pyrandall/") + assert agent.split('/')[1] == const.get_version() diff --git a/tests/unit/test_requests_http_validate.py b/tests/unit/test_requests_http_validate.py index 2762393..e8146b2 100644 --- a/tests/unit/test_requests_http_validate.py +++ b/tests/unit/test_requests_http_validate.py @@ -6,7 +6,6 @@ from pyrandall.reporter import Reporter from pyrandall.spec import RequestHttpSpec from pyrandall.types import Assertion, ExecutionMode -from tests.conftest import vcr @pytest.fixture @@ -26,7 +25,7 @@ def validator_1(): url="http://localhost:5000/foo/1", events=[], method="GET", - headers=[], + headers={}, assertions={"status_code": 404}, ) return RequestHttp(spec) @@ -38,7 +37,7 @@ def validator_3(): execution_mode=ExecutionMode.VALIDATING, events=[], method="GET", - headers=[], + headers={}, url="http://localhost:5000/foo/2", assertions={"body": b'{"foo": 2, "bar": false}'}, ) @@ -51,7 +50,7 @@ def validator_4(): execution_mode=ExecutionMode.VALIDATING, events=[], method="GET", - headers=[], + headers={}, url="http://localhost:5000/foo/2", assertions={"status_code": 201, "body": b'{"foo": 2, "bar": false}'}, ) @@ -64,14 +63,14 @@ def validator_5(): execution_mode=ExecutionMode.VALIDATING, events=[], method="GET", - headers=[], + headers={}, url="http://localhost:5000/foo/2", assertions={"status_code": 201, "body": b'{"foo": 2'}, ) return RequestHttp(spec) -def test_validate_makes_get(validator_1, reporter): +def test_validate_makes_get(validator_1, reporter, vcr): with vcr.use_cassette("test_http_executor_validate_makes_get") as cassette: result = validator_1.execute(reporter) @@ -96,7 +95,7 @@ def test_executor_fails_zero_assertions(reporter): assert result is False -def test_validate_makes_get_and_matches_body(validator_3, reporter): +def test_validate_makes_get_and_matches_body(validator_3, reporter, vcr): with vcr.use_cassette( "test_http_executor_validate_makes_get_and_matches_body" ) as cassette: @@ -114,7 +113,7 @@ def test_validate_makes_get_and_matches_body(validator_3, reporter): assert result -def test_validate__matches_body_not_status(validator_4, reporter): +def test_validate__matches_body_not_status(validator_4, reporter, vcr): with vcr.use_cassette( "test_http_executor_validate__matches_body_and_status" ) as cassette: @@ -133,7 +132,7 @@ def test_validate__matches_body_not_status(validator_4, reporter): @patch("pyrandall.executors.requests_http.Assertion") -def test_validate_body_and_status_do_not_match(assertion, validator_5, reporter_1): +def test_validate_body_and_status_do_not_match(assertion, validator_5, reporter_1, vcr): with vcr.use_cassette( "test_http_executor_validate_body_and_status_do_not_match" ) as cassette: diff --git a/tests/unit/test_spec.py b/tests/unit/test_spec.py index 20cc844..76232d0 100644 --- a/tests/unit/test_spec.py +++ b/tests/unit/test_spec.py @@ -7,7 +7,7 @@ @pytest.fixture def feature(): builder = SpecBuilder( - specfile="v2.yaml", + specfile=open("examples/scenarios/v2.yaml"), dataflow_path="examples/", default_request_url="http://localhost:5000", schemas_url="http://localhost:8899/schemas/", @@ -15,14 +15,9 @@ def feature(): return builder.feature() -def test_loads_valid_scenario_yaml(): - with pytest.raises(FileNotFoundError): - SpecBuilder("spec.yaml", dataflow_path="random_dir").feature() - - def test_request_url_missing(): with pytest.raises(ValueError) as e: - SpecBuilder("v2.yaml", dataflow_path="examples/").feature() + SpecBuilder(open("examples/scenarios/v2.yaml"), dataflow_path="examples/").feature() expected = ( "self.default_request_url is None. " "See README.md on how to configure a request URL." @@ -73,7 +68,7 @@ def test_creates_executors_for_validators_only(feature): def test_parse_events_plugins(): SpecBuilder( - specfile="v2.yaml", + specfile=open("examples/scenarios/v2.yaml"), dataflow_path="examples/", default_request_url="http://localhost:5000", schemas_url="http://localhost:8899/schemas/",