diff --git a/examples/CSharpDev/CSharpDev.csproj b/examples/CSharpDev/CSharpDev.csproj index 41785260..9c6198cc 100644 --- a/examples/CSharpDev/CSharpDev.csproj +++ b/examples/CSharpDev/CSharpDev.csproj @@ -13,7 +13,7 @@ - + Always diff --git a/examples/CSharpDev/ClientFactory/CustomDistributionExample.cs b/examples/CSharpDev/ClientFactory/ClientDistributionExample.cs similarity index 90% rename from examples/CSharpDev/ClientFactory/CustomDistributionExample.cs rename to examples/CSharpDev/ClientFactory/ClientDistributionExample.cs index 528409bb..e55ab519 100644 --- a/examples/CSharpDev/ClientFactory/CustomDistributionExample.cs +++ b/examples/CSharpDev/ClientFactory/ClientDistributionExample.cs @@ -1,13 +1,11 @@ -using System; -using System.Threading.Tasks; - -namespace CSharpDev.ClientFactory; +namespace CSharpDev.ClientFactory; -using NBomber; +using System; +using System.Threading.Tasks; using NBomber.Contracts; using NBomber.CSharp; -public class CustomDistributionExample +public class ClientDistributionExample { public static void Run() { @@ -38,6 +36,5 @@ public static void Run() NBomberRunner .RegisterScenarios(scenario) .Run(); - } } diff --git a/examples/CSharpDev/CustomReporting/CustomReporting.cs b/examples/CSharpDev/CustomReporting/CustomReporting.cs deleted file mode 100644 index b81a98fe..00000000 --- a/examples/CSharpDev/CustomReporting/CustomReporting.cs +++ /dev/null @@ -1,61 +0,0 @@ -using System; -using System.Threading.Tasks; - -using Microsoft.Extensions.Configuration; -using Serilog; - -using NBomber.Configuration; -using NBomber.Contracts; -using NBomber.Contracts.Stats; -using NBomber.CSharp; -using static NBomber.Time; - -namespace CSharpDev.CustomReporting -{ - class CustomReportingSink : IReportingSink - { - private ILogger _logger; - public string SinkName => nameof(CustomReportingSink); - - public Task Init(IBaseContext context, IConfiguration infraConfig) - { - _logger = context.Logger; - return Task.CompletedTask; - } - - public Task Start() => Task.CompletedTask; - public Task SaveRealtimeStats(ScenarioStats[] stats) => Task.CompletedTask; - public Task SaveFinalStats(NodeStats[] stats) => Task.CompletedTask; - public Task Stop() => Task.CompletedTask; - - public void Dispose() - { } - } - - public class CustomReporting - { - public static void Run() - { - var step = Step.Create("step", async context => - { - await Task.Delay(Seconds(0.1)); - return Response.Ok(sizeBytes: 100); - }); - - var scenario = ScenarioBuilder - .CreateScenario("simple_scenario", step) - .WithoutWarmUp() - .WithLoadSimulations(Simulation.KeepConstant(1, TimeSpan.FromMinutes(1))); - - NBomberRunner - .RegisterScenarios(scenario) - .WithTestSuite("reporting") - .WithTestName("custom_reporting_test") - .WithReportingSinks(new CustomReportingSink()) - .WithReportingInterval(Seconds(10)) - .WithReportFolder("./custom_reports") - .WithReportFormats(ReportFormat.Html, ReportFormat.Md, ReportFormat.Txt, ReportFormat.Csv) - .Run(); - } - } -} diff --git a/examples/CSharpDev/CustomReporting/CustomReportingExample.cs b/examples/CSharpDev/CustomReporting/CustomReportingExample.cs new file mode 100644 index 00000000..59551635 --- /dev/null +++ b/examples/CSharpDev/CustomReporting/CustomReportingExample.cs @@ -0,0 +1,60 @@ +using System; +using System.Threading.Tasks; + +using Microsoft.Extensions.Configuration; +using Serilog; + +using NBomber.Configuration; +using NBomber.Contracts; +using NBomber.Contracts.Stats; +using NBomber.CSharp; +using static NBomber.Time; + +namespace CSharpDev.CustomReporting; + +class CustomReportingSink : IReportingSink +{ + private ILogger _logger; + public string SinkName => nameof(CustomReportingSink); + + public Task Init(IBaseContext context, IConfiguration infraConfig) + { + _logger = context.Logger; + return Task.CompletedTask; + } + + public Task Start() => Task.CompletedTask; + public Task SaveRealtimeStats(ScenarioStats[] stats) => Task.CompletedTask; + public Task SaveFinalStats(NodeStats[] stats) => Task.CompletedTask; + public Task Stop() => Task.CompletedTask; + + public void Dispose() + { } +} + +public class CustomReportingExample +{ + public static void Run() + { + var step = Step.Create("step", async context => + { + await Task.Delay(Seconds(0.1)); + return Response.Ok(sizeBytes: 100); + }); + + var scenario = ScenarioBuilder + .CreateScenario("simple_scenario", step) + .WithoutWarmUp() + .WithLoadSimulations(Simulation.KeepConstant(1, TimeSpan.FromMinutes(1))); + + NBomberRunner + .RegisterScenarios(scenario) + .WithTestSuite("reporting") + .WithTestName("custom_reporting_test") + .WithReportingSinks(new CustomReportingSink()) + .WithReportingInterval(Seconds(10)) + .WithReportFolder("./custom_reports") + .WithReportFormats(ReportFormat.Html, ReportFormat.Md, ReportFormat.Txt, ReportFormat.Csv) + .Run(); + } +} diff --git a/examples/CSharpDev/DataFeed/DataFeedTest.cs b/examples/CSharpDev/DataFeed/DataFeedTest.cs index d902def6..ea65fdac 100644 --- a/examples/CSharpDev/DataFeed/DataFeedTest.cs +++ b/examples/CSharpDev/DataFeed/DataFeedTest.cs @@ -5,46 +5,45 @@ using NBomber.CSharp; using static NBomber.Time; -namespace CSharpDev.DataFeed +namespace CSharpDev.DataFeed; + +public class User { - public class User - { - public int Id { get; set; } - public string Name { get; set; } - } + public int Id { get; set; } + public string Name { get; set; } +} - public class DataFeedTest +public class DataFeedTest +{ + public static void Run() { - public static void Run() - { - var data = new[] {1, 2, 3, 4, 5}.ShuffleData(); - //var data = FeedData.FromJson("./DataFeed/users-feed-data.json"); - //var data = FeedData.FromCsv("./DataFeed/users-feed-data.csv"); + var data = new[] {1, 2, 3, 4, 5}.ShuffleData(); + //var data = FeedData.FromJson("./DataFeed/users-feed-data.json"); + //var data = FeedData.FromCsv("./DataFeed/users-feed-data.csv"); - var feed = Feed.CreateCircular("numbers", data); - //var feed = Feed.CreateCircularLazy("numbers", getData: context => data); + var feed = Feed.CreateCircular("numbers", data); + //var feed = Feed.CreateCircularLazy("numbers", getData: context => data); - //var feed = Feed.CreateConstant("numbers", data); - //var feed = Feed.CreateConstantLazy("numbers", data); + //var feed = Feed.CreateConstant("numbers", data); + //var feed = Feed.CreateConstantLazy("numbers", data); - //var feed = Feed.CreateRandom("numbers", data); - //var feed = Feed.CreateRandomLazy("numbers", data); + //var feed = Feed.CreateRandom("numbers", data); + //var feed = Feed.CreateRandomLazy("numbers", data); - var step = Step.Create("step", feed, async context => - { - await Task.Delay(Seconds(1)); + var step = Step.Create("step", feed, async context => + { + await Task.Delay(Seconds(1)); - context.Logger.Debug("Data from feed: {FeedItem}", context.FeedItem); - return Response.Ok(); - }); + context.Logger.Debug("Data from feed: {FeedItem}", context.FeedItem); + return Response.Ok(); + }); - var scenario = ScenarioBuilder - .CreateScenario("data_feed_scenario", step) - .WithLoadSimulations(Simulation.KeepConstant(1, TimeSpan.FromSeconds(1))); + var scenario = ScenarioBuilder + .CreateScenario("data_feed_scenario", step) + .WithLoadSimulations(Simulation.KeepConstant(1, TimeSpan.FromSeconds(1))); - NBomberRunner - .RegisterScenarios(scenario) - .Run(); - } + NBomberRunner + .RegisterScenarios(scenario) + .Run(); } } diff --git a/examples/CSharpDev/Features/CustomSettingsExample.cs b/examples/CSharpDev/Features/CustomSettingsExample.cs new file mode 100644 index 00000000..b684a9c6 --- /dev/null +++ b/examples/CSharpDev/Features/CustomSettingsExample.cs @@ -0,0 +1,57 @@ +namespace CSharpDev.Features; + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using NBomber.Contracts; +using NBomber.CSharp; + +public class CustomScenarioSettings +{ + public int TestField { get; set; } + public int PauseMs { get; set; } +} + +public class CustomSettingsExample +{ + static CustomScenarioSettings _customSettings = new CustomScenarioSettings(); + + static Task ScenarioInit(IScenarioContext context) + { + _customSettings = context.CustomSettings.Get(); + + context.Logger.Information( + "test init received CustomSettings.TestField '{TestField}'", + _customSettings.TestField + ); + + return Task.CompletedTask; + } + + public static void Run() + { + var step = Step.Create("step", async context => + { + await Task.Delay(TimeSpan.FromSeconds(0.1)); + + context.Logger.Debug( + "step received CustomSettings.TestField '{TestField}'", + _customSettings.TestField + ); + + return Response.Ok(); // this value will be passed as response for the next step + }); + + var customPause = Step.CreatePause(() => _customSettings.PauseMs); + + var scenario = ScenarioBuilder + .CreateScenario("my_scenario", step, customPause) + .WithInit(ScenarioInit); + + NBomberRunner + .RegisterScenarios(scenario) + .LoadConfig("./Features/config.json") + .Run(); + } +} + diff --git a/examples/CSharpDev/HelloWorld/CustomStepExecControlExample.cs b/examples/CSharpDev/Features/CustomStepExecControlExample.cs similarity index 94% rename from examples/CSharpDev/HelloWorld/CustomStepExecControlExample.cs rename to examples/CSharpDev/Features/CustomStepExecControlExample.cs index 81663291..b8b470f8 100644 --- a/examples/CSharpDev/HelloWorld/CustomStepExecControlExample.cs +++ b/examples/CSharpDev/Features/CustomStepExecControlExample.cs @@ -1,10 +1,9 @@ -using System; +namespace CSharpDev.Features; + +using System; using System.Threading.Tasks; using NBomber.Contracts; using NBomber.CSharp; -using ValueOption = NBomber.CSharp.ValueOption; - -namespace CSharpDev.HelloWorld; public class CustomStepExecControlExample { diff --git a/examples/CSharpDev/HelloWorld/config.json b/examples/CSharpDev/Features/config.json similarity index 100% rename from examples/CSharpDev/HelloWorld/config.json rename to examples/CSharpDev/Features/config.json diff --git a/examples/CSharpDev/HelloWorld/CustomSettingsExample.cs b/examples/CSharpDev/HelloWorld/CustomSettingsExample.cs deleted file mode 100644 index 2dd62951..00000000 --- a/examples/CSharpDev/HelloWorld/CustomSettingsExample.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System; -using System.Threading.Tasks; -using Microsoft.Extensions.Configuration; -using NBomber.Contracts; -using NBomber.CSharp; -using static NBomber.Time; - -namespace CSharpDev.HelloWorld -{ - public class CustomScenarioSettings - { - public int TestField { get; set; } - public int PauseMs { get; set; } - } - - public class CustomSettingsExample - { - static CustomScenarioSettings _customSettings = new CustomScenarioSettings(); - - static Task ScenarioInit(IScenarioContext context) - { - _customSettings = context.CustomSettings.Get(); - - context.Logger.Information( - "test init received CustomSettings.TestField '{TestField}'", - _customSettings.TestField - ); - - return Task.CompletedTask; - } - - public static void Run() - { - var step = Step.Create("step", async context => - { - await Task.Delay(Seconds(0.1)); - - context.Logger.Debug( - "step received CustomSettings.TestField '{TestField}'", - _customSettings.TestField - ); - - return Response.Ok(); // this value will be passed as response for the next step - }); - - var customPause = Step.CreatePause(() => _customSettings.PauseMs); - - var scenario = ScenarioBuilder - .CreateScenario("my_scenario", step, customPause) - .WithInit(ScenarioInit); - - NBomberRunner - .RegisterScenarios(scenario) - .LoadConfig("./HelloWorld/config.json") - .Run(); - } - } -} diff --git a/examples/CSharpDev/HelloWorld/HelloWorldExample.cs b/examples/CSharpDev/HelloWorld/HelloWorldExample.cs index 107f1541..a4a7b530 100644 --- a/examples/CSharpDev/HelloWorld/HelloWorldExample.cs +++ b/examples/CSharpDev/HelloWorld/HelloWorldExample.cs @@ -1,46 +1,44 @@ +namespace CSharpDev.HelloWorld; + using System; using System.Threading.Tasks; using NBomber.Contracts; using NBomber.CSharp; -using static NBomber.Time; -namespace CSharpDev.HelloWorld +public class HelloWorldExample { - public class HelloWorldExample + public static void Run() { - public static void Run() + var step1 = Step.Create("step_1", async context => { - var step1 = Step.Create("step_1", async context => - { - // you can do any logic here: go to http, websocket etc + // you can do any logic here: go to http, websocket etc - await Task.Delay(Seconds(0.1)); - return Response.Ok(42); // this value will be passed as response for the next step - }); + await Task.Delay(TimeSpan.FromSeconds(0.1)); + return Response.Ok(42); // this value will be passed as response for the next step + }); - var pause = Step.CreatePause(Milliseconds(100)); + var pause = Step.CreatePause(TimeSpan.FromMilliseconds(100)); - var step2 = Step.Create("step_2", async context => - { - var value = context.GetPreviousStepResponse(); // 42 - return Response.Ok(); - }); + var step2 = Step.Create("step_2", async context => + { + var value = context.GetPreviousStepResponse(); // 42 + return Response.Ok(); + }); - // here you create scenario and define (default) step order - // you also can define them in opposite direction, like [step2; step1] - // or even repeat [step1; step1; step1; step2] - var scenario = ScenarioBuilder - .CreateScenario("hello_world_scenario", step1, pause, step2) - .WithoutWarmUp() - .WithLoadSimulations( - Simulation.KeepConstant(copies: 1, during: TimeSpan.FromSeconds(30)) - ); + // here you create scenario and define (default) step order + // you also can define them in opposite direction, like [step2; step1] + // or even repeat [step1; step1; step1; step2] + var scenario = ScenarioBuilder + .CreateScenario("hello_world_scenario", step1, pause, step2) + .WithoutWarmUp() + .WithLoadSimulations( + Simulation.KeepConstant(copies: 1, during: TimeSpan.FromSeconds(30)) + ); - NBomberRunner - .RegisterScenarios(scenario) - .WithTestSuite("example") - .WithTestName("hello_world_test") - .Run(); - } + NBomberRunner + .RegisterScenarios(scenario) + .WithTestSuite("example") + .WithTestName("hello_world_test") + .Run(); } } diff --git a/examples/CSharpDev/Program.cs b/examples/CSharpDev/Program.cs index 7e914584..06691c68 100644 --- a/examples/CSharpDev/Program.cs +++ b/examples/CSharpDev/Program.cs @@ -1,5 +1,8 @@ using System; using CSharpDev.ClientFactory; +using CSharpDev.CustomReporting; +using CSharpDev.DataFeed; +using CSharpDev.Features; using CSharpDev.HelloWorld; namespace CSharpDev @@ -9,8 +12,11 @@ class Program static void Main(string[] args) { HelloWorldExample.Run(); - //CustomDistributionExample.Run(); - //CustomStepExecControlExample.Run(); + // ClientDistributionExample.Run(); + // CustomStepExecControlExample.Run(); + // DataFeedTest.Run(); + // CustomReportingExample.Run(); + // CustomSettingsExample.Run(); } } } diff --git a/examples/FSharpDev/FSharpDev.fsproj b/examples/FSharpDev/FSharpDev.fsproj index 177e83bd..62111693 100644 --- a/examples/FSharpDev/FSharpDev.fsproj +++ b/examples/FSharpDev/FSharpDev.fsproj @@ -17,7 +17,6 @@ Always - Always @@ -27,10 +26,6 @@ Always - - - - diff --git a/examples/FSharpDev/HelloWorld/CustomStepExecControlExample.fs b/examples/FSharpDev/HelloWorld/CustomStepExecControlExample.fs deleted file mode 100644 index 7495e22c..00000000 --- a/examples/FSharpDev/HelloWorld/CustomStepExecControlExample.fs +++ /dev/null @@ -1,42 +0,0 @@ -module FSharpDev.HelloWorld.CustomStepExecControlExample - -open System.Threading.Tasks - -open FSharp.Control.Tasks.NonAffine - -open NBomber -open NBomber.Contracts -open NBomber.FSharp - -let run () = - - let step1 = Step.create("step_1", fun context -> task { - context.Logger.Information($"{context.StepName} invoked") - do! Task.Delay(milliseconds 500) - return Response.ok() - }) - - let step2 = Step.create("step_2", fun context -> task { - context.Logger.Information($"{context.StepName} invoked") - do! Task.Delay(milliseconds 500) - return Response.ok() - }) - - let step3 = Step.create("step_3", fun context -> task { - context.Logger.Information($"{context.StepName} invoked") - do! Task.Delay(milliseconds 500) - return Response.ok() - }) - - Scenario.create "scenario" [step1; step2; step3] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(copies = 1, during = seconds 10)] - |> Scenario.withCustomStepExecControl(fun execControl -> - // step_1 will never be invoked - match execControl with - | ValueSome exec when not exec.PrevStepResponse.IsError -> ValueSome "step_2" - | _ -> ValueSome "step_3" - ) - |> NBomberRunner.registerScenario - |> NBomberRunner.run - |> ignore diff --git a/examples/FSharpDev/MqttTests/MqttScenario.fs b/examples/FSharpDev/MqttTests/MqttScenario.fs deleted file mode 100644 index a870d732..00000000 --- a/examples/FSharpDev/MqttTests/MqttScenario.fs +++ /dev/null @@ -1,12 +0,0 @@ -module FSharpDev.MqttTests.MqttScenario - -open NBomber.FSharp - -let run () = - - NBomberRunner.registerScenarios [ - PublisherScenario.create() - SubscriberScenario.create() - ] - |> NBomberRunner.run - |> ignore diff --git a/examples/FSharpDev/MqttTests/PublisherScenario.fs b/examples/FSharpDev/MqttTests/PublisherScenario.fs deleted file mode 100644 index 841b8f40..00000000 --- a/examples/FSharpDev/MqttTests/PublisherScenario.fs +++ /dev/null @@ -1,70 +0,0 @@ -module FSharpDev.MqttTests.PublisherScenario - -open System -open System.Text -open System.Threading - -open MQTTnet -open MQTTnet.Client.Connecting -open MQTTnet.Client.Options -open MQTTnet.Client.Publishing -open FSharp.Control.Tasks.NonAffine -open Newtonsoft.Json - -open NBomber -open NBomber.Contracts -open NBomber.FSharp - -[] -type Message = { - Number: int - PublishTime: DateTime -} - -let private createMqttFactory () = - ClientFactory.create( - name = "mqtt_publisher_factory", - initClient = fun (i, context) -> task { - let clientId = $"publisher_{i}" - let mqttFactory = MqttFactory() - - let clientOptions = - MqttClientOptionsBuilder() - .WithTcpServer("localhost") - .WithCleanSession() - .WithClientId(clientId) - .Build(); - - let client = mqttFactory.CreateMqttClient() - let! result = client.ConnectAsync(clientOptions, CancellationToken.None) - - if result.ResultCode <> MqttClientConnectResultCode.Success then - failwith $"MQTT connection code is: {result.ResultCode}, reason: {result.ReasonString}" - - return client - } - ) - -let create () = - - let mqttClientFactory = createMqttFactory() - - let step = Step.create("publish_step", - clientFactory = mqttClientFactory, - execute = fun context -> task { - - let msg = { Number = context.InvocationCount; PublishTime = DateTime.UtcNow } - let payload = msg |> JsonConvert.SerializeObject |> Encoding.ASCII.GetBytes - let mqttMsg = MqttApplicationMessage(Topic = "test_topic", Payload = payload) - let! result = context.Client.PublishAsync(mqttMsg, context.CancellationToken) - - return - if result.ReasonCode = MqttClientPublishReasonCode.Success then Response.ok() - else Response.fail(statusCode = int result.ReasonCode) - }) - - let pause = Step.createPause(seconds 0.5) - - Scenario.create "mqtt_publisher_scenario" [step; pause] - |> Scenario.withLoadSimulations [KeepConstant(1, seconds 10)] - |> Scenario.withoutWarmUp diff --git a/examples/FSharpDev/MqttTests/SubscriberScenario.fs b/examples/FSharpDev/MqttTests/SubscriberScenario.fs deleted file mode 100644 index 60db9126..00000000 --- a/examples/FSharpDev/MqttTests/SubscriberScenario.fs +++ /dev/null @@ -1,87 +0,0 @@ -module FSharpDev.MqttTests.SubscriberScenario - -open System -open System.Text -open System.Threading -open System.Threading.Tasks - -open MQTTnet -open MQTTnet.Client -open MQTTnet.Client.Connecting -open MQTTnet.Client.Options -open FSharp.Control.Tasks.NonAffine -open Newtonsoft.Json - -open NBomber -open NBomber.Contracts -open NBomber.Extensions.PushExtensions -open NBomber.FSharp - -[] -type Message = { - Number: int - PublishTime: DateTime -} - -let private _responseQueue = new PushResponseQueue() - -let private createMqttFactory (clientCount) = - ClientFactory.create( - name = "mqtt_subscriber_factory", - initClient = fun (i, ctx) -> task { - let clientId = $"subscriber_{i}" - let mqttFactory = MqttFactory() - - let clientOptions = - MqttClientOptionsBuilder() - .WithTcpServer("localhost") - .WithCleanSession() - .WithClientId(clientId) - .Build(); - - let client = mqttFactory.CreateMqttClient() - - client.UseApplicationMessageReceivedHandler(fun msg -> - _responseQueue.AddResponse(clientId, msg.ApplicationMessage.Payload) - Task.CompletedTask - ) - |> ignore - - let! result = client.ConnectAsync(clientOptions, CancellationToken.None) - let! subscribeResult = client.SubscribeAsync("test_topic") - - if result.ResultCode = MqttClientConnectResultCode.Success then - _responseQueue.InitQueueForClient(clientId) - else - failwith $"MQTT connection code is: {result.ResultCode}, reason: {result.ReasonString}" - - return client - }, - clientCount = clientCount - ) - -let create () = - - let clientCount = 1 - let mqttClientFactory = createMqttFactory(clientCount) - - let step = Step.create("receive_step", - clientFactory = mqttClientFactory, - execute = fun context -> task { - - let clientId = context.Client.Options.ClientId - let! response = _responseQueue.ReceiveResponse(clientId) - - let msg = - response.Payload :?> byte[] - |> Encoding.ASCII.GetString - |> JsonConvert.DeserializeObject - - let latency = response.ReceivedTime - msg.PublishTime - - return Response.ok(latencyMs = latency.TotalMilliseconds) - }) - - Scenario.create "mqtt_subscriber_scenario" [step] - |> Scenario.withLoadSimulations [KeepConstant(1, seconds 10)] - |> Scenario.withoutWarmUp diff --git a/examples/FSharpDev/MqttTests/docker-compose.yaml b/examples/FSharpDev/MqttTests/docker-compose.yaml deleted file mode 100644 index 4c11b690..00000000 --- a/examples/FSharpDev/MqttTests/docker-compose.yaml +++ /dev/null @@ -1,11 +0,0 @@ -version: '3.4' -services: - - emqx: - container_name: emqx - image: "emqx/emqx:latest" - ports: - - 1883:1883 - - 18083:18083 - - 8083:8083 - - 8084:8084 diff --git a/examples/FSharpDev/Program.fs b/examples/FSharpDev/Program.fs index feefd7f6..72ce247d 100644 --- a/examples/FSharpDev/Program.fs +++ b/examples/FSharpDev/Program.fs @@ -5,13 +5,11 @@ open FSharpDev.ClientFactory open FSharpDev.HelloWorld open FSharpDev.DataFeed open FSharpDev.HttpTests -open FSharpDev.MqttTests [] let main argv = //HelloWorldExample.run() - //CustomStepExecControlExample.run() //CustomSettingsExample.run() //DataFeedTest.run() //SimpleHttpTest.run() diff --git a/src/NBomber.Contracts/Stats.fs b/src/NBomber.Contracts/Stats.fs index 98606354..20d1e1ef 100644 --- a/src/NBomber.Contracts/Stats.fs +++ b/src/NBomber.Contracts/Stats.fs @@ -22,7 +22,6 @@ type NodeType = | SingleNode | Coordinator | Agent - | Cluster type OperationType = | None = 0 diff --git a/src/NBomber/Domain/Concurrency/ScenarioActor.fs b/src/NBomber/Domain/Concurrency/ScenarioActor.fs index 1dc06d01..e7a4b02b 100644 --- a/src/NBomber/Domain/Concurrency/ScenarioActor.fs +++ b/src/NBomber/Domain/Concurrency/ScenarioActor.fs @@ -11,7 +11,6 @@ open FSharp.Control.Tasks.NonAffine open NBomber open NBomber.Contracts open NBomber.Contracts.Internal -open NBomber.Domain open NBomber.Domain.DomainTypes open NBomber.Domain.Step open NBomber.Domain.Stats.ScenarioStatsActor @@ -23,6 +22,8 @@ type ActorDep = { Scenario: Scenario ScenarioStatsActor: IScenarioStatsActor ExecStopCommand: StopCommand -> unit + GetStepOrder: Scenario -> int[] + ExecSteps: StepDep -> RunningStep[] -> int[] -> Task // stepDep steps stepsOrder } type ScenarioActor(dep: ActorDep, scenarioInfo: ScenarioInfo) = @@ -58,8 +59,8 @@ type ScenarioActor(dep: ActorDep, scenarioInfo: ScenarioInfo) = _stepDep.Data.Clear() try - let stepsOrder = Scenario.getStepOrder dep.Scenario - do! RunningStep.execSteps _stepDep _steps stepsOrder + let stepsOrder = dep.GetStepOrder dep.Scenario + do! dep.ExecSteps _stepDep _steps stepsOrder with | ex -> _logger.Error(ex, $"Unhandled exception for Scenario: {dep.Scenario.ScenarioName}") diff --git a/src/NBomber/Domain/Concurrency/Scheduler/ConstantActorScheduler.fs b/src/NBomber/Domain/Concurrency/Scheduler/ConstantActorScheduler.fs index 143ffd27..fe585ea6 100644 --- a/src/NBomber/Domain/Concurrency/Scheduler/ConstantActorScheduler.fs +++ b/src/NBomber/Domain/Concurrency/Scheduler/ConstantActorScheduler.fs @@ -12,6 +12,7 @@ type SchedulerCommand = | RemoveActor of removeCount:int | StopScheduler +// dep * actorPool * scheduledActorCount type SchedulerExec = ActorDep -> ScenarioActor list -> int -> ScenarioActor list let removeFromScheduler scheduledActorsCount removeCount = diff --git a/src/NBomber/Domain/Concurrency/Scheduler/OneTimeActorScheduler.fs b/src/NBomber/Domain/Concurrency/Scheduler/OneTimeActorScheduler.fs index c1734e43..98605ef1 100644 --- a/src/NBomber/Domain/Concurrency/Scheduler/OneTimeActorScheduler.fs +++ b/src/NBomber/Domain/Concurrency/Scheduler/OneTimeActorScheduler.fs @@ -10,6 +10,7 @@ type SchedulerCommand = | StartActors of actors:ScenarioActor list | RentActors of actorCount:int +// dep * actorPool * scheduledActorCount type SchedulerExec = ActorDep -> ScenarioActor list -> int -> ScenarioActor list // todo: add tests diff --git a/src/NBomber/Domain/Errors.fs b/src/NBomber/Domain/Errors.fs index 9afdfbbb..bbbd9c42 100644 --- a/src/NBomber/Domain/Errors.fs +++ b/src/NBomber/Domain/Errors.fs @@ -39,6 +39,7 @@ type ValidationError = | SimulationIsBiggerThanMax of simulation:string | CopiesCountIsZeroOrNegative of simulation:string | RateIsZeroOrNegative of simulation:string + | EnterpriseOnlyFeature of message:string type AppError = | Domain of DomainError @@ -125,6 +126,7 @@ type AppError = | DuplicateScenarioNamesInConfig scenarioNames -> $"Scenario names are not unique in JSON config: '{String.concatWithComma scenarioNames}'" + | EnterpriseOnlyFeature message -> message static member toString (error: AppError) = match error with diff --git a/src/NBomber/Domain/Scenario.fs b/src/NBomber/Domain/Scenario.fs index 21e3b3b5..b162c5ed 100644 --- a/src/NBomber/Domain/Scenario.fs +++ b/src/NBomber/Domain/Scenario.fs @@ -196,12 +196,7 @@ let createDefaultStepOrder (stepOrderIndex: Dictionary) (scenario: C |> Seq.toArray let getStepOrder (scenario: Scenario) = - match scenario.CustomStepOrder with - | Some getStepOrder -> - getStepOrder() - |> Array.map(fun stName -> scenario.StepOrderIndex[stName]) - - | None -> scenario.DefaultStepOrder + scenario.DefaultStepOrder let createScenario (scn: Contracts.Scenario) = result { let! timeline = scn.LoadSimulations |> LoadTimeLine.createWithDuration diff --git a/src/NBomber/Domain/Step.fs b/src/NBomber/Domain/Step.fs index ca2161c8..17ef57ae 100644 --- a/src/NBomber/Domain/Step.fs +++ b/src/NBomber/Domain/Step.fs @@ -61,7 +61,7 @@ module StepContext = StopScenario = fun (scnName,reason) -> StopScenario(scnName, reason) |> dep.ExecStopCommand StopCurrentTest = fun reason -> StopTest(reason) |> dep.ExecStopCommand } - let inline create (untyped: UntypedStepContext) = { + let create (untyped: UntypedStepContext) = { new IStepContext<'TClient,'TFeedItem> with member _.StepName = untyped.StepName member _.ScenarioInfo = untyped.ScenarioInfo @@ -83,7 +83,7 @@ module StepContext = module StepClientContext = - let inline create (untyped: UntypedStepContext) (clientCount: int) = { + let create (untyped: UntypedStepContext) (clientCount: int) = { new IStepClientContext<'TFeedItem> with member _.StepName = untyped.StepName member _.ScenarioInfo = untyped.ScenarioInfo @@ -113,21 +113,14 @@ module RunningStep = let create (dep: StepDep) (stepIndex: int) (step: Step) = { StepIndex = stepIndex; Value = step; Context = StepContext.createUntyped dep step } - let getClient (context: UntypedStepContext) - (clientPool: ClientPool option) - (clientDistribution: (IStepClientContext -> int) option) = + let getClient (context: UntypedStepContext) (clientPool: ClientPool option) = - match clientPool, clientDistribution with - | Some pool, Some getClientIndex -> - let ctx = StepClientContext.create context pool.ClientCount - let index = getClientIndex ctx - pool.InitializedClients[index] - - | Some pool, None -> + match clientPool with + | Some pool -> let index = context.ScenarioInfo.ThreadNumber % pool.InitializedClients.Length pool.InitializedClients[index] - | _, _ -> Unchecked.defaultof<_> + | _ -> Unchecked.defaultof<_> let updateContext (step: RunningStep) (data: Dictionary) = let st = step.Value @@ -143,7 +136,7 @@ module RunningStep = context.Data <- data context.FeedItem <- feedItem // context.Client should be set as the last field because init order matter here - context.Client <- getClient context st.ClientPool st.ClientDistribution + context.Client <- getClient context st.ClientPool step @@ -200,30 +193,6 @@ module RunningStep = return ValueNone } - let execCustomExec (dep: StepDep) (steps: RunningStep[]) (execControl: IStepExecControlContext voption -> string voption) = task { - let mutable stop = false - let mutable execContext = ValueNone - while stop || not dep.CancellationToken.IsCancellationRequested do - let nextStep = execControl execContext - match nextStep with - | ValueSome stepName -> - let stepIndex = dep.Scenario.StepOrderIndex[stepName] - let step = updateContext steps[stepIndex] dep.Data - let! response = execStep dep step - - match response with - | ValueSome resp -> - execContext <- ValueSome { - new IStepExecControlContext with - member _.PrevStepContext = StepContext.create step.Context - member _.PrevStepResponse = resp - } - - | ValueNone -> stop <- true - - | ValueNone -> stop <- true - } - let execRegularExec (dep: StepDep) (steps: RunningStep[]) (stepsOrder: int[]) = task { let mutable stop = false for stepIndex in stepsOrder do @@ -237,6 +206,4 @@ module RunningStep = } let execSteps (dep: StepDep) (steps: RunningStep[]) (stepsOrder: int[]) = - match dep.Scenario.CustomStepExecControl with - | Some execControl -> execCustomExec dep steps execControl - | None -> execRegularExec dep steps stepsOrder + execRegularExec dep steps stepsOrder diff --git a/src/NBomber/DomainServices/NBomberContext.fs b/src/NBomber/DomainServices/NBomberContext.fs index 4aa1abfa..64ea40b9 100644 --- a/src/NBomber/DomainServices/NBomberContext.fs +++ b/src/NBomber/DomainServices/NBomberContext.fs @@ -7,13 +7,14 @@ open System.Globalization open FsToolkit.ErrorHandling open NBomber +open NBomber.Extensions.InternalExtensions +open NBomber.Extensions.Operator.Result open NBomber.Configuration open NBomber.Contracts open NBomber.Contracts.Stats open NBomber.Errors open NBomber.Domain -open NBomber.Extensions.InternalExtensions -open NBomber.Extensions.Operator.Result +open NBomber.Infra.Dependency // we keep ClientFactorySettings settings here instead of take them from ScenariosSettings // since after init (for case when the same ClientFactory assigned to several Scenarios) @@ -38,6 +39,64 @@ type SessionArgs = { UseHintsAnalyzer = true } +module EnterpriseValidation = + + let validateReportingSinks (dep: IGlobalDependency) = + match dep.NodeType with + | SingleNode when dep.ReportingSinks.Length > 0 -> + Error(EnterpriseOnlyFeature "ReportingSinks feature supported only for the Enterprise version") + | _ -> + Ok() + + let validateCustomStepExecControl (context: NBomberContext) = + let scenarios = + context.RegisteredScenarios + |> List.filter(fun x -> x.CustomStepExecControl.IsSome) + |> List.map(fun x -> x.ScenarioName) + + if scenarios.Length > 0 then + let names = scenarios |> String.concatWithComma + Error(EnterpriseOnlyFeature $"Scenario: '{names}' is using CustomStepExecControl feature that supported only for the Enterprise version") + else + Ok() + + let validateCustomStepOrder (context: NBomberContext) = + let scenarios = + context.RegisteredScenarios + |> List.filter(fun x -> x.CustomStepOrder.IsSome) + |> List.map(fun x -> x.ScenarioName) + + if scenarios.Length > 0 then + let names = scenarios |> String.concatWithComma + Error(EnterpriseOnlyFeature $"Scenario: '{names}' is using CustomStepOrder feature that supported only for the Enterprise version") + else + Ok() + + let validateClientDistribution (context: NBomberContext) = + let steps = + context.RegisteredScenarios + |> List.collect(fun x -> x.Steps) + |> Seq.cast + |> Seq.filter(fun x -> x.ClientDistribution.IsSome) + |> Seq.map(fun x -> x.StepName) + |> Seq.toList + + if steps.Length > 0 then + let names = steps |> String.concatWithComma + Error(EnterpriseOnlyFeature $"Step: '{names}' is using ClientDistribution feature that supported only for the Enterprise version") + else + Ok() + + let validate (dep: IGlobalDependency) (context: NBomberContext) = + result { + do! validateReportingSinks dep + do! validateCustomStepExecControl context + do! validateCustomStepOrder context + do! validateClientDistribution context + return context + } + |> Result.mapError AppError.create + module Validation = let checkAvailableTargets (scenarios: Scenario list) (targetScenarios: string list) = @@ -266,7 +325,7 @@ let createSessionArgs (testInfo: TestInfo) (scenarios: DomainTypes.Scenario list UseHintsAnalyzer = useHintsAnalyzer } } - |> Result.mapError(AppError.create) + |> Result.mapError AppError.create let createScenarios (context: NBomberContext) = context.RegisteredScenarios |> Scenario.createScenarios diff --git a/src/NBomber/DomainServices/NBomberRunner.fs b/src/NBomber/DomainServices/NBomberRunner.fs index bab51d49..592cc87a 100644 --- a/src/NBomber/DomainServices/NBomberRunner.fs +++ b/src/NBomber/DomainServices/NBomberRunner.fs @@ -6,6 +6,8 @@ open NBomber open NBomber.Contracts open NBomber.Contracts.Stats open NBomber.Errors +open NBomber.Domain +open NBomber.Domain.Step open NBomber.Infra open NBomber.Infra.Dependency open NBomber.DomainServices.Reports @@ -18,14 +20,16 @@ let runSession (testInfo: TestInfo) (nodeInfo: NodeInfo) (context: NBomberContex dep.Logger.Information(Constants.NBomberWelcomeText, nodeInfo.NBomberVersion, testInfo.SessionId) dep.Logger.Information "NBomber started as single node" - let! scenarios = context |> NBomberContext.createScenarios - let! sessionArgs = context |> NBomberContext.createSessionArgs testInfo scenarios - use testHost = new TestHost(dep, scenarios) + let! ctx = NBomberContext.EnterpriseValidation.validate dep context + + let! scenarios = ctx |> NBomberContext.createScenarios + let! sessionArgs = ctx |> NBomberContext.createSessionArgs testInfo scenarios + use testHost = new TestHost(dep, scenarios, Scenario.getStepOrder, RunningStep.execSteps) let! result = testHost.RunSession(sessionArgs) let finalStats = Report.build dep.Logger result testHost.TargetScenarios - |> Report.save dep context result.FinalStats + |> Report.save dep ctx result.FinalStats return { result with FinalStats = finalStats } } diff --git a/src/NBomber/DomainServices/TestHost/TestHost.fs b/src/NBomber/DomainServices/TestHost/TestHost.fs index 02524417..05b7cbe0 100644 --- a/src/NBomber/DomainServices/TestHost/TestHost.fs +++ b/src/NBomber/DomainServices/TestHost/TestHost.fs @@ -14,6 +14,7 @@ open NBomber.Contracts.Stats open NBomber.Errors open NBomber.Domain open NBomber.Domain.DomainTypes +open NBomber.Domain.Step open NBomber.Domain.Stats open NBomber.Domain.Stats.ScenarioStatsActor open NBomber.Domain.Concurrency.ScenarioActor @@ -24,7 +25,10 @@ open NBomber.DomainServices open NBomber.DomainServices.NBomberContext open NBomber.DomainServices.TestHost.TestHostReportingActor -type internal TestHost(dep: IGlobalDependency, registeredScenarios: Scenario list) as this = +type internal TestHost(dep: IGlobalDependency, + registeredScenarios: Scenario list, + getStepOrder: Scenario -> int[], + execSteps: StepDep -> RunningStep[] -> int[] -> Task) as this = let _logger = dep.Logger.ForContext() let mutable _stopped = false @@ -52,7 +56,9 @@ type internal TestHost(dep: IGlobalDependency, registeredScenarios: Scenario lis | StopTest reason -> this.StopScenarios(reason) |> ignore let createScenarioSchedulers (targetScenarios: Scenario list) - (createStatsActor: ILogger -> Scenario -> TimeSpan -> IScenarioStatsActor) = + (createStatsActor: ILogger -> Scenario -> TimeSpan -> IScenarioStatsActor) + (getStepOrder: Scenario -> int[]) + (execSteps: StepDep -> RunningStep[] -> int[] -> Task) = let createScheduler (cancelToken: CancellationToken) (scn: Scenario) = let actorDep = { @@ -62,6 +68,8 @@ type internal TestHost(dep: IGlobalDependency, registeredScenarios: Scenario lis Scenario = scn ScenarioStatsActor = createStatsActor _logger scn _sessionArgs.ReportingInterval ExecStopCommand = execStopCommand + GetStepOrder = getStepOrder + ExecSteps = execSteps } new ScenarioScheduler(actorDep) @@ -232,7 +240,7 @@ type internal TestHost(dep: IGlobalDependency, registeredScenarios: Scenario lis member _.GetHints(finalStats) = _targetScenarios |> getHints finalStats member _.CreateScenarioSchedulers(createStatsActor: ILogger -> Scenario -> TimeSpan -> IScenarioStatsActor) = - createScenarioSchedulers _targetScenarios createStatsActor + createScenarioSchedulers _targetScenarios createStatsActor getStepOrder execSteps member _.RunSession(sessionArgs: SessionArgs) = taskResult { let targetScenarios = registeredScenarios |> TestHostScenario.getTargetScenarios sessionArgs @@ -257,8 +265,8 @@ type internal TestHost(dep: IGlobalDependency, registeredScenarios: Scenario lis let! finalStats = reportingActor.GetFinalStats(getCurrentNodeInfo()) let! timeLineHistory = reportingActor.GetTimeLineHistory() let hints = _targetScenarios |> getHints finalStats - - return { FinalStats = finalStats; TimeLineHistory = Array.ofList timeLineHistory; Hints = hints } + let result = { FinalStats = finalStats; TimeLineHistory = Array.ofList timeLineHistory; Hints = hints } + return result } interface IDisposable with diff --git a/src/NBomber/Extensions/PushExtensions.fs b/src/NBomber/Extensions/PushExtensions.fs deleted file mode 100644 index 2420798f..00000000 --- a/src/NBomber/Extensions/PushExtensions.fs +++ /dev/null @@ -1,107 +0,0 @@ -namespace NBomber.Extensions.PushExtensions - -open System -open System.Collections.Generic -open System.Diagnostics -open System.Runtime.CompilerServices -open System.Threading.Tasks -open System.Threading.Tasks.Dataflow - -open NBomber.Extensions.InternalExtensions - -type PushResponse = { - ClientId: string - Payload: obj - ReceivedTime: DateTime -} - -type ClientId = string - -type internal CurrentTime() = - - let _timer = Stopwatch() - let _initTime = DateTime.UtcNow - - do _timer.Start() - - member _.UtcNow = _initTime + _timer.Elapsed - -type internal ActorMessage = - | InitQueueForClient of awaiterTsc:TaskCompletionSource * ClientId - | ReceivedPushResponse of PushResponse - | SubscribeOnResponse of awaiterTsc:TaskCompletionSource * ClientId - | ClearQueue of awaiterTsc:TaskCompletionSource - -type internal ActorState = - { Clients: Dictionary option> - ClientResponses: Dictionary> } - - static member init () = - { Clients = Dictionary option>() - ClientResponses = Dictionary>() } - - static member receive (state: ActorState) (msg: ActorMessage) = - match msg with - | InitQueueForClient (awaiterTsc, clientId) -> - state.Clients[clientId] <- None - state.ClientResponses[clientId] <- Queue() - awaiterTsc.TrySetResult() |> ignore - - | ReceivedPushResponse pushResponse -> - let clientResponses = state.ClientResponses[pushResponse.ClientId] - clientResponses.Enqueue(pushResponse) - - match state.Clients.TryGetValue(pushResponse.ClientId) with - | true, Some awaiterTsc -> - let latestResponse = clientResponses.Dequeue() - awaiterTsc.TrySetResult(latestResponse) |> ignore - state.Clients[pushResponse.ClientId] <- None - - | _ -> () - - | SubscribeOnResponse (awaiterTsc, clientId) -> - let clientResponses = state.ClientResponses[clientId] - if clientResponses.Count > 0 then - let response = clientResponses.Dequeue() - awaiterTsc.TrySetResult(response) |> ignore - else - state.Clients[clientId] <- Some awaiterTsc - - | ClearQueue awaiterTsc -> - state.Clients.Values - |> Seq.iter (fun awaiterTcs -> awaiterTcs |> Option.iter (fun x -> x.TrySetCanceled() |> ignore)) - - state.Clients.Clear() - state.ClientResponses.Clear() - awaiterTsc.TrySetResult() |> ignore - - state - -type PushResponseQueue() = - - let mutable _state = ActorState.init() - let _actor = ActionBlock(fun msg -> _state <- ActorState.receive _state msg) - let _currentTime = CurrentTime() - - [] - member _.InitQueueForClient(clientId: string) = - let awaiterTsc = TaskCompletionSource() - _actor.Post(InitQueueForClient(awaiterTsc, clientId)) |> ignore - awaiterTsc.Task.Wait() - - [] - member _.ReceiveResponse(clientId: string) = - let awaiterTsc = TaskCompletionSource() - _actor.Post(SubscribeOnResponse(awaiterTsc, clientId)) |> ignore - awaiterTsc.Task - - [] - member _.AddResponse(clientId: string, payload: obj) = - let pushResponse = { ClientId = clientId; Payload = payload; ReceivedTime = _currentTime.UtcNow } - _actor.Post(ReceivedPushResponse pushResponse) |> ignore - - interface IDisposable with - member _.Dispose() = - let awaiterTsc = TaskCompletionSource() - _actor.Post(ClearQueue awaiterTsc) |> ignore - awaiterTsc.Task.Wait() diff --git a/src/NBomber/NBomber.fsproj b/src/NBomber/NBomber.fsproj index e49da754..bc46b80c 100644 --- a/src/NBomber/NBomber.fsproj +++ b/src/NBomber/NBomber.fsproj @@ -33,7 +33,6 @@ - @@ -90,7 +89,7 @@ - + @@ -109,4 +108,7 @@ + + + diff --git a/tests/NBomber.IntegrationTests/ClientDistributionTests.fs b/tests/NBomber.IntegrationTests/ClientDistributionTests.fs deleted file mode 100644 index cc0ad577..00000000 --- a/tests/NBomber.IntegrationTests/ClientDistributionTests.fs +++ /dev/null @@ -1,173 +0,0 @@ -module Tests.ClientDistribution - -open System -open System.Threading.Tasks - -open Xunit -open Swensen.Unquote -open FsToolkit.ErrorHandling -open FSharp.Control.Tasks.NonAffine - -open NBomber -open NBomber.Contracts -open NBomber.FSharp -open NBomber.Extensions.InternalExtensions - -[] -let ``should allow distribute client with custom distribution``() = - - let factory = ClientFactory.create( - name = "test_pool", - initClient = (fun (number,context) -> Task.FromResult number), - clientCount = 10 - ) - - let step = Step.create("step", - clientFactory = factory, - clientDistribution = (fun context -> 5), // always return client with ID = 5 - execute = fun context -> task { - - do! Task.Delay(milliseconds 100) - - return - if context.Client = 5 then Response.ok() - else Response.fail() - }) - - Scenario.create "test" [step] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(copies = 100, during = seconds 3)] - |> NBomberRunner.registerScenario - |> NBomberRunner.run - |> Result.getOk - |> fun nodeStats -> - let stepStats = nodeStats.ScenarioStats[0] - test <@ stepStats.OkCount > 0 @> - test <@ stepStats.FailCount = 0 @> - -[] -let ``should support default distribution = ScenarioInfo.ThreadNumber % InitializedClients.Length``() = - - let clientCount = 50 - - let factory = ClientFactory.create( - name = "test_pool", - initClient = (fun (number,context) -> Task.FromResult number), - clientCount = clientCount - ) - - let step = Step.create("step", - clientFactory = factory, - execute = fun context -> task { - - do! Task.Delay(milliseconds 100) - - let clientId = context.ScenarioInfo.ThreadNumber % clientCount - - return - if context.Client = clientId then Response.ok() - else Response.fail() - }) - - Scenario.create "test" [step] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(copies = 100, during = seconds 5)] - |> NBomberRunner.registerScenario - |> NBomberRunner.run - |> Result.getOk - |> fun nodeStats -> - let stepStats = nodeStats.ScenarioStats[0] - test <@ stepStats.OkCount > 0 @> - test <@ stepStats.FailCount = 0 @> - -[] -let ``should validate if ClientFactory.IsNone && ClientDistribution.IsSome``() = - - try - Step.create("step", - clientDistribution = (fun context -> 5), - execute = fun context -> task { - - do! Task.Delay(milliseconds 100) - - return Response.ok() - }) - |> ignore - - failwith "validation bug" - - with - | ex -> - test <@ ex.Message.Contains("clientFactory") @> - -[] -let ``should provide corresponding FeedItem``() = - - let feed = [0;1;2;3;4;5] |> Feed.createCircular "test" - - let factory = ClientFactory.create( - name = "test_pool", - initClient = (fun (number,context) -> Task.FromResult number), - clientCount = 6 - ) - - let step = Step.create("step", - feed = feed, - clientFactory = factory, - clientDistribution = (fun context -> context.FeedItem), - execute = fun context -> task { - - do! Task.Delay(milliseconds 100) - - if context.Client = context.FeedItem - then return Response.ok() - else - return Response.fail() - }) - - Scenario.create "test" [step] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(copies = 1, during = seconds 2)] - |> NBomberRunner.registerScenario - |> NBomberRunner.run - |> Result.getOk - |> fun nodeStats -> - let stepStats = nodeStats.ScenarioStats[0] - test <@ stepStats.OkCount > 0 @> - test <@ stepStats.FailCount = 0 @> - -[] -let ``should handle errors without stopping the test``() = - - let factory = ClientFactory.create( - name = "test_pool", - initClient = (fun (number,context) -> Task.FromResult number), - clientCount = 5 - ) - - let step = Step.create("step", - clientFactory = factory, - clientDistribution = (fun context -> - failwith "client exception" - context.FeedItem - ), - execute = fun context -> task { - - do! Task.Delay(milliseconds 100) - - if context.Client = context.FeedItem - then return Response.ok() - else - return Response.fail() - }) - - Scenario.create "test" [step] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(copies = 1, during = seconds 1)] - |> NBomberRunner.registerScenario - |> NBomberRunner.run - |> Result.getOk - |> fun nodeStats -> - let stepStats = nodeStats.ScenarioStats[0] - test <@ stepStats.OkCount = 0 @> - test <@ stepStats.FailCount > 0 @> diff --git a/tests/NBomber.IntegrationTests/Concurrency/ConstantActorSchedulerTests.fs b/tests/NBomber.IntegrationTests/Concurrency/ConstantActorSchedulerTests.fs index 9448619a..0a6151ab 100644 --- a/tests/NBomber.IntegrationTests/Concurrency/ConstantActorSchedulerTests.fs +++ b/tests/NBomber.IntegrationTests/Concurrency/ConstantActorSchedulerTests.fs @@ -15,6 +15,7 @@ open NBomber.Contracts open NBomber.FSharp open NBomber.Domain open NBomber.Domain.Stats.ScenarioStatsActor +open NBomber.Domain.Step open NBomber.Domain.Concurrency open NBomber.Domain.Concurrency.ScenarioActor open NBomber.Domain.Concurrency.Scheduler.ConstantActorScheduler @@ -37,6 +38,8 @@ let internal baseDep = { Scenario = baseScenario ScenarioStatsActor = ScenarioStatsActor(logger, baseScenario, Constants.DefaultReportingInterval) ExecStopCommand = fun _ -> () + GetStepOrder = Scenario.getStepOrder + ExecSteps = RunningStep.execSteps } [] diff --git a/tests/NBomber.IntegrationTests/Concurrency/OneTimeActorSchedulerTests.fs b/tests/NBomber.IntegrationTests/Concurrency/OneTimeActorSchedulerTests.fs index 29e16ac6..1076699f 100644 --- a/tests/NBomber.IntegrationTests/Concurrency/OneTimeActorSchedulerTests.fs +++ b/tests/NBomber.IntegrationTests/Concurrency/OneTimeActorSchedulerTests.fs @@ -15,6 +15,7 @@ open NBomber.Contracts open NBomber.FSharp open NBomber.Domain open NBomber.Domain.Stats.ScenarioStatsActor +open NBomber.Domain.Step open NBomber.Domain.Concurrency open NBomber.Domain.Concurrency.ScenarioActor open NBomber.Domain.Concurrency.Scheduler.OneTimeActorScheduler @@ -37,6 +38,8 @@ let internal baseDep = { Scenario = baseScenario ScenarioStatsActor = ScenarioStatsActor(logger, baseScenario, Constants.DefaultReportingInterval) ExecStopCommand = fun _ -> () + GetStepOrder = Scenario.getStepOrder + ExecSteps = RunningStep.execSteps } [] diff --git a/tests/NBomber.IntegrationTests/Configuration/step_order_config.json b/tests/NBomber.IntegrationTests/Configuration/step_order_config.json deleted file mode 100644 index 9b7a7d26..00000000 --- a/tests/NBomber.IntegrationTests/Configuration/step_order_config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "GlobalSettings": { - "ScenariosSettings": [ - { - "ScenarioName": "1", - "CustomStepOrder": ["step_1"] - } - ] - } -} diff --git a/tests/NBomber.IntegrationTests/Configuration/step_order_invalid_config.json b/tests/NBomber.IntegrationTests/Configuration/step_order_invalid_config.json deleted file mode 100644 index 884ce434..00000000 --- a/tests/NBomber.IntegrationTests/Configuration/step_order_invalid_config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "GlobalSettings": { - "ScenariosSettings": [ - { - "ScenarioName": "1", - "CustomStepOrder": ["step_not_found"] - } - ] - } -} diff --git a/tests/NBomber.IntegrationTests/CustomStepExecControlTests.fs b/tests/NBomber.IntegrationTests/CustomStepExecControlTests.fs deleted file mode 100644 index e8261f59..00000000 --- a/tests/NBomber.IntegrationTests/CustomStepExecControlTests.fs +++ /dev/null @@ -1,200 +0,0 @@ -module Tests.CustomStepExecControl - -open System -open System.Threading.Tasks - -open Xunit -open FsCheck.Xunit -open Swensen.Unquote -open FSharp.Control.Tasks.NonAffine -open Microsoft.Extensions.Configuration - -open NBomber -open NBomber.Extensions.InternalExtensions -open NBomber.Configuration -open NBomber.Contracts -open NBomber.Contracts.Stats -open NBomber.FSharp -open NBomber.Domain -open NBomber.Domain.DomainTypes - -[] -let ``should work correctly`` () = - - let mutable initialStepCount = 0 - let mutable step1Count = 0 - let mutable step2Count = 0 - - let step1 = Step.create("step_1", fun context -> task { - step1Count <- step1Count + 1 - do! Task.Delay(milliseconds 10) - return Response.ok() - }) - - let step2 = Step.create("step_2", fun context -> task { - step2Count <- step2Count + 1 - do! Task.Delay(milliseconds 10) - return Response.ok() - }) - - Scenario.create "scenario" [step1; step2] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(copies = 1, during = seconds 1)] - |> Scenario.withCustomStepExecControl(fun context -> - match context with - | ValueSome ctx -> - if ctx.PrevStepResponse.IsError then ValueNone - else ValueSome "step_1" - - | ValueNone -> - initialStepCount <- initialStepCount + 1 - ValueSome "step_1" - ) - |> NBomberRunner.registerScenario - |> NBomberRunner.withReportFolder "./steps-tests/11/" - |> NBomberRunner.run - |> ignore - - test <@ initialStepCount = 1 @> - test <@ step1Count > 0 @> - test <@ step2Count = 0 @> - -[] -let ``should restart execution when ValueNone returned`` () = - - let mutable counter = 0 - - let step1 = Step.create("step_1", fun context -> task { - do! Task.Delay(milliseconds 10) - return Response.ok() - }) - - let step2 = Step.create("step_2", fun context -> task { - do! Task.Delay(milliseconds 10) - return Response.ok() - }) - - Scenario.create "scenario" [step1; step2] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(copies = 1, during = seconds 1)] - |> Scenario.withCustomStepExecControl(fun context -> - match context with - | ValueSome c -> - counter <- counter + 1 - ValueSome c.PrevStepContext.StepName - - | ValueNone -> ValueNone - ) - |> NBomberRunner.registerScenario - |> NBomberRunner.withReportFolder "./steps-tests/11/" - |> NBomberRunner.run - |> ignore - - test <@ counter = 0 @> - -[] -let ``should restart execution when invalid step name returned`` () = - - let mutable counter = 0 - - let step1 = Step.create("step_1", fun context -> task { - do! Task.Delay(milliseconds 10) - return Response.ok() - }) - - let step2 = Step.create("step_2", fun context -> task { - do! Task.Delay(milliseconds 10) - return Response.ok() - }) - - Scenario.create "scenario" [step1; step2] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(copies = 1, during = seconds 1)] - |> Scenario.withCustomStepExecControl(fun context -> ValueSome "invalid_name") - |> NBomberRunner.registerScenario - |> NBomberRunner.withReportFolder "./steps-tests/11/" - |> NBomberRunner.run - |> ignore - - test <@ counter = 0 @> - -[] -let ``should correctly populate previous step response`` () = - - let mutable step1RespReceived = false - let mutable step2RespReceived = false - - let step1 = Step.create("step_1", fun context -> task { - do! Task.Delay(milliseconds 10) - return Response.ok(sizeBytes = 20) - }) - - let step2 = Step.create("step_2", fun context -> task { - do! Task.Delay(milliseconds 10) - return Response.fail(latencyMs = 10) - }) - - Scenario.create "scenario" [step1; step2] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(copies = 1, during = seconds 1)] - |> Scenario.withCustomStepExecControl(fun context -> - match context with - | ValueSome c when c.PrevStepContext.StepName = "step_1" && not c.PrevStepResponse.IsError && c.PrevStepResponse.SizeBytes = 20 -> - step1RespReceived <- true - ValueSome "step_2" - - | ValueSome c when c.PrevStepContext.StepName = "step_2" && c.PrevStepResponse.LatencyMs = 10 && c.PrevStepResponse.IsError -> - step2RespReceived <- true - ValueNone - - | ValueSome _ -> failwith "bug" - | ValueNone -> ValueSome "step_1" - ) - |> NBomberRunner.registerScenario - |> NBomberRunner.withReportFolder "./steps-tests/11/" - |> NBomberRunner.run - |> ignore - - test <@ step1RespReceived = true @> - test <@ step2RespReceived = true @> - -[] -let ``should allow jumping into step even in case of previous step error`` () = - - let mutable step3Invoked = false - - let step1 = Step.create("step_1", fun context -> task { - do! Task.Delay(milliseconds 10) - return Response.ok(sizeBytes = 20) - }) - - let step2 = Step.create("step_2", fun context -> task { - do! Task.Delay(milliseconds 10) - return Response.fail(latencyMs = 10) // by default, step_3 shouldn't be invoked - }) - - let step3 = Step.create("step_3", fun context -> task { - do! Task.Delay(milliseconds 10) - return Response.ok(latencyMs = 10) - }) - - Scenario.create "scenario" [step1; step2; step3] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(copies = 1, during = seconds 1)] - |> Scenario.withCustomStepExecControl(fun context -> - match context with - | ValueSome c when c.PrevStepContext.StepName = "step_1" -> ValueSome "step_2" - | ValueSome c when c.PrevStepContext.StepName = "step_2" -> ValueSome "step_3" - | ValueSome c when c.PrevStepContext.StepName = "step_3" -> - step3Invoked <- true - ValueNone - - | ValueSome _ -> failwith "bug" - | ValueNone -> ValueSome "step_1" - ) - |> NBomberRunner.registerScenario - |> NBomberRunner.withReportFolder "./steps-tests/11/" - |> NBomberRunner.run - |> ignore - - test <@ step3Invoked = true @> diff --git a/tests/NBomber.IntegrationTests/NBomber.IntegrationTests.fsproj b/tests/NBomber.IntegrationTests/NBomber.IntegrationTests.fsproj index 30f43cd8..0ea5e315 100644 --- a/tests/NBomber.IntegrationTests/NBomber.IntegrationTests.fsproj +++ b/tests/NBomber.IntegrationTests/NBomber.IntegrationTests.fsproj @@ -11,7 +11,6 @@ - PreserveNewest @@ -21,12 +20,6 @@ PreserveNewest - - PreserveNewest - - - PreserveNewest - PreserveNewest @@ -46,8 +39,6 @@ - - diff --git a/tests/NBomber.IntegrationTests/Plugins/PluginTests.fs b/tests/NBomber.IntegrationTests/Plugins/PluginTests.fs index 6b4c3bb8..902f06e5 100644 --- a/tests/NBomber.IntegrationTests/Plugins/PluginTests.fs +++ b/tests/NBomber.IntegrationTests/Plugins/PluginTests.fs @@ -4,11 +4,15 @@ open System open System.Data open System.Threading.Tasks -open FSharp.Control.Tasks.NonAffine +open Serilog +open Serilog.Sinks.InMemory +open Serilog.Sinks.InMemory.Assertions open Swensen.Unquote open Xunit +open FSharp.Control.Tasks.NonAffine open NBomber +open NBomber.Extensions.InternalExtensions open NBomber.Contracts open NBomber.Contracts.Stats open NBomber.Domain @@ -48,49 +52,6 @@ module internal PluginTestHelper = [scenario1; scenario2] -module internal PluginStatisticsHelper = - - let private getPluginStatisticsColumns (prefix: string) = - let colKey = new DataColumn("Key", Type.GetType("System.String")) - colKey.Caption <- sprintf "%sColumnKey" prefix - - let colValue = new DataColumn("Value", Type.GetType("System.String")) - colValue.Caption <- sprintf "%sColumnValue" prefix - - let colType = new DataColumn("Type", Type.GetType("System.String")) - colType.Caption <- sprintf "%sColumnType" prefix - - [| colKey; colValue; colType |] - - let private getPluginStatisticsRows (count: int) (prefix: string) (table: DataTable) = [| - for i in 1 .. count do - let row = table.NewRow() - row["Key"] <- sprintf "%sRowKey%i" prefix i - row["Value"] <- sprintf "%sRowValue%i" prefix i - row["Type"] <- sprintf "%sRowType%i" prefix i - yield row - |] - - let private createTable (prefix: string) = - let tableName = sprintf "%sTable" prefix - let table = new DataTable(tableName) - - prefix - |> getPluginStatisticsColumns - |> table.Columns.AddRange - - table - |> getPluginStatisticsRows 10 prefix - |> Array.iter(fun x -> x |> table.Rows.Add) - - table - - let createPluginStats () = - let pluginStats = new DataSet() - pluginStats.Tables.Add(createTable("PluginStatistics1")) - pluginStats.Tables.Add(createTable("PluginStatistics2")) - pluginStats - [] let ``Init should be invoked once`` () = @@ -240,52 +201,63 @@ let ``StopTest should be invoked once`` () = test <@ pluginFinishTestInvokedCounter = 1 @> [] -let ``stats should be passed to IReportingSink`` () = +let ``PluginStats should return empty data set in case of execution timeout`` () = + let inMemorySink = InMemorySink() + let loggerConfig = fun () -> LoggerConfiguration().WriteTo.Sink(inMemorySink) - let scenarios = PluginTestHelper.createScenarios() - let mutable _nodeStats = Array.empty - - let plugin = { + let timeoutPlugin = { new IWorkerPlugin with member _.PluginName = "TestPlugin" + member _.Init(_, _) = Task.CompletedTask member _.Start() = Task.CompletedTask - member _.GetStats(_) = PluginStatisticsHelper.createPluginStats() |> Task.FromResult + member _.GetStats(_) = task { + do! Task.Delay(seconds 10) // we waiting more than default timeout = 5 sec + return new DataSet() + } member _.GetHints() = Array.empty member _.Stop() = Task.CompletedTask member _.Dispose() = () } - let reportingSink = { - new IReportingSink with - member _.SinkName = "TestSink" - member _.Init(_, _) = Task.CompletedTask - member _.Start() = Task.CompletedTask - - member _.SaveRealtimeStats(_) = Task.CompletedTask + let step1 = Step.create("step 1", fun _ -> Task.FromResult(Response.ok())) + Scenario.create "1" [step1] + |> Scenario.withoutWarmUp + |> Scenario.withLoadSimulations [KeepConstant(1, seconds 10)] + |> NBomberRunner.registerScenario + |> NBomberRunner.withLoggerConfig loggerConfig + |> NBomberRunner.withWorkerPlugins [timeoutPlugin] + |> NBomberRunner.run + |> Result.getOk + |> fun nodeStats -> + test <@ Array.isEmpty nodeStats.PluginStats @> + inMemorySink.Should().HaveMessage("Getting plugin stats failed with the timeout error", "because timeout has been reached") |> ignore - member _.SaveFinalStats(stats) = - _nodeStats <- stats - Task.CompletedTask +[] +let ``PluginStats should return empty data set in case of internal exception`` () = + let inMemorySink = InMemorySink() + let loggerConfig = fun () -> LoggerConfiguration().WriteTo.Sink(inMemorySink) + let exceptionPlugin = { + new IWorkerPlugin with + member _.PluginName = "TestPlugin" + member _.Init(_, _) = Task.CompletedTask + member _.Start() = Task.CompletedTask + member _.GetStats(_) = failwith "test exception" // we throw exception + member _.GetHints() = Array.empty member _.Stop() = Task.CompletedTask member _.Dispose() = () } - NBomberRunner.registerScenarios scenarios - |> NBomberRunner.withReportingSinks [reportingSink] - |> NBomberRunner.withReportingInterval(seconds 10) - |> NBomberRunner.withWorkerPlugins [plugin] + let step1 = Step.create("step 1", fun _ -> Task.FromResult(Response.ok())) + Scenario.create "1" [step1] + |> Scenario.withoutWarmUp + |> Scenario.withLoadSimulations [KeepConstant(1, seconds 2)] + |> NBomberRunner.registerScenario + |> NBomberRunner.withLoggerConfig loggerConfig + |> NBomberRunner.withWorkerPlugins [exceptionPlugin] |> NBomberRunner.run - |> Result.mapError failwith - |> ignore - - let pluginStats = _nodeStats[0].PluginStats[0] - let table1 = pluginStats.Tables["PluginStatistics1Table"] - let table2 = pluginStats.Tables["PluginStatistics2Table"] - - // assert on IReportingSink - test <@ table1.Columns.Count > 0 @> - test <@ table1.Rows.Count > 0 @> - test <@ table2.Columns.Count > 0 @> - test <@ table2.Rows.Count > 0 @> + |> Result.getOk + |> fun nodeStats -> + test <@ Array.isEmpty nodeStats.PluginStats @> + inMemorySink.Should().HaveMessage("Getting plugin stats failed with the following error", "because exception was thrown") |> ignore diff --git a/tests/NBomber.IntegrationTests/PushResponseQueueTests.fs b/tests/NBomber.IntegrationTests/PushResponseQueueTests.fs deleted file mode 100644 index 0e7715ee..00000000 --- a/tests/NBomber.IntegrationTests/PushResponseQueueTests.fs +++ /dev/null @@ -1,90 +0,0 @@ -module Tests.PushExtensions - -open System -open System.Threading.Tasks -open Xunit -open Swensen.Unquote -open NBomber -open NBomber.Extensions.PushExtensions - -[] -let ``PushResponseQueue ReceiveResponse should return always a new task`` () = - use responseQueue = new PushResponseQueue() - - responseQueue.InitQueueForClient("id") - responseQueue.InitQueueForClient("new_id") - - let tsk1 = responseQueue.ReceiveResponse("id") - let tsk2 = responseQueue.ReceiveResponse("id") - let tsk3 = responseQueue.ReceiveResponse("new_id") - - let set = Set.ofSeq [tsk1.Id; tsk2.Id; tsk3.Id] - test <@ set.Count = 3 @> - -[] -let ``PushResponseQueue should handle waiting on response correctly`` () = - let clientId = "clientId" - let payload = "payload" - - use responseQueue = new PushResponseQueue() - responseQueue.InitQueueForClient(clientId) - - let task = responseQueue.ReceiveResponse(clientId) - Task.Delay(seconds 1).Wait() - - if not task.IsCompleted then - responseQueue.AddResponse(clientId, payload) - - let response = string task.Result.Payload - test <@ response = payload @> - -[] -let ``PushResponseQueue should queue responses (in order) if no client is found`` () = - let clientId = "clientId" - - use responseQueue = new PushResponseQueue() - responseQueue.InitQueueForClient(clientId) - - // register the time when requests were sent - let addingTime = DateTime.UtcNow - - // adding responses that should be buffered - [0..5] |> Seq.iter (fun payload -> responseQueue.AddResponse(clientId, payload)) - Task.Delay(seconds 1).Wait() - - [0..5] - |> Seq.map (fun _ -> responseQueue.ReceiveResponse(clientId)) - |> Task.WhenAll - |> fun allResponses -> - - [0..5] - |> Seq.iter (fun i -> - let payload = allResponses.Result[i].Payload - let receivedTime = allResponses.Result[i].ReceivedTime - let shiftedTime = addingTime.AddMilliseconds(100) - test <@ payload = i @> - test <@ receivedTime < shiftedTime @> - ) - -[] -let ``PushResponseQueue should handle concurrency without deadlocks`` () = - - use responseQueue = new PushResponseQueue() - - // we create only 5 clients - [0..4] - |> Seq.iter (fun clientId -> responseQueue.InitQueueForClient(string clientId)) - - // and create 300 threads that concurrently add/receive - Parallel.For(1, 300, fun clientId -> - let id = string (clientId % 4) - responseQueue.AddResponse(id, id) - responseQueue.AddResponse(id, id) - - let response1 = responseQueue.ReceiveResponse(id).Result - let response2 = responseQueue.ReceiveResponse(id).Result - - test <@ id = string response1.Payload @> - test <@ id = string response2.Payload @> - ) - |> ignore diff --git a/tests/NBomber.IntegrationTests/Reporting/ReportingSinkTests.fs b/tests/NBomber.IntegrationTests/Reporting/ReportingSinkTests.fs deleted file mode 100644 index 9a48caae..00000000 --- a/tests/NBomber.IntegrationTests/Reporting/ReportingSinkTests.fs +++ /dev/null @@ -1,300 +0,0 @@ -module Tests.ReportingSink - -open System.Data -open System.Threading.Tasks - -open Serilog -open Serilog.Sinks.InMemory -open Serilog.Sinks.InMemory.Assertions -open Swensen.Unquote -open Xunit -open FSharp.Control.Tasks.NonAffine - -open NBomber -open NBomber.Contracts -open NBomber.Contracts.Stats -open NBomber.Domain -open NBomber.FSharp -open NBomber.Extensions.InternalExtensions - -//todo: test that multiply sink will be invoked correctly -//todo: test that stop timer stops sending metrics in case when stopping is still executing -//todo: test cluster stats - -[] -let ``SaveRealtimeStats should receive correct stats`` () = - - let _realtimeStats = ResizeArray() - let mutable _finalStats = Unchecked.defaultof - - let reportingSink = { - new IReportingSink with - member _.SinkName = "TestSink" - member _.Init(_, _) = Task.CompletedTask - member _.Start() = Task.CompletedTask - - member _.SaveRealtimeStats(stats) = - _realtimeStats.AddRange(stats) - Task.CompletedTask - - member _.SaveFinalStats(stats) = Task.CompletedTask - member _.Stop() = Task.CompletedTask - member _.Dispose() = () - } - - let okStep = Step.create("ok step", fun _ -> task { - do! Task.Delay(milliseconds 100) - return Response.ok() - }) - - let scenario1 = - Scenario.create "scenario_1" [okStep] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(copies = 1, during = seconds 30)] - - NBomberRunner.registerScenarios [scenario1] - |> NBomberRunner.withReportFolder "./reporting-sinks/3/" - |> NBomberRunner.withReportingSinks [reportingSink] - |> NBomberRunner.withReportingInterval(seconds 5) - |> NBomberRunner.run - |> Result.getOk - |> fun finalStats -> - let realtime = _realtimeStats.ToArray() - - test <@ realtime.Length > 0 @> - test <@ realtime |> Array.forall(fun x -> x.CurrentOperation = OperationType.Bombing) @> - test <@ realtime |> Array.forall(fun x -> x.OkCount > 0) @> - test <@ realtime |> Array.forall(fun x -> x.StepStats[0].Ok.Request.Count > 0) @> - test <@ realtime |> Array.forall(fun x -> x.StepStats[0].Ok.Request.RPS > 0.0) @> - test <@ realtime |> Array.forall(fun x -> x.FailCount = 0) @> - test <@ realtime |> Array.forall(fun x -> x.StepStats[0].Fail.Request.Count = 0) @> - test <@ realtime |> Array.forall(fun x -> x.StepStats[0].Fail.Request.RPS = 0.0) @> - - test <@ finalStats.NodeInfo.CurrentOperation = OperationType.Complete @> - test <@ finalStats.ScenarioStats[0].StepStats[0].Ok.Request.Count > 0 @> - test <@ finalStats.ScenarioStats[0].StepStats[0].Ok.Request.RPS > 0.0 @> - test <@ finalStats.ScenarioStats[0].StepStats[0].Fail.Request.Count = 0 @> - test <@ finalStats.ScenarioStats[0].StepStats[0].Fail.Request.RPS = 0.0 @> - -[] -let ``SaveRealtimeStats should receive calculated stats by intervals`` () = - - let _realtimeStats = ResizeArray() - - let reportingSink = { - new IReportingSink with - member _.SinkName = "TestSink" - member _.Init(_, _) = Task.CompletedTask - member _.Start() = Task.CompletedTask - - member _.SaveRealtimeStats(stats) = - _realtimeStats.Add(stats) - Task.CompletedTask - - member _.SaveFinalStats(stats) = Task.CompletedTask - member _.Stop() = Task.CompletedTask - member _.Dispose() = () - } - - let mutable delay = seconds 1 - let mutable size = 1000 - - let okStep = Step.create("ok step", timeout = seconds 5, execute = fun context -> task { - do! Task.Delay delay - - if context.InvocationCount = 5 then - delay <- milliseconds 500 - size <- 500 - - if context.InvocationCount = 15 then - delay <- milliseconds 100 - size <- 100 - - return Response.ok(sizeBytes = size) - }) - - let scenario1 = - Scenario.create "scenario_1" [okStep] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(copies = 1, during = seconds 30)] - - NBomberRunner.registerScenarios [scenario1] - |> NBomberRunner.withReportFolder "./reporting-sinks/3/" - |> NBomberRunner.withReportingSinks [reportingSink] - |> NBomberRunner.withReportingInterval(seconds 5) - |> NBomberRunner.run - |> Result.getOk - |> fun nodeStats -> - - let first = _realtimeStats[0] - let last = _realtimeStats[_realtimeStats.Count - 1] - - test <@ first[0].StepStats[0].Ok.Latency.MaxMs > last[0].StepStats[0].Ok.Latency.MaxMs @> - test <@ first[0].StepStats[0].Ok.Latency.MaxMs >= 1000.0 @> - test <@ last[0].StepStats[0].Ok.Latency.MaxMs <= 1000.0 @> - - test <@ first[0].StepStats[0].Ok.Request.RPS <= 1.0 @> - test <@ last[0].StepStats[0].Ok.Request.RPS <= 20.0 && last[0].StepStats[0].Ok.Request.RPS >= 5.0 @> - - test <@ first[0].StepStats[0].Ok.DataTransfer.MaxBytes > last[0].StepStats[0].Ok.DataTransfer.MaxBytes @> - test <@ first[0].StepStats[0].Ok.DataTransfer.MaxBytes >= 1000 @> - test <@ last[0].StepStats[0].Ok.DataTransfer.MaxBytes <= 1000 @> - -[] -let ``SaveRealtimeStats should receive correct calculated stats for long running steps`` () = - - let _realtimeStats = ResizeArray() - - let reportingSink = { - new IReportingSink with - member _.SinkName = "TestSink" - member _.Init(_, _) = Task.CompletedTask - member _.Start() = Task.CompletedTask - - member _.SaveRealtimeStats(stats) = - _realtimeStats.Add(stats) - Task.CompletedTask - - member _.SaveFinalStats(stats) = Task.CompletedTask - member _.Stop() = Task.CompletedTask - member _.Dispose() = () - } - - let fastStep = Step.create("fast_step", timeout = seconds 30, execute = fun context -> task { - do! Task.Delay(seconds 1) - return Response.ok() - }) - - let longStep = Step.create("long_step", timeout = seconds 60, execute = fun context -> task { - do! Task.Delay(seconds 30) - return Response.ok() - }) - - let scenario1 = - Scenario.create "scenario_1" [fastStep; longStep] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(copies = 10, during = seconds 20)] - - NBomberRunner.registerScenarios [scenario1] - |> NBomberRunner.withReportFolder "./reporting-sinks/3/" - |> NBomberRunner.withReportingSinks [reportingSink] - |> NBomberRunner.withReportingInterval(seconds 5) - |> NBomberRunner.run - |> Result.getOk - |> fun nodeStats -> - let first = _realtimeStats.[0] - - test <@ first.[0].StepStats.[0].Ok.Request.Count = 10 @> - test <@ first.[0].StepStats.[0].Ok.Request.RPS = 2 @> - -[] -let ``SaveFinalStats should receive correct stats`` () = - - let _finalStats = ResizeArray() - - let reportingSink = { - new IReportingSink with - member _.SinkName = "TestSink" - member _.Init(_, _) = Task.CompletedTask - member _.Start() = Task.CompletedTask - member _.SaveRealtimeStats(stats) = Task.CompletedTask - - member _.SaveFinalStats(stats) = - _finalStats.AddRange(stats) - Task.CompletedTask - - member _.Stop() = Task.CompletedTask - member _.Dispose() = () - } - - let okStep = Step.create("ok step", fun _ -> task { - do! Task.Delay(milliseconds 100) - return Response.ok() - }) - - let scenario1 = - Scenario.create "scenario_1" [okStep] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(copies = 1, during = seconds 30)] - - NBomberRunner.registerScenarios [scenario1] - |> NBomberRunner.withReportFolder "./reporting-sinks/3/" - |> NBomberRunner.withReportingSinks [reportingSink] - |> NBomberRunner.withReportingInterval(seconds 5) - |> NBomberRunner.run - |> Result.getOk - |> fun nodeStats -> - let final = _finalStats.ToArray() - test <@ final[0] = nodeStats @> - test <@ nodeStats.Duration = seconds 30 @> - test <@ nodeStats.ScenarioStats |> Array.filter(fun x -> x.CurrentOperation = OperationType.Complete) |> Array.length = 1 @> - - test <@ nodeStats.ScenarioStats |> Array.forall(fun x -> x.OkCount > 0) @> - test <@ nodeStats.ScenarioStats |> Array.forall(fun x -> x.StepStats[0].Ok.Request.Count > 0) @> - test <@ nodeStats.ScenarioStats |> Array.forall(fun x -> x.StepStats[0].Ok.Request.RPS > 0.0) @> - - test <@ nodeStats.ScenarioStats |> Array.forall(fun x -> x.FailCount = 0) @> - test <@ nodeStats.ScenarioStats |> Array.forall(fun x -> x.StepStats[0].Fail.Request.Count = 0) @> - test <@ nodeStats.ScenarioStats |> Array.forall(fun x -> x.StepStats[0].Fail.Request.RPS = 0.0) @> - -[] -let ``PluginStats should return empty data set in case of execution timeout`` () = - let inMemorySink = InMemorySink() - let loggerConfig = fun () -> LoggerConfiguration().WriteTo.Sink(inMemorySink) - - let timeoutPlugin = { - new IWorkerPlugin with - member _.PluginName = "TestPlugin" - - member _.Init(_, _) = Task.CompletedTask - member _.Start() = Task.CompletedTask - member _.GetStats(_) = task { - do! Task.Delay(seconds 10) // we waiting more than default timeout = 5 sec - return new DataSet() - } - member _.GetHints() = Array.empty - member _.Stop() = Task.CompletedTask - member _.Dispose() = () - } - - let step1 = Step.create("step 1", fun _ -> Task.FromResult(Response.ok())) - Scenario.create "1" [step1] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(1, seconds 10)] - |> NBomberRunner.registerScenario - |> NBomberRunner.withLoggerConfig loggerConfig - |> NBomberRunner.withWorkerPlugins [timeoutPlugin] - |> NBomberRunner.run - |> Result.getOk - |> fun nodeStats -> - test <@ Array.isEmpty nodeStats.PluginStats @> - inMemorySink.Should().HaveMessage("Getting plugin stats failed with the timeout error", "because timeout has been reached") |> ignore - -[] -let ``PluginStats should return empty data set in case of internal exception`` () = - let inMemorySink = InMemorySink() - let loggerConfig = fun () -> LoggerConfiguration().WriteTo.Sink(inMemorySink) - - let exceptionPlugin = { - new IWorkerPlugin with - member _.PluginName = "TestPlugin" - member _.Init(_, _) = Task.CompletedTask - member _.Start() = Task.CompletedTask - member _.GetStats(_) = failwith "test exception" // we throw exception - member _.GetHints() = Array.empty - member _.Stop() = Task.CompletedTask - member _.Dispose() = () - } - - let step1 = Step.create("step 1", fun _ -> Task.FromResult(Response.ok())) - Scenario.create "1" [step1] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(1, seconds 2)] - |> NBomberRunner.registerScenario - |> NBomberRunner.withLoggerConfig loggerConfig - |> NBomberRunner.withWorkerPlugins [exceptionPlugin] - |> NBomberRunner.run - |> Result.getOk - |> fun nodeStats -> - test <@ Array.isEmpty nodeStats.PluginStats @> - inMemorySink.Should().HaveMessage("Getting plugin stats failed with the following error", "because exception was thrown") |> ignore diff --git a/tests/NBomber.IntegrationTests/ScenarioTests.fs b/tests/NBomber.IntegrationTests/ScenarioTests.fs index b2fb4442..22ec5455 100644 --- a/tests/NBomber.IntegrationTests/ScenarioTests.fs +++ b/tests/NBomber.IntegrationTests/ScenarioTests.fs @@ -341,82 +341,6 @@ let ``check that scenario should be ok if it has no steps but init function exis |> Result.getOk |> ignore -[] -let ``withCustomStepOrder should allow to run steps with custom order`` () = - - let step1 = Step.create("step_1", fun context -> task { - do! Task.Delay(milliseconds 10) - return Response.ok() - }) - - let step2 = Step.create("step_2", fun context -> task { - do! Task.Delay(milliseconds 10) - return Response.ok() - }) - - Scenario.create "1" [step1; step2] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(1, seconds 2)] - |> Scenario.withCustomStepOrder(fun () -> [| "step_2" |]) - |> NBomberRunner.registerScenario - |> NBomberRunner.run - |> Result.getOk - |> fun stats -> - let stepsStats = stats.GetScenarioStats("1").StepStats - test <@ stepsStats[0].Ok.Request.Count = 0 @> - test <@ stepsStats[1].Ok.Request.Count > 0 @> - -[] -let ``CustomStepOrder should be supported via config.json `` () = - - let step1 = Step.create("step_1", fun context -> task { - do! Task.Delay(milliseconds 10) - return Response.ok() - }) - - let step2 = Step.create("step_2", fun context -> task { - do! Task.Delay(milliseconds 10) - return Response.ok() - }) - - Scenario.create "1" [step1; step2] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(1, seconds 2)] - |> Scenario.withCustomStepOrder(fun () -> [| "step_2" |]) - |> NBomberRunner.registerScenario - |> NBomberRunner.loadConfig "Configuration/step_order_config.json" // "StepOrder": [step_1] - |> NBomberRunner.run - |> Result.getOk - |> fun nodeStats -> - - let stepsStats = - nodeStats.ScenarioStats[0].StepStats - |> Seq.filter(fun s -> s.Ok.Request.Count > 0 || s.Fail.Request.Count > 0) - - test <@ stepsStats |> Seq.forall(fun s -> s.StepName = "step_1") @> - -[] -let ``CustomStepOrderSettings should be validated `` () = - - let step1 = Step.create("step_1", fun context -> task { - do! Task.Delay(milliseconds 10) - return Response.ok() - }) - - let step2 = Step.create("step_2", fun context -> task { - do! Task.Delay(milliseconds 10) - return Response.ok() - }) - - Scenario.create "1" [step1; step2] - |> Scenario.withoutWarmUp - |> Scenario.withLoadSimulations [KeepConstant(1, seconds 2)] - |> NBomberRunner.registerScenario - |> NBomberRunner.loadConfig "Configuration/step_order_invalid_config.json" // "StepOrder": [step_not_found] - |> NBomberRunner.run - |> Result.getError - |> fun error -> test <@ error.Contains("Scenario: '1' contains not found step: 'step_not_found'") @> - [] let ``ScenarioSettings should be validated on duplicates `` () =