diff --git a/cmd/ie/commands/test.go b/cmd/ie/commands/test.go index a8e1da5d..88810e77 100644 --- a/cmd/ie/commands/test.go +++ b/cmd/ie/commands/test.go @@ -20,6 +20,8 @@ func init() { String("subscription", "", "Sets the subscription ID used by a scenarios azure-cli commands. Will rely on the default subscription if not set.") testCommand.PersistentFlags(). String("working-directory", ".", "Sets the working directory for innovation engine to operate out of. Restores the current working directory when finished.") + testCommand.PersistentFlags(). + String("report", "", "The path to generate a report of the scenario execution. The contents of the report are in JSON and will only be generated when this flag is set.") testCommand.PersistentFlags(). StringArray("var", []string{}, "Sets an environment variable for the scenario. Format: --var =") @@ -40,6 +42,7 @@ var testCommand = &cobra.Command{ subscription, _ := cmd.Flags().GetString("subscription") workingDirectory, _ := cmd.Flags().GetString("working-directory") environment, _ := cmd.Flags().GetString("environment") + generateReport, _ := cmd.Flags().GetString("report") environmentVariables, _ := cmd.Flags().GetStringArray("var") @@ -67,6 +70,7 @@ var testCommand = &cobra.Command{ CorrelationId: "", WorkingDirectory: workingDirectory, Environment: environment, + ReportFile: generateReport, }) if err != nil { logging.GlobalLogger.Errorf("Error creating engine %s", err) diff --git a/docs/specs/test-reporting.md b/docs/specs/test-reporting.md new file mode 100644 index 00000000..98a59c87 --- /dev/null +++ b/docs/specs/test-reporting.md @@ -0,0 +1,216 @@ +# Test Reports + +## Summary + +When users are testing their executable documentation using `ie test`, being +able to see the results of their execution is important, especially in instances +where the test fails. At the moment, there are only two ways to see what +happened during the test execution: + +1. The user can look through the standard output from `ie test ` + (Most common). +1. The user can look through `ie.log` (Not as common). + +While these methods are effective for troubleshooting most issues, they also have +a few issues: + +1. Storing the output of `ie test` in a file doesn't provide a good way to + navigate the output and see the results of the test. +1. The log file `ie.log` is not user-friendly and can be difficult to navigate, especially + so if the user is invoking multiple `ie test` commands as the log file is cumulative. +1. It's not easy to reproduce the execution of a specific scenario, as most of + the variables declared by the scenario are randomized and don't have their + values rendered in the output. +1. The output of `ie test` is not easily shareable with others. + +To address these issues, we propose the introduction of test reports, a feature +for `ie test` that will generate a report of the scenario execution in JSON +format so that users can easily navigate the results of the test, reproduce +specific runs, and share the results with others. + +## Requirements + +- [x] The user can generate a test report by running `ie test //report=` +- [x] Reports capture the yaml metadata of the scenario. +- [x] Reports store the variables declared in the scenario and their values. +- [x] The report is generated in JSON format. +- [ ] Just like the scenarios that generated them, Reports are executable. +- [x] Outputs of the codeblocks executed are stored in the report. +- [x] Expected outputs for codeblocks are stored in the report. + +## Technical specifications + +- The report will be generated in JSON format, but in the future we may consider + other formats like yaml or HTML. JSON format was chosen for v1 because it is + easy to parse, and it is a common format for sharing data. +- Users must specify `//report=` to generate a report. If the path is not + specified, the report will not be generated. + +### Report schema + +The actual JSON schema is a work in progress, and will not be released with +the initial implementation, so we will list out the actual JSON with +documentation about each field until then. + +```json +{ + // Name of the scenario + "name": "Test reporting doc", + // Properties found in the yaml header + "properties": { + "ms.author": "vmarcella", + "otherProperty": "otherValue" + }, + + // Variables declared in the scenario + "environmentVariables": { + "NEW_VAR": "1" + }, + // Whether the test was successful or not + "success": true, + // Error message if the test failed + "error": "", + // The step number where the test failed (-1 if successful) + "failedAtStep": -1, + "steps": [ + // The entire step + { + // The codeblock for the step + "codeBlock": { + // The language of the codeblock + "language": "bash", + // The content of the codeblock + "content": "echo \"Hello, world!\"\n", + // The header paired with the codeblock + "header": "First step", + // The paragraph paired with the codeblock + "description": "This step will show you how to do something.", + // The expected output for the codeblock + "resultBlock": { + // The language of the expected output + "language": "text", + // The content of the expected output + "content": "Hello, world!\n", + // The expected similarity score of the output (between 0 - 1) + "expectedSimilarityScore": 1, + // The expected regex pattern of the output + "expectedRegexPattern": null + } + }, + // Codeblock number underneath the step (Should be ignored for now) + "codeBlockNumber": 0, + // Error message if the step failed (Would be same as top level error) + "error": null, + // Standard error output from executing the step + "stdErr": "", + // Standard output from executing the step + "stdOut": "Hello, world!\n", + // The name of the step + "stepName": "First step", + // The step number + "stepNumber": 0, + // Whether the step was successful or not + "success": true, + // The computed similarity score of the output (between 0 - 1) + "similarityScore": 0 + }, + { + "codeBlock": { + "language": "bash", + "content": "export NEW_VAR=1\n", + "header": "Second step", + "description": "This step will show you how to do something else.", + "resultBlock": { + "language": "", + "content": "", + "expectedSimilarityScore": 0, + "expectedRegexPattern": null + } + }, + "codeBlockNumber": 0, + "error": null, + "stdErr": "", + "stdOut": "", + "stepName": "Second step", + "stepNumber": 1, + "success": true, + "similarityScore": 0 + } + ] +} +``` + +## Examples + +Assuming you're running this command from the root of the repository: + +```bash +ie test scenarios/testing/reporting.md --report=report.json >/dev/null && cat report.json +``` + +The output of the command above should look like this: + + + + +```json +{ + "name": "Test reporting doc", + "properties": { + "ms.author": "vmarcella", + "otherProperty": "otherValue" + }, + "environmentVariables": { + "NEW_VAR": "1" + }, + "success": true, + "error": "", + "failedAtStep": -1, + "steps": [ + { + "codeBlock": { + "language": "bash", + "content": "echo \"Hello, world!\"\n", + "header": "First step", + "description": "This step will show you how to do something.", + "resultBlock": { + "language": "text", + "content": "Hello, world!\n", + "expectedSimilarityScore": 1, + "expectedRegexPattern": null + } + }, + "codeBlockNumber": 0, + "error": null, + "stdErr": "", + "stdOut": "Hello, world!\n", + "stepName": "First step", + "stepNumber": 0, + "success": true, + "similarityScore": 1 + }, + { + "codeBlock": { + "language": "bash", + "content": "export NEW_VAR=1\n", + "header": "Second step", + "description": "This step will show you how to do something else.", + "resultBlock": { + "language": "", + "content": "", + "expectedSimilarityScore": 0, + "expectedRegexPattern": null + } + }, + "codeBlockNumber": 0, + "error": null, + "stdErr": "", + "stdOut": "", + "stepName": "Second step", + "stepNumber": 1, + "success": true, + "similarityScore": 1 + } + ] +} +``` diff --git a/internal/engine/common/codeblock.go b/internal/engine/common/codeblock.go index b1340436..89360233 100644 --- a/internal/engine/common/codeblock.go +++ b/internal/engine/common/codeblock.go @@ -5,14 +5,15 @@ import "github.com/Azure/InnovationEngine/internal/parsers" // State for the codeblock in interactive mode. Used to keep track of the // state of each codeblock. type StatefulCodeBlock struct { - CodeBlock parsers.CodeBlock - CodeBlockNumber int - Error error - StdErr string - StdOut string - StepName string - StepNumber int - Success bool + CodeBlock parsers.CodeBlock `json:"codeBlock"` + CodeBlockNumber int `json:"codeBlockNumber"` + Error error `json:"error"` + StdErr string `json:"stdErr"` + StdOut string `json:"stdOut"` + StepName string `json:"stepName"` + StepNumber int `json:"stepNumber"` + Success bool `json:"success"` + SimilarityScore float64 `json:"similarityScore"` } // Checks if a codeblock was executed by looking at the diff --git a/internal/engine/common/commands.go b/internal/engine/common/commands.go index 2e550883..ef2784cb 100644 --- a/internal/engine/common/commands.go +++ b/internal/engine/common/commands.go @@ -12,15 +12,17 @@ import ( // Emitted when a command has been executed successfully. type SuccessfulCommandMessage struct { - StdOut string - StdErr string + StdOut string + StdErr string + SimilarityScore float64 } // Emitted when a command has failed to execute. type FailedCommandMessage struct { - StdOut string - StdErr string - Error error + StdOut string + StdErr string + Error error + SimilarityScore float64 } type ExitMessage struct { @@ -49,9 +51,10 @@ func ExecuteCodeBlockAsync(codeBlock parsers.CodeBlock, env map[string]string) t if err != nil { logging.GlobalLogger.Errorf("Error executing command:\n %s", err.Error()) return FailedCommandMessage{ - StdOut: output.StdOut, - StdErr: output.StdErr, - Error: err, + StdOut: output.StdOut, + StdErr: output.StdErr, + Error: err, + SimilarityScore: 0, } } @@ -62,7 +65,7 @@ func ExecuteCodeBlockAsync(codeBlock parsers.CodeBlock, env map[string]string) t expectedRegex := codeBlock.ExpectedOutput.ExpectedRegex expectedOutputLanguage := codeBlock.ExpectedOutput.Language - outputComparisonError := CompareCommandOutputs( + score, outputComparisonError := CompareCommandOutputs( actualOutput, expectedOutput, expectedSimilarity, @@ -77,17 +80,19 @@ func ExecuteCodeBlockAsync(codeBlock parsers.CodeBlock, env map[string]string) t ) return FailedCommandMessage{ - StdOut: output.StdOut, - StdErr: output.StdErr, - Error: outputComparisonError, + StdOut: output.StdOut, + StdErr: output.StdErr, + Error: outputComparisonError, + SimilarityScore: score, } } logging.GlobalLogger.Infof("Command output to stdout:\n %s", output.StdOut) return SuccessfulCommandMessage{ - StdOut: output.StdOut, - StdErr: output.StdErr, + StdOut: output.StdOut, + StdErr: output.StdErr, + SimilarityScore: score, } } } diff --git a/internal/engine/common/outputs.go b/internal/engine/common/outputs.go index 22ea96a0..64badc89 100644 --- a/internal/engine/common/outputs.go +++ b/internal/engine/common/outputs.go @@ -18,17 +18,17 @@ func CompareCommandOutputs( expectedSimilarity float64, expectedRegex *regexp.Regexp, expectedOutputLanguage string, -) error { +) (float64, error) { if expectedRegex != nil { if !expectedRegex.MatchString(actualOutput) { - return fmt.Errorf( + return 0.0, fmt.Errorf( ui.ErrorMessageStyle.Render( fmt.Sprintf("Expected output does not match: %q.", expectedRegex), ), ) } - return nil + return 0.0, nil } if strings.ToLower(expectedOutputLanguage) == "json" { @@ -39,7 +39,7 @@ func CompareCommandOutputs( ) results, err := lib.CompareJsonStrings(actualOutput, expectedOutput, expectedSimilarity) if err != nil { - return err + return results.Score, err } logging.GlobalLogger.Debugf( @@ -49,24 +49,27 @@ func CompareCommandOutputs( ) if !results.AboveThreshold { - return fmt.Errorf( + return results.Score, fmt.Errorf( ui.ErrorMessageStyle.Render( - "Expected output does not match actual output. Got: %s\n Expected: %s"), - actualOutput, - expectedOutput, + "Expected output does not match actual output.\nGot:\n%s\nExpected:\n%s\nExpected Score:%s\nActual Score:%s", + ), + ui.VerboseStyle.Render(actualOutput), + ui.VerboseStyle.Render(expectedOutput), + ui.VerboseStyle.Render(fmt.Sprintf("%f", expectedSimilarity)), + ui.VerboseStyle.Render(fmt.Sprintf("%f", results.Score)), ) } - return nil + return results.Score, nil } // Default case, using similarity on non JSON block. score := smetrics.JaroWinkler(expectedOutput, actualOutput, 0.7, 4) if expectedSimilarity > score { - return fmt.Errorf( + return score, fmt.Errorf( ui.ErrorMessageStyle.Render( - "Expected output does not match actual output.\nGot:\n%s\nExpected:\n%s\nExpected Score:%s\nActualScore:%s", + "Expected output does not match actual output.\nGot:\n%s\nExpected:\n%s\nExpected Score:%s\nActual Score:%s", ), ui.VerboseStyle.Render(actualOutput), ui.VerboseStyle.Render(expectedOutput), @@ -75,5 +78,5 @@ func CompareCommandOutputs( ) } - return nil + return score, nil } diff --git a/internal/engine/common/reports.go b/internal/engine/common/reports.go new file mode 100644 index 00000000..3b9befe3 --- /dev/null +++ b/internal/engine/common/reports.go @@ -0,0 +1,79 @@ +package common + +import ( + "encoding/json" + "os" + + "github.com/Azure/InnovationEngine/internal/logging" +) + +type Report struct { + Name string `json:"name"` + Properties map[string]interface{} `json:"properties"` + EnvironmentVariables map[string]string `json:"environmentVariables"` + Success bool `json:"success"` + Error string `json:"error"` + FailedAtStep int `json:"failedAtStep"` + CodeBlocks []StatefulCodeBlock `json:"steps"` +} + +func (report *Report) WithProperties(properties map[string]interface{}) *Report { + report.Properties = properties + return report +} + +func (report *Report) WithEnvironmentVariables(envVars map[string]string) *Report { + report.EnvironmentVariables = envVars + return report +} + +func (report *Report) WithCodeBlocks(codeBlocks []StatefulCodeBlock) *Report { + report.CodeBlocks = codeBlocks + return report +} + +func (report *Report) WithError(err error) *Report { + if err == nil { + return report + } + + report.Error = err.Error() + report.Success = false + return report +} + +// TODO(vmarcella): Implement this to write the report to JSON. +func (report *Report) WriteToJSONFile(outputPath string) error { + jsonReport, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + logging.GlobalLogger.Infof("Generated the test report:\n %s", jsonReport) + + file, err := os.Create(outputPath) + if err != nil { + return err + } + + defer file.Close() + + _, err = file.Write(jsonReport) + if err != nil { + return err + } + + logging.GlobalLogger.Infof("Wrote the test report to %s", outputPath) + + return nil +} + +func BuildReport(name string) Report { + return Report{ + Name: name, + Properties: make(map[string]interface{}), + EnvironmentVariables: make(map[string]string), + Success: true, + Error: "", + FailedAtStep: -1, + } +} diff --git a/internal/engine/common/scenario.go b/internal/engine/common/scenario.go index c349756b..d51e3344 100644 --- a/internal/engine/common/scenario.go +++ b/internal/engine/common/scenario.go @@ -27,6 +27,7 @@ type Scenario struct { Name string MarkdownAst ast.Node Steps []Step + Properties map[string]interface{} Environment map[string]string } @@ -116,6 +117,7 @@ func CreateScenarioFromMarkdown( // Convert the markdonw into an AST and extract the scenario variables. markdown := parsers.ParseMarkdownIntoAst(source) + properties := parsers.ExtractYamlMetadataFromAst(markdown) scenarioVariables := parsers.ExtractScenarioVariablesFromAst(markdown, source) for key, value := range scenarioVariables { environmentVariables[key] = value @@ -202,6 +204,7 @@ func CreateScenarioFromMarkdown( Name: title, Environment: environmentVariables, Steps: steps, + Properties: properties, MarkdownAst: markdown, }, nil } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index a376fc13..bdf918c0 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -14,7 +14,6 @@ import ( "github.com/Azure/InnovationEngine/internal/lib" "github.com/Azure/InnovationEngine/internal/lib/fs" "github.com/Azure/InnovationEngine/internal/logging" - "github.com/Azure/InnovationEngine/internal/shells" "github.com/Azure/InnovationEngine/internal/ui" tea "github.com/charmbracelet/bubbletea" ) @@ -28,6 +27,7 @@ type EngineConfiguration struct { Environment string WorkingDirectory string RenderValues bool + ReportFile string } type Engine struct { @@ -60,6 +60,8 @@ func (e *Engine) TestScenario(scenario *common.Scenario) error { az.SetCorrelationId(e.Configuration.CorrelationId, scenario.Environment) stepsToExecute := filterDeletionCommands(scenario.Steps, e.Configuration.DoNotDelete) + initialEnvironmentVariables := lib.GetEnvironmentVariables() + model, err := test.NewTestModeModel( scenario.Name, e.Configuration.Subscription, @@ -91,17 +93,52 @@ func (e *Engine) TestScenario(scenario *common.Scenario) error { // TODO(vmarcella): After testing is complete, we should generate a report. model, ok := finalModel.(test.TestModeModel) - if !ok { err = errors.Join(err, fmt.Errorf("failed to cast tea.Model to TestModeModel")) return err + } + + if e.Configuration.ReportFile != "" { + allEnvironmentVariables, envErr := lib.LoadEnvironmentStateFile( + lib.DefaultEnvironmentStateFile, + ) + if envErr != nil { + logging.GlobalLogger.Errorf("Failed to load environment state file: %s", err) + err = errors.Join(err, fmt.Errorf("failed to load environment state file: %s", err)) + return err + } + variablesDeclaredByScenario := lib.DiffMapsByKey( + allEnvironmentVariables, + initialEnvironmentVariables, + ) + + report := common.BuildReport(scenario.Name) + err = report. + WithProperties(scenario.Properties). + WithEnvironmentVariables(variablesDeclaredByScenario). + WithError(model.GetFailure()). + WithCodeBlocks(model.GetCodeBlocks()). + WriteToJSONFile(e.Configuration.ReportFile) + if err != nil { + err = errors.Join(err, fmt.Errorf("failed to write report to file: %s", err)) + return err + } + + model.CommandLines = append( + model.CommandLines, + "Report written to "+e.Configuration.ReportFile, + ) } + err = errors.Join(err, model.GetFailure()) + if err != nil { + logging.GlobalLogger.Errorf("Failed to run ie test %s", err) + } fmt.Println(strings.Join(model.CommandLines, "\n")) - return err + return nil }) } @@ -146,14 +183,14 @@ func (e *Engine) InteractWithScenario(scenario *common.Scenario) error { logging.GlobalLogger.Info( "Cleaning environment variable file located at /tmp/env-vars", ) - err := shells.CleanEnvironmentStateFile() + err := lib.CleanEnvironmentStateFile(lib.DefaultEnvironmentStateFile) if err != nil { logging.GlobalLogger.Errorf("Error cleaning environment variables: %s", err.Error()) return err } default: - shells.ResetStoredEnvironmentVariables() + lib.DeleteEnvironmentStateFile(lib.DefaultEnvironmentStateFile) } if err != nil { diff --git a/internal/engine/execution.go b/internal/engine/execution.go index 7a07ab68..283a4d53 100644 --- a/internal/engine/execution.go +++ b/internal/engine/execution.go @@ -183,7 +183,7 @@ func (e *Engine) ExecuteAndRenderSteps(steps []common.Step, env map[string]strin expectedRegex := block.ExpectedOutput.ExpectedRegex expectedOutputLanguage := block.ExpectedOutput.Language - outputComparisonError := common.CompareCommandOutputs(actualOutput, expectedOutput, expectedSimilarity, expectedRegex, expectedOutputLanguage) + _, outputComparisonError := common.CompareCommandOutputs(actualOutput, expectedOutput, expectedSimilarity, expectedRegex, expectedOutputLanguage) if outputComparisonError != nil { logging.GlobalLogger.Errorf("Error comparing command outputs: %s", outputComparisonError.Error()) @@ -310,14 +310,14 @@ func (e *Engine) ExecuteAndRenderSteps(steps []common.Step, env map[string]strin logging.GlobalLogger.Info( "Cleaning environment variable file located at /tmp/env-vars", ) - err := shells.CleanEnvironmentStateFile() + err := lib.CleanEnvironmentStateFile(lib.DefaultEnvironmentStateFile) if err != nil { logging.GlobalLogger.Errorf("Error cleaning environment variables: %s", err.Error()) return err } default: - shells.ResetStoredEnvironmentVariables() + lib.DeleteEnvironmentStateFile(lib.DefaultEnvironmentStateFile) } return nil diff --git a/internal/engine/test/model.go b/internal/engine/test/model.go index 57b97183..f1621534 100644 --- a/internal/engine/test/model.go +++ b/internal/engine/test/model.go @@ -54,6 +54,32 @@ func (model TestModeModel) GetFailure() error { ) } +func (model TestModeModel) GetScenarioTitle() string { + return model.scenarioTitle +} + +// Get the environment that the scenario is running in. +func (model TestModeModel) GetEnvironment() string { + return model.environment +} + +// Get the code blocks that were executed in the scenario. +func (model TestModeModel) GetCodeBlocks() []common.StatefulCodeBlock { + var codeBlocks []common.StatefulCodeBlock + for _, codeBlock := range model.codeBlockState { + codeBlocks = append(codeBlocks, codeBlock) + } + return codeBlocks +} + +func (model TestModeModel) GetEnvironmentVariables() map[string]string { + return model.environmentVariables +} + +func (model TestModeModel) GetDeclaredEnvironmentVariables() map[string]string { + return model.environmentVariables +} + // Init the test mode model by executing the first code block. func (model TestModeModel) Init() tea.Cmd { return common.ExecuteCodeBlockAsync( @@ -91,6 +117,7 @@ func (model TestModeModel) Update(message tea.Msg) (tea.Model, tea.Cmd) { codeBlockState.StdOut = message.StdOut codeBlockState.StdErr = message.StdErr codeBlockState.Success = true + codeBlockState.SimilarityScore = message.SimilarityScore model.codeBlockState[step] = codeBlockState logging.GlobalLogger.Infof("Finished executing:\n %s", codeBlockState.CodeBlock.Content) @@ -148,6 +175,7 @@ func (model TestModeModel) Update(message tea.Msg) (tea.Model, tea.Cmd) { codeBlockState.StdErr = message.StdErr codeBlockState.Error = message.Error codeBlockState.Success = false + codeBlockState.SimilarityScore = message.SimilarityScore model.codeBlockState[step] = codeBlockState model.CommandLines = append(model.CommandLines, codeBlockState.StdErr+message.Error.Error()) diff --git a/internal/engine/testing.go b/internal/engine/testing.go deleted file mode 100644 index 3c46c388..00000000 --- a/internal/engine/testing.go +++ /dev/null @@ -1,132 +0,0 @@ -package engine - -import ( - "errors" - "fmt" - "time" - - "github.com/Azure/InnovationEngine/internal/az" - "github.com/Azure/InnovationEngine/internal/engine/common" - "github.com/Azure/InnovationEngine/internal/lib" - "github.com/Azure/InnovationEngine/internal/logging" - "github.com/Azure/InnovationEngine/internal/parsers" - "github.com/Azure/InnovationEngine/internal/patterns" - "github.com/Azure/InnovationEngine/internal/shells" - "github.com/Azure/InnovationEngine/internal/terminal" - "github.com/Azure/InnovationEngine/internal/ui" -) - -func (e *Engine) TestSteps(steps []common.Step, env map[string]string) error { - var resourceGroupName string - stepsToExecute := filterDeletionCommands(steps, true) - err := az.SetSubscription(e.Configuration.Subscription) - if err != nil { - logging.GlobalLogger.Errorf("Invalid Config: Failed to set subscription: %s", err) - return err - } - - var testRunnerError error = nil -testRunner: - for stepNumber, step := range stepsToExecute { - stepTitle := fmt.Sprintf(" %d. %s\n", stepNumber+1, step.Name) - fmt.Println(ui.StepTitleStyle.Render(stepTitle)) - terminal.MoveCursorPositionUp(1) - terminal.HideCursor() - - for _, block := range step.CodeBlocks { - // execute the command as a goroutine to allow for the spinner to be - // rendered while the command is executing. - done := make(chan error) - var commandOutput shells.CommandOutput - go func(block parsers.CodeBlock) { - logging.GlobalLogger.Infof("Executing command: %s", block.Content) - output, err := shells.ExecuteBashCommand(block.Content, shells.BashCommandConfiguration{EnvironmentVariables: lib.CopyMap(env), InheritEnvironment: true, InteractiveCommand: false, WriteToHistory: true}) - logging.GlobalLogger.Infof("Command stdout: %s", output.StdOut) - logging.GlobalLogger.Infof("Command stderr: %s", output.StdErr) - commandOutput = output - done <- err - }(block) - - frame := 0 - var err error - - loop: - // While the command is executing, render the spinner. - for { - select { - case err = <-done: - terminal.ShowCursor() - - if err == nil { - actualOutput := commandOutput.StdOut - expectedOutput := block.ExpectedOutput.Content - expectedSimilarity := block.ExpectedOutput.ExpectedSimilarity - expectedRegex := block.ExpectedOutput.ExpectedRegex - expectedOutputLanguage := block.ExpectedOutput.Language - - err := common.CompareCommandOutputs(actualOutput, expectedOutput, expectedSimilarity, expectedRegex, expectedOutputLanguage) - if err != nil { - logging.GlobalLogger.Errorf("Error comparing command outputs: %s", err.Error()) - fmt.Print(ui.ErrorStyle.Render("Error when comparing the command outputs: %s\n", err.Error())) - } - - // Extract the resource group name from the command output if - // it's not already set. - if resourceGroupName == "" && patterns.AzCommand.MatchString(block.Content) { - tmpResourceGroup := az.FindResourceGroupName(commandOutput.StdOut) - if tmpResourceGroup != "" { - logging.GlobalLogger.Infof("Found resource group: %s", tmpResourceGroup) - resourceGroupName = tmpResourceGroup - } - } - - fmt.Printf("\r %s \n", ui.CheckStyle.Render("✔")) - terminal.MoveCursorPositionDown(1) - } else { - - fmt.Printf("\r %s \n", ui.ErrorStyle.Render("✗")) - terminal.MoveCursorPositionDown(1) - fmt.Printf(" %s\n", ui.ErrorStyle.Render("Error executing command: %s\n", err.Error())) - - logging.GlobalLogger.Errorf("Error executing command: %s", err.Error()) - - testRunnerError = err - break testRunner - } - - break loop - default: - frame = (frame + 1) % len(spinnerFrames) - fmt.Printf("\r %s", ui.SpinnerStyle.Render(string(spinnerFrames[frame]))) - time.Sleep(spinnerRefresh) - } - } - } - } - - // If the resource group name was set, delete it. - if resourceGroupName != "" { - fmt.Printf("\n") - fmt.Printf("Deleting resource group: %s\n", resourceGroupName) - command := fmt.Sprintf("az group delete --name %s --yes --no-wait", resourceGroupName) - output, err := shells.ExecuteBashCommand( - command, - shells.BashCommandConfiguration{ - EnvironmentVariables: lib.CopyMap(env), - InheritEnvironment: true, - InteractiveCommand: false, - WriteToHistory: true, - }, - ) - if err != nil { - fmt.Print(ui.ErrorStyle.Render("Error deleting resource group: %s\n", err.Error())) - logging.GlobalLogger.Errorf("Error deleting resource group: %s", err.Error()) - testRunnerError = errors.Join(testRunnerError, err) - } - - fmt.Print(output.StdOut) - } - - shells.ResetStoredEnvironmentVariables() - return testRunnerError -} diff --git a/internal/lib/env.go b/internal/lib/env.go new file mode 100644 index 00000000..c1af2d95 --- /dev/null +++ b/internal/lib/env.go @@ -0,0 +1,99 @@ +package lib + +import ( + "bufio" + "fmt" + "os" + "regexp" + "strings" + + "github.com/Azure/InnovationEngine/internal/lib/fs" +) + +// Get environment variables from the current process. +func GetEnvironmentVariables() map[string]string { + envMap := make(map[string]string) + for _, env := range os.Environ() { + pair := strings.SplitN(env, "=", 2) + if len(pair) == 2 { + envMap[pair[0]] = pair[1] + } + } + + return envMap +} + +// Location where the environment state from commands are to be captured +// and sent to for being able to share state across commands. +var DefaultEnvironmentStateFile = "/tmp/env-vars" + +// Loads a file that contains environment variables +func LoadEnvironmentStateFile(path string) (map[string]string, error) { + if !fs.FileExists(path) { + return nil, fmt.Errorf("env file '%s' does not exist", path) + } + + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to open env file '%s': %w", path, err) + } + + defer file.Close() + + scanner := bufio.NewScanner(file) + env := make(map[string]string) + + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, "=") { + parts := strings.SplitN(line, "=", 2) // Split at the first "=" only + value := parts[1] + if len(value) >= 2 && value[0] == '"' && value[len(value)-1] == '"' { + // Remove leading and trailing quotes + value = value[1 : len(value)-1] + } + env[parts[0]] = value + } + } + return env, nil +} + +func CleanEnvironmentStateFile(path string) error { + env, err := LoadEnvironmentStateFile(path) + if err != nil { + return err + } + + env = filterInvalidKeys(env) + + file, err := os.Create(path) + if err != nil { + return err + } + + writer := bufio.NewWriter(file) + for k, v := range env { + _, err := fmt.Fprintf(writer, "%s=\"%s\"\n", k, v) + if err != nil { + return err + } + } + return writer.Flush() +} + +var environmentVariableName = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$") + +func filterInvalidKeys(envMap map[string]string) map[string]string { + validEnvMap := make(map[string]string) + for key, value := range envMap { + if environmentVariableName.MatchString(key) { + validEnvMap[key] = value + } + } + return validEnvMap +} + +// Deletes the stored environment variables file. +func DeleteEnvironmentStateFile(path string) error { + return os.Remove(path) +} diff --git a/internal/lib/env_test.go b/internal/lib/env_test.go new file mode 100644 index 00000000..6b83dcd5 --- /dev/null +++ b/internal/lib/env_test.go @@ -0,0 +1,68 @@ +package lib + +import "testing" + +func TestEnvironmentVariableValidationAndFiltering(t *testing.T) { + // Test key validation + t.Run("Key Validation", func(t *testing.T) { + validCases := []struct { + key string + expected bool + }{ + {"ValidKey", true}, + {"VALID_VARIABLE", true}, + {"_AnotherValidKey", true}, + {"123Key", false}, // Starts with a digit + {"key-with-hyphen", false}, // Contains a hyphen + {"key.with.dot", false}, // Contains a period + {"Fabric_NET-0-[Delegated]", false}, // From cloud shell environment. + } + + for _, tc := range validCases { + t.Run(tc.key, func(t *testing.T) { + result := environmentVariableName.MatchString(tc.key) + if result != tc.expected { + t.Errorf( + "Expected isValidKey(%s) to be %v, got %v", + tc.key, + tc.expected, + result, + ) + } + }) + } + }) + + // Test key filtering + t.Run("Key Filtering", func(t *testing.T) { + envMap := map[string]string{ + "ValidKey": "value1", + "_AnotherValidKey": "value2", + "123Key": "value3", + "key-with-hyphen": "value4", + "key.with.dot": "value5", + "Fabric_NET-0-[Delegated]": "false", // From cloud shell environment. + } + + validEnvMap := filterInvalidKeys(envMap) + + expectedValidEnvMap := map[string]string{ + "ValidKey": "value1", + "_AnotherValidKey": "value2", + } + + if len(validEnvMap) != len(expectedValidEnvMap) { + t.Errorf( + "Expected validEnvMap to have %d keys, got %d", + len(expectedValidEnvMap), + len(validEnvMap), + ) + } + + for key, value := range validEnvMap { + if expectedValue, ok := expectedValidEnvMap[key]; !ok || value != expectedValue { + t.Errorf("Expected validEnvMap[%s] to be %s, got %s", key, expectedValue, value) + } + } + }) +} diff --git a/internal/lib/maps.go b/internal/lib/maps.go index 7e4388a5..fc1f9d7b 100644 --- a/internal/lib/maps.go +++ b/internal/lib/maps.go @@ -18,3 +18,48 @@ func MergeMaps(a, b map[string]string) map[string]string { return merged } + +// Compares two maps by key and returns the difference between the two. +// This comparison doesn't take into account the value of the key, only the +// presence of the key. +func DiffMapsByKey(a, b map[string]string) map[string]string { + diff := make(map[string]string) + + if len(a) > len(b) { + for k, v := range a { + if b[k] == "" && v != "" { + diff[k] = v + } + } + } else { + for k, v := range b { + if a[k] == "" && v != "" { + diff[k] = v + } + } + } + + return diff +} + +// Compares two maps by key and returns the difference between the two based +// on the value of the key. +func DiffMapsByValue(a, b map[string]string) map[string]string { + diff := make(map[string]string) + + if len(a) > len(b) { + for k, v := range a { + if b[k] != v { + diff[k] = v + } + } + } else { + for k, v := range b { + if a[k] != v { + diff[k] = v + } + } + } + + return diff +} diff --git a/internal/parsers/markdown.go b/internal/parsers/markdown.go index f1ea85d1..ab964b11 100644 --- a/internal/parsers/markdown.go +++ b/internal/parsers/markdown.go @@ -34,6 +34,7 @@ func ParseMarkdownIntoAst(source []byte) ast.Node { return document } +// Extract the metadata from the AST of a markdown document. func ExtractYamlMetadataFromAst(node ast.Node) map[string]interface{} { return node.OwnerDocument().Meta() } @@ -42,19 +43,19 @@ func ExtractYamlMetadataFromAst(node ast.Node) map[string]interface{} { // for scenarios that have expected output that should be validated against the // actual output. type ExpectedOutputBlock struct { - Language string - Content string - ExpectedSimilarity float64 - ExpectedRegex *regexp.Regexp + Language string `json:"language"` + Content string `json:"content"` + ExpectedSimilarity float64 `json:"expectedSimilarityScore"` + ExpectedRegex *regexp.Regexp `json:"expectedRegexPattern"` } // The representation of a code block in a markdown file. type CodeBlock struct { - Language string - Content string - Header string - Description string - ExpectedOutput ExpectedOutputBlock + Language string `json:"language"` + Content string `json:"content"` + Header string `json:"header"` + Description string `json:"description"` + ExpectedOutput ExpectedOutputBlock `json:"resultBlock"` } // Assumes the title of the scenario is the first h1 header in the diff --git a/internal/shells/bash.go b/internal/shells/bash.go index e49b0cec..ec151565 100644 --- a/internal/shells/bash.go +++ b/internal/shells/bash.go @@ -1,54 +1,17 @@ package shells import ( - "bufio" "bytes" "fmt" "os" "os/exec" - "regexp" "strings" "golang.org/x/sys/unix" "github.com/Azure/InnovationEngine/internal/lib" - "github.com/Azure/InnovationEngine/internal/lib/fs" ) -// Location where the environment state from commands is captured and sent to -// for being able to share state across commands. -var environmentStateFile = "/tmp/env-vars" - -func loadEnvFile(path string) (map[string]string, error) { - if !fs.FileExists(path) { - return nil, fmt.Errorf("env file '%s' does not exist", path) - } - - file, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("failed to open env file '%s': %w", path, err) - } - - defer file.Close() - - scanner := bufio.NewScanner(file) - env := make(map[string]string) - - for scanner.Scan() { - line := scanner.Text() - if strings.Contains(line, "=") { - parts := strings.SplitN(line, "=", 2) // Split at the first "=" only - value := parts[1] - if len(value) >= 2 && value[0] == '"' && value[len(value)-1] == '"' { - // Remove leading and trailing quotes - value = value[1 : len(value)-1] - } - env[parts[0]] = value - } - } - return env, nil -} - func appendToBashHistory(command string, filePath string) error { file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) if err != nil { @@ -72,46 +35,6 @@ func appendToBashHistory(command string, filePath string) error { return nil } -// Resets the stored environment variables file. -func ResetStoredEnvironmentVariables() error { - return os.Remove(environmentStateFile) -} - -var environmentVariableName = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$") - -func filterInvalidKeys(envMap map[string]string) map[string]string { - validEnvMap := make(map[string]string) - for key, value := range envMap { - if environmentVariableName.MatchString(key) { - validEnvMap[key] = value - } - } - return validEnvMap -} - -func CleanEnvironmentStateFile() error { - env, err := loadEnvFile(environmentStateFile) - if err != nil { - return err - } - - env = filterInvalidKeys(env) - - file, err := os.Create(environmentStateFile) - if err != nil { - return err - } - - writer := bufio.NewWriter(file) - for k, v := range env { - _, err := fmt.Fprintf(writer, "%s=\"%s\"\n", k, v) - if err != nil { - return err - } - } - return writer.Flush() -} - type CommandOutput struct { StdOut string StdErr string @@ -135,7 +58,7 @@ func executeBashCommandImpl( "set -e", command, "IE_LAST_COMMAND_EXIT_CODE=\"$?\"", - "env > " + environmentStateFile, + "env > " + lib.DefaultEnvironmentStateFile, "exit $IE_LAST_COMMAND_EXIT_CODE", } @@ -164,7 +87,7 @@ func executeBashCommandImpl( // after a command is executed within a file and then loading that file // before executing the next command. This allows us to share state between // isolated command calls. - envFromPreviousStep, err := loadEnvFile(environmentStateFile) + envFromPreviousStep, err := lib.LoadEnvironmentStateFile(lib.DefaultEnvironmentStateFile) if err == nil { merged := lib.MergeMaps(config.EnvironmentVariables, envFromPreviousStep) for k, v := range merged { diff --git a/internal/shells/bash_test.go b/internal/shells/bash_test.go index 2cc88734..2040ab62 100644 --- a/internal/shells/bash_test.go +++ b/internal/shells/bash_test.go @@ -4,71 +4,6 @@ import ( "testing" ) -func TestEnvironmentVariableValidationAndFiltering(t *testing.T) { - // Test key validation - t.Run("Key Validation", func(t *testing.T) { - validCases := []struct { - key string - expected bool - }{ - {"ValidKey", true}, - {"VALID_VARIABLE", true}, - {"_AnotherValidKey", true}, - {"123Key", false}, // Starts with a digit - {"key-with-hyphen", false}, // Contains a hyphen - {"key.with.dot", false}, // Contains a period - {"Fabric_NET-0-[Delegated]", false}, // From cloud shell environment. - } - - for _, tc := range validCases { - t.Run(tc.key, func(t *testing.T) { - result := environmentVariableName.MatchString(tc.key) - if result != tc.expected { - t.Errorf( - "Expected isValidKey(%s) to be %v, got %v", - tc.key, - tc.expected, - result, - ) - } - }) - } - }) - - // Test key filtering - t.Run("Key Filtering", func(t *testing.T) { - envMap := map[string]string{ - "ValidKey": "value1", - "_AnotherValidKey": "value2", - "123Key": "value3", - "key-with-hyphen": "value4", - "key.with.dot": "value5", - "Fabric_NET-0-[Delegated]": "false", // From cloud shell environment. - } - - validEnvMap := filterInvalidKeys(envMap) - - expectedValidEnvMap := map[string]string{ - "ValidKey": "value1", - "_AnotherValidKey": "value2", - } - - if len(validEnvMap) != len(expectedValidEnvMap) { - t.Errorf( - "Expected validEnvMap to have %d keys, got %d", - len(expectedValidEnvMap), - len(validEnvMap), - ) - } - - for key, value := range validEnvMap { - if expectedValue, ok := expectedValidEnvMap[key]; !ok || value != expectedValue { - t.Errorf("Expected validEnvMap[%s] to be %s, got %s", key, expectedValue, value) - } - } - }) -} - func TestBashCommandExecution(t *testing.T) { // Ensures that if a command succeeds, the output is returned. t.Run("Valid command execution", func(t *testing.T) { @@ -106,7 +41,6 @@ func TestBashCommandExecution(t *testing.T) { if err == nil { t.Errorf("Expected an error to occur, but the command succeeded.") } - }) // Test the execution of commands with multiple subcommands. @@ -146,7 +80,6 @@ func TestBashCommandExecution(t *testing.T) { if err == nil { t.Errorf("Expected an error to occur, but the command succeeded.") } - }) // Ensures that commands can access environment variables passed into @@ -164,7 +97,6 @@ func TestBashCommandExecution(t *testing.T) { WriteToHistory: false, }, ) - if err != nil { t.Errorf("Expected err to be nil, got %v", err) } @@ -173,5 +105,4 @@ func TestBashCommandExecution(t *testing.T) { t.Errorf("Expected result to be non-empty, got '%s'", result.StdOut) } }) - } diff --git a/scenarios/testing/reporting.md b/scenarios/testing/reporting.md new file mode 100644 index 00000000..e68d0660 --- /dev/null +++ b/scenarios/testing/reporting.md @@ -0,0 +1,28 @@ +--- +ms.author: vmarcella +otherProperty: otherValue +--- + +# Test reporting doc + +## First step + +This step will show you how to do something. + +```bash +echo "Hello, world!" +``` + + + +```text +Hello, world! +``` + +## Second step + +This step will show you how to do something else. + +```bash +export NEW_VAR=1 +```