From f3e396950eb2793c50552fa65374e1d0042251da Mon Sep 17 00:00:00 2001 From: vmarcella Date: Thu, 18 Jul 2024 15:09:03 -0700 Subject: [PATCH 01/11] [add] report structs. --- internal/engine/test/reporting.go | 38 +++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 internal/engine/test/reporting.go diff --git a/internal/engine/test/reporting.go b/internal/engine/test/reporting.go new file mode 100644 index 00000000..538e9bbf --- /dev/null +++ b/internal/engine/test/reporting.go @@ -0,0 +1,38 @@ +package test + +// Represents the structure of + +type TestedCodeBlocks struct { + CodeBlock string `json:"codeBlock"` + ExpectedOutput string `json:"expectedOutput"` + ActualOutput string `json:"actualOutput"` + ComparisonScore float64 `json:"score"` + Success bool `json:"success"` + Error string `json:"error"` +} + +type TestedStep struct { + Header string `json:"header"` + Description string `json:"description"` + CodeBlocks []TestedCodeBlocks `json:"codeBlocks"` +} + +// A generated report from the execution of `ie test` on a markdown document. +type Report struct { + Name string `json:"name"` + Properties map[string]string `json:"properties"` + EnvironmentVariables map[string]string `json:"environmentVariables"` + Success bool `json:"success"` + Error string `json:"error"` + FailedAt int `json:"failedAt"` + Steps []TestedStep `json:"steps"` +} + +// TODO(vmarcella): Build out the rest of the test reporting JSON. +func BuildReport() Report { + return Report{ + Name: "", + Properties: make(map[string]string), + EnvironmentVariables: make(map[string]string), + } +} From bb7806508279c1760b6d27b52805f4b36f05bfff Mon Sep 17 00:00:00 2001 From: vmarcella Date: Tue, 30 Jul 2024 12:19:43 -0700 Subject: [PATCH 02/11] [update] scenarios to carry the yaml properties on them, add scaffolding for test reporting, and update the configuration to allow the engine to conditionally render test reports. --- internal/engine/common/reports.go | 43 ++++++++++ internal/engine/common/scenario.go | 3 + internal/engine/engine.go | 9 ++ internal/engine/test/model.go | 26 ++++++ internal/engine/test/reporting.go | 38 --------- internal/engine/testing.go | 132 ----------------------------- internal/lib/maps.go | 12 +++ 7 files changed, 93 insertions(+), 170 deletions(-) create mode 100644 internal/engine/common/reports.go delete mode 100644 internal/engine/test/reporting.go delete mode 100644 internal/engine/testing.go diff --git a/internal/engine/common/reports.go b/internal/engine/common/reports.go new file mode 100644 index 00000000..86f2b1d0 --- /dev/null +++ b/internal/engine/common/reports.go @@ -0,0 +1,43 @@ +package common + +// Reports are summaries of the execution of a markdown document. They will +// include the name of the document, the properties found in the yaml header of +// the document, the environment variables set by the document, and general +// information about the execution. +type Report struct { + Name string `json:"name"` + Properties map[string]interface{} `json:"properties"` + EnvironmentVariables map[string]string `json:"environmentVariables"` + Success bool `json:"success"` + Error string `json:"error"` + FailedAt int `json:"failedAt"` + codeBlocks []StatefulCodeBlock `json:"codeBlocks"` +} + +func (report *Report) WithProperties(properties map[string]interface{}) *Report { + report.Properties = properties + return report +} + +func (report *Report) WithEnvironmentVariables(envVars map[string]string) *Report { + report.EnvironmentVariables = envVars + return report +} + +func (report *Report) WithCodeBlocks(codeBlocks []StatefulCodeBlock) *Report { + report.codeBlocks = codeBlocks + return report +} + +// TODO(vmarcella): Implement this to write the report to JSON. +func (report *Report) WriteToJSONFile(outputPath string) error { + return nil +} + +func BuildReport(name string) Report { + return Report{ + Name: name, + Properties: make(map[string]interface{}), + EnvironmentVariables: make(map[string]string), + } +} diff --git a/internal/engine/common/scenario.go b/internal/engine/common/scenario.go index db01dfe9..19726d91 100644 --- a/internal/engine/common/scenario.go +++ b/internal/engine/common/scenario.go @@ -27,6 +27,7 @@ type Scenario struct { Name string MarkdownAst ast.Node Steps []Step + Properties map[string]interface{} Environment map[string]string } @@ -116,6 +117,7 @@ func CreateScenarioFromMarkdown( // Convert the markdonw into an AST and extract the scenario variables. markdown := parsers.ParseMarkdownIntoAst(source) + properties := parsers.ExtractYamlMetadataFromAst(markdown) scenarioVariables := parsers.ExtractScenarioVariablesFromAst(markdown, source) for key, value := range scenarioVariables { environmentVariables[key] = value @@ -195,6 +197,7 @@ func CreateScenarioFromMarkdown( Name: title, Environment: environmentVariables, Steps: steps, + Properties: properties, MarkdownAst: markdown, }, nil } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index a376fc13..5941f177 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -28,6 +28,7 @@ type EngineConfiguration struct { Environment string WorkingDirectory string RenderValues bool + GenerateReport bool } type Engine struct { @@ -95,8 +96,16 @@ func (e *Engine) TestScenario(scenario *common.Scenario) error { if !ok { err = errors.Join(err, fmt.Errorf("failed to cast tea.Model to TestModeModel")) return err + } + if e.Configuration.GenerateReport { + report := common.BuildReport(scenario.Name) + report. + WithProperties(scenario.Properties). + WithEnvironmentVariables(model.GetEnvironmentVariables()). + WriteToJSONFile("/tmp/report.json") } + err = errors.Join(err, model.GetFailure()) fmt.Println(strings.Join(model.CommandLines, "\n")) diff --git a/internal/engine/test/model.go b/internal/engine/test/model.go index 57b97183..c971f730 100644 --- a/internal/engine/test/model.go +++ b/internal/engine/test/model.go @@ -54,6 +54,32 @@ func (model TestModeModel) GetFailure() error { ) } +func (model TestModeModel) GetScenarioTitle() string { + return model.scenarioTitle +} + +// Get the environment that the scenario is running in. +func (model TestModeModel) GetEnvironment() string { + return model.environment +} + +// Get the code blocks that were executed in the scenario. +func (model TestModeModel) GetCodeBlocks() []common.StatefulCodeBlock { + codeBlocks := make([]common.StatefulCodeBlock, len(model.codeBlockState)) + for _, codeBlock := range model.codeBlockState { + codeBlocks = append(codeBlocks, codeBlock) + } + return codeBlocks +} + +func (model TestModeModel) GetEnvironmentVariables() map[string]string { + return model.environmentVariables +} + +func (model TestModeModel) GetDeclaredEnvironmentVariables() map[string]string { + return model.environmentVariables +} + // Init the test mode model by executing the first code block. func (model TestModeModel) Init() tea.Cmd { return common.ExecuteCodeBlockAsync( diff --git a/internal/engine/test/reporting.go b/internal/engine/test/reporting.go deleted file mode 100644 index 538e9bbf..00000000 --- a/internal/engine/test/reporting.go +++ /dev/null @@ -1,38 +0,0 @@ -package test - -// Represents the structure of - -type TestedCodeBlocks struct { - CodeBlock string `json:"codeBlock"` - ExpectedOutput string `json:"expectedOutput"` - ActualOutput string `json:"actualOutput"` - ComparisonScore float64 `json:"score"` - Success bool `json:"success"` - Error string `json:"error"` -} - -type TestedStep struct { - Header string `json:"header"` - Description string `json:"description"` - CodeBlocks []TestedCodeBlocks `json:"codeBlocks"` -} - -// A generated report from the execution of `ie test` on a markdown document. -type Report struct { - Name string `json:"name"` - Properties map[string]string `json:"properties"` - EnvironmentVariables map[string]string `json:"environmentVariables"` - Success bool `json:"success"` - Error string `json:"error"` - FailedAt int `json:"failedAt"` - Steps []TestedStep `json:"steps"` -} - -// TODO(vmarcella): Build out the rest of the test reporting JSON. -func BuildReport() Report { - return Report{ - Name: "", - Properties: make(map[string]string), - EnvironmentVariables: make(map[string]string), - } -} diff --git a/internal/engine/testing.go b/internal/engine/testing.go deleted file mode 100644 index 3c46c388..00000000 --- a/internal/engine/testing.go +++ /dev/null @@ -1,132 +0,0 @@ -package engine - -import ( - "errors" - "fmt" - "time" - - "github.com/Azure/InnovationEngine/internal/az" - "github.com/Azure/InnovationEngine/internal/engine/common" - "github.com/Azure/InnovationEngine/internal/lib" - "github.com/Azure/InnovationEngine/internal/logging" - "github.com/Azure/InnovationEngine/internal/parsers" - "github.com/Azure/InnovationEngine/internal/patterns" - "github.com/Azure/InnovationEngine/internal/shells" - "github.com/Azure/InnovationEngine/internal/terminal" - "github.com/Azure/InnovationEngine/internal/ui" -) - -func (e *Engine) TestSteps(steps []common.Step, env map[string]string) error { - var resourceGroupName string - stepsToExecute := filterDeletionCommands(steps, true) - err := az.SetSubscription(e.Configuration.Subscription) - if err != nil { - logging.GlobalLogger.Errorf("Invalid Config: Failed to set subscription: %s", err) - return err - } - - var testRunnerError error = nil -testRunner: - for stepNumber, step := range stepsToExecute { - stepTitle := fmt.Sprintf(" %d. %s\n", stepNumber+1, step.Name) - fmt.Println(ui.StepTitleStyle.Render(stepTitle)) - terminal.MoveCursorPositionUp(1) - terminal.HideCursor() - - for _, block := range step.CodeBlocks { - // execute the command as a goroutine to allow for the spinner to be - // rendered while the command is executing. - done := make(chan error) - var commandOutput shells.CommandOutput - go func(block parsers.CodeBlock) { - logging.GlobalLogger.Infof("Executing command: %s", block.Content) - output, err := shells.ExecuteBashCommand(block.Content, shells.BashCommandConfiguration{EnvironmentVariables: lib.CopyMap(env), InheritEnvironment: true, InteractiveCommand: false, WriteToHistory: true}) - logging.GlobalLogger.Infof("Command stdout: %s", output.StdOut) - logging.GlobalLogger.Infof("Command stderr: %s", output.StdErr) - commandOutput = output - done <- err - }(block) - - frame := 0 - var err error - - loop: - // While the command is executing, render the spinner. - for { - select { - case err = <-done: - terminal.ShowCursor() - - if err == nil { - actualOutput := commandOutput.StdOut - expectedOutput := block.ExpectedOutput.Content - expectedSimilarity := block.ExpectedOutput.ExpectedSimilarity - expectedRegex := block.ExpectedOutput.ExpectedRegex - expectedOutputLanguage := block.ExpectedOutput.Language - - err := common.CompareCommandOutputs(actualOutput, expectedOutput, expectedSimilarity, expectedRegex, expectedOutputLanguage) - if err != nil { - logging.GlobalLogger.Errorf("Error comparing command outputs: %s", err.Error()) - fmt.Print(ui.ErrorStyle.Render("Error when comparing the command outputs: %s\n", err.Error())) - } - - // Extract the resource group name from the command output if - // it's not already set. - if resourceGroupName == "" && patterns.AzCommand.MatchString(block.Content) { - tmpResourceGroup := az.FindResourceGroupName(commandOutput.StdOut) - if tmpResourceGroup != "" { - logging.GlobalLogger.Infof("Found resource group: %s", tmpResourceGroup) - resourceGroupName = tmpResourceGroup - } - } - - fmt.Printf("\r %s \n", ui.CheckStyle.Render("✔")) - terminal.MoveCursorPositionDown(1) - } else { - - fmt.Printf("\r %s \n", ui.ErrorStyle.Render("✗")) - terminal.MoveCursorPositionDown(1) - fmt.Printf(" %s\n", ui.ErrorStyle.Render("Error executing command: %s\n", err.Error())) - - logging.GlobalLogger.Errorf("Error executing command: %s", err.Error()) - - testRunnerError = err - break testRunner - } - - break loop - default: - frame = (frame + 1) % len(spinnerFrames) - fmt.Printf("\r %s", ui.SpinnerStyle.Render(string(spinnerFrames[frame]))) - time.Sleep(spinnerRefresh) - } - } - } - } - - // If the resource group name was set, delete it. - if resourceGroupName != "" { - fmt.Printf("\n") - fmt.Printf("Deleting resource group: %s\n", resourceGroupName) - command := fmt.Sprintf("az group delete --name %s --yes --no-wait", resourceGroupName) - output, err := shells.ExecuteBashCommand( - command, - shells.BashCommandConfiguration{ - EnvironmentVariables: lib.CopyMap(env), - InheritEnvironment: true, - InteractiveCommand: false, - WriteToHistory: true, - }, - ) - if err != nil { - fmt.Print(ui.ErrorStyle.Render("Error deleting resource group: %s\n", err.Error())) - logging.GlobalLogger.Errorf("Error deleting resource group: %s", err.Error()) - testRunnerError = errors.Join(testRunnerError, err) - } - - fmt.Print(output.StdOut) - } - - shells.ResetStoredEnvironmentVariables() - return testRunnerError -} diff --git a/internal/lib/maps.go b/internal/lib/maps.go index 7e4388a5..b47e26cd 100644 --- a/internal/lib/maps.go +++ b/internal/lib/maps.go @@ -18,3 +18,15 @@ func MergeMaps(a, b map[string]string) map[string]string { return merged } + +// Returns the difference between two maps. +func DiffMaps(a, b map[string]string) map[string]string { + diff := make(map[string]string) + for k, v := range a { + if b[k] != v { + diff[k] = v + } + } + + return diff +} From 6267429cfb1f33b120c6ef82fe2ed2a20697fac6 Mon Sep 17 00:00:00 2001 From: vmarcella Date: Tue, 30 Jul 2024 13:40:57 -0700 Subject: [PATCH 03/11] [add] env.go for handling all environment variable related logic, update all other modules accordingly. --- internal/engine/common/reports.go | 11 +++- internal/engine/engine.go | 11 ++-- internal/engine/execution.go | 4 +- internal/lib/env.go | 99 +++++++++++++++++++++++++++++++ internal/lib/env_test.go | 68 +++++++++++++++++++++ internal/shells/bash.go | 81 +------------------------ internal/shells/bash_test.go | 69 --------------------- 7 files changed, 187 insertions(+), 156 deletions(-) create mode 100644 internal/lib/env.go create mode 100644 internal/lib/env_test.go diff --git a/internal/engine/common/reports.go b/internal/engine/common/reports.go index 86f2b1d0..2302741a 100644 --- a/internal/engine/common/reports.go +++ b/internal/engine/common/reports.go @@ -10,7 +10,7 @@ type Report struct { EnvironmentVariables map[string]string `json:"environmentVariables"` Success bool `json:"success"` Error string `json:"error"` - FailedAt int `json:"failedAt"` + FailedAtStep int `json:"failedAtStep"` codeBlocks []StatefulCodeBlock `json:"codeBlocks"` } @@ -29,6 +29,12 @@ func (report *Report) WithCodeBlocks(codeBlocks []StatefulCodeBlock) *Report { return report } +func (report *Report) WithError(err error) *Report { + report.Error = err.Error() + report.Success = false + return report +} + // TODO(vmarcella): Implement this to write the report to JSON. func (report *Report) WriteToJSONFile(outputPath string) error { return nil @@ -39,5 +45,8 @@ func BuildReport(name string) Report { Name: name, Properties: make(map[string]interface{}), EnvironmentVariables: make(map[string]string), + Success: true, + Error: "", + FailedAtStep: -1, } } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 5941f177..9ef023f5 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -14,7 +14,6 @@ import ( "github.com/Azure/InnovationEngine/internal/lib" "github.com/Azure/InnovationEngine/internal/lib/fs" "github.com/Azure/InnovationEngine/internal/logging" - "github.com/Azure/InnovationEngine/internal/shells" "github.com/Azure/InnovationEngine/internal/ui" tea "github.com/charmbracelet/bubbletea" ) @@ -28,7 +27,7 @@ type EngineConfiguration struct { Environment string WorkingDirectory string RenderValues bool - GenerateReport bool + GenerateReport string } type Engine struct { @@ -98,7 +97,7 @@ func (e *Engine) TestScenario(scenario *common.Scenario) error { return err } - if e.Configuration.GenerateReport { + if e.Configuration.GenerateReport != "" { report := common.BuildReport(scenario.Name) report. WithProperties(scenario.Properties). @@ -135,6 +134,8 @@ func (e *Engine) InteractWithScenario(scenario *common.Scenario) error { common.Program = tea.NewProgram(model, tea.WithAltScreen(), tea.WithMouseCellMotion()) + // initialEnvironmentVariables := lib.GetEnvironmentVariables() + var finalModel tea.Model var ok bool finalModel, err = common.Program.Run() @@ -155,14 +156,14 @@ func (e *Engine) InteractWithScenario(scenario *common.Scenario) error { logging.GlobalLogger.Info( "Cleaning environment variable file located at /tmp/env-vars", ) - err := shells.CleanEnvironmentStateFile() + err := lib.CleanEnvironmentStateFile(lib.DefaultEnvironmentStateFile) if err != nil { logging.GlobalLogger.Errorf("Error cleaning environment variables: %s", err.Error()) return err } default: - shells.ResetStoredEnvironmentVariables() + lib.DeleteEnvironmentStateFile(lib.DefaultEnvironmentStateFile) } if err != nil { diff --git a/internal/engine/execution.go b/internal/engine/execution.go index 7a07ab68..c2075152 100644 --- a/internal/engine/execution.go +++ b/internal/engine/execution.go @@ -310,14 +310,14 @@ func (e *Engine) ExecuteAndRenderSteps(steps []common.Step, env map[string]strin logging.GlobalLogger.Info( "Cleaning environment variable file located at /tmp/env-vars", ) - err := shells.CleanEnvironmentStateFile() + err := lib.CleanEnvironmentStateFile(lib.DefaultEnvironmentStateFile) if err != nil { logging.GlobalLogger.Errorf("Error cleaning environment variables: %s", err.Error()) return err } default: - shells.ResetStoredEnvironmentVariables() + lib.DeleteEnvironmentStateFile(lib.DefaultEnvironmentStateFile) } return nil diff --git a/internal/lib/env.go b/internal/lib/env.go new file mode 100644 index 00000000..c1af2d95 --- /dev/null +++ b/internal/lib/env.go @@ -0,0 +1,99 @@ +package lib + +import ( + "bufio" + "fmt" + "os" + "regexp" + "strings" + + "github.com/Azure/InnovationEngine/internal/lib/fs" +) + +// Get environment variables from the current process. +func GetEnvironmentVariables() map[string]string { + envMap := make(map[string]string) + for _, env := range os.Environ() { + pair := strings.SplitN(env, "=", 2) + if len(pair) == 2 { + envMap[pair[0]] = pair[1] + } + } + + return envMap +} + +// Location where the environment state from commands are to be captured +// and sent to for being able to share state across commands. +var DefaultEnvironmentStateFile = "/tmp/env-vars" + +// Loads a file that contains environment variables +func LoadEnvironmentStateFile(path string) (map[string]string, error) { + if !fs.FileExists(path) { + return nil, fmt.Errorf("env file '%s' does not exist", path) + } + + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to open env file '%s': %w", path, err) + } + + defer file.Close() + + scanner := bufio.NewScanner(file) + env := make(map[string]string) + + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, "=") { + parts := strings.SplitN(line, "=", 2) // Split at the first "=" only + value := parts[1] + if len(value) >= 2 && value[0] == '"' && value[len(value)-1] == '"' { + // Remove leading and trailing quotes + value = value[1 : len(value)-1] + } + env[parts[0]] = value + } + } + return env, nil +} + +func CleanEnvironmentStateFile(path string) error { + env, err := LoadEnvironmentStateFile(path) + if err != nil { + return err + } + + env = filterInvalidKeys(env) + + file, err := os.Create(path) + if err != nil { + return err + } + + writer := bufio.NewWriter(file) + for k, v := range env { + _, err := fmt.Fprintf(writer, "%s=\"%s\"\n", k, v) + if err != nil { + return err + } + } + return writer.Flush() +} + +var environmentVariableName = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$") + +func filterInvalidKeys(envMap map[string]string) map[string]string { + validEnvMap := make(map[string]string) + for key, value := range envMap { + if environmentVariableName.MatchString(key) { + validEnvMap[key] = value + } + } + return validEnvMap +} + +// Deletes the stored environment variables file. +func DeleteEnvironmentStateFile(path string) error { + return os.Remove(path) +} diff --git a/internal/lib/env_test.go b/internal/lib/env_test.go new file mode 100644 index 00000000..6b83dcd5 --- /dev/null +++ b/internal/lib/env_test.go @@ -0,0 +1,68 @@ +package lib + +import "testing" + +func TestEnvironmentVariableValidationAndFiltering(t *testing.T) { + // Test key validation + t.Run("Key Validation", func(t *testing.T) { + validCases := []struct { + key string + expected bool + }{ + {"ValidKey", true}, + {"VALID_VARIABLE", true}, + {"_AnotherValidKey", true}, + {"123Key", false}, // Starts with a digit + {"key-with-hyphen", false}, // Contains a hyphen + {"key.with.dot", false}, // Contains a period + {"Fabric_NET-0-[Delegated]", false}, // From cloud shell environment. + } + + for _, tc := range validCases { + t.Run(tc.key, func(t *testing.T) { + result := environmentVariableName.MatchString(tc.key) + if result != tc.expected { + t.Errorf( + "Expected isValidKey(%s) to be %v, got %v", + tc.key, + tc.expected, + result, + ) + } + }) + } + }) + + // Test key filtering + t.Run("Key Filtering", func(t *testing.T) { + envMap := map[string]string{ + "ValidKey": "value1", + "_AnotherValidKey": "value2", + "123Key": "value3", + "key-with-hyphen": "value4", + "key.with.dot": "value5", + "Fabric_NET-0-[Delegated]": "false", // From cloud shell environment. + } + + validEnvMap := filterInvalidKeys(envMap) + + expectedValidEnvMap := map[string]string{ + "ValidKey": "value1", + "_AnotherValidKey": "value2", + } + + if len(validEnvMap) != len(expectedValidEnvMap) { + t.Errorf( + "Expected validEnvMap to have %d keys, got %d", + len(expectedValidEnvMap), + len(validEnvMap), + ) + } + + for key, value := range validEnvMap { + if expectedValue, ok := expectedValidEnvMap[key]; !ok || value != expectedValue { + t.Errorf("Expected validEnvMap[%s] to be %s, got %s", key, expectedValue, value) + } + } + }) +} diff --git a/internal/shells/bash.go b/internal/shells/bash.go index e49b0cec..ec151565 100644 --- a/internal/shells/bash.go +++ b/internal/shells/bash.go @@ -1,54 +1,17 @@ package shells import ( - "bufio" "bytes" "fmt" "os" "os/exec" - "regexp" "strings" "golang.org/x/sys/unix" "github.com/Azure/InnovationEngine/internal/lib" - "github.com/Azure/InnovationEngine/internal/lib/fs" ) -// Location where the environment state from commands is captured and sent to -// for being able to share state across commands. -var environmentStateFile = "/tmp/env-vars" - -func loadEnvFile(path string) (map[string]string, error) { - if !fs.FileExists(path) { - return nil, fmt.Errorf("env file '%s' does not exist", path) - } - - file, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("failed to open env file '%s': %w", path, err) - } - - defer file.Close() - - scanner := bufio.NewScanner(file) - env := make(map[string]string) - - for scanner.Scan() { - line := scanner.Text() - if strings.Contains(line, "=") { - parts := strings.SplitN(line, "=", 2) // Split at the first "=" only - value := parts[1] - if len(value) >= 2 && value[0] == '"' && value[len(value)-1] == '"' { - // Remove leading and trailing quotes - value = value[1 : len(value)-1] - } - env[parts[0]] = value - } - } - return env, nil -} - func appendToBashHistory(command string, filePath string) error { file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) if err != nil { @@ -72,46 +35,6 @@ func appendToBashHistory(command string, filePath string) error { return nil } -// Resets the stored environment variables file. -func ResetStoredEnvironmentVariables() error { - return os.Remove(environmentStateFile) -} - -var environmentVariableName = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$") - -func filterInvalidKeys(envMap map[string]string) map[string]string { - validEnvMap := make(map[string]string) - for key, value := range envMap { - if environmentVariableName.MatchString(key) { - validEnvMap[key] = value - } - } - return validEnvMap -} - -func CleanEnvironmentStateFile() error { - env, err := loadEnvFile(environmentStateFile) - if err != nil { - return err - } - - env = filterInvalidKeys(env) - - file, err := os.Create(environmentStateFile) - if err != nil { - return err - } - - writer := bufio.NewWriter(file) - for k, v := range env { - _, err := fmt.Fprintf(writer, "%s=\"%s\"\n", k, v) - if err != nil { - return err - } - } - return writer.Flush() -} - type CommandOutput struct { StdOut string StdErr string @@ -135,7 +58,7 @@ func executeBashCommandImpl( "set -e", command, "IE_LAST_COMMAND_EXIT_CODE=\"$?\"", - "env > " + environmentStateFile, + "env > " + lib.DefaultEnvironmentStateFile, "exit $IE_LAST_COMMAND_EXIT_CODE", } @@ -164,7 +87,7 @@ func executeBashCommandImpl( // after a command is executed within a file and then loading that file // before executing the next command. This allows us to share state between // isolated command calls. - envFromPreviousStep, err := loadEnvFile(environmentStateFile) + envFromPreviousStep, err := lib.LoadEnvironmentStateFile(lib.DefaultEnvironmentStateFile) if err == nil { merged := lib.MergeMaps(config.EnvironmentVariables, envFromPreviousStep) for k, v := range merged { diff --git a/internal/shells/bash_test.go b/internal/shells/bash_test.go index 2cc88734..2040ab62 100644 --- a/internal/shells/bash_test.go +++ b/internal/shells/bash_test.go @@ -4,71 +4,6 @@ import ( "testing" ) -func TestEnvironmentVariableValidationAndFiltering(t *testing.T) { - // Test key validation - t.Run("Key Validation", func(t *testing.T) { - validCases := []struct { - key string - expected bool - }{ - {"ValidKey", true}, - {"VALID_VARIABLE", true}, - {"_AnotherValidKey", true}, - {"123Key", false}, // Starts with a digit - {"key-with-hyphen", false}, // Contains a hyphen - {"key.with.dot", false}, // Contains a period - {"Fabric_NET-0-[Delegated]", false}, // From cloud shell environment. - } - - for _, tc := range validCases { - t.Run(tc.key, func(t *testing.T) { - result := environmentVariableName.MatchString(tc.key) - if result != tc.expected { - t.Errorf( - "Expected isValidKey(%s) to be %v, got %v", - tc.key, - tc.expected, - result, - ) - } - }) - } - }) - - // Test key filtering - t.Run("Key Filtering", func(t *testing.T) { - envMap := map[string]string{ - "ValidKey": "value1", - "_AnotherValidKey": "value2", - "123Key": "value3", - "key-with-hyphen": "value4", - "key.with.dot": "value5", - "Fabric_NET-0-[Delegated]": "false", // From cloud shell environment. - } - - validEnvMap := filterInvalidKeys(envMap) - - expectedValidEnvMap := map[string]string{ - "ValidKey": "value1", - "_AnotherValidKey": "value2", - } - - if len(validEnvMap) != len(expectedValidEnvMap) { - t.Errorf( - "Expected validEnvMap to have %d keys, got %d", - len(expectedValidEnvMap), - len(validEnvMap), - ) - } - - for key, value := range validEnvMap { - if expectedValue, ok := expectedValidEnvMap[key]; !ok || value != expectedValue { - t.Errorf("Expected validEnvMap[%s] to be %s, got %s", key, expectedValue, value) - } - } - }) -} - func TestBashCommandExecution(t *testing.T) { // Ensures that if a command succeeds, the output is returned. t.Run("Valid command execution", func(t *testing.T) { @@ -106,7 +41,6 @@ func TestBashCommandExecution(t *testing.T) { if err == nil { t.Errorf("Expected an error to occur, but the command succeeded.") } - }) // Test the execution of commands with multiple subcommands. @@ -146,7 +80,6 @@ func TestBashCommandExecution(t *testing.T) { if err == nil { t.Errorf("Expected an error to occur, but the command succeeded.") } - }) // Ensures that commands can access environment variables passed into @@ -164,7 +97,6 @@ func TestBashCommandExecution(t *testing.T) { WriteToHistory: false, }, ) - if err != nil { t.Errorf("Expected err to be nil, got %v", err) } @@ -173,5 +105,4 @@ func TestBashCommandExecution(t *testing.T) { t.Errorf("Expected result to be non-empty, got '%s'", result.StdOut) } }) - } From fe47b6ae8e6f6a8299e4eb8d0a658ab245487700 Mon Sep 17 00:00:00 2001 From: vmarcella Date: Tue, 30 Jul 2024 13:55:54 -0700 Subject: [PATCH 04/11] [add] cli flag for generating the report, capture the initial environment state before the scenario starts execution, and add report output to test for now. --- cmd/ie/commands/test.go | 4 ++++ internal/engine/common/reports.go | 12 ++++++++++++ internal/engine/engine.go | 21 +++++++++++++++++---- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/cmd/ie/commands/test.go b/cmd/ie/commands/test.go index a8e1da5d..40c8eade 100644 --- a/cmd/ie/commands/test.go +++ b/cmd/ie/commands/test.go @@ -20,6 +20,8 @@ func init() { String("subscription", "", "Sets the subscription ID used by a scenarios azure-cli commands. Will rely on the default subscription if not set.") testCommand.PersistentFlags(). String("working-directory", ".", "Sets the working directory for innovation engine to operate out of. Restores the current working directory when finished.") + testCommand.PersistentFlags(). + String("generate-report", "", "The path to generate a report of the scenario execution. The contents of the report are in JSON and will only be generated when this flag is set.") testCommand.PersistentFlags(). StringArray("var", []string{}, "Sets an environment variable for the scenario. Format: --var =") @@ -40,6 +42,7 @@ var testCommand = &cobra.Command{ subscription, _ := cmd.Flags().GetString("subscription") workingDirectory, _ := cmd.Flags().GetString("working-directory") environment, _ := cmd.Flags().GetString("environment") + generateReport, _ := cmd.Flags().GetString("generate-report") environmentVariables, _ := cmd.Flags().GetStringArray("var") @@ -67,6 +70,7 @@ var testCommand = &cobra.Command{ CorrelationId: "", WorkingDirectory: workingDirectory, Environment: environment, + GenerateReport: generateReport, }) if err != nil { logging.GlobalLogger.Errorf("Error creating engine %s", err) diff --git a/internal/engine/common/reports.go b/internal/engine/common/reports.go index 2302741a..954a5a3b 100644 --- a/internal/engine/common/reports.go +++ b/internal/engine/common/reports.go @@ -1,5 +1,10 @@ package common +import ( + "encoding/json" + "fmt" +) + // Reports are summaries of the execution of a markdown document. They will // include the name of the document, the properties found in the yaml header of // the document, the environment variables set by the document, and general @@ -37,6 +42,13 @@ func (report *Report) WithError(err error) *Report { // TODO(vmarcella): Implement this to write the report to JSON. func (report *Report) WriteToJSONFile(outputPath string) error { + jsonReport, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + + fmt.Println(string(jsonReport)) + return nil } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 9ef023f5..3f021a22 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -60,6 +60,8 @@ func (e *Engine) TestScenario(scenario *common.Scenario) error { az.SetCorrelationId(e.Configuration.CorrelationId, scenario.Environment) stepsToExecute := filterDeletionCommands(scenario.Steps, e.Configuration.DoNotDelete) + initialEnvironmentVariables := lib.GetEnvironmentVariables() + model, err := test.NewTestModeModel( scenario.Name, e.Configuration.Subscription, @@ -97,12 +99,25 @@ func (e *Engine) TestScenario(scenario *common.Scenario) error { return err } + allEnvironmentVariables, err := lib.LoadEnvironmentStateFile( + lib.DefaultEnvironmentStateFile, + ) + if err != nil { + logging.GlobalLogger.Errorf("Failed to load environment state file: %s", err) + return err + } + + variablesDeclaredByScenario := lib.DiffMaps( + allEnvironmentVariables, + initialEnvironmentVariables, + ) + if e.Configuration.GenerateReport != "" { report := common.BuildReport(scenario.Name) report. WithProperties(scenario.Properties). - WithEnvironmentVariables(model.GetEnvironmentVariables()). - WriteToJSONFile("/tmp/report.json") + WithEnvironmentVariables(variablesDeclaredByScenario). + WriteToJSONFile(e.Configuration.GenerateReport) } err = errors.Join(err, model.GetFailure()) @@ -134,8 +149,6 @@ func (e *Engine) InteractWithScenario(scenario *common.Scenario) error { common.Program = tea.NewProgram(model, tea.WithAltScreen(), tea.WithMouseCellMotion()) - // initialEnvironmentVariables := lib.GetEnvironmentVariables() - var finalModel tea.Model var ok bool finalModel, err = common.Program.Run() From 2201860d478bad23900a0bbef2691f52987368c7 Mon Sep 17 00:00:00 2001 From: vmarcella Date: Tue, 30 Jul 2024 14:33:00 -0700 Subject: [PATCH 05/11] [update] tags so that codeblocks can be rendered in the JSON. --- internal/engine/common/codeblock.go | 16 +++++++-------- internal/engine/common/reports.go | 8 ++++++-- internal/engine/engine.go | 12 ++++++----- internal/engine/test/model.go | 3 ++- internal/lib/maps.go | 32 +++++++++++++++++++++++++++-- internal/parsers/markdown.go | 18 ++++++++-------- 6 files changed, 62 insertions(+), 27 deletions(-) diff --git a/internal/engine/common/codeblock.go b/internal/engine/common/codeblock.go index b1340436..9942ff27 100644 --- a/internal/engine/common/codeblock.go +++ b/internal/engine/common/codeblock.go @@ -5,14 +5,14 @@ import "github.com/Azure/InnovationEngine/internal/parsers" // State for the codeblock in interactive mode. Used to keep track of the // state of each codeblock. type StatefulCodeBlock struct { - CodeBlock parsers.CodeBlock - CodeBlockNumber int - Error error - StdErr string - StdOut string - StepName string - StepNumber int - Success bool + CodeBlock parsers.CodeBlock `json:"codeBlock"` + CodeBlockNumber int `json:"codeBlockNumber"` + Error error `json:"error"` + StdErr string `json:"stdErr"` + StdOut string `json:"stdOut"` + StepName string `json:"stepName"` + StepNumber int `json:"stepNumber"` + Success bool `json:"success"` } // Checks if a codeblock was executed by looking at the diff --git a/internal/engine/common/reports.go b/internal/engine/common/reports.go index 954a5a3b..9e5f88ab 100644 --- a/internal/engine/common/reports.go +++ b/internal/engine/common/reports.go @@ -16,7 +16,7 @@ type Report struct { Success bool `json:"success"` Error string `json:"error"` FailedAtStep int `json:"failedAtStep"` - codeBlocks []StatefulCodeBlock `json:"codeBlocks"` + CodeBlocks []StatefulCodeBlock `json:"codeBlocks"` } func (report *Report) WithProperties(properties map[string]interface{}) *Report { @@ -30,11 +30,15 @@ func (report *Report) WithEnvironmentVariables(envVars map[string]string) *Repor } func (report *Report) WithCodeBlocks(codeBlocks []StatefulCodeBlock) *Report { - report.codeBlocks = codeBlocks + report.CodeBlocks = codeBlocks return report } func (report *Report) WithError(err error) *Report { + if err == nil { + return report + } + report.Error = err.Error() report.Success = false return report diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 3f021a22..300d72ca 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -107,16 +107,18 @@ func (e *Engine) TestScenario(scenario *common.Scenario) error { return err } - variablesDeclaredByScenario := lib.DiffMaps( - allEnvironmentVariables, - initialEnvironmentVariables, - ) - if e.Configuration.GenerateReport != "" { + variablesDeclaredByScenario := lib.DiffMapsByKey( + allEnvironmentVariables, + initialEnvironmentVariables, + ) + report := common.BuildReport(scenario.Name) report. WithProperties(scenario.Properties). WithEnvironmentVariables(variablesDeclaredByScenario). + WithError(model.GetFailure()). + WithCodeBlocks(model.GetCodeBlocks()). WriteToJSONFile(e.Configuration.GenerateReport) } diff --git a/internal/engine/test/model.go b/internal/engine/test/model.go index c971f730..e07dbbad 100644 --- a/internal/engine/test/model.go +++ b/internal/engine/test/model.go @@ -65,10 +65,11 @@ func (model TestModeModel) GetEnvironment() string { // Get the code blocks that were executed in the scenario. func (model TestModeModel) GetCodeBlocks() []common.StatefulCodeBlock { - codeBlocks := make([]common.StatefulCodeBlock, len(model.codeBlockState)) + var codeBlocks []common.StatefulCodeBlock for _, codeBlock := range model.codeBlockState { codeBlocks = append(codeBlocks, codeBlock) } + fmt.Println(codeBlocks) return codeBlocks } diff --git a/internal/lib/maps.go b/internal/lib/maps.go index b47e26cd..3408bb81 100644 --- a/internal/lib/maps.go +++ b/internal/lib/maps.go @@ -1,5 +1,7 @@ package lib +import "fmt" + // Makes a copy of a map func CopyMap(m map[string]string) map[string]string { result := make(map[string]string) @@ -19,9 +21,35 @@ func MergeMaps(a, b map[string]string) map[string]string { return merged } -// Returns the difference between two maps. -func DiffMaps(a, b map[string]string) map[string]string { +// Compares two maps by key and returns the difference between the two. +func DiffMapsByKey(a, b map[string]string) map[string]string { + diff := make(map[string]string) + + aLength := len(a) + bLength := len(b) + + if aLength > bLength { + for k, v := range a { + if b[k] == "" && v != "" { + fmt.Print("diff: ", k, v) + diff[k] = v + } + } + } else { + for k, v := range b { + if a[k] == "" && v != "" { + fmt.Print("diff: ", k, v) + diff[k] = v + } + } + } + + return diff +} + +func DiffMapsByValue(a, b map[string]string) map[string]string { diff := make(map[string]string) + for k, v := range a { if b[k] != v { diff[k] = v diff --git a/internal/parsers/markdown.go b/internal/parsers/markdown.go index f1ea85d1..440e9dd5 100644 --- a/internal/parsers/markdown.go +++ b/internal/parsers/markdown.go @@ -42,19 +42,19 @@ func ExtractYamlMetadataFromAst(node ast.Node) map[string]interface{} { // for scenarios that have expected output that should be validated against the // actual output. type ExpectedOutputBlock struct { - Language string - Content string - ExpectedSimilarity float64 - ExpectedRegex *regexp.Regexp + Language string `json:"language"` + Content string `json:"content"` + ExpectedSimilarity float64 `json:"expectedSimilarity"` + ExpectedRegex *regexp.Regexp `json:"expectedRegex"` } // The representation of a code block in a markdown file. type CodeBlock struct { - Language string - Content string - Header string - Description string - ExpectedOutput ExpectedOutputBlock + Language string `json:"language"` + Content string `json:"content"` + Header string `json:"header"` + Description string `json:"description"` + ExpectedOutput ExpectedOutputBlock `json:"expectedOutput"` } // Assumes the title of the scenario is the first h1 header in the From 4562f1c44824fa4815b8031742b7c435c1af3557 Mon Sep 17 00:00:00 2001 From: vmarcella Date: Fri, 2 Aug 2024 12:51:36 -0700 Subject: [PATCH 06/11] [update] name of report flag. --- cmd/ie/commands/test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ie/commands/test.go b/cmd/ie/commands/test.go index 40c8eade..a7284145 100644 --- a/cmd/ie/commands/test.go +++ b/cmd/ie/commands/test.go @@ -21,7 +21,7 @@ func init() { testCommand.PersistentFlags(). String("working-directory", ".", "Sets the working directory for innovation engine to operate out of. Restores the current working directory when finished.") testCommand.PersistentFlags(). - String("generate-report", "", "The path to generate a report of the scenario execution. The contents of the report are in JSON and will only be generated when this flag is set.") + String("report", "", "The path to generate a report of the scenario execution. The contents of the report are in JSON and will only be generated when this flag is set.") testCommand.PersistentFlags(). StringArray("var", []string{}, "Sets an environment variable for the scenario. Format: --var =") From c37c4ef5f10500963d8e2b15981e0e55a7de5c4a Mon Sep 17 00:00:00 2001 From: vmarcella Date: Fri, 2 Aug 2024 13:14:04 -0700 Subject: [PATCH 07/11] [update] report to be written to JSON file, rename some tags, and fix error handling during report generation. --- internal/engine/common/codeblock.go | 1 + internal/engine/common/reports.go | 25 +++++++++++++++------- internal/engine/engine.go | 32 +++++++++++++++++++---------- internal/engine/test/model.go | 1 - internal/lib/maps.go | 27 ++++++++++++++---------- internal/parsers/markdown.go | 7 ++++--- 6 files changed, 60 insertions(+), 33 deletions(-) diff --git a/internal/engine/common/codeblock.go b/internal/engine/common/codeblock.go index 9942ff27..89360233 100644 --- a/internal/engine/common/codeblock.go +++ b/internal/engine/common/codeblock.go @@ -13,6 +13,7 @@ type StatefulCodeBlock struct { StepName string `json:"stepName"` StepNumber int `json:"stepNumber"` Success bool `json:"success"` + SimilarityScore float64 `json:"similarityScore"` } // Checks if a codeblock was executed by looking at the diff --git a/internal/engine/common/reports.go b/internal/engine/common/reports.go index 9e5f88ab..3b9befe3 100644 --- a/internal/engine/common/reports.go +++ b/internal/engine/common/reports.go @@ -2,13 +2,11 @@ package common import ( "encoding/json" - "fmt" + "os" + + "github.com/Azure/InnovationEngine/internal/logging" ) -// Reports are summaries of the execution of a markdown document. They will -// include the name of the document, the properties found in the yaml header of -// the document, the environment variables set by the document, and general -// information about the execution. type Report struct { Name string `json:"name"` Properties map[string]interface{} `json:"properties"` @@ -16,7 +14,7 @@ type Report struct { Success bool `json:"success"` Error string `json:"error"` FailedAtStep int `json:"failedAtStep"` - CodeBlocks []StatefulCodeBlock `json:"codeBlocks"` + CodeBlocks []StatefulCodeBlock `json:"steps"` } func (report *Report) WithProperties(properties map[string]interface{}) *Report { @@ -50,8 +48,21 @@ func (report *Report) WriteToJSONFile(outputPath string) error { if err != nil { return err } + logging.GlobalLogger.Infof("Generated the test report:\n %s", jsonReport) + + file, err := os.Create(outputPath) + if err != nil { + return err + } + + defer file.Close() + + _, err = file.Write(jsonReport) + if err != nil { + return err + } - fmt.Println(string(jsonReport)) + logging.GlobalLogger.Infof("Wrote the test report to %s", outputPath) return nil } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 300d72ca..e303d857 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -27,7 +27,7 @@ type EngineConfiguration struct { Environment string WorkingDirectory string RenderValues bool - GenerateReport string + ReportFile string } type Engine struct { @@ -99,27 +99,37 @@ func (e *Engine) TestScenario(scenario *common.Scenario) error { return err } - allEnvironmentVariables, err := lib.LoadEnvironmentStateFile( - lib.DefaultEnvironmentStateFile, - ) - if err != nil { - logging.GlobalLogger.Errorf("Failed to load environment state file: %s", err) - return err - } + if e.Configuration.ReportFile != "" { + allEnvironmentVariables, err := lib.LoadEnvironmentStateFile( + lib.DefaultEnvironmentStateFile, + ) + if err != nil { + logging.GlobalLogger.Errorf("Failed to load environment state file: %s", err) + err = errors.Join(err, fmt.Errorf("failed to load environment state file: %s", err)) + return err + } - if e.Configuration.GenerateReport != "" { variablesDeclaredByScenario := lib.DiffMapsByKey( allEnvironmentVariables, initialEnvironmentVariables, ) report := common.BuildReport(scenario.Name) - report. + err = report. WithProperties(scenario.Properties). WithEnvironmentVariables(variablesDeclaredByScenario). WithError(model.GetFailure()). WithCodeBlocks(model.GetCodeBlocks()). - WriteToJSONFile(e.Configuration.GenerateReport) + WriteToJSONFile(e.Configuration.ReportFile) + if err != nil { + err = errors.Join(err, fmt.Errorf("failed to write report to file: %s", err)) + return err + } + + model.CommandLines = append( + model.CommandLines, + "Report written to "+e.Configuration.ReportFile, + ) } err = errors.Join(err, model.GetFailure()) diff --git a/internal/engine/test/model.go b/internal/engine/test/model.go index e07dbbad..6467925f 100644 --- a/internal/engine/test/model.go +++ b/internal/engine/test/model.go @@ -69,7 +69,6 @@ func (model TestModeModel) GetCodeBlocks() []common.StatefulCodeBlock { for _, codeBlock := range model.codeBlockState { codeBlocks = append(codeBlocks, codeBlock) } - fmt.Println(codeBlocks) return codeBlocks } diff --git a/internal/lib/maps.go b/internal/lib/maps.go index 3408bb81..fc1f9d7b 100644 --- a/internal/lib/maps.go +++ b/internal/lib/maps.go @@ -1,7 +1,5 @@ package lib -import "fmt" - // Makes a copy of a map func CopyMap(m map[string]string) map[string]string { result := make(map[string]string) @@ -22,23 +20,20 @@ func MergeMaps(a, b map[string]string) map[string]string { } // Compares two maps by key and returns the difference between the two. +// This comparison doesn't take into account the value of the key, only the +// presence of the key. func DiffMapsByKey(a, b map[string]string) map[string]string { diff := make(map[string]string) - aLength := len(a) - bLength := len(b) - - if aLength > bLength { + if len(a) > len(b) { for k, v := range a { if b[k] == "" && v != "" { - fmt.Print("diff: ", k, v) diff[k] = v } } } else { for k, v := range b { if a[k] == "" && v != "" { - fmt.Print("diff: ", k, v) diff[k] = v } } @@ -47,12 +42,22 @@ func DiffMapsByKey(a, b map[string]string) map[string]string { return diff } +// Compares two maps by key and returns the difference between the two based +// on the value of the key. func DiffMapsByValue(a, b map[string]string) map[string]string { diff := make(map[string]string) - for k, v := range a { - if b[k] != v { - diff[k] = v + if len(a) > len(b) { + for k, v := range a { + if b[k] != v { + diff[k] = v + } + } + } else { + for k, v := range b { + if a[k] != v { + diff[k] = v + } } } diff --git a/internal/parsers/markdown.go b/internal/parsers/markdown.go index 440e9dd5..ab964b11 100644 --- a/internal/parsers/markdown.go +++ b/internal/parsers/markdown.go @@ -34,6 +34,7 @@ func ParseMarkdownIntoAst(source []byte) ast.Node { return document } +// Extract the metadata from the AST of a markdown document. func ExtractYamlMetadataFromAst(node ast.Node) map[string]interface{} { return node.OwnerDocument().Meta() } @@ -44,8 +45,8 @@ func ExtractYamlMetadataFromAst(node ast.Node) map[string]interface{} { type ExpectedOutputBlock struct { Language string `json:"language"` Content string `json:"content"` - ExpectedSimilarity float64 `json:"expectedSimilarity"` - ExpectedRegex *regexp.Regexp `json:"expectedRegex"` + ExpectedSimilarity float64 `json:"expectedSimilarityScore"` + ExpectedRegex *regexp.Regexp `json:"expectedRegexPattern"` } // The representation of a code block in a markdown file. @@ -54,7 +55,7 @@ type CodeBlock struct { Content string `json:"content"` Header string `json:"header"` Description string `json:"description"` - ExpectedOutput ExpectedOutputBlock `json:"expectedOutput"` + ExpectedOutput ExpectedOutputBlock `json:"resultBlock"` } // Assumes the title of the scenario is the first h1 header in the From 0397c59c5400373a18ec0e283a14762ac574f678 Mon Sep 17 00:00:00 2001 From: vmarcella Date: Fri, 2 Aug 2024 13:19:11 -0700 Subject: [PATCH 08/11] [update] the name of the flag being retrieved. --- cmd/ie/commands/test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/ie/commands/test.go b/cmd/ie/commands/test.go index a7284145..88810e77 100644 --- a/cmd/ie/commands/test.go +++ b/cmd/ie/commands/test.go @@ -42,7 +42,7 @@ var testCommand = &cobra.Command{ subscription, _ := cmd.Flags().GetString("subscription") workingDirectory, _ := cmd.Flags().GetString("working-directory") environment, _ := cmd.Flags().GetString("environment") - generateReport, _ := cmd.Flags().GetString("generate-report") + generateReport, _ := cmd.Flags().GetString("report") environmentVariables, _ := cmd.Flags().GetStringArray("var") @@ -70,7 +70,7 @@ var testCommand = &cobra.Command{ CorrelationId: "", WorkingDirectory: workingDirectory, Environment: environment, - GenerateReport: generateReport, + ReportFile: generateReport, }) if err != nil { logging.GlobalLogger.Errorf("Error creating engine %s", err) From 8f6ddba1bf97eb93c163b6d48b13b9c9ee1340c4 Mon Sep 17 00:00:00 2001 From: vmarcella Date: Fri, 2 Aug 2024 13:42:45 -0700 Subject: [PATCH 09/11] [update] similarity score to be attached to the test report. --- internal/engine/common/commands.go | 33 +++++++++++++++++------------- internal/engine/common/outputs.go | 16 +++++++-------- internal/engine/execution.go | 2 +- internal/engine/test/model.go | 2 ++ 4 files changed, 30 insertions(+), 23 deletions(-) diff --git a/internal/engine/common/commands.go b/internal/engine/common/commands.go index 2e550883..ef2784cb 100644 --- a/internal/engine/common/commands.go +++ b/internal/engine/common/commands.go @@ -12,15 +12,17 @@ import ( // Emitted when a command has been executed successfully. type SuccessfulCommandMessage struct { - StdOut string - StdErr string + StdOut string + StdErr string + SimilarityScore float64 } // Emitted when a command has failed to execute. type FailedCommandMessage struct { - StdOut string - StdErr string - Error error + StdOut string + StdErr string + Error error + SimilarityScore float64 } type ExitMessage struct { @@ -49,9 +51,10 @@ func ExecuteCodeBlockAsync(codeBlock parsers.CodeBlock, env map[string]string) t if err != nil { logging.GlobalLogger.Errorf("Error executing command:\n %s", err.Error()) return FailedCommandMessage{ - StdOut: output.StdOut, - StdErr: output.StdErr, - Error: err, + StdOut: output.StdOut, + StdErr: output.StdErr, + Error: err, + SimilarityScore: 0, } } @@ -62,7 +65,7 @@ func ExecuteCodeBlockAsync(codeBlock parsers.CodeBlock, env map[string]string) t expectedRegex := codeBlock.ExpectedOutput.ExpectedRegex expectedOutputLanguage := codeBlock.ExpectedOutput.Language - outputComparisonError := CompareCommandOutputs( + score, outputComparisonError := CompareCommandOutputs( actualOutput, expectedOutput, expectedSimilarity, @@ -77,17 +80,19 @@ func ExecuteCodeBlockAsync(codeBlock parsers.CodeBlock, env map[string]string) t ) return FailedCommandMessage{ - StdOut: output.StdOut, - StdErr: output.StdErr, - Error: outputComparisonError, + StdOut: output.StdOut, + StdErr: output.StdErr, + Error: outputComparisonError, + SimilarityScore: score, } } logging.GlobalLogger.Infof("Command output to stdout:\n %s", output.StdOut) return SuccessfulCommandMessage{ - StdOut: output.StdOut, - StdErr: output.StdErr, + StdOut: output.StdOut, + StdErr: output.StdErr, + SimilarityScore: score, } } } diff --git a/internal/engine/common/outputs.go b/internal/engine/common/outputs.go index 22ea96a0..e7c2ed7d 100644 --- a/internal/engine/common/outputs.go +++ b/internal/engine/common/outputs.go @@ -18,17 +18,17 @@ func CompareCommandOutputs( expectedSimilarity float64, expectedRegex *regexp.Regexp, expectedOutputLanguage string, -) error { +) (float64, error) { if expectedRegex != nil { if !expectedRegex.MatchString(actualOutput) { - return fmt.Errorf( + return 0.0, fmt.Errorf( ui.ErrorMessageStyle.Render( fmt.Sprintf("Expected output does not match: %q.", expectedRegex), ), ) } - return nil + return 0.0, nil } if strings.ToLower(expectedOutputLanguage) == "json" { @@ -39,7 +39,7 @@ func CompareCommandOutputs( ) results, err := lib.CompareJsonStrings(actualOutput, expectedOutput, expectedSimilarity) if err != nil { - return err + return results.Score, err } logging.GlobalLogger.Debugf( @@ -49,7 +49,7 @@ func CompareCommandOutputs( ) if !results.AboveThreshold { - return fmt.Errorf( + return results.Score, fmt.Errorf( ui.ErrorMessageStyle.Render( "Expected output does not match actual output. Got: %s\n Expected: %s"), actualOutput, @@ -57,14 +57,14 @@ func CompareCommandOutputs( ) } - return nil + return results.Score, nil } // Default case, using similarity on non JSON block. score := smetrics.JaroWinkler(expectedOutput, actualOutput, 0.7, 4) if expectedSimilarity > score { - return fmt.Errorf( + return score, fmt.Errorf( ui.ErrorMessageStyle.Render( "Expected output does not match actual output.\nGot:\n%s\nExpected:\n%s\nExpected Score:%s\nActualScore:%s", ), @@ -75,5 +75,5 @@ func CompareCommandOutputs( ) } - return nil + return score, nil } diff --git a/internal/engine/execution.go b/internal/engine/execution.go index c2075152..283a4d53 100644 --- a/internal/engine/execution.go +++ b/internal/engine/execution.go @@ -183,7 +183,7 @@ func (e *Engine) ExecuteAndRenderSteps(steps []common.Step, env map[string]strin expectedRegex := block.ExpectedOutput.ExpectedRegex expectedOutputLanguage := block.ExpectedOutput.Language - outputComparisonError := common.CompareCommandOutputs(actualOutput, expectedOutput, expectedSimilarity, expectedRegex, expectedOutputLanguage) + _, outputComparisonError := common.CompareCommandOutputs(actualOutput, expectedOutput, expectedSimilarity, expectedRegex, expectedOutputLanguage) if outputComparisonError != nil { logging.GlobalLogger.Errorf("Error comparing command outputs: %s", outputComparisonError.Error()) diff --git a/internal/engine/test/model.go b/internal/engine/test/model.go index 6467925f..f1621534 100644 --- a/internal/engine/test/model.go +++ b/internal/engine/test/model.go @@ -117,6 +117,7 @@ func (model TestModeModel) Update(message tea.Msg) (tea.Model, tea.Cmd) { codeBlockState.StdOut = message.StdOut codeBlockState.StdErr = message.StdErr codeBlockState.Success = true + codeBlockState.SimilarityScore = message.SimilarityScore model.codeBlockState[step] = codeBlockState logging.GlobalLogger.Infof("Finished executing:\n %s", codeBlockState.CodeBlock.Content) @@ -174,6 +175,7 @@ func (model TestModeModel) Update(message tea.Msg) (tea.Model, tea.Cmd) { codeBlockState.StdErr = message.StdErr codeBlockState.Error = message.Error codeBlockState.Success = false + codeBlockState.SimilarityScore = message.SimilarityScore model.codeBlockState[step] = codeBlockState model.CommandLines = append(model.CommandLines, codeBlockState.StdErr+message.Error.Error()) From 1a659595647fde9a72d788185c94a4c0dd33df9f Mon Sep 17 00:00:00 2001 From: vmarcella Date: Fri, 2 Aug 2024 14:12:13 -0700 Subject: [PATCH 10/11] [fix] output comparison errors for json payloads, prevent duplicate errors from occurring inside test output after a run. --- internal/engine/common/outputs.go | 11 +++++++---- internal/engine/engine.go | 10 ++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/internal/engine/common/outputs.go b/internal/engine/common/outputs.go index e7c2ed7d..64badc89 100644 --- a/internal/engine/common/outputs.go +++ b/internal/engine/common/outputs.go @@ -51,9 +51,12 @@ func CompareCommandOutputs( if !results.AboveThreshold { return results.Score, fmt.Errorf( ui.ErrorMessageStyle.Render( - "Expected output does not match actual output. Got: %s\n Expected: %s"), - actualOutput, - expectedOutput, + "Expected output does not match actual output.\nGot:\n%s\nExpected:\n%s\nExpected Score:%s\nActual Score:%s", + ), + ui.VerboseStyle.Render(actualOutput), + ui.VerboseStyle.Render(expectedOutput), + ui.VerboseStyle.Render(fmt.Sprintf("%f", expectedSimilarity)), + ui.VerboseStyle.Render(fmt.Sprintf("%f", results.Score)), ) } @@ -66,7 +69,7 @@ func CompareCommandOutputs( if expectedSimilarity > score { return score, fmt.Errorf( ui.ErrorMessageStyle.Render( - "Expected output does not match actual output.\nGot:\n%s\nExpected:\n%s\nExpected Score:%s\nActualScore:%s", + "Expected output does not match actual output.\nGot:\n%s\nExpected:\n%s\nExpected Score:%s\nActual Score:%s", ), ui.VerboseStyle.Render(actualOutput), ui.VerboseStyle.Render(expectedOutput), diff --git a/internal/engine/engine.go b/internal/engine/engine.go index e303d857..bdf918c0 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -93,17 +93,16 @@ func (e *Engine) TestScenario(scenario *common.Scenario) error { // TODO(vmarcella): After testing is complete, we should generate a report. model, ok := finalModel.(test.TestModeModel) - if !ok { err = errors.Join(err, fmt.Errorf("failed to cast tea.Model to TestModeModel")) return err } if e.Configuration.ReportFile != "" { - allEnvironmentVariables, err := lib.LoadEnvironmentStateFile( + allEnvironmentVariables, envErr := lib.LoadEnvironmentStateFile( lib.DefaultEnvironmentStateFile, ) - if err != nil { + if envErr != nil { logging.GlobalLogger.Errorf("Failed to load environment state file: %s", err) err = errors.Join(err, fmt.Errorf("failed to load environment state file: %s", err)) return err @@ -133,10 +132,13 @@ func (e *Engine) TestScenario(scenario *common.Scenario) error { } err = errors.Join(err, model.GetFailure()) + if err != nil { + logging.GlobalLogger.Errorf("Failed to run ie test %s", err) + } fmt.Println(strings.Join(model.CommandLines, "\n")) - return err + return nil }) } From 0c141b1fc64dabd74686582cd4515c8484312f40 Mon Sep 17 00:00:00 2001 From: vmarcella Date: Fri, 2 Aug 2024 14:12:44 -0700 Subject: [PATCH 11/11] [add] specification and example doc used in it. --- docs/specs/test-reporting.md | 216 +++++++++++++++++++++++++++++++++ scenarios/testing/reporting.md | 28 +++++ 2 files changed, 244 insertions(+) create mode 100644 docs/specs/test-reporting.md create mode 100644 scenarios/testing/reporting.md diff --git a/docs/specs/test-reporting.md b/docs/specs/test-reporting.md new file mode 100644 index 00000000..98a59c87 --- /dev/null +++ b/docs/specs/test-reporting.md @@ -0,0 +1,216 @@ +# Test Reports + +## Summary + +When users are testing their executable documentation using `ie test`, being +able to see the results of their execution is important, especially in instances +where the test fails. At the moment, there are only two ways to see what +happened during the test execution: + +1. The user can look through the standard output from `ie test ` + (Most common). +1. The user can look through `ie.log` (Not as common). + +While these methods are effective for troubleshooting most issues, they also have +a few issues: + +1. Storing the output of `ie test` in a file doesn't provide a good way to + navigate the output and see the results of the test. +1. The log file `ie.log` is not user-friendly and can be difficult to navigate, especially + so if the user is invoking multiple `ie test` commands as the log file is cumulative. +1. It's not easy to reproduce the execution of a specific scenario, as most of + the variables declared by the scenario are randomized and don't have their + values rendered in the output. +1. The output of `ie test` is not easily shareable with others. + +To address these issues, we propose the introduction of test reports, a feature +for `ie test` that will generate a report of the scenario execution in JSON +format so that users can easily navigate the results of the test, reproduce +specific runs, and share the results with others. + +## Requirements + +- [x] The user can generate a test report by running `ie test //report=` +- [x] Reports capture the yaml metadata of the scenario. +- [x] Reports store the variables declared in the scenario and their values. +- [x] The report is generated in JSON format. +- [ ] Just like the scenarios that generated them, Reports are executable. +- [x] Outputs of the codeblocks executed are stored in the report. +- [x] Expected outputs for codeblocks are stored in the report. + +## Technical specifications + +- The report will be generated in JSON format, but in the future we may consider + other formats like yaml or HTML. JSON format was chosen for v1 because it is + easy to parse, and it is a common format for sharing data. +- Users must specify `//report=` to generate a report. If the path is not + specified, the report will not be generated. + +### Report schema + +The actual JSON schema is a work in progress, and will not be released with +the initial implementation, so we will list out the actual JSON with +documentation about each field until then. + +```json +{ + // Name of the scenario + "name": "Test reporting doc", + // Properties found in the yaml header + "properties": { + "ms.author": "vmarcella", + "otherProperty": "otherValue" + }, + + // Variables declared in the scenario + "environmentVariables": { + "NEW_VAR": "1" + }, + // Whether the test was successful or not + "success": true, + // Error message if the test failed + "error": "", + // The step number where the test failed (-1 if successful) + "failedAtStep": -1, + "steps": [ + // The entire step + { + // The codeblock for the step + "codeBlock": { + // The language of the codeblock + "language": "bash", + // The content of the codeblock + "content": "echo \"Hello, world!\"\n", + // The header paired with the codeblock + "header": "First step", + // The paragraph paired with the codeblock + "description": "This step will show you how to do something.", + // The expected output for the codeblock + "resultBlock": { + // The language of the expected output + "language": "text", + // The content of the expected output + "content": "Hello, world!\n", + // The expected similarity score of the output (between 0 - 1) + "expectedSimilarityScore": 1, + // The expected regex pattern of the output + "expectedRegexPattern": null + } + }, + // Codeblock number underneath the step (Should be ignored for now) + "codeBlockNumber": 0, + // Error message if the step failed (Would be same as top level error) + "error": null, + // Standard error output from executing the step + "stdErr": "", + // Standard output from executing the step + "stdOut": "Hello, world!\n", + // The name of the step + "stepName": "First step", + // The step number + "stepNumber": 0, + // Whether the step was successful or not + "success": true, + // The computed similarity score of the output (between 0 - 1) + "similarityScore": 0 + }, + { + "codeBlock": { + "language": "bash", + "content": "export NEW_VAR=1\n", + "header": "Second step", + "description": "This step will show you how to do something else.", + "resultBlock": { + "language": "", + "content": "", + "expectedSimilarityScore": 0, + "expectedRegexPattern": null + } + }, + "codeBlockNumber": 0, + "error": null, + "stdErr": "", + "stdOut": "", + "stepName": "Second step", + "stepNumber": 1, + "success": true, + "similarityScore": 0 + } + ] +} +``` + +## Examples + +Assuming you're running this command from the root of the repository: + +```bash +ie test scenarios/testing/reporting.md --report=report.json >/dev/null && cat report.json +``` + +The output of the command above should look like this: + + + + +```json +{ + "name": "Test reporting doc", + "properties": { + "ms.author": "vmarcella", + "otherProperty": "otherValue" + }, + "environmentVariables": { + "NEW_VAR": "1" + }, + "success": true, + "error": "", + "failedAtStep": -1, + "steps": [ + { + "codeBlock": { + "language": "bash", + "content": "echo \"Hello, world!\"\n", + "header": "First step", + "description": "This step will show you how to do something.", + "resultBlock": { + "language": "text", + "content": "Hello, world!\n", + "expectedSimilarityScore": 1, + "expectedRegexPattern": null + } + }, + "codeBlockNumber": 0, + "error": null, + "stdErr": "", + "stdOut": "Hello, world!\n", + "stepName": "First step", + "stepNumber": 0, + "success": true, + "similarityScore": 1 + }, + { + "codeBlock": { + "language": "bash", + "content": "export NEW_VAR=1\n", + "header": "Second step", + "description": "This step will show you how to do something else.", + "resultBlock": { + "language": "", + "content": "", + "expectedSimilarityScore": 0, + "expectedRegexPattern": null + } + }, + "codeBlockNumber": 0, + "error": null, + "stdErr": "", + "stdOut": "", + "stepName": "Second step", + "stepNumber": 1, + "success": true, + "similarityScore": 1 + } + ] +} +``` diff --git a/scenarios/testing/reporting.md b/scenarios/testing/reporting.md new file mode 100644 index 00000000..e68d0660 --- /dev/null +++ b/scenarios/testing/reporting.md @@ -0,0 +1,28 @@ +--- +ms.author: vmarcella +otherProperty: otherValue +--- + +# Test reporting doc + +## First step + +This step will show you how to do something. + +```bash +echo "Hello, world!" +``` + + + +```text +Hello, world! +``` + +## Second step + +This step will show you how to do something else. + +```bash +export NEW_VAR=1 +```