feat(test): New UI implementation for ie test & refactor various parts of the engine (#198)

This PR:
* Updates `ie test` to use bubbletea for rendering it's output, similar
to `ie interactive` and soon `ie execute`.
* Refactors `ie interactive` and `ie test` implementations to exist in
their own folders inside of `engine` (Which will be renamed to `ie` in
the future).
* Refactors code commonly used across execute, interactive, and test
modes into a new package found in `internal/engine/common`.
* Decoupled models from requiring a pointer to `Engine` as a parameter
in preparation for when it will be deprecated in the future.
* Reimplements `shells.ExecuteCodeBlock` to be a variable alias of
`shells.executeCodeBlockImpl`. This allows for us to easily mock the the
`ExecuteCodeBlock` implementation easily in tests. This allows us to do
things like track calls to commands, record what commands were failed,
mimic failures from executing commands, etc without having to actually
execute commands. This is particularly useful in cases where we want to
test the behavior of executing an Azure CLI command without actually
executing the command itself.
* Fixed an issue with the regex used to locate resource groups inside of
command outputs and added tests to ensure that it works.
* Adds a new environment `github-action`, specifically for running `ie
test` inside of github actions runners. If this flag is not supplied,
`ie test` will crash due to bubbletea attempting to open tty when there
are no ttys available. (See actions/runner#241
for more information on GH actions not providing a TTY).
*

Author: vmarcella

.github/workflows/scenario-testing.yaml

@@ -39,10 +39,11 @@ jobs:
     environment: ScenarioTesting
     steps:
       - uses: actions/checkout@v2
-      - name: Build all targets.
+      - name: Build & test all targets.
         run: |
           make build-all
           make test-all WITH_COVERAGE=true
+          make test-local-scenarios ENVIRONMENT=github-action
       - name: Upload test coverage
         uses: actions/upload-artifact@v2
         if: github.event_name == 'pull_request'
@@ -65,6 +66,6 @@ jobs:
             apk add --no-cache make git openssh openssl helm curl jq
             make test-upstream-scenarios SUBSCRIPTION=${{ secrets.AZURE_SUBSCRIPTION }}
       - name: Display ie.log file
-        if: (success() || failure()) && github.event_name != 'pull_request'
+        if: (success() || failure())
         run: |
           cat ie.log

Makefile

@@ -22,12 +22,11 @@ install-ie:
 # ------------------------------ Test targets ----------------------------------
 
 WITH_COVERAGE := false
-
 test-all:
 	@go clean -testcache
 ifeq ($(WITH_COVERAGE), true)
 	@echo "Running all tests with coverage..."
-	@go test -v -coverprofile=coverage.out ./...
+	@go test -v -coverpkg=./... -coverprofile=coverage.out ./...
 	@go tool cover -html=coverage.out -o coverage.html
 else
 	@echo "Running all tests..."
@@ -38,16 +37,27 @@ endif
 SUBSCRIPTION ?= 00000000-0000-0000-0000-000000000000
 SCENARIO ?= ./README.md
 WORKING_DIRECTORY ?= $(PWD)
+ENVIRONMENT ?= local
 test-scenario:
 	@echo "Running scenario $(SCENARIO)"
-	$(IE_BINARY) test $(SCENARIO) --subscription $(SUBSCRIPTION) --working-directory $(WORKING_DIRECTORY)
+ifeq ($(SUBSCRIPTION), 00000000-0000-0000-0000-000000000000)
+	$(IE_BINARY) test $(SCENARIO) --working-directory $(WORKING_DIRECTORY) --environment $(ENVIRONMENT)
+else
+	$(IE_BINARY) test $(SCENARIO) --subscription $(SUBSCRIPTION) --working-directory $(WORKING_DIRECTORY) --enviroment $(ENVIRONMENT)
+endif
 
 test-scenarios:
 	@echo "Testing out the scenarios"
 	for dir in ./scenarios/ocd/*/; do \
 		($(MAKE) test-scenario SCENARIO="$${dir}README.md" SUBCRIPTION="$(SUBSCRIPTION)") || exit $$?; \
 	done
 
+test-local-scenarios:
+	@echo "Testing out the local scenarios"
+	for file in ./scenarios/testing/*.md; do \
+		($(MAKE) test-scenario SCENARIO="$${file}") || exit $$?; \
+	done
+
 test-upstream-scenarios:
 	@echo "Pulling the upstream scenarios"
 	@git config --global --add safe.directory /home/runner/work/InnovationEngine/InnovationEngine

cmd/ie/commands/execute.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/Azure/InnovationEngine/internal/engine"
+	"github.com/Azure/InnovationEngine/internal/engine/common"
 	"github.com/Azure/InnovationEngine/internal/logging"
 	"github.com/spf13/cobra"
 )
@@ -92,7 +93,7 @@ var executeCommand = &cobra.Command{
 		}
 
 		// Parse the markdown file and create a scenario
-		scenario, err := engine.CreateScenarioFromMarkdown(
+		scenario, err := common.CreateScenarioFromMarkdown(
 			markdownFile,
 			[]string{"bash", "azurecli", "azurecli-interactive", "terraform"},
 			cliEnvironmentVariables,
@@ -112,7 +113,6 @@ var executeCommand = &cobra.Command{
 			WorkingDirectory: workingDirectory,
 			RenderValues:     renderValues,
 		})
-
 		if err != nil {
 			logging.GlobalLogger.Errorf("Error creating engine: %s", err)
 			fmt.Printf("Error creating engine: %s", err)

cmd/ie/commands/inspect.go

@@ -5,7 +5,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/Azure/InnovationEngine/internal/engine"
+	"github.com/Azure/InnovationEngine/internal/engine/common"
 	"github.com/Azure/InnovationEngine/internal/logging"
 	"github.com/Azure/InnovationEngine/internal/ui"
 	"github.com/spf13/cobra"
@@ -59,7 +59,7 @@ var inspectCommand = &cobra.Command{
 			cliEnvironmentVariables[keyValuePair[0]] = keyValuePair[1]
 		}
 		// Parse the markdown file and create a scenario
-		scenario, err := engine.CreateScenarioFromMarkdown(
+		scenario, err := common.CreateScenarioFromMarkdown(
 			markdownFile,
 			[]string{"bash", "azurecli", "azurecli-inspect", "terraform"},
 			cliEnvironmentVariables,
@@ -104,6 +104,5 @@ var inspectCommand = &cobra.Command{
 				fmt.Println()
 			}
 		}
-
 	},
 }

cmd/ie/commands/interactive.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/Azure/InnovationEngine/internal/engine"
+	"github.com/Azure/InnovationEngine/internal/engine/common"
 	"github.com/Azure/InnovationEngine/internal/logging"
 	"github.com/spf13/cobra"
 )
@@ -69,7 +70,7 @@ var interactiveCommand = &cobra.Command{
 			cliEnvironmentVariables[keyValuePair[0]] = keyValuePair[1]
 		}
 		// Parse the markdown file and create a scenario
-		scenario, err := engine.CreateScenarioFromMarkdown(
+		scenario, err := common.CreateScenarioFromMarkdown(
 			markdownFile,
 			[]string{"bash", "azurecli", "azurecli-interactive", "terraform"},
 			cliEnvironmentVariables,
@@ -89,7 +90,6 @@ var interactiveCommand = &cobra.Command{
 			WorkingDirectory: workingDirectory,
 			RenderValues:     renderValues,
 		})
-
 		if err != nil {
 			logging.GlobalLogger.Errorf("Error creating engine: %s", err)
 			fmt.Printf("Error creating engine: %s", err)

cmd/ie/commands/test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 
 	"github.com/Azure/InnovationEngine/internal/engine"
+	"github.com/Azure/InnovationEngine/internal/engine/common"
 	"github.com/Azure/InnovationEngine/internal/logging"
 	"github.com/spf13/cobra"
 )
@@ -23,9 +24,8 @@ func init() {
 var testCommand = &cobra.Command{
 	Use:   "test",
 	Args:  cobra.MinimumNArgs(1),
-	Short: "Test document commands against it's expected outputs.",
+	Short: "Test document commands against their expected outputs.",
 	Run: func(cmd *cobra.Command, args []string) {
-
 		markdownFile := args[0]
 		if markdownFile == "" {
 			cmd.Help()
@@ -35,22 +35,23 @@ var testCommand = &cobra.Command{
 		verbose, _ := cmd.Flags().GetBool("verbose")
 		subscription, _ := cmd.Flags().GetString("subscription")
 		workingDirectory, _ := cmd.Flags().GetString("working-directory")
+		environment, _ := cmd.Flags().GetString("environment")
 
 		innovationEngine, err := engine.NewEngine(engine.EngineConfiguration{
 			Verbose:          verbose,
 			DoNotDelete:      false,
 			Subscription:     subscription,
 			CorrelationId:    "",
 			WorkingDirectory: workingDirectory,
+			Environment:      environment,
 		})
-
 		if err != nil {
 			logging.GlobalLogger.Errorf("Error creating engine %s", err)
 			fmt.Printf("Error creating engine %s", err)
 			os.Exit(1)
 		}
 
-		scenario, err := engine.CreateScenarioFromMarkdown(
+		scenario, err := common.CreateScenarioFromMarkdown(
 			markdownFile,
 			[]string{"bash", "azurecli", "azurecli-interactive", "terraform"},
 			nil,

cmd/ie/commands/to-bash.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/Azure/InnovationEngine/internal/engine"
+	"github.com/Azure/InnovationEngine/internal/engine/common"
 	"github.com/Azure/InnovationEngine/internal/engine/environments"
 	"github.com/Azure/InnovationEngine/internal/logging"
 	"github.com/spf13/cobra"
@@ -50,11 +50,10 @@ var toBashCommand = &cobra.Command{
 		}
 
 		// Parse the markdown file and create a scenario
-		scenario, err := engine.CreateScenarioFromMarkdown(
+		scenario, err := common.CreateScenarioFromMarkdown(
 			markdownFile,
 			[]string{"bash", "azurecli", "azurecli-interactive", "terraform"},
 			cliEnvironmentVariables)
-
 		if err != nil {
 			logging.GlobalLogger.Errorf("Error creating scenario: %s", err)
 			fmt.Printf("Error creating scenario: %s", err)
@@ -66,7 +65,6 @@ var toBashCommand = &cobra.Command{
 		if environments.IsAzureEnvironment(environment) {
 			script := AzureScript{Script: scenario.ToShellScript()}
 			scriptJson, err := json.Marshal(script)
-
 			if err != nil {
 				logging.GlobalLogger.Errorf("Error converting to json: %s", err)
 				fmt.Printf("Error converting to json: %s", err)
@@ -79,7 +77,6 @@ var toBashCommand = &cobra.Command{
 		}
 
 		return nil
-
 	},
 }
 

internal/az/group.go

@@ -17,7 +17,6 @@ func FindAllDeployedResourceURIs(resourceGroup string) []string {
 			WriteToHistory:       true,
 		},
 	)
-
 	if err != nil {
 		logging.GlobalLogger.Error("Failed to list deployments", err)
 	}

internal/az/group_test.go

@@ -0,0 +1,46 @@
+package az
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFindingResourceGroups(t *testing.T) {
+	testCases := []struct {
+		resourceGroupString       string
+		expectedResourceGroupName string
+	}{
+		// RG string that ends with a slash
+		{
+			resourceGroupString:       "resourceGroups/rg1/",
+			expectedResourceGroupName: "rg1",
+		},
+		// RG string that ends with a space and starts new text
+		{
+			resourceGroupString:       "resourceGroups/rg1 /subscriptions/",
+			expectedResourceGroupName: "rg1",
+		},
+
+		// RG string that includes nested resources and extraneous text.
+		{
+			resourceGroupString:       "/subscriptions/9b70acd9-975f-44ba-bad6-255a2c8bda37/resourceGroups/myResourceGroup-rg/providers/Microsoft.ContainerRegistry/registries/mydnsrandomnamebbbhe  ffc55a9e-ed2a-4b60-b034-45228dfe7db5  2024-06-11T09:41:36.631310+00:00",
+			expectedResourceGroupName: "myResourceGroup-rg",
+		},
+		// RG string that is surrounded by quotes.
+		{
+			resourceGroupString:       `"id": "/subscriptions/0a2c89a7-a44e-4cd0-b6ec-868432ad1d13/resourceGroups/myResourceGroup"`,
+			expectedResourceGroupName: "myResourceGroup",
+		},
+		// RG string that has no match.
+		{
+			resourceGroupString:       "NoMatch",
+			expectedResourceGroupName: "",
+		},
+	}
+
+	for _, tc := range testCases {
+		resourceGroupName := FindResourceGroupName(tc.resourceGroupString)
+		assert.Equal(t, tc.expectedResourceGroupName, resourceGroupName)
+	}
+}

internal/engine/common/codeblock.go

@@ -0,0 +1,22 @@
+package common
+
+import "github.com/Azure/InnovationEngine/internal/parsers"
+
+// State for the codeblock in interactive mode. Used to keep track of the
+// state of each codeblock.
+type StatefulCodeBlock struct {
+	CodeBlock       parsers.CodeBlock
+	CodeBlockNumber int
+	Error           error
+	StdErr          string
+	StdOut          string
+	StepName        string
+	StepNumber      int
+	Success         bool
+}
+
+// Checks if a codeblock was executed by looking at the
+// output, errors, and if success is true.
+func (s StatefulCodeBlock) WasExecuted() bool {
+	return s.StdOut != "" || s.StdErr != "" || s.Error != nil || s.Success
+}

internal/engine/commands.go internal/engine/common/commands.go

@@ -1,4 +1,4 @@
-package engine
+package common
 
 import (
 	"fmt"
@@ -23,6 +23,16 @@ type FailedCommandMessage struct {
 	Error  error
 }
 
+type ExitMessage struct {
+	EncounteredFailure bool
+}
+
+func Exit(encounteredFailure bool) tea.Cmd {
+	return func() tea.Msg {
+		return ExitMessage{EncounteredFailure: encounteredFailure}
+	}
+}
+
 // Executes a bash command and returns a tea message with the output. This function
 // will be executed asycnhronously.
 func ExecuteCodeBlockAsync(codeBlock parsers.CodeBlock, env map[string]string) tea.Cmd {
@@ -52,7 +62,7 @@ func ExecuteCodeBlockAsync(codeBlock parsers.CodeBlock, env map[string]string) t
 		expectedRegex := codeBlock.ExpectedOutput.ExpectedRegex
 		expectedOutputLanguage := codeBlock.ExpectedOutput.Language
 
-		outputComparisonError := compareCommandOutputs(
+		outputComparisonError := CompareCommandOutputs(
 			actualOutput,
 			expectedOutput,
 			expectedSimilarity,
@@ -86,16 +96,19 @@ func ExecuteCodeBlockAsync(codeBlock parsers.CodeBlock, env map[string]string) t
 // finishes executing.
 func ExecuteCodeBlockSync(codeBlock parsers.CodeBlock, env map[string]string) tea.Msg {
 	logging.GlobalLogger.Info("Executing command synchronously: ", codeBlock.Content)
-	program.ReleaseTerminal()
+	Program.ReleaseTerminal()
 
-	output, err := shells.ExecuteBashCommand(codeBlock.Content, shells.BashCommandConfiguration{
-		EnvironmentVariables: env,
-		InheritEnvironment:   true,
-		InteractiveCommand:   true,
-		WriteToHistory:       true,
-	})
+	output, err := shells.ExecuteBashCommand(
+		codeBlock.Content,
+		shells.BashCommandConfiguration{
+			EnvironmentVariables: env,
+			InheritEnvironment:   true,
+			InteractiveCommand:   true,
+			WriteToHistory:       true,
+		},
+	)
 
-	program.RestoreTerminal()
+	Program.RestoreTerminal()
 
 	if err != nil {
 		return FailedCommandMessage{
@@ -113,7 +126,7 @@ func ExecuteCodeBlockSync(codeBlock parsers.CodeBlock, env map[string]string) te
 }
 
 // clearScreen returns a command that clears the terminal screen and positions the cursor at the top-left corner
-func clearScreen() tea.Cmd {
+func ClearScreen() tea.Cmd {
 	return func() tea.Msg {
 		fmt.Print(
 			"\033[H\033[2J",
@@ -124,13 +137,13 @@ func clearScreen() tea.Cmd {
 
 // Updates the azure status with the current state of the interactive mode
 // model.
-func updateAzureStatus(model InteractiveModeModel) tea.Cmd {
+func UpdateAzureStatus(azureStatus environments.AzureDeploymentStatus, environment string) tea.Cmd {
 	return func() tea.Msg {
 		logging.GlobalLogger.Tracef(
 			"Attempting to update the azure status: %+v",
-			model.azureStatus,
+			azureStatus,
 		)
-		environments.ReportAzureStatus(model.azureStatus, model.environment)
+		environments.ReportAzureStatus(azureStatus, environment)
 		return AzureStatusUpdatedMessage{}
 	}
 }

internal/engine/common/globals.go

@@ -0,0 +1,9 @@
+package common
+
+import tea "github.com/charmbracelet/bubbletea"
+
+// TODO: Ideally we won't need a global program variable. We should
+// refactor this in the future such that each tea program is localized to the
+// function that creates it and ExecuteCodeBlockSync doesn't mutate the global
+// program variable.
+var Program *tea.Program = nil