wip: test passes with stuff commented out because dry-run prints the wrong format still

Signed-off-by: Carolyn Van Slyck <[email protected]>

Author: carolynvs

pkg/porter/porter.go

@@ -64,6 +64,7 @@ func NewFor(c *config.Config, store storage.Store, secretStorage secrets.Store)
 	credStorage := storage.NewCredentialStore(storageManager, secretStorage)
 	paramStorage := storage.NewParameterStore(storageManager, secretStorage)
 	sanitizerService := storage.NewSanitizer(paramStorage, secretStorage)
+	secretStorage.SetPorterStrategy(NewPorterSecretStrategy(installationStorage, sanitizerService))
 	storageManager.Initialize(sanitizerService) // we have a bit of a dependency problem here that it would be great to figure out eventually
 
 	return &Porter{

pkg/porter/porter_strategy.go

@@ -2,8 +2,8 @@ package porter
 
 import (
 	"context"
-	"fmt"
-	"regexp"
+
+	"go.mongodb.org/mongo-driver/bson"
 
 	"get.porter.sh/porter/pkg/storage"
 
@@ -16,12 +16,15 @@ import (
 // It is not written as a plugin because it is much more straightforward to
 // retrieve the data already loaded in the running Porter instance than to start
 // another one, load its config and requery the database.
+// This should always run in-process within porter and never as an out-of-process plugin.
 type PorterSecretStrategy struct {
 	installations storage.InstallationProvider
+	sanitizer     *storage.Sanitizer
 }
 
-// regular expression for parsing a workflow wiring string, such as workflow.jobs.db.outputs.connstr
-var workflowWiringRegex = regexp.MustCompile(`workflow\.jobs\.([^\.]+)\.(.+)`)
+func NewPorterSecretStrategy(installations storage.InstallationProvider, sanitizer *storage.Sanitizer) PorterSecretStrategy {
+	return PorterSecretStrategy{installations: installations, sanitizer: sanitizer}
+}
 
 func (s PorterSecretStrategy) Resolve(ctx context.Context, keyName string, keyValue string) (string, error) {
 	ctx, span := tracing.StartSpan(ctx)
@@ -30,23 +33,75 @@ func (s PorterSecretStrategy) Resolve(ctx context.Context, keyName string, keyVa
 	// TODO(PEP003): It would be great when we configure this strategy that we also do host, so that host secret resolution isn't deferred to the plugins
 	// i.e. we can configure a secret strategy and still be able to resolve directly in porter any host values.
 	if keyName != "porter" {
-		return "", fmt.Errorf("attempted to resolve secrets of type %s from the porter strategy", keyName)
+		return "", span.Errorf("attempted to resolve secrets of type %s from the porter strategy", keyName)
 	}
 
 	wiring, err := v2.ParseWorkflowWiring(keyValue)
 	if err != nil {
-		return "", fmt.Errorf("invalid workflow wiring was passed to the porter strategy, %s", keyValue)
+		return "", span.Errorf("invalid workflow wiring was passed to the porter strategy, %s", keyValue)
 	}
 
+	// We support retrieving certain data from Porter's database:
+	// workflow.WORKFLOWID.jobs.JOBKEY.outputs.OUTPUT
+	// The WORKFLOWID is set to the current executing workflow by the workflow engine before running the job
+	// It is not stored in the database and is always set dynamically when the job is run.
+
 	// TODO(PEP003): How do we want to re-resolve credentials passed to the root bundle? They aren't recorded so it's not a simple lookup
 	if wiring.Parameter != "" {
 		// TODO(PEP003): Resolve a parameter from another job that has not run yet
+		// IS THIS ACTUALLY A PROBLEM? We pass creds/params from the root job, which we need to deal with, but otherwise we only pass outputs from non-root jobs
 		// 1. Find the workflow definition from the db (need a way to track "current" workflow)
 		// 2. Grab the job based on the jobid in the workflow wiring
 		// 3. First check the parameters field for the param, resolve just that if available, otherwise resolve parameter sets and get it from there
 		// it sure would help if we remembered what params are in each set
+
+		return "", nil
 	} else if wiring.Output != "" {
 		// TODO(PEP003): Resolve the output from an already executed job
+
+		// Lookup the result and run associated with the job run in that workflow
+		w, err := s.installations.GetWorkflow(ctx, wiring.WorkflowID)
+		if err != nil {
+			return "", span.Errorf("error retrieving workflow %s: %w", wiring.WorkflowID, err)
+		}
+
+		// Prepare internal data structures of the workflow
+		w.Prepare()
+
+		// locate the job in the workflow
+		j, err := w.GetJob(wiring.JobKey)
+		if err != nil {
+			return "", span.Errorf("error retrieving job from workflow %s: %w", wiring.WorkflowID)
+		}
+
+		if j.Status.LastResultID == "" {
+			return "", span.Errorf("error retrieving job status for %s in workflow %s, no result recorded yet", wiring.JobKey, wiring.WorkflowID)
+		}
+
+		outputs, err := s.installations.FindOutputs(ctx, storage.FindOptions{
+			Sort:  []string{"-_id"},
+			Skip:  0,
+			Limit: 1,
+			Filter: bson.M{
+				"resultId": j.Status.LastResultID,
+				"name":     wiring.Output,
+			},
+		})
+		if err != nil {
+			// TODO(PEP003): Move a lot of these values into the span attributes instead of in the error message
+			return "", span.Errorf("error retrieving output %s from result %s for job %s in workflow %s: %w", wiring.Output, j.Status.LastResultID, wiring.JobKey, wiring.WorkflowID)
+		}
+
+		if len(outputs) == 0 {
+			return "", span.Errorf("no output named %s, found for result %s for job %s in workflow %s", wiring.Output, j.Status.LastResultID, wiring.JobKey, wiring.WorkflowID)
+		}
+
+		output, err := s.sanitizer.RestoreOutput(ctx, outputs[1])
+		if err != nil {
+			return "", span.Errorf("error restoring output named %s, found for result %s for job %s in workflow %s", wiring.Output, j.Status.LastResultID, wiring.JobKey, wiring.WorkflowID)
+		}
+
+		return string(output.Value), nil
 	}
 
 	panic("not implemented")

pkg/runtime/runtime_manifest.go

@@ -45,8 +45,9 @@ type RuntimeManifest struct {
 	// do advanced stuff with the manifest, like just read out the yaml for a particular step.
 	editor *yaml.Editor
 
-	// bundles is map of the dependencies bundle definitions, keyed by the alias used in the root manifest
-	bundles map[string]cnab.ExtendedBundle
+	// depsv1_bundles is map of the dependencies bundle definitions, keyed by the alias used in the root manifest
+	// This is only populated for a bundle built for dependencies v1
+	depsv1_bundles map[string]cnab.ExtendedBundle
 
 	steps           manifest.Steps
 	outputs         map[string]string
@@ -74,9 +75,12 @@ func (m *RuntimeManifest) Validate() error {
 		return err
 	}
 
-	err = m.loadDependencyDefinitions()
-	if err != nil {
-		return err
+	fmt.Println("carolyn was here")
+	if m.bundle.HasDependenciesV1() {
+		err = m.loadDependencyDefinitions()
+		if err != nil {
+			return err
+		}
 	}
 
 	err = m.setStepsByAction()
@@ -112,7 +116,11 @@ func (m *RuntimeManifest) GetInstallationName() string {
 }
 
 func (m *RuntimeManifest) loadDependencyDefinitions() error {
-	m.bundles = make(map[string]cnab.ExtendedBundle, len(m.Dependencies.Requires))
+	if !m.bundle.HasDependenciesV1() {
+		return nil
+	}
+
+	m.depsv1_bundles = make(map[string]cnab.ExtendedBundle, len(m.Dependencies.Requires))
 	for _, dep := range m.Dependencies.Requires {
 		bunD, err := GetDependencyDefinition(m.config.Context, dep.Name)
 		if err != nil {
@@ -124,7 +132,7 @@ func (m *RuntimeManifest) loadDependencyDefinitions() error {
 			return fmt.Errorf("error unmarshaling bundle definition for dependency %s: %w", dep.Name, err)
 		}
 
-		m.bundles[dep.Name] = cnab.NewBundle(*bun)
+		m.depsv1_bundles[dep.Name] = cnab.NewBundle(*bun)
 	}
 
 	return nil
@@ -290,15 +298,17 @@ func (m *RuntimeManifest) buildSourceData() (map[string]interface{}, error) {
 	}
 
 	deps := make(map[string]interface{})
-	bun["dependencies"] = deps
-	for alias, depB := range m.bundles {
-		// bundle.dependencies.ALIAS.outputs.NAME
-		depBun := make(map[string]interface{})
-		deps[alias] = depBun
+	if m.bundle.HasDependenciesV1() {
+		bun["dependencies"] = deps
+		for alias, depB := range m.depsv1_bundles {
+			// bundle.dependencies.ALIAS.outputs.NAME
+			depBun := make(map[string]interface{})
+			deps[alias] = depBun
 
-		depBun["name"] = depB.Name
-		depBun["version"] = depB.Version
-		depBun["description"] = depB.Description
+			depBun["name"] = depB.Name
+			depBun["version"] = depB.Version
+			depBun["description"] = depB.Description
+		}
 	}
 
 	bun["outputs"] = m.outputs
@@ -338,6 +348,10 @@ func (m *RuntimeManifest) buildSourceData() (map[string]interface{}, error) {
 		for _, s := range sources.ListSourcesByPriority() {
 			switch ps := s.(type) {
 			case cnab.DependencyOutputParameterSource:
+				if m.bundle.HasDependenciesV1() {
+					return nil, fmt.Errorf("bundle was not built for dependencies v1 but uses a dependency parameter source which is invalid")
+				}
+
 				outRef := manifest.DependencyOutputReference{Dependency: ps.Dependency, Output: ps.OutputName}
 
 				// Ignore anything that isn't templated, because that's what we are building the source data for
@@ -362,7 +376,7 @@ func (m *RuntimeManifest) buildSourceData() (map[string]interface{}, error) {
 				depOutputs[ps.OutputName] = value
 
 				// Determine if the dependency's output is defined as sensitive
-				depB := m.bundles[ps.Dependency]
+				depB := m.depsv1_bundles[ps.Dependency]
 				if ok, _ := depB.IsOutputSensitive(ps.OutputName); ok {
 					m.setSensitiveValue(value)
 				}

pkg/secrets/helpers.go

@@ -5,7 +5,7 @@ import inmemory "get.porter.sh/porter/pkg/secrets/plugins/in-memory"
 var _ Store = &TestSecretsProvider{}
 
 type TestSecretsProvider struct {
-	PluginAdapter
+	*PluginAdapter
 
 	secrets *inmemory.Store
 }

pkg/secrets/plugin_adapter.go

@@ -7,36 +7,47 @@ import (
 	"get.porter.sh/porter/pkg/secrets/plugins"
 )
 
-var _ Store = PluginAdapter{}
+var _ Store = &PluginAdapter{}
+
+type PorterSecretStrategy interface {
+	Resolve(ctx context.Context, keyName string, keyValue string) (string, error)
+}
 
 // PluginAdapter converts between the low-level plugins.SecretsProtocol and
 // the secrets.Store interface.
 type PluginAdapter struct {
-	plugin plugins.SecretsProtocol
+	plugin       plugins.SecretsProtocol
+	porterPlugin PorterSecretStrategy
+}
+
+func (a *PluginAdapter) SetPorterStrategy(strategy PorterSecretStrategy) {
+	a.porterPlugin = strategy
 }
 
 // NewPluginAdapter wraps the specified storage plugin.
-func NewPluginAdapter(plugin plugins.SecretsProtocol) PluginAdapter {
-	return PluginAdapter{plugin: plugin}
+func NewPluginAdapter(plugin plugins.SecretsProtocol) *PluginAdapter {
+	return &PluginAdapter{
+		plugin: plugin,
+	}
 }
 
-func (a PluginAdapter) Close() error {
+func (a *PluginAdapter) Close() error {
 	if closer, ok := a.plugin.(io.Closer); ok {
 		return closer.Close()
 	}
 	return nil
 }
 
-func (a PluginAdapter) Resolve(ctx context.Context, keyName string, keyValue string) (string, error) {
-	// Instead of calling out to a plugin, resolve the value from Porter's database
+func (a *PluginAdapter) Resolve(ctx context.Context, keyName string, keyValue string) (string, error) {
+	// Intercept requests for Porter to resolve an internal value and run the plugin in-process.
 	// This supports bundle workflows where we are sourcing data from other runs, e.g. passing a connection string from a dependency to another bundle
 	if keyName == "porter" {
-
+		return a.porterPlugin.Resolve(ctx, keyName, keyValue)
 	}
 
 	return a.plugin.Resolve(ctx, keyName, keyValue)
 }
 
-func (a PluginAdapter) Create(ctx context.Context, keyName string, keyValue string, value string) error {
+func (a *PluginAdapter) Create(ctx context.Context, keyName string, keyValue string, value string) error {
 	return a.plugin.Create(ctx, keyName, keyValue, value)
 }

pkg/secrets/plugins/porter/store.go

@@ -0,0 +1,51 @@
+package inmemory
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"regexp"
+
+	"get.porter.sh/porter/pkg/secrets/plugins"
+	"get.porter.sh/porter/pkg/storage"
+	"github.com/cnabio/cnab-go/secrets/host"
+)
+
+var _ plugins.SecretsProtocol = &Store{}
+
+// Store implements an in-process plugin for retrieving values from Porter's database
+// This never runs in an external process, or even as an internal plugin, because it requires loading all of Porter's config.
+type Store struct {
+	Installations storage.InstallationStore
+}
+
+func NewStore() *Store {
+	return &Store{}
+}
+
+var workflowSecretRegexp = regexp.MustCompile(`workflow\.([^.]+)?\.jobs\.([^.]+)?\.outputs\.(.+)`)
+
+func (s *Store) Resolve(ctx context.Context, keyName string, keyValue string) (string, error) {
+	if keyName == "porter" {
+		// We support retrieving certain data from Porter's database:
+		// workflow.WORKFLOWID.jobs.JOBKEY.outputs.OUTPUT
+		// The WORKFLOWID is set to the current executing workflow by the workflow engine before running the job
+		// It is not stored in the database and is always set dynamically when the job is run.
+
+		matches := workflowSecretRegexp.FindStringSubmatch(keyValue)
+		if len(matches) != 4 {
+			return "", fmt.Errorf("invalid porter secret mapping value: expected the format workflow.WORKFLOWID.jobs.JOBKEY.outputs.OUTPUT but got %s", keyValue)
+		}
+
+		// Lookup the result and run associated with the job run in that workflow
+
+	}
+
+	// Fallback to the host secret plugin
+	hostStore := host.SecretStore{}
+	return hostStore.Resolve(keyName, keyValue)
+}
+
+func (s *Store) Create(ctx context.Context, keyName string, keyValue string, value string) error {
+	return errors.New("The porter secrets plugin does not support the create function, because it is for internal use within Porter only. Chek your porter configuration file and make sure that you are using a supported secrets plugin and not the porter secrets plugin directly.")
+}

pkg/secrets/secrets.go

@@ -27,4 +27,8 @@ type Store interface {
 	// - keyName=key, keyValue=conn-string, value=redis://foo
 	// - keyName=path, keyValue=/tmp/connstring.txt, value=redis://foo
 	Create(ctx context.Context, keyName string, keyValue string, value string) error
+
+	// SetPorterStrategy gives the secret store the ability to resolve the porter secret strategy
+	// using Porter's database.
+	SetPorterStrategy(strategy PorterSecretStrategy)
 }

pkg/storage/installation_provider.go

@@ -55,6 +55,10 @@ type InstallationProvider interface {
 	// GetLastRun returns the last run of an Installation.
 	GetLastRun(ctx context.Context, namespace string, installation string) (Run, error)
 
+	// FindOutputs applies the find operation against outputs collection
+	// using the specified options.
+	FindOutputs(ctx context.Context, opts FindOptions) ([]Output, error)
+
 	// GetLastOutput returns the most recent value (last) of the specified
 	// Output associated with the installation.
 	GetLastOutput(ctx context.Context, namespace string, installation string, name string) (Output, error)

pkg/storage/installation_store.go

@@ -142,6 +142,15 @@ func (s InstallationStore) FindInstallations(ctx context.Context, findOpts FindO
 	return out, err
 }
 
+func (s InstallationStore) FindOutputs(ctx context.Context, findOpts FindOptions) ([]Output, error) {
+	_, log := tracing.StartSpan(ctx)
+	defer log.EndSpan()
+
+	var out []Output
+	err := s.store.Find(ctx, CollectionOutputs, findOpts, &out)
+	return out, err
+}
+
 func (s InstallationStore) GetInstallation(ctx context.Context, namespace string, name string) (Installation, error) {
 	var out Installation