Add extensible source providers

cmd/build/build.go

@@ -38,8 +38,8 @@ func init() {
 	build := &cobra.Command{
 		Use:   "build",
 		Short: "Build and distribute configured bundles",
-		Run: func(cmd *cobra.Command, args []string) {
-			ctx := cmd.Context()
+		Run: func(c *cobra.Command, args []string) {
+			ctx := c.Context()
 			lc := params.logging
 			if !params.noninteractive {
 				// interactive sessions get a nice report, so we suppress error logs
@@ -85,6 +85,7 @@ func init() {
 				WithPersistenceDir(params.persistenceDir).
 				WithConfig(config).
 				WithBuiltinFS(libraries.FS).
+				WithSourceProviders(cmd.SourceProviders).
 				WithSingleShot(true).
 				WithLogger(log).
 				WithNoninteractive(params.noninteractive).

cmd/commands.go

@@ -5,6 +5,8 @@ import (
 	"path"
 
 	"github.com/spf13/cobra"
+
+	pkgsync "github.com/open-policy-agent/opa-control-plane/pkg/sync"
 )
 
 // RootCommand is the base CLI command that all subcommands are added to.
@@ -13,3 +15,9 @@ var RootCommand = &cobra.Command{
 	Short: "OPA Control Plane",
 	Long:  "An open source control plane for Open Policy Agent (OPA).",
 }
+
+// SourceProviders holds the source types that sources' providers entries can
+// use in the build and run commands. It is empty in opactl; programs that
+// embed OCP's commands register their source providers here before
+// executing RootCommand.
+var SourceProviders = pkgsync.NewSourceProviderRegistry()

cmd/run/run.go

@@ -34,8 +34,8 @@ func init() {
 	run := &cobra.Command{
 		Use:   "run",
 		Short: "Run the OPA Control Plane service",
-		Run: func(cmd *cobra.Command, args []string) {
-			ctx := cmd.Context()
+		Run: func(c *cobra.Command, args []string) {
+			ctx := c.Context()
 
 			log := logging.NewLogger(params.logging)
 
@@ -64,6 +64,7 @@ func init() {
 				WithPersistenceDir(params.persistenceDir).
 				WithConfig(config).
 				WithBuiltinFS(libraries.FS).
+				WithSourceProviders(cmd.SourceProviders).
 				WithLogger(log).
 				WithMigrateDB(params.migrateDB || sqlite) // always run migrations with sqlite
 
@@ -78,6 +79,7 @@ func init() {
 			go func() {
 				if err := server.New().
 					WithDatabase(svc.Database()).
+					WithSourceProviders(svc.SourceProviders()).
 					WithReadiness(svc.Ready).
 					WithConfig(config).
 					WithMetrics(m).

config/schema.json

@@ -395,6 +395,33 @@
       },
       "type": "object"
     },
+    "ConfigProvider": {
+      "required": [
+        "name",
+        "type"
+      ],
+      "additionalProperties": true,
+      "properties": {
+        "name": {
+          "minLength": 1,
+          "type": "string"
+        },
+        "path": {
+          "type": "string"
+        },
+        "type": {
+          "minLength": 1,
+          "type": "string"
+        }
+      },
+      "type": "object"
+    },
+    "ConfigProviders": {
+      "items": {
+        "$ref": "#/definitions/ConfigProvider"
+      },
+      "type": "array"
+    },
     "ConfigRequirement": {
       "additionalProperties": false,
       "properties": {
@@ -505,6 +532,9 @@
         "paths": {
           "$ref": "#/definitions/ConfigStringSet"
         },
+        "providers": {
+          "$ref": "#/definitions/ConfigProviders"
+        },
         "requirements": {
           "$ref": "#/definitions/ConfigRequirements"
         }

e2e/cli/build_providers_basic.txtar

@@ -0,0 +1,41 @@
+# A providers entry, handled by the E2E test source provider (see
+# provider_test.go): its data goes under its path, its Rego keeps its
+# package, and its metadata is available to the revision template.
+exec opactl-with-providers build --config config.d/bundle.yml --data-dir tmp --non-interactive
+! stderr .
+! stdout .
+
+exec tar tf bundles/app/bundle.tar.gz
+cmp stdout exp/tarball
+
+exec tar xf bundles/app/bundle.tar.gz
+exec jq -e '.revision == "v1" and (.roots | sort) == ["authz", "users"]' .manifest
+exec jq -e '. == {"users": {"alice": ["admin"]}}' data.json
+
+-- config.d/bundle.yml --
+bundles:
+  app:
+    object_storage:
+      filesystem:
+        path: bundles/app/bundle.tar.gz
+    revision: input.sources.app.providers.users.version
+    requirements:
+    - source: app
+sources:
+  app:
+    providers:
+    - name: users
+      type: e2e.files
+      path: users
+      files:
+        data.json: '{"alice": ["admin"]}'
+        authz.rego: |
+          package authz
+          import rego.v1
+          allow if input.user in data.users.alice
+      metadata:
+        version: v1
+-- exp/tarball --
+/data.json
+/app/users/authz.rego
+/.manifest

e2e/cli/build_providers_contribution.txtar

@@ -0,0 +1,33 @@
+# A providers entry contributing manifest metadata, a root it has no files
+# for, and its Rego version (the policy uses v1 syntax without importing
+# rego.v1).
+exec opactl-with-providers build --config config.d/bundle.yml --data-dir tmp --non-interactive
+! stderr .
+! stdout .
+
+exec tar xf bundles/app/bundle.tar.gz
+exec jq -e '.rego_version == 1 and .metadata == {"example": {"version": "v1"}} and (.roots | sort) == ["authz", "lazy/remote"]' .manifest
+
+-- config.d/bundle.yml --
+bundles:
+  app:
+    object_storage:
+      filesystem:
+        path: bundles/app/bundle.tar.gz
+    requirements:
+    - source: app
+sources:
+  app:
+    providers:
+    - name: users
+      type: e2e.files
+      files:
+        authz.rego: |
+          package authz
+          allow if input.user == "alice"
+      contribution:
+        metadata:
+          example:
+            version: v1
+        roots: [lazy/remote]
+        rego_version: 1

e2e/cli/build_providers_path_change.txtar

@@ -0,0 +1,45 @@
+# Changing an entry's path leaves nothing from the old path in the bundle:
+# the entry's directory is emptied before every sync. Both builds share the
+# same data directory.
+exec opactl-with-providers build --config config.d/old.yml --data-dir tmp --non-interactive
+exec tar xf bundles/app/bundle.tar.gz
+exec jq -e '. == {"old": {"v": 1}}' data.json
+
+exec opactl-with-providers build --config config.d/new.yml --data-dir tmp --non-interactive
+! stderr .
+! stdout .
+exec tar xf bundles/app/bundle.tar.gz
+exec jq -e '. == {"new": {"v": 2}}' data.json
+
+-- config.d/old.yml --
+bundles:
+  app:
+    object_storage:
+      filesystem:
+        path: bundles/app/bundle.tar.gz
+    requirements:
+    - source: app
+sources:
+  app:
+    providers:
+    - name: users
+      type: e2e.files
+      path: old
+      files:
+        data.json: '{"v": 1}'
+-- config.d/new.yml --
+bundles:
+  app:
+    object_storage:
+      filesystem:
+        path: bundles/app/bundle.tar.gz
+    requirements:
+    - source: app
+sources:
+  app:
+    providers:
+    - name: users
+      type: e2e.files
+      path: new
+      files:
+        data.json: '{"v": 2}'

e2e/cli/build_providers_unknown_type.txtar

@@ -0,0 +1,20 @@
+# opactl registers no custom source types, so any providers entry is
+# rejected when the configuration is loaded.
+! exec $OPACTL build --config config.d/bundle.yml --data-dir tmp --non-interactive
+stderr '^invalid configuration: source "app": provider "users": unknown type "example.custom-source"'
+! stdout .
+
+-- config.d/bundle.yml --
+bundles:
+  hello-world:
+    object_storage:
+      filesystem:
+        path: bundles/hello-world/bundle.tar.gz
+    requirements:
+    - source: app
+sources:
+  app:
+    providers:
+    - name: users
+      type: example.custom-source
+      param1: value

e2e/cli/build_providers_with_git.txtar

@@ -0,0 +1,46 @@
+# A git repository and a providers entry in the same source.
+cd repo
+env HOME=.
+exec git init .
+exec git branch -m main
+exec git config --global user.name "botbotbot"
+exec git config --global user.email "bot@bot.bot"
+exec git add .
+exec git commit -m initial-commit
+cd ..
+
+exec opactl-with-providers build --config config.d/bundle.yml --data-dir tmp --non-interactive
+! stderr .
+! stdout .
+
+exec tar xf bundles/app/bundle.tar.gz
+exec jq -e '.revision == "git-v1" and (.roots | sort) == ["rules", "users"]' .manifest
+exec jq -e '. == {"users": {"alice": ["admin"]}}' data.json
+exists app/policy.rego
+
+-- repo/policy.rego --
+package rules
+import rego.v1
+allow if input.user in data.users.alice
+-- config.d/bundle.yml --
+bundles:
+  app:
+    object_storage:
+      filesystem:
+        path: bundles/app/bundle.tar.gz
+    revision: $"git-{input.sources.app.providers.users.version}"
+    requirements:
+    - source: app
+sources:
+  app:
+    git:
+      repo: ./repo/
+      reference: refs/heads/main
+    providers:
+    - name: users
+      type: e2e.files
+      path: users
+      files:
+        data.json: '{"alice": ["admin"]}'
+      metadata:
+        version: v1

e2e/cli/main_test.go

@@ -17,8 +17,36 @@ import (
 	"time"
 
 	"github.com/rogpeppe/go-internal/testscript"
+
+	"github.com/open-policy-agent/opa-control-plane/cmd"
+	// The same subcommands as opactl's main package.
+	_ "github.com/open-policy-agent/opa-control-plane/cmd/backtest"
+	_ "github.com/open-policy-agent/opa-control-plane/cmd/build"
+	_ "github.com/open-policy-agent/opa-control-plane/cmd/compare"
+	_ "github.com/open-policy-agent/opa-control-plane/cmd/db"
+	_ "github.com/open-policy-agent/opa-control-plane/cmd/migrate"
+	_ "github.com/open-policy-agent/opa-control-plane/cmd/run"
+	_ "github.com/open-policy-agent/opa-control-plane/cmd/version"
 )
 
+// TestMain makes "opactl-with-providers" available to test scripts: OCP's
+// commands, with the E2E test source provider (see provider_test.go)
+// registered. opactl itself registers none, so scripts exercising providers
+// entries use this instead of $OPACTL.
+func TestMain(m *testing.M) {
+	testscript.Main(m, map[string]func(){
+		"opactl-with-providers": func() {
+			if err := cmd.SourceProviders.Register(filesProvider{}); err != nil {
+				fmt.Fprintln(os.Stderr, err)
+				os.Exit(1)
+			}
+			if err := cmd.RootCommand.Execute(); err != nil {
+				os.Exit(1)
+			}
+		},
+	})
+}
+
 func testServer() *httptest.Server {
 	mux := http.NewServeMux()
 	mux.HandleFunc("GET /headers", func(w http.ResponseWriter, r *http.Request) {