feat: scraper plugins (#1200)

* feat: ScrapePlugin CRD

* feat: load plugins

[skip ci]

* feat: inject plugin to base scraper

* feat: cache scrape plugins

* chore: refactor applying of plugins

Author: adityathebe

api/context.go

@@ -88,13 +88,15 @@ func (ctx ScrapeContext) WithValue(key, val any) ScrapeContext {
 
 }
 
-func (ctx ScrapeContext) WithScrapeConfig(scraper *v1.ScrapeConfig) ScrapeContext {
-	ctx.scrapeConfig = scraper
+func (ctx ScrapeContext) WithScrapeConfig(scraper *v1.ScrapeConfig, plugins ...v1.ScrapePluginSpec) ScrapeContext {
+	sc := scraper.DeepCopy()
+	sc.Spec = sc.Spec.ApplyPlugin(plugins)
 
-	ctx.Context = ctx.WithObject(scraper.ObjectMeta)
+	ctx.scrapeConfig = sc
+	ctx.Context = ctx.WithObject(sc.ObjectMeta)
 
 	// Try to use the temp cache if it exits
-	if c, exists := scraperTempCache.Load(lo.FromPtr(scraper.GetPersistedID())); exists {
+	if c, exists := scraperTempCache.Load(lo.FromPtr(sc.GetPersistedID())); exists {
 		ctx.temp = c.(*TempCache)
 	}
 	return ctx

api/v1/common.go

@@ -241,6 +241,18 @@ type BaseScraper struct {
 	Properties []ConfigProperties `json:"properties,omitempty" template:"true"`
 }
 
+func (base BaseScraper) ApplyPlugins(plugins ...ScrapePluginSpec) BaseScraper {
+	for _, p := range plugins {
+		base.Transform.Change.Exclude = append(base.Transform.Change.Exclude, p.Change.Exclude...)
+		base.Transform.Change.Mapping = append(base.Transform.Change.Mapping, p.Change.Mapping...)
+
+		base.Transform.Relationship = append(base.Transform.Relationship, p.Relationship...)
+		base.Properties = append(base.Properties, p.Properties...)
+	}
+
+	return base
+}
+
 func (base BaseScraper) WithoutTransform() BaseScraper {
 	base.Transform = Transform{}
 	return base

api/v1/scrapeplugin_types.go

@@ -0,0 +1,92 @@
+package v1
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/flanksource/duty/models"
+	"github.com/google/uuid"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// ScrapePluginStatus defines the observed state of Plugin
+type ScrapePluginStatus struct {
+	ObservedGeneration int64 `json:"observedGeneration,omitempty" protobuf:"varint,3,opt,name=observedGeneration"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+
+// ScrapePlugin is the Schema for the scraper plugins
+type ScrapePlugin struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   ScrapePluginSpec   `json:"spec,omitempty"`
+	Status ScrapePluginStatus `json:"status,omitempty"`
+}
+
+type ScrapePluginSpec struct {
+	Change TransformChange `json:"changes,omitempty"`
+
+	// Relationship allows you to form relationships between config items using selectors.
+	Relationship []RelationshipConfig `json:"relationship,omitempty"`
+
+	// Properties are custom templatable properties for the scraped config items
+	// grouped by the config type.
+	Properties []ConfigProperties `json:"properties,omitempty" template:"true"`
+}
+
+func (t ScrapePlugin) ToModel() (*models.ScrapePlugin, error) {
+	var id uuid.UUID
+	if v, err := uuid.Parse(string(t.GetUID())); err == nil {
+		id = v
+	}
+
+	specJSON, err := json.Marshal(t.Spec)
+	if err != nil {
+		return nil, err
+	}
+
+	return &models.ScrapePlugin{
+		ID:        id,
+		Name:      t.Name,
+		Namespace: t.Namespace,
+		Spec:      specJSON,
+		CreatedAt: t.CreationTimestamp.Time,
+	}, nil
+}
+
+func (t ScrapePlugin) LoggerName() string {
+	return fmt.Sprintf("plugin.%s.%s", t.Namespace, t.Name)
+}
+
+func (t ScrapePlugin) GetContext() map[string]any {
+	return map[string]any{
+		"namespace":  t.Namespace,
+		"name":       t.Name,
+		"scraper_id": t.GetPersistedID(),
+	}
+}
+
+func (t *ScrapePlugin) GetPersistedID() *uuid.UUID {
+	if t.GetUID() == "" {
+		return nil
+	}
+
+	u, _ := uuid.Parse(string(t.GetUID()))
+	return &u
+}
+
+//+kubebuilder:object:root=true
+
+// ScrapePluginList contains a list of Plugin
+type ScrapePluginList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []ScrapePlugin `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&ScrapePlugin{}, &ScrapePluginList{})
+}

api/v1/types.go

@@ -72,6 +72,64 @@ type ScraperSpec struct {
 	Full bool `json:"full,omitempty"`
 }
 
+func (c ScraperSpec) ApplyPlugin(plugins []ScrapePluginSpec) ScraperSpec {
+	spec := c.DeepCopy()
+
+	for i := range spec.GCP {
+		spec.GCP[i].BaseScraper = spec.GCP[i].BaseScraper.ApplyPlugins(plugins...)
+	}
+
+	for i := range spec.AWS {
+		spec.AWS[i].BaseScraper = spec.AWS[i].BaseScraper.ApplyPlugins(plugins...)
+	}
+
+	for i := range spec.File {
+		spec.File[i].BaseScraper = spec.File[i].BaseScraper.ApplyPlugins(plugins...)
+	}
+
+	for i := range spec.Kubernetes {
+		spec.Kubernetes[i].BaseScraper = spec.Kubernetes[i].BaseScraper.ApplyPlugins(plugins...)
+	}
+
+	for i := range spec.KubernetesFile {
+		spec.KubernetesFile[i].BaseScraper = spec.KubernetesFile[i].BaseScraper.ApplyPlugins(plugins...)
+	}
+
+	for i := range spec.AzureDevops {
+		spec.AzureDevops[i].BaseScraper = spec.AzureDevops[i].BaseScraper.ApplyPlugins(plugins...)
+	}
+
+	for i := range spec.GithubActions {
+		spec.GithubActions[i].BaseScraper = spec.GithubActions[i].BaseScraper.ApplyPlugins(plugins...)
+	}
+
+	for i := range spec.Azure {
+		spec.Azure[i].BaseScraper = spec.Azure[i].BaseScraper.ApplyPlugins(plugins...)
+	}
+
+	for i := range spec.SQL {
+		spec.SQL[i].BaseScraper = spec.SQL[i].BaseScraper.ApplyPlugins(plugins...)
+	}
+
+	for i := range spec.Slack {
+		spec.Slack[i].BaseScraper = spec.Slack[i].BaseScraper.ApplyPlugins(plugins...)
+	}
+
+	for i := range spec.Trivy {
+		spec.Trivy[i].BaseScraper = spec.Trivy[i].BaseScraper.ApplyPlugins(plugins...)
+	}
+
+	for i := range spec.Terraform {
+		spec.Terraform[i].BaseScraper = spec.Terraform[i].BaseScraper.ApplyPlugins(plugins...)
+	}
+
+	for i := range spec.HTTP {
+		spec.HTTP[i].BaseScraper = spec.HTTP[i].BaseScraper.ApplyPlugins(plugins...)
+	}
+
+	return *spec
+}
+
 func (c ScraperSpec) GenerateName() (string, error) {
 	return utils.Hash(c)
 }