Merge pull request #196 from flanksource/fix-jobs

moshloop · web-flow · commit 9b160da18b91 · 2023-04-03T00:06:05.000+03:00
Fix jobs
diff --git a/api/v1/interface.go b/api/v1/interface.go
@@ -17,6 +17,7 @@ import (
 // +kubebuilder:object:generate=false
 type Scraper interface {
 	Scrape(ctx *ScrapeContext, config ConfigScraper) ScrapeResults
+	CanScrape(config ConfigScraper) bool
 }
 
 // Analyzer ...
diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go
@@ -968,7 +968,10 @@ func (aws Scraper) ami(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults
 	}
 }
 
-// Scrape ...
+func (aws Scraper) CanScrape(configs v1.ConfigScraper) bool {
+	return len(configs.AWS) > 0
+}
+
 func (aws Scraper) Scrape(ctx *v1.ScrapeContext, config v1.ConfigScraper) v1.ScrapeResults {
 	results := &v1.ScrapeResults{}
 
diff --git a/scrapers/aws/cost.go b/scrapers/aws/cost.go
@@ -8,7 +8,7 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/service/sts"
 	"github.com/flanksource/commons/logger"
-	"github.com/flanksource/config-db/api/v1"
+	v1 "github.com/flanksource/config-db/api/v1"
 	"github.com/flanksource/config-db/db"
 	athena "github.com/uber/athenadriver/go"
 )
@@ -129,6 +129,15 @@ func FetchCosts(ctx *v1.ScrapeContext, config v1.AWS) ([]LineItemRow, error) {
 
 type CostScraper struct{}
 
+func (awsCost CostScraper) CanScrape(config v1.ConfigScraper) bool {
+	for _, awsConfig := range config.AWS {
+		if awsConfig.CostReporting.S3BucketPath != "" || awsConfig.CostReporting.Table != "" {
+			return true
+		}
+	}
+	return false
+}
+
 func (awsCost CostScraper) Scrape(ctx *v1.ScrapeContext, config v1.ConfigScraper) v1.ScrapeResults {
 	var results v1.ScrapeResults
 
diff --git a/scrapers/azure/azure.go b/scrapers/azure/azure.go
@@ -21,7 +21,10 @@ type Scraper struct {
 	config *v1.Azure
 }
 
-// Scrape ...
+func (azure Scraper) CanScrape(configs v1.ConfigScraper) bool {
+	return len(configs.Azure) > 0
+}
+
 func (azure Scraper) Scrape(ctx *v1.ScrapeContext, configs v1.ConfigScraper) v1.ScrapeResults {
 	var results v1.ScrapeResults
 	for _, config := range configs.Azure {
diff --git a/scrapers/azure/devops/pipelines.go b/scrapers/azure/devops/pipelines.go
@@ -12,6 +12,10 @@ const PipelineRun = "AzureDevops::PipelineRun"
 type AzureDevopsScraper struct {
 }
 
+func (ado AzureDevopsScraper) CanScrape(configs v1.ConfigScraper) bool {
+	return len(configs.AzureDevops) > 0
+}
+
 // Scrape ...
 func (ado AzureDevopsScraper) Scrape(ctx *v1.ScrapeContext, configs v1.ConfigScraper) v1.ScrapeResults {
 
diff --git a/scrapers/file/file.go b/scrapers/file/file.go
@@ -63,7 +63,10 @@ func convertToLocalPath(uri string) string {
 	return p + path.Base(_uri.Path) + "-" + hex.EncodeToString(hash[:])[0:8]
 }
 
-// Scrape ...
+func (file FileScraper) CanScrape(configs v1.ConfigScraper) bool {
+	return len(configs.File) > 0
+}
+
 func (file FileScraper) Scrape(ctx *v1.ScrapeContext, configs v1.ConfigScraper) v1.ScrapeResults {
 	pwd, _ := os.Getwd()
 	cacheDir := path.Join(pwd, ".config-db", "cache", "files")
diff --git a/scrapers/github/workflows.go b/scrapers/github/workflows.go
@@ -14,6 +14,10 @@ const WorkflowRun = "GitHubActions::WorkflowRun"
 type GithubActionsScraper struct {
 }
 
+func (gh GithubActionsScraper) CanScrape(configs v1.ConfigScraper) bool {
+	return len(configs.GithubActions) > 0
+}
+
 // Scrape fetches github workflows and workflow runs from github API and converts the action executions (workflow runs) to change events.
 func (gh GithubActionsScraper) Scrape(ctx *v1.ScrapeContext, configs v1.ConfigScraper) v1.ScrapeResults {
 	results := v1.ScrapeResults{}
diff --git a/scrapers/kubernetes/kubernetes.go b/scrapers/kubernetes/kubernetes.go
@@ -20,7 +20,10 @@ type KubernetesScraper struct {
 
 const ExternalTypePrefix = "Kubernetes::"
 
-// Scrape ...
+func (kubernetes KubernetesScraper) CanScrape(configs v1.ConfigScraper) bool {
+	return len(configs.Kubernetes) > 0
+}
+
 func (kubernetes KubernetesScraper) Scrape(ctx *v1.ScrapeContext, configs v1.ConfigScraper) v1.ScrapeResults {
 	var (
 		results       v1.ScrapeResults
diff --git a/scrapers/kubernetes/kubernetes_file.go b/scrapers/kubernetes/kubernetes_file.go
@@ -117,6 +117,10 @@ func findBySelector(ctx *v1.ScrapeContext, client *kubernetes.Clientset, config
 	return pods, nil
 }
 
+func (kubernetes KubernetesFileScraper) CanScrape(configs v1.ConfigScraper) bool {
+	return len(configs.KubernetesFile) > 0
+}
+
 // Scrape ...
 func (kubernetes KubernetesFileScraper) Scrape(ctx *v1.ScrapeContext, configs v1.ConfigScraper) v1.ScrapeResults {
 	results := v1.ScrapeResults{}
diff --git a/scrapers/processors/json.go b/scrapers/processors/json.go
@@ -13,6 +13,7 @@ import (
 	"github.com/ohler55/ojg/jp"
 	"github.com/ohler55/ojg/oj"
 	"github.com/pkg/errors"
+	kyaml "sigs.k8s.io/yaml"
 )
 
 type Mask struct {
@@ -176,6 +177,12 @@ func (e Extract) Extract(inputs ...v1.ScrapeResult) ([]v1.ScrapeResult, error) {
 				return results, errors.Wrapf(err, "Failed parse properties %s", input)
 			}
 			input.Config = props.Map()
+		} else if input.Format == "yaml" {
+			contentByte, err := kyaml.YAMLToJSON([]byte(input.Config.(string)))
+			if err != nil {
+				return results, errors.Wrapf(err, "Failed parse yaml %s", input)
+			}
+			input.Config = string(contentByte)
 		} else if input.Format != "" {
 			input.Config = map[string]any{
 				"format":  input.Format,
diff --git a/scrapers/runscrapers.go b/scrapers/runscrapers.go
@@ -23,6 +23,9 @@ func Run(ctx *v1.ScrapeContext, configs ...v1.ConfigScraper) ([]v1.ScrapeResult,
 	results := []v1.ScrapeResult{}
 	for _, config := range configs {
 		for _, scraper := range All {
+			if !scraper.CanScrape(config) {
+				continue
+			}
 			jobHistory := models.JobHistory{
 				Name:         fmt.Sprintf("scraper:%T", scraper),
 				ResourceType: "config_scraper",
diff --git a/scrapers/sql/sql.go b/scrapers/sql/sql.go
@@ -18,7 +18,10 @@ import (
 type SqlScraper struct {
 }
 
-// Scrape ...
+func (s SqlScraper) CanScrape(configs v1.ConfigScraper) bool {
+	return len(configs.SQL) > 0
+}
+
 func (s SqlScraper) Scrape(ctx *v1.ScrapeContext, configs v1.ConfigScraper) v1.ScrapeResults {
 	var results v1.ScrapeResults
 	for _, _config := range configs.SQL {
diff --git a/utils/kube/exec.go b/utils/kube/exec.go
@@ -5,8 +5,6 @@ import (
 	"context"
 	"fmt"
 
-	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/remotecommand"
@@ -20,18 +18,15 @@ func ExecutePodf(ctx context.Context, client *kubernetes.Clientset, rc *rest.Con
 		Name(pod).
 		Namespace(namespace).
 		SubResource("exec").
-		Param("container", container)
-
-	scheme := runtime.NewScheme()
-	ParameterCodec := runtime.NewParameterCodec(scheme)
-	req.VersionedParams(&v1.PodExecOptions{
-		Container: container,
-		Command:   command,
-		Stdin:     false,
-		Stdout:    true,
-		Stderr:    true,
-		TTY:       tty,
-	}, ParameterCodec)
+		Param("container", container).
+		Param("stdin", fmt.Sprintf("%t", false)).
+		Param("stdout", fmt.Sprintf("%t", true)).
+		Param("stderr", fmt.Sprintf("%t", true)).
+		Param("tty", fmt.Sprintf("%t", tty))
+
+	for _, c := range command {
+		req.Param("command", c)
+	}
 
 	exec, err := remotecommand.NewSPDYExecutor(rc, "POST", req.URL())
 	if err != nil {
@@ -47,8 +42,9 @@ func ExecutePodf(ctx context.Context, client *kubernetes.Clientset, rc *rest.Con
 
 	_stdout := safeString(&stdout)
 	_stderr := safeString(&stderr)
+
 	if err != nil {
-		return _stdout, _stderr, fmt.Errorf("exec returned an error: %+v", err)
+		return "", "", fmt.Errorf("failed to execute command: %v, stdout=%s stderr=%s", err, _stdout, _stderr)
 	}
 
 	return _stdout, _stderr, nil

Original file line number	Diff line number	Diff line change
`@@ -17,6 +17,7 @@ import (`
`17`	`17`	`// +kubebuilder:object:generate=false`
`18`	`18`	`type Scraper interface {`
`19`	`19`	`Scrape(ctx *ScrapeContext, config ConfigScraper) ScrapeResults`
	`20`	`+ CanScrape(config ConfigScraper) bool`
`20`	`21`	`}`
`21`	`22`
`22`	`23`	`// Analyzer ...`
Original file line number	Diff line number	Diff line change
`@@ -968,7 +968,10 @@ func (aws Scraper) ami(ctx AWSContext, config v1.AWS, results v1.ScrapeResults`
`968`	`968`	`}`
`969`	`969`	`}`
`970`	`970`
`971`		`-// Scrape ...`
	`971`	`+func (aws Scraper) CanScrape(configs v1.ConfigScraper) bool {`
	`972`	`+ return len(configs.AWS) > 0`
	`973`	`+}`
	`974`	`+`
`972`	`975`	`func (aws Scraper) Scrape(ctx *v1.ScrapeContext, config v1.ConfigScraper) v1.ScrapeResults {`
`973`	`976`	`results := &v1.ScrapeResults{}`
`974`	`977`
Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,10 @@ const PipelineRun = "AzureDevops::PipelineRun"`
`12`	`12`	`type AzureDevopsScraper struct {`
`13`	`13`	`}`
`14`	`14`
	`15`	`+func (ado AzureDevopsScraper) CanScrape(configs v1.ConfigScraper) bool {`
	`16`	`+ return len(configs.AzureDevops) > 0`
	`17`	`+}`
	`18`	`+`
`15`	`19`	`// Scrape ...`
`16`	`20`	`func (ado AzureDevopsScraper) Scrape(ctx *v1.ScrapeContext, configs v1.ConfigScraper) v1.ScrapeResults {`
`17`	`21`