Dynamically generate the list of Prometheus servers

Author: prymitive

.github/spellcheck/wordlist.txt

@@ -9,6 +9,7 @@ cloudflare
 Cloudflare
 config
 Deduplicate
+deduplicated
 dir
 durations
 endraw

Makefile

@@ -43,7 +43,7 @@ test:
 		-coverprofile=$(COVER_PROFILE) \
 		-coverpkg=./... \
 		-race \
-		-count=5 \
+		-count=3 \
 		-timeout=15m \
 		./...
 

cmd/pint/ci.go

@@ -100,13 +100,17 @@ func actionCI(c *cli.Context) error {
 		return err
 	}
 
-	for _, prom := range meta.cfg.PrometheusServers {
-		prom.StartWorkers()
-	}
-	defer meta.cleanup()
-
 	ctx := context.WithValue(context.Background(), config.CommandKey, config.CICommand)
-	summary := checkRules(ctx, meta.workers, meta.cfg, entries)
+
+	gen := config.NewPrometheusGenerator(meta.cfg, metricsRegistry)
+	defer gen.Stop()
+
+	slog.Debug("Generated all Prometheus servers", slog.Int("count", gen.Count()))
+
+	summary, err := checkRules(ctx, meta.workers, gen, meta.cfg, entries)
+	if err != nil {
+		return err
+	}
 
 	if c.Bool(requireOwnerFlag) {
 		summary.Report(verifyOwners(entries, meta.cfg.Owners.CompileAllowed())...)

cmd/pint/lint.go

@@ -60,13 +60,15 @@ func actionLint(c *cli.Context) error {
 		return err
 	}
 
-	for _, prom := range meta.cfg.PrometheusServers {
-		prom.StartWorkers()
-	}
-	defer meta.cleanup()
-
 	ctx := context.WithValue(context.Background(), config.CommandKey, config.LintCommand)
-	summary := checkRules(ctx, meta.workers, meta.cfg, entries)
+
+	gen := config.NewPrometheusGenerator(meta.cfg, metricsRegistry)
+	defer gen.Stop()
+
+	summary, err := checkRules(ctx, meta.workers, gen, meta.cfg, entries)
+	if err != nil {
+		return err
+	}
 
 	if c.Bool(requireOwnerFlag) {
 		summary.Report(verifyOwners(entries, meta.cfg.Owners.CompileAllowed())...)

cmd/pint/main.go

@@ -82,12 +82,6 @@ type actionMeta struct {
 	workers int
 }
 
-func (meta actionMeta) cleanup() {
-	for _, prom := range meta.cfg.PrometheusServers {
-		prom.Close()
-	}
-}
-
 func actionSetup(c *cli.Context) (meta actionMeta, err error) {
 	err = initLogger(c.String(logLevelFlag), c.Bool(noColorFlag))
 	if err != nil {

cmd/pint/main_test.go

@@ -297,6 +297,7 @@ func tlsCert(ts *testscript.TestScript, _ bool, args []string) {
 			StreetAddress: []string{""},
 			PostalCode:    []string{""},
 		},
+		DNSNames:     []string{name},
 		IPAddresses:  []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
 		NotBefore:    time.Now(),
 		NotAfter:     time.Now().AddDate(0, 0, 1),

cmd/pint/metrics.go

@@ -3,6 +3,8 @@ package main
 import "github.com/prometheus/client_golang/prometheus"
 
 var (
+	metricsRegistry = prometheus.NewRegistry()
+
 	pintVersion = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "pint_version",

cmd/pint/scan.go

@@ -54,7 +54,16 @@ func tryDecodingYamlError(err error) (l int, s string) {
 	return 1, s
 }
 
-func checkRules(ctx context.Context, workers int, cfg config.Config, entries []discovery.Entry) (summary reporter.Summary) {
+func checkRules(ctx context.Context, workers int, gen *config.PrometheusGenerator, cfg config.Config, entries []discovery.Entry) (summary reporter.Summary, err error) {
+	if len(entries) > 0 {
+		if err = gen.Generate(ctx); err != nil {
+			return summary, err
+		}
+		slog.Debug("Generated all Prometheus servers", slog.Int("count", gen.Count()))
+	} else {
+		slog.Info("No rules found, skipping Prometheus discovery")
+	}
+
 	checkIterationChecks.Set(0)
 	checkIterationChecksDone.Set(0)
 
@@ -108,7 +117,7 @@ func checkRules(ctx context.Context, workers int, cfg config.Config, entries []d
 					)
 				}
 
-				checkList := cfg.GetChecksForRule(ctx, entry.SourcePath, entry.Rule, entry.DisabledChecks)
+				checkList := cfg.GetChecksForRule(ctx, gen, entry.SourcePath, entry.Rule, entry.DisabledChecks)
 				for _, check := range checkList {
 					checkIterationChecks.Inc()
 					check := check
@@ -143,7 +152,7 @@ func checkRules(ctx context.Context, workers int, cfg config.Config, entries []d
 
 	lastRunTime.SetToCurrentTime()
 
-	return summary
+	return summary, nil
 }
 
 type scanJob struct {

cmd/pint/tests/0001_match_path.txt

@@ -4,6 +4,7 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 rules/0002.yml:2 Bug: job label is required and should be preserved when aggregating "^.+$" rules, remove job from without() (promql/aggregate)
  2 |   expr: sum(foo) without(job)
 

cmd/pint/tests/0002_nothing_to_lint.txt

@@ -4,4 +4,5 @@ pint.error  --no-color lint rules
 cmp stderr stderr.txt
 
 -- stderr.txt --
+level=INFO msg="Finding all rules to check" paths=["rules"]
 level=ERROR msg="Fatal error" err="no matching files"

cmd/pint/tests/0003_lint_workdir.txt

@@ -5,6 +5,7 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 rules/0001.yml:2 Warning: job label is required and should be preserved when aggregating "^.+$" rules, remove job from without() (promql/aggregate)
  2 |   expr: sum(rate(fl_cf_html_bytes_in[10m])) WITHOUT (colo_id, instance, node_type, region, node_status, job, colo_name)
 

cmd/pint/tests/0004_fail_invalid_yaml.txt

@@ -4,6 +4,7 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 level=ERROR msg="Failed to parse file content" err="yaml: line 4: did not find expected key" path=rules/bad.yaml lines=1-7
 rules/bad.yaml:4 Fatal: did not find expected key (yaml/parse)
  4 | 

cmd/pint/tests/0005_false_positive.txt

@@ -4,6 +4,7 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 -- rules/0001.yml --
 - record: "colo:test1"
   expr: topk(6, sum(rate(edgeworker_subrequest_errorCount{cordon="free"}[5m])) BY (zoneId,job))

cmd/pint/tests/0006_rr_labels.txt

@@ -4,6 +4,7 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 rules/0001.yml:8 Fatal: incomplete rule, no alert or record key (yaml/parse)
  8 |   - expr: sum(foo)
 

cmd/pint/tests/0007_alerts.txt

@@ -4,6 +4,7 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 rules/0001.yml:1-2 Bug: url annotation is required (alerts/annotation)
  1 | - alert: Always
  2 |   expr: up

cmd/pint/tests/0008_recording_rule_prometheus.txt

@@ -4,6 +4,7 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 rules/0001.yml:5 Bug: instance label should be removed when aggregating "^colo(?:_.+)?:.+$" rules, remove instance from by() (promql/aggregate)
  5 |       expr: sum by (instance) (http_inprogress_requests)
 

cmd/pint/tests/0009_alerting_rule_prometheus.txt

@@ -4,6 +4,7 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 rules/0001.yml:11-13 Bug: link annotation is required (alerts/annotation)
  11 |     annotations:
  12 |       summary: "Instance {{ $labels.instance }} down"

cmd/pint/tests/0010_syntax_check.txt

@@ -4,6 +4,7 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 level=ERROR msg="Failed to parse file content" err="yaml: line 6: did not find expected '-' indicator" path=rules/1.yaml lines=1-12
 rules/1.yaml:6 Fatal: did not find expected '-' indicator (yaml/parse)
  6 | 

cmd/pint/tests/0011_ignore_rules.txt

@@ -4,6 +4,7 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 rules/1.yaml:5 Fatal: syntax error: unexpected right parenthesis ')' (promql/syntax)
  5 |   expr: sum(errors_total) by )
 

cmd/pint/tests/0012_issue_20.txt

@@ -4,6 +4,7 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 level=WARN msg="Tried to read more lines than present in the source file, this is likely due to '
 ' usage in some rules, see https://github.com/cloudflare/pint/issues/20 for details" path=rules/1.yaml
 rules/1.yaml:9-13 Warning: runbook_url annotation is required (alerts/annotation)

cmd/pint/tests/0014_issue49_2.txt

@@ -4,6 +4,7 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 -- rules/0001.yaml --
 - record: down
   expr: up == 0

cmd/pint/tests/0018_match_alerting.txt

@@ -4,7 +4,9 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 level=DEBUG msg="File parsed" path=rules/0001.yml rules=2
+level=DEBUG msg="Generated all Prometheus servers" count=0
 level=DEBUG msg="Found recording rule" path=rules/0001.yml record=colo:recording lines=1-2
 level=DEBUG msg="Configured checks for rule" enabled=["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp"] path=rules/0001.yml rule=colo:recording
 level=DEBUG msg="Found alerting rule" path=rules/0001.yml alert=colo:alerting lines=4-5

cmd/pint/tests/0019_match_recording.txt

@@ -4,7 +4,9 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 level=DEBUG msg="File parsed" path=rules/0001.yml rules=2
+level=DEBUG msg="Generated all Prometheus servers" count=0
 level=DEBUG msg="Found recording rule" path=rules/0001.yml record=colo:recording lines=1-2
 level=DEBUG msg="Configured checks for rule" enabled=["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp","promql/aggregate(job:true)"] path=rules/0001.yml rule=colo:recording
 level=DEBUG msg="Found alerting rule" path=rules/0001.yml alert=colo:alerting lines=4-5

cmd/pint/tests/0020_ignore_kind.txt

@@ -4,7 +4,9 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 level=DEBUG msg="File parsed" path=rules/0001.yml rules=2
+level=DEBUG msg="Generated all Prometheus servers" count=0
 level=DEBUG msg="Found recording rule" path=rules/0001.yml record=colo:recording lines=4-5
 level=DEBUG msg="Configured checks for rule" enabled=["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp","promql/aggregate(job:true)"] path=rules/0001.yml rule=colo:recording
 level=DEBUG msg="Found alerting rule" path=rules/0001.yml alert=colo:alerting lines=7-8

cmd/pint/tests/0022_ignore_multi.txt

@@ -4,6 +4,7 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 rules/1.yaml:2 Warning: dropped label should be removed when aggregating "^.+$" rules, remove dropped from by() (promql/aggregate)
  2 |   expr: sum(errors_total) by(keep,dropped)
 

cmd/pint/tests/0024_color_output.txt

@@ -5,6 +5,7 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 rules/0001.yml:2 Warning: job label is required and should be preserved when aggregating "^.+$" rules, remove job from without() (promql/aggregate)
  2 |   expr: sum(rate(fl_cf_html_bytes_in[10m])) WITHOUT (colo_id, instance, node_type, region, node_status, job, colo_name)
 

cmd/pint/tests/0027_ci_branch.txt

@@ -21,6 +21,7 @@ cmp stderr ../stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check on current git branch using git blame" base=main
 level=INFO msg="Problems found" Fatal=1
 rules.yml:2 Fatal: syntax error: unexpected identifier "bi" (promql/syntax)
  2 |   expr: sum(foo) bi(job)

cmd/pint/tests/0028_ci_git_error.txt

@@ -23,11 +23,9 @@ cmp stderr ../stderr.txt
 level=INFO msg="Loading configuration file" path=.pint.hcl
 level=DEBUG msg="Running git command" args=["rev-parse","--abbrev-ref","HEAD"]
 level=DEBUG msg="Got branch information" base=notmain current=v2
+level=INFO msg="Finding all rules to check on current git branch using git blame" base=notmain
 level=DEBUG msg="Running git command" args=["log","--format=%H","--no-abbrev-commit","--reverse","notmain..HEAD"]
-level=ERROR msg="Fatal error" err="failed to get the list of commits to scan: fatal: ambiguous argument 'notmain..HEAD': unknown revision or path not in the working tree.
-Use '--' to separate paths from revisions, like this:
-'git <command> [<revision>...] -- [<file>...]'
-"
+level=ERROR msg="Fatal error" err="failed to get the list of commits to scan: fatal: ambiguous argument 'notmain..HEAD': unknown revision or path not in the working tree.\nUse '--' to separate paths from revisions, like this:\n'git <command> [<revision>...] -- [<file>...]'\n"
 -- src/v1.yml --
 - record: rule1
   expr: sum(foo) by(job)

cmd/pint/tests/0029_ci_too_many_commits.txt

@@ -28,6 +28,7 @@ cmp stderr ../stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check on current git branch using git blame" base=main
 level=ERROR msg="Fatal error" err="number of commits to check (3) is higher than maxCommits (2), exiting"
 -- src/v1.yml --
 - record: rule1

cmd/pint/tests/0037_disable_checks.txt

@@ -4,9 +4,12 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 level=DEBUG msg="File parsed" path=rules/0001.yml rules=3
-level=DEBUG msg="Starting query workers" name=prom uri=http://127.0.0.1 workers=16
+level=INFO msg="Configured new Prometheus server" name=prom uris=1
+level=DEBUG msg="Generated all Prometheus servers" count=1
 level=DEBUG msg="Found alerting rule" path=rules/0001.yml alert=default-for lines=1-3
+level=DEBUG msg="Starting query workers" name=prom uri=http://127.0.0.1 workers=16
 level=DEBUG msg="Configured checks for rule" enabled=["promql/syntax","alerts/template","promql/fragile","promql/regexp","promql/vector_matching(prom)","rule/duplicate(prom)","labels/conflict(prom)"] path=rules/0001.yml rule=default-for
 level=DEBUG msg="Found recording rule" path=rules/0001.yml record=sum-job lines=5-6
 level=DEBUG msg="Configured checks for rule" enabled=["promql/syntax","alerts/template","promql/fragile","promql/regexp","promql/vector_matching(prom)","rule/duplicate(prom)","labels/conflict(prom)","promql/aggregate(job:true)"] path=rules/0001.yml rule=sum-job

cmd/pint/tests/0038_disable_checks_regex.txt

@@ -4,6 +4,7 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 rules/0001.yml:6 Warning: job label is required and should be preserved when aggregating "^.+$" rules, use by(job, ...) (promql/aggregate)
  6 |   expr: sum(foo)
 

cmd/pint/tests/0039_prom_selected_path.txt

@@ -4,8 +4,10 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 level=DEBUG msg="File parsed" path=rules/0001.yml rules=3
-level=DEBUG msg="Starting query workers" name=disabled uri=http://127.0.0.1:123 workers=16
+level=INFO msg="Configured new Prometheus server" name=disabled uris=1
+level=DEBUG msg="Generated all Prometheus servers" count=1
 level=DEBUG msg="Found alerting rule" path=rules/0001.yml alert=first lines=1-3
 level=DEBUG msg="Configured checks for rule" enabled=["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp"] path=rules/0001.yml rule=first
 level=DEBUG msg="Found recording rule" path=rules/0001.yml record=second lines=5-6
@@ -16,7 +18,6 @@ rules/0001.yml:6 Warning: job label is required and should be preserved when agg
  6 |   expr: sum(bar)
 
 level=INFO msg="Problems found" Warning=1
-level=DEBUG msg="Stopping query workers" name=disabled uri=http://127.0.0.1:123
 -- rules/0001.yml --
 - alert: first
   expr: foo > 1

cmd/pint/tests/0040_rule_match_label.txt

@@ -4,7 +4,9 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 level=DEBUG msg="File parsed" path=rules/rules.yml rules=4
+level=DEBUG msg="Generated all Prometheus servers" count=0
 level=DEBUG msg="Found recording rule" path=rules/rules.yml record=ignore lines=1-2
 level=DEBUG msg="Configured checks for rule" enabled=["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp"] path=rules/rules.yml rule=ignore
 level=DEBUG msg="Found recording rule" path=rules/rules.yml record=match lines=4-7

cmd/pint/tests/0042_watch_metrics.txt

@@ -178,17 +178,3 @@ pint_rules_parsed_total{kind="recording"}
 # HELP pint_version Version information
 # TYPE pint_version gauge
 pint_version{version="unknown"}
-# HELP prometheus_template_text_expansion_failures_total The total number of template text expansion failures.
-# TYPE prometheus_template_text_expansion_failures_total counter
-prometheus_template_text_expansion_failures_total
-# HELP prometheus_template_text_expansions_total The total number of template text expansions.
-# TYPE prometheus_template_text_expansions_total counter
-prometheus_template_text_expansions_total
-# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served.
-# TYPE promhttp_metric_handler_requests_in_flight gauge
-promhttp_metric_handler_requests_in_flight
-# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code.
-# TYPE promhttp_metric_handler_requests_total counter
-promhttp_metric_handler_requests_total{code="200"}
-promhttp_metric_handler_requests_total{code="500"}
-promhttp_metric_handler_requests_total{code="503"}

cmd/pint/tests/0052_match_multiple.txt

@@ -4,7 +4,9 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 level=DEBUG msg="File parsed" path=rules/0001.yml rules=2
+level=DEBUG msg="Generated all Prometheus servers" count=0
 level=DEBUG msg="Found recording rule" path=rules/0001.yml record=colo:recording lines=4-5
 level=DEBUG msg="Configured checks for rule" enabled=["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp","promql/aggregate(job:true)"] path=rules/0001.yml rule=colo:recording
 level=DEBUG msg="Found alerting rule" path=rules/0001.yml alert=colo:alerting lines=7-8

cmd/pint/tests/0053_ignore_multiple.txt

@@ -4,7 +4,9 @@ cmp stderr stderr.txt
 
 -- stderr.txt --
 level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
 level=DEBUG msg="File parsed" path=rules/0001.yml rules=2
+level=DEBUG msg="Generated all Prometheus servers" count=0
 level=DEBUG msg="Found recording rule" path=rules/0001.yml record=colo:recording lines=4-5
 level=DEBUG msg="Configured checks for rule" enabled=["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp"] path=rules/0001.yml rule=colo:recording
 level=DEBUG msg="Found alerting rule" path=rules/0001.yml alert=colo:alerting lines=7-8