sql: automatically cleanup automatic stats jobs

Automatic stats jobs are frequent -- sometimes almost continious -- background process in
many clusters, constantly being created and running as part of its normal operation. In some
clusters these completed jobs have been observed to make up more than 90% of the
retained content of the jobs system, but in most user-visible surfaces, they are
usually filtered out or excluded as they have typically been found to be of low
relevance or utility to users looking for more notable, discrete cluster events
tied to less continious jobs.

This change introduces automatic, immediate cleanup of automatic stats jobs so that
they are eagerly removed rather than being retained for the full job retention period.
This should dramatically reduce the number of these in the jobs system to just those
are executing and could be relevant to the cluster's operation, and those which were
manually run (and thus likely of concern to a user), as well as those which failed
which could also be of interest.

Release note (ops change): successfully completed automatic SQL stats collecton jobs
now automatically purged rather than being retained for the full default job retention
period.
Epic: CRDB-55121.

Author: dt

pkg/sql/create_stats.go

@@ -91,6 +91,12 @@ var errorOnConcurrentCreateStats = settings.RegisterBoolSetting(
 	false,
 	settings.WithPublic)
 
+var automaticStatsJobAutoCleanup = settings.RegisterBoolSetting(
+	settings.ApplicationLevel,
+	"sql.stats.automatic_stats_job_auto_cleanup.enabled",
+	"set to true to enable automatic cleanup of completed AUTO CREATE STATISTICS jobs",
+	true)
+
 const nonIndexColHistogramBuckets = 2
 
 // createStatsNode is a planNode implemented in terms of a function. The
@@ -203,6 +209,12 @@ func (n *createStatsNode) runJob(ctx context.Context) error {
 				return nil
 			}
 		}
+	} else if automaticStatsJobAutoCleanup.Get(n.p.ExecCfg().SV()) {
+		if name := job.Details().(jobspb.CreateStatsDetails).Name; name == jobspb.AutoStatsName || name == jobspb.AutoPartialStatsName {
+			if err := n.p.ExecCfg().JobRegistry.DeleteTerminalJobByID(ctx, job.ID()); err != nil {
+				log.Dev.Warningf(ctx, "failed to auto-delete terminal automatic stats job: %v", err)
+			}
+		}
 	}
 	return err
 }

pkg/sql/create_stats_test.go

@@ -303,6 +303,7 @@ func TestAutoPartialStatsJobDescription(t *testing.T) {
 	// Disable automatic statistics collection.
 	sqlRunner.Exec(t, `SET CLUSTER SETTING sql.stats.automatic_collection.enabled = false;`)
 	sqlRunner.Exec(t, `SET CLUSTER SETTING sql.stats.automatic_partial_collection.enabled = false;`)
+	sqlRunner.Exec(t, `SET CLUSTER SETTING sql.stats.automatic_stats_job_auto_cleanup.enabled = false;`)
 
 	// Create a test table.
 	sqlRunner.Exec(t, `CREATE TABLE test (id INT PRIMARY KEY, value INT);`)