sql/opt: add cluster setting sql.stats.canary_fraction

See release note for details. Note that this cluster setting doesn't
apply to internal queries.

Release note (sql change): introduce the cluster setting `sql.stats.canary_fraction`
which takes a float number within range [0, 1]. Its value determines
what fraction of queries will use "canary statistics" (newly collected
stats within their canary window) versus "stable statistics"
(previously proven stats). For example, a value of 0.2 means 20% of
queries will test canary stats while 80% use stable stats. The
selection is atomic per query: if a query is chosen for canary
evaluation, it will use canary statistics for ALL tables it references
(where available). A query never uses a mix of canary and stable
statistics.

Author: ZhouXing19

pkg/sql/opt/BUILD.bazel

@@ -29,6 +29,7 @@ go_library(
         "//pkg/clusterversion",
         "//pkg/security/username",
         "//pkg/server/telemetry",
+        "//pkg/settings",
         "//pkg/sql/catalog",
         "//pkg/sql/catalog/catpb",
         "//pkg/sql/catalog/colinfo",

pkg/sql/opt/memo/memo.go

@@ -322,7 +322,7 @@ func (m *Memo) Init(ctx context.Context, evalCtx *eval.Context) {
 		rowSecurity:                                evalCtx.SessionData().RowSecurity,
 		txnIsoLevel:                                evalCtx.TxnIsoLevel,
 	}
-	m.metadata.Init()
+	m.metadata.Init(evalCtx)
 	m.logPropsBuilder.init(ctx, evalCtx, m)
 }
 

pkg/sql/opt/metadata.go

@@ -9,10 +9,12 @@ import (
 	"context"
 	"fmt"
 	"math/bits"
+	"math/rand"
 	"strings"
 
 	"github.com/cockroachdb/cockroach/pkg/clusterversion"
 	"github.com/cockroachdb/cockroach/pkg/security/username"
+	"github.com/cockroachdb/cockroach/pkg/settings"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/multiregion"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/typedesc"
@@ -46,6 +48,44 @@ type routineDep struct {
 	invocationTypes []*types.T
 }
 
+// canaryFraction controls the probabilistic sampling rate for queries
+// participating in the canary statistics rollout feature.
+//
+// This cluster-level setting determines what fraction of queries will use
+// "canary statistics" (newly collected stats within their canary window)
+// versus "stable statistics" (previously proven stats). For example, a value
+// of 0.2 means 20% of queries will test canary stats while 80% use stable stats.
+//
+// The selection is atomic per query: if a query is chosen for canary evaluation,
+// it will use canary statistics for ALL tables it references (where available).
+// A query never uses a mix of canary and stable statistics.
+var canaryFraction = settings.RegisterFloatSetting(
+	settings.ApplicationLevel,
+	"sql.stats.canary_fraction",
+	"probability that a query will use canary statistics instead of stable statistics (0.0-1.0)",
+	0.1,
+	settings.FloatInRange(0, 1),
+)
+
+// canaryRollDice performs the probabilistic check to determine if a query
+// should use the "canary path" for statistics.
+// This selection is atomic per query.
+func canaryRollDice(evalCtx *eval.Context) bool {
+	threshold := canaryFraction.Get(&evalCtx.Settings.SV)
+
+	// If the fraction is 0, never use canary stats.
+	if threshold == 0 {
+		return false
+	}
+	// If the fraction is 1, always use canary stats.
+	if threshold == 1 {
+		return true
+	}
+
+	actual := rand.Float64()
+	return actual < threshold
+}
+
 // Metadata assigns unique ids to the columns, tables, and other metadata used
 // for global identification within the scope of a particular query. These ids
 // tend to be small integers that can be efficiently stored and manipulated.
@@ -153,12 +193,35 @@ type Metadata struct {
 		depDigest cat.DependencyDigest
 	}
 
+	// useCanaryStats indicates whether this query participates in the canary
+	// statistics rollout feature. When set to true, the optimizer attempts to use
+	// "canary statistics" for all tables referenced by the query.
+	//
+	// This flag is determined probabilistically during query planning based on the
+	// sql.stats.canary_fraction cluster setting. The selection is atomic per query:
+	// either all tables use canary stats (when available) or all use stable stats.
+	//
+	// Canary statistics are newly collected table statistics that are still within
+	// their configured "canary window" (canary_stats_window storage parameter).
+	// These stats provide a controlled way to gradually roll out new statistics
+	// before promoting them to stable, allowing for manual intervention if
+	// performance regressions are detected.
+	//
+	// Stable statistics are the previously established statistics that have either
+	// been promoted from canary status or were collected before canary mode was
+	// enabled for the table.
+	//
+	// Fallback behavior: If a table lacks distinct canary statistics (e.g., only
+	// one statistics version exists, or canary stats have expired), the optimizer
+	// will use the available stable statistics even when this flag is true.
+	useCanaryStats bool
+
 	// NOTE! When adding fields here, update Init (if reusing allocated
 	// data structures is desired), CopyFrom and TestMetadata.
 }
 
 // Init prepares the metadata for use (or reuse).
-func (md *Metadata) Init() {
+func (md *Metadata) Init(evalCtx *eval.Context) {
 	// Clear the metadata objects to release memory (this clearing pattern is
 	// optimized by Go).
 	schemas := md.schemas
@@ -239,6 +302,11 @@ func (md *Metadata) Init() {
 	md.objectRefsByName = objectRefsByName
 	md.privileges = privileges
 	md.builtinRefsByName = builtinRefsByName
+	// TODO(janexing): figure out if this gate can overkill. Can any routines
+	// or builtin functions access tables as an internal query?
+	if !evalCtx.SessionData().Internal {
+		md.useCanaryStats = canaryRollDice(evalCtx)
+	}
 }
 
 // CopyFrom initializes the metadata with a copy of the provided metadata.
@@ -337,6 +405,7 @@ func (md *Metadata) CopyFrom(from *Metadata, copyScalarFn func(Expr) Expr) {
 	md.withBindings = nil
 
 	md.rlsMeta = from.rlsMeta.Copy()
+	md.useCanaryStats = from.useCanaryStats
 }
 
 // MDDepName stores either the unresolved DataSourceName or the StableID from

pkg/sql/opt/metadata_test.go

@@ -63,7 +63,7 @@ func TestMetadata(t *testing.T) {
 		},
 	}
 
-	md.Init()
+	md.Init(&evalCtx)
 	if md.AddSchema(testCat.Schema()) != schID {
 		t.Fatalf("unexpected schema id")
 	}

pkg/sql/opt/testutils/BUILD.bazel

@@ -29,7 +29,9 @@ go_test(
     srcs = ["scalar_vars_test.go"],
     embed = [":testutils"],
     deps = [
+        "//pkg/settings/cluster",
         "//pkg/sql/opt",
+        "//pkg/sql/sem/eval",
         "@com_github_stretchr_testify//assert",
     ],
 )

pkg/sql/opt/testutils/scalar_vars_test.go

@@ -11,7 +11,9 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/sql/opt"
+	"github.com/cockroachdb/cockroach/pkg/sql/sem/eval"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -37,7 +39,8 @@ func TestScalarVars(t *testing.T) {
 	}
 
 	vars := "a int, b string not null, c decimal"
-	md.Init()
+	evalCtx := eval.MakeTestingEvalContext(cluster.MakeTestingClusterSettings())
+	md.Init(&evalCtx)
 	if err := sv.Init(&md, strings.Split(vars, ", ")); err != nil {
 		t.Fatal(err)
 	}