From 194414327f07d345ce49898aa4d8c8a07eeb9c59 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 30 Sep 2025 15:50:46 +0800 Subject: [PATCH 01/29] This is an automated cherry-pick of #63409 Signed-off-by: ti-chi-bot --- .../snap_client/systable_restore_test.go | 313 +++ pkg/bindinfo/binding.go | 109 + pkg/bindinfo/binding_cache.go | 30 + pkg/bindinfo/binding_operator.go | 253 ++ pkg/bindinfo/global_handle_test.go | 15 + pkg/bindinfo/tests/BUILD.bazel | 9 + pkg/bindinfo/tests/bind_test.go | 3 + pkg/bindinfo/tests/bind_usage_info_test.go | 124 + pkg/bindinfo/utils.go | 347 +++ pkg/domain/domain.go | 22 + .../test/clustertablestest/BUILD.bazel | 1 + .../clustertablestest/cluster_tables_test.go | 7 +- pkg/session/bootstrap.go | 6 + pkg/session/bootstrap_test.go | 213 ++ pkg/session/bootstraptest/BUILD.bazel | 4 + .../bootstraptest/bootstrap_upgrade_test.go | 93 + pkg/session/upgrade.go | 2039 +++++++++++++++++ 17 files changed, 3587 insertions(+), 1 deletion(-) create mode 100644 pkg/bindinfo/binding_operator.go create mode 100644 pkg/bindinfo/tests/bind_usage_info_test.go create mode 100644 pkg/bindinfo/utils.go create mode 100644 pkg/session/upgrade.go diff --git a/br/pkg/restore/snap_client/systable_restore_test.go b/br/pkg/restore/snap_client/systable_restore_test.go index c0df26e1d2386..d619691880dfc 100644 --- a/br/pkg/restore/snap_client/systable_restore_test.go +++ b/br/pkg/restore/snap_client/systable_restore_test.go @@ -116,5 +116,318 @@ func TestCheckSysTableCompatibility(t *testing.T) { // // The above variables are in the file br/pkg/restore/systable_restore.go func TestMonitorTheSystemTableIncremental(t *testing.T) { +<<<<<<< HEAD require.Equal(t, int64(220), session.CurrentBootstrapVersion) +======= + require.Equal(t, int64(253), session.CurrentBootstrapVersion) +} + +func TestIsStatsTemporaryTable(t *testing.T) { + require.False(t, snapclient.IsStatsTemporaryTable("", "")) + require.False(t, snapclient.IsStatsTemporaryTable("", "stats_meta")) + require.False(t, snapclient.IsStatsTemporaryTable("mysql", "stats_meta")) + require.False(t, snapclient.IsStatsTemporaryTable("__TiDB_BR_Temporary_test", "stats_meta")) + require.True(t, snapclient.IsStatsTemporaryTable("__TiDB_BR_Temporary_mysql", "stats_meta")) + require.False(t, snapclient.IsStatsTemporaryTable("__TiDB_BR_Temporary_mysql", "test")) +} + +func TestGetDBNameIfStatsTemporaryTable(t *testing.T) { + _, ok := snapclient.GetDBNameIfStatsTemporaryTable("", "") + require.False(t, ok) + _, ok = snapclient.GetDBNameIfStatsTemporaryTable("", "stats_meta") + require.False(t, ok) + _, ok = snapclient.GetDBNameIfStatsTemporaryTable("mysql", "stats_meta") + require.False(t, ok) + _, ok = snapclient.GetDBNameIfStatsTemporaryTable("__TiDB_BR_Temporary_test", "stats_meta") + require.False(t, ok) + name, ok := snapclient.GetDBNameIfStatsTemporaryTable("__TiDB_BR_Temporary_mysql", "stats_meta") + require.True(t, ok) + require.Equal(t, "mysql", name) + _, ok = snapclient.GetDBNameIfStatsTemporaryTable("__TiDB_BR_Temporary_mysql", "test") + require.False(t, ok) +} + +func TestTemporaryTableCheckerForStatsTemporaryTable(t *testing.T) { + checker := snapclient.NewTemporaryTableChecker(true, false) + _, ok := checker.CheckTemporaryTables("", "") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("", "stats_meta") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("mysql", "stats_meta") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_test", "stats_meta") + require.False(t, ok) + name, ok := checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "stats_meta") + require.True(t, ok) + require.Equal(t, "mysql", name) + _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "test") + require.False(t, ok) + + _, ok = checker.CheckTemporaryTables("", "user") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("mysql", "user") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_test", "user") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "user") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "test") + require.False(t, ok) +} + +func TestIsRenameableSysTemporaryTable(t *testing.T) { + require.False(t, snapclient.IsRenameableSysTemporaryTable("", "")) + require.False(t, snapclient.IsRenameableSysTemporaryTable("", "user")) + require.False(t, snapclient.IsRenameableSysTemporaryTable("mysql", "user")) + require.False(t, snapclient.IsRenameableSysTemporaryTable("__TiDB_BR_Temporary_test", "user")) + require.True(t, snapclient.IsRenameableSysTemporaryTable("__TiDB_BR_Temporary_mysql", "user")) + require.False(t, snapclient.IsRenameableSysTemporaryTable("__TiDB_BR_Temporary_mysql", "test")) +} + +func TestGetDBNameIfRenameableSysTemporaryTable(t *testing.T) { + _, ok := snapclient.GetDBNameIfRenameableSysTemporaryTable("", "") + require.False(t, ok) + _, ok = snapclient.GetDBNameIfRenameableSysTemporaryTable("", "user") + require.False(t, ok) + _, ok = snapclient.GetDBNameIfRenameableSysTemporaryTable("mysql", "user") + require.False(t, ok) + _, ok = snapclient.GetDBNameIfRenameableSysTemporaryTable("__TiDB_BR_Temporary_test", "user") + require.False(t, ok) + name, ok := snapclient.GetDBNameIfRenameableSysTemporaryTable("__TiDB_BR_Temporary_mysql", "user") + require.True(t, ok) + require.Equal(t, "mysql", name) + _, ok = snapclient.GetDBNameIfRenameableSysTemporaryTable("__TiDB_BR_Temporary_mysql", "test") + require.False(t, ok) +} + +func TestTemporaryTableCheckerForRenameableSysTemporaryTable(t *testing.T) { + checker := snapclient.NewTemporaryTableChecker(false, true) + _, ok := checker.CheckTemporaryTables("", "") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("", "user") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("mysql", "user") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_test", "user") + require.False(t, ok) + name, ok := checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "user") + require.True(t, ok) + require.Equal(t, "mysql", name) + _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "test") + require.False(t, ok) + + _, ok = checker.CheckTemporaryTables("", "stats_meta") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("mysql", "stats_meta") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_test", "stats_meta") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "stats_meta") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "test") + require.False(t, ok) +} + +func TestTemporaryTableChecker(t *testing.T) { + checker := snapclient.NewTemporaryTableChecker(true, true) + _, ok := checker.CheckTemporaryTables("", "") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("", "user") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("mysql", "user") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_test", "user") + require.False(t, ok) + name, ok := checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "user") + require.True(t, ok) + require.Equal(t, "mysql", name) + _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "test") + require.False(t, ok) + + _, ok = checker.CheckTemporaryTables("", "stats_meta") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("mysql", "stats_meta") + require.False(t, ok) + _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_test", "stats_meta") + require.False(t, ok) + name, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "stats_meta") + require.True(t, ok) + require.Equal(t, "mysql", name) + _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "test") + require.False(t, ok) +} + +func TestGenerateMoveRenamedTableSQLPair(t *testing.T) { + renameSQL := snapclient.GenerateMoveRenamedTableSQLPair(123, map[string]map[string]struct{}{ + "mysql": {"stats_meta": struct{}{}, "stats_buckets": struct{}{}, "stats_top_n": struct{}{}}, + }) + require.Contains(t, renameSQL, "mysql.stats_meta TO __TiDB_BR_Temporary_mysql.stats_meta_deleted_123") + require.Contains(t, renameSQL, "__TiDB_BR_Temporary_mysql.stats_meta TO mysql.stats_meta") + require.Contains(t, renameSQL, "mysql.stats_buckets TO __TiDB_BR_Temporary_mysql.stats_buckets_deleted_123") + require.Contains(t, renameSQL, "__TiDB_BR_Temporary_mysql.stats_buckets TO mysql.stats_buckets") + require.Contains(t, renameSQL, "mysql.stats_top_n TO __TiDB_BR_Temporary_mysql.stats_top_n_deleted_123") + require.Contains(t, renameSQL, "__TiDB_BR_Temporary_mysql.stats_top_n TO mysql.stats_top_n") +} + +func TestUpdateStatsTableSchema(t *testing.T) { + ctx := context.Background() + expectedSQLs := []string{} + execution := func(_ context.Context, sql string) error { + require.Equal(t, expectedSQLs[0], sql) + expectedSQLs = expectedSQLs[1:] + return nil + } + + // schema name is mismatch + err := snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ + "test": {"stats_meta": struct{}{}}, + }, snapclient.SchemaVersionPairT{ + UpstreamVersionMajor: 7, + UpstreamVersionMinor: 5, + DownstreamVersionMajor: 8, + DownstreamVersionMinor: 5, + }, execution) + require.NoError(t, err) + err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ + "test": {"stats_meta": struct{}{}}, + }, snapclient.SchemaVersionPairT{ + UpstreamVersionMajor: 8, + UpstreamVersionMinor: 5, + DownstreamVersionMajor: 7, + DownstreamVersionMinor: 5, + }, execution) + require.NoError(t, err) + + // table name is mismatch + err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ + "mysql": {"stats_meta2": struct{}{}}, + }, snapclient.SchemaVersionPairT{ + UpstreamVersionMajor: 7, + UpstreamVersionMinor: 5, + DownstreamVersionMajor: 8, + DownstreamVersionMinor: 5, + }, execution) + require.NoError(t, err) + err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ + "mysql": {"stats_meta2": struct{}{}}, + }, snapclient.SchemaVersionPairT{ + UpstreamVersionMajor: 8, + UpstreamVersionMinor: 5, + DownstreamVersionMajor: 7, + DownstreamVersionMinor: 5, + }, execution) + require.NoError(t, err) + + // version range is mismatch + err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ + "mysql": {"stats_meta": struct{}{}}, + }, snapclient.SchemaVersionPairT{ + UpstreamVersionMajor: 7, + UpstreamVersionMinor: 5, + DownstreamVersionMajor: 8, + DownstreamVersionMinor: 1, + }, execution) + require.NoError(t, err) + err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ + "mysql": {"stats_meta": struct{}{}}, + }, snapclient.SchemaVersionPairT{ + UpstreamVersionMajor: 8, + UpstreamVersionMinor: 1, + DownstreamVersionMajor: 7, + DownstreamVersionMinor: 5, + }, execution) + require.NoError(t, err) + err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ + "mysql": {"stats_meta": struct{}{}}, + }, snapclient.SchemaVersionPairT{ + UpstreamVersionMajor: 9, + UpstreamVersionMinor: 1, + DownstreamVersionMajor: 9, + DownstreamVersionMinor: 5, + }, execution) + require.NoError(t, err) + err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ + "mysql": {"stats_meta": struct{}{}}, + }, snapclient.SchemaVersionPairT{ + UpstreamVersionMajor: 9, + UpstreamVersionMinor: 5, + DownstreamVersionMajor: 9, + DownstreamVersionMinor: 1, + }, execution) + require.NoError(t, err) + + // match + expectedSQLs = []string{ + "ALTER TABLE __TiDB_BR_Temporary_mysql.stats_meta ADD COLUMN IF NOT EXISTS last_stats_histograms_version bigint unsigned DEFAULT NULL", + "ALTER TABLE __TiDB_BR_Temporary_mysql.stats_meta DROP COLUMN IF EXISTS last_stats_histograms_version", + "ALTER TABLE __TiDB_BR_Temporary_mysql.stats_meta ADD COLUMN IF NOT EXISTS last_stats_histograms_version bigint unsigned DEFAULT NULL", + "ALTER TABLE __TiDB_BR_Temporary_mysql.stats_meta DROP COLUMN IF EXISTS last_stats_histograms_version", + } + err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ + "mysql": {"stats_meta": struct{}{}}, + }, snapclient.SchemaVersionPairT{ + UpstreamVersionMajor: 7, + UpstreamVersionMinor: 5, + DownstreamVersionMajor: 8, + DownstreamVersionMinor: 5, + }, execution) + require.NoError(t, err) + err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ + "mysql": {"stats_meta": struct{}{}}, + }, snapclient.SchemaVersionPairT{ + UpstreamVersionMajor: 8, + UpstreamVersionMinor: 5, + DownstreamVersionMajor: 8, + DownstreamVersionMinor: 1, + }, execution) + require.NoError(t, err) + err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ + "mysql": {"stats_meta": struct{}{}, "test": struct{}{}}, + "test": {"stats_meta": struct{}{}}, + }, snapclient.SchemaVersionPairT{ + UpstreamVersionMajor: 8, + UpstreamVersionMinor: 1, + DownstreamVersionMajor: 8, + DownstreamVersionMinor: 5, + }, execution) + require.NoError(t, err) + err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ + "mysql": {"stats_meta": struct{}{}, "test": struct{}{}}, + "test": {"stats_meta": struct{}{}}, + }, snapclient.SchemaVersionPairT{ + UpstreamVersionMajor: 8, + UpstreamVersionMinor: 5, + DownstreamVersionMajor: 7, + DownstreamVersionMinor: 5, + }, execution) + require.NoError(t, err) + err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ + "mysql": {"stats_meta": struct{}{}, "test": struct{}{}}, + "test": {"stats_meta": struct{}{}}, + }, snapclient.SchemaVersionPairT{ + UpstreamVersionMajor: 8, + UpstreamVersionMinor: 5, + DownstreamVersionMajor: 8, + DownstreamVersionMinor: 5, + }, execution) + require.NoError(t, err) +} + +func TestNotifyUpdateAllUsersPrivilege(t *testing.T) { + notifier := func() error { + return errors.Errorf("test") + } + err := snapclient.NotifyUpdateAllUsersPrivilege(map[string]map[string]struct{}{ + "test": {"user": {}}, + }, notifier) + require.NoError(t, err) + err = snapclient.NotifyUpdateAllUsersPrivilege(map[string]map[string]struct{}{ + "mysql": {"use": {}, "test": {}}, + }, notifier) + require.NoError(t, err) + err = snapclient.NotifyUpdateAllUsersPrivilege(map[string]map[string]struct{}{ + "mysql": {"test": {}, "user": {}, "db": {}}, + }, notifier) + require.Error(t, err) +>>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) } diff --git a/pkg/bindinfo/binding.go b/pkg/bindinfo/binding.go index 8f3586f676175..159425b0ac69f 100644 --- a/pkg/bindinfo/binding.go +++ b/pkg/bindinfo/binding.go @@ -15,6 +15,14 @@ package bindinfo import ( +<<<<<<< HEAD +======= + "context" + "fmt" + "strings" + "sync" + "sync/atomic" +>>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) "time" "unsafe" @@ -74,6 +82,10 @@ type Binding struct { // TableNames records all schema and table names in this binding statement, which are used for fuzzy matching. TableNames []*ast.TableName `json:"-"` + + // UsageInfo is to track the usage information `last_used_time` of this binding + // and it will be updated when this binding is used. + UsageInfo bindingInfoUsageInfo } func (b *Binding) isSame(rb *Binding) bool { @@ -96,9 +108,106 @@ func (b *Binding) IsBindingAvailable() bool { return b.IsBindingEnabled() || b.Status == Disabled } +<<<<<<< HEAD // SinceUpdateTime returns the duration since last update time. Export for test. func (b *Binding) SinceUpdateTime() (time.Duration, error) { updateTime, err := b.UpdateTime.GoTime(time.Local) +======= +// UpdateLastUsedAt is to update binding usage info when this binding is used. +func (b *Binding) UpdateLastUsedAt() { + now := time.Now() + b.UsageInfo.LastUsedAt.Store(&now) +} + +// UpdateLastSavedAt is to update the last saved time +func (b *Binding) UpdateLastSavedAt(ts *time.Time) { + b.UsageInfo.LastSavedAt.Store(ts) +} + +type bindingInfoUsageInfo struct { + // LastUsedAt records the last time when this binding is used. + // It is nil if this binding has never been used. + // It is updated when this binding is used. + // It is used to update the `last_used_time` field in mysql.bind_info table. + LastUsedAt atomic.Pointer[time.Time] + // LastSavedAt records the last time when this binding is saved into storage. + LastSavedAt atomic.Pointer[time.Time] +} + +var ( + // GetBindingHandle is a function to get the global binding handle. + // It is mainly used to resolve cycle import issue. + GetBindingHandle func(sctx sessionctx.Context) BindingHandle +) + +// BindingMatchInfo records necessary information for cross-db binding matching. +// This is mainly for plan cache to avoid normalizing the same statement repeatedly. +type BindingMatchInfo struct { + NoDBDigest string + TableNames []*ast.TableName +} + +// MatchSQLBindingForPlanCache matches binding for plan cache. +func MatchSQLBindingForPlanCache(sctx sessionctx.Context, stmtNode ast.StmtNode, info *BindingMatchInfo) (bindingSQL string) { + binding, matched, _ := matchSQLBinding(sctx, stmtNode, info) + if matched { + bindingSQL = binding.BindSQL + } + return +} + +// MatchSQLBinding returns the matched binding for this statement. +func MatchSQLBinding(sctx sessionctx.Context, stmtNode ast.StmtNode) (binding *Binding, matched bool, scope string) { + return matchSQLBinding(sctx, stmtNode, nil) +} + +func matchSQLBinding(sctx sessionctx.Context, stmtNode ast.StmtNode, info *BindingMatchInfo) (binding *Binding, matched bool, scope string) { + useBinding := sctx.GetSessionVars().UsePlanBaselines + if !useBinding || stmtNode == nil { + return + } + // When the domain is initializing, the bind will be nil. + if sctx.Value(SessionBindInfoKeyType) == nil { + return + } + + // record the normalization result into info to avoid repeat normalization next time. + var noDBDigest string + var tableNames []*ast.TableName + if info == nil || info.TableNames == nil || info.NoDBDigest == "" { + _, noDBDigest = NormalizeStmtForBinding(stmtNode, "", true) + tableNames = CollectTableNames(stmtNode) + if info != nil { + info.NoDBDigest = noDBDigest + info.TableNames = tableNames + } + } else { + noDBDigest = info.NoDBDigest + tableNames = info.TableNames + } + + sessionHandle := sctx.Value(SessionBindInfoKeyType).(SessionBindingHandle) + if binding, matched := sessionHandle.MatchSessionBinding(sctx, noDBDigest, tableNames); matched { + return binding, matched, metrics.ScopeSession + } + globalHandle := GetBindingHandle(sctx) + if globalHandle == nil { + return + } + binding, matched = globalHandle.MatchingBinding(sctx, noDBDigest, tableNames) + if matched { + // After hitting the cache, update the usage time of the bind. + binding.UpdateLastUsedAt() + return binding, matched, metrics.ScopeGlobal + } + + return +} + +func noDBDigestFromBinding(binding *Binding) (string, error) { + p := parser.New() + stmt, err := p.ParseOneStmt(binding.BindSQL, binding.Charset, binding.Collation) +>>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) if err != nil { return 0, err } diff --git a/pkg/bindinfo/binding_cache.go b/pkg/bindinfo/binding_cache.go index 7e3211665daa5..5346c1e1a6248 100644 --- a/pkg/bindinfo/binding_cache.go +++ b/pkg/bindinfo/binding_cache.go @@ -58,6 +58,18 @@ type FuzzyBindingCache interface { Copy() (c FuzzyBindingCache, err error) BindingCache +<<<<<<< HEAD +======= + + // LoadFromStorageToCache loads global bindings from storage to the memory cache. + LoadFromStorageToCache(fullLoad bool) (err error) + + // UpdateBindingUsageInfoToStorage is to update the binding usage info into storage + UpdateBindingUsageInfoToStorage() error + + // LastUpdateTime returns the last update time. + LastUpdateTime() types.Time +>>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) } type fuzzyBindingCache struct { @@ -149,6 +161,7 @@ func (fbc *fuzzyBindingCache) getFromMemory(sctx sessionctx.Context, fuzzyDigest return matchedBinding, isMatched, missingSQLDigest } +<<<<<<< HEAD func (fbc *fuzzyBindingCache) loadFromStore(sctx sessionctx.Context, missingSQLDigest []string) { if intest.InTest && sctx.Value(LoadBindingNothing) != nil { return @@ -156,6 +169,23 @@ func (fbc *fuzzyBindingCache) loadFromStore(sctx sessionctx.Context, missingSQLD defer func(start time.Time) { sctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("loading binding from storage takes " + time.Since(start).String())) }(time.Now()) +======= +// UpdateBindingUsageInfoToStorage is to update the binding usage info into storage +func (u *bindingCacheUpdater) UpdateBindingUsageInfoToStorage() error { + defer func() { + if r := recover(); r != nil { + bindingLogger().Warn("panic when update usage info for binding", zap.Any("recover", r)) + } + }() + bindings := u.GetAllBindings() + return updateBindingUsageInfoToStorage(u.sPool, bindings) +} + +// LastUpdateTime returns the last update time. +func (u *bindingCacheUpdater) LastUpdateTime() types.Time { + return u.lastUpdateTime.Load().(types.Time) +} +>>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) for _, sqlDigest := range missingSQLDigest { start := time.Now() diff --git a/pkg/bindinfo/binding_operator.go b/pkg/bindinfo/binding_operator.go new file mode 100644 index 0000000000000..8e4d5c00d6fed --- /dev/null +++ b/pkg/bindinfo/binding_operator.go @@ -0,0 +1,253 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bindinfo + +import ( + "strings" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/sessionctx" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util" +) + +// BindingOperator is used to operate (create/drop/update/GC) bindings. +type BindingOperator interface { + // CreateBinding creates a Bindings to the storage and the cache. + // It replaces all the exists bindings for the same normalized SQL. + CreateBinding(sctx sessionctx.Context, bindings []*Binding) (err error) + + // DropBinding drop Bindings to the storage and Bindings int the cache. + DropBinding(sqlDigests []string) (deletedRows uint64, err error) + + // SetBindingStatus set a Bindings's status to the storage and bind cache. + SetBindingStatus(newStatus, sqlDigest string) (ok bool, err error) + + // GCBinding physically removes the deleted bind records in mysql.bind_info. + GCBinding() (err error) +} + +type bindingOperator struct { + cache BindingCacheUpdater + sPool util.DestroyableSessionPool +} + +func newBindingOperator(sPool util.DestroyableSessionPool, cache BindingCacheUpdater) BindingOperator { + return &bindingOperator{ + sPool: sPool, + cache: cache, + } +} + +// CreateBinding creates a Bindings to the storage and the cache. +// It replaces all the exists bindings for the same normalized SQL. +func (op *bindingOperator) CreateBinding(sctx sessionctx.Context, bindings []*Binding) (err error) { + for _, binding := range bindings { + if err := prepareHints(sctx, binding); err != nil { + return err + } + } + defer func() { + if err == nil { + err = op.cache.LoadFromStorageToCache(false) + } + }() + + return callWithSCtx(op.sPool, true, func(sctx sessionctx.Context) error { + // Lock mysql.bind_info to synchronize with CreateBinding / AddBinding / DropBinding on other tidb instances. + if err = lockBindInfoTable(sctx); err != nil { + return err + } + + for i, binding := range bindings { + now := types.NewTime(types.FromGoTime(time.Now()), mysql.TypeTimestamp, 6) + + updateTs := now.String() + _, err = exec( + sctx, + `UPDATE mysql.bind_info SET status = %?, update_time = %? WHERE original_sql = %? AND update_time < %?`, + StatusDeleted, + updateTs, + binding.OriginalSQL, + updateTs, + ) + if err != nil { + return err + } + + binding.CreateTime = now + binding.UpdateTime = now + + // TODO: update the sql_mode or sctx.types.Flag to let execution engine returns errors like dataTooLong, + // overflow directly. + + // Insert the Bindings to the storage. + var sqlDigest, planDigest any // null by default + if binding.SQLDigest != "" { + sqlDigest = binding.SQLDigest + } + if binding.PlanDigest != "" { + planDigest = binding.PlanDigest + } + _, err = exec( + sctx, + `INSERT INTO mysql.bind_info( + original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest +) VALUES (%?,%?, %?, %?, %?, %?, %?, %?, %?, %?, %?)`, + binding.OriginalSQL, + binding.BindSQL, + strings.ToLower(binding.Db), + binding.Status, + binding.CreateTime.String(), + binding.UpdateTime.String(), + binding.Charset, + binding.Collation, + binding.Source, + sqlDigest, + planDigest, + ) + failpoint.Inject("CreateGlobalBindingNthFail", func(val failpoint.Value) { + n := val.(int) + if n == i { + err = errors.NewNoStackError("An injected error") + } + }) + if err != nil { + return err + } + + warnings, _, err := execRows(sctx, "show warnings") + if err != nil { + return err + } + if len(warnings) != 0 { + return errors.New(warnings[0].GetString(2)) + } + } + return nil + }) +} + +// DropBinding drop Bindings to the storage and Bindings int the cache. +func (op *bindingOperator) DropBinding(sqlDigests []string) (deletedRows uint64, err error) { + if len(sqlDigests) == 0 { + return 0, errors.New("sql digest is empty") + } + defer func() { + if err == nil { + err = op.cache.LoadFromStorageToCache(false) + } + }() + + err = callWithSCtx(op.sPool, true, func(sctx sessionctx.Context) error { + // Lock mysql.bind_info to synchronize with CreateBinding / AddBinding / DropBinding on other tidb instances. + if err = lockBindInfoTable(sctx); err != nil { + return err + } + + for _, sqlDigest := range sqlDigests { + updateTs := types.NewTime(types.FromGoTime(time.Now()), mysql.TypeTimestamp, 6).String() + _, err = exec( + sctx, + `UPDATE mysql.bind_info SET status = %?, update_time = %? WHERE sql_digest = %? AND update_time < %? AND status != %?`, + StatusDeleted, + updateTs, + sqlDigest, + updateTs, + StatusDeleted, + ) + if err != nil { + return err + } + deletedRows += sctx.GetSessionVars().StmtCtx.AffectedRows() + } + return nil + }) + if err != nil { + deletedRows = 0 + } + return deletedRows, err +} + +// SetBindingStatus set a Bindings's status to the storage and bind cache. +func (op *bindingOperator) SetBindingStatus(newStatus, sqlDigest string) (ok bool, err error) { + var ( + updateTs types.Time + oldStatus0, oldStatus1 string + ) + if newStatus == StatusDisabled { + // For compatibility reasons, when we need to 'set binding disabled for ', + // we need to consider both the 'enabled' and 'using' status. + oldStatus0 = StatusUsing + oldStatus1 = StatusEnabled + } else if newStatus == StatusEnabled { + // In order to unify the code, two identical old statuses are set. + oldStatus0 = StatusDisabled + oldStatus1 = StatusDisabled + } + + defer func() { + if err == nil { + err = op.cache.LoadFromStorageToCache(false) + } + }() + + err = callWithSCtx(op.sPool, true, func(sctx sessionctx.Context) error { + // Lock mysql.bind_info to synchronize with SetBindingStatus on other tidb instances. + if err = lockBindInfoTable(sctx); err != nil { + return err + } + + updateTs = types.NewTime(types.FromGoTime(time.Now()), mysql.TypeTimestamp, 6) + updateTsStr := updateTs.String() + + _, err = exec(sctx, `UPDATE mysql.bind_info SET status = %?, update_time = %? WHERE sql_digest = %? AND update_time < %? AND status IN (%?, %?)`, + newStatus, updateTsStr, sqlDigest, updateTsStr, oldStatus0, oldStatus1) + return err + }) + return +} + +// GCBinding physically removes the deleted bind records in mysql.bind_info. +func (op *bindingOperator) GCBinding() (err error) { + return callWithSCtx(op.sPool, true, func(sctx sessionctx.Context) error { + // Lock mysql.bind_info to synchronize with CreateBinding / AddBinding / DropBinding on other tidb instances. + if err = lockBindInfoTable(sctx); err != nil { + return err + } + + // To make sure that all the deleted bind records have been acknowledged to all tidb, + // we only garbage collect those records with update_time before 10 leases. + updateTime := time.Now().Add(-(10 * Lease)) + updateTimeStr := types.NewTime(types.FromGoTime(updateTime), mysql.TypeTimestamp, 6).String() + _, err = exec(sctx, `DELETE FROM mysql.bind_info WHERE status = 'deleted' and update_time < %?`, updateTimeStr) + return err + }) +} + +// lockBindInfoTable simulates `LOCK TABLE mysql.bind_info WRITE` by acquiring a pessimistic lock on a +// special builtin row of mysql.bind_info. Note that this function must be called with h.sctx.Lock() held. +// We can replace this implementation to normal `LOCK TABLE mysql.bind_info WRITE` if that feature is +// generally available later. +// This lock would enforce the CREATE / DROP GLOBAL BINDING statements to be executed sequentially, +// even if they come from different tidb instances. +func lockBindInfoTable(sctx sessionctx.Context) error { + // h.sctx already locked. + _, err := exec(sctx, LockBindInfoSQL) + return err +} diff --git a/pkg/bindinfo/global_handle_test.go b/pkg/bindinfo/global_handle_test.go index edb7af4a7a5d7..0c4fe08e88eea 100644 --- a/pkg/bindinfo/global_handle_test.go +++ b/pkg/bindinfo/global_handle_test.go @@ -94,8 +94,13 @@ func TestBindingLastUpdateTimeWithInvalidBind(t *testing.T) { updateTime0 := rows0[0][1] require.Equal(t, updateTime0, "0000-00-00 00:00:00") +<<<<<<< HEAD:pkg/bindinfo/global_handle_test.go tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t`', 'select * from `test` . `t` use index(`idx`)', 'test', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + bindinfo.Manual + "', '', '')") +======= + tk.MustExec("insert into mysql.bind_info (original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest) values('select * from `test` . `t`', 'invalid_binding', 'test', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + + bindinfo.SourceManual + "', '', '')") +>>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)):pkg/bindinfo/binding_operator_test.go tk.MustExec("use test") tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int)") @@ -262,11 +267,16 @@ func TestSetBindingStatusWithoutBindingInCache(t *testing.T) { // Simulate creating bindings on other machines _, sqlDigest := parser.NormalizeDigestForBinding("select * from `test` . `t` where `a` > ?") +<<<<<<< HEAD:pkg/bindinfo/global_handle_test.go tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t` where `a` > ?', 'SELECT /*+ USE_INDEX(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` > 10', 'test', 'deleted', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t` where `a` > ?', 'SELECT /*+ USE_INDEX(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` > 10', 'test', 'enabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") dom.BindHandle().Clear() +======= + tk.MustExec("insert into mysql.bind_info (original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest) values('select * from `test` . `t` where `a` > ?', 'SELECT /*+ USE_INDEX(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` > 10', 'test', 'enabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" + + bindinfo.SourceManual + "', '" + sqlDigest.String() + "', '')") +>>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)):pkg/bindinfo/binding_operator_test.go tk.MustExec("set binding disabled for select * from t where a > 10") tk.MustExec("admin reload bindings") rows := tk.MustQuery("show global bindings").Rows() @@ -277,11 +287,16 @@ func TestSetBindingStatusWithoutBindingInCache(t *testing.T) { internal.UtilCleanBindingEnv(tk, dom) // Simulate creating bindings on other machines +<<<<<<< HEAD:pkg/bindinfo/global_handle_test.go tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t` where `a` > ?', 'SELECT * FROM `test`.`t` WHERE `a` > 10', 'test', 'deleted', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t` where `a` > ?', 'SELECT * FROM `test`.`t` WHERE `a` > 10', 'test', 'disabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") dom.BindHandle().Clear() +======= + tk.MustExec("insert into mysql.bind_info (original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest) values('select * from `test` . `t` where `a` > ?', 'SELECT * FROM `test`.`t` WHERE `a` > 10', 'test', 'disabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" + + bindinfo.SourceManual + "', '" + sqlDigest.String() + "', '')") +>>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)):pkg/bindinfo/binding_operator_test.go tk.MustExec("set binding enabled for select * from t where a > 10") tk.MustExec("admin reload bindings") rows = tk.MustQuery("show global bindings").Rows() diff --git a/pkg/bindinfo/tests/BUILD.bazel b/pkg/bindinfo/tests/BUILD.bazel index c131284da79e7..c7a02ed42cd5c 100644 --- a/pkg/bindinfo/tests/BUILD.bazel +++ b/pkg/bindinfo/tests/BUILD.bazel @@ -5,11 +5,20 @@ go_test( timeout = "moderate", srcs = [ "bind_test.go", +<<<<<<< HEAD +======= + "bind_usage_info_test.go", + "cross_db_binding_test.go", +>>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) "main_test.go", ], flaky = True, race = "on", +<<<<<<< HEAD shard_count = 19, +======= + shard_count = 23, +>>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) deps = [ "//pkg/bindinfo", "//pkg/bindinfo/internal", diff --git a/pkg/bindinfo/tests/bind_test.go b/pkg/bindinfo/tests/bind_test.go index 780b8f17786bb..e99e50f23122d 100644 --- a/pkg/bindinfo/tests/bind_test.go +++ b/pkg/bindinfo/tests/bind_test.go @@ -40,8 +40,11 @@ import ( func TestPrepareCacheWithBinding(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) +<<<<<<< HEAD tk.MustExec(`set tidb_enable_prepared_plan_cache=1`) tk.MustExec("set tidb_cost_model_version=2") +======= +>>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) tk.MustExec("use test") tk.MustExec("drop table if exists t1, t2") tk.MustExec("create table t1(a int, b int, c int, key idx_b(b), key idx_c(c))") diff --git a/pkg/bindinfo/tests/bind_usage_info_test.go b/pkg/bindinfo/tests/bind_usage_info_test.go new file mode 100644 index 0000000000000..d1c54e8e62bd9 --- /dev/null +++ b/pkg/bindinfo/tests/bind_usage_info_test.go @@ -0,0 +1,124 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import ( + "fmt" + "testing" + "time" + + "github.com/pingcap/tidb/pkg/bindinfo" + "github.com/pingcap/tidb/pkg/testkit" + "github.com/stretchr/testify/require" +) + +func TestBindUsageInfo(t *testing.T) { + checklist := []string{ + "5ce1df6eadf8b24222668b1bd2e995b72d4c88e6fe9340d8b13e834703e28c32", + "5d3975ef2160c1e0517353798dac90a9914095d82c025e7cd97bd55aeb804798", + "9d3995845aef70ba086d347f38a4e14c9705e966f7c5793b9fa92194bca2bbef", + "aa3c510b94b9d680f729252ca88415794c8a4f52172c5f9e06c27bee57d08329", + } + bindinfo.UpdateBindingUsageInfoBatchSize = 2 + bindinfo.MaxWriteInterval = 100 * time.Microsecond + store, dom := testkit.CreateMockStoreAndDomain(t) + bindingHandle := dom.BindingHandle() + tk := testkit.NewTestKit(t, store) + + tk.MustExec(`use test`) + tk.MustExec(`set @@tidb_opt_enable_fuzzy_binding=1`) + tk.MustExec("create table t1(a int, b int, c int, key idx_b(b), key idx_c(c))") + tk.MustExec("create table t2(a int, b int, c int, key idx_b(b), key idx_c(c))") + tk.MustExec("create table t3(a int, b int, c int, key idx_b(b), key idx_c(c))") + tk.MustExec("create table t4(a int, b int, c int, key idx_b(b), key idx_c(c))") + tk.MustExec("create table t5(a int, b int, c int, key idx_b(b), key idx_c(c))") + + tk.MustExec("prepare stmt1 from 'delete from t1 where b = 1 and c > 1';") + tk.MustExec("prepare stmt2 from 'delete t1, t2 from t1 inner join t2 on t1.b = t2.b';") + tk.MustExec("prepare stmt3 from 'update t1 set a = 1 where b = 1 and c > 1';") + tk.MustExec("execute stmt1;") + tk.MustExec("create global binding for delete from t1 where b = 1 and c > 1 using delete /*+ use_index(t1,idx_c) */ from t1 where b = 1 and c > 1") + tk.MustExec("create global binding for delete t1, t2 from t1 inner join t2 on t1.b = t2.b using delete /*+ inl_join(t1) */ t1, t2 from t1 inner join t2 on t1.b = t2.b") + tk.MustExec("create global binding for update t1 set a = 1 where b = 1 and c > 1 using update /*+ use_index(t1,idx_c) */ t1 set a = 1 where b = 1 and c > 1") + // cross database binding + tk.MustExec(`create global binding using select /*+ leading(t1, t2, t3, t4, t5) */ * from *.t1, *.t2, *.t3, *.t4, *.t5`) + tk.MustExec("select * from t1, t2, t3, t4, t5") + tk.MustExec("execute stmt1;") + origin := tk.MustQuery(fmt.Sprintf(`select sql_digest,last_used_date from mysql.bind_info where original_sql != '%s' order by sql_digest`, bindinfo.BuiltinPseudoSQL4BindLock)) + origin.Check(testkit.Rows( + "5ce1df6eadf8b24222668b1bd2e995b72d4c88e6fe9340d8b13e834703e28c32 ", + "5d3975ef2160c1e0517353798dac90a9914095d82c025e7cd97bd55aeb804798 ", + "9d3995845aef70ba086d347f38a4e14c9705e966f7c5793b9fa92194bca2bbef ", + "aa3c510b94b9d680f729252ca88415794c8a4f52172c5f9e06c27bee57d08329 ")) + time.Sleep(50 * time.Microsecond) + require.NoError(t, bindingHandle.UpdateBindingUsageInfoToStorage()) + result := tk.MustQuery(fmt.Sprintf(`select sql_digest,last_used_date from mysql.bind_info where original_sql != '%s' order by sql_digest`, bindinfo.BuiltinPseudoSQL4BindLock)) + t.Log("result:", result.Rows()) + // The last_used_date should be updated. + require.True(t, !origin.Equal(result.Rows())) + var first *testkit.Result + for range 5 { + tk.MustExec("execute stmt1;") + tk.MustExec("execute stmt2;") + tk.MustExec("execute stmt3;") + tk.MustExec("select * from t1, t2, t3, t4, t5") + time.Sleep(1 * time.Second) + // Set all last_used_date to null to simulate that the bindinfo in storage is not updated. + resetAllLastUsedData(tk) + require.NoError(t, bindingHandle.UpdateBindingUsageInfoToStorage()) + checkBindinfoInMemory(t, bindingHandle, checklist) + tk.MustQuery(fmt.Sprintf(`select last_used_date from mysql.bind_info where original_sql != '%s' and last_used_date is null`, bindinfo.BuiltinPseudoSQL4BindLock)).Check(testkit.Rows()) + result := tk.MustQuery(fmt.Sprintf(`select sql_digest,last_used_date from mysql.bind_info where original_sql != '%s' order by sql_digest`, bindinfo.BuiltinPseudoSQL4BindLock)) + t.Log("result:", result.Rows()) + if first == nil { + first = result + } else { + // in fact, The result of each for-loop should be the same. + require.True(t, first.Equal(result.Rows())) + } + } + // Set all last_used_date to null to simulate that the bindinfo in storage is not updated. + resetAllLastUsedData(tk) + for range 5 { + time.Sleep(1 * time.Second) + // No used, so last_used_date should not be updated. + require.NoError(t, bindingHandle.UpdateBindingUsageInfoToStorage()) + tk.MustQuery(`select last_used_date from mysql.bind_info where last_used_date is not null`).Check(testkit.Rows()) + } + tk.MustExec("execute stmt1;") + tk.MustExec("execute stmt2;") + tk.MustExec("execute stmt3;") + time.Sleep(1 * time.Second) + require.NoError(t, bindingHandle.UpdateBindingUsageInfoToStorage()) + // it has been updated again. + rows := tk.MustQuery( + fmt.Sprintf(`select * from mysql.bind_info where original_sql != '%s' and last_used_date is not null`, bindinfo.BuiltinPseudoSQL4BindLock)).Rows() + require.Len(t, rows, 3) +} + +func resetAllLastUsedData(tk *testkit.TestKit) { + tk.MustExec(fmt.Sprintf(`update mysql.bind_info set last_used_date = null where original_sql != '%s'`, bindinfo.BuiltinPseudoSQL4BindLock)) +} + +func checkBindinfoInMemory(t *testing.T, bindingHandle bindinfo.BindingHandle, checklist []string) { + for _, digest := range checklist { + binding := bindingHandle.GetBinding(digest) + require.NotNil(t, binding) + lastSaved := binding.UsageInfo.LastSavedAt.Load() + if lastSaved != nil { + require.GreaterOrEqual(t, *binding.UsageInfo.LastSavedAt.Load(), *binding.UsageInfo.LastUsedAt.Load()) + } + } +} diff --git a/pkg/bindinfo/utils.go b/pkg/bindinfo/utils.go new file mode 100644 index 0000000000000..4ae780121c093 --- /dev/null +++ b/pkg/bindinfo/utils.go @@ -0,0 +1,347 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bindinfo + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/format" + "github.com/pingcap/tidb/pkg/parser/terror" + "github.com/pingcap/tidb/pkg/planner/core/resolve" + "github.com/pingcap/tidb/pkg/sessionctx" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util" + "github.com/pingcap/tidb/pkg/util/chunk" + "github.com/pingcap/tidb/pkg/util/hint" + "github.com/pingcap/tidb/pkg/util/intest" + "github.com/pingcap/tidb/pkg/util/logutil" + utilparser "github.com/pingcap/tidb/pkg/util/parser" + "github.com/pingcap/tidb/pkg/util/sqlexec" + "go.uber.org/zap" +) + +func callWithSCtx(sPool util.DestroyableSessionPool, wrapTxn bool, f func(sctx sessionctx.Context) error) (err error) { + resource, err := sPool.Get() + if err != nil { + return err + } + defer func() { + if err == nil { // only recycle when no error + sPool.Put(resource) + } else { + // Note: Otherwise, the session will be leaked. + sPool.Destroy(resource) + } + }() + sctx := resource.(sessionctx.Context) + if wrapTxn { + if _, err = exec(sctx, "BEGIN PESSIMISTIC"); err != nil { + return + } + defer func() { + if err == nil { + _, err = exec(sctx, "COMMIT") + } else { + _, err1 := exec(sctx, "ROLLBACK") + terror.Log(errors.Trace(err1)) + } + }() + } + + err = f(sctx) + return +} + +// exec is a helper function to execute sql and return RecordSet. +func exec(sctx sessionctx.Context, sql string, args ...any) (sqlexec.RecordSet, error) { + sqlExec := sctx.GetSQLExecutor() + return sqlExec.ExecuteInternal(kv.WithInternalSourceType(context.Background(), kv.InternalTxnBindInfo), sql, args...) +} + +// execRows is a helper function to execute sql and return rows and fields. +func execRows(sctx sessionctx.Context, sql string, args ...any) (rows []chunk.Row, fields []*resolve.ResultField, err error) { + sqlExec := sctx.GetRestrictedSQLExecutor() + return sqlExec.ExecRestrictedSQL(kv.WithInternalSourceType(context.Background(), kv.InternalTxnBindInfo), + []sqlexec.OptionFuncAlias{sqlexec.ExecOptionUseCurSession}, sql, args...) +} + +// bindingLogger with category "sql-bind" is used to log statistic related messages. +func bindingLogger() *zap.Logger { + return logutil.BgLogger().With(zap.String("category", "sql-bind")) +} + +// GenerateBindingSQL generates binding sqls from stmt node and plan hints. +func GenerateBindingSQL(stmtNode ast.StmtNode, planHint string, defaultDB string) string { + // We need to evolve plan based on the current sql, not the original sql which may have different parameters. + // So here we would remove the hint and inject the current best plan hint. + hint.BindHint(stmtNode, &hint.HintsSet{}) + bindSQL := RestoreDBForBinding(stmtNode, defaultDB) + if bindSQL == "" { + return "" + } + switch n := stmtNode.(type) { + case *ast.DeleteStmt: + deleteIdx := strings.Index(bindSQL, "DELETE") + // Remove possible `explain` prefix. + bindSQL = bindSQL[deleteIdx:] + return strings.Replace(bindSQL, "DELETE", fmt.Sprintf("DELETE /*+ %s*/", planHint), 1) + case *ast.UpdateStmt: + updateIdx := strings.Index(bindSQL, "UPDATE") + // Remove possible `explain` prefix. + bindSQL = bindSQL[updateIdx:] + return strings.Replace(bindSQL, "UPDATE", fmt.Sprintf("UPDATE /*+ %s*/", planHint), 1) + case *ast.SelectStmt: + var selectIdx int + if n.With != nil { + var withSb strings.Builder + withIdx := strings.Index(bindSQL, "WITH") + restoreCtx := format.NewRestoreCtx(format.RestoreStringSingleQuotes|format.RestoreSpacesAroundBinaryOperation|format.RestoreStringWithoutCharset|format.RestoreNameBackQuotes, &withSb) + restoreCtx.DefaultDB = defaultDB + if err := n.With.Restore(restoreCtx); err != nil { + bindingLogger().Debug("restore SQL failed", zap.Error(err)) + return "" + } + withEnd := withIdx + len(withSb.String()) + tmp := strings.Replace(bindSQL[withEnd:], "SELECT", fmt.Sprintf("SELECT /*+ %s*/", planHint), 1) + return strings.Join([]string{bindSQL[withIdx:withEnd], tmp}, "") + } + selectIdx = strings.Index(bindSQL, "SELECT") + // Remove possible `explain` prefix. + bindSQL = bindSQL[selectIdx:] + return strings.Replace(bindSQL, "SELECT", fmt.Sprintf("SELECT /*+ %s*/", planHint), 1) + case *ast.InsertStmt: + insertIdx := int(0) + if n.IsReplace { + insertIdx = strings.Index(bindSQL, "REPLACE") + } else { + insertIdx = strings.Index(bindSQL, "INSERT") + } + // Remove possible `explain` prefix. + bindSQL = bindSQL[insertIdx:] + return strings.Replace(bindSQL, "SELECT", fmt.Sprintf("SELECT /*+ %s*/", planHint), 1) + } + bindingLogger().Debug("unexpected statement type when generating bind SQL", zap.Any("statement", stmtNode)) + return "" +} + +func readBindingsFromStorage(sPool util.DestroyableSessionPool, condition string, args ...any) (bindings []*Binding, err error) { + selectStmt := fmt.Sprintf(`SELECT original_sql, bind_sql, default_db, status, create_time, + update_time, charset, collation, source, sql_digest, plan_digest FROM mysql.bind_info + %s`, condition) + + err = callWithSCtx(sPool, false, func(sctx sessionctx.Context) error { + rows, _, err := execRows(sctx, selectStmt, args...) + if err != nil { + return err + } + bindings = make([]*Binding, 0, len(rows)) + for _, row := range rows { + // Skip the builtin record which is designed for binding synchronization. + if row.GetString(0) == BuiltinPseudoSQL4BindLock { + continue + } + binding := newBindingFromStorage(row) + if hErr := prepareHints(sctx, binding); hErr != nil { + bindingLogger().Warn("failed to generate bind record from data row", zap.Error(hErr)) + continue + } + bindings = append(bindings, binding) + } + return nil + }) + return +} + +var ( + // UpdateBindingUsageInfoBatchSize indicates the batch size when updating binding usage info to storage. + UpdateBindingUsageInfoBatchSize = 100 + // MaxWriteInterval indicates the interval at which a write operation needs to be performed after a binding has not been read. + MaxWriteInterval = 6 * time.Hour +) + +func updateBindingUsageInfoToStorage(sPool util.DestroyableSessionPool, bindings []*Binding) error { + toWrite := make([]*Binding, 0, UpdateBindingUsageInfoBatchSize) + now := time.Now() + cnt := 0 + defer func() { + if cnt > 0 { + bindingLogger().Info("update binding usage info to storage", zap.Int("count", cnt), zap.Duration("duration", time.Since(now))) + } + }() + for _, binding := range bindings { + lastUsed := binding.UsageInfo.LastUsedAt.Load() + if lastUsed == nil { + continue + } + lastSaved := binding.UsageInfo.LastSavedAt.Load() + if shouldUpdateBinding(lastSaved, lastUsed) { + toWrite = append(toWrite, binding) + cnt++ + } + if len(toWrite) == UpdateBindingUsageInfoBatchSize { + err := updateBindingUsageInfoToStorageInternal(sPool, toWrite) + if err != nil { + return err + } + toWrite = toWrite[:0] + } + } + if len(toWrite) > 0 { + err := updateBindingUsageInfoToStorageInternal(sPool, toWrite) + if err != nil { + return err + } + } + return nil +} + +func shouldUpdateBinding(lastSaved, lastUsed *time.Time) bool { + if lastSaved == nil { + // If it has never been written before, it will be written. + return true + } + // If a certain amount of time specified by MaxWriteInterval has passed since the last record was written, + // and it has been used in between, it will be written. + return time.Since(*lastSaved) >= MaxWriteInterval && lastUsed.After(*lastSaved) +} + +func updateBindingUsageInfoToStorageInternal(sPool util.DestroyableSessionPool, bindings []*Binding) error { + err := callWithSCtx(sPool, true, func(sctx sessionctx.Context) (err error) { + if err = lockBindInfoTable(sctx); err != nil { + return errors.Trace(err) + } + // lockBindInfoTable is to prefetch the rows and lock them, it is good for performance when + // there are many bindings to update with multi tidb nodes. + // in the performance test, it takes 26.24s to update 44679 bindings. + if err = addLockForBinds(sctx, bindings); err != nil { + return errors.Trace(err) + } + for _, binding := range bindings { + lastUsed := binding.UsageInfo.LastUsedAt.Load() + intest.Assert(lastUsed != nil) + err = saveBindingUsage(sctx, binding.SQLDigest, binding.PlanDigest, *lastUsed) + if err != nil { + return errors.Trace(err) + } + } + return nil + }) + if err == nil { + ts := time.Now() + for _, binding := range bindings { + binding.UpdateLastSavedAt(&ts) + } + } + return err +} + +func addLockForBinds(sctx sessionctx.Context, bindings []*Binding) error { + condition := make([]string, 0, len(bindings)) + for _, binding := range bindings { + sqlDigest := binding.SQLDigest + planDigest := binding.PlanDigest + sql := fmt.Sprintf("('%s'", sqlDigest) + if planDigest == "" { + sql += ",NULL)" + } else { + sql += fmt.Sprintf(",'%s')", planDigest) + } + condition = append(condition, sql) + } + locksql := "select 1 from mysql.bind_info use index(digest_index) where (plan_digest, sql_digest) in (" + + strings.Join(condition, " , ") + ") for update" + _, err := exec(sctx, locksql) + if err != nil { + return errors.Trace(err) + } + return nil +} + +func saveBindingUsage(sctx sessionctx.Context, sqldigest, planDigest string, ts time.Time) error { + lastUsedTime := ts.UTC().Format(types.TimeFormat) + var sql = "UPDATE mysql.bind_info USE INDEX(digest_index) SET last_used_date = CONVERT_TZ(%?, '+00:00', @@TIME_ZONE) WHERE sql_digest = %?" + if planDigest == "" { + sql += " AND plan_digest IS NULL" + } else { + sql += fmt.Sprintf(" AND plan_digest = '%s'", planDigest) + } + _, err := exec( + sctx, + sql, + lastUsedTime, sqldigest, + ) + return err +} + +// newBindingFromStorage builds Bindings from a tuple in storage. +func newBindingFromStorage(row chunk.Row) *Binding { + status := row.GetString(3) + // For compatibility, the 'Using' status binding will be converted to the 'Enabled' status binding. + if status == StatusUsing { + status = StatusEnabled + } + return &Binding{ + OriginalSQL: row.GetString(0), + Db: strings.ToLower(row.GetString(2)), + BindSQL: row.GetString(1), + Status: status, + CreateTime: row.GetTime(4), + UpdateTime: row.GetTime(5), + Charset: row.GetString(6), + Collation: row.GetString(7), + Source: row.GetString(8), + SQLDigest: row.GetString(9), + PlanDigest: row.GetString(10), + } +} + +// getBindingPlanDigest does the best efforts to fill binding's plan_digest. +func getBindingPlanDigest(sctx sessionctx.Context, schema, bindingSQL string) (planDigest string) { + defer func() { + if r := recover(); r != nil { + bindingLogger().Error("panic when filling plan digest for binding", + zap.String("binding_sql", bindingSQL), zap.Reflect("panic", r)) + } + }() + + vars := sctx.GetSessionVars() + defer func(originalBaseline bool, originalDB string) { + vars.UsePlanBaselines = originalBaseline + vars.CurrentDB = originalDB + }(vars.UsePlanBaselines, vars.CurrentDB) + vars.UsePlanBaselines = false + vars.CurrentDB = schema + + p := utilparser.GetParser() + defer utilparser.DestroyParser(p) + p.SetSQLMode(vars.SQLMode) + p.SetParserConfig(vars.BuildParserConfig()) + + charset, collation := vars.GetCharsetInfo() + if stmt, err := p.ParseOneStmt(bindingSQL, charset, collation); err == nil { + if !hasParam(stmt) { + // if there is '?' from `create binding using select a from t where a=?`, + // the final plan digest might be incorrect. + planDigest, _ = CalculatePlanDigest(sctx, stmt) + } + } + return +} diff --git a/pkg/domain/domain.go b/pkg/domain/domain.go index 7e8586a029b47..61fc9acaea251 100644 --- a/pkg/domain/domain.go +++ b/pkg/domain/domain.go @@ -2076,9 +2076,11 @@ func (do *Domain) globalBindHandleWorkerLoop(owner owner.Manager) { bindWorkerTicker := time.NewTicker(bindinfo.Lease) gcBindTicker := time.NewTicker(100 * bindinfo.Lease) + writeBindingUsageTicker := time.NewTicker(100 * bindinfo.Lease) defer func() { bindWorkerTicker.Stop() gcBindTicker.Stop() + writeBindingUsageTicker.Stop() }() for { select { @@ -2105,11 +2107,31 @@ func (do *Domain) globalBindHandleWorkerLoop(owner owner.Manager) { if err != nil { logutil.BgLogger().Error("GC bind record failed", zap.Error(err)) } + case <-writeBindingUsageTicker.C: + bindHandle := do.BindingHandle() + err := bindHandle.UpdateBindingUsageInfoToStorage() + if err != nil { + logutil.BgLogger().Warn("BindingHandle.UpdateBindingUsageInfoToStorage", zap.Error(err)) + } + // randomize the next write interval to avoid thundering herd problem + // if there are many tidb servers. The next write interval is [3h, 6h]. + writeBindingUsageTicker.Reset( + randomDuration( + 3*60*60, // 3h + 6*60*60, // 6h + ), + ) } } }, "globalBindHandleWorkerLoop") } +func randomDuration(minSeconds, maxSeconds int) time.Duration { + randomIntervalSeconds := rand.Intn(maxSeconds-minSeconds+1) + minSeconds + newDuration := time.Duration(randomIntervalSeconds) * time.Second + return newDuration +} + // TelemetryLoop create a goroutine that reports usage data in a loop, it should be called only once // in BootstrapSession. func (do *Domain) TelemetryLoop(ctx sessionctx.Context) { diff --git a/pkg/infoschema/test/clustertablestest/BUILD.bazel b/pkg/infoschema/test/clustertablestest/BUILD.bazel index d61f60d5edf02..2e970c389b81a 100644 --- a/pkg/infoschema/test/clustertablestest/BUILD.bazel +++ b/pkg/infoschema/test/clustertablestest/BUILD.bazel @@ -11,6 +11,7 @@ go_test( flaky = True, shard_count = 50, deps = [ + "//pkg/bindinfo", "//pkg/config", "//pkg/domain", "//pkg/errno", diff --git a/pkg/infoschema/test/clustertablestest/cluster_tables_test.go b/pkg/infoschema/test/clustertablestest/cluster_tables_test.go index ddb50e7a9bf42..41e694cabeda2 100644 --- a/pkg/infoschema/test/clustertablestest/cluster_tables_test.go +++ b/pkg/infoschema/test/clustertablestest/cluster_tables_test.go @@ -33,6 +33,7 @@ import ( "github.com/pingcap/fn" "github.com/pingcap/kvproto/pkg/deadlock" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/tidb/pkg/bindinfo" "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/domain" "github.com/pingcap/tidb/pkg/executor" @@ -1531,7 +1532,11 @@ func TestSetBindingStatusBySQLDigest(t *testing.T) { sql = "select * from t where t.a = 1" tk.MustExec(sql) tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("1")) - + bindinfo.MaxWriteInterval = 1 * time.Microsecond + time.Sleep(1 * time.Second) + require.NoError(t, s.dom.BindingHandle().UpdateBindingUsageInfoToStorage()) + tk.MustQuery(fmt.Sprintf(`select last_used_date from mysql.bind_info where original_sql != '%s' and last_used_date is null`, + bindinfo.BuiltinPseudoSQL4BindLock)).Check(testkit.Rows()) sqlDigest := tk.MustQuery("show global bindings").Rows() tk.MustExec(fmt.Sprintf("set binding disabled for sql digest '%s'", sqlDigest[0][9])) tk.MustExec(sql) diff --git a/pkg/session/bootstrap.go b/pkg/session/bootstrap.go index 533eca398a898..87bc41c175459 100644 --- a/pkg/session/bootstrap.go +++ b/pkg/session/bootstrap.go @@ -297,8 +297,14 @@ const ( charset TEXT NOT NULL, collation TEXT NOT NULL, source VARCHAR(10) NOT NULL DEFAULT 'unknown', +<<<<<<< HEAD sql_digest varchar(64), plan_digest varchar(64), +======= + sql_digest varchar(64) DEFAULT NULL, + plan_digest varchar(64) DEFAULT NULL, + last_used_date date DEFAULT NULL, +>>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) INDEX sql_index(original_sql(700),default_db(68)) COMMENT "accelerate the speed when add global binding query", INDEX time_index(update_time) COMMENT "accelerate the speed when querying with last update time" ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;` diff --git a/pkg/session/bootstrap_test.go b/pkg/session/bootstrap_test.go index d70c7da7bb365..a9c92f831ad69 100644 --- a/pkg/session/bootstrap_test.go +++ b/pkg/session/bootstrap_test.go @@ -25,6 +25,10 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/pingcap/tidb/pkg/bindinfo" +<<<<<<< HEAD +======= + "github.com/pingcap/tidb/pkg/config/kerneltype" +>>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) "github.com/pingcap/tidb/pkg/ddl" "github.com/pingcap/tidb/pkg/domain" "github.com/pingcap/tidb/pkg/expression/sessionexpr" @@ -126,6 +130,12 @@ func TestBootstrap(t *testing.T) { require.NoError(t, err) require.Equal(t, 0, req.NumRows()) se.Close() + r = MustExecToRecodeSet(t, se, fmt.Sprintf("select * from mysql.bind_info where original_sql = '%s'", bindinfo.BuiltinPseudoSQL4BindLock)) + req = r.NewChunk(nil) + err = r.Next(ctx, req) + require.NoError(t, err) + require.Equal(t, 1, req.NumRows()) + se.Close() } func globalVarsCount() int64 { @@ -2585,3 +2595,206 @@ func TestTiDBUpgradeToVer219(t *testing.T) { require.Contains(t, string(chk.GetRow(0).GetBytes(1)), "idx_schema_table_state") require.Contains(t, string(chk.GetRow(0).GetBytes(1)), "idx_schema_table_partition_state") } +<<<<<<< HEAD +======= + +func TestTiDBUpgradeToVer252(t *testing.T) { + // NOTE: this case needed to be passed in both classic and next-gen kernel. + // in the first release of next-gen kernel, the version is 250. + ctx := context.Background() + store, dom := CreateStoreAndBootstrap(t) + defer func() { require.NoError(t, store.Close()) }() + + ver250 := version250 + seV250 := CreateSessionAndSetID(t, store) + txn, err := store.Begin() + require.NoError(t, err) + m := meta.NewMutator(txn) + err = m.FinishBootstrap(int64(ver250)) + require.NoError(t, err) + revertVersionAndVariables(t, seV250, ver250) + err = txn.Commit(ctx) + require.NoError(t, err) + store.SetOption(StoreBootstrappedKey, nil) + + getBindInfoSQLFn := func(se sessionapi.Session) string { + res := MustExecToRecodeSet(t, se, "show create table mysql.bind_info") + chk := res.NewChunk(nil) + err = res.Next(ctx, chk) + require.NoError(t, err) + require.Equal(t, 1, chk.NumRows()) + return string(chk.GetRow(0).GetBytes(1)) + } + createTblSQL := getBindInfoSQLFn(seV250) + require.Contains(t, createTblSQL, "`create_time` timestamp(6)") + require.Contains(t, createTblSQL, "`update_time` timestamp(6)") + // revert it back to timestamp(3) for testing. we must set below fields to + // simulate the real session in upgrade process. + seV250.SetValue(sessionctx.Initing, true) + seV250.GetSessionVars().SQLMode = mysql.ModeNone + res := MustExecToRecodeSet(t, seV250, "select create_time,update_time from mysql.bind_info") + chk := res.NewChunk(nil) + err = res.Next(ctx, chk) + for i := range chk.NumRows() { + getTime := chk.GetRow(i).GetTime(0) + getTime = chk.GetRow(i).GetTime(1) + _ = getTime + } + mustExecute(seV250, "alter table mysql.bind_info modify create_time timestamp(3)") + mustExecute(seV250, "alter table mysql.bind_info modify update_time timestamp(3)") + createTblSQL = getBindInfoSQLFn(seV250) + require.Contains(t, createTblSQL, "`create_time` timestamp(3)") + require.Contains(t, createTblSQL, "`update_time` timestamp(3)") + + // do upgrade to latest version + dom.Close() + domCurVer, err := BootstrapSession(store) + require.NoError(t, err) + defer domCurVer.Close() + seCurVer := CreateSessionAndSetID(t, store) + ver, err := getBootstrapVersion(seCurVer) + require.NoError(t, err) + require.Equal(t, currentBootstrapVersion, ver) + // check if the columns have been changed to timestamp(6) + createTblSQL = getBindInfoSQLFn(seCurVer) + require.Contains(t, createTblSQL, "`create_time` timestamp(6)") + require.Contains(t, createTblSQL, "`update_time` timestamp(6)") +} + +func TestWriteClusterIDToMySQLTiDBWhenUpgradingTo242(t *testing.T) { + if kerneltype.IsNextGen() { + t.Skip("Skip this case because there is no upgrade in the first release of next-gen kernel") + } + + ctx := context.Background() + store, dom := CreateStoreAndBootstrap(t) + defer func() { require.NoError(t, store.Close()) }() + + // `cluster_id` is inserted for a new TiDB cluster. + se := CreateSessionAndSetID(t, store) + r := MustExecToRecodeSet(t, se, `select VARIABLE_VALUE from mysql.tidb where VARIABLE_NAME='cluster_id'`) + req := r.NewChunk(nil) + err := r.Next(ctx, req) + require.NoError(t, err) + require.Equal(t, 1, req.NumRows()) + require.NotEmpty(t, req.GetRow(0).GetBytes(0)) + require.NoError(t, r.Close()) + se.Close() + + // bootstrap as version241 + ver241 := version241 + seV241 := CreateSessionAndSetID(t, store) + txn, err := store.Begin() + require.NoError(t, err) + m := meta.NewMutator(txn) + err = m.FinishBootstrap(int64(ver241)) + require.NoError(t, err) + revertVersionAndVariables(t, seV241, ver241) + // remove the cluster_id entry from mysql.tidb table + MustExec(t, seV241, "delete from mysql.tidb where variable_name='cluster_id'") + err = txn.Commit(ctx) + require.NoError(t, err) + store.SetOption(StoreBootstrappedKey, nil) + ver, err := getBootstrapVersion(seV241) + require.NoError(t, err) + require.Equal(t, int64(ver241), ver) + seV241.Close() + + // upgrade to current version + dom.Close() + domCurVer, err := BootstrapSession(store) + require.NoError(t, err) + defer domCurVer.Close() + seCurVer := CreateSessionAndSetID(t, store) + ver, err = getBootstrapVersion(seCurVer) + require.NoError(t, err) + require.Equal(t, currentBootstrapVersion, ver) + + // check if the cluster_id has been set in the `mysql.tidb` table during upgrade + r = MustExecToRecodeSet(t, seCurVer, `select VARIABLE_VALUE from mysql.tidb where VARIABLE_NAME='cluster_id'`) + req = r.NewChunk(nil) + err = r.Next(ctx, req) + require.NoError(t, err) + require.Equal(t, 1, req.NumRows()) + require.NotEmpty(t, req.GetRow(0).GetBytes(0)) + require.NoError(t, r.Close()) + seCurVer.Close() +} + +func TestBindInfoUniqueIndex(t *testing.T) { + if kerneltype.IsNextGen() { + t.Skip("Skip this case because there is no upgrade in the first release of next-gen kernel") + } + + ctx := context.Background() + store, dom := CreateStoreAndBootstrap(t) + defer func() { require.NoError(t, store.Close()) }() + + // bootstrap as version245 + ver245 := version245 + seV245 := CreateSessionAndSetID(t, store) + txn, err := store.Begin() + require.NoError(t, err) + m := meta.NewMutator(txn) + err = m.FinishBootstrap(int64(ver245)) + require.NoError(t, err) + revertVersionAndVariables(t, seV245, ver245) + err = txn.Commit(ctx) + require.NoError(t, err) + store.SetOption(StoreBootstrappedKey, nil) + + // remove the unique index on mysql.bind_info for testing + MustExec(t, seV245, "alter table mysql.bind_info drop index digest_index") + + // insert duplicated values into mysql.bind_info + for _, sqlDigest := range []string{"null", "'x'", "'y'"} { + for _, planDigest := range []string{"null", "'x'", "'y'"} { + insertStmt := fmt.Sprintf(`insert into mysql.bind_info values ( + "sql", "bind_sql", "db", "disabled", NOW(), NOW(), "", "", "", %s, %s, null)`, + sqlDigest, planDigest) + MustExec(t, seV245, insertStmt) + MustExec(t, seV245, insertStmt) + } + } + + // upgrade to current version + dom.Close() + domCurVer, err := BootstrapSession(store) + require.NoError(t, err) + defer domCurVer.Close() + seCurVer := CreateSessionAndSetID(t, store) + ver, err := getBootstrapVersion(seCurVer) + require.NoError(t, err) + require.Equal(t, currentBootstrapVersion, ver) +} + +func TestVersionedBootstrapSchemas(t *testing.T) { + require.True(t, slices.IsSortedFunc(versionedBootstrapSchemas, func(a, b versionedBootstrapSchema) int { + return cmp.Compare(a.ver, b.ver) + }), "versionedBootstrapSchemas should be sorted by version") + + // make sure that later change won't affect existing version schemas. + require.Len(t, versionedBootstrapSchemas[0].databases[0].Tables, 52) + require.Len(t, versionedBootstrapSchemas[0].databases[1].Tables, 0) + + allIDs := make([]int64, 0, len(versionedBootstrapSchemas)) + var allTableCount int + for _, vbs := range versionedBootstrapSchemas { + for _, db := range vbs.databases { + require.Greater(t, db.ID, metadef.ReservedGlobalIDLowerBound) + require.LessOrEqual(t, db.ID, metadef.ReservedGlobalIDUpperBound) + allIDs = append(allIDs, db.ID) + + testTableBasicInfoSlice(t, db.Tables) + allTableCount += len(db.Tables) + for _, tbl := range db.Tables { + allIDs = append(allIDs, tbl.ID) + } + } + } + require.Len(t, tablesInSystemDatabase, allTableCount, + "versionedBootstrapSchemas should have the same number of tables as tablesInSystemDatabase") + slices.Sort(allIDs) + require.IsIncreasing(t, allIDs, "versionedBootstrapSchemas should not have duplicate IDs") +} +>>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) diff --git a/pkg/session/bootstraptest/BUILD.bazel b/pkg/session/bootstraptest/BUILD.bazel index f770b484eb0d5..91a6719ca6b09 100644 --- a/pkg/session/bootstraptest/BUILD.bazel +++ b/pkg/session/bootstraptest/BUILD.bazel @@ -8,7 +8,11 @@ go_test( "main_test.go", ], flaky = True, +<<<<<<< HEAD:pkg/session/bootstraptest/BUILD.bazel shard_count = 12, +======= + shard_count = 15, +>>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)):pkg/session/test/bootstraptest/BUILD.bazel deps = [ "//pkg/config", "//pkg/ddl", diff --git a/pkg/session/bootstraptest/bootstrap_upgrade_test.go b/pkg/session/bootstraptest/bootstrap_upgrade_test.go index 21612418249c1..8dba8a92523bb 100644 --- a/pkg/session/bootstraptest/bootstrap_upgrade_test.go +++ b/pkg/session/bootstraptest/bootstrap_upgrade_test.go @@ -875,3 +875,96 @@ func TestUpgradeWithCrossJoinDisabled(t *testing.T) { require.NoError(t, store.Close()) }() } +<<<<<<< HEAD:pkg/session/bootstraptest/bootstrap_upgrade_test.go +======= + +func TestUpgradeBDRPrimary(t *testing.T) { + fromVersion := 244 + if kerneltype.IsNextGen() { + fromVersion = 250 + } + store, dom := session.CreateStoreAndBootstrap(t) + defer func() { require.NoError(t, store.Close()) }() + seVLow := session.CreateSessionAndSetID(t, store) + txn, err := store.Begin() + require.NoError(t, err) + m := meta.NewMutator(txn) + err = m.FinishBootstrap(int64(fromVersion)) + require.NoError(t, err) + err = txn.Commit(context.Background()) + revertVersionAndVariables(t, seVLow, fromVersion) + require.NoError(t, err) + session.MustExec(t, seVLow, "ADMIN SET BDR ROLE PRIMARY") + store.SetOption(session.StoreBootstrappedKey, nil) + ver, err := session.GetBootstrapVersion(seVLow) + require.NoError(t, err) + require.Equal(t, int64(fromVersion), ver) + dom.Close() + newVer, err := session.BootstrapSession(store) + require.NoError(t, err) + ver, err = session.GetBootstrapVersion(seVLow) + require.NoError(t, err) + require.Equal(t, session.CurrentBootstrapVersion, ver) + newVer.Close() +} + +func TestUpgradeBDRSecondary(t *testing.T) { + fromVersion := 244 + if kerneltype.IsNextGen() { + fromVersion = 250 + } + store, dom := session.CreateStoreAndBootstrap(t) + defer func() { require.NoError(t, store.Close()) }() + seV244 := session.CreateSessionAndSetID(t, store) + txn, err := store.Begin() + require.NoError(t, err) + m := meta.NewMutator(txn) + err = m.FinishBootstrap(int64(fromVersion)) + require.NoError(t, err) + err = txn.Commit(context.Background()) + revertVersionAndVariables(t, seV244, fromVersion) + require.NoError(t, err) + session.MustExec(t, seV244, "ADMIN SET BDR ROLE SECONDARY") + store.SetOption(session.StoreBootstrappedKey, nil) + ver, err := session.GetBootstrapVersion(seV244) + require.NoError(t, err) + require.Equal(t, int64(fromVersion), ver) + dom.Close() + newVer, err := session.BootstrapSession(store) + require.NoError(t, err) + ver, err = session.GetBootstrapVersion(seV244) + require.NoError(t, err) + require.Equal(t, session.CurrentBootstrapVersion, ver) + newVer.Close() +} + +func TestUpgradeBindInfo(t *testing.T) { + fromVersion := 251 + store, dom := session.CreateStoreAndBootstrap(t) + defer func() { require.NoError(t, store.Close()) }() + seV251 := session.CreateSessionAndSetID(t, store) + txn, err := store.Begin() + require.NoError(t, err) + m := meta.NewMutator(txn) + err = m.FinishBootstrap(int64(fromVersion)) + require.NoError(t, err) + err = txn.Commit(context.Background()) + revertVersionAndVariables(t, seV251, fromVersion) + require.NoError(t, err) + session.MustExec(t, seV251, "ADMIN RELOAD BINDINGS;") + store.SetOption(session.StoreBootstrappedKey, nil) + ver, err := session.GetBootstrapVersion(seV251) + require.NoError(t, err) + require.Equal(t, int64(fromVersion), ver) + dom.Close() + newVer, err := session.BootstrapSession(store) + require.NoError(t, err) + seLatestV := session.CreateSessionAndSetID(t, store) + ver, err = session.GetBootstrapVersion(seLatestV) + require.NoError(t, err) + require.Equal(t, session.CurrentBootstrapVersion, ver) + session.MustExec(t, seLatestV, "ADMIN RELOAD BINDINGS;") + newVer.Close() + seLatestV.Close() +} +>>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)):pkg/session/test/bootstraptest/bootstrap_upgrade_test.go diff --git a/pkg/session/upgrade.go b/pkg/session/upgrade.go new file mode 100644 index 0000000000000..f8317ee02e0f5 --- /dev/null +++ b/pkg/session/upgrade.go @@ -0,0 +1,2039 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package session + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/pingcap/tidb/pkg/bindinfo" + "github.com/pingcap/tidb/pkg/config" + "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/infoschema" + "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/meta" + "github.com/pingcap/tidb/pkg/parser" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/parser/terror" + "github.com/pingcap/tidb/pkg/session/sessionapi" + "github.com/pingcap/tidb/pkg/sessionctx/vardef" + "github.com/pingcap/tidb/pkg/sessionctx/variable" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/chunk" + "github.com/pingcap/tidb/pkg/util/dbterror" + "github.com/pingcap/tidb/pkg/util/dbterror/plannererrors" + "github.com/pingcap/tidb/pkg/util/logutil" + utilparser "github.com/pingcap/tidb/pkg/util/parser" + "github.com/pingcap/tidb/pkg/util/sqlexec" + "github.com/pingcap/tidb/pkg/util/timeutil" + "go.uber.org/zap" +) + +const ( + // Const for TiDB server version 2. + version2 = 2 + version3 = 3 + version4 = 4 + version5 = 5 + version6 = 6 + version7 = 7 + version8 = 8 + version9 = 9 + version10 = 10 + version11 = 11 + version12 = 12 + version13 = 13 + version14 = 14 + version15 = 15 + version16 = 16 + version17 = 17 + version18 = 18 + version19 = 19 + version20 = 20 + version21 = 21 + version22 = 22 + version23 = 23 + version24 = 24 + version25 = 25 + version26 = 26 + version27 = 27 + version28 = 28 + // version29 only need to be run when the current version is 28. + version29 = 29 + version30 = 30 + version31 = 31 + version32 = 32 + version33 = 33 + version34 = 34 + version35 = 35 + version36 = 36 + version37 = 37 + version38 = 38 + // version39 will be redone in version46 so it's skipped here. + // version40 is the version that introduce new collation in TiDB, + // see https://github.com/pingcap/tidb/pull/14574 for more details. + version40 = 40 + version41 = 41 + // version42 add storeType and reason column in expr_pushdown_blacklist + version42 = 42 + // version43 updates global variables related to statement summary. + version43 = 43 + // version44 delete tidb_isolation_read_engines from mysql.global_variables to avoid unexpected behavior after upgrade. + version44 = 44 + // version45 introduces CONFIG_PRIV for SET CONFIG statements. + version45 = 45 + // version46 fix a bug in v3.1.1. + version46 = 46 + // version47 add Source to bindings to indicate the way binding created. + version47 = 47 + // version48 reset all deprecated concurrency related system-variables if they were all default value. + // version49 introduces mysql.stats_extended table. + // Both version48 and version49 will be redone in version55 and version56 so they're skipped here. + // version50 add mysql.schema_index_usage table. + version50 = 50 + // version51 introduces CreateTablespacePriv to mysql.user. + // version51 will be redone in version63 so it's skipped here. + // version52 change mysql.stats_histograms cm_sketch column from blob to blob(6291456) + version52 = 52 + // version53 introduce Global variable tidb_enable_strict_double_type_check + version53 = 53 + // version54 writes a variable `mem_quota_query` to mysql.tidb if it's a cluster upgraded from v3.0.x to v4.0.9+. + version54 = 54 + // version55 fixes the bug that upgradeToVer48 would be missed when upgrading from v4.0 to a new version + version55 = 55 + // version56 fixes the bug that upgradeToVer49 would be missed when upgrading from v4.0 to a new version + version56 = 56 + // version57 fixes the bug of concurrent create / drop binding + version57 = 57 + // version58 add `Repl_client_priv` and `Repl_slave_priv` to `mysql.user` + // version58 will be redone in version64 so it's skipped here. + // version59 add writes a variable `oom-action` to mysql.tidb if it's a cluster upgraded from v3.0.x to v4.0.11+. + version59 = 59 + // version60 redesigns `mysql.stats_extended` + version60 = 60 + // version61 will be redone in version67 + // version62 add column ndv for mysql.stats_buckets. + version62 = 62 + // version63 fixes the bug that upgradeToVer51 would be missed when upgrading from v4.0 to a new version + version63 = 63 + // version64 is redone upgradeToVer58 after upgradeToVer63, this is to preserve the order of the columns in mysql.user + version64 = 64 + // version65 add mysql.stats_fm_sketch table. + version65 = 65 + // version66 enables the feature `track_aggregate_memory_usage` by default. + version66 = 66 + // version67 restore all SQL bindings. + version67 = 67 + // version68 update the global variable 'tidb_enable_clustered_index' from 'off' to 'int_only'. + version68 = 68 + // version69 adds mysql.global_grants for DYNAMIC privileges + version69 = 69 + // version70 adds mysql.user.plugin to allow multiple authentication plugins + version70 = 70 + // version71 forces tidb_multi_statement_mode=OFF when tidb_multi_statement_mode=WARN + // This affects upgrades from v4.0 where the default was WARN. + version71 = 71 + // version72 adds snapshot column for mysql.stats_meta + version72 = 72 + // version73 adds mysql.capture_plan_baselines_blacklist table + version73 = 73 + // version74 changes global variable `tidb_stmt_summary_max_stmt_count` value from 200 to 3000. + version74 = 74 + // version75 update mysql.*.host from char(60) to char(255) + version75 = 75 + // version76 update mysql.columns_priv from SET('Select','Insert','Update') to SET('Select','Insert','Update','References') + version76 = 76 + // version77 adds mysql.column_stats_usage table + version77 = 77 + // version78 updates mysql.stats_buckets.lower_bound, mysql.stats_buckets.upper_bound and mysql.stats_histograms.last_analyze_pos from BLOB to LONGBLOB. + version78 = 78 + // version79 adds the mysql.table_cache_meta table + version79 = 79 + // version80 fixes the issue https://github.com/pingcap/tidb/issues/25422. + // If the TiDB upgrading from the 4.x to a newer version, we keep the tidb_analyze_version to 1. + version80 = 80 + // version81 insert "tidb_enable_index_merge|off" to mysql.GLOBAL_VARIABLES if there is no tidb_enable_index_merge. + // This will only happens when we upgrade a cluster before 4.0.0 to 4.0.0+. + version81 = 81 + // version82 adds the mysql.analyze_options table + version82 = 82 + // version83 adds the tables mysql.stats_history + version83 = 83 + // version84 adds the tables mysql.stats_meta_history + version84 = 84 + // version85 updates bindings with status 'using' in mysql.bind_info table to 'enabled' status + version85 = 85 + // version86 update mysql.tables_priv from SET('Select','Insert','Update') to SET('Select','Insert','Update','References'). + version86 = 86 + // version87 adds the mysql.analyze_jobs table + version87 = 87 + // version88 fixes the issue https://github.com/pingcap/tidb/issues/33650. + version88 = 88 + // version89 adds the tables mysql.advisory_locks + version89 = 89 + // version90 converts enable-batch-dml, mem-quota-query, query-log-max-len, committer-concurrency, run-auto-analyze, and oom-action to a sysvar + version90 = 90 + // version91 converts prepared-plan-cache to sysvars + version91 = 91 + // version92 for concurrent ddl. + version92 = 92 + // version93 converts oom-use-tmp-storage to a sysvar + version93 = 93 + version94 = 94 + // version95 add a column `User_attributes` to `mysql.user` + version95 = 95 + // version97 sets tidb_opt_range_max_size to 0 when a cluster upgrades from some version lower than v6.4.0 to v6.4.0+. + // It promises the compatibility of building ranges behavior. + version97 = 97 + // version98 add a column `Token_issuer` to `mysql.user` + version98 = 98 + version99 = 99 + // version100 converts server-memory-quota to a sysvar + version100 = 100 + // version101 add mysql.plan_replayer_status table + version101 = 101 + // version102 add mysql.plan_replayer_task table + version102 = 102 + // version103 adds the tables mysql.stats_table_locked + version103 = 103 + // version104 add `sql_digest` and `plan_digest` to `bind_info` + version104 = 104 + // version105 insert "tidb_cost_model_version|1" to mysql.GLOBAL_VARIABLES if there is no tidb_cost_model_version. + // This will only happens when we upgrade a cluster before 6.0. + version105 = 105 + // version106 add mysql.password_history, and Password_reuse_history, Password_reuse_time into mysql.user. + version106 = 106 + // version107 add columns related to password expiration into mysql.user + version107 = 107 + // version108 adds the table tidb_ttl_table_status + version108 = 108 + // version109 sets tidb_enable_gc_aware_memory_track to off when a cluster upgrades from some version lower than v6.5.0. + version109 = 109 + // ... + // [version110, version129] is the version range reserved for patches of 6.5.x + // ... + // version110 sets tidb_stats_load_pseudo_timeout to ON when a cluster upgrades from some version lower than v6.5.0. + version110 = 110 + // version130 add column source to mysql.stats_meta_history + version130 = 130 + // version131 adds the table tidb_ttl_task and tidb_ttl_job_history + version131 = 131 + // version132 modifies the view tidb_mdl_view + version132 = 132 + // version133 sets tidb_server_memory_limit to "80%" + version133 = 133 + // version134 modifies the following global variables default value: + // - foreign_key_checks: off -> on + // - tidb_enable_foreign_key: off -> on + // - tidb_store_batch_size: 0 -> 4 + version134 = 134 + // version135 sets tidb_opt_advanced_join_hint to off when a cluster upgrades from some version lower than v7.0. + version135 = 135 + // version136 prepare the tables for the distributed task. + version136 = 136 + // version137 introduces some reserved resource groups + version137 = 137 + // version 138 set tidb_enable_null_aware_anti_join to true + version138 = 138 + // version 139 creates mysql.load_data_jobs table for LOAD DATA statement + // deprecated in version184 + version139 = 139 + // version 140 add column task_key to mysql.tidb_global_task + version140 = 140 + // version 141 + // set the value of `tidb_session_plan_cache_size` to "tidb_prepared_plan_cache_size" if there is no `tidb_session_plan_cache_size`. + // update tidb_load_based_replica_read_threshold from 0 to 4 + // This will only happens when we upgrade a cluster before 7.1. + version141 = 141 + // version 142 insert "tidb_enable_non_prepared_plan_cache|0" to mysql.GLOBAL_VARIABLES if there is no tidb_enable_non_prepared_plan_cache. + // This will only happens when we upgrade a cluster before 6.5. + version142 = 142 + // version 143 add column `error` to `mysql.tidb_global_task` and `mysql.tidb_background_subtask` + version143 = 143 + // version 144 turn off `tidb_plan_cache_invalidation_on_fresh_stats`, which is introduced in 7.1-rc, + // if it's upgraded from an existing old version cluster. + version144 = 144 + // version 145 to only add a version make we know when we support upgrade state. + version145 = 145 + // version 146 add index for mysql.stats_meta_history and mysql.stats_history. + version146 = 146 + // ... + // [version147, version166] is the version range reserved for patches of 7.1.x + // ... + // version 167 add column `step` to `mysql.tidb_background_subtask` + version167 = 167 + version168 = 168 + // version 169 + // create table `mysql.tidb_runaway_quarantined_watch` and table `mysql.tidb_runaway_queries` + // to save runaway query records and persist runaway watch at 7.2 version. + // but due to ver171 recreate `mysql.tidb_runaway_watch`, + // no need to create table `mysql.tidb_runaway_quarantined_watch`, so delete it. + version169 = 169 + version170 = 170 + // version 171 + // keep the tidb_server length same as instance in other tables. + version171 = 171 + // version 172 + // create table `mysql.tidb_runaway_watch` and table `mysql.tidb_runaway_watch_done` + // to persist runaway watch and deletion of runaway watch at 7.3. + version172 = 172 + // version 173 add column `summary` to `mysql.tidb_background_subtask`. + version173 = 173 + // version 174 + // add column `step`, `error`; delete unique key; and add key idx_state_update_time + // to `mysql.tidb_background_subtask_history`. + version174 = 174 + + // version 175 + // update normalized bindings of `in (?)` to `in (...)` to solve #44298. + version175 = 175 + + // version 176 + // add `mysql.tidb_global_task_history` + version176 = 176 + + // version 177 + // add `mysql.dist_framework_meta` + version177 = 177 + + // version 178 + // write mDDLTableVersion into `mysql.tidb` table + version178 = 178 + + // version 179 + // enlarge `VARIABLE_VALUE` of `mysql.global_variables` from `varchar(1024)` to `varchar(16383)`. + version179 = 179 + + // ... + // [version180, version189] is the version range reserved for patches of 7.5.x + // ... + + // version 190 + // add priority/create_time/end_time to `mysql.tidb_global_task`/`mysql.tidb_global_task_history` + // add concurrency/create_time/end_time/digest to `mysql.tidb_background_subtask`/`mysql.tidb_background_subtask_history` + // add idx_exec_id(exec_id), uk_digest to `mysql.tidb_background_subtask` + // add cpu_count to mysql.dist_framework_meta + // modify `mysql.dist_framework_meta` host from VARCHAR(100) to VARCHAR(261) + // modify `mysql.tidb_background_subtask`/`mysql.tidb_background_subtask_history` exec_id from varchar(256) to VARCHAR(261) + // modify `mysql.tidb_global_task`/`mysql.tidb_global_task_history` dispatcher_id from varchar(256) to VARCHAR(261) + version190 = 190 + + // version 191 + // set tidb_txn_mode to Optimistic when tidb_txn_mode is not set. + version191 = 191 + + // version 192 + // add new system table `mysql.request_unit_by_group`, which is used for + // historical RU consumption by resource group per day. + version192 = 192 + + // version 193 + // replace `mysql.tidb_mdl_view` table + version193 = 193 + + // version 194 + // remove `mysql.load_data_jobs` table + version194 = 194 + + // version 195 + // drop `mysql.schema_index_usage` table + // create `sys` schema + // create `sys.schema_unused_indexes` table + version195 = 195 + + // version 196 + // add column `target_scope` for 'mysql.tidb_global_task` table + // add column `target_scope` for 'mysql.tidb_global_task_history` table + version196 = 196 + + // version 197 + // replace `mysql.tidb_mdl_view` table + version197 = 197 + + // version 198 + // add column `owner_id` for `mysql.tidb_mdl_info` table + version198 = 198 + + // ... + // [version199, version208] is the version range reserved for patches of 8.1.x + // ... + + // version 209 + // sets `tidb_resource_control_strict_mode` to off when a cluster upgrades from some version lower than v8.2. + version209 = 209 + // version210 indicates that if TiDB is upgraded from a lower version(lower than 8.3.0), the tidb_analyze_column_options will be set to ALL. + version210 = 210 + + // version211 add column `summary` to `mysql.tidb_background_subtask_history`. + version211 = 211 + + // version212 changed a lots of runaway related table. + // 1. switchGroup: add column `switch_group_name` to `mysql.tidb_runaway_watch` and `mysql.tidb_runaway_watch_done`. + // 2. modify column `plan_digest` type, modify column `time` to `start_time, + // modify column `original_sql` to `sample_sql` to `mysql.tidb_runaway_queries`. + // 3. modify column length of `action`. + // 4. add column `rule` to `mysql.tidb_runaway_watch`, `mysql.tidb_runaway_watch_done` and `mysql.tidb_runaway_queries`. + version212 = 212 + + // version 213 + // create `mysql.tidb_pitr_id_map` table + version213 = 213 + + // version 214 + // create `mysql.index_advisor_results` table + version214 = 214 + + // If the TiDB upgrading from the a version before v7.0 to a newer version, we keep the tidb_enable_inl_join_inner_multi_pattern to 0. + version215 = 215 + + // version 216 + // changes variable `tidb_scatter_region` value from ON to "table" and OFF to "". + version216 = 216 + + // version 217 + // Keep tidb_schema_cache_size to 0 if this variable does not exist (upgrading from old version pre 8.1). + version217 = 217 + + // version 218 + // enable fast_create_table on default + version218 = 218 + + // ... + // [version219, version238] is the version range reserved for patches of 8.5.x + // ... + + // next version should start with 239 + + // version 239 + // add modify_params to tidb_global_task and tidb_global_task_history. + version239 = 239 + + // version 240 + // Add indexes to mysql.analyze_jobs to speed up the query. + version240 = 240 + + // Add index on user field for some mysql tables. + version241 = 241 + + // version 242 + // insert `cluster_id` into the `mysql.tidb` table. + // Add workload-based learning system tables + version242 = 242 + + // Add max_node_count column to tidb_global_task and tidb_global_task_history. + // Add extra_params to tidb_global_task and tidb_global_task_history. + version243 = 243 + + // version244 add Max_user_connections into mysql.user. + version244 = 244 + + // version245 updates column types of mysql.bind_info. + version245 = 245 + + // version246 adds new unique index for mysql.bind_info. + version246 = 246 + + // version 247 + // Add last_stats_histograms_version to mysql.stats_meta. + version247 = 247 + + // version 248 + // Update mysql.tidb_pitr_id_map to add restore_id as a primary key field + version248 = 248 + version249 = 249 + + // version250 add keyspace to tidb_global_task and tidb_global_task_history. + version250 = 250 + + // version 251 + // Add group_key to mysql.tidb_import_jobs. + version251 = 251 + + // version 252 + // Update FSP of mysql.bind_info timestamp columns to microsecond precision. + version252 = 252 + + // version253 + // Add last_used_date to mysql.bind_info + version253 = 253 +) + +// versionedUpgradeFunction is a struct that holds the upgrade function related +// to a specific bootstrap version. +// we will run the upgrade function fn when the current bootstrapped version is +// less than the version in this struct +type versionedUpgradeFunction struct { + version int64 + fn func(sessionapi.Session, int64) +} + +// currentBootstrapVersion is defined as a variable, so we can modify its value for testing. +// please make sure this is the largest version +var currentBootstrapVersion int64 = version253 + +var ( + // this list must be ordered by version in ascending order, and the function + // name must follow the same pattern as `upgradeToVer`. + upgradeToVerFunctions = []versionedUpgradeFunction{ + {version: version2, fn: upgradeToVer2}, + {version: version3, fn: upgradeToVer3}, + {version: version4, fn: upgradeToVer4}, + {version: version5, fn: upgradeToVer5}, + {version: version6, fn: upgradeToVer6}, + {version: version7, fn: upgradeToVer7}, + {version: version8, fn: upgradeToVer8}, + {version: version9, fn: upgradeToVer9}, + {version: version10, fn: upgradeToVer10}, + {version: version11, fn: upgradeToVer11}, + {version: version12, fn: upgradeToVer12}, + {version: version13, fn: upgradeToVer13}, + {version: version14, fn: upgradeToVer14}, + {version: version15, fn: upgradeToVer15}, + {version: version16, fn: upgradeToVer16}, + {version: version17, fn: upgradeToVer17}, + {version: version18, fn: upgradeToVer18}, + {version: version19, fn: upgradeToVer19}, + {version: version20, fn: upgradeToVer20}, + {version: version21, fn: upgradeToVer21}, + {version: version22, fn: upgradeToVer22}, + {version: version23, fn: upgradeToVer23}, + {version: version24, fn: upgradeToVer24}, + {version: version25, fn: upgradeToVer25}, + {version: version26, fn: upgradeToVer26}, + {version: version27, fn: upgradeToVer27}, + {version: version28, fn: upgradeToVer28}, + {version: version29, fn: upgradeToVer29}, + {version: version30, fn: upgradeToVer30}, + {version: version31, fn: upgradeToVer31}, + {version: version32, fn: upgradeToVer32}, + {version: version33, fn: upgradeToVer33}, + {version: version34, fn: upgradeToVer34}, + {version: version35, fn: upgradeToVer35}, + {version: version36, fn: upgradeToVer36}, + {version: version37, fn: upgradeToVer37}, + {version: version38, fn: upgradeToVer38}, + // We will redo upgradeToVer39 in upgradeToVer46, + // so upgradeToVer39 is skipped here. + {version: version40, fn: upgradeToVer40}, + {version: version41, fn: upgradeToVer41}, + {version: version42, fn: upgradeToVer42}, + {version: version43, fn: upgradeToVer43}, + {version: version44, fn: upgradeToVer44}, + {version: version45, fn: upgradeToVer45}, + {version: version46, fn: upgradeToVer46}, + {version: version47, fn: upgradeToVer47}, + // We will redo upgradeToVer48 and upgradeToVer49 in upgradeToVer55 and upgradeToVer56, + // so upgradeToVer48 and upgradeToVer49 is skipped here. + {version: version50, fn: upgradeToVer50}, + // We will redo upgradeToVer51 in upgradeToVer63, it is skipped here. + {version: version52, fn: upgradeToVer52}, + {version: version53, fn: upgradeToVer53}, + {version: version54, fn: upgradeToVer54}, + {version: version55, fn: upgradeToVer55}, + {version: version56, fn: upgradeToVer56}, + {version: version57, fn: upgradeToVer57}, + // We will redo upgradeToVer58 in upgradeToVer64, it is skipped here. + {version: version59, fn: upgradeToVer59}, + {version: version60, fn: upgradeToVer60}, + // We will redo upgradeToVer61 in upgradeToVer67, it is skipped here. + {version: version62, fn: upgradeToVer62}, + {version: version63, fn: upgradeToVer63}, + {version: version64, fn: upgradeToVer64}, + {version: version65, fn: upgradeToVer65}, + {version: version66, fn: upgradeToVer66}, + {version: version67, fn: upgradeToVer67}, + {version: version68, fn: upgradeToVer68}, + {version: version69, fn: upgradeToVer69}, + {version: version70, fn: upgradeToVer70}, + {version: version71, fn: upgradeToVer71}, + {version: version72, fn: upgradeToVer72}, + {version: version73, fn: upgradeToVer73}, + {version: version74, fn: upgradeToVer74}, + {version: version75, fn: upgradeToVer75}, + {version: version76, fn: upgradeToVer76}, + {version: version77, fn: upgradeToVer77}, + {version: version78, fn: upgradeToVer78}, + {version: version79, fn: upgradeToVer79}, + {version: version80, fn: upgradeToVer80}, + {version: version81, fn: upgradeToVer81}, + {version: version82, fn: upgradeToVer82}, + {version: version83, fn: upgradeToVer83}, + {version: version84, fn: upgradeToVer84}, + {version: version85, fn: upgradeToVer85}, + {version: version86, fn: upgradeToVer86}, + {version: version87, fn: upgradeToVer87}, + {version: version88, fn: upgradeToVer88}, + {version: version89, fn: upgradeToVer89}, + {version: version90, fn: upgradeToVer90}, + {version: version91, fn: upgradeToVer91}, + {version: version93, fn: upgradeToVer93}, + {version: version94, fn: upgradeToVer94}, + {version: version95, fn: upgradeToVer95}, + // We will redo upgradeToVer96 in upgradeToVer100, it is skipped here. + {version: version97, fn: upgradeToVer97}, + {version: version98, fn: upgradeToVer98}, + {version: version100, fn: upgradeToVer100}, + {version: version101, fn: upgradeToVer101}, + {version: version102, fn: upgradeToVer102}, + {version: version103, fn: upgradeToVer103}, + {version: version104, fn: upgradeToVer104}, + {version: version105, fn: upgradeToVer105}, + {version: version106, fn: upgradeToVer106}, + {version: version107, fn: upgradeToVer107}, + {version: version108, fn: upgradeToVer108}, + {version: version109, fn: upgradeToVer109}, + {version: version110, fn: upgradeToVer110}, + {version: version130, fn: upgradeToVer130}, + {version: version131, fn: upgradeToVer131}, + {version: version132, fn: upgradeToVer132}, + {version: version133, fn: upgradeToVer133}, + {version: version134, fn: upgradeToVer134}, + {version: version135, fn: upgradeToVer135}, + {version: version136, fn: upgradeToVer136}, + {version: version137, fn: upgradeToVer137}, + {version: version138, fn: upgradeToVer138}, + {version: version139, fn: upgradeToVer139}, + {version: version140, fn: upgradeToVer140}, + {version: version141, fn: upgradeToVer141}, + {version: version142, fn: upgradeToVer142}, + {version: version143, fn: upgradeToVer143}, + {version: version144, fn: upgradeToVer144}, + // We will only use Ver145 to differentiate versions, so it is skipped here. + {version: version146, fn: upgradeToVer146}, + {version: version167, fn: upgradeToVer167}, + {version: version168, fn: upgradeToVer168}, + {version: version169, fn: upgradeToVer169}, + {version: version170, fn: upgradeToVer170}, + {version: version171, fn: upgradeToVer171}, + {version: version172, fn: upgradeToVer172}, + {version: version173, fn: upgradeToVer173}, + {version: version174, fn: upgradeToVer174}, + {version: version175, fn: upgradeToVer175}, + {version: version176, fn: upgradeToVer176}, + {version: version177, fn: upgradeToVer177}, + {version: version178, fn: upgradeToVer178}, + {version: version179, fn: upgradeToVer179}, + {version: version190, fn: upgradeToVer190}, + {version: version191, fn: upgradeToVer191}, + {version: version192, fn: upgradeToVer192}, + {version: version193, fn: upgradeToVer193}, + {version: version194, fn: upgradeToVer194}, + {version: version195, fn: upgradeToVer195}, + {version: version196, fn: upgradeToVer196}, + {version: version197, fn: upgradeToVer197}, + {version: version198, fn: upgradeToVer198}, + {version: version209, fn: upgradeToVer209}, + {version: version210, fn: upgradeToVer210}, + {version: version211, fn: upgradeToVer211}, + {version: version212, fn: upgradeToVer212}, + {version: version213, fn: upgradeToVer213}, + {version: version214, fn: upgradeToVer214}, + {version: version215, fn: upgradeToVer215}, + {version: version216, fn: upgradeToVer216}, + {version: version217, fn: upgradeToVer217}, + {version: version218, fn: upgradeToVer218}, + {version: version239, fn: upgradeToVer239}, + {version: version240, fn: upgradeToVer240}, + {version: version241, fn: upgradeToVer241}, + {version: version242, fn: upgradeToVer242}, + {version: version243, fn: upgradeToVer243}, + {version: version244, fn: upgradeToVer244}, + {version: version245, fn: upgradeToVer245}, + {version: version246, fn: upgradeToVer246}, + {version: version247, fn: upgradeToVer247}, + {version: version248, fn: upgradeToVer248}, + {version: version249, fn: upgradeToVer249}, + {version: version250, fn: upgradeToVer250}, + {version: version251, fn: upgradeToVer251}, + {version: version252, fn: upgradeToVer252}, + {version: version253, fn: upgradeToVer253}, + } +) + +// upgradeToVer2 updates to version 2. +func upgradeToVer2(s sessionapi.Session, _ int64) { + // Version 2 add two system variable for DistSQL concurrency controlling. + // Insert distsql related system variable. + distSQLVars := []string{vardef.TiDBDistSQLScanConcurrency} + values := make([]string, 0, len(distSQLVars)) + for _, v := range distSQLVars { + value := fmt.Sprintf(`("%s", "%s")`, v, variable.GetSysVar(v).Value) + values = append(values, value) + } + sql := fmt.Sprintf("INSERT HIGH_PRIORITY IGNORE INTO %s.%s VALUES %s;", mysql.SystemDB, mysql.GlobalVariablesTable, + strings.Join(values, ", ")) + mustExecute(s, sql) +} + +// upgradeToVer3 updates to version 3. +func upgradeToVer3(s sessionapi.Session, _ int64) { + // Version 3 fix tx_read_only variable value. + mustExecute(s, "UPDATE HIGH_PRIORITY %n.%n SET variable_value = '0' WHERE variable_name = 'tx_read_only';", mysql.SystemDB, mysql.GlobalVariablesTable) +} + +// upgradeToVer4 updates to version 4. +func upgradeToVer4(s sessionapi.Session, _ int64) { + mustExecute(s, CreateStatsMetaTable) +} + +func upgradeToVer5(s sessionapi.Session, _ int64) { + mustExecute(s, CreateStatsHistogramsTable) + mustExecute(s, CreateStatsBucketsTable) +} + +func upgradeToVer6(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Super_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Show_db_priv`", infoschema.ErrColumnExists) + // For reasons of compatibility, set the non-exists privilege column value to 'Y', as TiDB doesn't check them in older versions. + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Super_priv='Y'") +} + +func upgradeToVer7(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Process_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Drop_priv`", infoschema.ErrColumnExists) + // For reasons of compatibility, set the non-exists privilege column value to 'Y', as TiDB doesn't check them in older versions. + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Process_priv='Y'") +} + +func upgradeToVer8(s sessionapi.Session, ver int64) { + ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) + // This is a dummy upgrade, it checks whether upgradeToVer7 success, if not, do it again. + if _, err := s.ExecuteInternal(ctx, "SELECT HIGH_PRIORITY `Process_priv` FROM mysql.user LIMIT 0"); err == nil { + return + } + upgradeToVer7(s, ver) +} + +func upgradeToVer9(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Trigger_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Create_user_priv`", infoschema.ErrColumnExists) + // For reasons of compatibility, set the non-exists privilege column value to 'Y', as TiDB doesn't check them in older versions. + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Trigger_priv='Y'") +} + +func doReentrantDDL(s sessionapi.Session, sql string, ignorableErrs ...error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(internalSQLTimeout)*time.Second) + ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnBootstrap) + _, err := s.ExecuteInternal(ctx, sql) + defer cancel() + for _, ignorableErr := range ignorableErrs { + if terror.ErrorEqual(err, ignorableErr) { + return + } + } + if err != nil { + logutil.BgLogger().Fatal("doReentrantDDL error", zap.Error(err)) + } +} + +func upgradeToVer10(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.stats_buckets CHANGE COLUMN `value` `upper_bound` BLOB NOT NULL", infoschema.ErrColumnNotExists, infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.stats_buckets ADD COLUMN `lower_bound` BLOB", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms ADD COLUMN `null_count` BIGINT(64) NOT NULL DEFAULT 0", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms DROP COLUMN distinct_ratio", dbterror.ErrCantDropFieldOrKey) + doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms DROP COLUMN use_count_to_estimate", dbterror.ErrCantDropFieldOrKey) +} + +func upgradeToVer11(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `References_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Grant_priv`", infoschema.ErrColumnExists) + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET References_priv='Y'") +} + +func upgradeToVer12(s sessionapi.Session, _ int64) { + ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) + _, err := s.ExecuteInternal(ctx, "BEGIN") + terror.MustNil(err) + sql := "SELECT HIGH_PRIORITY user, host, password FROM mysql.user WHERE password != ''" + rs, err := s.ExecuteInternal(ctx, sql) + if terror.ErrorEqual(err, plannererrors.ErrUnknownColumn) { + sql := "SELECT HIGH_PRIORITY user, host, authentication_string FROM mysql.user WHERE authentication_string != ''" + rs, err = s.ExecuteInternal(ctx, sql) + } + terror.MustNil(err) + sqls := make([]string, 0, 1) + defer terror.Call(rs.Close) + req := rs.NewChunk(nil) + it := chunk.NewIterator4Chunk(req) + err = rs.Next(ctx, req) + for err == nil && req.NumRows() != 0 { + for row := it.Begin(); row != it.End(); row = it.Next() { + user := row.GetString(0) + host := row.GetString(1) + pass := row.GetString(2) + var newPass string + newPass, err = oldPasswordUpgrade(pass) + terror.MustNil(err) + updateSQL := fmt.Sprintf(`UPDATE HIGH_PRIORITY mysql.user SET password = "%s" WHERE user="%s" AND host="%s"`, newPass, user, host) + sqls = append(sqls, updateSQL) + } + err = rs.Next(ctx, req) + } + terror.MustNil(err) + + for _, sql := range sqls { + mustExecute(s, sql) + } + + sql = fmt.Sprintf(`INSERT HIGH_PRIORITY INTO %s.%s VALUES ("%s", "%d", "TiDB bootstrap version.") ON DUPLICATE KEY UPDATE VARIABLE_VALUE="%d"`, + mysql.SystemDB, mysql.TiDBTable, tidbServerVersionVar, version12, version12) + mustExecute(s, sql) + + mustExecute(s, "COMMIT") +} + +func upgradeToVer13(s sessionapi.Session, _ int64) { + sqls := []string{ + "ALTER TABLE mysql.user ADD COLUMN `Create_tmp_table_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Super_priv`", + "ALTER TABLE mysql.user ADD COLUMN `Lock_tables_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Create_tmp_table_priv`", + "ALTER TABLE mysql.user ADD COLUMN `Create_view_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Execute_priv`", + "ALTER TABLE mysql.user ADD COLUMN `Show_view_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Create_view_priv`", + "ALTER TABLE mysql.user ADD COLUMN `Create_routine_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Show_view_priv`", + "ALTER TABLE mysql.user ADD COLUMN `Alter_routine_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Create_routine_priv`", + "ALTER TABLE mysql.user ADD COLUMN `Event_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Create_user_priv`", + } + for _, sql := range sqls { + doReentrantDDL(s, sql, infoschema.ErrColumnExists) + } + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Create_tmp_table_priv='Y',Lock_tables_priv='Y',Create_routine_priv='Y',Alter_routine_priv='Y',Event_priv='Y' WHERE Super_priv='Y'") + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Create_view_priv='Y',Show_view_priv='Y' WHERE Create_priv='Y'") +} + +func upgradeToVer14(s sessionapi.Session, _ int64) { + sqls := []string{ + "ALTER TABLE mysql.db ADD COLUMN `References_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Grant_priv`", + "ALTER TABLE mysql.db ADD COLUMN `Create_tmp_table_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Alter_priv`", + "ALTER TABLE mysql.db ADD COLUMN `Lock_tables_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Create_tmp_table_priv`", + "ALTER TABLE mysql.db ADD COLUMN `Create_view_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Lock_tables_priv`", + "ALTER TABLE mysql.db ADD COLUMN `Show_view_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Create_view_priv`", + "ALTER TABLE mysql.db ADD COLUMN `Create_routine_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Show_view_priv`", + "ALTER TABLE mysql.db ADD COLUMN `Alter_routine_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Create_routine_priv`", + "ALTER TABLE mysql.db ADD COLUMN `Event_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Execute_priv`", + "ALTER TABLE mysql.db ADD COLUMN `Trigger_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Event_priv`", + } + for _, sql := range sqls { + doReentrantDDL(s, sql, infoschema.ErrColumnExists) + } +} + +func upgradeToVer15(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateGCDeleteRangeTable) +} + +func upgradeToVer16(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms ADD COLUMN `cm_sketch` BLOB", infoschema.ErrColumnExists) +} + +func upgradeToVer17(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user MODIFY User CHAR(32)") +} + +func upgradeToVer18(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms ADD COLUMN `tot_col_size` BIGINT(64) NOT NULL DEFAULT 0", infoschema.ErrColumnExists) +} + +func upgradeToVer19(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.db MODIFY User CHAR(32)") + doReentrantDDL(s, "ALTER TABLE mysql.tables_priv MODIFY User CHAR(32)") + doReentrantDDL(s, "ALTER TABLE mysql.columns_priv MODIFY User CHAR(32)") +} + +func upgradeToVer20(s sessionapi.Session, _ int64) { + // NOTE: Feedback is deprecated, but we still need to create this table for compatibility. + doReentrantDDL(s, CreateStatsFeedbackTable) +} + +func upgradeToVer21(s sessionapi.Session, _ int64) { + mustExecute(s, CreateGCDeleteRangeDoneTable) + + doReentrantDDL(s, "ALTER TABLE mysql.gc_delete_range DROP INDEX job_id", dbterror.ErrCantDropFieldOrKey) + doReentrantDDL(s, "ALTER TABLE mysql.gc_delete_range ADD UNIQUE INDEX delete_range_index (job_id, element_id)", dbterror.ErrDupKeyName) + doReentrantDDL(s, "ALTER TABLE mysql.gc_delete_range DROP INDEX element_id", dbterror.ErrCantDropFieldOrKey) +} + +func upgradeToVer22(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms ADD COLUMN `stats_ver` BIGINT(64) NOT NULL DEFAULT 0", infoschema.ErrColumnExists) +} + +func upgradeToVer23(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms ADD COLUMN `flag` BIGINT(64) NOT NULL DEFAULT 0", infoschema.ErrColumnExists) +} + +// writeSystemTZ writes system timezone info into mysql.tidb +func writeSystemTZ(s sessionapi.Session) { + mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, "TiDB Global System Timezone.") ON DUPLICATE KEY UPDATE VARIABLE_VALUE= %?`, + mysql.SystemDB, + mysql.TiDBTable, + tidbSystemTZ, + timeutil.InferSystemTZ(), + timeutil.InferSystemTZ(), + ) +} + +// upgradeToVer24 initializes `System` timezone according to docs/design/2018-09-10-adding-tz-env.md +func upgradeToVer24(s sessionapi.Session, _ int64) { + writeSystemTZ(s) +} + +// upgradeToVer25 updates tidb_max_chunk_size to new low bound value 32 if previous value is small than 32. +func upgradeToVer25(s sessionapi.Session, _ int64) { + sql := fmt.Sprintf("UPDATE HIGH_PRIORITY %[1]s.%[2]s SET VARIABLE_VALUE = '%[4]d' WHERE VARIABLE_NAME = '%[3]s' AND VARIABLE_VALUE < %[4]d", + mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBMaxChunkSize, vardef.DefInitChunkSize) + mustExecute(s, sql) +} + +func upgradeToVer26(s sessionapi.Session, _ int64) { + mustExecute(s, CreateRoleEdgesTable) + mustExecute(s, CreateDefaultRolesTable) + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Create_role_priv` ENUM('N','Y') DEFAULT 'N'", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Drop_role_priv` ENUM('N','Y') DEFAULT 'N'", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Account_locked` ENUM('N','Y') DEFAULT 'N'", infoschema.ErrColumnExists) + // user with Create_user_Priv privilege should have Create_view_priv and Show_view_priv after upgrade to v3.0 + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Create_role_priv='Y',Drop_role_priv='Y' WHERE Create_user_priv='Y'") + // user with Create_Priv privilege should have Create_view_priv and Show_view_priv after upgrade to v3.0 + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Create_view_priv='Y',Show_view_priv='Y' WHERE Create_priv='Y'") +} + +func upgradeToVer27(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms ADD COLUMN `correlation` DOUBLE NOT NULL DEFAULT 0", infoschema.ErrColumnExists) +} + +func upgradeToVer28(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateBindInfoTable) +} + +func upgradeToVer29(s sessionapi.Session, ver int64) { + // upgradeToVer29 only need to be run when the current version is 28. + if ver != version28 { + return + } + doReentrantDDL(s, "ALTER TABLE mysql.bind_info CHANGE create_time create_time TIMESTAMP(3)") + doReentrantDDL(s, "ALTER TABLE mysql.bind_info CHANGE update_time update_time TIMESTAMP(3)") + doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD INDEX sql_index (original_sql(1024),default_db(1024))", dbterror.ErrDupKeyName) +} + +func upgradeToVer30(s sessionapi.Session, _ int64) { + mustExecute(s, CreateStatsTopNTable) +} + +func upgradeToVer31(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms ADD COLUMN `last_analyze_pos` BLOB DEFAULT NULL", infoschema.ErrColumnExists) +} + +func upgradeToVer32(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.tables_priv MODIFY table_priv SET('Select','Insert','Update','Delete','Create','Drop','Grant', 'Index', 'Alter', 'Create View', 'Show View', 'Trigger', 'References')") +} + +func upgradeToVer33(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateExprPushdownBlacklistTable) +} + +func upgradeToVer34(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateOptRuleBlacklistTable) +} + +func upgradeToVer35(s sessionapi.Session, _ int64) { + sql := fmt.Sprintf("UPDATE HIGH_PRIORITY %s.%s SET VARIABLE_NAME = '%s' WHERE VARIABLE_NAME = 'tidb_back_off_weight'", + mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBBackOffWeight) + mustExecute(s, sql) +} + +func upgradeToVer36(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Shutdown_priv` ENUM('N','Y') DEFAULT 'N'", infoschema.ErrColumnExists) + // A root user will have those privileges after upgrading. + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Shutdown_priv='Y' WHERE Super_priv='Y'") + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Create_tmp_table_priv='Y',Lock_tables_priv='Y',Create_routine_priv='Y',Alter_routine_priv='Y',Event_priv='Y' WHERE Super_priv='Y'") +} + +func upgradeToVer37(s sessionapi.Session, _ int64) { + // when upgrade from old tidb and no 'tidb_enable_window_function' in GLOBAL_VARIABLES, init it with 0. + sql := fmt.Sprintf("INSERT IGNORE INTO %s.%s (`VARIABLE_NAME`, `VARIABLE_VALUE`) VALUES ('%s', '%d')", + mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBEnableWindowFunction, 0) + mustExecute(s, sql) +} + +func upgradeToVer38(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateGlobalPrivTable) +} + +func writeNewCollationParameter(s sessionapi.Session, flag bool) { + comment := "If the new collations are enabled. Do not edit it." + b := varFalse + if flag { + b = varTrue + } + mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, %?) ON DUPLICATE KEY UPDATE VARIABLE_VALUE=%?`, + mysql.SystemDB, mysql.TiDBTable, TidbNewCollationEnabled, b, comment, b, + ) +} + +func upgradeToVer40(s sessionapi.Session, _ int64) { + // There is no way to enable new collation for an existing TiDB cluster. + writeNewCollationParameter(s, false) +} + +func upgradeToVer41(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user CHANGE `password` `authentication_string` TEXT", infoschema.ErrColumnExists, infoschema.ErrColumnNotExists) + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `password` TEXT as (`authentication_string`)", infoschema.ErrColumnExists) +} + +// writeDefaultExprPushDownBlacklist writes default expr pushdown blacklist into mysql.expr_pushdown_blacklist +func writeDefaultExprPushDownBlacklist(s sessionapi.Session) { + mustExecute(s, "INSERT HIGH_PRIORITY INTO mysql.expr_pushdown_blacklist VALUES"+ + "('date_add','tiflash', 'DST(daylight saving time) does not take effect in TiFlash date_add')") +} + +func upgradeToVer42(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.expr_pushdown_blacklist ADD COLUMN `store_type` CHAR(100) NOT NULL DEFAULT 'tikv,tiflash,tidb'", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.expr_pushdown_blacklist ADD COLUMN `reason` VARCHAR(200)", infoschema.ErrColumnExists) + writeDefaultExprPushDownBlacklist(s) +} + +// Convert statement summary global variables to non-empty values. +func writeStmtSummaryVars(s sessionapi.Session) { + sql := "UPDATE %n.%n SET variable_value= %? WHERE variable_name= %? AND variable_value=''" + mustExecute(s, sql, mysql.SystemDB, mysql.GlobalVariablesTable, variable.BoolToOnOff(vardef.DefTiDBEnableStmtSummary), vardef.TiDBEnableStmtSummary) + mustExecute(s, sql, mysql.SystemDB, mysql.GlobalVariablesTable, variable.BoolToOnOff(vardef.DefTiDBStmtSummaryInternalQuery), vardef.TiDBStmtSummaryInternalQuery) + mustExecute(s, sql, mysql.SystemDB, mysql.GlobalVariablesTable, strconv.Itoa(vardef.DefTiDBStmtSummaryRefreshInterval), vardef.TiDBStmtSummaryRefreshInterval) + mustExecute(s, sql, mysql.SystemDB, mysql.GlobalVariablesTable, strconv.Itoa(vardef.DefTiDBStmtSummaryHistorySize), vardef.TiDBStmtSummaryHistorySize) + mustExecute(s, sql, mysql.SystemDB, mysql.GlobalVariablesTable, strconv.FormatUint(uint64(vardef.DefTiDBStmtSummaryMaxStmtCount), 10), vardef.TiDBStmtSummaryMaxStmtCount) + mustExecute(s, sql, mysql.SystemDB, mysql.GlobalVariablesTable, strconv.FormatUint(uint64(vardef.DefTiDBStmtSummaryMaxSQLLength), 10), vardef.TiDBStmtSummaryMaxSQLLength) +} + +func upgradeToVer43(s sessionapi.Session, _ int64) { + writeStmtSummaryVars(s) +} + +func upgradeToVer44(s sessionapi.Session, _ int64) { + mustExecute(s, "DELETE FROM mysql.global_variables where variable_name = \"tidb_isolation_read_engines\"") +} + +func upgradeToVer45(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Config_priv` ENUM('N','Y') DEFAULT 'N'", infoschema.ErrColumnExists) + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Config_priv='Y' WHERE Super_priv='Y'") +} + +// In v3.1.1, we wrongly replace the context of upgradeToVer39 with upgradeToVer44. If we upgrade from v3.1.1 to a newer version, +// upgradeToVer39 will be missed. So we redo upgradeToVer39 here to make sure the upgrading from v3.1.1 succeed. +func upgradeToVer46(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Reload_priv` ENUM('N','Y') DEFAULT 'N'", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `File_priv` ENUM('N','Y') DEFAULT 'N'", infoschema.ErrColumnExists) + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Reload_priv='Y' WHERE Super_priv='Y'") + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET File_priv='Y' WHERE Super_priv='Y'") +} + +func upgradeToVer47(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD COLUMN `source` varchar(10) NOT NULL default 'unknown'", infoschema.ErrColumnExists) +} + +func upgradeToVer50(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateSchemaIndexUsageTable) +} + +func upgradeToVer52(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms MODIFY cm_sketch BLOB(6291456)") +} + +func upgradeToVer53(s sessionapi.Session, _ int64) { + // when upgrade from old tidb and no `tidb_enable_strict_double_type_check` in GLOBAL_VARIABLES, init it with 1` + sql := fmt.Sprintf("INSERT IGNORE INTO %s.%s (`VARIABLE_NAME`, `VARIABLE_VALUE`) VALUES ('%s', '%d')", + mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBEnableStrictDoubleTypeCheck, 0) + mustExecute(s, sql) +} + +func upgradeToVer54(s sessionapi.Session, ver int64) { + // The mem-query-quota default value is 32GB by default in v3.0, and 1GB by + // default in v4.0. + // If a cluster is upgraded from v3.0.x (bootstrapVer <= version38) to + // v4.0.9+, we'll write the default value to mysql.tidb. Thus we can get the + // default value of mem-quota-query, and promise the compatibility even if + // the tidb-server restarts. + // If it's a newly deployed cluster, we do not need to write the value into + // mysql.tidb, since no compatibility problem will happen. + + // This bootstrap task becomes obsolete in TiDB 5.0+, because it appears that the + // default value of mem-quota-query changes back to 1GB. In TiDB 6.1+ mem-quota-query + // is no longer a config option, but instead a system variable (tidb_mem_quota_query). + + if ver <= version38 { + writeMemoryQuotaQuery(s) + } +} + +// When cherry-pick upgradeToVer52 to v4.0, we wrongly name it upgradeToVer48. +// If we upgrade from v4.0 to a newer version, the real upgradeToVer48 will be missed. +// So we redo upgradeToVer48 here to make sure the upgrading from v4.0 succeeds. +func upgradeToVer55(s sessionapi.Session, _ int64) { + defValues := map[string]string{ + vardef.TiDBIndexLookupConcurrency: "4", + vardef.TiDBIndexLookupJoinConcurrency: "4", + vardef.TiDBHashAggFinalConcurrency: "4", + vardef.TiDBHashAggPartialConcurrency: "4", + vardef.TiDBWindowConcurrency: "4", + vardef.TiDBProjectionConcurrency: "4", + vardef.TiDBHashJoinConcurrency: "5", + } + names := make([]string, 0, len(defValues)) + for n := range defValues { + names = append(names, n) + } + + selectSQL := "select HIGH_PRIORITY * from mysql.global_variables where variable_name in ('" + strings.Join(names, quoteCommaQuote) + "')" + ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) + rs, err := s.ExecuteInternal(ctx, selectSQL) + terror.MustNil(err) + defer terror.Call(rs.Close) + req := rs.NewChunk(nil) + it := chunk.NewIterator4Chunk(req) + err = rs.Next(ctx, req) + for err == nil && req.NumRows() != 0 { + for row := it.Begin(); row != it.End(); row = it.Next() { + n := strings.ToLower(row.GetString(0)) + v := row.GetString(1) + if defValue, ok := defValues[n]; !ok || defValue != v { + return + } + } + err = rs.Next(ctx, req) + } + terror.MustNil(err) + + mustExecute(s, "BEGIN") + v := strconv.Itoa(vardef.ConcurrencyUnset) + sql := fmt.Sprintf("UPDATE %s.%s SET variable_value='%%s' WHERE variable_name='%%s'", mysql.SystemDB, mysql.GlobalVariablesTable) + for _, name := range names { + mustExecute(s, fmt.Sprintf(sql, v, name)) + } + mustExecute(s, "COMMIT") +} + +// When cherry-pick upgradeToVer54 to v4.0, we wrongly name it upgradeToVer49. +// If we upgrade from v4.0 to a newer version, the real upgradeToVer49 will be missed. +// So we redo upgradeToVer49 here to make sure the upgrading from v4.0 succeeds. +func upgradeToVer56(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateStatsExtendedTable) +} + +func upgradeToVer57(s sessionapi.Session, _ int64) { + insertBuiltinBindInfoRow(s) +} + +func insertBuiltinBindInfoRow(s sessionapi.Session) { + mustExecute(s, `INSERT HIGH_PRIORITY INTO mysql.bind_info(original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source) + VALUES (%?, %?, "mysql", %?, "0000-00-00 00:00:00", "0000-00-00 00:00:00", "", "", %?)`, + bindinfo.BuiltinPseudoSQL4BindLock, bindinfo.BuiltinPseudoSQL4BindLock, bindinfo.StatusBuiltin, bindinfo.StatusBuiltin, + ) +} + +func upgradeToVer59(s sessionapi.Session, _ int64) { + // The oom-action default value is log by default in v3.0, and cancel by + // default in v4.0.11+. + // If a cluster is upgraded from v3.0.x (bootstrapVer <= version59) to + // v4.0.11+, we'll write the default value to mysql.tidb. Thus we can get + // the default value of oom-action, and promise the compatibility even if + // the tidb-server restarts. + // If it's a newly deployed cluster, we do not need to write the value into + // mysql.tidb, since no compatibility problem will happen. + writeOOMAction(s) +} + +func upgradeToVer60(s sessionapi.Session, _ int64) { + mustExecute(s, "DROP TABLE IF EXISTS mysql.stats_extended") + doReentrantDDL(s, CreateStatsExtendedTable) +} + +type bindInfo struct { + bindSQL string + status string + createTime types.Time + charset string + collation string + source string +} + +func upgradeToVer67(s sessionapi.Session, _ int64) { + bindMap := make(map[string]bindInfo) + var err error + mustExecute(s, "BEGIN PESSIMISTIC") + + defer func() { + if err != nil { + mustExecute(s, "ROLLBACK") + return + } + + mustExecute(s, "COMMIT") + }() + mustExecute(s, bindinfo.LockBindInfoSQL) + ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) + var rs sqlexec.RecordSet + rs, err = s.ExecuteInternal(ctx, + `SELECT bind_sql, default_db, status, create_time, charset, collation, source + FROM mysql.bind_info + WHERE source != 'builtin' + ORDER BY update_time DESC`) + if err != nil { + logutil.BgLogger().Fatal("upgradeToVer67 error", zap.Error(err)) + } + req := rs.NewChunk(nil) + iter := chunk.NewIterator4Chunk(req) + p := parser.New() + now := types.NewTime(types.FromGoTime(time.Now()), mysql.TypeTimestamp, 3) + for { + err = rs.Next(context.TODO(), req) + if err != nil { + logutil.BgLogger().Fatal("upgradeToVer67 error", zap.Error(err)) + } + if req.NumRows() == 0 { + break + } + updateBindInfo(iter, p, bindMap) + } + terror.Call(rs.Close) + + mustExecute(s, "DELETE FROM mysql.bind_info where source != 'builtin'") + for original, bind := range bindMap { + mustExecute(s, fmt.Sprintf("INSERT INTO mysql.bind_info VALUES(%s, %s, '', %s, %s, %s, %s, %s, %s)", + expression.Quote(original), + expression.Quote(bind.bindSQL), + expression.Quote(bind.status), + expression.Quote(bind.createTime.String()), + expression.Quote(now.String()), + expression.Quote(bind.charset), + expression.Quote(bind.collation), + expression.Quote(bind.source), + )) + } +} + +func updateBindInfo(iter *chunk.Iterator4Chunk, p *parser.Parser, bindMap map[string]bindInfo) { + for row := iter.Begin(); row != iter.End(); row = iter.Next() { + bind := row.GetString(0) + db := row.GetString(1) + status := row.GetString(2) + + if status != bindinfo.StatusEnabled && status != bindinfo.StatusUsing && status != bindinfo.StatusBuiltin { + continue + } + + charset := row.GetString(4) + collation := row.GetString(5) + stmt, err := p.ParseOneStmt(bind, charset, collation) + if err != nil { + logutil.BgLogger().Fatal("updateBindInfo error", zap.Error(err)) + } + originWithDB := parser.Normalize(utilparser.RestoreWithDefaultDB(stmt, db, bind), "ON") + if _, ok := bindMap[originWithDB]; ok { + // The results are sorted in descending order of time. + // And in the following cases, duplicate originWithDB may occur + // originalText |bindText |DB + // `select * from t` |`select /*+ use_index(t, idx) */ * from t` |`test` + // `select * from test.t` |`select /*+ use_index(t, idx) */ * from test.t`|`` + // Therefore, if repeated, we can skip to keep the latest binding. + continue + } + bindMap[originWithDB] = bindInfo{ + bindSQL: utilparser.RestoreWithDefaultDB(stmt, db, bind), + status: status, + createTime: row.GetTime(3), + charset: charset, + collation: collation, + source: row.GetString(6), + } + } +} + +func writeMemoryQuotaQuery(s sessionapi.Session) { + comment := "memory_quota_query is 32GB by default in v3.0.x, 1GB by default in v4.0.x+" + mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, %?) ON DUPLICATE KEY UPDATE VARIABLE_VALUE=%?`, + mysql.SystemDB, mysql.TiDBTable, tidbDefMemoryQuotaQuery, 32<<30, comment, 32<<30, + ) +} + +func upgradeToVer62(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.stats_buckets ADD COLUMN `ndv` bigint not null default 0", infoschema.ErrColumnExists) +} + +func upgradeToVer63(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Create_tablespace_priv` ENUM('N','Y') DEFAULT 'N'", infoschema.ErrColumnExists) + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Create_tablespace_priv='Y' where Super_priv='Y'") +} + +func upgradeToVer64(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Repl_slave_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Execute_priv`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Repl_client_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Repl_slave_priv`", infoschema.ErrColumnExists) + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Repl_slave_priv='Y',Repl_client_priv='Y' where Super_priv='Y'") +} + +func upgradeToVer65(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateStatsFMSketchTable) +} + +func upgradeToVer66(s sessionapi.Session, _ int64) { + mustExecute(s, "set @@global.tidb_track_aggregate_memory_usage = 1") +} + +func upgradeToVer68(s sessionapi.Session, _ int64) { + mustExecute(s, "DELETE FROM mysql.global_variables where VARIABLE_NAME = 'tidb_enable_clustered_index' and VARIABLE_VALUE = 'OFF'") +} + +func upgradeToVer69(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateGlobalGrantsTable) +} + +func upgradeToVer70(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN plugin CHAR(64) AFTER authentication_string", infoschema.ErrColumnExists) + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET plugin='mysql_native_password'") +} + +func upgradeToVer71(s sessionapi.Session, _ int64) { + mustExecute(s, "UPDATE mysql.global_variables SET VARIABLE_VALUE='OFF' WHERE VARIABLE_NAME = 'tidb_multi_statement_mode' AND VARIABLE_VALUE = 'WARN'") +} + +func upgradeToVer72(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.stats_meta ADD COLUMN snapshot BIGINT(64) UNSIGNED NOT NULL DEFAULT 0", infoschema.ErrColumnExists) +} + +func upgradeToVer73(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateCapturePlanBaselinesBlacklistTable) +} + +func upgradeToVer74(s sessionapi.Session, _ int64) { + // The old default value of `tidb_stmt_summary_max_stmt_count` is 200, we want to enlarge this to the new default value when TiDB upgrade. + mustExecute(s, fmt.Sprintf("UPDATE mysql.global_variables SET VARIABLE_VALUE='%[1]v' WHERE VARIABLE_NAME = 'tidb_stmt_summary_max_stmt_count' AND CAST(VARIABLE_VALUE AS SIGNED) = 200", vardef.DefTiDBStmtSummaryMaxStmtCount)) +} + +func upgradeToVer75(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user MODIFY COLUMN Host CHAR(255)") + doReentrantDDL(s, "ALTER TABLE mysql.global_priv MODIFY COLUMN Host CHAR(255)") + doReentrantDDL(s, "ALTER TABLE mysql.db MODIFY COLUMN Host CHAR(255)") + doReentrantDDL(s, "ALTER TABLE mysql.tables_priv MODIFY COLUMN Host CHAR(255)") + doReentrantDDL(s, "ALTER TABLE mysql.columns_priv MODIFY COLUMN Host CHAR(255)") +} + +func upgradeToVer76(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.columns_priv MODIFY COLUMN Column_priv SET('Select','Insert','Update','References')") +} + +func upgradeToVer77(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateColumnStatsUsageTable) +} + +func upgradeToVer78(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.stats_buckets MODIFY upper_bound LONGBLOB NOT NULL") + doReentrantDDL(s, "ALTER TABLE mysql.stats_buckets MODIFY lower_bound LONGBLOB") + doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms MODIFY last_analyze_pos LONGBLOB DEFAULT NULL") +} + +func upgradeToVer79(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateTableCacheMetaTable) +} + +func upgradeToVer80(s sessionapi.Session, _ int64) { + // Check if tidb_analyze_version exists in mysql.GLOBAL_VARIABLES. + // If not, insert "tidb_analyze_version | 1" since this is the old behavior before we introduce this variable. + initGlobalVariableIfNotExists(s, vardef.TiDBAnalyzeVersion, 1) +} + +// For users that upgrade TiDB from a pre-4.0 version, we want to disable index merge by default. +// This helps minimize query plan regressions. +func upgradeToVer81(s sessionapi.Session, _ int64) { + // Check if tidb_enable_index_merge exists in mysql.GLOBAL_VARIABLES. + // If not, insert "tidb_enable_index_merge | off". + initGlobalVariableIfNotExists(s, vardef.TiDBEnableIndexMerge, vardef.Off) +} + +func upgradeToVer82(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateAnalyzeOptionsTable) +} + +func upgradeToVer83(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateStatsHistoryTable) +} + +func upgradeToVer84(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateStatsMetaHistoryTable) +} + +func upgradeToVer85(s sessionapi.Session, _ int64) { + mustExecute(s, fmt.Sprintf("UPDATE HIGH_PRIORITY mysql.bind_info SET status= '%s' WHERE status = '%s'", bindinfo.StatusEnabled, bindinfo.StatusUsing)) +} + +func upgradeToVer86(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.tables_priv MODIFY COLUMN Column_priv SET('Select','Insert','Update','References')") +} + +func upgradeToVer87(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateAnalyzeJobsTable) +} + +func upgradeToVer88(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user CHANGE `Repl_slave_priv` `Repl_slave_priv` ENUM('N','Y') NOT NULL DEFAULT 'N' AFTER `Execute_priv`") + doReentrantDDL(s, "ALTER TABLE mysql.user CHANGE `Repl_client_priv` `Repl_client_priv` ENUM('N','Y') NOT NULL DEFAULT 'N' AFTER `Repl_slave_priv`") +} + +func upgradeToVer89(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateAdvisoryLocksTable) +} + +// importConfigOption is a one-time import. +// It is intended to be used to convert a config option to a sysvar. +// It reads the config value from the tidb-server executing the bootstrap +// (not guaranteed to be the same on all servers), and writes a message +// to the error log. The message is important since the behavior is weird +// (changes to the config file will no longer take effect past this point). +func importConfigOption(s sessionapi.Session, configName, svName, valStr string) { + message := fmt.Sprintf("%s is now configured by the system variable %s. One-time importing the value specified in tidb.toml file", configName, svName) + logutil.BgLogger().Warn(message, zap.String("value", valStr)) + // We use insert ignore, since if its a duplicate we don't want to overwrite any user-set values. + sql := fmt.Sprintf("INSERT IGNORE INTO %s.%s (`VARIABLE_NAME`, `VARIABLE_VALUE`) VALUES ('%s', '%s')", + mysql.SystemDB, mysql.GlobalVariablesTable, svName, valStr) + mustExecute(s, sql) +} + +func upgradeToVer90(s sessionapi.Session, _ int64) { + valStr := variable.BoolToOnOff(config.GetGlobalConfig().EnableBatchDML) + importConfigOption(s, "enable-batch-dml", vardef.TiDBEnableBatchDML, valStr) + valStr = fmt.Sprint(config.GetGlobalConfig().MemQuotaQuery) + importConfigOption(s, "mem-quota-query", vardef.TiDBMemQuotaQuery, valStr) + valStr = fmt.Sprint(config.GetGlobalConfig().Log.QueryLogMaxLen) + importConfigOption(s, "query-log-max-len", vardef.TiDBQueryLogMaxLen, valStr) + valStr = fmt.Sprint(config.GetGlobalConfig().Performance.CommitterConcurrency) + importConfigOption(s, "committer-concurrency", vardef.TiDBCommitterConcurrency, valStr) + valStr = variable.BoolToOnOff(config.GetGlobalConfig().Performance.RunAutoAnalyze) + importConfigOption(s, "run-auto-analyze", vardef.TiDBEnableAutoAnalyze, valStr) + valStr = config.GetGlobalConfig().OOMAction + importConfigOption(s, "oom-action", vardef.TiDBMemOOMAction, valStr) +} + +func upgradeToVer91(s sessionapi.Session, _ int64) { + valStr := variable.BoolToOnOff(config.GetGlobalConfig().PreparedPlanCache.Enabled) + importConfigOption(s, "prepared-plan-cache.enable", vardef.TiDBEnablePrepPlanCache, valStr) + + valStr = strconv.Itoa(int(config.GetGlobalConfig().PreparedPlanCache.Capacity)) + importConfigOption(s, "prepared-plan-cache.capacity", vardef.TiDBPrepPlanCacheSize, valStr) + + valStr = strconv.FormatFloat(config.GetGlobalConfig().PreparedPlanCache.MemoryGuardRatio, 'f', -1, 64) + importConfigOption(s, "prepared-plan-cache.memory-guard-ratio", vardef.TiDBPrepPlanCacheMemoryGuardRatio, valStr) +} + +func upgradeToVer93(s sessionapi.Session, _ int64) { + valStr := variable.BoolToOnOff(config.GetGlobalConfig().OOMUseTmpStorage) + importConfigOption(s, "oom-use-tmp-storage", vardef.TiDBEnableTmpStorageOnOOM, valStr) +} + +func upgradeToVer94(s sessionapi.Session, _ int64) { + mustExecute(s, CreateTiDBMDLView) +} + +func upgradeToVer95(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN IF NOT EXISTS `User_attributes` JSON") +} + +func upgradeToVer97(s sessionapi.Session, _ int64) { + // Check if tidb_opt_range_max_size exists in mysql.GLOBAL_VARIABLES. + // If not, insert "tidb_opt_range_max_size | 0" since this is the old behavior before we introduce this variable. + initGlobalVariableIfNotExists(s, vardef.TiDBOptRangeMaxSize, 0) +} + +func upgradeToVer98(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN IF NOT EXISTS `Token_issuer` varchar(255)") +} + +func upgradeToVer99Before(s sessionapi.Session) { + mustExecute(s, "INSERT HIGH_PRIORITY IGNORE INTO %n.%n VALUES (%?, %?);", + mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBEnableMDL, 0) +} + +func upgradeToVer99After(s sessionapi.Session) { + sql := fmt.Sprintf("UPDATE HIGH_PRIORITY %[1]s.%[2]s SET VARIABLE_VALUE = %[4]d WHERE VARIABLE_NAME = '%[3]s'", + mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBEnableMDL, 1) + mustExecute(s, sql) + err := kv.RunInNewTxn(kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL), s.GetStore(), true, func(_ context.Context, txn kv.Transaction) error { + t := meta.NewMutator(txn) + return t.SetMetadataLock(true) + }) + terror.MustNil(err) +} + +func upgradeToVer100(s sessionapi.Session, _ int64) { + valStr := strconv.Itoa(int(config.GetGlobalConfig().Performance.ServerMemoryQuota)) + importConfigOption(s, "performance.server-memory-quota", vardef.TiDBServerMemoryLimit, valStr) +} + +func upgradeToVer101(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreatePlanReplayerStatusTable) +} + +func upgradeToVer102(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreatePlanReplayerTaskTable) +} + +func upgradeToVer103(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateStatsTableLockedTable) +} + +func upgradeToVer104(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD COLUMN IF NOT EXISTS `sql_digest` varchar(64)") + doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD COLUMN IF NOT EXISTS `plan_digest` varchar(64)") +} + +// For users that upgrade TiDB from a pre-6.0 version, we want to disable tidb cost model2 by default to keep plans unchanged. +func upgradeToVer105(s sessionapi.Session, _ int64) { + initGlobalVariableIfNotExists(s, vardef.TiDBCostModelVersion, "1") +} + +func upgradeToVer106(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreatePasswordHistoryTable) + doReentrantDDL(s, "Alter table mysql.user add COLUMN IF NOT EXISTS `Password_reuse_history` smallint unsigned DEFAULT NULL AFTER `Create_Tablespace_Priv` ") + doReentrantDDL(s, "Alter table mysql.user add COLUMN IF NOT EXISTS `Password_reuse_time` smallint unsigned DEFAULT NULL AFTER `Password_reuse_history`") +} + +func upgradeToVer107(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN IF NOT EXISTS `Password_expired` ENUM('N','Y') NOT NULL DEFAULT 'N'") + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN IF NOT EXISTS `Password_last_changed` TIMESTAMP DEFAULT CURRENT_TIMESTAMP()") + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN IF NOT EXISTS `Password_lifetime` SMALLINT UNSIGNED DEFAULT NULL") +} + +func upgradeToVer108(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateTiDBTTLTableStatusTable) +} + +// For users that upgrade TiDB from a 6.2-6.4 version, we want to disable tidb gc_aware_memory_track by default. +func upgradeToVer109(s sessionapi.Session, _ int64) { + mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", + mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBEnableGCAwareMemoryTrack, 0) +} + +// For users that upgrade TiDB from a 5.4-6.4 version, we want to enable tidb tidb_stats_load_pseudo_timeout by default. +func upgradeToVer110(s sessionapi.Session, _ int64) { + mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", + mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBStatsLoadPseudoTimeout, 1) +} + +func upgradeToVer130(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.stats_meta_history ADD COLUMN IF NOT EXISTS `source` varchar(40) NOT NULL after `version`;") +} + +func upgradeToVer131(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateTiDBTTLTaskTable) + doReentrantDDL(s, CreateTiDBTTLJobHistoryTable) +} + +func upgradeToVer132(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateTiDBMDLView) +} + +func upgradeToVer133(s sessionapi.Session, _ int64) { + mustExecute(s, "UPDATE HIGH_PRIORITY %n.%n set VARIABLE_VALUE = %? where VARIABLE_NAME = %? and VARIABLE_VALUE = %?;", + mysql.SystemDB, mysql.GlobalVariablesTable, vardef.DefTiDBServerMemoryLimit, vardef.TiDBServerMemoryLimit, "0") +} + +func upgradeToVer134(s sessionapi.Session, _ int64) { + mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", mysql.SystemDB, mysql.GlobalVariablesTable, vardef.ForeignKeyChecks, vardef.On) + mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBEnableForeignKey, vardef.On) + mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBEnableHistoricalStats, vardef.On) + mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBEnablePlanReplayerCapture, vardef.On) + mustExecute(s, "UPDATE HIGH_PRIORITY %n.%n SET VARIABLE_VALUE = %? WHERE VARIABLE_NAME = %? AND VARIABLE_VALUE = %?;", mysql.SystemDB, mysql.GlobalVariablesTable, "4", vardef.TiDBStoreBatchSize, "0") +} + +// For users that upgrade TiDB from a pre-7.0 version, we want to set tidb_opt_advanced_join_hint to off by default to keep plans unchanged. +func upgradeToVer135(s sessionapi.Session, _ int64) { + initGlobalVariableIfNotExists(s, vardef.TiDBOptAdvancedJoinHint, false) +} + +func upgradeToVer136(s sessionapi.Session, _ int64) { + mustExecute(s, CreateTiDBGlobalTaskTable) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask DROP INDEX namespace", dbterror.ErrCantDropFieldOrKey) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD INDEX idx_task_key(task_key)", dbterror.ErrDupKeyName) +} + +func upgradeToVer137(_ sessionapi.Session, _ int64) { + // NOOP, we don't depend on ddl to init the default group due to backward compatible issue. +} + +// For users that upgrade TiDB from a version below 7.0, we want to enable tidb tidb_enable_null_aware_anti_join by default. +func upgradeToVer138(s sessionapi.Session, _ int64) { + mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBOptimizerEnableNAAJ, vardef.On) +} + +func upgradeToVer139(sessionapi.Session, int64) {} + +func upgradeToVer140(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN `task_key` VARCHAR(256) NOT NULL AFTER `id`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD UNIQUE KEY task_key(task_key)", dbterror.ErrDupKeyName) +} + +// upgradeToVer141 sets the value of `tidb_session_plan_cache_size` as `tidb_prepared_plan_cache_size` for compatibility, +// and update tidb_load_based_replica_read_threshold from 0 to 4. +func upgradeToVer141(s sessionapi.Session, _ int64) { + ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) + rs, err := s.ExecuteInternal(ctx, "SELECT VARIABLE_VALUE FROM %n.%n WHERE VARIABLE_NAME=%?;", + mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBPrepPlanCacheSize) + terror.MustNil(err) + req := rs.NewChunk(nil) + err = rs.Next(ctx, req) + if err != nil || req.NumRows() == 0 { + return + } + row := req.GetRow(0) + if row.IsNull(0) { + return + } + val := row.GetString(0) + + mustExecute(s, "INSERT HIGH_PRIORITY IGNORE INTO %n.%n VALUES (%?, %?);", + mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBSessionPlanCacheSize, val) + mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBLoadBasedReplicaReadThreshold, vardef.DefTiDBLoadBasedReplicaReadThreshold.String()) +} + +func upgradeToVer142(s sessionapi.Session, _ int64) { + initGlobalVariableIfNotExists(s, vardef.TiDBEnableNonPreparedPlanCache, vardef.Off) +} + +func upgradeToVer143(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN `error` BLOB", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD COLUMN `error` BLOB", infoschema.ErrColumnExists) +} + +func upgradeToVer144(s sessionapi.Session, _ int64) { + initGlobalVariableIfNotExists(s, vardef.TiDBPlanCacheInvalidationOnFreshStats, vardef.Off) +} + +func upgradeToVer146(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.stats_meta_history ADD INDEX idx_create_time (create_time)", dbterror.ErrDupKeyName) + doReentrantDDL(s, "ALTER TABLE mysql.stats_history ADD INDEX idx_create_time (create_time)", dbterror.ErrDupKeyName) +} + +func upgradeToVer167(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD COLUMN `step` INT AFTER `id`", infoschema.ErrColumnExists) +} + +func upgradeToVer168(s sessionapi.Session, _ int64) { + mustExecute(s, CreateTiDBImportJobsTable) +} + +func upgradeToVer169(s sessionapi.Session, _ int64) { + mustExecute(s, CreateTiDBRunawayQueriesTable) +} + +func upgradeToVer170(s sessionapi.Session, _ int64) { + mustExecute(s, CreateTiDBTimersTable) +} + +func upgradeToVer171(s sessionapi.Session, _ int64) { + mustExecute(s, "ALTER TABLE mysql.tidb_runaway_queries CHANGE COLUMN `tidb_server` `tidb_server` varchar(512)") +} + +func upgradeToVer172(s sessionapi.Session, _ int64) { + mustExecute(s, "DROP TABLE IF EXISTS mysql.tidb_runaway_quarantined_watch") + mustExecute(s, CreateTiDBRunawayWatchTable) + mustExecute(s, CreateTiDBRunawayWatchDoneTable) +} + +func upgradeToVer173(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD COLUMN `summary` JSON", infoschema.ErrColumnExists) +} + +func upgradeToVer174(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD COLUMN `step` INT AFTER `id`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD COLUMN `error` BLOB", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history DROP INDEX `namespace`", dbterror.ErrCantDropFieldOrKey) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD INDEX `idx_task_key`(`task_key`)", dbterror.ErrDupKeyName) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD INDEX `idx_state_update_time`(`state_update_time`)", dbterror.ErrDupKeyName) +} + +// upgradeToVer175 updates normalized bindings of `in (?)` to `in (...)` to solve +// the issue #44298 that bindings for `in (?)` can't work for `in (?, ?, ?)`. +// After this update, multiple bindings may have the same `original_sql`, but it's OK, and +// for safety, don't remove duplicated bindings when upgrading. +func upgradeToVer175(s sessionapi.Session, _ int64) { + var err error + mustExecute(s, "BEGIN PESSIMISTIC") + defer func() { + if err != nil { + mustExecute(s, "ROLLBACK") + return + } + mustExecute(s, "COMMIT") + }() + ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) + rs, err := s.ExecuteInternal(ctx, "SELECT original_sql, bind_sql FROM mysql.bind_info WHERE source != 'builtin'") + if err != nil { + logutil.BgLogger().Fatal("upgradeToVer175 error", zap.Error(err)) + return + } + req := rs.NewChunk(nil) + updateStmts := make([]string, 0, 4) + for { + err = rs.Next(ctx, req) + if err != nil { + logutil.BgLogger().Fatal("upgradeToVer175 error", zap.Error(err)) + return + } + if req.NumRows() == 0 { + break + } + for i := range req.NumRows() { + originalNormalizedSQL, bindSQL := req.GetRow(i).GetString(0), req.GetRow(i).GetString(1) + newNormalizedSQL := parser.NormalizeForBinding(bindSQL, false) + // update `in (?)` to `in (...)` + if originalNormalizedSQL == newNormalizedSQL { + continue // no need to update + } + // must run those update statements outside this loop, otherwise may cause some concurrency problems, + // since the current statement over this session has not been finished yet. + updateStmts = append(updateStmts, fmt.Sprintf("UPDATE mysql.bind_info SET original_sql='%s' WHERE original_sql='%s'", newNormalizedSQL, originalNormalizedSQL)) + } + req.Reset() + } + if err := rs.Close(); err != nil { + logutil.BgLogger().Fatal("upgradeToVer175 error", zap.Error(err)) + } + for _, updateStmt := range updateStmts { + mustExecute(s, updateStmt) + } +} + +func upgradeToVer176(s sessionapi.Session, _ int64) { + mustExecute(s, CreateTiDBGlobalTaskHistoryTable) +} + +func upgradeToVer177(s sessionapi.Session, _ int64) { + // ignore error when upgrading from v7.4 to higher version. + doReentrantDDL(s, CreateDistFrameworkMetaTable, infoschema.ErrTableExists) + err := s.GetSessionVars().GlobalVarsAccessor.SetGlobalSysVar(context.Background(), vardef.TiDBEnableAsyncMergeGlobalStats, vardef.Off) + if err != nil { + logutil.BgLogger().Fatal("upgradeToVer177 error", zap.Error(err)) + } +} + +// writeDDLTableVersion writes mDDLTableVersion into mysql.tidb +func writeDDLTableVersion(s sessionapi.Session) { + var err error + var ddlTableVersion meta.DDLTableVersion + err = kv.RunInNewTxn(kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap), s.GetStore(), true, func(_ context.Context, txn kv.Transaction) error { + t := meta.NewMutator(txn) + ddlTableVersion, err = t.GetDDLTableVersion() + return err + }) + terror.MustNil(err) + mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, "DDL Table Version. Do not delete.") ON DUPLICATE KEY UPDATE VARIABLE_VALUE= %?`, + mysql.SystemDB, + mysql.TiDBTable, + tidbDDLTableVersion, + ddlTableVersion, + ddlTableVersion, + ) +} + +func upgradeToVer178(s sessionapi.Session, _ int64) { + writeDDLTableVersion(s) +} + +func upgradeToVer179(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.global_variables MODIFY COLUMN `VARIABLE_VALUE` varchar(16383)") +} + +func upgradeToVer190(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN `priority` INT DEFAULT 1 AFTER `state`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN `create_time` TIMESTAMP AFTER `priority`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN `end_time` TIMESTAMP AFTER `state_update_time`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN `priority` INT DEFAULT 1 AFTER `state`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN `create_time` TIMESTAMP AFTER `priority`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN `end_time` TIMESTAMP AFTER `state_update_time`", infoschema.ErrColumnExists) + + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD COLUMN `concurrency` INT AFTER `checkpoint`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD COLUMN `create_time` TIMESTAMP AFTER `concurrency`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD COLUMN `end_time` TIMESTAMP AFTER `state_update_time`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD COLUMN `ordinal` int AFTER `meta`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD COLUMN `concurrency` INT AFTER `checkpoint`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD COLUMN `create_time` TIMESTAMP AFTER `concurrency`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD COLUMN `end_time` TIMESTAMP AFTER `state_update_time`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD COLUMN `ordinal` int AFTER `meta`", infoschema.ErrColumnExists) + + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD INDEX idx_exec_id(exec_id)", dbterror.ErrDupKeyName) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD UNIQUE INDEX uk_task_key_step_ordinal(task_key, step, ordinal)", dbterror.ErrDupKeyName) + + doReentrantDDL(s, "ALTER TABLE mysql.dist_framework_meta ADD COLUMN `cpu_count` INT DEFAULT 0 AFTER `role`", infoschema.ErrColumnExists) + + doReentrantDDL(s, "ALTER TABLE mysql.dist_framework_meta MODIFY COLUMN `host` VARCHAR(261)") + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask MODIFY COLUMN `exec_id` VARCHAR(261)") + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history MODIFY COLUMN `exec_id` VARCHAR(261)") + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task MODIFY COLUMN `dispatcher_id` VARCHAR(261)") + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history MODIFY COLUMN `dispatcher_id` VARCHAR(261)") +} + +func upgradeToVer191(s sessionapi.Session, _ int64) { + sql := fmt.Sprintf("INSERT HIGH_PRIORITY IGNORE INTO %s.%s VALUES('%s', '%s')", + mysql.SystemDB, mysql.GlobalVariablesTable, + vardef.TiDBTxnMode, vardef.OptimisticTxnMode) + mustExecute(s, sql) +} + +func upgradeToVer192(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateRequestUnitByGroupTable) +} + +func upgradeToVer193(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateTiDBMDLView) +} + +func upgradeToVer194(s sessionapi.Session, _ int64) { + mustExecute(s, "DROP TABLE IF EXISTS mysql.load_data_jobs") +} + +func upgradeToVer195(s sessionapi.Session, _ int64) { + doReentrantDDL(s, DropMySQLIndexUsageTable) +} + +func upgradeToVer196(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN target_scope VARCHAR(256) DEFAULT '' AFTER `step`;", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN target_scope VARCHAR(256) DEFAULT '' AFTER `step`;", infoschema.ErrColumnExists) +} + +func upgradeToVer197(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateTiDBMDLView) +} + +func upgradeToVer198(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.tidb_mdl_info ADD COLUMN owner_id VARCHAR(64) NOT NULL DEFAULT '';", infoschema.ErrColumnExists) +} + +func upgradeToVer209(s sessionapi.Session, _ int64) { + initGlobalVariableIfNotExists(s, vardef.TiDBResourceControlStrictMode, vardef.Off) +} + +func upgradeToVer210(s sessionapi.Session, _ int64) { + // Check if tidb_analyze_column_options exists in mysql.GLOBAL_VARIABLES. + // If not, set tidb_analyze_column_options to ALL since this is the old behavior before we introduce this variable. + initGlobalVariableIfNotExists(s, vardef.TiDBAnalyzeColumnOptions, ast.AllColumns.String()) + + // Check if tidb_opt_projection_push_down exists in mysql.GLOBAL_VARIABLES. + // If not, set tidb_opt_projection_push_down to Off since this is the old behavior before we introduce this variable. + initGlobalVariableIfNotExists(s, vardef.TiDBOptProjectionPushDown, vardef.Off) +} + +func upgradeToVer211(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD COLUMN `summary` JSON", infoschema.ErrColumnExists) +} + +func upgradeToVer212(s sessionapi.Session, ver int64) { + // need to ensure curVersion has the column before rename. + // version169 created `tidb_runaway_queries` table + // version172 created `tidb_runaway_watch` and `tidb_runaway_watch_done` tables + if ver < version172 { + return + } + // version212 changed a lots of runaway related table. + // 1. switchGroup: add column `switch_group_name` to `mysql.tidb_runaway_watch` and `mysql.tidb_runaway_watch_done`. + doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_watch ADD COLUMN `switch_group_name` VARCHAR(32) DEFAULT '' AFTER `action`;", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_watch_done ADD COLUMN `switch_group_name` VARCHAR(32) DEFAULT '' AFTER `action`;", infoschema.ErrColumnExists) + // 2. modify column `plan_digest` type, modify column `time` to `start_time, + // modify column `original_sql` to `sample_sql` and unique union key to `mysql.tidb_runaway_queries`. + // add column `sql_digest`. + doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries ADD COLUMN `sql_digest` varchar(64) DEFAULT '' AFTER `original_sql`;", infoschema.ErrColumnExists) + // add column `repeats`. + doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries ADD COLUMN `repeats` int DEFAULT 1 AFTER `time`;", infoschema.ErrColumnExists) + // rename column name from `time` to `start_time`, will auto rebuild the index. + doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries RENAME COLUMN `time` TO `start_time`", infoschema.ErrColumnNotExists) + // rename column `original_sql` to `sample_sql`. + doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries RENAME COLUMN `original_sql` TO `sample_sql`", infoschema.ErrColumnNotExists) + // modify column type of `plan_digest`. + doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries MODIFY COLUMN `plan_digest` varchar(64) DEFAULT '';", infoschema.ErrColumnExists) + // 3. modify column length of `action`. + doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries MODIFY COLUMN `action` VARCHAR(64) NOT NULL;", infoschema.ErrColumnExists) + // 4. add column `rule` to `mysql.tidb_runaway_watch`, `mysql.tidb_runaway_watch_done` and `mysql.tidb_runaway_queries`. + doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_watch ADD COLUMN `rule` VARCHAR(512) DEFAULT '' AFTER `switch_group_name`;", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_watch_done ADD COLUMN `rule` VARCHAR(512) DEFAULT '' AFTER `switch_group_name`;", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries ADD COLUMN `rule` VARCHAR(512) DEFAULT '' AFTER `tidb_server`;", infoschema.ErrColumnExists) +} + +func upgradeToVer213(s sessionapi.Session, _ int64) { + mustExecute(s, CreateTiDBPITRIDMapTable) +} + +func upgradeToVer214(s sessionapi.Session, _ int64) { + mustExecute(s, CreateIndexAdvisorResultsTable) + mustExecute(s, CreateTiDBKernelOptionsTable) +} + +func upgradeToVer215(s sessionapi.Session, _ int64) { + initGlobalVariableIfNotExists(s, vardef.TiDBEnableINLJoinInnerMultiPattern, vardef.Off) +} + +func upgradeToVer216(s sessionapi.Session, _ int64) { + mustExecute(s, "UPDATE mysql.global_variables SET VARIABLE_VALUE='' WHERE VARIABLE_NAME = 'tidb_scatter_region' AND VARIABLE_VALUE = 'OFF'") + mustExecute(s, "UPDATE mysql.global_variables SET VARIABLE_VALUE='table' WHERE VARIABLE_NAME = 'tidb_scatter_region' AND VARIABLE_VALUE = 'ON'") +} + +func upgradeToVer217(s sessionapi.Session, _ int64) { + // If tidb_schema_cache_size does not exist, insert a record and set the value to 0 + // Otherwise do nothing. + mustExecute(s, "INSERT IGNORE INTO mysql.global_variables VALUES ('tidb_schema_cache_size', 0)") +} + +func upgradeToVer218(_ sessionapi.Session, _ int64) { + // empty, just make lint happy. +} + +func upgradeToVer239(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN modify_params json AFTER `error`;", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN modify_params json AFTER `error`;", infoschema.ErrColumnExists) +} + +const ( + // addAnalyzeJobsSchemaTableStateIndex is a DDL statement that adds an index on (table_schema, table_name, state) + // columns to mysql.analyze_jobs table. This index is currently unused since queries filter on partition_name='', + // even for non-partitioned tables. It is kept for potential future optimization where queries could use this + // simpler index directly for non-partitioned tables. + addAnalyzeJobsSchemaTableStateIndex = "ALTER TABLE mysql.analyze_jobs ADD INDEX idx_schema_table_state (table_schema, table_name, state)" + // addAnalyzeJobsSchemaTablePartitionStateIndex adds an index on (table_schema, table_name, partition_name, state) to mysql.analyze_jobs + addAnalyzeJobsSchemaTablePartitionStateIndex = "ALTER TABLE mysql.analyze_jobs ADD INDEX idx_schema_table_partition_state (table_schema, table_name, partition_name, state)" +) + +func upgradeToVer240(s sessionapi.Session, _ int64) { + doReentrantDDL(s, addAnalyzeJobsSchemaTableStateIndex, dbterror.ErrDupKeyName) + doReentrantDDL(s, addAnalyzeJobsSchemaTablePartitionStateIndex, dbterror.ErrDupKeyName) +} + +func upgradeToVer241(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user ADD INDEX i_user (user)", dbterror.ErrDupKeyName) + doReentrantDDL(s, "ALTER TABLE mysql.global_priv ADD INDEX i_user (user)", dbterror.ErrDupKeyName) + doReentrantDDL(s, "ALTER TABLE mysql.db ADD INDEX i_user (user)", dbterror.ErrDupKeyName) + doReentrantDDL(s, "ALTER TABLE mysql.tables_priv ADD INDEX i_user (user)", dbterror.ErrDupKeyName) + doReentrantDDL(s, "ALTER TABLE mysql.columns_priv ADD INDEX i_user (user)", dbterror.ErrDupKeyName) + doReentrantDDL(s, "ALTER TABLE mysql.global_grants ADD INDEX i_user (user)", dbterror.ErrDupKeyName) + doReentrantDDL(s, "ALTER TABLE mysql.default_roles ADD INDEX i_user (user)", dbterror.ErrDupKeyName) +} + +// writeClusterID writes cluster id into mysql.tidb +func writeClusterID(s sessionapi.Session) { + clusterID := s.GetStore().GetClusterID() + + mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, "TiDB Cluster ID.") ON DUPLICATE KEY UPDATE VARIABLE_VALUE= %?`, + mysql.SystemDB, + mysql.TiDBTable, + tidbClusterID, + clusterID, + clusterID, + ) +} + +func upgradeToVer242(s sessionapi.Session, _ int64) { + writeClusterID(s) + mustExecute(s, CreateTiDBWorkloadValuesTable) +} + +func upgradeToVer243(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN max_node_count INT DEFAULT 0 AFTER `modify_params`;", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN max_node_count INT DEFAULT 0 AFTER `modify_params`;", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN extra_params json AFTER max_node_count;", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN extra_params json AFTER max_node_count;", infoschema.ErrColumnExists) +} + +func upgradeToVer244(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN IF NOT EXISTS `Max_user_connections` INT UNSIGNED NOT NULL DEFAULT 0 AFTER `Password_lifetime`") +} + +func upgradeToVer245(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.bind_info MODIFY COLUMN original_sql LONGTEXT NOT NULL") + doReentrantDDL(s, "ALTER TABLE mysql.bind_info MODIFY COLUMN bind_sql LONGTEXT NOT NULL") +} + +func upgradeToVer246(s sessionapi.Session, _ int64) { + // log duplicated digests that will be set to null. + ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) + rs, err := s.ExecuteInternal(ctx, + `select plan_digest, sql_digest from mysql.bind_info group by plan_digest, sql_digest having count(1) > 1`) + if err != nil { + logutil.BgLogger().Fatal("failed to get duplicated plan and sql digests", zap.Error(err)) + return + } + req := rs.NewChunk(nil) + duplicatedDigests := make(map[string]struct{}) + for { + err = rs.Next(ctx, req) + if err != nil { + logutil.BgLogger().Fatal("failed to get duplicated plan and sql digests", zap.Error(err)) + return + } + if req.NumRows() == 0 { + break + } + for i := range req.NumRows() { + planDigest, sqlDigest := req.GetRow(i).GetString(0), req.GetRow(i).GetString(1) + duplicatedDigests[sqlDigest+", "+planDigest] = struct{}{} + } + req.Reset() + } + if err := rs.Close(); err != nil { + logutil.BgLogger().Warn("failed to close record set", zap.Error(err)) + } + if len(duplicatedDigests) > 0 { + digestList := make([]string, 0, len(duplicatedDigests)) + for k := range duplicatedDigests { + digestList = append(digestList, "("+k+")") + } + logutil.BgLogger().Warn("set the following (plan digest, sql digest) in mysql.bind_info to null " + + "for adding new unique index: " + strings.Join(digestList, ", ")) + } + + // to avoid the failure of adding the unique index, remove duplicated rows on these 2 digest columns first. + // in most cases, there should be no duplicated rows, since now we only store one binding for each sql_digest. + // compared with upgrading failure, it's OK to set these 2 columns to null. + doReentrantDDL(s, `UPDATE mysql.bind_info SET plan_digest=null, sql_digest=null + WHERE (plan_digest, sql_digest) in ( + select plan_digest, sql_digest from mysql.bind_info + group by plan_digest, sql_digest having count(1) > 1)`) + doReentrantDDL(s, "ALTER TABLE mysql.bind_info MODIFY COLUMN sql_digest VARCHAR(64) DEFAULT NULL") + doReentrantDDL(s, "ALTER TABLE mysql.bind_info MODIFY COLUMN plan_digest VARCHAR(64) DEFAULT NULL") + doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD UNIQUE INDEX digest_index(plan_digest, sql_digest)", dbterror.ErrDupKeyName) +} + +func upgradeToVer247(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.stats_meta ADD COLUMN last_stats_histograms_version bigint unsigned DEFAULT NULL", infoschema.ErrColumnExists) +} + +func upgradeToVer248(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.tidb_pitr_id_map ADD COLUMN restore_id BIGINT NOT NULL DEFAULT 0", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_pitr_id_map DROP PRIMARY KEY") + doReentrantDDL(s, "ALTER TABLE mysql.tidb_pitr_id_map ADD PRIMARY KEY(restore_id, restored_ts, upstream_cluster_id, segment_id)") +} + +func upgradeToVer249(s sessionapi.Session, _ int64) { + doReentrantDDL(s, CreateTiDBRestoreRegistryTable) +} + +func upgradeToVer250(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN `keyspace` varchar(64) DEFAULT '' AFTER `extra_params`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD INDEX idx_keyspace(keyspace)", dbterror.ErrDupKeyName) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN `keyspace` varchar(64) DEFAULT '' AFTER `extra_params`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD INDEX idx_keyspace(keyspace)", dbterror.ErrDupKeyName) +} + +func upgradeToVer251(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.tidb_import_jobs ADD COLUMN `group_key` VARCHAR(256) NOT NULL DEFAULT '' AFTER `created_by`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_import_jobs ADD INDEX idx_group_key(group_key)", dbterror.ErrDupKeyName) +} + +func upgradeToVer252(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.bind_info CHANGE create_time create_time TIMESTAMP(6)") + doReentrantDDL(s, "ALTER TABLE mysql.bind_info CHANGE update_time update_time TIMESTAMP(6)") +} + +func upgradeToVer253(s sessionapi.Session, _ int64) { + doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD COLUMN last_used_date DATE DEFAULT NULL AFTER `plan_digest`", infoschema.ErrColumnExists) +} From 281ea2f56799d55a5fcef5c29f141f08d7b9af4a Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Fri, 10 Oct 2025 11:44:12 +0800 Subject: [PATCH 02/29] update Signed-off-by: Weizhen Wang --- .../snap_client/systable_restore_test.go | 313 ------------------ pkg/bindinfo/binding.go | 109 ------ pkg/bindinfo/binding_cache.go | 30 -- pkg/bindinfo/global_handle_test.go | 15 - pkg/bindinfo/tests/BUILD.bazel | 9 - pkg/bindinfo/tests/bind_test.go | 3 - pkg/session/bootstrap.go | 5 - 7 files changed, 484 deletions(-) diff --git a/br/pkg/restore/snap_client/systable_restore_test.go b/br/pkg/restore/snap_client/systable_restore_test.go index d619691880dfc..c0df26e1d2386 100644 --- a/br/pkg/restore/snap_client/systable_restore_test.go +++ b/br/pkg/restore/snap_client/systable_restore_test.go @@ -116,318 +116,5 @@ func TestCheckSysTableCompatibility(t *testing.T) { // // The above variables are in the file br/pkg/restore/systable_restore.go func TestMonitorTheSystemTableIncremental(t *testing.T) { -<<<<<<< HEAD require.Equal(t, int64(220), session.CurrentBootstrapVersion) -======= - require.Equal(t, int64(253), session.CurrentBootstrapVersion) -} - -func TestIsStatsTemporaryTable(t *testing.T) { - require.False(t, snapclient.IsStatsTemporaryTable("", "")) - require.False(t, snapclient.IsStatsTemporaryTable("", "stats_meta")) - require.False(t, snapclient.IsStatsTemporaryTable("mysql", "stats_meta")) - require.False(t, snapclient.IsStatsTemporaryTable("__TiDB_BR_Temporary_test", "stats_meta")) - require.True(t, snapclient.IsStatsTemporaryTable("__TiDB_BR_Temporary_mysql", "stats_meta")) - require.False(t, snapclient.IsStatsTemporaryTable("__TiDB_BR_Temporary_mysql", "test")) -} - -func TestGetDBNameIfStatsTemporaryTable(t *testing.T) { - _, ok := snapclient.GetDBNameIfStatsTemporaryTable("", "") - require.False(t, ok) - _, ok = snapclient.GetDBNameIfStatsTemporaryTable("", "stats_meta") - require.False(t, ok) - _, ok = snapclient.GetDBNameIfStatsTemporaryTable("mysql", "stats_meta") - require.False(t, ok) - _, ok = snapclient.GetDBNameIfStatsTemporaryTable("__TiDB_BR_Temporary_test", "stats_meta") - require.False(t, ok) - name, ok := snapclient.GetDBNameIfStatsTemporaryTable("__TiDB_BR_Temporary_mysql", "stats_meta") - require.True(t, ok) - require.Equal(t, "mysql", name) - _, ok = snapclient.GetDBNameIfStatsTemporaryTable("__TiDB_BR_Temporary_mysql", "test") - require.False(t, ok) -} - -func TestTemporaryTableCheckerForStatsTemporaryTable(t *testing.T) { - checker := snapclient.NewTemporaryTableChecker(true, false) - _, ok := checker.CheckTemporaryTables("", "") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("", "stats_meta") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("mysql", "stats_meta") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_test", "stats_meta") - require.False(t, ok) - name, ok := checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "stats_meta") - require.True(t, ok) - require.Equal(t, "mysql", name) - _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "test") - require.False(t, ok) - - _, ok = checker.CheckTemporaryTables("", "user") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("mysql", "user") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_test", "user") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "user") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "test") - require.False(t, ok) -} - -func TestIsRenameableSysTemporaryTable(t *testing.T) { - require.False(t, snapclient.IsRenameableSysTemporaryTable("", "")) - require.False(t, snapclient.IsRenameableSysTemporaryTable("", "user")) - require.False(t, snapclient.IsRenameableSysTemporaryTable("mysql", "user")) - require.False(t, snapclient.IsRenameableSysTemporaryTable("__TiDB_BR_Temporary_test", "user")) - require.True(t, snapclient.IsRenameableSysTemporaryTable("__TiDB_BR_Temporary_mysql", "user")) - require.False(t, snapclient.IsRenameableSysTemporaryTable("__TiDB_BR_Temporary_mysql", "test")) -} - -func TestGetDBNameIfRenameableSysTemporaryTable(t *testing.T) { - _, ok := snapclient.GetDBNameIfRenameableSysTemporaryTable("", "") - require.False(t, ok) - _, ok = snapclient.GetDBNameIfRenameableSysTemporaryTable("", "user") - require.False(t, ok) - _, ok = snapclient.GetDBNameIfRenameableSysTemporaryTable("mysql", "user") - require.False(t, ok) - _, ok = snapclient.GetDBNameIfRenameableSysTemporaryTable("__TiDB_BR_Temporary_test", "user") - require.False(t, ok) - name, ok := snapclient.GetDBNameIfRenameableSysTemporaryTable("__TiDB_BR_Temporary_mysql", "user") - require.True(t, ok) - require.Equal(t, "mysql", name) - _, ok = snapclient.GetDBNameIfRenameableSysTemporaryTable("__TiDB_BR_Temporary_mysql", "test") - require.False(t, ok) -} - -func TestTemporaryTableCheckerForRenameableSysTemporaryTable(t *testing.T) { - checker := snapclient.NewTemporaryTableChecker(false, true) - _, ok := checker.CheckTemporaryTables("", "") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("", "user") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("mysql", "user") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_test", "user") - require.False(t, ok) - name, ok := checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "user") - require.True(t, ok) - require.Equal(t, "mysql", name) - _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "test") - require.False(t, ok) - - _, ok = checker.CheckTemporaryTables("", "stats_meta") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("mysql", "stats_meta") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_test", "stats_meta") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "stats_meta") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "test") - require.False(t, ok) -} - -func TestTemporaryTableChecker(t *testing.T) { - checker := snapclient.NewTemporaryTableChecker(true, true) - _, ok := checker.CheckTemporaryTables("", "") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("", "user") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("mysql", "user") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_test", "user") - require.False(t, ok) - name, ok := checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "user") - require.True(t, ok) - require.Equal(t, "mysql", name) - _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "test") - require.False(t, ok) - - _, ok = checker.CheckTemporaryTables("", "stats_meta") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("mysql", "stats_meta") - require.False(t, ok) - _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_test", "stats_meta") - require.False(t, ok) - name, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "stats_meta") - require.True(t, ok) - require.Equal(t, "mysql", name) - _, ok = checker.CheckTemporaryTables("__TiDB_BR_Temporary_mysql", "test") - require.False(t, ok) -} - -func TestGenerateMoveRenamedTableSQLPair(t *testing.T) { - renameSQL := snapclient.GenerateMoveRenamedTableSQLPair(123, map[string]map[string]struct{}{ - "mysql": {"stats_meta": struct{}{}, "stats_buckets": struct{}{}, "stats_top_n": struct{}{}}, - }) - require.Contains(t, renameSQL, "mysql.stats_meta TO __TiDB_BR_Temporary_mysql.stats_meta_deleted_123") - require.Contains(t, renameSQL, "__TiDB_BR_Temporary_mysql.stats_meta TO mysql.stats_meta") - require.Contains(t, renameSQL, "mysql.stats_buckets TO __TiDB_BR_Temporary_mysql.stats_buckets_deleted_123") - require.Contains(t, renameSQL, "__TiDB_BR_Temporary_mysql.stats_buckets TO mysql.stats_buckets") - require.Contains(t, renameSQL, "mysql.stats_top_n TO __TiDB_BR_Temporary_mysql.stats_top_n_deleted_123") - require.Contains(t, renameSQL, "__TiDB_BR_Temporary_mysql.stats_top_n TO mysql.stats_top_n") -} - -func TestUpdateStatsTableSchema(t *testing.T) { - ctx := context.Background() - expectedSQLs := []string{} - execution := func(_ context.Context, sql string) error { - require.Equal(t, expectedSQLs[0], sql) - expectedSQLs = expectedSQLs[1:] - return nil - } - - // schema name is mismatch - err := snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ - "test": {"stats_meta": struct{}{}}, - }, snapclient.SchemaVersionPairT{ - UpstreamVersionMajor: 7, - UpstreamVersionMinor: 5, - DownstreamVersionMajor: 8, - DownstreamVersionMinor: 5, - }, execution) - require.NoError(t, err) - err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ - "test": {"stats_meta": struct{}{}}, - }, snapclient.SchemaVersionPairT{ - UpstreamVersionMajor: 8, - UpstreamVersionMinor: 5, - DownstreamVersionMajor: 7, - DownstreamVersionMinor: 5, - }, execution) - require.NoError(t, err) - - // table name is mismatch - err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ - "mysql": {"stats_meta2": struct{}{}}, - }, snapclient.SchemaVersionPairT{ - UpstreamVersionMajor: 7, - UpstreamVersionMinor: 5, - DownstreamVersionMajor: 8, - DownstreamVersionMinor: 5, - }, execution) - require.NoError(t, err) - err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ - "mysql": {"stats_meta2": struct{}{}}, - }, snapclient.SchemaVersionPairT{ - UpstreamVersionMajor: 8, - UpstreamVersionMinor: 5, - DownstreamVersionMajor: 7, - DownstreamVersionMinor: 5, - }, execution) - require.NoError(t, err) - - // version range is mismatch - err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ - "mysql": {"stats_meta": struct{}{}}, - }, snapclient.SchemaVersionPairT{ - UpstreamVersionMajor: 7, - UpstreamVersionMinor: 5, - DownstreamVersionMajor: 8, - DownstreamVersionMinor: 1, - }, execution) - require.NoError(t, err) - err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ - "mysql": {"stats_meta": struct{}{}}, - }, snapclient.SchemaVersionPairT{ - UpstreamVersionMajor: 8, - UpstreamVersionMinor: 1, - DownstreamVersionMajor: 7, - DownstreamVersionMinor: 5, - }, execution) - require.NoError(t, err) - err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ - "mysql": {"stats_meta": struct{}{}}, - }, snapclient.SchemaVersionPairT{ - UpstreamVersionMajor: 9, - UpstreamVersionMinor: 1, - DownstreamVersionMajor: 9, - DownstreamVersionMinor: 5, - }, execution) - require.NoError(t, err) - err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ - "mysql": {"stats_meta": struct{}{}}, - }, snapclient.SchemaVersionPairT{ - UpstreamVersionMajor: 9, - UpstreamVersionMinor: 5, - DownstreamVersionMajor: 9, - DownstreamVersionMinor: 1, - }, execution) - require.NoError(t, err) - - // match - expectedSQLs = []string{ - "ALTER TABLE __TiDB_BR_Temporary_mysql.stats_meta ADD COLUMN IF NOT EXISTS last_stats_histograms_version bigint unsigned DEFAULT NULL", - "ALTER TABLE __TiDB_BR_Temporary_mysql.stats_meta DROP COLUMN IF EXISTS last_stats_histograms_version", - "ALTER TABLE __TiDB_BR_Temporary_mysql.stats_meta ADD COLUMN IF NOT EXISTS last_stats_histograms_version bigint unsigned DEFAULT NULL", - "ALTER TABLE __TiDB_BR_Temporary_mysql.stats_meta DROP COLUMN IF EXISTS last_stats_histograms_version", - } - err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ - "mysql": {"stats_meta": struct{}{}}, - }, snapclient.SchemaVersionPairT{ - UpstreamVersionMajor: 7, - UpstreamVersionMinor: 5, - DownstreamVersionMajor: 8, - DownstreamVersionMinor: 5, - }, execution) - require.NoError(t, err) - err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ - "mysql": {"stats_meta": struct{}{}}, - }, snapclient.SchemaVersionPairT{ - UpstreamVersionMajor: 8, - UpstreamVersionMinor: 5, - DownstreamVersionMajor: 8, - DownstreamVersionMinor: 1, - }, execution) - require.NoError(t, err) - err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ - "mysql": {"stats_meta": struct{}{}, "test": struct{}{}}, - "test": {"stats_meta": struct{}{}}, - }, snapclient.SchemaVersionPairT{ - UpstreamVersionMajor: 8, - UpstreamVersionMinor: 1, - DownstreamVersionMajor: 8, - DownstreamVersionMinor: 5, - }, execution) - require.NoError(t, err) - err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ - "mysql": {"stats_meta": struct{}{}, "test": struct{}{}}, - "test": {"stats_meta": struct{}{}}, - }, snapclient.SchemaVersionPairT{ - UpstreamVersionMajor: 8, - UpstreamVersionMinor: 5, - DownstreamVersionMajor: 7, - DownstreamVersionMinor: 5, - }, execution) - require.NoError(t, err) - err = snapclient.UpdateStatsTableSchema(ctx, map[string]map[string]struct{}{ - "mysql": {"stats_meta": struct{}{}, "test": struct{}{}}, - "test": {"stats_meta": struct{}{}}, - }, snapclient.SchemaVersionPairT{ - UpstreamVersionMajor: 8, - UpstreamVersionMinor: 5, - DownstreamVersionMajor: 8, - DownstreamVersionMinor: 5, - }, execution) - require.NoError(t, err) -} - -func TestNotifyUpdateAllUsersPrivilege(t *testing.T) { - notifier := func() error { - return errors.Errorf("test") - } - err := snapclient.NotifyUpdateAllUsersPrivilege(map[string]map[string]struct{}{ - "test": {"user": {}}, - }, notifier) - require.NoError(t, err) - err = snapclient.NotifyUpdateAllUsersPrivilege(map[string]map[string]struct{}{ - "mysql": {"use": {}, "test": {}}, - }, notifier) - require.NoError(t, err) - err = snapclient.NotifyUpdateAllUsersPrivilege(map[string]map[string]struct{}{ - "mysql": {"test": {}, "user": {}, "db": {}}, - }, notifier) - require.Error(t, err) ->>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) } diff --git a/pkg/bindinfo/binding.go b/pkg/bindinfo/binding.go index 159425b0ac69f..8f3586f676175 100644 --- a/pkg/bindinfo/binding.go +++ b/pkg/bindinfo/binding.go @@ -15,14 +15,6 @@ package bindinfo import ( -<<<<<<< HEAD -======= - "context" - "fmt" - "strings" - "sync" - "sync/atomic" ->>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) "time" "unsafe" @@ -82,10 +74,6 @@ type Binding struct { // TableNames records all schema and table names in this binding statement, which are used for fuzzy matching. TableNames []*ast.TableName `json:"-"` - - // UsageInfo is to track the usage information `last_used_time` of this binding - // and it will be updated when this binding is used. - UsageInfo bindingInfoUsageInfo } func (b *Binding) isSame(rb *Binding) bool { @@ -108,106 +96,9 @@ func (b *Binding) IsBindingAvailable() bool { return b.IsBindingEnabled() || b.Status == Disabled } -<<<<<<< HEAD // SinceUpdateTime returns the duration since last update time. Export for test. func (b *Binding) SinceUpdateTime() (time.Duration, error) { updateTime, err := b.UpdateTime.GoTime(time.Local) -======= -// UpdateLastUsedAt is to update binding usage info when this binding is used. -func (b *Binding) UpdateLastUsedAt() { - now := time.Now() - b.UsageInfo.LastUsedAt.Store(&now) -} - -// UpdateLastSavedAt is to update the last saved time -func (b *Binding) UpdateLastSavedAt(ts *time.Time) { - b.UsageInfo.LastSavedAt.Store(ts) -} - -type bindingInfoUsageInfo struct { - // LastUsedAt records the last time when this binding is used. - // It is nil if this binding has never been used. - // It is updated when this binding is used. - // It is used to update the `last_used_time` field in mysql.bind_info table. - LastUsedAt atomic.Pointer[time.Time] - // LastSavedAt records the last time when this binding is saved into storage. - LastSavedAt atomic.Pointer[time.Time] -} - -var ( - // GetBindingHandle is a function to get the global binding handle. - // It is mainly used to resolve cycle import issue. - GetBindingHandle func(sctx sessionctx.Context) BindingHandle -) - -// BindingMatchInfo records necessary information for cross-db binding matching. -// This is mainly for plan cache to avoid normalizing the same statement repeatedly. -type BindingMatchInfo struct { - NoDBDigest string - TableNames []*ast.TableName -} - -// MatchSQLBindingForPlanCache matches binding for plan cache. -func MatchSQLBindingForPlanCache(sctx sessionctx.Context, stmtNode ast.StmtNode, info *BindingMatchInfo) (bindingSQL string) { - binding, matched, _ := matchSQLBinding(sctx, stmtNode, info) - if matched { - bindingSQL = binding.BindSQL - } - return -} - -// MatchSQLBinding returns the matched binding for this statement. -func MatchSQLBinding(sctx sessionctx.Context, stmtNode ast.StmtNode) (binding *Binding, matched bool, scope string) { - return matchSQLBinding(sctx, stmtNode, nil) -} - -func matchSQLBinding(sctx sessionctx.Context, stmtNode ast.StmtNode, info *BindingMatchInfo) (binding *Binding, matched bool, scope string) { - useBinding := sctx.GetSessionVars().UsePlanBaselines - if !useBinding || stmtNode == nil { - return - } - // When the domain is initializing, the bind will be nil. - if sctx.Value(SessionBindInfoKeyType) == nil { - return - } - - // record the normalization result into info to avoid repeat normalization next time. - var noDBDigest string - var tableNames []*ast.TableName - if info == nil || info.TableNames == nil || info.NoDBDigest == "" { - _, noDBDigest = NormalizeStmtForBinding(stmtNode, "", true) - tableNames = CollectTableNames(stmtNode) - if info != nil { - info.NoDBDigest = noDBDigest - info.TableNames = tableNames - } - } else { - noDBDigest = info.NoDBDigest - tableNames = info.TableNames - } - - sessionHandle := sctx.Value(SessionBindInfoKeyType).(SessionBindingHandle) - if binding, matched := sessionHandle.MatchSessionBinding(sctx, noDBDigest, tableNames); matched { - return binding, matched, metrics.ScopeSession - } - globalHandle := GetBindingHandle(sctx) - if globalHandle == nil { - return - } - binding, matched = globalHandle.MatchingBinding(sctx, noDBDigest, tableNames) - if matched { - // After hitting the cache, update the usage time of the bind. - binding.UpdateLastUsedAt() - return binding, matched, metrics.ScopeGlobal - } - - return -} - -func noDBDigestFromBinding(binding *Binding) (string, error) { - p := parser.New() - stmt, err := p.ParseOneStmt(binding.BindSQL, binding.Charset, binding.Collation) ->>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) if err != nil { return 0, err } diff --git a/pkg/bindinfo/binding_cache.go b/pkg/bindinfo/binding_cache.go index 5346c1e1a6248..7e3211665daa5 100644 --- a/pkg/bindinfo/binding_cache.go +++ b/pkg/bindinfo/binding_cache.go @@ -58,18 +58,6 @@ type FuzzyBindingCache interface { Copy() (c FuzzyBindingCache, err error) BindingCache -<<<<<<< HEAD -======= - - // LoadFromStorageToCache loads global bindings from storage to the memory cache. - LoadFromStorageToCache(fullLoad bool) (err error) - - // UpdateBindingUsageInfoToStorage is to update the binding usage info into storage - UpdateBindingUsageInfoToStorage() error - - // LastUpdateTime returns the last update time. - LastUpdateTime() types.Time ->>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) } type fuzzyBindingCache struct { @@ -161,7 +149,6 @@ func (fbc *fuzzyBindingCache) getFromMemory(sctx sessionctx.Context, fuzzyDigest return matchedBinding, isMatched, missingSQLDigest } -<<<<<<< HEAD func (fbc *fuzzyBindingCache) loadFromStore(sctx sessionctx.Context, missingSQLDigest []string) { if intest.InTest && sctx.Value(LoadBindingNothing) != nil { return @@ -169,23 +156,6 @@ func (fbc *fuzzyBindingCache) loadFromStore(sctx sessionctx.Context, missingSQLD defer func(start time.Time) { sctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("loading binding from storage takes " + time.Since(start).String())) }(time.Now()) -======= -// UpdateBindingUsageInfoToStorage is to update the binding usage info into storage -func (u *bindingCacheUpdater) UpdateBindingUsageInfoToStorage() error { - defer func() { - if r := recover(); r != nil { - bindingLogger().Warn("panic when update usage info for binding", zap.Any("recover", r)) - } - }() - bindings := u.GetAllBindings() - return updateBindingUsageInfoToStorage(u.sPool, bindings) -} - -// LastUpdateTime returns the last update time. -func (u *bindingCacheUpdater) LastUpdateTime() types.Time { - return u.lastUpdateTime.Load().(types.Time) -} ->>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) for _, sqlDigest := range missingSQLDigest { start := time.Now() diff --git a/pkg/bindinfo/global_handle_test.go b/pkg/bindinfo/global_handle_test.go index 0c4fe08e88eea..edb7af4a7a5d7 100644 --- a/pkg/bindinfo/global_handle_test.go +++ b/pkg/bindinfo/global_handle_test.go @@ -94,13 +94,8 @@ func TestBindingLastUpdateTimeWithInvalidBind(t *testing.T) { updateTime0 := rows0[0][1] require.Equal(t, updateTime0, "0000-00-00 00:00:00") -<<<<<<< HEAD:pkg/bindinfo/global_handle_test.go tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t`', 'select * from `test` . `t` use index(`idx`)', 'test', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + bindinfo.Manual + "', '', '')") -======= - tk.MustExec("insert into mysql.bind_info (original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest) values('select * from `test` . `t`', 'invalid_binding', 'test', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + - bindinfo.SourceManual + "', '', '')") ->>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)):pkg/bindinfo/binding_operator_test.go tk.MustExec("use test") tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int)") @@ -267,16 +262,11 @@ func TestSetBindingStatusWithoutBindingInCache(t *testing.T) { // Simulate creating bindings on other machines _, sqlDigest := parser.NormalizeDigestForBinding("select * from `test` . `t` where `a` > ?") -<<<<<<< HEAD:pkg/bindinfo/global_handle_test.go tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t` where `a` > ?', 'SELECT /*+ USE_INDEX(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` > 10', 'test', 'deleted', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t` where `a` > ?', 'SELECT /*+ USE_INDEX(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` > 10', 'test', 'enabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") dom.BindHandle().Clear() -======= - tk.MustExec("insert into mysql.bind_info (original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest) values('select * from `test` . `t` where `a` > ?', 'SELECT /*+ USE_INDEX(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` > 10', 'test', 'enabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" + - bindinfo.SourceManual + "', '" + sqlDigest.String() + "', '')") ->>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)):pkg/bindinfo/binding_operator_test.go tk.MustExec("set binding disabled for select * from t where a > 10") tk.MustExec("admin reload bindings") rows := tk.MustQuery("show global bindings").Rows() @@ -287,16 +277,11 @@ func TestSetBindingStatusWithoutBindingInCache(t *testing.T) { internal.UtilCleanBindingEnv(tk, dom) // Simulate creating bindings on other machines -<<<<<<< HEAD:pkg/bindinfo/global_handle_test.go tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t` where `a` > ?', 'SELECT * FROM `test`.`t` WHERE `a` > 10', 'test', 'deleted', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t` where `a` > ?', 'SELECT * FROM `test`.`t` WHERE `a` > 10', 'test', 'disabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") dom.BindHandle().Clear() -======= - tk.MustExec("insert into mysql.bind_info (original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest) values('select * from `test` . `t` where `a` > ?', 'SELECT * FROM `test`.`t` WHERE `a` > 10', 'test', 'disabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" + - bindinfo.SourceManual + "', '" + sqlDigest.String() + "', '')") ->>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)):pkg/bindinfo/binding_operator_test.go tk.MustExec("set binding enabled for select * from t where a > 10") tk.MustExec("admin reload bindings") rows = tk.MustQuery("show global bindings").Rows() diff --git a/pkg/bindinfo/tests/BUILD.bazel b/pkg/bindinfo/tests/BUILD.bazel index c7a02ed42cd5c..c131284da79e7 100644 --- a/pkg/bindinfo/tests/BUILD.bazel +++ b/pkg/bindinfo/tests/BUILD.bazel @@ -5,20 +5,11 @@ go_test( timeout = "moderate", srcs = [ "bind_test.go", -<<<<<<< HEAD -======= - "bind_usage_info_test.go", - "cross_db_binding_test.go", ->>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) "main_test.go", ], flaky = True, race = "on", -<<<<<<< HEAD shard_count = 19, -======= - shard_count = 23, ->>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) deps = [ "//pkg/bindinfo", "//pkg/bindinfo/internal", diff --git a/pkg/bindinfo/tests/bind_test.go b/pkg/bindinfo/tests/bind_test.go index e99e50f23122d..780b8f17786bb 100644 --- a/pkg/bindinfo/tests/bind_test.go +++ b/pkg/bindinfo/tests/bind_test.go @@ -40,11 +40,8 @@ import ( func TestPrepareCacheWithBinding(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) -<<<<<<< HEAD tk.MustExec(`set tidb_enable_prepared_plan_cache=1`) tk.MustExec("set tidb_cost_model_version=2") -======= ->>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) tk.MustExec("use test") tk.MustExec("drop table if exists t1, t2") tk.MustExec("create table t1(a int, b int, c int, key idx_b(b), key idx_c(c))") diff --git a/pkg/session/bootstrap.go b/pkg/session/bootstrap.go index 87bc41c175459..c895e5915171d 100644 --- a/pkg/session/bootstrap.go +++ b/pkg/session/bootstrap.go @@ -297,14 +297,9 @@ const ( charset TEXT NOT NULL, collation TEXT NOT NULL, source VARCHAR(10) NOT NULL DEFAULT 'unknown', -<<<<<<< HEAD sql_digest varchar(64), plan_digest varchar(64), -======= - sql_digest varchar(64) DEFAULT NULL, - plan_digest varchar(64) DEFAULT NULL, last_used_date date DEFAULT NULL, ->>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) INDEX sql_index(original_sql(700),default_db(68)) COMMENT "accelerate the speed when add global binding query", INDEX time_index(update_time) COMMENT "accelerate the speed when querying with last update time" ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;` From b7df67bab8650d846785098a8a14033edb8686e4 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Fri, 10 Oct 2025 19:18:50 +0800 Subject: [PATCH 03/29] update Signed-off-by: Weizhen Wang --- .../{tests => }/bind_usage_info_test.go | 26 +- pkg/bindinfo/binding.go | 26 + pkg/bindinfo/binding_operator.go | 253 -- pkg/bindinfo/global_handle.go | 18 +- pkg/bindinfo/util.go | 129 ++ pkg/bindinfo/utils.go | 347 --- pkg/domain/domain.go | 2 +- pkg/session/bootstrap.go | 14 +- pkg/session/bootstrap_test.go | 4 - pkg/session/upgrade.go | 2039 ----------------- 10 files changed, 190 insertions(+), 2668 deletions(-) rename pkg/bindinfo/{tests => }/bind_usage_info_test.go (85%) delete mode 100644 pkg/bindinfo/binding_operator.go delete mode 100644 pkg/bindinfo/utils.go delete mode 100644 pkg/session/upgrade.go diff --git a/pkg/bindinfo/tests/bind_usage_info_test.go b/pkg/bindinfo/bind_usage_info_test.go similarity index 85% rename from pkg/bindinfo/tests/bind_usage_info_test.go rename to pkg/bindinfo/bind_usage_info_test.go index d1c54e8e62bd9..841612c727719 100644 --- a/pkg/bindinfo/tests/bind_usage_info_test.go +++ b/pkg/bindinfo/bind_usage_info_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package tests +package bindinfo_test import ( "fmt" @@ -25,18 +25,12 @@ import ( ) func TestBindUsageInfo(t *testing.T) { - checklist := []string{ - "5ce1df6eadf8b24222668b1bd2e995b72d4c88e6fe9340d8b13e834703e28c32", - "5d3975ef2160c1e0517353798dac90a9914095d82c025e7cd97bd55aeb804798", - "9d3995845aef70ba086d347f38a4e14c9705e966f7c5793b9fa92194bca2bbef", - "aa3c510b94b9d680f729252ca88415794c8a4f52172c5f9e06c27bee57d08329", - } bindinfo.UpdateBindingUsageInfoBatchSize = 2 bindinfo.MaxWriteInterval = 100 * time.Microsecond - store, dom := testkit.CreateMockStoreAndDomain(t) - bindingHandle := dom.BindingHandle() - tk := testkit.NewTestKit(t, store) + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + bindingHandle := bindinfo.NewGlobalBindingHandle(&mockSessionPool{tk.Session()}) tk.MustExec(`use test`) tk.MustExec(`set @@tidb_opt_enable_fuzzy_binding=1`) tk.MustExec("create table t1(a int, b int, c int, key idx_b(b), key idx_c(c))") @@ -78,7 +72,6 @@ func TestBindUsageInfo(t *testing.T) { // Set all last_used_date to null to simulate that the bindinfo in storage is not updated. resetAllLastUsedData(tk) require.NoError(t, bindingHandle.UpdateBindingUsageInfoToStorage()) - checkBindinfoInMemory(t, bindingHandle, checklist) tk.MustQuery(fmt.Sprintf(`select last_used_date from mysql.bind_info where original_sql != '%s' and last_used_date is null`, bindinfo.BuiltinPseudoSQL4BindLock)).Check(testkit.Rows()) result := tk.MustQuery(fmt.Sprintf(`select sql_digest,last_used_date from mysql.bind_info where original_sql != '%s' order by sql_digest`, bindinfo.BuiltinPseudoSQL4BindLock)) t.Log("result:", result.Rows()) @@ -111,14 +104,3 @@ func TestBindUsageInfo(t *testing.T) { func resetAllLastUsedData(tk *testkit.TestKit) { tk.MustExec(fmt.Sprintf(`update mysql.bind_info set last_used_date = null where original_sql != '%s'`, bindinfo.BuiltinPseudoSQL4BindLock)) } - -func checkBindinfoInMemory(t *testing.T, bindingHandle bindinfo.BindingHandle, checklist []string) { - for _, digest := range checklist { - binding := bindingHandle.GetBinding(digest) - require.NotNil(t, binding) - lastSaved := binding.UsageInfo.LastSavedAt.Load() - if lastSaved != nil { - require.GreaterOrEqual(t, *binding.UsageInfo.LastSavedAt.Load(), *binding.UsageInfo.LastUsedAt.Load()) - } - } -} diff --git a/pkg/bindinfo/binding.go b/pkg/bindinfo/binding.go index 8f3586f676175..862226ea9d4d8 100644 --- a/pkg/bindinfo/binding.go +++ b/pkg/bindinfo/binding.go @@ -15,6 +15,7 @@ package bindinfo import ( + "sync/atomic" "time" "unsafe" @@ -74,6 +75,10 @@ type Binding struct { // TableNames records all schema and table names in this binding statement, which are used for fuzzy matching. TableNames []*ast.TableName `json:"-"` + + // UsageInfo is to track the usage information `last_used_time` of this binding + // and it will be updated when this binding is used. + UsageInfo bindingInfoUsageInfo } func (b *Binding) isSame(rb *Binding) bool { @@ -235,3 +240,24 @@ func (b *Binding) size() float64 { res := len(b.OriginalSQL) + len(b.Db) + len(b.BindSQL) + len(b.Status) + 2*int(unsafe.Sizeof(b.CreateTime)) + len(b.Charset) + len(b.Collation) + len(b.ID) return float64(res) } + +// UpdateLastUsedAt is to update binding usage info when this binding is used. +func (b *Binding) UpdateLastUsedAt() { + now := time.Now() + b.UsageInfo.LastUsedAt.Store(&now) +} + +// UpdateLastSavedAt is to update the last saved time +func (b *Binding) UpdateLastSavedAt(ts *time.Time) { + b.UsageInfo.LastSavedAt.Store(ts) +} + +type bindingInfoUsageInfo struct { + // LastUsedAt records the last time when this binding is used. + // It is nil if this binding has never been used. + // It is updated when this binding is used. + // It is used to update the `last_used_time` field in mysql.bind_info table. + LastUsedAt atomic.Pointer[time.Time] + // LastSavedAt records the last time when this binding is saved into storage. + LastSavedAt atomic.Pointer[time.Time] +} diff --git a/pkg/bindinfo/binding_operator.go b/pkg/bindinfo/binding_operator.go deleted file mode 100644 index 8e4d5c00d6fed..0000000000000 --- a/pkg/bindinfo/binding_operator.go +++ /dev/null @@ -1,253 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bindinfo - -import ( - "strings" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/tidb/pkg/parser/mysql" - "github.com/pingcap/tidb/pkg/sessionctx" - "github.com/pingcap/tidb/pkg/types" - "github.com/pingcap/tidb/pkg/util" -) - -// BindingOperator is used to operate (create/drop/update/GC) bindings. -type BindingOperator interface { - // CreateBinding creates a Bindings to the storage and the cache. - // It replaces all the exists bindings for the same normalized SQL. - CreateBinding(sctx sessionctx.Context, bindings []*Binding) (err error) - - // DropBinding drop Bindings to the storage and Bindings int the cache. - DropBinding(sqlDigests []string) (deletedRows uint64, err error) - - // SetBindingStatus set a Bindings's status to the storage and bind cache. - SetBindingStatus(newStatus, sqlDigest string) (ok bool, err error) - - // GCBinding physically removes the deleted bind records in mysql.bind_info. - GCBinding() (err error) -} - -type bindingOperator struct { - cache BindingCacheUpdater - sPool util.DestroyableSessionPool -} - -func newBindingOperator(sPool util.DestroyableSessionPool, cache BindingCacheUpdater) BindingOperator { - return &bindingOperator{ - sPool: sPool, - cache: cache, - } -} - -// CreateBinding creates a Bindings to the storage and the cache. -// It replaces all the exists bindings for the same normalized SQL. -func (op *bindingOperator) CreateBinding(sctx sessionctx.Context, bindings []*Binding) (err error) { - for _, binding := range bindings { - if err := prepareHints(sctx, binding); err != nil { - return err - } - } - defer func() { - if err == nil { - err = op.cache.LoadFromStorageToCache(false) - } - }() - - return callWithSCtx(op.sPool, true, func(sctx sessionctx.Context) error { - // Lock mysql.bind_info to synchronize with CreateBinding / AddBinding / DropBinding on other tidb instances. - if err = lockBindInfoTable(sctx); err != nil { - return err - } - - for i, binding := range bindings { - now := types.NewTime(types.FromGoTime(time.Now()), mysql.TypeTimestamp, 6) - - updateTs := now.String() - _, err = exec( - sctx, - `UPDATE mysql.bind_info SET status = %?, update_time = %? WHERE original_sql = %? AND update_time < %?`, - StatusDeleted, - updateTs, - binding.OriginalSQL, - updateTs, - ) - if err != nil { - return err - } - - binding.CreateTime = now - binding.UpdateTime = now - - // TODO: update the sql_mode or sctx.types.Flag to let execution engine returns errors like dataTooLong, - // overflow directly. - - // Insert the Bindings to the storage. - var sqlDigest, planDigest any // null by default - if binding.SQLDigest != "" { - sqlDigest = binding.SQLDigest - } - if binding.PlanDigest != "" { - planDigest = binding.PlanDigest - } - _, err = exec( - sctx, - `INSERT INTO mysql.bind_info( - original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest -) VALUES (%?,%?, %?, %?, %?, %?, %?, %?, %?, %?, %?)`, - binding.OriginalSQL, - binding.BindSQL, - strings.ToLower(binding.Db), - binding.Status, - binding.CreateTime.String(), - binding.UpdateTime.String(), - binding.Charset, - binding.Collation, - binding.Source, - sqlDigest, - planDigest, - ) - failpoint.Inject("CreateGlobalBindingNthFail", func(val failpoint.Value) { - n := val.(int) - if n == i { - err = errors.NewNoStackError("An injected error") - } - }) - if err != nil { - return err - } - - warnings, _, err := execRows(sctx, "show warnings") - if err != nil { - return err - } - if len(warnings) != 0 { - return errors.New(warnings[0].GetString(2)) - } - } - return nil - }) -} - -// DropBinding drop Bindings to the storage and Bindings int the cache. -func (op *bindingOperator) DropBinding(sqlDigests []string) (deletedRows uint64, err error) { - if len(sqlDigests) == 0 { - return 0, errors.New("sql digest is empty") - } - defer func() { - if err == nil { - err = op.cache.LoadFromStorageToCache(false) - } - }() - - err = callWithSCtx(op.sPool, true, func(sctx sessionctx.Context) error { - // Lock mysql.bind_info to synchronize with CreateBinding / AddBinding / DropBinding on other tidb instances. - if err = lockBindInfoTable(sctx); err != nil { - return err - } - - for _, sqlDigest := range sqlDigests { - updateTs := types.NewTime(types.FromGoTime(time.Now()), mysql.TypeTimestamp, 6).String() - _, err = exec( - sctx, - `UPDATE mysql.bind_info SET status = %?, update_time = %? WHERE sql_digest = %? AND update_time < %? AND status != %?`, - StatusDeleted, - updateTs, - sqlDigest, - updateTs, - StatusDeleted, - ) - if err != nil { - return err - } - deletedRows += sctx.GetSessionVars().StmtCtx.AffectedRows() - } - return nil - }) - if err != nil { - deletedRows = 0 - } - return deletedRows, err -} - -// SetBindingStatus set a Bindings's status to the storage and bind cache. -func (op *bindingOperator) SetBindingStatus(newStatus, sqlDigest string) (ok bool, err error) { - var ( - updateTs types.Time - oldStatus0, oldStatus1 string - ) - if newStatus == StatusDisabled { - // For compatibility reasons, when we need to 'set binding disabled for ', - // we need to consider both the 'enabled' and 'using' status. - oldStatus0 = StatusUsing - oldStatus1 = StatusEnabled - } else if newStatus == StatusEnabled { - // In order to unify the code, two identical old statuses are set. - oldStatus0 = StatusDisabled - oldStatus1 = StatusDisabled - } - - defer func() { - if err == nil { - err = op.cache.LoadFromStorageToCache(false) - } - }() - - err = callWithSCtx(op.sPool, true, func(sctx sessionctx.Context) error { - // Lock mysql.bind_info to synchronize with SetBindingStatus on other tidb instances. - if err = lockBindInfoTable(sctx); err != nil { - return err - } - - updateTs = types.NewTime(types.FromGoTime(time.Now()), mysql.TypeTimestamp, 6) - updateTsStr := updateTs.String() - - _, err = exec(sctx, `UPDATE mysql.bind_info SET status = %?, update_time = %? WHERE sql_digest = %? AND update_time < %? AND status IN (%?, %?)`, - newStatus, updateTsStr, sqlDigest, updateTsStr, oldStatus0, oldStatus1) - return err - }) - return -} - -// GCBinding physically removes the deleted bind records in mysql.bind_info. -func (op *bindingOperator) GCBinding() (err error) { - return callWithSCtx(op.sPool, true, func(sctx sessionctx.Context) error { - // Lock mysql.bind_info to synchronize with CreateBinding / AddBinding / DropBinding on other tidb instances. - if err = lockBindInfoTable(sctx); err != nil { - return err - } - - // To make sure that all the deleted bind records have been acknowledged to all tidb, - // we only garbage collect those records with update_time before 10 leases. - updateTime := time.Now().Add(-(10 * Lease)) - updateTimeStr := types.NewTime(types.FromGoTime(updateTime), mysql.TypeTimestamp, 6).String() - _, err = exec(sctx, `DELETE FROM mysql.bind_info WHERE status = 'deleted' and update_time < %?`, updateTimeStr) - return err - }) -} - -// lockBindInfoTable simulates `LOCK TABLE mysql.bind_info WRITE` by acquiring a pessimistic lock on a -// special builtin row of mysql.bind_info. Note that this function must be called with h.sctx.Lock() held. -// We can replace this implementation to normal `LOCK TABLE mysql.bind_info WRITE` if that feature is -// generally available later. -// This lock would enforce the CREATE / DROP GLOBAL BINDING statements to be executed sequentially, -// even if they come from different tidb instances. -func lockBindInfoTable(sctx sessionctx.Context) error { - // h.sctx already locked. - _, err := exec(sctx, LockBindInfoSQL) - return err -} diff --git a/pkg/bindinfo/global_handle.go b/pkg/bindinfo/global_handle.go index 304e4895406b4..79f2bf1483539 100644 --- a/pkg/bindinfo/global_handle.go +++ b/pkg/bindinfo/global_handle.go @@ -106,6 +106,9 @@ type GlobalBindingHandle interface { // CaptureBaselines is used to automatically capture plan baselines. CaptureBaselines() + // UpdateBindingUsageInfoToStorage is to update the binding usage info into storage + UpdateBindingUsageInfoToStorage() error + variable.Statistics } @@ -255,6 +258,17 @@ func (h *globalBindingHandle) LoadFromStorageToCache(fullLoad bool) (err error) }) } +// UpdateBindingUsageInfoToStorage is to update the binding usage info into storage +func (u *globalBindingHandle) UpdateBindingUsageInfoToStorage() error { + defer func() { + if r := recover(); r != nil { + logutil.BindLogger().Warn("panic when update usage info for binding", zap.Any("recover", r)) + } + }() + bindings := u.GetAllGlobalBindings() + return u.updateBindingUsageInfoToStorage(bindings) +} + // CreateGlobalBinding creates a Bindings to the storage and the cache. // It replaces all the exists bindings for the same normalized SQL. func (h *globalBindingHandle) CreateGlobalBinding(sctx sessionctx.Context, bindings []*Binding) (err error) { @@ -297,7 +311,9 @@ func (h *globalBindingHandle) CreateGlobalBinding(sctx sessionctx.Context, bindi // Insert the Bindings to the storage. _, err = exec( sctx, - `INSERT INTO mysql.bind_info VALUES (%?,%?, %?, %?, %?, %?, %?, %?, %?, %?, %?)`, + `INSERT INTO mysql.bind_info( + original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest +) VALUES (%?,%?, %?, %?, %?, %?, %?, %?, %?, %?, %?)`, binding.OriginalSQL, binding.BindSQL, strings.ToLower(binding.Db), diff --git a/pkg/bindinfo/util.go b/pkg/bindinfo/util.go index 050c606bc5af2..4ab9862566136 100644 --- a/pkg/bindinfo/util.go +++ b/pkg/bindinfo/util.go @@ -16,12 +16,20 @@ package bindinfo import ( "context" + "fmt" + "strings" + "time" + "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/planner/core/resolve" "github.com/pingcap/tidb/pkg/sessionctx" + "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/chunk" + "github.com/pingcap/tidb/pkg/util/intest" + "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/sqlexec" + "go.uber.org/zap" ) // exec is a helper function to execute sql and return RecordSet. @@ -36,3 +44,124 @@ func execRows(sctx sessionctx.Context, sql string, args ...any) (rows []chunk.Ro return sqlExec.ExecRestrictedSQL(kv.WithInternalSourceType(context.Background(), kv.InternalTxnBindInfo), []sqlexec.OptionFuncAlias{sqlexec.ExecOptionUseCurSession}, sql, args...) } + +var ( + // UpdateBindingUsageInfoBatchSize indicates the batch size when updating binding usage info to storage. + UpdateBindingUsageInfoBatchSize = 100 + // MaxWriteInterval indicates the interval at which a write operation needs to be performed after a binding has not been read. + MaxWriteInterval = 6 * time.Hour +) + +func (u *globalBindingHandle) updateBindingUsageInfoToStorage(bindings []Binding) error { + toWrite := make([]Binding, 0, UpdateBindingUsageInfoBatchSize) + now := time.Now() + cnt := 0 + defer func() { + if cnt > 0 { + logutil.BgLogger().Info("update binding usage info to storage", zap.Int("count", cnt), zap.Duration("duration", time.Since(now))) + } + }() + for _, binding := range bindings { + lastUsed := binding.UsageInfo.LastUsedAt.Load() + if lastUsed == nil { + continue + } + lastSaved := binding.UsageInfo.LastSavedAt.Load() + if shouldUpdateBinding(lastSaved, lastUsed) { + toWrite = append(toWrite, binding) + cnt++ + } + if len(toWrite) == UpdateBindingUsageInfoBatchSize { + err := u.updateBindingUsageInfoToStorageInternal(toWrite) + if err != nil { + return err + } + toWrite = toWrite[:0] + } + } + if len(toWrite) > 0 { + err := u.updateBindingUsageInfoToStorageInternal(toWrite) + if err != nil { + return err + } + } + return nil +} + +func (h *globalBindingHandle) updateBindingUsageInfoToStorageInternal(bindings []Binding) error { + err := h.callWithSCtx(true, func(sctx sessionctx.Context) (err error) { + if err = lockBindInfoTable(sctx); err != nil { + return errors.Trace(err) + } + // lockBindInfoTable is to prefetch the rows and lock them, it is good for performance when + // there are many bindings to update with multi tidb nodes. + // in the performance test, it takes 26.24s to update 44679 bindings. + if err = addLockForBinds(sctx, bindings); err != nil { + return errors.Trace(err) + } + for _, binding := range bindings { + lastUsed := binding.UsageInfo.LastUsedAt.Load() + intest.Assert(lastUsed != nil) + err = saveBindingUsage(sctx, binding.SQLDigest, binding.PlanDigest, *lastUsed) + if err != nil { + return errors.Trace(err) + } + } + return nil + }) + if err == nil { + ts := time.Now() + for _, binding := range bindings { + binding.UpdateLastSavedAt(&ts) + } + } + return err +} + +func shouldUpdateBinding(lastSaved, lastUsed *time.Time) bool { + if lastSaved == nil { + // If it has never been written before, it will be written. + return true + } + // If a certain amount of time specified by MaxWriteInterval has passed since the last record was written, + // and it has been used in between, it will be written. + return time.Since(*lastSaved) >= MaxWriteInterval && lastUsed.After(*lastSaved) +} + +func addLockForBinds(sctx sessionctx.Context, bindings []Binding) error { + condition := make([]string, 0, len(bindings)) + for _, binding := range bindings { + sqlDigest := binding.SQLDigest + planDigest := binding.PlanDigest + sql := fmt.Sprintf("('%s'", sqlDigest) + if planDigest == "" { + sql += ",NULL)" + } else { + sql += fmt.Sprintf(",'%s')", planDigest) + } + condition = append(condition, sql) + } + locksql := "select 1 from mysql.bind_info use index(digest_index) where (plan_digest, sql_digest) in (" + + strings.Join(condition, " , ") + ") for update" + _, err := exec(sctx, locksql) + if err != nil { + return errors.Trace(err) + } + return nil +} + +func saveBindingUsage(sctx sessionctx.Context, sqldigest, planDigest string, ts time.Time) error { + lastUsedTime := ts.UTC().Format(types.TimeFormat) + var sql = "UPDATE mysql.bind_info USE INDEX(digest_index) SET last_used_date = CONVERT_TZ(%?, '+00:00', @@TIME_ZONE) WHERE sql_digest = %?" + if planDigest == "" { + sql += " AND plan_digest IS NULL" + } else { + sql += fmt.Sprintf(" AND plan_digest = '%s'", planDigest) + } + _, err := exec( + sctx, + sql, + lastUsedTime, sqldigest, + ) + return err +} diff --git a/pkg/bindinfo/utils.go b/pkg/bindinfo/utils.go deleted file mode 100644 index 4ae780121c093..0000000000000 --- a/pkg/bindinfo/utils.go +++ /dev/null @@ -1,347 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bindinfo - -import ( - "context" - "fmt" - "strings" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/tidb/pkg/kv" - "github.com/pingcap/tidb/pkg/parser/ast" - "github.com/pingcap/tidb/pkg/parser/format" - "github.com/pingcap/tidb/pkg/parser/terror" - "github.com/pingcap/tidb/pkg/planner/core/resolve" - "github.com/pingcap/tidb/pkg/sessionctx" - "github.com/pingcap/tidb/pkg/types" - "github.com/pingcap/tidb/pkg/util" - "github.com/pingcap/tidb/pkg/util/chunk" - "github.com/pingcap/tidb/pkg/util/hint" - "github.com/pingcap/tidb/pkg/util/intest" - "github.com/pingcap/tidb/pkg/util/logutil" - utilparser "github.com/pingcap/tidb/pkg/util/parser" - "github.com/pingcap/tidb/pkg/util/sqlexec" - "go.uber.org/zap" -) - -func callWithSCtx(sPool util.DestroyableSessionPool, wrapTxn bool, f func(sctx sessionctx.Context) error) (err error) { - resource, err := sPool.Get() - if err != nil { - return err - } - defer func() { - if err == nil { // only recycle when no error - sPool.Put(resource) - } else { - // Note: Otherwise, the session will be leaked. - sPool.Destroy(resource) - } - }() - sctx := resource.(sessionctx.Context) - if wrapTxn { - if _, err = exec(sctx, "BEGIN PESSIMISTIC"); err != nil { - return - } - defer func() { - if err == nil { - _, err = exec(sctx, "COMMIT") - } else { - _, err1 := exec(sctx, "ROLLBACK") - terror.Log(errors.Trace(err1)) - } - }() - } - - err = f(sctx) - return -} - -// exec is a helper function to execute sql and return RecordSet. -func exec(sctx sessionctx.Context, sql string, args ...any) (sqlexec.RecordSet, error) { - sqlExec := sctx.GetSQLExecutor() - return sqlExec.ExecuteInternal(kv.WithInternalSourceType(context.Background(), kv.InternalTxnBindInfo), sql, args...) -} - -// execRows is a helper function to execute sql and return rows and fields. -func execRows(sctx sessionctx.Context, sql string, args ...any) (rows []chunk.Row, fields []*resolve.ResultField, err error) { - sqlExec := sctx.GetRestrictedSQLExecutor() - return sqlExec.ExecRestrictedSQL(kv.WithInternalSourceType(context.Background(), kv.InternalTxnBindInfo), - []sqlexec.OptionFuncAlias{sqlexec.ExecOptionUseCurSession}, sql, args...) -} - -// bindingLogger with category "sql-bind" is used to log statistic related messages. -func bindingLogger() *zap.Logger { - return logutil.BgLogger().With(zap.String("category", "sql-bind")) -} - -// GenerateBindingSQL generates binding sqls from stmt node and plan hints. -func GenerateBindingSQL(stmtNode ast.StmtNode, planHint string, defaultDB string) string { - // We need to evolve plan based on the current sql, not the original sql which may have different parameters. - // So here we would remove the hint and inject the current best plan hint. - hint.BindHint(stmtNode, &hint.HintsSet{}) - bindSQL := RestoreDBForBinding(stmtNode, defaultDB) - if bindSQL == "" { - return "" - } - switch n := stmtNode.(type) { - case *ast.DeleteStmt: - deleteIdx := strings.Index(bindSQL, "DELETE") - // Remove possible `explain` prefix. - bindSQL = bindSQL[deleteIdx:] - return strings.Replace(bindSQL, "DELETE", fmt.Sprintf("DELETE /*+ %s*/", planHint), 1) - case *ast.UpdateStmt: - updateIdx := strings.Index(bindSQL, "UPDATE") - // Remove possible `explain` prefix. - bindSQL = bindSQL[updateIdx:] - return strings.Replace(bindSQL, "UPDATE", fmt.Sprintf("UPDATE /*+ %s*/", planHint), 1) - case *ast.SelectStmt: - var selectIdx int - if n.With != nil { - var withSb strings.Builder - withIdx := strings.Index(bindSQL, "WITH") - restoreCtx := format.NewRestoreCtx(format.RestoreStringSingleQuotes|format.RestoreSpacesAroundBinaryOperation|format.RestoreStringWithoutCharset|format.RestoreNameBackQuotes, &withSb) - restoreCtx.DefaultDB = defaultDB - if err := n.With.Restore(restoreCtx); err != nil { - bindingLogger().Debug("restore SQL failed", zap.Error(err)) - return "" - } - withEnd := withIdx + len(withSb.String()) - tmp := strings.Replace(bindSQL[withEnd:], "SELECT", fmt.Sprintf("SELECT /*+ %s*/", planHint), 1) - return strings.Join([]string{bindSQL[withIdx:withEnd], tmp}, "") - } - selectIdx = strings.Index(bindSQL, "SELECT") - // Remove possible `explain` prefix. - bindSQL = bindSQL[selectIdx:] - return strings.Replace(bindSQL, "SELECT", fmt.Sprintf("SELECT /*+ %s*/", planHint), 1) - case *ast.InsertStmt: - insertIdx := int(0) - if n.IsReplace { - insertIdx = strings.Index(bindSQL, "REPLACE") - } else { - insertIdx = strings.Index(bindSQL, "INSERT") - } - // Remove possible `explain` prefix. - bindSQL = bindSQL[insertIdx:] - return strings.Replace(bindSQL, "SELECT", fmt.Sprintf("SELECT /*+ %s*/", planHint), 1) - } - bindingLogger().Debug("unexpected statement type when generating bind SQL", zap.Any("statement", stmtNode)) - return "" -} - -func readBindingsFromStorage(sPool util.DestroyableSessionPool, condition string, args ...any) (bindings []*Binding, err error) { - selectStmt := fmt.Sprintf(`SELECT original_sql, bind_sql, default_db, status, create_time, - update_time, charset, collation, source, sql_digest, plan_digest FROM mysql.bind_info - %s`, condition) - - err = callWithSCtx(sPool, false, func(sctx sessionctx.Context) error { - rows, _, err := execRows(sctx, selectStmt, args...) - if err != nil { - return err - } - bindings = make([]*Binding, 0, len(rows)) - for _, row := range rows { - // Skip the builtin record which is designed for binding synchronization. - if row.GetString(0) == BuiltinPseudoSQL4BindLock { - continue - } - binding := newBindingFromStorage(row) - if hErr := prepareHints(sctx, binding); hErr != nil { - bindingLogger().Warn("failed to generate bind record from data row", zap.Error(hErr)) - continue - } - bindings = append(bindings, binding) - } - return nil - }) - return -} - -var ( - // UpdateBindingUsageInfoBatchSize indicates the batch size when updating binding usage info to storage. - UpdateBindingUsageInfoBatchSize = 100 - // MaxWriteInterval indicates the interval at which a write operation needs to be performed after a binding has not been read. - MaxWriteInterval = 6 * time.Hour -) - -func updateBindingUsageInfoToStorage(sPool util.DestroyableSessionPool, bindings []*Binding) error { - toWrite := make([]*Binding, 0, UpdateBindingUsageInfoBatchSize) - now := time.Now() - cnt := 0 - defer func() { - if cnt > 0 { - bindingLogger().Info("update binding usage info to storage", zap.Int("count", cnt), zap.Duration("duration", time.Since(now))) - } - }() - for _, binding := range bindings { - lastUsed := binding.UsageInfo.LastUsedAt.Load() - if lastUsed == nil { - continue - } - lastSaved := binding.UsageInfo.LastSavedAt.Load() - if shouldUpdateBinding(lastSaved, lastUsed) { - toWrite = append(toWrite, binding) - cnt++ - } - if len(toWrite) == UpdateBindingUsageInfoBatchSize { - err := updateBindingUsageInfoToStorageInternal(sPool, toWrite) - if err != nil { - return err - } - toWrite = toWrite[:0] - } - } - if len(toWrite) > 0 { - err := updateBindingUsageInfoToStorageInternal(sPool, toWrite) - if err != nil { - return err - } - } - return nil -} - -func shouldUpdateBinding(lastSaved, lastUsed *time.Time) bool { - if lastSaved == nil { - // If it has never been written before, it will be written. - return true - } - // If a certain amount of time specified by MaxWriteInterval has passed since the last record was written, - // and it has been used in between, it will be written. - return time.Since(*lastSaved) >= MaxWriteInterval && lastUsed.After(*lastSaved) -} - -func updateBindingUsageInfoToStorageInternal(sPool util.DestroyableSessionPool, bindings []*Binding) error { - err := callWithSCtx(sPool, true, func(sctx sessionctx.Context) (err error) { - if err = lockBindInfoTable(sctx); err != nil { - return errors.Trace(err) - } - // lockBindInfoTable is to prefetch the rows and lock them, it is good for performance when - // there are many bindings to update with multi tidb nodes. - // in the performance test, it takes 26.24s to update 44679 bindings. - if err = addLockForBinds(sctx, bindings); err != nil { - return errors.Trace(err) - } - for _, binding := range bindings { - lastUsed := binding.UsageInfo.LastUsedAt.Load() - intest.Assert(lastUsed != nil) - err = saveBindingUsage(sctx, binding.SQLDigest, binding.PlanDigest, *lastUsed) - if err != nil { - return errors.Trace(err) - } - } - return nil - }) - if err == nil { - ts := time.Now() - for _, binding := range bindings { - binding.UpdateLastSavedAt(&ts) - } - } - return err -} - -func addLockForBinds(sctx sessionctx.Context, bindings []*Binding) error { - condition := make([]string, 0, len(bindings)) - for _, binding := range bindings { - sqlDigest := binding.SQLDigest - planDigest := binding.PlanDigest - sql := fmt.Sprintf("('%s'", sqlDigest) - if planDigest == "" { - sql += ",NULL)" - } else { - sql += fmt.Sprintf(",'%s')", planDigest) - } - condition = append(condition, sql) - } - locksql := "select 1 from mysql.bind_info use index(digest_index) where (plan_digest, sql_digest) in (" + - strings.Join(condition, " , ") + ") for update" - _, err := exec(sctx, locksql) - if err != nil { - return errors.Trace(err) - } - return nil -} - -func saveBindingUsage(sctx sessionctx.Context, sqldigest, planDigest string, ts time.Time) error { - lastUsedTime := ts.UTC().Format(types.TimeFormat) - var sql = "UPDATE mysql.bind_info USE INDEX(digest_index) SET last_used_date = CONVERT_TZ(%?, '+00:00', @@TIME_ZONE) WHERE sql_digest = %?" - if planDigest == "" { - sql += " AND plan_digest IS NULL" - } else { - sql += fmt.Sprintf(" AND plan_digest = '%s'", planDigest) - } - _, err := exec( - sctx, - sql, - lastUsedTime, sqldigest, - ) - return err -} - -// newBindingFromStorage builds Bindings from a tuple in storage. -func newBindingFromStorage(row chunk.Row) *Binding { - status := row.GetString(3) - // For compatibility, the 'Using' status binding will be converted to the 'Enabled' status binding. - if status == StatusUsing { - status = StatusEnabled - } - return &Binding{ - OriginalSQL: row.GetString(0), - Db: strings.ToLower(row.GetString(2)), - BindSQL: row.GetString(1), - Status: status, - CreateTime: row.GetTime(4), - UpdateTime: row.GetTime(5), - Charset: row.GetString(6), - Collation: row.GetString(7), - Source: row.GetString(8), - SQLDigest: row.GetString(9), - PlanDigest: row.GetString(10), - } -} - -// getBindingPlanDigest does the best efforts to fill binding's plan_digest. -func getBindingPlanDigest(sctx sessionctx.Context, schema, bindingSQL string) (planDigest string) { - defer func() { - if r := recover(); r != nil { - bindingLogger().Error("panic when filling plan digest for binding", - zap.String("binding_sql", bindingSQL), zap.Reflect("panic", r)) - } - }() - - vars := sctx.GetSessionVars() - defer func(originalBaseline bool, originalDB string) { - vars.UsePlanBaselines = originalBaseline - vars.CurrentDB = originalDB - }(vars.UsePlanBaselines, vars.CurrentDB) - vars.UsePlanBaselines = false - vars.CurrentDB = schema - - p := utilparser.GetParser() - defer utilparser.DestroyParser(p) - p.SetSQLMode(vars.SQLMode) - p.SetParserConfig(vars.BuildParserConfig()) - - charset, collation := vars.GetCharsetInfo() - if stmt, err := p.ParseOneStmt(bindingSQL, charset, collation); err == nil { - if !hasParam(stmt) { - // if there is '?' from `create binding using select a from t where a=?`, - // the final plan digest might be incorrect. - planDigest, _ = CalculatePlanDigest(sctx, stmt) - } - } - return -} diff --git a/pkg/domain/domain.go b/pkg/domain/domain.go index 61fc9acaea251..ffb901b7caae9 100644 --- a/pkg/domain/domain.go +++ b/pkg/domain/domain.go @@ -2108,7 +2108,7 @@ func (do *Domain) globalBindHandleWorkerLoop(owner owner.Manager) { logutil.BgLogger().Error("GC bind record failed", zap.Error(err)) } case <-writeBindingUsageTicker.C: - bindHandle := do.BindingHandle() + bindHandle := do.BindHandle() err := bindHandle.UpdateBindingUsageInfoToStorage() if err != nil { logutil.BgLogger().Warn("BindingHandle.UpdateBindingUsageInfoToStorage", zap.Error(err)) diff --git a/pkg/session/bootstrap.go b/pkg/session/bootstrap.go index c895e5915171d..ffd08fe501a2a 100644 --- a/pkg/session/bootstrap.go +++ b/pkg/session/bootstrap.go @@ -1207,12 +1207,16 @@ const ( // Add last_stats_histograms_version to mysql.stats_meta. version220 = 220 + // version 221 + // Add last_used_date to mysql.bind_info + version221 = 221 + // next version should start with 239 ) // currentBootstrapVersion is defined as a variable, so we can modify its value for testing. // please make sure this is the largest version -var currentBootstrapVersion int64 = version220 +var currentBootstrapVersion int64 = version221 // DDL owner key's expired time is ManagerSessionTTL seconds, we should wait the time and give more time to have a chance to finish it. var internalSQLTimeout = owner.ManagerSessionTTL + 15 @@ -1388,6 +1392,7 @@ var ( upgradeToVer218, upgradeToVer219, upgradeToVer220, + upgradeToVer221, } ) @@ -3264,6 +3269,13 @@ func upgradeToVer220(s sessiontypes.Session, ver int64) { doReentrantDDL(s, "ALTER TABLE mysql.stats_meta ADD COLUMN last_stats_histograms_version bigint unsigned DEFAULT NULL", infoschema.ErrColumnExists) } +func upgradeToVer221(s sessiontypes.Session, ver int64) { + if ver >= version221 { + return + } + doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD COLUMN last_used_date DATE DEFAULT NULL AFTER `plan_digest`", infoschema.ErrColumnExists) +} + // initGlobalVariableIfNotExists initialize a global variable with specific val if it does not exist. func initGlobalVariableIfNotExists(s sessiontypes.Session, name string, val any) { ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) diff --git a/pkg/session/bootstrap_test.go b/pkg/session/bootstrap_test.go index a9c92f831ad69..1e53b1384e9ac 100644 --- a/pkg/session/bootstrap_test.go +++ b/pkg/session/bootstrap_test.go @@ -25,10 +25,6 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/pingcap/tidb/pkg/bindinfo" -<<<<<<< HEAD -======= - "github.com/pingcap/tidb/pkg/config/kerneltype" ->>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) "github.com/pingcap/tidb/pkg/ddl" "github.com/pingcap/tidb/pkg/domain" "github.com/pingcap/tidb/pkg/expression/sessionexpr" diff --git a/pkg/session/upgrade.go b/pkg/session/upgrade.go deleted file mode 100644 index f8317ee02e0f5..0000000000000 --- a/pkg/session/upgrade.go +++ /dev/null @@ -1,2039 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package session - -import ( - "context" - "fmt" - "strconv" - "strings" - "time" - - "github.com/pingcap/tidb/pkg/bindinfo" - "github.com/pingcap/tidb/pkg/config" - "github.com/pingcap/tidb/pkg/expression" - "github.com/pingcap/tidb/pkg/infoschema" - "github.com/pingcap/tidb/pkg/kv" - "github.com/pingcap/tidb/pkg/meta" - "github.com/pingcap/tidb/pkg/parser" - "github.com/pingcap/tidb/pkg/parser/ast" - "github.com/pingcap/tidb/pkg/parser/mysql" - "github.com/pingcap/tidb/pkg/parser/terror" - "github.com/pingcap/tidb/pkg/session/sessionapi" - "github.com/pingcap/tidb/pkg/sessionctx/vardef" - "github.com/pingcap/tidb/pkg/sessionctx/variable" - "github.com/pingcap/tidb/pkg/types" - "github.com/pingcap/tidb/pkg/util/chunk" - "github.com/pingcap/tidb/pkg/util/dbterror" - "github.com/pingcap/tidb/pkg/util/dbterror/plannererrors" - "github.com/pingcap/tidb/pkg/util/logutil" - utilparser "github.com/pingcap/tidb/pkg/util/parser" - "github.com/pingcap/tidb/pkg/util/sqlexec" - "github.com/pingcap/tidb/pkg/util/timeutil" - "go.uber.org/zap" -) - -const ( - // Const for TiDB server version 2. - version2 = 2 - version3 = 3 - version4 = 4 - version5 = 5 - version6 = 6 - version7 = 7 - version8 = 8 - version9 = 9 - version10 = 10 - version11 = 11 - version12 = 12 - version13 = 13 - version14 = 14 - version15 = 15 - version16 = 16 - version17 = 17 - version18 = 18 - version19 = 19 - version20 = 20 - version21 = 21 - version22 = 22 - version23 = 23 - version24 = 24 - version25 = 25 - version26 = 26 - version27 = 27 - version28 = 28 - // version29 only need to be run when the current version is 28. - version29 = 29 - version30 = 30 - version31 = 31 - version32 = 32 - version33 = 33 - version34 = 34 - version35 = 35 - version36 = 36 - version37 = 37 - version38 = 38 - // version39 will be redone in version46 so it's skipped here. - // version40 is the version that introduce new collation in TiDB, - // see https://github.com/pingcap/tidb/pull/14574 for more details. - version40 = 40 - version41 = 41 - // version42 add storeType and reason column in expr_pushdown_blacklist - version42 = 42 - // version43 updates global variables related to statement summary. - version43 = 43 - // version44 delete tidb_isolation_read_engines from mysql.global_variables to avoid unexpected behavior after upgrade. - version44 = 44 - // version45 introduces CONFIG_PRIV for SET CONFIG statements. - version45 = 45 - // version46 fix a bug in v3.1.1. - version46 = 46 - // version47 add Source to bindings to indicate the way binding created. - version47 = 47 - // version48 reset all deprecated concurrency related system-variables if they were all default value. - // version49 introduces mysql.stats_extended table. - // Both version48 and version49 will be redone in version55 and version56 so they're skipped here. - // version50 add mysql.schema_index_usage table. - version50 = 50 - // version51 introduces CreateTablespacePriv to mysql.user. - // version51 will be redone in version63 so it's skipped here. - // version52 change mysql.stats_histograms cm_sketch column from blob to blob(6291456) - version52 = 52 - // version53 introduce Global variable tidb_enable_strict_double_type_check - version53 = 53 - // version54 writes a variable `mem_quota_query` to mysql.tidb if it's a cluster upgraded from v3.0.x to v4.0.9+. - version54 = 54 - // version55 fixes the bug that upgradeToVer48 would be missed when upgrading from v4.0 to a new version - version55 = 55 - // version56 fixes the bug that upgradeToVer49 would be missed when upgrading from v4.0 to a new version - version56 = 56 - // version57 fixes the bug of concurrent create / drop binding - version57 = 57 - // version58 add `Repl_client_priv` and `Repl_slave_priv` to `mysql.user` - // version58 will be redone in version64 so it's skipped here. - // version59 add writes a variable `oom-action` to mysql.tidb if it's a cluster upgraded from v3.0.x to v4.0.11+. - version59 = 59 - // version60 redesigns `mysql.stats_extended` - version60 = 60 - // version61 will be redone in version67 - // version62 add column ndv for mysql.stats_buckets. - version62 = 62 - // version63 fixes the bug that upgradeToVer51 would be missed when upgrading from v4.0 to a new version - version63 = 63 - // version64 is redone upgradeToVer58 after upgradeToVer63, this is to preserve the order of the columns in mysql.user - version64 = 64 - // version65 add mysql.stats_fm_sketch table. - version65 = 65 - // version66 enables the feature `track_aggregate_memory_usage` by default. - version66 = 66 - // version67 restore all SQL bindings. - version67 = 67 - // version68 update the global variable 'tidb_enable_clustered_index' from 'off' to 'int_only'. - version68 = 68 - // version69 adds mysql.global_grants for DYNAMIC privileges - version69 = 69 - // version70 adds mysql.user.plugin to allow multiple authentication plugins - version70 = 70 - // version71 forces tidb_multi_statement_mode=OFF when tidb_multi_statement_mode=WARN - // This affects upgrades from v4.0 where the default was WARN. - version71 = 71 - // version72 adds snapshot column for mysql.stats_meta - version72 = 72 - // version73 adds mysql.capture_plan_baselines_blacklist table - version73 = 73 - // version74 changes global variable `tidb_stmt_summary_max_stmt_count` value from 200 to 3000. - version74 = 74 - // version75 update mysql.*.host from char(60) to char(255) - version75 = 75 - // version76 update mysql.columns_priv from SET('Select','Insert','Update') to SET('Select','Insert','Update','References') - version76 = 76 - // version77 adds mysql.column_stats_usage table - version77 = 77 - // version78 updates mysql.stats_buckets.lower_bound, mysql.stats_buckets.upper_bound and mysql.stats_histograms.last_analyze_pos from BLOB to LONGBLOB. - version78 = 78 - // version79 adds the mysql.table_cache_meta table - version79 = 79 - // version80 fixes the issue https://github.com/pingcap/tidb/issues/25422. - // If the TiDB upgrading from the 4.x to a newer version, we keep the tidb_analyze_version to 1. - version80 = 80 - // version81 insert "tidb_enable_index_merge|off" to mysql.GLOBAL_VARIABLES if there is no tidb_enable_index_merge. - // This will only happens when we upgrade a cluster before 4.0.0 to 4.0.0+. - version81 = 81 - // version82 adds the mysql.analyze_options table - version82 = 82 - // version83 adds the tables mysql.stats_history - version83 = 83 - // version84 adds the tables mysql.stats_meta_history - version84 = 84 - // version85 updates bindings with status 'using' in mysql.bind_info table to 'enabled' status - version85 = 85 - // version86 update mysql.tables_priv from SET('Select','Insert','Update') to SET('Select','Insert','Update','References'). - version86 = 86 - // version87 adds the mysql.analyze_jobs table - version87 = 87 - // version88 fixes the issue https://github.com/pingcap/tidb/issues/33650. - version88 = 88 - // version89 adds the tables mysql.advisory_locks - version89 = 89 - // version90 converts enable-batch-dml, mem-quota-query, query-log-max-len, committer-concurrency, run-auto-analyze, and oom-action to a sysvar - version90 = 90 - // version91 converts prepared-plan-cache to sysvars - version91 = 91 - // version92 for concurrent ddl. - version92 = 92 - // version93 converts oom-use-tmp-storage to a sysvar - version93 = 93 - version94 = 94 - // version95 add a column `User_attributes` to `mysql.user` - version95 = 95 - // version97 sets tidb_opt_range_max_size to 0 when a cluster upgrades from some version lower than v6.4.0 to v6.4.0+. - // It promises the compatibility of building ranges behavior. - version97 = 97 - // version98 add a column `Token_issuer` to `mysql.user` - version98 = 98 - version99 = 99 - // version100 converts server-memory-quota to a sysvar - version100 = 100 - // version101 add mysql.plan_replayer_status table - version101 = 101 - // version102 add mysql.plan_replayer_task table - version102 = 102 - // version103 adds the tables mysql.stats_table_locked - version103 = 103 - // version104 add `sql_digest` and `plan_digest` to `bind_info` - version104 = 104 - // version105 insert "tidb_cost_model_version|1" to mysql.GLOBAL_VARIABLES if there is no tidb_cost_model_version. - // This will only happens when we upgrade a cluster before 6.0. - version105 = 105 - // version106 add mysql.password_history, and Password_reuse_history, Password_reuse_time into mysql.user. - version106 = 106 - // version107 add columns related to password expiration into mysql.user - version107 = 107 - // version108 adds the table tidb_ttl_table_status - version108 = 108 - // version109 sets tidb_enable_gc_aware_memory_track to off when a cluster upgrades from some version lower than v6.5.0. - version109 = 109 - // ... - // [version110, version129] is the version range reserved for patches of 6.5.x - // ... - // version110 sets tidb_stats_load_pseudo_timeout to ON when a cluster upgrades from some version lower than v6.5.0. - version110 = 110 - // version130 add column source to mysql.stats_meta_history - version130 = 130 - // version131 adds the table tidb_ttl_task and tidb_ttl_job_history - version131 = 131 - // version132 modifies the view tidb_mdl_view - version132 = 132 - // version133 sets tidb_server_memory_limit to "80%" - version133 = 133 - // version134 modifies the following global variables default value: - // - foreign_key_checks: off -> on - // - tidb_enable_foreign_key: off -> on - // - tidb_store_batch_size: 0 -> 4 - version134 = 134 - // version135 sets tidb_opt_advanced_join_hint to off when a cluster upgrades from some version lower than v7.0. - version135 = 135 - // version136 prepare the tables for the distributed task. - version136 = 136 - // version137 introduces some reserved resource groups - version137 = 137 - // version 138 set tidb_enable_null_aware_anti_join to true - version138 = 138 - // version 139 creates mysql.load_data_jobs table for LOAD DATA statement - // deprecated in version184 - version139 = 139 - // version 140 add column task_key to mysql.tidb_global_task - version140 = 140 - // version 141 - // set the value of `tidb_session_plan_cache_size` to "tidb_prepared_plan_cache_size" if there is no `tidb_session_plan_cache_size`. - // update tidb_load_based_replica_read_threshold from 0 to 4 - // This will only happens when we upgrade a cluster before 7.1. - version141 = 141 - // version 142 insert "tidb_enable_non_prepared_plan_cache|0" to mysql.GLOBAL_VARIABLES if there is no tidb_enable_non_prepared_plan_cache. - // This will only happens when we upgrade a cluster before 6.5. - version142 = 142 - // version 143 add column `error` to `mysql.tidb_global_task` and `mysql.tidb_background_subtask` - version143 = 143 - // version 144 turn off `tidb_plan_cache_invalidation_on_fresh_stats`, which is introduced in 7.1-rc, - // if it's upgraded from an existing old version cluster. - version144 = 144 - // version 145 to only add a version make we know when we support upgrade state. - version145 = 145 - // version 146 add index for mysql.stats_meta_history and mysql.stats_history. - version146 = 146 - // ... - // [version147, version166] is the version range reserved for patches of 7.1.x - // ... - // version 167 add column `step` to `mysql.tidb_background_subtask` - version167 = 167 - version168 = 168 - // version 169 - // create table `mysql.tidb_runaway_quarantined_watch` and table `mysql.tidb_runaway_queries` - // to save runaway query records and persist runaway watch at 7.2 version. - // but due to ver171 recreate `mysql.tidb_runaway_watch`, - // no need to create table `mysql.tidb_runaway_quarantined_watch`, so delete it. - version169 = 169 - version170 = 170 - // version 171 - // keep the tidb_server length same as instance in other tables. - version171 = 171 - // version 172 - // create table `mysql.tidb_runaway_watch` and table `mysql.tidb_runaway_watch_done` - // to persist runaway watch and deletion of runaway watch at 7.3. - version172 = 172 - // version 173 add column `summary` to `mysql.tidb_background_subtask`. - version173 = 173 - // version 174 - // add column `step`, `error`; delete unique key; and add key idx_state_update_time - // to `mysql.tidb_background_subtask_history`. - version174 = 174 - - // version 175 - // update normalized bindings of `in (?)` to `in (...)` to solve #44298. - version175 = 175 - - // version 176 - // add `mysql.tidb_global_task_history` - version176 = 176 - - // version 177 - // add `mysql.dist_framework_meta` - version177 = 177 - - // version 178 - // write mDDLTableVersion into `mysql.tidb` table - version178 = 178 - - // version 179 - // enlarge `VARIABLE_VALUE` of `mysql.global_variables` from `varchar(1024)` to `varchar(16383)`. - version179 = 179 - - // ... - // [version180, version189] is the version range reserved for patches of 7.5.x - // ... - - // version 190 - // add priority/create_time/end_time to `mysql.tidb_global_task`/`mysql.tidb_global_task_history` - // add concurrency/create_time/end_time/digest to `mysql.tidb_background_subtask`/`mysql.tidb_background_subtask_history` - // add idx_exec_id(exec_id), uk_digest to `mysql.tidb_background_subtask` - // add cpu_count to mysql.dist_framework_meta - // modify `mysql.dist_framework_meta` host from VARCHAR(100) to VARCHAR(261) - // modify `mysql.tidb_background_subtask`/`mysql.tidb_background_subtask_history` exec_id from varchar(256) to VARCHAR(261) - // modify `mysql.tidb_global_task`/`mysql.tidb_global_task_history` dispatcher_id from varchar(256) to VARCHAR(261) - version190 = 190 - - // version 191 - // set tidb_txn_mode to Optimistic when tidb_txn_mode is not set. - version191 = 191 - - // version 192 - // add new system table `mysql.request_unit_by_group`, which is used for - // historical RU consumption by resource group per day. - version192 = 192 - - // version 193 - // replace `mysql.tidb_mdl_view` table - version193 = 193 - - // version 194 - // remove `mysql.load_data_jobs` table - version194 = 194 - - // version 195 - // drop `mysql.schema_index_usage` table - // create `sys` schema - // create `sys.schema_unused_indexes` table - version195 = 195 - - // version 196 - // add column `target_scope` for 'mysql.tidb_global_task` table - // add column `target_scope` for 'mysql.tidb_global_task_history` table - version196 = 196 - - // version 197 - // replace `mysql.tidb_mdl_view` table - version197 = 197 - - // version 198 - // add column `owner_id` for `mysql.tidb_mdl_info` table - version198 = 198 - - // ... - // [version199, version208] is the version range reserved for patches of 8.1.x - // ... - - // version 209 - // sets `tidb_resource_control_strict_mode` to off when a cluster upgrades from some version lower than v8.2. - version209 = 209 - // version210 indicates that if TiDB is upgraded from a lower version(lower than 8.3.0), the tidb_analyze_column_options will be set to ALL. - version210 = 210 - - // version211 add column `summary` to `mysql.tidb_background_subtask_history`. - version211 = 211 - - // version212 changed a lots of runaway related table. - // 1. switchGroup: add column `switch_group_name` to `mysql.tidb_runaway_watch` and `mysql.tidb_runaway_watch_done`. - // 2. modify column `plan_digest` type, modify column `time` to `start_time, - // modify column `original_sql` to `sample_sql` to `mysql.tidb_runaway_queries`. - // 3. modify column length of `action`. - // 4. add column `rule` to `mysql.tidb_runaway_watch`, `mysql.tidb_runaway_watch_done` and `mysql.tidb_runaway_queries`. - version212 = 212 - - // version 213 - // create `mysql.tidb_pitr_id_map` table - version213 = 213 - - // version 214 - // create `mysql.index_advisor_results` table - version214 = 214 - - // If the TiDB upgrading from the a version before v7.0 to a newer version, we keep the tidb_enable_inl_join_inner_multi_pattern to 0. - version215 = 215 - - // version 216 - // changes variable `tidb_scatter_region` value from ON to "table" and OFF to "". - version216 = 216 - - // version 217 - // Keep tidb_schema_cache_size to 0 if this variable does not exist (upgrading from old version pre 8.1). - version217 = 217 - - // version 218 - // enable fast_create_table on default - version218 = 218 - - // ... - // [version219, version238] is the version range reserved for patches of 8.5.x - // ... - - // next version should start with 239 - - // version 239 - // add modify_params to tidb_global_task and tidb_global_task_history. - version239 = 239 - - // version 240 - // Add indexes to mysql.analyze_jobs to speed up the query. - version240 = 240 - - // Add index on user field for some mysql tables. - version241 = 241 - - // version 242 - // insert `cluster_id` into the `mysql.tidb` table. - // Add workload-based learning system tables - version242 = 242 - - // Add max_node_count column to tidb_global_task and tidb_global_task_history. - // Add extra_params to tidb_global_task and tidb_global_task_history. - version243 = 243 - - // version244 add Max_user_connections into mysql.user. - version244 = 244 - - // version245 updates column types of mysql.bind_info. - version245 = 245 - - // version246 adds new unique index for mysql.bind_info. - version246 = 246 - - // version 247 - // Add last_stats_histograms_version to mysql.stats_meta. - version247 = 247 - - // version 248 - // Update mysql.tidb_pitr_id_map to add restore_id as a primary key field - version248 = 248 - version249 = 249 - - // version250 add keyspace to tidb_global_task and tidb_global_task_history. - version250 = 250 - - // version 251 - // Add group_key to mysql.tidb_import_jobs. - version251 = 251 - - // version 252 - // Update FSP of mysql.bind_info timestamp columns to microsecond precision. - version252 = 252 - - // version253 - // Add last_used_date to mysql.bind_info - version253 = 253 -) - -// versionedUpgradeFunction is a struct that holds the upgrade function related -// to a specific bootstrap version. -// we will run the upgrade function fn when the current bootstrapped version is -// less than the version in this struct -type versionedUpgradeFunction struct { - version int64 - fn func(sessionapi.Session, int64) -} - -// currentBootstrapVersion is defined as a variable, so we can modify its value for testing. -// please make sure this is the largest version -var currentBootstrapVersion int64 = version253 - -var ( - // this list must be ordered by version in ascending order, and the function - // name must follow the same pattern as `upgradeToVer`. - upgradeToVerFunctions = []versionedUpgradeFunction{ - {version: version2, fn: upgradeToVer2}, - {version: version3, fn: upgradeToVer3}, - {version: version4, fn: upgradeToVer4}, - {version: version5, fn: upgradeToVer5}, - {version: version6, fn: upgradeToVer6}, - {version: version7, fn: upgradeToVer7}, - {version: version8, fn: upgradeToVer8}, - {version: version9, fn: upgradeToVer9}, - {version: version10, fn: upgradeToVer10}, - {version: version11, fn: upgradeToVer11}, - {version: version12, fn: upgradeToVer12}, - {version: version13, fn: upgradeToVer13}, - {version: version14, fn: upgradeToVer14}, - {version: version15, fn: upgradeToVer15}, - {version: version16, fn: upgradeToVer16}, - {version: version17, fn: upgradeToVer17}, - {version: version18, fn: upgradeToVer18}, - {version: version19, fn: upgradeToVer19}, - {version: version20, fn: upgradeToVer20}, - {version: version21, fn: upgradeToVer21}, - {version: version22, fn: upgradeToVer22}, - {version: version23, fn: upgradeToVer23}, - {version: version24, fn: upgradeToVer24}, - {version: version25, fn: upgradeToVer25}, - {version: version26, fn: upgradeToVer26}, - {version: version27, fn: upgradeToVer27}, - {version: version28, fn: upgradeToVer28}, - {version: version29, fn: upgradeToVer29}, - {version: version30, fn: upgradeToVer30}, - {version: version31, fn: upgradeToVer31}, - {version: version32, fn: upgradeToVer32}, - {version: version33, fn: upgradeToVer33}, - {version: version34, fn: upgradeToVer34}, - {version: version35, fn: upgradeToVer35}, - {version: version36, fn: upgradeToVer36}, - {version: version37, fn: upgradeToVer37}, - {version: version38, fn: upgradeToVer38}, - // We will redo upgradeToVer39 in upgradeToVer46, - // so upgradeToVer39 is skipped here. - {version: version40, fn: upgradeToVer40}, - {version: version41, fn: upgradeToVer41}, - {version: version42, fn: upgradeToVer42}, - {version: version43, fn: upgradeToVer43}, - {version: version44, fn: upgradeToVer44}, - {version: version45, fn: upgradeToVer45}, - {version: version46, fn: upgradeToVer46}, - {version: version47, fn: upgradeToVer47}, - // We will redo upgradeToVer48 and upgradeToVer49 in upgradeToVer55 and upgradeToVer56, - // so upgradeToVer48 and upgradeToVer49 is skipped here. - {version: version50, fn: upgradeToVer50}, - // We will redo upgradeToVer51 in upgradeToVer63, it is skipped here. - {version: version52, fn: upgradeToVer52}, - {version: version53, fn: upgradeToVer53}, - {version: version54, fn: upgradeToVer54}, - {version: version55, fn: upgradeToVer55}, - {version: version56, fn: upgradeToVer56}, - {version: version57, fn: upgradeToVer57}, - // We will redo upgradeToVer58 in upgradeToVer64, it is skipped here. - {version: version59, fn: upgradeToVer59}, - {version: version60, fn: upgradeToVer60}, - // We will redo upgradeToVer61 in upgradeToVer67, it is skipped here. - {version: version62, fn: upgradeToVer62}, - {version: version63, fn: upgradeToVer63}, - {version: version64, fn: upgradeToVer64}, - {version: version65, fn: upgradeToVer65}, - {version: version66, fn: upgradeToVer66}, - {version: version67, fn: upgradeToVer67}, - {version: version68, fn: upgradeToVer68}, - {version: version69, fn: upgradeToVer69}, - {version: version70, fn: upgradeToVer70}, - {version: version71, fn: upgradeToVer71}, - {version: version72, fn: upgradeToVer72}, - {version: version73, fn: upgradeToVer73}, - {version: version74, fn: upgradeToVer74}, - {version: version75, fn: upgradeToVer75}, - {version: version76, fn: upgradeToVer76}, - {version: version77, fn: upgradeToVer77}, - {version: version78, fn: upgradeToVer78}, - {version: version79, fn: upgradeToVer79}, - {version: version80, fn: upgradeToVer80}, - {version: version81, fn: upgradeToVer81}, - {version: version82, fn: upgradeToVer82}, - {version: version83, fn: upgradeToVer83}, - {version: version84, fn: upgradeToVer84}, - {version: version85, fn: upgradeToVer85}, - {version: version86, fn: upgradeToVer86}, - {version: version87, fn: upgradeToVer87}, - {version: version88, fn: upgradeToVer88}, - {version: version89, fn: upgradeToVer89}, - {version: version90, fn: upgradeToVer90}, - {version: version91, fn: upgradeToVer91}, - {version: version93, fn: upgradeToVer93}, - {version: version94, fn: upgradeToVer94}, - {version: version95, fn: upgradeToVer95}, - // We will redo upgradeToVer96 in upgradeToVer100, it is skipped here. - {version: version97, fn: upgradeToVer97}, - {version: version98, fn: upgradeToVer98}, - {version: version100, fn: upgradeToVer100}, - {version: version101, fn: upgradeToVer101}, - {version: version102, fn: upgradeToVer102}, - {version: version103, fn: upgradeToVer103}, - {version: version104, fn: upgradeToVer104}, - {version: version105, fn: upgradeToVer105}, - {version: version106, fn: upgradeToVer106}, - {version: version107, fn: upgradeToVer107}, - {version: version108, fn: upgradeToVer108}, - {version: version109, fn: upgradeToVer109}, - {version: version110, fn: upgradeToVer110}, - {version: version130, fn: upgradeToVer130}, - {version: version131, fn: upgradeToVer131}, - {version: version132, fn: upgradeToVer132}, - {version: version133, fn: upgradeToVer133}, - {version: version134, fn: upgradeToVer134}, - {version: version135, fn: upgradeToVer135}, - {version: version136, fn: upgradeToVer136}, - {version: version137, fn: upgradeToVer137}, - {version: version138, fn: upgradeToVer138}, - {version: version139, fn: upgradeToVer139}, - {version: version140, fn: upgradeToVer140}, - {version: version141, fn: upgradeToVer141}, - {version: version142, fn: upgradeToVer142}, - {version: version143, fn: upgradeToVer143}, - {version: version144, fn: upgradeToVer144}, - // We will only use Ver145 to differentiate versions, so it is skipped here. - {version: version146, fn: upgradeToVer146}, - {version: version167, fn: upgradeToVer167}, - {version: version168, fn: upgradeToVer168}, - {version: version169, fn: upgradeToVer169}, - {version: version170, fn: upgradeToVer170}, - {version: version171, fn: upgradeToVer171}, - {version: version172, fn: upgradeToVer172}, - {version: version173, fn: upgradeToVer173}, - {version: version174, fn: upgradeToVer174}, - {version: version175, fn: upgradeToVer175}, - {version: version176, fn: upgradeToVer176}, - {version: version177, fn: upgradeToVer177}, - {version: version178, fn: upgradeToVer178}, - {version: version179, fn: upgradeToVer179}, - {version: version190, fn: upgradeToVer190}, - {version: version191, fn: upgradeToVer191}, - {version: version192, fn: upgradeToVer192}, - {version: version193, fn: upgradeToVer193}, - {version: version194, fn: upgradeToVer194}, - {version: version195, fn: upgradeToVer195}, - {version: version196, fn: upgradeToVer196}, - {version: version197, fn: upgradeToVer197}, - {version: version198, fn: upgradeToVer198}, - {version: version209, fn: upgradeToVer209}, - {version: version210, fn: upgradeToVer210}, - {version: version211, fn: upgradeToVer211}, - {version: version212, fn: upgradeToVer212}, - {version: version213, fn: upgradeToVer213}, - {version: version214, fn: upgradeToVer214}, - {version: version215, fn: upgradeToVer215}, - {version: version216, fn: upgradeToVer216}, - {version: version217, fn: upgradeToVer217}, - {version: version218, fn: upgradeToVer218}, - {version: version239, fn: upgradeToVer239}, - {version: version240, fn: upgradeToVer240}, - {version: version241, fn: upgradeToVer241}, - {version: version242, fn: upgradeToVer242}, - {version: version243, fn: upgradeToVer243}, - {version: version244, fn: upgradeToVer244}, - {version: version245, fn: upgradeToVer245}, - {version: version246, fn: upgradeToVer246}, - {version: version247, fn: upgradeToVer247}, - {version: version248, fn: upgradeToVer248}, - {version: version249, fn: upgradeToVer249}, - {version: version250, fn: upgradeToVer250}, - {version: version251, fn: upgradeToVer251}, - {version: version252, fn: upgradeToVer252}, - {version: version253, fn: upgradeToVer253}, - } -) - -// upgradeToVer2 updates to version 2. -func upgradeToVer2(s sessionapi.Session, _ int64) { - // Version 2 add two system variable for DistSQL concurrency controlling. - // Insert distsql related system variable. - distSQLVars := []string{vardef.TiDBDistSQLScanConcurrency} - values := make([]string, 0, len(distSQLVars)) - for _, v := range distSQLVars { - value := fmt.Sprintf(`("%s", "%s")`, v, variable.GetSysVar(v).Value) - values = append(values, value) - } - sql := fmt.Sprintf("INSERT HIGH_PRIORITY IGNORE INTO %s.%s VALUES %s;", mysql.SystemDB, mysql.GlobalVariablesTable, - strings.Join(values, ", ")) - mustExecute(s, sql) -} - -// upgradeToVer3 updates to version 3. -func upgradeToVer3(s sessionapi.Session, _ int64) { - // Version 3 fix tx_read_only variable value. - mustExecute(s, "UPDATE HIGH_PRIORITY %n.%n SET variable_value = '0' WHERE variable_name = 'tx_read_only';", mysql.SystemDB, mysql.GlobalVariablesTable) -} - -// upgradeToVer4 updates to version 4. -func upgradeToVer4(s sessionapi.Session, _ int64) { - mustExecute(s, CreateStatsMetaTable) -} - -func upgradeToVer5(s sessionapi.Session, _ int64) { - mustExecute(s, CreateStatsHistogramsTable) - mustExecute(s, CreateStatsBucketsTable) -} - -func upgradeToVer6(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Super_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Show_db_priv`", infoschema.ErrColumnExists) - // For reasons of compatibility, set the non-exists privilege column value to 'Y', as TiDB doesn't check them in older versions. - mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Super_priv='Y'") -} - -func upgradeToVer7(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Process_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Drop_priv`", infoschema.ErrColumnExists) - // For reasons of compatibility, set the non-exists privilege column value to 'Y', as TiDB doesn't check them in older versions. - mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Process_priv='Y'") -} - -func upgradeToVer8(s sessionapi.Session, ver int64) { - ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) - // This is a dummy upgrade, it checks whether upgradeToVer7 success, if not, do it again. - if _, err := s.ExecuteInternal(ctx, "SELECT HIGH_PRIORITY `Process_priv` FROM mysql.user LIMIT 0"); err == nil { - return - } - upgradeToVer7(s, ver) -} - -func upgradeToVer9(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Trigger_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Create_user_priv`", infoschema.ErrColumnExists) - // For reasons of compatibility, set the non-exists privilege column value to 'Y', as TiDB doesn't check them in older versions. - mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Trigger_priv='Y'") -} - -func doReentrantDDL(s sessionapi.Session, sql string, ignorableErrs ...error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(internalSQLTimeout)*time.Second) - ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnBootstrap) - _, err := s.ExecuteInternal(ctx, sql) - defer cancel() - for _, ignorableErr := range ignorableErrs { - if terror.ErrorEqual(err, ignorableErr) { - return - } - } - if err != nil { - logutil.BgLogger().Fatal("doReentrantDDL error", zap.Error(err)) - } -} - -func upgradeToVer10(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.stats_buckets CHANGE COLUMN `value` `upper_bound` BLOB NOT NULL", infoschema.ErrColumnNotExists, infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.stats_buckets ADD COLUMN `lower_bound` BLOB", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms ADD COLUMN `null_count` BIGINT(64) NOT NULL DEFAULT 0", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms DROP COLUMN distinct_ratio", dbterror.ErrCantDropFieldOrKey) - doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms DROP COLUMN use_count_to_estimate", dbterror.ErrCantDropFieldOrKey) -} - -func upgradeToVer11(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `References_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Grant_priv`", infoschema.ErrColumnExists) - mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET References_priv='Y'") -} - -func upgradeToVer12(s sessionapi.Session, _ int64) { - ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) - _, err := s.ExecuteInternal(ctx, "BEGIN") - terror.MustNil(err) - sql := "SELECT HIGH_PRIORITY user, host, password FROM mysql.user WHERE password != ''" - rs, err := s.ExecuteInternal(ctx, sql) - if terror.ErrorEqual(err, plannererrors.ErrUnknownColumn) { - sql := "SELECT HIGH_PRIORITY user, host, authentication_string FROM mysql.user WHERE authentication_string != ''" - rs, err = s.ExecuteInternal(ctx, sql) - } - terror.MustNil(err) - sqls := make([]string, 0, 1) - defer terror.Call(rs.Close) - req := rs.NewChunk(nil) - it := chunk.NewIterator4Chunk(req) - err = rs.Next(ctx, req) - for err == nil && req.NumRows() != 0 { - for row := it.Begin(); row != it.End(); row = it.Next() { - user := row.GetString(0) - host := row.GetString(1) - pass := row.GetString(2) - var newPass string - newPass, err = oldPasswordUpgrade(pass) - terror.MustNil(err) - updateSQL := fmt.Sprintf(`UPDATE HIGH_PRIORITY mysql.user SET password = "%s" WHERE user="%s" AND host="%s"`, newPass, user, host) - sqls = append(sqls, updateSQL) - } - err = rs.Next(ctx, req) - } - terror.MustNil(err) - - for _, sql := range sqls { - mustExecute(s, sql) - } - - sql = fmt.Sprintf(`INSERT HIGH_PRIORITY INTO %s.%s VALUES ("%s", "%d", "TiDB bootstrap version.") ON DUPLICATE KEY UPDATE VARIABLE_VALUE="%d"`, - mysql.SystemDB, mysql.TiDBTable, tidbServerVersionVar, version12, version12) - mustExecute(s, sql) - - mustExecute(s, "COMMIT") -} - -func upgradeToVer13(s sessionapi.Session, _ int64) { - sqls := []string{ - "ALTER TABLE mysql.user ADD COLUMN `Create_tmp_table_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Super_priv`", - "ALTER TABLE mysql.user ADD COLUMN `Lock_tables_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Create_tmp_table_priv`", - "ALTER TABLE mysql.user ADD COLUMN `Create_view_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Execute_priv`", - "ALTER TABLE mysql.user ADD COLUMN `Show_view_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Create_view_priv`", - "ALTER TABLE mysql.user ADD COLUMN `Create_routine_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Show_view_priv`", - "ALTER TABLE mysql.user ADD COLUMN `Alter_routine_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Create_routine_priv`", - "ALTER TABLE mysql.user ADD COLUMN `Event_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Create_user_priv`", - } - for _, sql := range sqls { - doReentrantDDL(s, sql, infoschema.ErrColumnExists) - } - mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Create_tmp_table_priv='Y',Lock_tables_priv='Y',Create_routine_priv='Y',Alter_routine_priv='Y',Event_priv='Y' WHERE Super_priv='Y'") - mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Create_view_priv='Y',Show_view_priv='Y' WHERE Create_priv='Y'") -} - -func upgradeToVer14(s sessionapi.Session, _ int64) { - sqls := []string{ - "ALTER TABLE mysql.db ADD COLUMN `References_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Grant_priv`", - "ALTER TABLE mysql.db ADD COLUMN `Create_tmp_table_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Alter_priv`", - "ALTER TABLE mysql.db ADD COLUMN `Lock_tables_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Create_tmp_table_priv`", - "ALTER TABLE mysql.db ADD COLUMN `Create_view_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Lock_tables_priv`", - "ALTER TABLE mysql.db ADD COLUMN `Show_view_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Create_view_priv`", - "ALTER TABLE mysql.db ADD COLUMN `Create_routine_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Show_view_priv`", - "ALTER TABLE mysql.db ADD COLUMN `Alter_routine_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Create_routine_priv`", - "ALTER TABLE mysql.db ADD COLUMN `Event_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Execute_priv`", - "ALTER TABLE mysql.db ADD COLUMN `Trigger_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Event_priv`", - } - for _, sql := range sqls { - doReentrantDDL(s, sql, infoschema.ErrColumnExists) - } -} - -func upgradeToVer15(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateGCDeleteRangeTable) -} - -func upgradeToVer16(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms ADD COLUMN `cm_sketch` BLOB", infoschema.ErrColumnExists) -} - -func upgradeToVer17(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user MODIFY User CHAR(32)") -} - -func upgradeToVer18(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms ADD COLUMN `tot_col_size` BIGINT(64) NOT NULL DEFAULT 0", infoschema.ErrColumnExists) -} - -func upgradeToVer19(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.db MODIFY User CHAR(32)") - doReentrantDDL(s, "ALTER TABLE mysql.tables_priv MODIFY User CHAR(32)") - doReentrantDDL(s, "ALTER TABLE mysql.columns_priv MODIFY User CHAR(32)") -} - -func upgradeToVer20(s sessionapi.Session, _ int64) { - // NOTE: Feedback is deprecated, but we still need to create this table for compatibility. - doReentrantDDL(s, CreateStatsFeedbackTable) -} - -func upgradeToVer21(s sessionapi.Session, _ int64) { - mustExecute(s, CreateGCDeleteRangeDoneTable) - - doReentrantDDL(s, "ALTER TABLE mysql.gc_delete_range DROP INDEX job_id", dbterror.ErrCantDropFieldOrKey) - doReentrantDDL(s, "ALTER TABLE mysql.gc_delete_range ADD UNIQUE INDEX delete_range_index (job_id, element_id)", dbterror.ErrDupKeyName) - doReentrantDDL(s, "ALTER TABLE mysql.gc_delete_range DROP INDEX element_id", dbterror.ErrCantDropFieldOrKey) -} - -func upgradeToVer22(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms ADD COLUMN `stats_ver` BIGINT(64) NOT NULL DEFAULT 0", infoschema.ErrColumnExists) -} - -func upgradeToVer23(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms ADD COLUMN `flag` BIGINT(64) NOT NULL DEFAULT 0", infoschema.ErrColumnExists) -} - -// writeSystemTZ writes system timezone info into mysql.tidb -func writeSystemTZ(s sessionapi.Session) { - mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, "TiDB Global System Timezone.") ON DUPLICATE KEY UPDATE VARIABLE_VALUE= %?`, - mysql.SystemDB, - mysql.TiDBTable, - tidbSystemTZ, - timeutil.InferSystemTZ(), - timeutil.InferSystemTZ(), - ) -} - -// upgradeToVer24 initializes `System` timezone according to docs/design/2018-09-10-adding-tz-env.md -func upgradeToVer24(s sessionapi.Session, _ int64) { - writeSystemTZ(s) -} - -// upgradeToVer25 updates tidb_max_chunk_size to new low bound value 32 if previous value is small than 32. -func upgradeToVer25(s sessionapi.Session, _ int64) { - sql := fmt.Sprintf("UPDATE HIGH_PRIORITY %[1]s.%[2]s SET VARIABLE_VALUE = '%[4]d' WHERE VARIABLE_NAME = '%[3]s' AND VARIABLE_VALUE < %[4]d", - mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBMaxChunkSize, vardef.DefInitChunkSize) - mustExecute(s, sql) -} - -func upgradeToVer26(s sessionapi.Session, _ int64) { - mustExecute(s, CreateRoleEdgesTable) - mustExecute(s, CreateDefaultRolesTable) - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Create_role_priv` ENUM('N','Y') DEFAULT 'N'", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Drop_role_priv` ENUM('N','Y') DEFAULT 'N'", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Account_locked` ENUM('N','Y') DEFAULT 'N'", infoschema.ErrColumnExists) - // user with Create_user_Priv privilege should have Create_view_priv and Show_view_priv after upgrade to v3.0 - mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Create_role_priv='Y',Drop_role_priv='Y' WHERE Create_user_priv='Y'") - // user with Create_Priv privilege should have Create_view_priv and Show_view_priv after upgrade to v3.0 - mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Create_view_priv='Y',Show_view_priv='Y' WHERE Create_priv='Y'") -} - -func upgradeToVer27(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms ADD COLUMN `correlation` DOUBLE NOT NULL DEFAULT 0", infoschema.ErrColumnExists) -} - -func upgradeToVer28(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateBindInfoTable) -} - -func upgradeToVer29(s sessionapi.Session, ver int64) { - // upgradeToVer29 only need to be run when the current version is 28. - if ver != version28 { - return - } - doReentrantDDL(s, "ALTER TABLE mysql.bind_info CHANGE create_time create_time TIMESTAMP(3)") - doReentrantDDL(s, "ALTER TABLE mysql.bind_info CHANGE update_time update_time TIMESTAMP(3)") - doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD INDEX sql_index (original_sql(1024),default_db(1024))", dbterror.ErrDupKeyName) -} - -func upgradeToVer30(s sessionapi.Session, _ int64) { - mustExecute(s, CreateStatsTopNTable) -} - -func upgradeToVer31(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms ADD COLUMN `last_analyze_pos` BLOB DEFAULT NULL", infoschema.ErrColumnExists) -} - -func upgradeToVer32(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.tables_priv MODIFY table_priv SET('Select','Insert','Update','Delete','Create','Drop','Grant', 'Index', 'Alter', 'Create View', 'Show View', 'Trigger', 'References')") -} - -func upgradeToVer33(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateExprPushdownBlacklistTable) -} - -func upgradeToVer34(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateOptRuleBlacklistTable) -} - -func upgradeToVer35(s sessionapi.Session, _ int64) { - sql := fmt.Sprintf("UPDATE HIGH_PRIORITY %s.%s SET VARIABLE_NAME = '%s' WHERE VARIABLE_NAME = 'tidb_back_off_weight'", - mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBBackOffWeight) - mustExecute(s, sql) -} - -func upgradeToVer36(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Shutdown_priv` ENUM('N','Y') DEFAULT 'N'", infoschema.ErrColumnExists) - // A root user will have those privileges after upgrading. - mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Shutdown_priv='Y' WHERE Super_priv='Y'") - mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Create_tmp_table_priv='Y',Lock_tables_priv='Y',Create_routine_priv='Y',Alter_routine_priv='Y',Event_priv='Y' WHERE Super_priv='Y'") -} - -func upgradeToVer37(s sessionapi.Session, _ int64) { - // when upgrade from old tidb and no 'tidb_enable_window_function' in GLOBAL_VARIABLES, init it with 0. - sql := fmt.Sprintf("INSERT IGNORE INTO %s.%s (`VARIABLE_NAME`, `VARIABLE_VALUE`) VALUES ('%s', '%d')", - mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBEnableWindowFunction, 0) - mustExecute(s, sql) -} - -func upgradeToVer38(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateGlobalPrivTable) -} - -func writeNewCollationParameter(s sessionapi.Session, flag bool) { - comment := "If the new collations are enabled. Do not edit it." - b := varFalse - if flag { - b = varTrue - } - mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, %?) ON DUPLICATE KEY UPDATE VARIABLE_VALUE=%?`, - mysql.SystemDB, mysql.TiDBTable, TidbNewCollationEnabled, b, comment, b, - ) -} - -func upgradeToVer40(s sessionapi.Session, _ int64) { - // There is no way to enable new collation for an existing TiDB cluster. - writeNewCollationParameter(s, false) -} - -func upgradeToVer41(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user CHANGE `password` `authentication_string` TEXT", infoschema.ErrColumnExists, infoschema.ErrColumnNotExists) - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `password` TEXT as (`authentication_string`)", infoschema.ErrColumnExists) -} - -// writeDefaultExprPushDownBlacklist writes default expr pushdown blacklist into mysql.expr_pushdown_blacklist -func writeDefaultExprPushDownBlacklist(s sessionapi.Session) { - mustExecute(s, "INSERT HIGH_PRIORITY INTO mysql.expr_pushdown_blacklist VALUES"+ - "('date_add','tiflash', 'DST(daylight saving time) does not take effect in TiFlash date_add')") -} - -func upgradeToVer42(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.expr_pushdown_blacklist ADD COLUMN `store_type` CHAR(100) NOT NULL DEFAULT 'tikv,tiflash,tidb'", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.expr_pushdown_blacklist ADD COLUMN `reason` VARCHAR(200)", infoschema.ErrColumnExists) - writeDefaultExprPushDownBlacklist(s) -} - -// Convert statement summary global variables to non-empty values. -func writeStmtSummaryVars(s sessionapi.Session) { - sql := "UPDATE %n.%n SET variable_value= %? WHERE variable_name= %? AND variable_value=''" - mustExecute(s, sql, mysql.SystemDB, mysql.GlobalVariablesTable, variable.BoolToOnOff(vardef.DefTiDBEnableStmtSummary), vardef.TiDBEnableStmtSummary) - mustExecute(s, sql, mysql.SystemDB, mysql.GlobalVariablesTable, variable.BoolToOnOff(vardef.DefTiDBStmtSummaryInternalQuery), vardef.TiDBStmtSummaryInternalQuery) - mustExecute(s, sql, mysql.SystemDB, mysql.GlobalVariablesTable, strconv.Itoa(vardef.DefTiDBStmtSummaryRefreshInterval), vardef.TiDBStmtSummaryRefreshInterval) - mustExecute(s, sql, mysql.SystemDB, mysql.GlobalVariablesTable, strconv.Itoa(vardef.DefTiDBStmtSummaryHistorySize), vardef.TiDBStmtSummaryHistorySize) - mustExecute(s, sql, mysql.SystemDB, mysql.GlobalVariablesTable, strconv.FormatUint(uint64(vardef.DefTiDBStmtSummaryMaxStmtCount), 10), vardef.TiDBStmtSummaryMaxStmtCount) - mustExecute(s, sql, mysql.SystemDB, mysql.GlobalVariablesTable, strconv.FormatUint(uint64(vardef.DefTiDBStmtSummaryMaxSQLLength), 10), vardef.TiDBStmtSummaryMaxSQLLength) -} - -func upgradeToVer43(s sessionapi.Session, _ int64) { - writeStmtSummaryVars(s) -} - -func upgradeToVer44(s sessionapi.Session, _ int64) { - mustExecute(s, "DELETE FROM mysql.global_variables where variable_name = \"tidb_isolation_read_engines\"") -} - -func upgradeToVer45(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Config_priv` ENUM('N','Y') DEFAULT 'N'", infoschema.ErrColumnExists) - mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Config_priv='Y' WHERE Super_priv='Y'") -} - -// In v3.1.1, we wrongly replace the context of upgradeToVer39 with upgradeToVer44. If we upgrade from v3.1.1 to a newer version, -// upgradeToVer39 will be missed. So we redo upgradeToVer39 here to make sure the upgrading from v3.1.1 succeed. -func upgradeToVer46(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Reload_priv` ENUM('N','Y') DEFAULT 'N'", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `File_priv` ENUM('N','Y') DEFAULT 'N'", infoschema.ErrColumnExists) - mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Reload_priv='Y' WHERE Super_priv='Y'") - mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET File_priv='Y' WHERE Super_priv='Y'") -} - -func upgradeToVer47(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD COLUMN `source` varchar(10) NOT NULL default 'unknown'", infoschema.ErrColumnExists) -} - -func upgradeToVer50(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateSchemaIndexUsageTable) -} - -func upgradeToVer52(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms MODIFY cm_sketch BLOB(6291456)") -} - -func upgradeToVer53(s sessionapi.Session, _ int64) { - // when upgrade from old tidb and no `tidb_enable_strict_double_type_check` in GLOBAL_VARIABLES, init it with 1` - sql := fmt.Sprintf("INSERT IGNORE INTO %s.%s (`VARIABLE_NAME`, `VARIABLE_VALUE`) VALUES ('%s', '%d')", - mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBEnableStrictDoubleTypeCheck, 0) - mustExecute(s, sql) -} - -func upgradeToVer54(s sessionapi.Session, ver int64) { - // The mem-query-quota default value is 32GB by default in v3.0, and 1GB by - // default in v4.0. - // If a cluster is upgraded from v3.0.x (bootstrapVer <= version38) to - // v4.0.9+, we'll write the default value to mysql.tidb. Thus we can get the - // default value of mem-quota-query, and promise the compatibility even if - // the tidb-server restarts. - // If it's a newly deployed cluster, we do not need to write the value into - // mysql.tidb, since no compatibility problem will happen. - - // This bootstrap task becomes obsolete in TiDB 5.0+, because it appears that the - // default value of mem-quota-query changes back to 1GB. In TiDB 6.1+ mem-quota-query - // is no longer a config option, but instead a system variable (tidb_mem_quota_query). - - if ver <= version38 { - writeMemoryQuotaQuery(s) - } -} - -// When cherry-pick upgradeToVer52 to v4.0, we wrongly name it upgradeToVer48. -// If we upgrade from v4.0 to a newer version, the real upgradeToVer48 will be missed. -// So we redo upgradeToVer48 here to make sure the upgrading from v4.0 succeeds. -func upgradeToVer55(s sessionapi.Session, _ int64) { - defValues := map[string]string{ - vardef.TiDBIndexLookupConcurrency: "4", - vardef.TiDBIndexLookupJoinConcurrency: "4", - vardef.TiDBHashAggFinalConcurrency: "4", - vardef.TiDBHashAggPartialConcurrency: "4", - vardef.TiDBWindowConcurrency: "4", - vardef.TiDBProjectionConcurrency: "4", - vardef.TiDBHashJoinConcurrency: "5", - } - names := make([]string, 0, len(defValues)) - for n := range defValues { - names = append(names, n) - } - - selectSQL := "select HIGH_PRIORITY * from mysql.global_variables where variable_name in ('" + strings.Join(names, quoteCommaQuote) + "')" - ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) - rs, err := s.ExecuteInternal(ctx, selectSQL) - terror.MustNil(err) - defer terror.Call(rs.Close) - req := rs.NewChunk(nil) - it := chunk.NewIterator4Chunk(req) - err = rs.Next(ctx, req) - for err == nil && req.NumRows() != 0 { - for row := it.Begin(); row != it.End(); row = it.Next() { - n := strings.ToLower(row.GetString(0)) - v := row.GetString(1) - if defValue, ok := defValues[n]; !ok || defValue != v { - return - } - } - err = rs.Next(ctx, req) - } - terror.MustNil(err) - - mustExecute(s, "BEGIN") - v := strconv.Itoa(vardef.ConcurrencyUnset) - sql := fmt.Sprintf("UPDATE %s.%s SET variable_value='%%s' WHERE variable_name='%%s'", mysql.SystemDB, mysql.GlobalVariablesTable) - for _, name := range names { - mustExecute(s, fmt.Sprintf(sql, v, name)) - } - mustExecute(s, "COMMIT") -} - -// When cherry-pick upgradeToVer54 to v4.0, we wrongly name it upgradeToVer49. -// If we upgrade from v4.0 to a newer version, the real upgradeToVer49 will be missed. -// So we redo upgradeToVer49 here to make sure the upgrading from v4.0 succeeds. -func upgradeToVer56(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateStatsExtendedTable) -} - -func upgradeToVer57(s sessionapi.Session, _ int64) { - insertBuiltinBindInfoRow(s) -} - -func insertBuiltinBindInfoRow(s sessionapi.Session) { - mustExecute(s, `INSERT HIGH_PRIORITY INTO mysql.bind_info(original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source) - VALUES (%?, %?, "mysql", %?, "0000-00-00 00:00:00", "0000-00-00 00:00:00", "", "", %?)`, - bindinfo.BuiltinPseudoSQL4BindLock, bindinfo.BuiltinPseudoSQL4BindLock, bindinfo.StatusBuiltin, bindinfo.StatusBuiltin, - ) -} - -func upgradeToVer59(s sessionapi.Session, _ int64) { - // The oom-action default value is log by default in v3.0, and cancel by - // default in v4.0.11+. - // If a cluster is upgraded from v3.0.x (bootstrapVer <= version59) to - // v4.0.11+, we'll write the default value to mysql.tidb. Thus we can get - // the default value of oom-action, and promise the compatibility even if - // the tidb-server restarts. - // If it's a newly deployed cluster, we do not need to write the value into - // mysql.tidb, since no compatibility problem will happen. - writeOOMAction(s) -} - -func upgradeToVer60(s sessionapi.Session, _ int64) { - mustExecute(s, "DROP TABLE IF EXISTS mysql.stats_extended") - doReentrantDDL(s, CreateStatsExtendedTable) -} - -type bindInfo struct { - bindSQL string - status string - createTime types.Time - charset string - collation string - source string -} - -func upgradeToVer67(s sessionapi.Session, _ int64) { - bindMap := make(map[string]bindInfo) - var err error - mustExecute(s, "BEGIN PESSIMISTIC") - - defer func() { - if err != nil { - mustExecute(s, "ROLLBACK") - return - } - - mustExecute(s, "COMMIT") - }() - mustExecute(s, bindinfo.LockBindInfoSQL) - ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) - var rs sqlexec.RecordSet - rs, err = s.ExecuteInternal(ctx, - `SELECT bind_sql, default_db, status, create_time, charset, collation, source - FROM mysql.bind_info - WHERE source != 'builtin' - ORDER BY update_time DESC`) - if err != nil { - logutil.BgLogger().Fatal("upgradeToVer67 error", zap.Error(err)) - } - req := rs.NewChunk(nil) - iter := chunk.NewIterator4Chunk(req) - p := parser.New() - now := types.NewTime(types.FromGoTime(time.Now()), mysql.TypeTimestamp, 3) - for { - err = rs.Next(context.TODO(), req) - if err != nil { - logutil.BgLogger().Fatal("upgradeToVer67 error", zap.Error(err)) - } - if req.NumRows() == 0 { - break - } - updateBindInfo(iter, p, bindMap) - } - terror.Call(rs.Close) - - mustExecute(s, "DELETE FROM mysql.bind_info where source != 'builtin'") - for original, bind := range bindMap { - mustExecute(s, fmt.Sprintf("INSERT INTO mysql.bind_info VALUES(%s, %s, '', %s, %s, %s, %s, %s, %s)", - expression.Quote(original), - expression.Quote(bind.bindSQL), - expression.Quote(bind.status), - expression.Quote(bind.createTime.String()), - expression.Quote(now.String()), - expression.Quote(bind.charset), - expression.Quote(bind.collation), - expression.Quote(bind.source), - )) - } -} - -func updateBindInfo(iter *chunk.Iterator4Chunk, p *parser.Parser, bindMap map[string]bindInfo) { - for row := iter.Begin(); row != iter.End(); row = iter.Next() { - bind := row.GetString(0) - db := row.GetString(1) - status := row.GetString(2) - - if status != bindinfo.StatusEnabled && status != bindinfo.StatusUsing && status != bindinfo.StatusBuiltin { - continue - } - - charset := row.GetString(4) - collation := row.GetString(5) - stmt, err := p.ParseOneStmt(bind, charset, collation) - if err != nil { - logutil.BgLogger().Fatal("updateBindInfo error", zap.Error(err)) - } - originWithDB := parser.Normalize(utilparser.RestoreWithDefaultDB(stmt, db, bind), "ON") - if _, ok := bindMap[originWithDB]; ok { - // The results are sorted in descending order of time. - // And in the following cases, duplicate originWithDB may occur - // originalText |bindText |DB - // `select * from t` |`select /*+ use_index(t, idx) */ * from t` |`test` - // `select * from test.t` |`select /*+ use_index(t, idx) */ * from test.t`|`` - // Therefore, if repeated, we can skip to keep the latest binding. - continue - } - bindMap[originWithDB] = bindInfo{ - bindSQL: utilparser.RestoreWithDefaultDB(stmt, db, bind), - status: status, - createTime: row.GetTime(3), - charset: charset, - collation: collation, - source: row.GetString(6), - } - } -} - -func writeMemoryQuotaQuery(s sessionapi.Session) { - comment := "memory_quota_query is 32GB by default in v3.0.x, 1GB by default in v4.0.x+" - mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, %?) ON DUPLICATE KEY UPDATE VARIABLE_VALUE=%?`, - mysql.SystemDB, mysql.TiDBTable, tidbDefMemoryQuotaQuery, 32<<30, comment, 32<<30, - ) -} - -func upgradeToVer62(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.stats_buckets ADD COLUMN `ndv` bigint not null default 0", infoschema.ErrColumnExists) -} - -func upgradeToVer63(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Create_tablespace_priv` ENUM('N','Y') DEFAULT 'N'", infoschema.ErrColumnExists) - mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Create_tablespace_priv='Y' where Super_priv='Y'") -} - -func upgradeToVer64(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Repl_slave_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Execute_priv`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN `Repl_client_priv` ENUM('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' AFTER `Repl_slave_priv`", infoschema.ErrColumnExists) - mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET Repl_slave_priv='Y',Repl_client_priv='Y' where Super_priv='Y'") -} - -func upgradeToVer65(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateStatsFMSketchTable) -} - -func upgradeToVer66(s sessionapi.Session, _ int64) { - mustExecute(s, "set @@global.tidb_track_aggregate_memory_usage = 1") -} - -func upgradeToVer68(s sessionapi.Session, _ int64) { - mustExecute(s, "DELETE FROM mysql.global_variables where VARIABLE_NAME = 'tidb_enable_clustered_index' and VARIABLE_VALUE = 'OFF'") -} - -func upgradeToVer69(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateGlobalGrantsTable) -} - -func upgradeToVer70(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN plugin CHAR(64) AFTER authentication_string", infoschema.ErrColumnExists) - mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET plugin='mysql_native_password'") -} - -func upgradeToVer71(s sessionapi.Session, _ int64) { - mustExecute(s, "UPDATE mysql.global_variables SET VARIABLE_VALUE='OFF' WHERE VARIABLE_NAME = 'tidb_multi_statement_mode' AND VARIABLE_VALUE = 'WARN'") -} - -func upgradeToVer72(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.stats_meta ADD COLUMN snapshot BIGINT(64) UNSIGNED NOT NULL DEFAULT 0", infoschema.ErrColumnExists) -} - -func upgradeToVer73(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateCapturePlanBaselinesBlacklistTable) -} - -func upgradeToVer74(s sessionapi.Session, _ int64) { - // The old default value of `tidb_stmt_summary_max_stmt_count` is 200, we want to enlarge this to the new default value when TiDB upgrade. - mustExecute(s, fmt.Sprintf("UPDATE mysql.global_variables SET VARIABLE_VALUE='%[1]v' WHERE VARIABLE_NAME = 'tidb_stmt_summary_max_stmt_count' AND CAST(VARIABLE_VALUE AS SIGNED) = 200", vardef.DefTiDBStmtSummaryMaxStmtCount)) -} - -func upgradeToVer75(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user MODIFY COLUMN Host CHAR(255)") - doReentrantDDL(s, "ALTER TABLE mysql.global_priv MODIFY COLUMN Host CHAR(255)") - doReentrantDDL(s, "ALTER TABLE mysql.db MODIFY COLUMN Host CHAR(255)") - doReentrantDDL(s, "ALTER TABLE mysql.tables_priv MODIFY COLUMN Host CHAR(255)") - doReentrantDDL(s, "ALTER TABLE mysql.columns_priv MODIFY COLUMN Host CHAR(255)") -} - -func upgradeToVer76(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.columns_priv MODIFY COLUMN Column_priv SET('Select','Insert','Update','References')") -} - -func upgradeToVer77(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateColumnStatsUsageTable) -} - -func upgradeToVer78(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.stats_buckets MODIFY upper_bound LONGBLOB NOT NULL") - doReentrantDDL(s, "ALTER TABLE mysql.stats_buckets MODIFY lower_bound LONGBLOB") - doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms MODIFY last_analyze_pos LONGBLOB DEFAULT NULL") -} - -func upgradeToVer79(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateTableCacheMetaTable) -} - -func upgradeToVer80(s sessionapi.Session, _ int64) { - // Check if tidb_analyze_version exists in mysql.GLOBAL_VARIABLES. - // If not, insert "tidb_analyze_version | 1" since this is the old behavior before we introduce this variable. - initGlobalVariableIfNotExists(s, vardef.TiDBAnalyzeVersion, 1) -} - -// For users that upgrade TiDB from a pre-4.0 version, we want to disable index merge by default. -// This helps minimize query plan regressions. -func upgradeToVer81(s sessionapi.Session, _ int64) { - // Check if tidb_enable_index_merge exists in mysql.GLOBAL_VARIABLES. - // If not, insert "tidb_enable_index_merge | off". - initGlobalVariableIfNotExists(s, vardef.TiDBEnableIndexMerge, vardef.Off) -} - -func upgradeToVer82(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateAnalyzeOptionsTable) -} - -func upgradeToVer83(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateStatsHistoryTable) -} - -func upgradeToVer84(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateStatsMetaHistoryTable) -} - -func upgradeToVer85(s sessionapi.Session, _ int64) { - mustExecute(s, fmt.Sprintf("UPDATE HIGH_PRIORITY mysql.bind_info SET status= '%s' WHERE status = '%s'", bindinfo.StatusEnabled, bindinfo.StatusUsing)) -} - -func upgradeToVer86(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.tables_priv MODIFY COLUMN Column_priv SET('Select','Insert','Update','References')") -} - -func upgradeToVer87(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateAnalyzeJobsTable) -} - -func upgradeToVer88(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user CHANGE `Repl_slave_priv` `Repl_slave_priv` ENUM('N','Y') NOT NULL DEFAULT 'N' AFTER `Execute_priv`") - doReentrantDDL(s, "ALTER TABLE mysql.user CHANGE `Repl_client_priv` `Repl_client_priv` ENUM('N','Y') NOT NULL DEFAULT 'N' AFTER `Repl_slave_priv`") -} - -func upgradeToVer89(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateAdvisoryLocksTable) -} - -// importConfigOption is a one-time import. -// It is intended to be used to convert a config option to a sysvar. -// It reads the config value from the tidb-server executing the bootstrap -// (not guaranteed to be the same on all servers), and writes a message -// to the error log. The message is important since the behavior is weird -// (changes to the config file will no longer take effect past this point). -func importConfigOption(s sessionapi.Session, configName, svName, valStr string) { - message := fmt.Sprintf("%s is now configured by the system variable %s. One-time importing the value specified in tidb.toml file", configName, svName) - logutil.BgLogger().Warn(message, zap.String("value", valStr)) - // We use insert ignore, since if its a duplicate we don't want to overwrite any user-set values. - sql := fmt.Sprintf("INSERT IGNORE INTO %s.%s (`VARIABLE_NAME`, `VARIABLE_VALUE`) VALUES ('%s', '%s')", - mysql.SystemDB, mysql.GlobalVariablesTable, svName, valStr) - mustExecute(s, sql) -} - -func upgradeToVer90(s sessionapi.Session, _ int64) { - valStr := variable.BoolToOnOff(config.GetGlobalConfig().EnableBatchDML) - importConfigOption(s, "enable-batch-dml", vardef.TiDBEnableBatchDML, valStr) - valStr = fmt.Sprint(config.GetGlobalConfig().MemQuotaQuery) - importConfigOption(s, "mem-quota-query", vardef.TiDBMemQuotaQuery, valStr) - valStr = fmt.Sprint(config.GetGlobalConfig().Log.QueryLogMaxLen) - importConfigOption(s, "query-log-max-len", vardef.TiDBQueryLogMaxLen, valStr) - valStr = fmt.Sprint(config.GetGlobalConfig().Performance.CommitterConcurrency) - importConfigOption(s, "committer-concurrency", vardef.TiDBCommitterConcurrency, valStr) - valStr = variable.BoolToOnOff(config.GetGlobalConfig().Performance.RunAutoAnalyze) - importConfigOption(s, "run-auto-analyze", vardef.TiDBEnableAutoAnalyze, valStr) - valStr = config.GetGlobalConfig().OOMAction - importConfigOption(s, "oom-action", vardef.TiDBMemOOMAction, valStr) -} - -func upgradeToVer91(s sessionapi.Session, _ int64) { - valStr := variable.BoolToOnOff(config.GetGlobalConfig().PreparedPlanCache.Enabled) - importConfigOption(s, "prepared-plan-cache.enable", vardef.TiDBEnablePrepPlanCache, valStr) - - valStr = strconv.Itoa(int(config.GetGlobalConfig().PreparedPlanCache.Capacity)) - importConfigOption(s, "prepared-plan-cache.capacity", vardef.TiDBPrepPlanCacheSize, valStr) - - valStr = strconv.FormatFloat(config.GetGlobalConfig().PreparedPlanCache.MemoryGuardRatio, 'f', -1, 64) - importConfigOption(s, "prepared-plan-cache.memory-guard-ratio", vardef.TiDBPrepPlanCacheMemoryGuardRatio, valStr) -} - -func upgradeToVer93(s sessionapi.Session, _ int64) { - valStr := variable.BoolToOnOff(config.GetGlobalConfig().OOMUseTmpStorage) - importConfigOption(s, "oom-use-tmp-storage", vardef.TiDBEnableTmpStorageOnOOM, valStr) -} - -func upgradeToVer94(s sessionapi.Session, _ int64) { - mustExecute(s, CreateTiDBMDLView) -} - -func upgradeToVer95(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN IF NOT EXISTS `User_attributes` JSON") -} - -func upgradeToVer97(s sessionapi.Session, _ int64) { - // Check if tidb_opt_range_max_size exists in mysql.GLOBAL_VARIABLES. - // If not, insert "tidb_opt_range_max_size | 0" since this is the old behavior before we introduce this variable. - initGlobalVariableIfNotExists(s, vardef.TiDBOptRangeMaxSize, 0) -} - -func upgradeToVer98(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN IF NOT EXISTS `Token_issuer` varchar(255)") -} - -func upgradeToVer99Before(s sessionapi.Session) { - mustExecute(s, "INSERT HIGH_PRIORITY IGNORE INTO %n.%n VALUES (%?, %?);", - mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBEnableMDL, 0) -} - -func upgradeToVer99After(s sessionapi.Session) { - sql := fmt.Sprintf("UPDATE HIGH_PRIORITY %[1]s.%[2]s SET VARIABLE_VALUE = %[4]d WHERE VARIABLE_NAME = '%[3]s'", - mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBEnableMDL, 1) - mustExecute(s, sql) - err := kv.RunInNewTxn(kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL), s.GetStore(), true, func(_ context.Context, txn kv.Transaction) error { - t := meta.NewMutator(txn) - return t.SetMetadataLock(true) - }) - terror.MustNil(err) -} - -func upgradeToVer100(s sessionapi.Session, _ int64) { - valStr := strconv.Itoa(int(config.GetGlobalConfig().Performance.ServerMemoryQuota)) - importConfigOption(s, "performance.server-memory-quota", vardef.TiDBServerMemoryLimit, valStr) -} - -func upgradeToVer101(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreatePlanReplayerStatusTable) -} - -func upgradeToVer102(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreatePlanReplayerTaskTable) -} - -func upgradeToVer103(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateStatsTableLockedTable) -} - -func upgradeToVer104(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD COLUMN IF NOT EXISTS `sql_digest` varchar(64)") - doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD COLUMN IF NOT EXISTS `plan_digest` varchar(64)") -} - -// For users that upgrade TiDB from a pre-6.0 version, we want to disable tidb cost model2 by default to keep plans unchanged. -func upgradeToVer105(s sessionapi.Session, _ int64) { - initGlobalVariableIfNotExists(s, vardef.TiDBCostModelVersion, "1") -} - -func upgradeToVer106(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreatePasswordHistoryTable) - doReentrantDDL(s, "Alter table mysql.user add COLUMN IF NOT EXISTS `Password_reuse_history` smallint unsigned DEFAULT NULL AFTER `Create_Tablespace_Priv` ") - doReentrantDDL(s, "Alter table mysql.user add COLUMN IF NOT EXISTS `Password_reuse_time` smallint unsigned DEFAULT NULL AFTER `Password_reuse_history`") -} - -func upgradeToVer107(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN IF NOT EXISTS `Password_expired` ENUM('N','Y') NOT NULL DEFAULT 'N'") - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN IF NOT EXISTS `Password_last_changed` TIMESTAMP DEFAULT CURRENT_TIMESTAMP()") - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN IF NOT EXISTS `Password_lifetime` SMALLINT UNSIGNED DEFAULT NULL") -} - -func upgradeToVer108(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateTiDBTTLTableStatusTable) -} - -// For users that upgrade TiDB from a 6.2-6.4 version, we want to disable tidb gc_aware_memory_track by default. -func upgradeToVer109(s sessionapi.Session, _ int64) { - mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", - mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBEnableGCAwareMemoryTrack, 0) -} - -// For users that upgrade TiDB from a 5.4-6.4 version, we want to enable tidb tidb_stats_load_pseudo_timeout by default. -func upgradeToVer110(s sessionapi.Session, _ int64) { - mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", - mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBStatsLoadPseudoTimeout, 1) -} - -func upgradeToVer130(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.stats_meta_history ADD COLUMN IF NOT EXISTS `source` varchar(40) NOT NULL after `version`;") -} - -func upgradeToVer131(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateTiDBTTLTaskTable) - doReentrantDDL(s, CreateTiDBTTLJobHistoryTable) -} - -func upgradeToVer132(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateTiDBMDLView) -} - -func upgradeToVer133(s sessionapi.Session, _ int64) { - mustExecute(s, "UPDATE HIGH_PRIORITY %n.%n set VARIABLE_VALUE = %? where VARIABLE_NAME = %? and VARIABLE_VALUE = %?;", - mysql.SystemDB, mysql.GlobalVariablesTable, vardef.DefTiDBServerMemoryLimit, vardef.TiDBServerMemoryLimit, "0") -} - -func upgradeToVer134(s sessionapi.Session, _ int64) { - mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", mysql.SystemDB, mysql.GlobalVariablesTable, vardef.ForeignKeyChecks, vardef.On) - mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBEnableForeignKey, vardef.On) - mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBEnableHistoricalStats, vardef.On) - mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBEnablePlanReplayerCapture, vardef.On) - mustExecute(s, "UPDATE HIGH_PRIORITY %n.%n SET VARIABLE_VALUE = %? WHERE VARIABLE_NAME = %? AND VARIABLE_VALUE = %?;", mysql.SystemDB, mysql.GlobalVariablesTable, "4", vardef.TiDBStoreBatchSize, "0") -} - -// For users that upgrade TiDB from a pre-7.0 version, we want to set tidb_opt_advanced_join_hint to off by default to keep plans unchanged. -func upgradeToVer135(s sessionapi.Session, _ int64) { - initGlobalVariableIfNotExists(s, vardef.TiDBOptAdvancedJoinHint, false) -} - -func upgradeToVer136(s sessionapi.Session, _ int64) { - mustExecute(s, CreateTiDBGlobalTaskTable) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask DROP INDEX namespace", dbterror.ErrCantDropFieldOrKey) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD INDEX idx_task_key(task_key)", dbterror.ErrDupKeyName) -} - -func upgradeToVer137(_ sessionapi.Session, _ int64) { - // NOOP, we don't depend on ddl to init the default group due to backward compatible issue. -} - -// For users that upgrade TiDB from a version below 7.0, we want to enable tidb tidb_enable_null_aware_anti_join by default. -func upgradeToVer138(s sessionapi.Session, _ int64) { - mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBOptimizerEnableNAAJ, vardef.On) -} - -func upgradeToVer139(sessionapi.Session, int64) {} - -func upgradeToVer140(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN `task_key` VARCHAR(256) NOT NULL AFTER `id`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD UNIQUE KEY task_key(task_key)", dbterror.ErrDupKeyName) -} - -// upgradeToVer141 sets the value of `tidb_session_plan_cache_size` as `tidb_prepared_plan_cache_size` for compatibility, -// and update tidb_load_based_replica_read_threshold from 0 to 4. -func upgradeToVer141(s sessionapi.Session, _ int64) { - ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) - rs, err := s.ExecuteInternal(ctx, "SELECT VARIABLE_VALUE FROM %n.%n WHERE VARIABLE_NAME=%?;", - mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBPrepPlanCacheSize) - terror.MustNil(err) - req := rs.NewChunk(nil) - err = rs.Next(ctx, req) - if err != nil || req.NumRows() == 0 { - return - } - row := req.GetRow(0) - if row.IsNull(0) { - return - } - val := row.GetString(0) - - mustExecute(s, "INSERT HIGH_PRIORITY IGNORE INTO %n.%n VALUES (%?, %?);", - mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBSessionPlanCacheSize, val) - mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", mysql.SystemDB, mysql.GlobalVariablesTable, vardef.TiDBLoadBasedReplicaReadThreshold, vardef.DefTiDBLoadBasedReplicaReadThreshold.String()) -} - -func upgradeToVer142(s sessionapi.Session, _ int64) { - initGlobalVariableIfNotExists(s, vardef.TiDBEnableNonPreparedPlanCache, vardef.Off) -} - -func upgradeToVer143(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN `error` BLOB", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD COLUMN `error` BLOB", infoschema.ErrColumnExists) -} - -func upgradeToVer144(s sessionapi.Session, _ int64) { - initGlobalVariableIfNotExists(s, vardef.TiDBPlanCacheInvalidationOnFreshStats, vardef.Off) -} - -func upgradeToVer146(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.stats_meta_history ADD INDEX idx_create_time (create_time)", dbterror.ErrDupKeyName) - doReentrantDDL(s, "ALTER TABLE mysql.stats_history ADD INDEX idx_create_time (create_time)", dbterror.ErrDupKeyName) -} - -func upgradeToVer167(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD COLUMN `step` INT AFTER `id`", infoschema.ErrColumnExists) -} - -func upgradeToVer168(s sessionapi.Session, _ int64) { - mustExecute(s, CreateTiDBImportJobsTable) -} - -func upgradeToVer169(s sessionapi.Session, _ int64) { - mustExecute(s, CreateTiDBRunawayQueriesTable) -} - -func upgradeToVer170(s sessionapi.Session, _ int64) { - mustExecute(s, CreateTiDBTimersTable) -} - -func upgradeToVer171(s sessionapi.Session, _ int64) { - mustExecute(s, "ALTER TABLE mysql.tidb_runaway_queries CHANGE COLUMN `tidb_server` `tidb_server` varchar(512)") -} - -func upgradeToVer172(s sessionapi.Session, _ int64) { - mustExecute(s, "DROP TABLE IF EXISTS mysql.tidb_runaway_quarantined_watch") - mustExecute(s, CreateTiDBRunawayWatchTable) - mustExecute(s, CreateTiDBRunawayWatchDoneTable) -} - -func upgradeToVer173(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD COLUMN `summary` JSON", infoschema.ErrColumnExists) -} - -func upgradeToVer174(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD COLUMN `step` INT AFTER `id`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD COLUMN `error` BLOB", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history DROP INDEX `namespace`", dbterror.ErrCantDropFieldOrKey) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD INDEX `idx_task_key`(`task_key`)", dbterror.ErrDupKeyName) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD INDEX `idx_state_update_time`(`state_update_time`)", dbterror.ErrDupKeyName) -} - -// upgradeToVer175 updates normalized bindings of `in (?)` to `in (...)` to solve -// the issue #44298 that bindings for `in (?)` can't work for `in (?, ?, ?)`. -// After this update, multiple bindings may have the same `original_sql`, but it's OK, and -// for safety, don't remove duplicated bindings when upgrading. -func upgradeToVer175(s sessionapi.Session, _ int64) { - var err error - mustExecute(s, "BEGIN PESSIMISTIC") - defer func() { - if err != nil { - mustExecute(s, "ROLLBACK") - return - } - mustExecute(s, "COMMIT") - }() - ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) - rs, err := s.ExecuteInternal(ctx, "SELECT original_sql, bind_sql FROM mysql.bind_info WHERE source != 'builtin'") - if err != nil { - logutil.BgLogger().Fatal("upgradeToVer175 error", zap.Error(err)) - return - } - req := rs.NewChunk(nil) - updateStmts := make([]string, 0, 4) - for { - err = rs.Next(ctx, req) - if err != nil { - logutil.BgLogger().Fatal("upgradeToVer175 error", zap.Error(err)) - return - } - if req.NumRows() == 0 { - break - } - for i := range req.NumRows() { - originalNormalizedSQL, bindSQL := req.GetRow(i).GetString(0), req.GetRow(i).GetString(1) - newNormalizedSQL := parser.NormalizeForBinding(bindSQL, false) - // update `in (?)` to `in (...)` - if originalNormalizedSQL == newNormalizedSQL { - continue // no need to update - } - // must run those update statements outside this loop, otherwise may cause some concurrency problems, - // since the current statement over this session has not been finished yet. - updateStmts = append(updateStmts, fmt.Sprintf("UPDATE mysql.bind_info SET original_sql='%s' WHERE original_sql='%s'", newNormalizedSQL, originalNormalizedSQL)) - } - req.Reset() - } - if err := rs.Close(); err != nil { - logutil.BgLogger().Fatal("upgradeToVer175 error", zap.Error(err)) - } - for _, updateStmt := range updateStmts { - mustExecute(s, updateStmt) - } -} - -func upgradeToVer176(s sessionapi.Session, _ int64) { - mustExecute(s, CreateTiDBGlobalTaskHistoryTable) -} - -func upgradeToVer177(s sessionapi.Session, _ int64) { - // ignore error when upgrading from v7.4 to higher version. - doReentrantDDL(s, CreateDistFrameworkMetaTable, infoschema.ErrTableExists) - err := s.GetSessionVars().GlobalVarsAccessor.SetGlobalSysVar(context.Background(), vardef.TiDBEnableAsyncMergeGlobalStats, vardef.Off) - if err != nil { - logutil.BgLogger().Fatal("upgradeToVer177 error", zap.Error(err)) - } -} - -// writeDDLTableVersion writes mDDLTableVersion into mysql.tidb -func writeDDLTableVersion(s sessionapi.Session) { - var err error - var ddlTableVersion meta.DDLTableVersion - err = kv.RunInNewTxn(kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap), s.GetStore(), true, func(_ context.Context, txn kv.Transaction) error { - t := meta.NewMutator(txn) - ddlTableVersion, err = t.GetDDLTableVersion() - return err - }) - terror.MustNil(err) - mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, "DDL Table Version. Do not delete.") ON DUPLICATE KEY UPDATE VARIABLE_VALUE= %?`, - mysql.SystemDB, - mysql.TiDBTable, - tidbDDLTableVersion, - ddlTableVersion, - ddlTableVersion, - ) -} - -func upgradeToVer178(s sessionapi.Session, _ int64) { - writeDDLTableVersion(s) -} - -func upgradeToVer179(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.global_variables MODIFY COLUMN `VARIABLE_VALUE` varchar(16383)") -} - -func upgradeToVer190(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN `priority` INT DEFAULT 1 AFTER `state`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN `create_time` TIMESTAMP AFTER `priority`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN `end_time` TIMESTAMP AFTER `state_update_time`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN `priority` INT DEFAULT 1 AFTER `state`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN `create_time` TIMESTAMP AFTER `priority`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN `end_time` TIMESTAMP AFTER `state_update_time`", infoschema.ErrColumnExists) - - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD COLUMN `concurrency` INT AFTER `checkpoint`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD COLUMN `create_time` TIMESTAMP AFTER `concurrency`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD COLUMN `end_time` TIMESTAMP AFTER `state_update_time`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD COLUMN `ordinal` int AFTER `meta`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD COLUMN `concurrency` INT AFTER `checkpoint`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD COLUMN `create_time` TIMESTAMP AFTER `concurrency`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD COLUMN `end_time` TIMESTAMP AFTER `state_update_time`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD COLUMN `ordinal` int AFTER `meta`", infoschema.ErrColumnExists) - - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD INDEX idx_exec_id(exec_id)", dbterror.ErrDupKeyName) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD UNIQUE INDEX uk_task_key_step_ordinal(task_key, step, ordinal)", dbterror.ErrDupKeyName) - - doReentrantDDL(s, "ALTER TABLE mysql.dist_framework_meta ADD COLUMN `cpu_count` INT DEFAULT 0 AFTER `role`", infoschema.ErrColumnExists) - - doReentrantDDL(s, "ALTER TABLE mysql.dist_framework_meta MODIFY COLUMN `host` VARCHAR(261)") - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask MODIFY COLUMN `exec_id` VARCHAR(261)") - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history MODIFY COLUMN `exec_id` VARCHAR(261)") - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task MODIFY COLUMN `dispatcher_id` VARCHAR(261)") - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history MODIFY COLUMN `dispatcher_id` VARCHAR(261)") -} - -func upgradeToVer191(s sessionapi.Session, _ int64) { - sql := fmt.Sprintf("INSERT HIGH_PRIORITY IGNORE INTO %s.%s VALUES('%s', '%s')", - mysql.SystemDB, mysql.GlobalVariablesTable, - vardef.TiDBTxnMode, vardef.OptimisticTxnMode) - mustExecute(s, sql) -} - -func upgradeToVer192(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateRequestUnitByGroupTable) -} - -func upgradeToVer193(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateTiDBMDLView) -} - -func upgradeToVer194(s sessionapi.Session, _ int64) { - mustExecute(s, "DROP TABLE IF EXISTS mysql.load_data_jobs") -} - -func upgradeToVer195(s sessionapi.Session, _ int64) { - doReentrantDDL(s, DropMySQLIndexUsageTable) -} - -func upgradeToVer196(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN target_scope VARCHAR(256) DEFAULT '' AFTER `step`;", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN target_scope VARCHAR(256) DEFAULT '' AFTER `step`;", infoschema.ErrColumnExists) -} - -func upgradeToVer197(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateTiDBMDLView) -} - -func upgradeToVer198(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.tidb_mdl_info ADD COLUMN owner_id VARCHAR(64) NOT NULL DEFAULT '';", infoschema.ErrColumnExists) -} - -func upgradeToVer209(s sessionapi.Session, _ int64) { - initGlobalVariableIfNotExists(s, vardef.TiDBResourceControlStrictMode, vardef.Off) -} - -func upgradeToVer210(s sessionapi.Session, _ int64) { - // Check if tidb_analyze_column_options exists in mysql.GLOBAL_VARIABLES. - // If not, set tidb_analyze_column_options to ALL since this is the old behavior before we introduce this variable. - initGlobalVariableIfNotExists(s, vardef.TiDBAnalyzeColumnOptions, ast.AllColumns.String()) - - // Check if tidb_opt_projection_push_down exists in mysql.GLOBAL_VARIABLES. - // If not, set tidb_opt_projection_push_down to Off since this is the old behavior before we introduce this variable. - initGlobalVariableIfNotExists(s, vardef.TiDBOptProjectionPushDown, vardef.Off) -} - -func upgradeToVer211(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history ADD COLUMN `summary` JSON", infoschema.ErrColumnExists) -} - -func upgradeToVer212(s sessionapi.Session, ver int64) { - // need to ensure curVersion has the column before rename. - // version169 created `tidb_runaway_queries` table - // version172 created `tidb_runaway_watch` and `tidb_runaway_watch_done` tables - if ver < version172 { - return - } - // version212 changed a lots of runaway related table. - // 1. switchGroup: add column `switch_group_name` to `mysql.tidb_runaway_watch` and `mysql.tidb_runaway_watch_done`. - doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_watch ADD COLUMN `switch_group_name` VARCHAR(32) DEFAULT '' AFTER `action`;", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_watch_done ADD COLUMN `switch_group_name` VARCHAR(32) DEFAULT '' AFTER `action`;", infoschema.ErrColumnExists) - // 2. modify column `plan_digest` type, modify column `time` to `start_time, - // modify column `original_sql` to `sample_sql` and unique union key to `mysql.tidb_runaway_queries`. - // add column `sql_digest`. - doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries ADD COLUMN `sql_digest` varchar(64) DEFAULT '' AFTER `original_sql`;", infoschema.ErrColumnExists) - // add column `repeats`. - doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries ADD COLUMN `repeats` int DEFAULT 1 AFTER `time`;", infoschema.ErrColumnExists) - // rename column name from `time` to `start_time`, will auto rebuild the index. - doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries RENAME COLUMN `time` TO `start_time`", infoschema.ErrColumnNotExists) - // rename column `original_sql` to `sample_sql`. - doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries RENAME COLUMN `original_sql` TO `sample_sql`", infoschema.ErrColumnNotExists) - // modify column type of `plan_digest`. - doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries MODIFY COLUMN `plan_digest` varchar(64) DEFAULT '';", infoschema.ErrColumnExists) - // 3. modify column length of `action`. - doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries MODIFY COLUMN `action` VARCHAR(64) NOT NULL;", infoschema.ErrColumnExists) - // 4. add column `rule` to `mysql.tidb_runaway_watch`, `mysql.tidb_runaway_watch_done` and `mysql.tidb_runaway_queries`. - doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_watch ADD COLUMN `rule` VARCHAR(512) DEFAULT '' AFTER `switch_group_name`;", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_watch_done ADD COLUMN `rule` VARCHAR(512) DEFAULT '' AFTER `switch_group_name`;", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries ADD COLUMN `rule` VARCHAR(512) DEFAULT '' AFTER `tidb_server`;", infoschema.ErrColumnExists) -} - -func upgradeToVer213(s sessionapi.Session, _ int64) { - mustExecute(s, CreateTiDBPITRIDMapTable) -} - -func upgradeToVer214(s sessionapi.Session, _ int64) { - mustExecute(s, CreateIndexAdvisorResultsTable) - mustExecute(s, CreateTiDBKernelOptionsTable) -} - -func upgradeToVer215(s sessionapi.Session, _ int64) { - initGlobalVariableIfNotExists(s, vardef.TiDBEnableINLJoinInnerMultiPattern, vardef.Off) -} - -func upgradeToVer216(s sessionapi.Session, _ int64) { - mustExecute(s, "UPDATE mysql.global_variables SET VARIABLE_VALUE='' WHERE VARIABLE_NAME = 'tidb_scatter_region' AND VARIABLE_VALUE = 'OFF'") - mustExecute(s, "UPDATE mysql.global_variables SET VARIABLE_VALUE='table' WHERE VARIABLE_NAME = 'tidb_scatter_region' AND VARIABLE_VALUE = 'ON'") -} - -func upgradeToVer217(s sessionapi.Session, _ int64) { - // If tidb_schema_cache_size does not exist, insert a record and set the value to 0 - // Otherwise do nothing. - mustExecute(s, "INSERT IGNORE INTO mysql.global_variables VALUES ('tidb_schema_cache_size', 0)") -} - -func upgradeToVer218(_ sessionapi.Session, _ int64) { - // empty, just make lint happy. -} - -func upgradeToVer239(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN modify_params json AFTER `error`;", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN modify_params json AFTER `error`;", infoschema.ErrColumnExists) -} - -const ( - // addAnalyzeJobsSchemaTableStateIndex is a DDL statement that adds an index on (table_schema, table_name, state) - // columns to mysql.analyze_jobs table. This index is currently unused since queries filter on partition_name='', - // even for non-partitioned tables. It is kept for potential future optimization where queries could use this - // simpler index directly for non-partitioned tables. - addAnalyzeJobsSchemaTableStateIndex = "ALTER TABLE mysql.analyze_jobs ADD INDEX idx_schema_table_state (table_schema, table_name, state)" - // addAnalyzeJobsSchemaTablePartitionStateIndex adds an index on (table_schema, table_name, partition_name, state) to mysql.analyze_jobs - addAnalyzeJobsSchemaTablePartitionStateIndex = "ALTER TABLE mysql.analyze_jobs ADD INDEX idx_schema_table_partition_state (table_schema, table_name, partition_name, state)" -) - -func upgradeToVer240(s sessionapi.Session, _ int64) { - doReentrantDDL(s, addAnalyzeJobsSchemaTableStateIndex, dbterror.ErrDupKeyName) - doReentrantDDL(s, addAnalyzeJobsSchemaTablePartitionStateIndex, dbterror.ErrDupKeyName) -} - -func upgradeToVer241(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user ADD INDEX i_user (user)", dbterror.ErrDupKeyName) - doReentrantDDL(s, "ALTER TABLE mysql.global_priv ADD INDEX i_user (user)", dbterror.ErrDupKeyName) - doReentrantDDL(s, "ALTER TABLE mysql.db ADD INDEX i_user (user)", dbterror.ErrDupKeyName) - doReentrantDDL(s, "ALTER TABLE mysql.tables_priv ADD INDEX i_user (user)", dbterror.ErrDupKeyName) - doReentrantDDL(s, "ALTER TABLE mysql.columns_priv ADD INDEX i_user (user)", dbterror.ErrDupKeyName) - doReentrantDDL(s, "ALTER TABLE mysql.global_grants ADD INDEX i_user (user)", dbterror.ErrDupKeyName) - doReentrantDDL(s, "ALTER TABLE mysql.default_roles ADD INDEX i_user (user)", dbterror.ErrDupKeyName) -} - -// writeClusterID writes cluster id into mysql.tidb -func writeClusterID(s sessionapi.Session) { - clusterID := s.GetStore().GetClusterID() - - mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, "TiDB Cluster ID.") ON DUPLICATE KEY UPDATE VARIABLE_VALUE= %?`, - mysql.SystemDB, - mysql.TiDBTable, - tidbClusterID, - clusterID, - clusterID, - ) -} - -func upgradeToVer242(s sessionapi.Session, _ int64) { - writeClusterID(s) - mustExecute(s, CreateTiDBWorkloadValuesTable) -} - -func upgradeToVer243(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN max_node_count INT DEFAULT 0 AFTER `modify_params`;", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN max_node_count INT DEFAULT 0 AFTER `modify_params`;", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN extra_params json AFTER max_node_count;", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN extra_params json AFTER max_node_count;", infoschema.ErrColumnExists) -} - -func upgradeToVer244(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN IF NOT EXISTS `Max_user_connections` INT UNSIGNED NOT NULL DEFAULT 0 AFTER `Password_lifetime`") -} - -func upgradeToVer245(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.bind_info MODIFY COLUMN original_sql LONGTEXT NOT NULL") - doReentrantDDL(s, "ALTER TABLE mysql.bind_info MODIFY COLUMN bind_sql LONGTEXT NOT NULL") -} - -func upgradeToVer246(s sessionapi.Session, _ int64) { - // log duplicated digests that will be set to null. - ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) - rs, err := s.ExecuteInternal(ctx, - `select plan_digest, sql_digest from mysql.bind_info group by plan_digest, sql_digest having count(1) > 1`) - if err != nil { - logutil.BgLogger().Fatal("failed to get duplicated plan and sql digests", zap.Error(err)) - return - } - req := rs.NewChunk(nil) - duplicatedDigests := make(map[string]struct{}) - for { - err = rs.Next(ctx, req) - if err != nil { - logutil.BgLogger().Fatal("failed to get duplicated plan and sql digests", zap.Error(err)) - return - } - if req.NumRows() == 0 { - break - } - for i := range req.NumRows() { - planDigest, sqlDigest := req.GetRow(i).GetString(0), req.GetRow(i).GetString(1) - duplicatedDigests[sqlDigest+", "+planDigest] = struct{}{} - } - req.Reset() - } - if err := rs.Close(); err != nil { - logutil.BgLogger().Warn("failed to close record set", zap.Error(err)) - } - if len(duplicatedDigests) > 0 { - digestList := make([]string, 0, len(duplicatedDigests)) - for k := range duplicatedDigests { - digestList = append(digestList, "("+k+")") - } - logutil.BgLogger().Warn("set the following (plan digest, sql digest) in mysql.bind_info to null " + - "for adding new unique index: " + strings.Join(digestList, ", ")) - } - - // to avoid the failure of adding the unique index, remove duplicated rows on these 2 digest columns first. - // in most cases, there should be no duplicated rows, since now we only store one binding for each sql_digest. - // compared with upgrading failure, it's OK to set these 2 columns to null. - doReentrantDDL(s, `UPDATE mysql.bind_info SET plan_digest=null, sql_digest=null - WHERE (plan_digest, sql_digest) in ( - select plan_digest, sql_digest from mysql.bind_info - group by plan_digest, sql_digest having count(1) > 1)`) - doReentrantDDL(s, "ALTER TABLE mysql.bind_info MODIFY COLUMN sql_digest VARCHAR(64) DEFAULT NULL") - doReentrantDDL(s, "ALTER TABLE mysql.bind_info MODIFY COLUMN plan_digest VARCHAR(64) DEFAULT NULL") - doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD UNIQUE INDEX digest_index(plan_digest, sql_digest)", dbterror.ErrDupKeyName) -} - -func upgradeToVer247(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.stats_meta ADD COLUMN last_stats_histograms_version bigint unsigned DEFAULT NULL", infoschema.ErrColumnExists) -} - -func upgradeToVer248(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.tidb_pitr_id_map ADD COLUMN restore_id BIGINT NOT NULL DEFAULT 0", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_pitr_id_map DROP PRIMARY KEY") - doReentrantDDL(s, "ALTER TABLE mysql.tidb_pitr_id_map ADD PRIMARY KEY(restore_id, restored_ts, upstream_cluster_id, segment_id)") -} - -func upgradeToVer249(s sessionapi.Session, _ int64) { - doReentrantDDL(s, CreateTiDBRestoreRegistryTable) -} - -func upgradeToVer250(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN `keyspace` varchar(64) DEFAULT '' AFTER `extra_params`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD INDEX idx_keyspace(keyspace)", dbterror.ErrDupKeyName) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN `keyspace` varchar(64) DEFAULT '' AFTER `extra_params`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD INDEX idx_keyspace(keyspace)", dbterror.ErrDupKeyName) -} - -func upgradeToVer251(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.tidb_import_jobs ADD COLUMN `group_key` VARCHAR(256) NOT NULL DEFAULT '' AFTER `created_by`", infoschema.ErrColumnExists) - doReentrantDDL(s, "ALTER TABLE mysql.tidb_import_jobs ADD INDEX idx_group_key(group_key)", dbterror.ErrDupKeyName) -} - -func upgradeToVer252(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.bind_info CHANGE create_time create_time TIMESTAMP(6)") - doReentrantDDL(s, "ALTER TABLE mysql.bind_info CHANGE update_time update_time TIMESTAMP(6)") -} - -func upgradeToVer253(s sessionapi.Session, _ int64) { - doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD COLUMN last_used_date DATE DEFAULT NULL AFTER `plan_digest`", infoschema.ErrColumnExists) -} From 16745867ce10aadae03c4b0396275b37dceb5d90 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Fri, 10 Oct 2025 19:23:47 +0800 Subject: [PATCH 04/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/BUILD.bazel | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/bindinfo/BUILD.bazel b/pkg/bindinfo/BUILD.bazel index 129efc381fae8..a85dec4cf5336 100644 --- a/pkg/bindinfo/BUILD.bazel +++ b/pkg/bindinfo/BUILD.bazel @@ -36,6 +36,7 @@ go_library( "//pkg/util/hint", "//pkg/util/intest", "//pkg/util/kvcache", + "//pkg/util/logutil", "//pkg/util/mathutil", "//pkg/util/memory", "//pkg/util/parser", @@ -55,6 +56,7 @@ go_test( name = "bindinfo_test", timeout = "moderate", srcs = [ + "bind_usage_info_test.go", "binding_cache_test.go", "binding_match_test.go", "capture_test.go", From 4c09374be09f0c512aa582cfd864acb908e0c43c Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Fri, 10 Oct 2025 19:49:44 +0800 Subject: [PATCH 05/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/binding_match.go | 2 + pkg/session/bootstrap_test.go | 203 ------------------ pkg/session/bootstraptest/BUILD.bazel | 6 +- .../bootstraptest/bootstrap_upgrade_test.go | 75 +------ 4 files changed, 9 insertions(+), 277 deletions(-) diff --git a/pkg/bindinfo/binding_match.go b/pkg/bindinfo/binding_match.go index 82a2a7efe6183..8d79b337ab9d3 100644 --- a/pkg/bindinfo/binding_match.go +++ b/pkg/bindinfo/binding_match.go @@ -88,6 +88,8 @@ func matchSQLBinding(sctx sessionctx.Context, stmtNode ast.StmtNode, info *Bindi } binding, matched = globalHandle.MatchGlobalBinding(sctx, fuzzyDigest, tableNames) if matched { + // After hitting the cache, update the usage time of the bind. + binding.UpdateLastUsedAt() return binding, matched, metrics.ScopeGlobal } diff --git a/pkg/session/bootstrap_test.go b/pkg/session/bootstrap_test.go index 1e53b1384e9ac..2964ffb367e0b 100644 --- a/pkg/session/bootstrap_test.go +++ b/pkg/session/bootstrap_test.go @@ -2591,206 +2591,3 @@ func TestTiDBUpgradeToVer219(t *testing.T) { require.Contains(t, string(chk.GetRow(0).GetBytes(1)), "idx_schema_table_state") require.Contains(t, string(chk.GetRow(0).GetBytes(1)), "idx_schema_table_partition_state") } -<<<<<<< HEAD -======= - -func TestTiDBUpgradeToVer252(t *testing.T) { - // NOTE: this case needed to be passed in both classic and next-gen kernel. - // in the first release of next-gen kernel, the version is 250. - ctx := context.Background() - store, dom := CreateStoreAndBootstrap(t) - defer func() { require.NoError(t, store.Close()) }() - - ver250 := version250 - seV250 := CreateSessionAndSetID(t, store) - txn, err := store.Begin() - require.NoError(t, err) - m := meta.NewMutator(txn) - err = m.FinishBootstrap(int64(ver250)) - require.NoError(t, err) - revertVersionAndVariables(t, seV250, ver250) - err = txn.Commit(ctx) - require.NoError(t, err) - store.SetOption(StoreBootstrappedKey, nil) - - getBindInfoSQLFn := func(se sessionapi.Session) string { - res := MustExecToRecodeSet(t, se, "show create table mysql.bind_info") - chk := res.NewChunk(nil) - err = res.Next(ctx, chk) - require.NoError(t, err) - require.Equal(t, 1, chk.NumRows()) - return string(chk.GetRow(0).GetBytes(1)) - } - createTblSQL := getBindInfoSQLFn(seV250) - require.Contains(t, createTblSQL, "`create_time` timestamp(6)") - require.Contains(t, createTblSQL, "`update_time` timestamp(6)") - // revert it back to timestamp(3) for testing. we must set below fields to - // simulate the real session in upgrade process. - seV250.SetValue(sessionctx.Initing, true) - seV250.GetSessionVars().SQLMode = mysql.ModeNone - res := MustExecToRecodeSet(t, seV250, "select create_time,update_time from mysql.bind_info") - chk := res.NewChunk(nil) - err = res.Next(ctx, chk) - for i := range chk.NumRows() { - getTime := chk.GetRow(i).GetTime(0) - getTime = chk.GetRow(i).GetTime(1) - _ = getTime - } - mustExecute(seV250, "alter table mysql.bind_info modify create_time timestamp(3)") - mustExecute(seV250, "alter table mysql.bind_info modify update_time timestamp(3)") - createTblSQL = getBindInfoSQLFn(seV250) - require.Contains(t, createTblSQL, "`create_time` timestamp(3)") - require.Contains(t, createTblSQL, "`update_time` timestamp(3)") - - // do upgrade to latest version - dom.Close() - domCurVer, err := BootstrapSession(store) - require.NoError(t, err) - defer domCurVer.Close() - seCurVer := CreateSessionAndSetID(t, store) - ver, err := getBootstrapVersion(seCurVer) - require.NoError(t, err) - require.Equal(t, currentBootstrapVersion, ver) - // check if the columns have been changed to timestamp(6) - createTblSQL = getBindInfoSQLFn(seCurVer) - require.Contains(t, createTblSQL, "`create_time` timestamp(6)") - require.Contains(t, createTblSQL, "`update_time` timestamp(6)") -} - -func TestWriteClusterIDToMySQLTiDBWhenUpgradingTo242(t *testing.T) { - if kerneltype.IsNextGen() { - t.Skip("Skip this case because there is no upgrade in the first release of next-gen kernel") - } - - ctx := context.Background() - store, dom := CreateStoreAndBootstrap(t) - defer func() { require.NoError(t, store.Close()) }() - - // `cluster_id` is inserted for a new TiDB cluster. - se := CreateSessionAndSetID(t, store) - r := MustExecToRecodeSet(t, se, `select VARIABLE_VALUE from mysql.tidb where VARIABLE_NAME='cluster_id'`) - req := r.NewChunk(nil) - err := r.Next(ctx, req) - require.NoError(t, err) - require.Equal(t, 1, req.NumRows()) - require.NotEmpty(t, req.GetRow(0).GetBytes(0)) - require.NoError(t, r.Close()) - se.Close() - - // bootstrap as version241 - ver241 := version241 - seV241 := CreateSessionAndSetID(t, store) - txn, err := store.Begin() - require.NoError(t, err) - m := meta.NewMutator(txn) - err = m.FinishBootstrap(int64(ver241)) - require.NoError(t, err) - revertVersionAndVariables(t, seV241, ver241) - // remove the cluster_id entry from mysql.tidb table - MustExec(t, seV241, "delete from mysql.tidb where variable_name='cluster_id'") - err = txn.Commit(ctx) - require.NoError(t, err) - store.SetOption(StoreBootstrappedKey, nil) - ver, err := getBootstrapVersion(seV241) - require.NoError(t, err) - require.Equal(t, int64(ver241), ver) - seV241.Close() - - // upgrade to current version - dom.Close() - domCurVer, err := BootstrapSession(store) - require.NoError(t, err) - defer domCurVer.Close() - seCurVer := CreateSessionAndSetID(t, store) - ver, err = getBootstrapVersion(seCurVer) - require.NoError(t, err) - require.Equal(t, currentBootstrapVersion, ver) - - // check if the cluster_id has been set in the `mysql.tidb` table during upgrade - r = MustExecToRecodeSet(t, seCurVer, `select VARIABLE_VALUE from mysql.tidb where VARIABLE_NAME='cluster_id'`) - req = r.NewChunk(nil) - err = r.Next(ctx, req) - require.NoError(t, err) - require.Equal(t, 1, req.NumRows()) - require.NotEmpty(t, req.GetRow(0).GetBytes(0)) - require.NoError(t, r.Close()) - seCurVer.Close() -} - -func TestBindInfoUniqueIndex(t *testing.T) { - if kerneltype.IsNextGen() { - t.Skip("Skip this case because there is no upgrade in the first release of next-gen kernel") - } - - ctx := context.Background() - store, dom := CreateStoreAndBootstrap(t) - defer func() { require.NoError(t, store.Close()) }() - - // bootstrap as version245 - ver245 := version245 - seV245 := CreateSessionAndSetID(t, store) - txn, err := store.Begin() - require.NoError(t, err) - m := meta.NewMutator(txn) - err = m.FinishBootstrap(int64(ver245)) - require.NoError(t, err) - revertVersionAndVariables(t, seV245, ver245) - err = txn.Commit(ctx) - require.NoError(t, err) - store.SetOption(StoreBootstrappedKey, nil) - - // remove the unique index on mysql.bind_info for testing - MustExec(t, seV245, "alter table mysql.bind_info drop index digest_index") - - // insert duplicated values into mysql.bind_info - for _, sqlDigest := range []string{"null", "'x'", "'y'"} { - for _, planDigest := range []string{"null", "'x'", "'y'"} { - insertStmt := fmt.Sprintf(`insert into mysql.bind_info values ( - "sql", "bind_sql", "db", "disabled", NOW(), NOW(), "", "", "", %s, %s, null)`, - sqlDigest, planDigest) - MustExec(t, seV245, insertStmt) - MustExec(t, seV245, insertStmt) - } - } - - // upgrade to current version - dom.Close() - domCurVer, err := BootstrapSession(store) - require.NoError(t, err) - defer domCurVer.Close() - seCurVer := CreateSessionAndSetID(t, store) - ver, err := getBootstrapVersion(seCurVer) - require.NoError(t, err) - require.Equal(t, currentBootstrapVersion, ver) -} - -func TestVersionedBootstrapSchemas(t *testing.T) { - require.True(t, slices.IsSortedFunc(versionedBootstrapSchemas, func(a, b versionedBootstrapSchema) int { - return cmp.Compare(a.ver, b.ver) - }), "versionedBootstrapSchemas should be sorted by version") - - // make sure that later change won't affect existing version schemas. - require.Len(t, versionedBootstrapSchemas[0].databases[0].Tables, 52) - require.Len(t, versionedBootstrapSchemas[0].databases[1].Tables, 0) - - allIDs := make([]int64, 0, len(versionedBootstrapSchemas)) - var allTableCount int - for _, vbs := range versionedBootstrapSchemas { - for _, db := range vbs.databases { - require.Greater(t, db.ID, metadef.ReservedGlobalIDLowerBound) - require.LessOrEqual(t, db.ID, metadef.ReservedGlobalIDUpperBound) - allIDs = append(allIDs, db.ID) - - testTableBasicInfoSlice(t, db.Tables) - allTableCount += len(db.Tables) - for _, tbl := range db.Tables { - allIDs = append(allIDs, tbl.ID) - } - } - } - require.Len(t, tablesInSystemDatabase, allTableCount, - "versionedBootstrapSchemas should have the same number of tables as tablesInSystemDatabase") - slices.Sort(allIDs) - require.IsIncreasing(t, allIDs, "versionedBootstrapSchemas should not have duplicate IDs") -} ->>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)) diff --git a/pkg/session/bootstraptest/BUILD.bazel b/pkg/session/bootstraptest/BUILD.bazel index 91a6719ca6b09..251c35fb6ddf8 100644 --- a/pkg/session/bootstraptest/BUILD.bazel +++ b/pkg/session/bootstraptest/BUILD.bazel @@ -8,11 +8,7 @@ go_test( "main_test.go", ], flaky = True, -<<<<<<< HEAD:pkg/session/bootstraptest/BUILD.bazel - shard_count = 12, -======= - shard_count = 15, ->>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)):pkg/session/test/bootstraptest/BUILD.bazel + shard_count = 13, deps = [ "//pkg/config", "//pkg/ddl", diff --git a/pkg/session/bootstraptest/bootstrap_upgrade_test.go b/pkg/session/bootstraptest/bootstrap_upgrade_test.go index 8dba8a92523bb..333dc3727f21b 100644 --- a/pkg/session/bootstraptest/bootstrap_upgrade_test.go +++ b/pkg/session/bootstraptest/bootstrap_upgrade_test.go @@ -875,85 +875,23 @@ func TestUpgradeWithCrossJoinDisabled(t *testing.T) { require.NoError(t, store.Close()) }() } -<<<<<<< HEAD:pkg/session/bootstraptest/bootstrap_upgrade_test.go -======= - -func TestUpgradeBDRPrimary(t *testing.T) { - fromVersion := 244 - if kerneltype.IsNextGen() { - fromVersion = 250 - } - store, dom := session.CreateStoreAndBootstrap(t) - defer func() { require.NoError(t, store.Close()) }() - seVLow := session.CreateSessionAndSetID(t, store) - txn, err := store.Begin() - require.NoError(t, err) - m := meta.NewMutator(txn) - err = m.FinishBootstrap(int64(fromVersion)) - require.NoError(t, err) - err = txn.Commit(context.Background()) - revertVersionAndVariables(t, seVLow, fromVersion) - require.NoError(t, err) - session.MustExec(t, seVLow, "ADMIN SET BDR ROLE PRIMARY") - store.SetOption(session.StoreBootstrappedKey, nil) - ver, err := session.GetBootstrapVersion(seVLow) - require.NoError(t, err) - require.Equal(t, int64(fromVersion), ver) - dom.Close() - newVer, err := session.BootstrapSession(store) - require.NoError(t, err) - ver, err = session.GetBootstrapVersion(seVLow) - require.NoError(t, err) - require.Equal(t, session.CurrentBootstrapVersion, ver) - newVer.Close() -} - -func TestUpgradeBDRSecondary(t *testing.T) { - fromVersion := 244 - if kerneltype.IsNextGen() { - fromVersion = 250 - } - store, dom := session.CreateStoreAndBootstrap(t) - defer func() { require.NoError(t, store.Close()) }() - seV244 := session.CreateSessionAndSetID(t, store) - txn, err := store.Begin() - require.NoError(t, err) - m := meta.NewMutator(txn) - err = m.FinishBootstrap(int64(fromVersion)) - require.NoError(t, err) - err = txn.Commit(context.Background()) - revertVersionAndVariables(t, seV244, fromVersion) - require.NoError(t, err) - session.MustExec(t, seV244, "ADMIN SET BDR ROLE SECONDARY") - store.SetOption(session.StoreBootstrappedKey, nil) - ver, err := session.GetBootstrapVersion(seV244) - require.NoError(t, err) - require.Equal(t, int64(fromVersion), ver) - dom.Close() - newVer, err := session.BootstrapSession(store) - require.NoError(t, err) - ver, err = session.GetBootstrapVersion(seV244) - require.NoError(t, err) - require.Equal(t, session.CurrentBootstrapVersion, ver) - newVer.Close() -} func TestUpgradeBindInfo(t *testing.T) { - fromVersion := 251 + fromVersion := 220 store, dom := session.CreateStoreAndBootstrap(t) defer func() { require.NoError(t, store.Close()) }() - seV251 := session.CreateSessionAndSetID(t, store) + seV220 := session.CreateSessionAndSetID(t, store) txn, err := store.Begin() require.NoError(t, err) m := meta.NewMutator(txn) err = m.FinishBootstrap(int64(fromVersion)) require.NoError(t, err) err = txn.Commit(context.Background()) - revertVersionAndVariables(t, seV251, fromVersion) + revertVersionAndVariables(t, seV220, fromVersion) require.NoError(t, err) - session.MustExec(t, seV251, "ADMIN RELOAD BINDINGS;") - store.SetOption(session.StoreBootstrappedKey, nil) - ver, err := session.GetBootstrapVersion(seV251) + session.MustExec(t, seV220, "ADMIN RELOAD BINDINGS;") + session.UnsetStoreBootstrapped(store.UUID()) + ver, err := session.GetBootstrapVersion(seV220) require.NoError(t, err) require.Equal(t, int64(fromVersion), ver) dom.Close() @@ -967,4 +905,3 @@ func TestUpgradeBindInfo(t *testing.T) { newVer.Close() seLatestV.Close() } ->>>>>>> 70c7d5051c5 (bindinfo: add last_used_date to track bindinfo usage frequency (#63409)):pkg/session/test/bootstraptest/bootstrap_upgrade_test.go From 39ecefad5c48662fe18fe90ccd3e7d3779af71e7 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 11 Oct 2025 18:22:56 +0800 Subject: [PATCH 06/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/binding.go | 9 ++++----- pkg/bindinfo/util.go | 6 +++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pkg/bindinfo/binding.go b/pkg/bindinfo/binding.go index 862226ea9d4d8..6ebd1ce1d0b76 100644 --- a/pkg/bindinfo/binding.go +++ b/pkg/bindinfo/binding.go @@ -15,7 +15,6 @@ package bindinfo import ( - "sync/atomic" "time" "unsafe" @@ -244,12 +243,12 @@ func (b *Binding) size() float64 { // UpdateLastUsedAt is to update binding usage info when this binding is used. func (b *Binding) UpdateLastUsedAt() { now := time.Now() - b.UsageInfo.LastUsedAt.Store(&now) + b.UsageInfo.LastUsedAt = &now } // UpdateLastSavedAt is to update the last saved time func (b *Binding) UpdateLastSavedAt(ts *time.Time) { - b.UsageInfo.LastSavedAt.Store(ts) + b.UsageInfo.LastSavedAt = ts } type bindingInfoUsageInfo struct { @@ -257,7 +256,7 @@ type bindingInfoUsageInfo struct { // It is nil if this binding has never been used. // It is updated when this binding is used. // It is used to update the `last_used_time` field in mysql.bind_info table. - LastUsedAt atomic.Pointer[time.Time] + LastUsedAt *time.Time // LastSavedAt records the last time when this binding is saved into storage. - LastSavedAt atomic.Pointer[time.Time] + LastSavedAt *time.Time } diff --git a/pkg/bindinfo/util.go b/pkg/bindinfo/util.go index 4ab9862566136..f92fdd5ebfd34 100644 --- a/pkg/bindinfo/util.go +++ b/pkg/bindinfo/util.go @@ -62,11 +62,11 @@ func (u *globalBindingHandle) updateBindingUsageInfoToStorage(bindings []Binding } }() for _, binding := range bindings { - lastUsed := binding.UsageInfo.LastUsedAt.Load() + lastUsed := binding.UsageInfo.LastUsedAt if lastUsed == nil { continue } - lastSaved := binding.UsageInfo.LastSavedAt.Load() + lastSaved := binding.UsageInfo.LastSavedAt if shouldUpdateBinding(lastSaved, lastUsed) { toWrite = append(toWrite, binding) cnt++ @@ -100,7 +100,7 @@ func (h *globalBindingHandle) updateBindingUsageInfoToStorageInternal(bindings [ return errors.Trace(err) } for _, binding := range bindings { - lastUsed := binding.UsageInfo.LastUsedAt.Load() + lastUsed := binding.UsageInfo.LastUsedAt intest.Assert(lastUsed != nil) err = saveBindingUsage(sctx, binding.SQLDigest, binding.PlanDigest, *lastUsed) if err != nil { From 1cf0f4d45ce2e8a83aebc638d853215e5ef3069d Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 11 Oct 2025 19:58:46 +0800 Subject: [PATCH 07/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/global_handle.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/bindinfo/global_handle.go b/pkg/bindinfo/global_handle.go index 79f2bf1483539..91ba1193f6c58 100644 --- a/pkg/bindinfo/global_handle.go +++ b/pkg/bindinfo/global_handle.go @@ -259,14 +259,14 @@ func (h *globalBindingHandle) LoadFromStorageToCache(fullLoad bool) (err error) } // UpdateBindingUsageInfoToStorage is to update the binding usage info into storage -func (u *globalBindingHandle) UpdateBindingUsageInfoToStorage() error { +func (h *globalBindingHandle) UpdateBindingUsageInfoToStorage() error { defer func() { if r := recover(); r != nil { logutil.BindLogger().Warn("panic when update usage info for binding", zap.Any("recover", r)) } }() - bindings := u.GetAllGlobalBindings() - return u.updateBindingUsageInfoToStorage(bindings) + bindings := h.GetAllGlobalBindings() + return h.updateBindingUsageInfoToStorage(bindings) } // CreateGlobalBinding creates a Bindings to the storage and the cache. From edb8313fb665b983e4ce15b3cb5ba0dda18261ff Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 13 Oct 2025 00:25:37 +0800 Subject: [PATCH 08/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/bindinfo/util.go b/pkg/bindinfo/util.go index f92fdd5ebfd34..461de212a4d1d 100644 --- a/pkg/bindinfo/util.go +++ b/pkg/bindinfo/util.go @@ -88,8 +88,8 @@ func (u *globalBindingHandle) updateBindingUsageInfoToStorage(bindings []Binding return nil } -func (h *globalBindingHandle) updateBindingUsageInfoToStorageInternal(bindings []Binding) error { - err := h.callWithSCtx(true, func(sctx sessionctx.Context) (err error) { +func (u *globalBindingHandle) updateBindingUsageInfoToStorageInternal(bindings []Binding) error { + err := u.callWithSCtx(true, func(sctx sessionctx.Context) (err error) { if err = lockBindInfoTable(sctx); err != nil { return errors.Trace(err) } From 08ebc8140515c034084f699257e5a33cbb5d7888 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 13 Oct 2025 00:48:47 +0800 Subject: [PATCH 09/29] update Signed-off-by: Weizhen Wang --- build/nogo_config.json | 1 - 1 file changed, 1 deletion(-) diff --git a/build/nogo_config.json b/build/nogo_config.json index 6104177c02410..d6aca8f50a481 100644 --- a/build/nogo_config.json +++ b/build/nogo_config.json @@ -699,7 +699,6 @@ "pkg/planner/cascades": "planner/cascades code", "pkg/store/": "store code", "pkg/ttl/": "ttl code", - "pkg/bindinfo/": "bindinfo code", "pkg/domain/": "domain code" } }, From b2d7cff8702f07c8775a7ac40909b537f7cd22cc Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 13 Oct 2025 11:11:26 +0800 Subject: [PATCH 10/29] update Signed-off-by: Weizhen Wang --- pkg/infoschema/test/clustertablestest/cluster_tables_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/infoschema/test/clustertablestest/cluster_tables_test.go b/pkg/infoschema/test/clustertablestest/cluster_tables_test.go index 41e694cabeda2..f5ab98850a449 100644 --- a/pkg/infoschema/test/clustertablestest/cluster_tables_test.go +++ b/pkg/infoschema/test/clustertablestest/cluster_tables_test.go @@ -1534,7 +1534,7 @@ func TestSetBindingStatusBySQLDigest(t *testing.T) { tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("1")) bindinfo.MaxWriteInterval = 1 * time.Microsecond time.Sleep(1 * time.Second) - require.NoError(t, s.dom.BindingHandle().UpdateBindingUsageInfoToStorage()) + require.NoError(t, s.dom.BindHandle().UpdateBindingUsageInfoToStorage()) tk.MustQuery(fmt.Sprintf(`select last_used_date from mysql.bind_info where original_sql != '%s' and last_used_date is null`, bindinfo.BuiltinPseudoSQL4BindLock)).Check(testkit.Rows()) sqlDigest := tk.MustQuery("show global bindings").Rows() From 8c7c01e5fc81477bca359ec9e30cdf7815c9c41c Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 13 Oct 2025 16:40:28 +0800 Subject: [PATCH 11/29] update Signed-off-by: Weizhen Wang --- pkg/session/bootstrap_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/session/bootstrap_test.go b/pkg/session/bootstrap_test.go index 2964ffb367e0b..4fd26cea61356 100644 --- a/pkg/session/bootstrap_test.go +++ b/pkg/session/bootstrap_test.go @@ -2027,11 +2027,11 @@ func TestTiDBBindingInListToVer175(t *testing.T) { MustExec(t, seV174, "use test") MustExec(t, seV174, "create table t (a int, b int, c int, key(c))") _, digest := parser.NormalizeDigestForBinding("SELECT * FROM `test`.`t` WHERE `a` IN (1,2,3)") - MustExec(t, seV174, fmt.Sprintf("insert into mysql.bind_info values ('select * from `test` . `t` where `a` in ( ... )', 'SELECT /*+ use_index(`t` `c`)*/ * FROM `test`.`t` WHERE `a` IN (1,2,3)', 'test', 'enabled', '2023-09-13 14:41:38.319', '2023-09-13 14:41:35.319', 'utf8', 'utf8_general_ci', 'manual', '%s', '')", digest.String())) + MustExec(t, seV174, fmt.Sprintf("insert into mysql.bind_info values ('select * from `test` . `t` where `a` in ( ... )', 'SELECT /*+ use_index(`t` `c`)*/ * FROM `test`.`t` WHERE `a` IN (1,2,3)', 'test', 'enabled', '2023-09-13 14:41:38.319', '2023-09-13 14:41:35.319', 'utf8', 'utf8_general_ci', 'manual', '%s', '', null)", digest.String())) _, digest = parser.NormalizeDigestForBinding("SELECT * FROM `test`.`t` WHERE `a` IN (1)") - MustExec(t, seV174, fmt.Sprintf("insert into mysql.bind_info values ('select * from `test` . `t` where `a` in ( ? )', 'SELECT /*+ use_index(`t` `c`)*/ * FROM `test`.`t` WHERE `a` IN (1)', 'test', 'enabled', '2023-09-13 14:41:38.319', '2023-09-13 14:41:36.319', 'utf8', 'utf8_general_ci', 'manual', '%s', '')", digest.String())) + MustExec(t, seV174, fmt.Sprintf("insert into mysql.bind_info values ('select * from `test` . `t` where `a` in ( ? )', 'SELECT /*+ use_index(`t` `c`)*/ * FROM `test`.`t` WHERE `a` IN (1)', 'test', 'enabled', '2023-09-13 14:41:38.319', '2023-09-13 14:41:36.319', 'utf8', 'utf8_general_ci', 'manual', '%s', '', null)", digest.String())) _, digest = parser.NormalizeDigestForBinding("SELECT * FROM `test`.`t` WHERE `a` IN (1) AND `b` IN (1,2,3)") - MustExec(t, seV174, fmt.Sprintf("insert into mysql.bind_info values ('select * from `test` . `t` where `a` in ( ? ) and `b` in ( ... )', 'SELECT /*+ use_index(`t` `c`)*/ * FROM `test`.`t` WHERE `a` IN (1) AND `b` IN (1,2,3)', 'test', 'enabled', '2023-09-13 14:41:37.319', '2023-09-13 14:41:38.319', 'utf8', 'utf8_general_ci', 'manual', '%s', '')", digest.String())) + MustExec(t, seV174, fmt.Sprintf("insert into mysql.bind_info values ('select * from `test` . `t` where `a` in ( ? ) and `b` in ( ... )', 'SELECT /*+ use_index(`t` `c`)*/ * FROM `test`.`t` WHERE `a` IN (1) AND `b` IN (1,2,3)', 'test', 'enabled', '2023-09-13 14:41:37.319', '2023-09-13 14:41:38.319', 'utf8', 'utf8_general_ci', 'manual', '%s', '', null)", digest.String())) showBindings := func(s sessiontypes.Session) (records []string) { MustExec(t, s, "admin reload bindings") From 89f7374f36a168660529746bd8274e5013fe706e Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 13 Oct 2025 17:14:55 +0800 Subject: [PATCH 12/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/session_handle_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/bindinfo/session_handle_test.go b/pkg/bindinfo/session_handle_test.go index 5d5bc3d7bb1fa..0246dff46a37d 100644 --- a/pkg/bindinfo/session_handle_test.go +++ b/pkg/bindinfo/session_handle_test.go @@ -187,7 +187,7 @@ func TestBaselineDBLowerCase(t *testing.T) { // Simulate existing bindings with upper case default_db. _, sqlDigest := parser.NormalizeDigestForBinding("select * from `spm` . `t`") - tk.MustExec("insert into mysql.bind_info values('select * from `spm` . `t`', 'select * from `spm` . `t`', 'SPM', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + + tk.MustExec("insert into mysql.bind_info (original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest) values('select * from `spm` . `t`', 'select * from `spm` . `t`', 'SPM', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") tk.MustQuery("select original_sql, default_db from mysql.bind_info where original_sql = 'select * from `spm` . `t`'").Check(testkit.Rows( "select * from `spm` . `t` SPM", @@ -205,7 +205,7 @@ func TestBaselineDBLowerCase(t *testing.T) { internal.UtilCleanBindingEnv(tk, dom) // Simulate existing bindings with upper case default_db. - tk.MustExec("insert into mysql.bind_info values('select * from `spm` . `t`', 'select * from `spm` . `t`', 'SPM', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + + tk.MustExec("insert into mysql.bind_info (original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest) values('select * from `spm` . `t`', 'select * from `spm` . `t`', 'SPM', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") tk.MustQuery("select original_sql, default_db from mysql.bind_info where original_sql = 'select * from `spm` . `t`'").Check(testkit.Rows( "select * from `spm` . `t` SPM", From 188080b6435676f40b9a5e475b950032f561410e Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 14 Oct 2025 17:08:59 +0800 Subject: [PATCH 13/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/bind_usage_info_test.go | 4 ++-- pkg/bindinfo/binding.go | 25 +++++++++---------------- pkg/bindinfo/binding_cache.go | 5 ++++- pkg/bindinfo/util.go | 14 ++++++++------ 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/pkg/bindinfo/bind_usage_info_test.go b/pkg/bindinfo/bind_usage_info_test.go index 841612c727719..2f4f96e689c3e 100644 --- a/pkg/bindinfo/bind_usage_info_test.go +++ b/pkg/bindinfo/bind_usage_info_test.go @@ -27,10 +27,10 @@ import ( func TestBindUsageInfo(t *testing.T) { bindinfo.UpdateBindingUsageInfoBatchSize = 2 bindinfo.MaxWriteInterval = 100 * time.Microsecond - store := testkit.CreateMockStore(t) + store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) - bindingHandle := bindinfo.NewGlobalBindingHandle(&mockSessionPool{tk.Session()}) + bindingHandle := dom.BindHandle() tk.MustExec(`use test`) tk.MustExec(`set @@tidb_opt_enable_fuzzy_binding=1`) tk.MustExec("create table t1(a int, b int, c int, key idx_b(b), key idx_c(c))") diff --git a/pkg/bindinfo/binding.go b/pkg/bindinfo/binding.go index 6ebd1ce1d0b76..d563a8ebd5b6a 100644 --- a/pkg/bindinfo/binding.go +++ b/pkg/bindinfo/binding.go @@ -74,10 +74,13 @@ type Binding struct { // TableNames records all schema and table names in this binding statement, which are used for fuzzy matching. TableNames []*ast.TableName `json:"-"` - - // UsageInfo is to track the usage information `last_used_time` of this binding - // and it will be updated when this binding is used. - UsageInfo bindingInfoUsageInfo + // LastUsedAt records the last time when this binding is used. + // It is nil if this binding has never been used. + // It is updated when this binding is used. + // It is used to update the `last_used_time` field in mysql.bind_info table. + LastUsedAt *time.Time + // LastSavedAt records the last time when this binding is saved into storage. + LastSavedAt *time.Time } func (b *Binding) isSame(rb *Binding) bool { @@ -243,20 +246,10 @@ func (b *Binding) size() float64 { // UpdateLastUsedAt is to update binding usage info when this binding is used. func (b *Binding) UpdateLastUsedAt() { now := time.Now() - b.UsageInfo.LastUsedAt = &now + b.LastUsedAt = &now } // UpdateLastSavedAt is to update the last saved time func (b *Binding) UpdateLastSavedAt(ts *time.Time) { - b.UsageInfo.LastSavedAt = ts -} - -type bindingInfoUsageInfo struct { - // LastUsedAt records the last time when this binding is used. - // It is nil if this binding has never been used. - // It is updated when this binding is used. - // It is used to update the `last_used_time` field in mysql.bind_info table. - LastUsedAt *time.Time - // LastSavedAt records the last time when this binding is saved into storage. - LastSavedAt *time.Time + b.LastSavedAt = ts } diff --git a/pkg/bindinfo/binding_cache.go b/pkg/bindinfo/binding_cache.go index 7e3211665daa5..1b4a4677703a0 100644 --- a/pkg/bindinfo/binding_cache.go +++ b/pkg/bindinfo/binding_cache.go @@ -130,18 +130,21 @@ func (fbc *fuzzyBindingCache) getFromMemory(sctx sessionctx.Context, fuzzyDigest } } if bindings != nil { - for _, binding := range bindings { + for idx, binding := range bindings { numWildcards, matched := fuzzyMatchBindingTableName(sctx.GetSessionVars().CurrentDB, tableNames, binding.TableNames) if matched && numWildcards > 0 && sctx != nil && !enableFuzzyBinding { continue // fuzzy binding is disabled, skip this binding } if matched && numWildcards < leastWildcards { matchedBinding = binding + binding.UpdateLastUsedAt() + bindings[idx] = binding isMatched = true leastWildcards = numWildcards break } } + bindingCache.SetBinding(sqlDigest, bindings) // update the last used time } else { missingSQLDigest = append(missingSQLDigest, sqlDigest) } diff --git a/pkg/bindinfo/util.go b/pkg/bindinfo/util.go index 461de212a4d1d..2a2b8bf51dfb8 100644 --- a/pkg/bindinfo/util.go +++ b/pkg/bindinfo/util.go @@ -62,11 +62,11 @@ func (u *globalBindingHandle) updateBindingUsageInfoToStorage(bindings []Binding } }() for _, binding := range bindings { - lastUsed := binding.UsageInfo.LastUsedAt + lastUsed := binding.LastUsedAt if lastUsed == nil { continue } - lastSaved := binding.UsageInfo.LastSavedAt + lastSaved := binding.LastSavedAt if shouldUpdateBinding(lastSaved, lastUsed) { toWrite = append(toWrite, binding) cnt++ @@ -100,7 +100,7 @@ func (u *globalBindingHandle) updateBindingUsageInfoToStorageInternal(bindings [ return errors.Trace(err) } for _, binding := range bindings { - lastUsed := binding.UsageInfo.LastUsedAt + lastUsed := binding.LastUsedAt intest.Assert(lastUsed != nil) err = saveBindingUsage(sctx, binding.SQLDigest, binding.PlanDigest, *lastUsed) if err != nil { @@ -113,8 +113,10 @@ func (u *globalBindingHandle) updateBindingUsageInfoToStorageInternal(bindings [ ts := time.Now() for _, binding := range bindings { binding.UpdateLastSavedAt(&ts) + u.getCache().SetBinding(binding.SQLDigest, []Binding{binding}) } } + return err } @@ -141,7 +143,7 @@ func addLockForBinds(sctx sessionctx.Context, bindings []Binding) error { } condition = append(condition, sql) } - locksql := "select 1 from mysql.bind_info use index(digest_index) where (plan_digest, sql_digest) in (" + + locksql := "select 1 from mysql.bind_info where (plan_digest, sql_digest) in (" + strings.Join(condition, " , ") + ") for update" _, err := exec(sctx, locksql) if err != nil { @@ -152,9 +154,9 @@ func addLockForBinds(sctx sessionctx.Context, bindings []Binding) error { func saveBindingUsage(sctx sessionctx.Context, sqldigest, planDigest string, ts time.Time) error { lastUsedTime := ts.UTC().Format(types.TimeFormat) - var sql = "UPDATE mysql.bind_info USE INDEX(digest_index) SET last_used_date = CONVERT_TZ(%?, '+00:00', @@TIME_ZONE) WHERE sql_digest = %?" + var sql = "UPDATE mysql.bind_info SET last_used_date = CONVERT_TZ(%?, '+00:00', @@TIME_ZONE) WHERE sql_digest = %?" if planDigest == "" { - sql += " AND plan_digest IS NULL" + sql += " AND plan_digest = ''" } else { sql += fmt.Sprintf(" AND plan_digest = '%s'", planDigest) } From ccd609dace441906fc6c0907df491e1ea689bb21 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 14 Oct 2025 17:35:28 +0800 Subject: [PATCH 14/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/binding_cache.go | 5 ++++- pkg/bindinfo/util.go | 14 ++++++++------ pkg/session/bootstrap.go | 4 +++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/pkg/bindinfo/binding_cache.go b/pkg/bindinfo/binding_cache.go index 1b4a4677703a0..161d27dfad922 100644 --- a/pkg/bindinfo/binding_cache.go +++ b/pkg/bindinfo/binding_cache.go @@ -144,7 +144,10 @@ func (fbc *fuzzyBindingCache) getFromMemory(sctx sessionctx.Context, fuzzyDigest break } } - bindingCache.SetBinding(sqlDigest, bindings) // update the last used time + err := bindingCache.SetBinding(sqlDigest, bindings) // update the last used time + if err != nil { + logutil.BindLogger().Warn("bindingCache.SetBinding", zap.Error(err)) + } } else { missingSQLDigest = append(missingSQLDigest, sqlDigest) } diff --git a/pkg/bindinfo/util.go b/pkg/bindinfo/util.go index 2a2b8bf51dfb8..352f36d01b746 100644 --- a/pkg/bindinfo/util.go +++ b/pkg/bindinfo/util.go @@ -21,13 +21,13 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/tidb/pkg/bindinfo/internal/logutil" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/planner/core/resolve" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/chunk" "github.com/pingcap/tidb/pkg/util/intest" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/sqlexec" "go.uber.org/zap" ) @@ -58,7 +58,7 @@ func (u *globalBindingHandle) updateBindingUsageInfoToStorage(bindings []Binding cnt := 0 defer func() { if cnt > 0 { - logutil.BgLogger().Info("update binding usage info to storage", zap.Int("count", cnt), zap.Duration("duration", time.Since(now))) + logutil.BindLogger().Info("update binding usage info to storage", zap.Int("count", cnt), zap.Duration("duration", time.Since(now))) } }() for _, binding := range bindings { @@ -113,10 +113,12 @@ func (u *globalBindingHandle) updateBindingUsageInfoToStorageInternal(bindings [ ts := time.Now() for _, binding := range bindings { binding.UpdateLastSavedAt(&ts) - u.getCache().SetBinding(binding.SQLDigest, []Binding{binding}) + err := u.getCache().SetBinding(binding.SQLDigest, []Binding{binding}) + if err != nil { + logutil.BindLogger().Warn("update binding cache error", zap.Error(err)) + } } } - return err } @@ -143,7 +145,7 @@ func addLockForBinds(sctx sessionctx.Context, bindings []Binding) error { } condition = append(condition, sql) } - locksql := "select 1 from mysql.bind_info where (plan_digest, sql_digest) in (" + + locksql := "select 1 from mysql.bind_info use index(digest_index) where (plan_digest, sql_digest) in (" + strings.Join(condition, " , ") + ") for update" _, err := exec(sctx, locksql) if err != nil { @@ -154,7 +156,7 @@ func addLockForBinds(sctx sessionctx.Context, bindings []Binding) error { func saveBindingUsage(sctx sessionctx.Context, sqldigest, planDigest string, ts time.Time) error { lastUsedTime := ts.UTC().Format(types.TimeFormat) - var sql = "UPDATE mysql.bind_info SET last_used_date = CONVERT_TZ(%?, '+00:00', @@TIME_ZONE) WHERE sql_digest = %?" + var sql = "UPDATE mysql.bind_info USE INDEX(digest_index) SET last_used_date = CONVERT_TZ(%?, '+00:00', @@TIME_ZONE) WHERE sql_digest = %?" if planDigest == "" { sql += " AND plan_digest = ''" } else { diff --git a/pkg/session/bootstrap.go b/pkg/session/bootstrap.go index ffd08fe501a2a..cecb7e36e510b 100644 --- a/pkg/session/bootstrap.go +++ b/pkg/session/bootstrap.go @@ -301,7 +301,8 @@ const ( plan_digest varchar(64), last_used_date date DEFAULT NULL, INDEX sql_index(original_sql(700),default_db(68)) COMMENT "accelerate the speed when add global binding query", - INDEX time_index(update_time) COMMENT "accelerate the speed when querying with last update time" + INDEX time_index(update_time) COMMENT "accelerate the speed when querying with last update time", + UNIQUE INDEX digest_index(plan_digest, sql_digest) COMMENT "avoid duplicated records" ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;` // CreateRoleEdgesTable stores the role and user relationship information. @@ -3274,6 +3275,7 @@ func upgradeToVer221(s sessiontypes.Session, ver int64) { return } doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD COLUMN last_used_date DATE DEFAULT NULL AFTER `plan_digest`", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD UNIQUE INDEX digest_index(plan_digest, sql_digest)", dbterror.ErrDupKeyName) } // initGlobalVariableIfNotExists initialize a global variable with specific val if it does not exist. From 7fe5a75506989f6b5c4c7b6ec044b778d53db92a Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 14 Oct 2025 17:41:59 +0800 Subject: [PATCH 15/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/binding.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/bindinfo/binding.go b/pkg/bindinfo/binding.go index d563a8ebd5b6a..b664777ed561f 100644 --- a/pkg/bindinfo/binding.go +++ b/pkg/bindinfo/binding.go @@ -206,6 +206,8 @@ func merge(lBindings, rBindings Bindings) Bindings { if lbind.isSame(&rbind) { found = true if rbind.UpdateTime.Compare(lbind.UpdateTime) >= 0 { + rbind.LastUsedAt = lbind.LastUsedAt + rbind.LastSavedAt = lbind.LastSavedAt result[j] = rbind } break From bcb98f17524640cc5bae4879c609776745ce531b Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 14 Oct 2025 17:53:40 +0800 Subject: [PATCH 16/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/BUILD.bazel | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/bindinfo/BUILD.bazel b/pkg/bindinfo/BUILD.bazel index a85dec4cf5336..fdef9e31fc4d3 100644 --- a/pkg/bindinfo/BUILD.bazel +++ b/pkg/bindinfo/BUILD.bazel @@ -36,7 +36,6 @@ go_library( "//pkg/util/hint", "//pkg/util/intest", "//pkg/util/kvcache", - "//pkg/util/logutil", "//pkg/util/mathutil", "//pkg/util/memory", "//pkg/util/parser", From 211d53a1ade2f3d85092f617e70f1d903144e839 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 14 Oct 2025 18:33:58 +0800 Subject: [PATCH 17/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/util.go | 2 +- pkg/session/BUILD.bazel | 1 - pkg/session/bootstrap.go | 51 +++++- pkg/session/bootstrap_test.go | 160 ------------------ .../bootstraptest/bootstrap_upgrade_test.go | 39 +++-- 5 files changed, 74 insertions(+), 179 deletions(-) diff --git a/pkg/bindinfo/util.go b/pkg/bindinfo/util.go index 352f36d01b746..dbc4d1a14fc7f 100644 --- a/pkg/bindinfo/util.go +++ b/pkg/bindinfo/util.go @@ -158,7 +158,7 @@ func saveBindingUsage(sctx sessionctx.Context, sqldigest, planDigest string, ts lastUsedTime := ts.UTC().Format(types.TimeFormat) var sql = "UPDATE mysql.bind_info USE INDEX(digest_index) SET last_used_date = CONVERT_TZ(%?, '+00:00', @@TIME_ZONE) WHERE sql_digest = %?" if planDigest == "" { - sql += " AND plan_digest = ''" + sql += " AND plan_digest is null" } else { sql += fmt.Sprintf(" AND plan_digest = '%s'", planDigest) } diff --git a/pkg/session/BUILD.bazel b/pkg/session/BUILD.bazel index 40e654398422a..e9caed5823d73 100644 --- a/pkg/session/BUILD.bazel +++ b/pkg/session/BUILD.bazel @@ -164,7 +164,6 @@ go_test( "//pkg/expression/sessionexpr", "//pkg/kv", "//pkg/meta", - "//pkg/parser", "//pkg/parser/ast", "//pkg/parser/auth", "//pkg/session/types", diff --git a/pkg/session/bootstrap.go b/pkg/session/bootstrap.go index cecb7e36e510b..ca9dd232c6acf 100644 --- a/pkg/session/bootstrap.go +++ b/pkg/session/bootstrap.go @@ -297,8 +297,8 @@ const ( charset TEXT NOT NULL, collation TEXT NOT NULL, source VARCHAR(10) NOT NULL DEFAULT 'unknown', - sql_digest varchar(64), - plan_digest varchar(64), + sql_digest varchar(64) DEFAULT NULL, + plan_digest varchar(64) DEFAULT NULL, last_used_date date DEFAULT NULL, INDEX sql_index(original_sql(700),default_db(68)) COMMENT "accelerate the speed when add global binding query", INDEX time_index(update_time) COMMENT "accelerate the speed when querying with last update time", @@ -3274,8 +3274,53 @@ func upgradeToVer221(s sessiontypes.Session, ver int64) { if ver >= version221 { return } - doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD COLUMN last_used_date DATE DEFAULT NULL AFTER `plan_digest`", infoschema.ErrColumnExists) + // log duplicated digests that will be set to null. + ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) + rs, err := s.ExecuteInternal(ctx, + `select plan_digest, sql_digest from mysql.bind_info group by plan_digest, sql_digest having count(1) > 1`) + if err != nil { + logutil.BgLogger().Fatal("failed to get duplicated plan and sql digests", zap.Error(err)) + return + } + req := rs.NewChunk(nil) + duplicatedDigests := make(map[string]struct{}) + for { + err = rs.Next(ctx, req) + if err != nil { + logutil.BgLogger().Fatal("failed to get duplicated plan and sql digests", zap.Error(err)) + return + } + if req.NumRows() == 0 { + break + } + for i := 0; i < req.NumRows(); i++ { + planDigest, sqlDigest := req.GetRow(i).GetString(0), req.GetRow(i).GetString(1) + duplicatedDigests[sqlDigest+", "+planDigest] = struct{}{} + } + req.Reset() + } + if err := rs.Close(); err != nil { + logutil.BgLogger().Warn("failed to close record set", zap.Error(err)) + } + if len(duplicatedDigests) > 0 { + digestList := make([]string, 0, len(duplicatedDigests)) + for k := range duplicatedDigests { + digestList = append(digestList, "("+k+")") + } + logutil.BgLogger().Warn("set the following (plan digest, sql digest) in mysql.bind_info to null " + + "for adding new unique index: " + strings.Join(digestList, ", ")) + } + // to avoid the failure of adding the unique index, remove duplicated rows on these 2 digest columns first. + // in most cases, there should be no duplicated rows, since now we only store one binding for each sql_digest. + // compared with upgrading failure, it's OK to set these 2 columns to null. + doReentrantDDL(s, `UPDATE mysql.bind_info SET plan_digest=null, sql_digest=null + WHERE (plan_digest, sql_digest) in ( + select plan_digest, sql_digest from mysql.bind_info + group by plan_digest, sql_digest having count(1) > 1)`) + doReentrantDDL(s, "ALTER TABLE mysql.bind_info MODIFY COLUMN sql_digest VARCHAR(64) DEFAULT NULL") + doReentrantDDL(s, "ALTER TABLE mysql.bind_info MODIFY COLUMN plan_digest VARCHAR(64) DEFAULT NULL") doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD UNIQUE INDEX digest_index(plan_digest, sql_digest)", dbterror.ErrDupKeyName) + doReentrantDDL(s, "ALTER TABLE mysql.bind_info ADD COLUMN last_used_date DATE DEFAULT NULL AFTER `plan_digest`", infoschema.ErrColumnExists) } // initGlobalVariableIfNotExists initialize a global variable with specific val if it does not exist. diff --git a/pkg/session/bootstrap_test.go b/pkg/session/bootstrap_test.go index 4fd26cea61356..00ec6a39b3f63 100644 --- a/pkg/session/bootstrap_test.go +++ b/pkg/session/bootstrap_test.go @@ -17,7 +17,6 @@ package session import ( "context" "fmt" - "sort" "strings" "testing" "time" @@ -29,7 +28,6 @@ import ( "github.com/pingcap/tidb/pkg/domain" "github.com/pingcap/tidb/pkg/expression/sessionexpr" "github.com/pingcap/tidb/pkg/meta" - "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/auth" sessiontypes "github.com/pingcap/tidb/pkg/session/types" "github.com/pingcap/tidb/pkg/sessionctx" @@ -588,53 +586,6 @@ func TestStmtSummary(t *testing.T) { require.NoError(t, r.Close()) } -func TestUpdateDuplicateBindInfo(t *testing.T) { - ctx := context.Background() - store, dom := CreateStoreAndBootstrap(t) - defer func() { require.NoError(t, store.Close()) }() - defer dom.Close() - se := CreateSessionAndSetID(t, store) - MustExec(t, se, "alter table mysql.bind_info drop column if exists plan_digest") - MustExec(t, se, "alter table mysql.bind_info drop column if exists sql_digest") - - MustExec(t, se, `insert into mysql.bind_info values('select * from t', 'select /*+ use_index(t, idx_a)*/ * from t', 'test', 'enabled', '2021-01-04 14:50:58.257', '2021-01-04 14:50:58.257', 'utf8', 'utf8_general_ci', 'manual')`) - // The latest one. - MustExec(t, se, `insert into mysql.bind_info values('select * from test . t', 'select /*+ use_index(t, idx_b)*/ * from test.t', 'test', 'enabled', '2021-01-04 14:50:58.257', '2021-01-09 14:50:58.257', 'utf8', 'utf8_general_ci', 'manual')`) - - MustExec(t, se, `insert into mysql.bind_info values('select * from t where a < ?', 'select * from t use index(idx) where a < 1', 'test', 'deleted', '2021-06-04 17:04:43.333', '2021-06-04 17:04:43.335', 'utf8', 'utf8_general_ci', 'manual')`) - MustExec(t, se, `insert into mysql.bind_info values('select * from t where a < ?', 'select * from t ignore index(idx) where a < 1', 'test', 'enabled', '2021-06-04 17:04:43.335', '2021-06-04 17:04:43.335', 'utf8', 'utf8_general_ci', 'manual')`) - MustExec(t, se, `insert into mysql.bind_info values('select * from test . t where a <= ?', 'select * from test.t use index(idx) where a <= 1', '', 'deleted', '2021-06-04 17:04:43.345', '2021-06-04 17:04:45.334', 'utf8', 'utf8_general_ci', 'manual')`) - MustExec(t, se, `insert into mysql.bind_info values('select * from test . t where a <= ?', 'select * from test.t ignore index(idx) where a <= 1', '', 'enabled', '2021-06-04 17:04:45.334', '2021-06-04 17:04:45.334', 'utf8', 'utf8_general_ci', 'manual')`) - - upgradeToVer67(se, version66) - - r := MustExecToRecodeSet(t, se, `select original_sql, bind_sql, default_db, status, create_time from mysql.bind_info where source != 'builtin' order by create_time`) - req := r.NewChunk(nil) - require.NoError(t, r.Next(ctx, req)) - require.Equal(t, 3, req.NumRows()) - row := req.GetRow(0) - require.Equal(t, "select * from `test` . `t`", row.GetString(0)) - require.Equal(t, "SELECT /*+ use_index(`t` `idx_b`)*/ * FROM `test`.`t`", row.GetString(1)) - require.Equal(t, "", row.GetString(2)) - require.Equal(t, bindinfo.Enabled, row.GetString(3)) - require.Equal(t, "2021-01-04 14:50:58.257", row.GetTime(4).String()) - row = req.GetRow(1) - require.Equal(t, "select * from `test` . `t` where `a` < ?", row.GetString(0)) - require.Equal(t, "SELECT * FROM `test`.`t` IGNORE INDEX (`idx`) WHERE `a` < 1", row.GetString(1)) - require.Equal(t, "", row.GetString(2)) - require.Equal(t, bindinfo.Enabled, row.GetString(3)) - require.Equal(t, "2021-06-04 17:04:43.335", row.GetTime(4).String()) - row = req.GetRow(2) - require.Equal(t, "select * from `test` . `t` where `a` <= ?", row.GetString(0)) - require.Equal(t, "SELECT * FROM `test`.`t` IGNORE INDEX (`idx`) WHERE `a` <= 1", row.GetString(1)) - require.Equal(t, "", row.GetString(2)) - require.Equal(t, bindinfo.Enabled, row.GetString(3)) - require.Equal(t, "2021-06-04 17:04:45.334", row.GetTime(4).String()) - - require.NoError(t, r.Close()) - MustExec(t, se, "delete from mysql.bind_info where original_sql = 'select * from test . t'") -} - func TestUpgradeClusteredIndexDefaultValue(t *testing.T) { store, dom := CreateStoreAndBootstrap(t) defer func() { require.NoError(t, store.Close()) }() @@ -922,33 +873,6 @@ func testIndexMergeUpgradeFrom400To540(t *testing.T, enable bool) { } } -func TestUpgradeToVer85(t *testing.T) { - ctx := context.Background() - store, dom := CreateStoreAndBootstrap(t) - defer func() { require.NoError(t, store.Close()) }() - defer dom.Close() - se := CreateSessionAndSetID(t, store) - MustExec(t, se, "alter table mysql.bind_info drop column if exists plan_digest") - MustExec(t, se, "alter table mysql.bind_info drop column if exists sql_digest") - - MustExec(t, se, `insert into mysql.bind_info values('select * from t', 'select /*+ use_index(t, idx_a)*/ * from t', 'test', 'using', '2021-01-04 14:50:58.257', '2021-01-04 14:50:58.257', 'utf8', 'utf8_general_ci', 'manual')`) - MustExec(t, se, `insert into mysql.bind_info values('select * from t1', 'select /*+ use_index(t1, idx_a)*/ * from t1', 'test', 'enabled', '2021-01-05 14:50:58.257', '2021-01-05 14:50:58.257', 'utf8', 'utf8_general_ci', 'manual')`) - MustExec(t, se, `insert into mysql.bind_info values('select * from t2', 'select /*+ use_index(t2, idx_a)*/ * from t2', 'test', 'disabled', '2021-01-06 14:50:58.257', '2021-01-06 14:50:58.257', 'utf8', 'utf8_general_ci', 'manual')`) - MustExec(t, se, `insert into mysql.bind_info values('select * from t3', 'select /*+ use_index(t3, idx_a)*/ * from t3', 'test', 'deleted', '2021-01-07 14:50:58.257', '2021-01-07 14:50:58.257', 'utf8', 'utf8_general_ci', 'manual')`) - MustExec(t, se, `insert into mysql.bind_info values('select * from t4', 'select /*+ use_index(t4, idx_a)*/ * from t4', 'test', 'invalid', '2021-01-08 14:50:58.257', '2021-01-08 14:50:58.257', 'utf8', 'utf8_general_ci', 'manual')`) - upgradeToVer85(se, version84) - - r := MustExecToRecodeSet(t, se, `select count(*) from mysql.bind_info where status = 'enabled'`) - req := r.NewChunk(nil) - require.NoError(t, r.Next(ctx, req)) - require.Equal(t, 1, req.NumRows()) - row := req.GetRow(0) - require.Equal(t, int64(2), row.GetInt64(0)) - - require.NoError(t, r.Close()) - MustExec(t, se, "delete from mysql.bind_info where default_db = 'test'") -} - func TestTiDBEnablePagingVariable(t *testing.T) { store, dom := CreateStoreAndBootstrap(t) se := CreateSessionAndSetID(t, store) @@ -2005,90 +1929,6 @@ func TestTiDBUpgradeToVer170(t *testing.T) { dom.Close() } -func TestTiDBBindingInListToVer175(t *testing.T) { - ctx := context.Background() - store, dom := CreateStoreAndBootstrap(t) - defer func() { require.NoError(t, store.Close()) }() - - // bootstrap as version174 - ver174 := version174 - seV174 := CreateSessionAndSetID(t, store) - txn, err := store.Begin() - require.NoError(t, err) - m := meta.NewMutator(txn) - err = m.FinishBootstrap(int64(ver174)) - require.NoError(t, err) - revertVersionAndVariables(t, seV174, ver174) - err = txn.Commit(context.Background()) - require.NoError(t, err) - unsetStoreBootstrapped(store.UUID()) - - // create some bindings at version174 - MustExec(t, seV174, "use test") - MustExec(t, seV174, "create table t (a int, b int, c int, key(c))") - _, digest := parser.NormalizeDigestForBinding("SELECT * FROM `test`.`t` WHERE `a` IN (1,2,3)") - MustExec(t, seV174, fmt.Sprintf("insert into mysql.bind_info values ('select * from `test` . `t` where `a` in ( ... )', 'SELECT /*+ use_index(`t` `c`)*/ * FROM `test`.`t` WHERE `a` IN (1,2,3)', 'test', 'enabled', '2023-09-13 14:41:38.319', '2023-09-13 14:41:35.319', 'utf8', 'utf8_general_ci', 'manual', '%s', '', null)", digest.String())) - _, digest = parser.NormalizeDigestForBinding("SELECT * FROM `test`.`t` WHERE `a` IN (1)") - MustExec(t, seV174, fmt.Sprintf("insert into mysql.bind_info values ('select * from `test` . `t` where `a` in ( ? )', 'SELECT /*+ use_index(`t` `c`)*/ * FROM `test`.`t` WHERE `a` IN (1)', 'test', 'enabled', '2023-09-13 14:41:38.319', '2023-09-13 14:41:36.319', 'utf8', 'utf8_general_ci', 'manual', '%s', '', null)", digest.String())) - _, digest = parser.NormalizeDigestForBinding("SELECT * FROM `test`.`t` WHERE `a` IN (1) AND `b` IN (1,2,3)") - MustExec(t, seV174, fmt.Sprintf("insert into mysql.bind_info values ('select * from `test` . `t` where `a` in ( ? ) and `b` in ( ... )', 'SELECT /*+ use_index(`t` `c`)*/ * FROM `test`.`t` WHERE `a` IN (1) AND `b` IN (1,2,3)', 'test', 'enabled', '2023-09-13 14:41:37.319', '2023-09-13 14:41:38.319', 'utf8', 'utf8_general_ci', 'manual', '%s', '', null)", digest.String())) - - showBindings := func(s sessiontypes.Session) (records []string) { - MustExec(t, s, "admin reload bindings") - res := MustExecToRecodeSet(t, s, "show global bindings") - chk := res.NewChunk(nil) - for { - require.NoError(t, res.Next(ctx, chk)) - if chk.NumRows() == 0 { - break - } - for i := 0; i < chk.NumRows(); i++ { - originalSQL := chk.GetRow(i).GetString(0) - bindSQL := chk.GetRow(i).GetString(1) - records = append(records, fmt.Sprintf("%s:%s", bindSQL, originalSQL)) - } - } - require.NoError(t, res.Close()) - sort.Strings(records) - return - } - bindings := showBindings(seV174) - // on ver174, `in (1)` and `in (1,2,3)` have different normalized results: `in (?)` and `in (...)` - require.Equal(t, []string{"SELECT /*+ use_index(`t` `c`)*/ * FROM `test`.`t` WHERE `a` IN (1) AND `b` IN (1,2,3):select * from `test` . `t` where `a` in ( ? ) and `b` in ( ... )", - "SELECT /*+ use_index(`t` `c`)*/ * FROM `test`.`t` WHERE `a` IN (1):select * from `test` . `t` where `a` in ( ? )", - "SELECT /*+ use_index(`t` `c`)*/ * FROM `test`.`t` WHERE `a` IN (1,2,3):select * from `test` . `t` where `a` in ( ... )"}, bindings) - - // upgrade to ver175 - dom.Close() - domCurVer, err := BootstrapSession(store) - require.NoError(t, err) - defer domCurVer.Close() - seCurVer := CreateSessionAndSetID(t, store) - ver, err := getBootstrapVersion(seCurVer) - require.NoError(t, err) - require.Equal(t, currentBootstrapVersion, ver) - - // `in (?)` becomes to `in ( ... )` - bindings = showBindings(seCurVer) - require.Equal(t, []string{"SELECT /*+ use_index(`t` `c`)*/ * FROM `test`.`t` WHERE `a` IN (1) AND `b` IN (1,2,3):select * from `test` . `t` where `a` in ( ... ) and `b` in ( ... )", - "SELECT /*+ use_index(`t` `c`)*/ * FROM `test`.`t` WHERE `a` IN (1):select * from `test` . `t` where `a` in ( ... )"}, bindings) - - planFromBinding := func(s sessiontypes.Session, q string) { - MustExec(t, s, q) - res := MustExecToRecodeSet(t, s, "select @@last_plan_from_binding") - chk := res.NewChunk(nil) - require.NoError(t, res.Next(ctx, chk)) - require.Equal(t, int64(1), chk.GetRow(0).GetInt64(0)) - require.NoError(t, res.Close()) - } - planFromBinding(seCurVer, "select * from test.t where a in (1)") - planFromBinding(seCurVer, "select * from test.t where a in (1,2,3)") - planFromBinding(seCurVer, "select * from test.t where a in (1,2,3,4,5,6,7)") - planFromBinding(seCurVer, "select * from test.t where a in (1,2,3,4,5,6,7) and b in(1)") - planFromBinding(seCurVer, "select * from test.t where a in (1,2,3,4,5,6,7) and b in(1,2,3,4)") - planFromBinding(seCurVer, "select * from test.t where a in (7) and b in(1,2,3,4)") -} - func TestTiDBUpgradeToVer176(t *testing.T) { store, do := CreateStoreAndBootstrap(t) defer func() { diff --git a/pkg/session/bootstraptest/bootstrap_upgrade_test.go b/pkg/session/bootstraptest/bootstrap_upgrade_test.go index 333dc3727f21b..8d0dc094677b7 100644 --- a/pkg/session/bootstraptest/bootstrap_upgrade_test.go +++ b/pkg/session/bootstraptest/bootstrap_upgrade_test.go @@ -876,32 +876,43 @@ func TestUpgradeWithCrossJoinDisabled(t *testing.T) { }() } -func TestUpgradeBindInfo(t *testing.T) { - fromVersion := 220 +func TestBindInfoUniqueIndex(t *testing.T) { + ctx := context.Background() store, dom := session.CreateStoreAndBootstrap(t) defer func() { require.NoError(t, store.Close()) }() + // bootstrap as version220 + ver220 := 220 seV220 := session.CreateSessionAndSetID(t, store) txn, err := store.Begin() require.NoError(t, err) m := meta.NewMutator(txn) - err = m.FinishBootstrap(int64(fromVersion)) + err = m.FinishBootstrap(int64(ver220)) require.NoError(t, err) - err = txn.Commit(context.Background()) - revertVersionAndVariables(t, seV220, fromVersion) + revertVersionAndVariables(t, seV220, ver220) + err = txn.Commit(ctx) require.NoError(t, err) session.MustExec(t, seV220, "ADMIN RELOAD BINDINGS;") session.UnsetStoreBootstrapped(store.UUID()) - ver, err := session.GetBootstrapVersion(seV220) - require.NoError(t, err) - require.Equal(t, int64(fromVersion), ver) + // remove the unique index on mysql.bind_info for testing + session.MustExec(t, seV220, "alter table mysql.bind_info drop index digest_index") + // insert duplicated values into mysql.bind_info + for _, sqlDigest := range []string{"null", "'x'", "'y'"} { + for _, planDigest := range []string{"null", "'x'", "'y'"} { + insertStmt := fmt.Sprintf(`insert into mysql.bind_info values ( + "sql", "bind_sql", "db", "disabled", NOW(), NOW(), "", "", "", %s, %s, null)`, + sqlDigest, planDigest) + session.MustExec(t, seV220, insertStmt) + session.MustExec(t, seV220, insertStmt) + } + } + // upgrade to current version dom.Close() - newVer, err := session.BootstrapSession(store) + domCurVer, err := session.BootstrapSession(store) require.NoError(t, err) - seLatestV := session.CreateSessionAndSetID(t, store) - ver, err = session.GetBootstrapVersion(seLatestV) + defer domCurVer.Close() + seCurVer := session.CreateSessionAndSetID(t, store) + ver, err := session.GetBootstrapVersion(seCurVer) require.NoError(t, err) require.Equal(t, session.CurrentBootstrapVersion, ver) - session.MustExec(t, seLatestV, "ADMIN RELOAD BINDINGS;") - newVer.Close() - seLatestV.Close() + session.MustExec(t, seCurVer, "ADMIN RELOAD BINDINGS;") } From 917c637bfcef3a276f37223730f7d5565eacb43f Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 14 Oct 2025 19:11:45 +0800 Subject: [PATCH 18/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/global_handle.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/bindinfo/global_handle.go b/pkg/bindinfo/global_handle.go index 91ba1193f6c58..584088b68fa04 100644 --- a/pkg/bindinfo/global_handle.go +++ b/pkg/bindinfo/global_handle.go @@ -307,7 +307,13 @@ func (h *globalBindingHandle) CreateGlobalBinding(sctx sessionctx.Context, bindi binding.CreateTime = now binding.UpdateTime = now - + var sqlDigest, planDigest any // null by default + if binding.SQLDigest != "" { + sqlDigest = binding.SQLDigest + } + if binding.PlanDigest != "" { + planDigest = binding.PlanDigest + } // Insert the Bindings to the storage. _, err = exec( sctx, @@ -323,8 +329,8 @@ func (h *globalBindingHandle) CreateGlobalBinding(sctx sessionctx.Context, bindi binding.Charset, binding.Collation, binding.Source, - binding.SQLDigest, - binding.PlanDigest, + sqlDigest, + planDigest, ) failpoint.Inject("CreateGlobalBindingNthFail", func(val failpoint.Value) { n := val.(int) From 18030ff536d8cdfd4a4965f64b5f9c87da433877 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 14 Oct 2025 21:21:03 +0800 Subject: [PATCH 19/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/capture_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/bindinfo/capture_test.go b/pkg/bindinfo/capture_test.go index c39ab547cc9e8..a6346dfbc4853 100644 --- a/pkg/bindinfo/capture_test.go +++ b/pkg/bindinfo/capture_test.go @@ -384,7 +384,7 @@ func TestConcurrentCapture(t *testing.T) { // Simulate an existing binding generated by concurrent CREATE BINDING, which has not been synchronized to current tidb-server yet. // Actually, it is more common to be generated by concurrent baseline capture, I use Manual just for simpler test verification. - tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t`', 'select * from `test` . `t`', '', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + + tk.MustExec("insert into mysql.bind_info (original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest)values('select * from `test` . `t`', 'select * from `test` . `t`', '', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + bindinfo.Manual + "', '', '')") tk.MustQuery("select original_sql, source from mysql.bind_info where source != 'builtin'").Check(testkit.Rows( "select * from `test` . `t` manual", From 3343a62b8b44edb0da3764cd742a1fb9622cfe3e Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 15 Oct 2025 11:09:33 +0800 Subject: [PATCH 20/29] update Signed-off-by: Weizhen Wang --- br/pkg/restore/snap_client/systable_restore_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/br/pkg/restore/snap_client/systable_restore_test.go b/br/pkg/restore/snap_client/systable_restore_test.go index c0df26e1d2386..0abbf32e6e717 100644 --- a/br/pkg/restore/snap_client/systable_restore_test.go +++ b/br/pkg/restore/snap_client/systable_restore_test.go @@ -116,5 +116,5 @@ func TestCheckSysTableCompatibility(t *testing.T) { // // The above variables are in the file br/pkg/restore/systable_restore.go func TestMonitorTheSystemTableIncremental(t *testing.T) { - require.Equal(t, int64(220), session.CurrentBootstrapVersion) + require.Equal(t, int64(221), session.CurrentBootstrapVersion) } From 47def2e9799503d89795ab474074410b38e0cc56 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 15 Oct 2025 12:06:55 +0800 Subject: [PATCH 21/29] update Signed-off-by: Weizhen Wang --- pkg/executor/infoschema_reader_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/executor/infoschema_reader_test.go b/pkg/executor/infoschema_reader_test.go index bff7e86c738d0..ac14995abcdcc 100644 --- a/pkg/executor/infoschema_reader_test.go +++ b/pkg/executor/infoschema_reader_test.go @@ -708,7 +708,7 @@ func TestIndexUsageTable(t *testing.T) { testkit.RowsWithSep("|", "test|idt2|idx_4")) tk.MustQuery(`select count(*) from information_schema.tidb_index_usage;`).Check( - testkit.RowsWithSep("|", "80")) + testkit.RowsWithSep("|", "81")) tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, INDEX_NAME from information_schema.tidb_index_usage where TABLE_SCHEMA = 'test1';`).Check(testkit.Rows()) From ef8164d41368123f2a16bd388150383af87fa7ec Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 15 Oct 2025 12:27:38 +0800 Subject: [PATCH 22/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/global_handle_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/bindinfo/global_handle_test.go b/pkg/bindinfo/global_handle_test.go index edb7af4a7a5d7..428b1df5b9d9b 100644 --- a/pkg/bindinfo/global_handle_test.go +++ b/pkg/bindinfo/global_handle_test.go @@ -94,7 +94,7 @@ func TestBindingLastUpdateTimeWithInvalidBind(t *testing.T) { updateTime0 := rows0[0][1] require.Equal(t, updateTime0, "0000-00-00 00:00:00") - tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t`', 'select * from `test` . `t` use index(`idx`)', 'test', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + + tk.MustExec("insert into mysql.bind_info (original_sql,bind_sql,default_db,status,create_time,update_time,charset,collation,source, sql_digest, plan_digest)values('select * from `test` . `t`', 'select * from `test` . `t` use index(`idx`)', 'test', 'enabled', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + bindinfo.Manual + "', '', '')") tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -262,9 +262,9 @@ func TestSetBindingStatusWithoutBindingInCache(t *testing.T) { // Simulate creating bindings on other machines _, sqlDigest := parser.NormalizeDigestForBinding("select * from `test` . `t` where `a` > ?") - tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t` where `a` > ?', 'SELECT /*+ USE_INDEX(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` > 10', 'test', 'deleted', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + + tk.MustExec("insert into mysql.bind_info (original_sql,bind_sql,default_db,status,create_time,update_time,charset,collation,source, sql_digest, plan_digest)values('select * from `test` . `t` where `a` > ?', 'SELECT /*+ USE_INDEX(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` > 10', 'test', 'deleted', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") - tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t` where `a` > ?', 'SELECT /*+ USE_INDEX(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` > 10', 'test', 'enabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" + + tk.MustExec("insert into mysql.bind_info (original_sql,bind_sql,default_db,status,create_time,update_time,charset,collation,source, sql_digest, plan_digest)values('select * from `test` . `t` where `a` > ?', 'SELECT /*+ USE_INDEX(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` > 10', 'test', 'enabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") dom.BindHandle().Clear() tk.MustExec("set binding disabled for select * from t where a > 10") @@ -277,9 +277,9 @@ func TestSetBindingStatusWithoutBindingInCache(t *testing.T) { internal.UtilCleanBindingEnv(tk, dom) // Simulate creating bindings on other machines - tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t` where `a` > ?', 'SELECT * FROM `test`.`t` WHERE `a` > 10', 'test', 'deleted', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + + tk.MustExec("insert into mysql.bind_info (original_sql,bind_sql,default_db,status,create_time,update_time,charset,collation,source, sql_digest, plan_digest)values('select * from `test` . `t` where `a` > ?', 'SELECT * FROM `test`.`t` WHERE `a` > 10', 'test', 'deleted', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") - tk.MustExec("insert into mysql.bind_info values('select * from `test` . `t` where `a` > ?', 'SELECT * FROM `test`.`t` WHERE `a` > 10', 'test', 'disabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" + + tk.MustExec("insert into mysql.bind_info (original_sql,bind_sql,default_db,status,create_time,update_time,charset,collation,source, sql_digest, plan_digest)values('select * from `test` . `t` where `a` > ?', 'SELECT * FROM `test`.`t` WHERE `a` > 10', 'test', 'disabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") dom.BindHandle().Clear() tk.MustExec("set binding enabled for select * from t where a > 10") From 20427bada654751aa94feaae6672b075f2d70337 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 15 Oct 2025 13:41:29 +0800 Subject: [PATCH 23/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/binding.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/bindinfo/binding.go b/pkg/bindinfo/binding.go index b664777ed561f..3ecff62ca7b30 100644 --- a/pkg/bindinfo/binding.go +++ b/pkg/bindinfo/binding.go @@ -78,9 +78,9 @@ type Binding struct { // It is nil if this binding has never been used. // It is updated when this binding is used. // It is used to update the `last_used_time` field in mysql.bind_info table. - LastUsedAt *time.Time + LastUsedAt *time.Time `json:"-"` // LastSavedAt records the last time when this binding is saved into storage. - LastSavedAt *time.Time + LastSavedAt *time.Time `json:"-"` } func (b *Binding) isSame(rb *Binding) bool { From 8dd92ea01297d5221c69712485d80254798b2d9b Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 15 Oct 2025 14:08:45 +0800 Subject: [PATCH 24/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/global_handle_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/bindinfo/global_handle_test.go b/pkg/bindinfo/global_handle_test.go index 428b1df5b9d9b..e2deba815c5b5 100644 --- a/pkg/bindinfo/global_handle_test.go +++ b/pkg/bindinfo/global_handle_test.go @@ -264,8 +264,6 @@ func TestSetBindingStatusWithoutBindingInCache(t *testing.T) { _, sqlDigest := parser.NormalizeDigestForBinding("select * from `test` . `t` where `a` > ?") tk.MustExec("insert into mysql.bind_info (original_sql,bind_sql,default_db,status,create_time,update_time,charset,collation,source, sql_digest, plan_digest)values('select * from `test` . `t` where `a` > ?', 'SELECT /*+ USE_INDEX(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` > 10', 'test', 'deleted', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") - tk.MustExec("insert into mysql.bind_info (original_sql,bind_sql,default_db,status,create_time,update_time,charset,collation,source, sql_digest, plan_digest)values('select * from `test` . `t` where `a` > ?', 'SELECT /*+ USE_INDEX(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` > 10', 'test', 'enabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" + - bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") dom.BindHandle().Clear() tk.MustExec("set binding disabled for select * from t where a > 10") tk.MustExec("admin reload bindings") From 2d39a627ea0c31df480b50416d2cb410b38c9808 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 15 Oct 2025 14:16:51 +0800 Subject: [PATCH 25/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/global_handle_test.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/bindinfo/global_handle_test.go b/pkg/bindinfo/global_handle_test.go index e2deba815c5b5..6caad45288ddc 100644 --- a/pkg/bindinfo/global_handle_test.go +++ b/pkg/bindinfo/global_handle_test.go @@ -262,9 +262,8 @@ func TestSetBindingStatusWithoutBindingInCache(t *testing.T) { // Simulate creating bindings on other machines _, sqlDigest := parser.NormalizeDigestForBinding("select * from `test` . `t` where `a` > ?") - tk.MustExec("insert into mysql.bind_info (original_sql,bind_sql,default_db,status,create_time,update_time,charset,collation,source, sql_digest, plan_digest)values('select * from `test` . `t` where `a` > ?', 'SELECT /*+ USE_INDEX(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` > 10', 'test', 'deleted', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + + tk.MustExec("insert into mysql.bind_info (original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest) values('select * from `test` . `t` where `a` > ?', 'SELECT /*+ USE_INDEX(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` > 10', 'test', 'enabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") - dom.BindHandle().Clear() tk.MustExec("set binding disabled for select * from t where a > 10") tk.MustExec("admin reload bindings") rows := tk.MustQuery("show global bindings").Rows() @@ -275,9 +274,7 @@ func TestSetBindingStatusWithoutBindingInCache(t *testing.T) { internal.UtilCleanBindingEnv(tk, dom) // Simulate creating bindings on other machines - tk.MustExec("insert into mysql.bind_info (original_sql,bind_sql,default_db,status,create_time,update_time,charset,collation,source, sql_digest, plan_digest)values('select * from `test` . `t` where `a` > ?', 'SELECT * FROM `test`.`t` WHERE `a` > 10', 'test', 'deleted', '2000-01-01 09:00:00', '2000-01-01 09:00:00', '', '','" + - bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") - tk.MustExec("insert into mysql.bind_info (original_sql,bind_sql,default_db,status,create_time,update_time,charset,collation,source, sql_digest, plan_digest)values('select * from `test` . `t` where `a` > ?', 'SELECT * FROM `test`.`t` WHERE `a` > 10', 'test', 'disabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" + + tk.MustExec("insert into mysql.bind_info (original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest) values('select * from `test` . `t` where `a` > ?', 'SELECT * FROM `test`.`t` WHERE `a` > 10', 'test', 'disabled', '2000-01-02 09:00:00', '2000-01-02 09:00:00', '', '','" + bindinfo.Manual + "', '" + sqlDigest.String() + "', '')") dom.BindHandle().Clear() tk.MustExec("set binding enabled for select * from t where a > 10") From e2016e769131fadbf7c3c53d74d2a1f876a56c35 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 15 Oct 2025 14:51:55 +0800 Subject: [PATCH 26/29] update Signed-off-by: Weizhen Wang --- pkg/executor/infoschema_reader_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/executor/infoschema_reader_test.go b/pkg/executor/infoschema_reader_test.go index ac14995abcdcc..d469e49ad6890 100644 --- a/pkg/executor/infoschema_reader_test.go +++ b/pkg/executor/infoschema_reader_test.go @@ -661,7 +661,7 @@ func TestColumnTable(t *testing.T) { testkit.RowsWithSep("|", "test|tbl1|col_2")) tk.MustQuery(`select count(*) from information_schema.columns;`).Check( - testkit.RowsWithSep("|", "4986")) + testkit.RowsWithSep("|", "4987")) } func TestIndexUsageTable(t *testing.T) { From ca2283c46b1e1e86be0cd292daaa764b87cfd727 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 21 Oct 2025 11:16:38 +0800 Subject: [PATCH 27/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/bind_usage_info_test.go | 13 +++++++++++++ pkg/bindinfo/binding_cache.go | 15 ++++++++++----- pkg/bindinfo/global_handle.go | 3 +++ pkg/sessionctx/variable/sysvar.go | 14 ++++++++++++++ pkg/sessionctx/variable/tidb_vars.go | 18 ++++++++++-------- 5 files changed, 50 insertions(+), 13 deletions(-) diff --git a/pkg/bindinfo/bind_usage_info_test.go b/pkg/bindinfo/bind_usage_info_test.go index 2f4f96e689c3e..c603e1a16abc5 100644 --- a/pkg/bindinfo/bind_usage_info_test.go +++ b/pkg/bindinfo/bind_usage_info_test.go @@ -99,6 +99,19 @@ func TestBindUsageInfo(t *testing.T) { rows := tk.MustQuery( fmt.Sprintf(`select * from mysql.bind_info where original_sql != '%s' and last_used_date is not null`, bindinfo.BuiltinPseudoSQL4BindLock)).Rows() require.Len(t, rows, 3) + // Set all last_used_date to null to simulate that the bindinfo in storage is not updated. + resetAllLastUsedData(tk) + tk.MustExec(`set @@global.tidb_enable_binding_usage=0;`) + for range 5 { + tk.MustExec("execute stmt1;") + tk.MustExec("execute stmt2;") + tk.MustExec("execute stmt3;") + tk.MustExec("select * from t1, t2, t3, t4, t5") + time.Sleep(1 * time.Second) + require.NoError(t, bindingHandle.UpdateBindingUsageInfoToStorage()) + tk.MustQuery(fmt.Sprintf(`select last_used_date from mysql.bind_info where original_sql != '%s' and last_used_date is not null`, bindinfo.BuiltinPseudoSQL4BindLock)). + Check(testkit.Rows()) + } } func resetAllLastUsedData(tk *testkit.TestKit) { diff --git a/pkg/bindinfo/binding_cache.go b/pkg/bindinfo/binding_cache.go index 161d27dfad922..f2c3ae677e8c1 100644 --- a/pkg/bindinfo/binding_cache.go +++ b/pkg/bindinfo/binding_cache.go @@ -117,6 +117,7 @@ func (fbc *fuzzyBindingCache) getFromMemory(sctx sessionctx.Context, fuzzyDigest } leastWildcards := len(tableNames) + 1 enableFuzzyBinding := sctx.GetSessionVars().EnableFuzzyBinding + enableBindingUsage := variable.EnableBindingUsage.Load() for _, sqlDigest := range fbc.fuzzy2SQLDigests[fuzzyDigest] { bindings := bindingCache.GetBinding(sqlDigest) if intest.InTest { @@ -137,16 +138,20 @@ func (fbc *fuzzyBindingCache) getFromMemory(sctx sessionctx.Context, fuzzyDigest } if matched && numWildcards < leastWildcards { matchedBinding = binding - binding.UpdateLastUsedAt() - bindings[idx] = binding + if enableBindingUsage { + binding.UpdateLastUsedAt() + bindings[idx] = binding + } isMatched = true leastWildcards = numWildcards break } } - err := bindingCache.SetBinding(sqlDigest, bindings) // update the last used time - if err != nil { - logutil.BindLogger().Warn("bindingCache.SetBinding", zap.Error(err)) + if enableBindingUsage { + err := bindingCache.SetBinding(sqlDigest, bindings) // update the last used time + if err != nil { + logutil.BindLogger().Warn("bindingCache.SetBinding", zap.Error(err)) + } } } else { missingSQLDigest = append(missingSQLDigest, sqlDigest) diff --git a/pkg/bindinfo/global_handle.go b/pkg/bindinfo/global_handle.go index 584088b68fa04..c8c94e6ae170b 100644 --- a/pkg/bindinfo/global_handle.go +++ b/pkg/bindinfo/global_handle.go @@ -265,6 +265,9 @@ func (h *globalBindingHandle) UpdateBindingUsageInfoToStorage() error { logutil.BindLogger().Warn("panic when update usage info for binding", zap.Any("recover", r)) } }() + if !variable.EnableBindingUsage.Load() { + return nil + } bindings := h.GetAllGlobalBindings() return h.updateBindingUsageInfoToStorage(bindings) } diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index ecd88e46eeae2..e6eb3a475b94f 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -1687,6 +1687,18 @@ var defaultSysVars = []*SysVar{ vars.LoadBindingTimeout = uint64(timeoutMS) return nil }}, + { + Scope: ScopeGlobal, + Name: TiDBEnableBindingUsage, + Value: BoolToOnOff(DefTiDBEnableBindingUsage), + Type: TypeBool, + IsHintUpdatableVerified: false, + SetGlobal: func(_ context.Context, s *SessionVars, val string) error { + EnableBindingUsage.Store(TiDBOptOn(val)) + return nil + }, GetGlobal: func(_ context.Context, s *SessionVars) (string, error) { + return BoolToOnOff(EnableBindingUsage.Load()), nil + }}, { Scope: ScopeGlobal | ScopeSession, Name: MaxExecutionTime, @@ -3871,6 +3883,8 @@ const ( TiKVClientReadTimeout = "tikv_client_read_timeout" // TiDBLoadBindingTimeout is the name of the 'tidb_load_binding_timeout' system variable. TiDBLoadBindingTimeout = "tidb_load_binding_timeout" + // TiDBEnableBindingUsage is the name of the 'tidb_enable_binding_usage' system variable. + TiDBEnableBindingUsage = "tidb_enable_binding_usage" // ReadOnly is the name of the 'read_only' system variable. ReadOnly = "read_only" // DefaultAuthPlugin is the name of 'default_authentication_plugin' system variable. diff --git a/pkg/sessionctx/variable/tidb_vars.go b/pkg/sessionctx/variable/tidb_vars.go index 574c5c87cf6cc..8191f5ac152b3 100644 --- a/pkg/sessionctx/variable/tidb_vars.go +++ b/pkg/sessionctx/variable/tidb_vars.go @@ -1379,7 +1379,7 @@ const ( DefTiDBHashAggPartialConcurrency = ConcurrencyUnset DefTiDBHashAggFinalConcurrency = ConcurrencyUnset DefTiDBWindowConcurrency = ConcurrencyUnset - DefTiDBMergeJoinConcurrency = 1 // disable optimization by default + DefTiDBMergeJoinConcurrency = 1 // disable optimization by default DefTiDBStreamAggConcurrency = 1 DefTiDBForcePriority = mysql.NoPriority DefEnableWindowFunction = true @@ -1390,23 +1390,23 @@ const ( DefTiDBDDLSlowOprThreshold = 300 DefTiDBUseFastAnalyze = false DefTiDBSkipIsolationLevelCheck = false - DefTiDBExpensiveQueryTimeThreshold = 60 // 60s - DefTiDBExpensiveTxnTimeThreshold = 60 * 10 // 10 minutes + DefTiDBExpensiveQueryTimeThreshold = 60 // 60s + DefTiDBExpensiveTxnTimeThreshold = 60 * 10 // 10 minutes DefTiDBScatterRegion = ScatterOff DefTiDBWaitSplitRegionFinish = true - DefWaitSplitRegionTimeout = 300 // 300s + DefWaitSplitRegionTimeout = 300 // 300s DefTiDBEnableNoopFuncs = Off DefTiDBEnableNoopVariables = true DefTiDBAllowRemoveAutoInc = false DefTiDBUsePlanBaselines = true DefTiDBEvolvePlanBaselines = false - DefTiDBEvolvePlanTaskMaxTime = 600 // 600s + DefTiDBEvolvePlanTaskMaxTime = 600 // 600s DefTiDBEvolvePlanTaskStartTime = "00:00 +0000" DefTiDBEvolvePlanTaskEndTime = "23:59 +0000" - DefInnodbLockWaitTimeout = 50 // 50s + DefInnodbLockWaitTimeout = 50 // 50s DefTiDBStoreLimit = 0 - DefTiDBMetricSchemaStep = 60 // 60s - DefTiDBMetricSchemaRangeDuration = 60 // 60s + DefTiDBMetricSchemaStep = 60 // 60s + DefTiDBMetricSchemaRangeDuration = 60 // 60s DefTiDBFoundInPlanCache = false DefTiDBFoundInBinding = false DefTiDBEnableCollectExecutionInfo = true @@ -1614,6 +1614,7 @@ const ( DefTiDBEnableSharedLockPromotion = false DefTiDBTSOClientRPCMode = TSOClientRPCModeDefault DefTiDBLoadBindingTimeout = 200 + DefTiDBEnableBindingUsage = true ) // Process global variables. @@ -1736,6 +1737,7 @@ var ( SchemaCacheSize = atomic.NewUint64(DefTiDBSchemaCacheSize) SchemaCacheSizeOriginText = atomic.NewString(strconv.Itoa(DefTiDBSchemaCacheSize)) + EnableBindingUsage = atomic.NewBool(DefTiDBEnableBindingUsage) ) var ( From e12e5cb780e75d7cf6145537bb0134ed2b809b76 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 21 Oct 2025 11:32:29 +0800 Subject: [PATCH 28/29] update Signed-off-by: Weizhen Wang --- pkg/sessionctx/variable/tidb_vars.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/sessionctx/variable/tidb_vars.go b/pkg/sessionctx/variable/tidb_vars.go index 8191f5ac152b3..777b7a033da3a 100644 --- a/pkg/sessionctx/variable/tidb_vars.go +++ b/pkg/sessionctx/variable/tidb_vars.go @@ -1379,7 +1379,7 @@ const ( DefTiDBHashAggPartialConcurrency = ConcurrencyUnset DefTiDBHashAggFinalConcurrency = ConcurrencyUnset DefTiDBWindowConcurrency = ConcurrencyUnset - DefTiDBMergeJoinConcurrency = 1 // disable optimization by default + DefTiDBMergeJoinConcurrency = 1 // disable optimization by default DefTiDBStreamAggConcurrency = 1 DefTiDBForcePriority = mysql.NoPriority DefEnableWindowFunction = true @@ -1390,23 +1390,23 @@ const ( DefTiDBDDLSlowOprThreshold = 300 DefTiDBUseFastAnalyze = false DefTiDBSkipIsolationLevelCheck = false - DefTiDBExpensiveQueryTimeThreshold = 60 // 60s - DefTiDBExpensiveTxnTimeThreshold = 60 * 10 // 10 minutes + DefTiDBExpensiveQueryTimeThreshold = 60 // 60s + DefTiDBExpensiveTxnTimeThreshold = 60 * 10 // 10 minutes DefTiDBScatterRegion = ScatterOff DefTiDBWaitSplitRegionFinish = true - DefWaitSplitRegionTimeout = 300 // 300s + DefWaitSplitRegionTimeout = 300 // 300s DefTiDBEnableNoopFuncs = Off DefTiDBEnableNoopVariables = true DefTiDBAllowRemoveAutoInc = false DefTiDBUsePlanBaselines = true DefTiDBEvolvePlanBaselines = false - DefTiDBEvolvePlanTaskMaxTime = 600 // 600s + DefTiDBEvolvePlanTaskMaxTime = 600 // 600s DefTiDBEvolvePlanTaskStartTime = "00:00 +0000" DefTiDBEvolvePlanTaskEndTime = "23:59 +0000" - DefInnodbLockWaitTimeout = 50 // 50s + DefInnodbLockWaitTimeout = 50 // 50s DefTiDBStoreLimit = 0 - DefTiDBMetricSchemaStep = 60 // 60s - DefTiDBMetricSchemaRangeDuration = 60 // 60s + DefTiDBMetricSchemaStep = 60 // 60s + DefTiDBMetricSchemaRangeDuration = 60 // 60s DefTiDBFoundInPlanCache = false DefTiDBFoundInBinding = false DefTiDBEnableCollectExecutionInfo = true From 963f53d8639c5f5ef98a64f38689c5c6f4c50f6c Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 21 Oct 2025 11:53:32 +0800 Subject: [PATCH 29/29] update Signed-off-by: Weizhen Wang --- pkg/bindinfo/fuzzy_binding_test.go | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/pkg/bindinfo/fuzzy_binding_test.go b/pkg/bindinfo/fuzzy_binding_test.go index e3efa43f1a3f3..50bef6df8d296 100644 --- a/pkg/bindinfo/fuzzy_binding_test.go +++ b/pkg/bindinfo/fuzzy_binding_test.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/tidb/pkg/bindinfo" "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util" @@ -340,3 +341,42 @@ func TestFuzzyBindingPlanCache(t *testing.T) { tk.MustExec(`execute stmt using @v`) hasPlan("IndexFullScan", "index:c(c)") } + +func BenchmarkFuzzyBindingBasic(b *testing.B) { + b.ReportAllocs() + store := testkit.CreateMockStore(b) + tk1 := testkit.NewTestKit(b, store) + tk1.MustExec(`use test`) + tk1.MustExec(`create table t (a int, b int, c int, d int, e int, key(a), key(b), key(c), key(d), key(e))`) + tk1.MustExec(`create database test1`) + tk1.MustExec(`use test1`) + tk1.MustExec(`create table t (a int, b int, c int, d int, e int, key(a), key(b), key(c), key(d), key(e))`) + tk1.MustExec(`create database test2`) + tk1.MustExec(`use test2`) + tk1.MustExec(`create table t (a int, b int, c int, d int, e int, key(a), key(b), key(c), key(d), key(e))`) + + variable.EnableBindingUsage.Store(true) + for _, scope := range []string{"", "session", "global"} { + tk := testkit.NewTestKit(b, store) + for _, idx := range []string{"a", "b", "c", "d", "e"} { + tk.MustExec("use test") + tk.MustExec(fmt.Sprintf(`create %v binding using select /*+ use_index(t, %v) */ * from *.t`, scope, idx)) + for _, useDB := range []string{"test", "test1", "test2"} { + b.ResetTimer() + for i := 0; i < b.N; i++ { + tk.MustExec("use " + useDB) + for _, testDB := range []string{"", "test.", "test1.", "test2."} { + tk.MustExec(`set @@tidb_opt_enable_fuzzy_binding=1`) // enabled + tk.MustUseIndex(fmt.Sprintf("select * from %vt", testDB), idx) + tk.MustQuery(`select @@last_plan_from_binding`).Check(testkit.Rows("1")) + tk.MustUseIndex(fmt.Sprintf("select * from %vt", testDB), idx) + tk.MustQuery(`show warnings`).Check(testkit.Rows()) // no warning + tk.MustExec(`set @@tidb_opt_enable_fuzzy_binding=0`) // disabled + tk.MustQuery(fmt.Sprintf("select * from %vt", testDB)) + tk.MustQuery(`select @@last_plan_from_binding`).Check(testkit.Rows("0")) + } + } + } + } + } +}