chore: remove unique dump because no one use

Change-Id: Ic7e10c8401a75dddbd3fce8c2018ecc86e998824

Author: Thearas

cmd/dump.go

@@ -52,17 +52,15 @@ type Dump struct {
 	SSHPassword   string
 	SSHPrivateKey string
 
-	DumpSchema           bool
-	DumpStats            bool
-	DumpQuery            bool
-	QueryOutputMode      string
-	QueryUniqueNormalize bool
-	QueryMinDuration_    time.Duration
-	QueryMinDurationMs   int
-	QueryStates          []string
-	OnlySelect           bool
-	Strict               bool
-	From, To             string
+	DumpSchema         bool
+	DumpStats          bool
+	DumpQuery          bool
+	QueryMinDuration_  time.Duration
+	QueryMinDurationMs int
+	QueryStates        []string
+	OnlySelect         bool
+	Strict             bool
+	From, To           string
 
 	Clean bool
 }
@@ -144,8 +142,6 @@ func init() {
 	pFlags.BoolVar(&DumpConfig.DumpSchema, "dump-schema", false, "Dump schema")
 	pFlags.BoolVar(&DumpConfig.DumpStats, "dump-stats", true, "Dump schema stats, only take effect when '--dump-schema=true'")
 	pFlags.BoolVar(&DumpConfig.DumpQuery, "dump-query", false, "Dump query from audit log")
-	pFlags.StringVar(&DumpConfig.QueryOutputMode, "query-output-mode", "default", "Dump query output mode, one of [default, unique]")
-	pFlags.BoolVar(&DumpConfig.QueryUniqueNormalize, "query-unique-normalize", false, "Regard 'select 1 from b where a = 1' as 'select ? from b where a = ?' for unique, only take effect when '--query-output-mode=unique'")
 	pFlags.DurationVar(&DumpConfig.QueryMinDuration_, "query-min-duration", 0, "Dump queries which execution duration is greater than or equal to")
 	pFlags.StringSliceVar(&DumpConfig.QueryStates, "query-states", []string{}, "Dump queries with states, like 'ok', 'eof' and 'err'")
 	pFlags.BoolVar(&DumpConfig.OnlySelect, "only-select", true, "Only dump SELECT queries")
@@ -361,8 +357,6 @@ func dumpQueries(ctx context.Context) ([][]string, error) {
 		DBs:                GlobalConfig.DBs,
 		QueryMinDurationMs: DumpConfig.QueryMinDurationMs,
 		QueryStates:        DumpConfig.QueryStates,
-		Unique:             DumpConfig.QueryOutputMode == "unique",
-		UniqueNormalize:    DumpConfig.QueryUniqueNormalize,
 		Unescape:           DumpConfig.AuditLogUnescape,
 		OnlySelect:         DumpConfig.OnlySelect,
 		Strict:             DumpConfig.Strict,
@@ -380,9 +374,6 @@ func dumpQueriesFromTable(ctx context.Context, opts src.AuditLogScanOpts) ([][]s
 	if opts.From == "" || opts.To == "" {
 		return nil, errors.New("Must specific both '--from' and '--to' when dumping from audit log table")
 	}
-	if opts.Unique {
-		return nil, errors.New("Not yet support '--query-output-mode=unique' with '--audit-log-table'")
-	}
 
 	dbTable := strings.SplitN(DumpConfig.AuditLogTable, ".", 2)
 	dbname, table := dbTable[0], dbTable[1]

src/auditlog.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/dlclark/regexp2"
 	"github.com/edsrzf/mmap-go"
-	tidbparser "github.com/pingcap/tidb/pkg/parser"
 	"github.com/samber/lo"
 	"github.com/sirupsen/logrus"
 	"github.com/zeebo/blake3"
@@ -30,9 +29,6 @@ var (
 	// Tested on v2.0.x and v2.1.x. Not sure if it also works on others Doris version.
 	stmtMatchFmt = `^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d*) \[[^\]]+?\] \|Client=([^|]+?)\|User=([^|]+?)\|(?:.+?)\|Db=(%s?)\|State=%s\|(?:.+?)\|Time(?:\(ms\))?=(\d*)\|(?:.+?)\|QueryId=([a-z0-9-]+)\|IsQuery=%s\|(?:.+?)\|Stmt=(.+?)\|CpuTimeMS=`
 
-	// The cols read from audit log table
-	captureFieldCols = []string{"time", "client_ip", "user", "db", "query_time", "query_id", "stmt"}
-
 	unescapeReplacer = strings.NewReplacer(
 		"\\n", "\n",
 		"\\t", "\t",
@@ -60,9 +56,8 @@ type AuditLogScanOpts struct {
 	OnlySelect         bool
 	From, To           string
 
-	Unique, UniqueNormalize bool
-	Unescape                bool
-	Strict                  bool
+	Unescape bool
+	Strict   bool
 }
 
 func (opts *AuditLogScanOpts) sqlConditions() string {
@@ -250,13 +245,6 @@ func (s *SimpleAuditLogScanner) ScanOne(oneLog []byte) error {
 }
 
 func (s *SimpleAuditLogScanner) Result() []string {
-	if s.Unique {
-		// append number of occurrences of each sql
-		for _, v := range s.uniqsqls {
-			sqlCount := fmt.Sprintf(" -- count: %d", v.count)
-			s.sqls[v.sqlIdx] += sqlCount
-		}
-	}
 	return s.sqls
 }
 
@@ -281,30 +269,12 @@ func (s *SimpleAuditLogScanner) onMatch(caps []string, skipOptsFilter bool) {
 		stmt = unescapeReplacer.Replace(stmt)
 	}
 	if s.Strict && s.validateSQL(queryId, stmt) != nil {
-		print("asdsadad\n")
 		return
 	}
 
-	// unique sqls
-	if s.Unique {
-		var h [32]byte
-		if s.UniqueNormalize {
-			h = hashstr(s.hash, tidbparser.NormalizeKeepHint(stmt))
-		} else {
-			h = hashstr(s.hash, stmt)
-		}
-		if _, ok := s.uniqsqls[h]; ok {
-			s.uniqsqls[h].count++
-			return
-		} else {
-			s.uniqsqls[h] = &uniqSql{count: 1, sqlIdx: len(s.sqls)}
-		}
-	}
-
 	// add leading meta comment
 	outputStmt := EncodeReplaySql(time, client, user, db, queryId, stmt)
 
-	// not unique sqls
 	s.sqls = append(s.sqls, outputStmt)
 }
 

src/auditlog_test.go

@@ -23,8 +23,6 @@ func TestExtractQueriesFromAuditLogs(t *testing.T) {
 		queryMinCpuTimeMs int
 		queryStates       []string
 		parallel          int
-		unique            bool
-		uniqueNormalize   bool
 		unescape          bool
 		onlySelect        bool
 		strict            bool
@@ -48,19 +46,6 @@ func TestExtractQueriesFromAuditLogs(t *testing.T) {
 			},
 			want: []int{8},
 		},
-		{
-			name: "unique",
-			args: args{
-				auditlogPaths:   []string{"fixture/fe.audit.log"},
-				encoding:        "auto",
-				unique:          true,
-				uniqueNormalize: true,
-				unescape:        true,
-				onlySelect:      true,
-				strict:          true,
-			},
-			want: []int{7},
-		},
 		{
 			name: "not_only_select",
 			args: args{
@@ -95,8 +80,6 @@ func TestExtractQueriesFromAuditLogs(t *testing.T) {
 				OnlySelect:         tt.args.onlySelect,
 				From:               tt.args.from,
 				To:                 tt.args.to,
-				Unique:             tt.args.unique,
-				UniqueNormalize:    tt.args.uniqueNormalize,
 				Unescape:           tt.args.unescape,
 				Strict:             tt.args.strict,
 			}

src/db.go

@@ -445,8 +445,7 @@ func getDBAuditLogs(
 	conditions string,
 	limit, offset int,
 ) (string, string, error) {
-	stmt := fmt.Sprintf("SELECT %s FROM `%s`.`%s` WHERE %s LIMIT %d OFFSET %d ORDER BY time asc, query_id asc",
-		strings.Join(captureFieldCols, ", "),
+	stmt := fmt.Sprintf("SELECT time, client_ip, user, db, query_time, query_id, stmt FROM `%s`.`%s` WHERE %s LIMIT %d OFFSET %d ORDER BY time asc, query_id asc",
 		dbname,
 		table,
 		conditions,