From 3ce273047f7936246de35fd902fc8006a615f2f3 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 19 Nov 2024 17:42:21 +0100 Subject: [PATCH 01/22] feat: read connection id from the slow query log Signed-off-by: Andres Taylor --- go/data/query.go | 1 + go/data/slow_query_log_loader.go | 16 +++++++++----- go/data/slow_query_log_loader_test.go | 30 ++++++++++++++------------- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/go/data/query.go b/go/data/query.go index bd6f722..17d4c4e 100644 --- a/go/data/query.go +++ b/go/data/query.go @@ -42,6 +42,7 @@ type ( Type CmdType // These fields are only set if the log file is a slow query log + ConnectionID int QueryTime, LockTime float64 RowsSent, RowsExamined int Timestamp int64 diff --git a/go/data/slow_query_log_loader.go b/go/data/slow_query_log_loader.go index 4e533c5..5ad3882 100644 --- a/go/data/slow_query_log_loader.go +++ b/go/data/slow_query_log_loader.go @@ -129,7 +129,7 @@ func (s *slowQueryLogReaderState) processLine(line string, state *lineProcessorS } func (s *slowQueryLogReaderState) processCommentLine(line string, state *lineProcessorState) (bool, error) { - if strings.HasPrefix(line, "# Query_time:") { + if strings.HasPrefix(line, "# Query_time:") || strings.HasPrefix(line, "# User@Host:") { if err := parseQueryMetrics(line, &state.currentQuery); err != nil { return false, err } @@ -190,17 +190,16 @@ func parseQueryMetrics(line string, q *Query) error { for i < len(fields) { field := fields[i] if !strings.HasSuffix(field, ":") { - return fmt.Errorf("unexpected field format '%s'", field) + i++ + continue } - // Remove the trailing colon to get the key key := strings.TrimSuffix(field, ":") if i+1 >= len(fields) { return fmt.Errorf("missing value for key '%s'", key) } value := fields[i+1] - // Assign to Query struct based on key switch key { case "Query_time": fval, err := strconv.ParseFloat(value, 64) @@ -214,6 +213,12 @@ func parseQueryMetrics(line string, q *Query) error { return fmt.Errorf("invalid Lock_time value '%s'", value) } q.LockTime = fval + case "Id": + ival, err := strconv.Atoi(value) + if err != nil { + return fmt.Errorf("invalid connection id value '%s'", value) + } + q.ConnectionID = ival case "Rows_sent": ival, err := strconv.Atoi(value) if err != nil { @@ -227,7 +232,7 @@ func parseQueryMetrics(line string, q *Query) error { } q.RowsExamined = ival } - i += 2 // Move to the next key-value pair + i += 2 } return nil @@ -285,6 +290,7 @@ func parseQuery(rs Query) (*Query, error) { RowsSent: rs.RowsSent, RowsExamined: rs.RowsExamined, Timestamp: rs.Timestamp, + ConnectionID: rs.ConnectionID, } if len(s) < 3 { diff --git a/go/data/slow_query_log_loader_test.go b/go/data/slow_query_log_loader_test.go index 32de1b0..ac018a8 100644 --- a/go/data/slow_query_log_loader_test.go +++ b/go/data/slow_query_log_loader_test.go @@ -28,21 +28,23 @@ func TestLoadSlowQueryLogWithMetadata(t *testing.T) { require.NoError(t, err) expected := []Query{ - {FirstWord: "/bin/mysqld,", Query: "/bin/mysqld, Version: 8.0.26 (Source distribution). started with:\nTcp port: 3306 Unix socket: /tmp/mysql.sock\nTime Id Command Argument\nuse testdb;", Line: 1, Type: 0, QueryTime: 0.000153, LockTime: 6.3e-05, RowsSent: 1, RowsExamined: 1, Timestamp: 0}, + {FirstWord: "/bin/mysqld,", Query: "/bin/mysqld, Version: 8.0.26 (Source distribution). started with:\nTcp port: 3306 Unix socket: /tmp/mysql.sock\nTime Id Command Argument\nuse testdb;", Line: 1, Type: 0, QueryTime: 0.000153, LockTime: 6.3e-05, RowsSent: 1, RowsExamined: 1, Timestamp: 0, ConnectionID: 780496}, {FirstWord: "SET", Query: "SET timestamp=1690891201;", Line: 8}, {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2343274;", Line: 9}, - {FirstWord: "FLUSH", Query: "FLUSH SLOW LOGS;", Line: 19, QueryTime: 0.005047, LockTime: 0, RowsSent: 0, RowsExamined: 0, Timestamp: 1690891201}, - {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2343272;", Line: 24, QueryTime: 0.000162, LockTime: 6.7e-05, RowsSent: 1, RowsExamined: 1, Timestamp: 1690891201}, - {FirstWord: "select", Query: "select s1_0.id, s1_0.code, s1_0.token, s1_0.date from stores s1_0 where s1_0.id=11393;", Line: 29, QueryTime: 0.000583, LockTime: 0.000322, RowsSent: 1, RowsExamined: 1, Timestamp: 1690891201}, - {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2343265;", Line: 34, QueryTime: 0.000148, LockTime: 6.2e-05, RowsSent: 1, RowsExamined: 1, Timestamp: 1690891201}, - {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2343188;", Line: 39, QueryTime: 0.000159, LockTime: 6.5e-05, RowsSent: 1, RowsExamined: 1, Timestamp: 1690891201}, - {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2343180;", Line: 44, QueryTime: 0.000152, LockTime: 6.3e-05, RowsSent: 1, RowsExamined: 1, Timestamp: 1690891201}, - {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2343011;", Line: 49, QueryTime: 0.000149, LockTime: 6.1e-05, RowsSent: 666, RowsExamined: 1, Timestamp: 1690891201}, - {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2342469;", Line: 54, QueryTime: 0.000153, LockTime: 6.2e-05, RowsSent: 1, RowsExamined: 1, Timestamp: 1690891201}, - {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2342465;", Line: 59, QueryTime: 0.000151, LockTime: 6.2e-05, RowsSent: 1, RowsExamined: 1, Timestamp: 1690891201}, - {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2342439;", Line: 64, QueryTime: 0.000148, LockTime: 6.1e-05, RowsSent: 1, RowsExamined: 731, Timestamp: 1690891201}, - {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2342389;", Line: 69, QueryTime: 0.000163, LockTime: 6.7e-05, RowsSent: 1, RowsExamined: 1, Timestamp: 1690891201}, + {FirstWord: "FLUSH", Query: "FLUSH SLOW LOGS;", Line: 19, QueryTime: 0.005047, LockTime: 0, RowsSent: 0, RowsExamined: 0, Timestamp: 1690891201, ConnectionID: 341291}, + {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2343272;", Line: 24, QueryTime: 0.000162, LockTime: 6.7e-05, RowsSent: 1, RowsExamined: 1, Timestamp: 1690891201, ConnectionID: 780496}, + {FirstWord: "select", Query: "select s1_0.id, s1_0.code, s1_0.token, s1_0.date from stores s1_0 where s1_0.id=11393;", Line: 29, QueryTime: 0.000583, LockTime: 0.000322, RowsSent: 1, RowsExamined: 1, Timestamp: 1690891201, ConnectionID: 780506}, + {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2343265;", Line: 34, QueryTime: 0.000148, LockTime: 6.2e-05, RowsSent: 1, RowsExamined: 1, Timestamp: 1690891201, ConnectionID: 780496}, + {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2343188;", Line: 39, QueryTime: 0.000159, LockTime: 6.5e-05, RowsSent: 1, RowsExamined: 1, Timestamp: 1690891201, ConnectionID: 780496}, + {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2343180;", Line: 44, QueryTime: 0.000152, LockTime: 6.3e-05, RowsSent: 1, RowsExamined: 1, Timestamp: 1690891201, ConnectionID: 780496}, + {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2343011;", Line: 49, QueryTime: 0.000149, LockTime: 6.1e-05, RowsSent: 666, RowsExamined: 1, Timestamp: 1690891201, ConnectionID: 780496}, + {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2342469;", Line: 54, QueryTime: 0.000153, LockTime: 6.2e-05, RowsSent: 1, RowsExamined: 1, Timestamp: 1690891201, ConnectionID: 780496}, + {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2342465;", Line: 59, QueryTime: 0.000151, LockTime: 6.2e-05, RowsSent: 1, RowsExamined: 1, Timestamp: 1690891201, ConnectionID: 780496}, + {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2342439;", Line: 64, QueryTime: 0.000148, LockTime: 6.1e-05, RowsSent: 1, RowsExamined: 731, Timestamp: 1690891201, ConnectionID: 780496}, + {FirstWord: "select", Query: "select m1_0.id, m1_0.name, m1_0.value, m1_0.date from items m1_0 where m1_0.id=2342389;", Line: 69, QueryTime: 0.000163, LockTime: 6.7e-05, RowsSent: 1, RowsExamined: 1, Timestamp: 1690891201, ConnectionID: 780496}, + } + for i, expectedQuery := range expected { + query := queries[i] + require.Equal(t, expectedQuery, query, "Unexpected query at index %d", i) } - - require.Equal(t, expected, queries) } From ca78c5e33b43bde15df87f9337768d4da8eda82f Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Wed, 20 Nov 2024 10:29:50 +0100 Subject: [PATCH 02/22] feat: add connection id to mysql general log reader Signed-off-by: Andres Taylor --- go/data/query_log_parse.go | 34 +++++++++++++++++++++++++-------- go/data/query_log_parse_test.go | 23 ++++++++++++++-------- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/go/data/query_log_parse.go b/go/data/query_log_parse.go index 6042687..23cec63 100644 --- a/go/data/query_log_parse.go +++ b/go/data/query_log_parse.go @@ -23,6 +23,7 @@ import ( "io" "os" "regexp" + "strconv" "sync" ) @@ -41,8 +42,9 @@ type ( mysqlLogReaderState struct { logReaderState - prevQuery string - queryStart int + prevQuery string + queryStart int + prevConnectionID int } ) @@ -98,6 +100,12 @@ func (s *mysqlLogReaderState) Next() (Query, bool) { if matches[3] == "Query" { s.prevQuery = matches[4] s.queryStart = s.lineNumber + connID, err := strconv.Atoi(matches[2]) + if err != nil { + s.err = fmt.Errorf("invalid connection id at line %d: %w", s.lineNumber, err) + return Query{}, false + } + s.prevConnectionID = connID } } s.closed = true @@ -137,26 +145,36 @@ func (s *logReaderState) readLine() (string, bool, error) { func (s *mysqlLogReaderState) finalizeQuery() Query { query := Query{ - Query: s.prevQuery, - Line: s.queryStart, - Type: QueryT, + Query: s.prevQuery, + Line: s.queryStart, + Type: QueryT, + ConnectionID: s.prevConnectionID, } s.prevQuery = "" + s.prevConnectionID = 0 return query } func (s *mysqlLogReaderState) processQuery(matches []string) Query { query := Query{ - Query: s.prevQuery, - Line: s.queryStart, - Type: QueryT, + Query: s.prevQuery, + Line: s.queryStart, + Type: QueryT, + ConnectionID: s.prevConnectionID, } s.prevQuery = "" + s.prevConnectionID = 0 // If the new line is a query, store it for next iteration if matches[3] == "Query" { s.prevQuery = matches[4] s.queryStart = s.lineNumber + connID, err := strconv.Atoi(matches[2]) + if err != nil { + s.err = fmt.Errorf("invalid connection id at line %d: %w", s.lineNumber, err) + return Query{} + } + s.prevConnectionID = connID } return query } diff --git a/go/data/query_log_parse_test.go b/go/data/query_log_parse_test.go index d0eaf5d..b7497d6 100644 --- a/go/data/query_log_parse_test.go +++ b/go/data/query_log_parse_test.go @@ -19,6 +19,7 @@ package data import ( "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -26,6 +27,9 @@ func TestParseMySQLQueryLog(t *testing.T) { loader := MySQLLogLoader{}.Load("../testdata/mysql.query.log") gotQueries, err := makeSlice(loader) require.NoError(t, err) + for _, query := range gotQueries { + assert.NotZero(t, query.ConnectionID, query.Query) + } require.Equal(t, 1517, len(gotQueries), "expected 1517 queries") //nolint:testifylint // too many elements for the output to be readable } @@ -35,13 +39,15 @@ func TestSmallSnippet(t *testing.T) { require.NoError(t, err) expected := []Query{ { - Query: "SET GLOBAL log_output = 'FILE'", - Line: 4, - Type: QueryT, + Query: "SET GLOBAL log_output = 'FILE'", + Line: 4, + Type: QueryT, + ConnectionID: 32, }, { - Query: "show databases", - Line: 5, - Type: QueryT, + Query: "show databases", + Line: 5, + Type: QueryT, + ConnectionID: 32, }, { Query: `UPDATE _vt.schema_migrations SET @@ -67,8 +73,9 @@ WHERE AND retries=0 ) LIMIT 1`, - Line: 6, - Type: QueryT, + Line: 6, + Type: QueryT, + ConnectionID: 24, }, } From 96c7fab5cb8afac6df62e1cad3a83d42cea13f64 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Wed, 20 Nov 2024 12:17:21 +0100 Subject: [PATCH 03/22] feat: read sessionUUID from vtgate query log Signed-off-by: Andres Taylor --- go/cmd/keys.go | 2 +- go/data/vtgate_log_parse.go | 67 +++++++++++++++++----- go/data/vtgate_log_parse_test.go | 9 ++- go/testdata/vtgate.query.log | 14 ++--- go/testdata/vtgate.query.log.parsed.bv.txt | 50 ++++++++-------- go/testdata/vtgate.query.log.parsed.txt | 50 ++++++++-------- 6 files changed, 118 insertions(+), 74 deletions(-) diff --git a/go/cmd/keys.go b/go/cmd/keys.go index 8569db8..00e39eb 100644 --- a/go/cmd/keys.go +++ b/go/cmd/keys.go @@ -55,7 +55,7 @@ func keysCmd() *cobra.Command { func addInputTypeFlag(cmd *cobra.Command, s *string) { *s = "sql" - cmd.Flags().StringVar(s, "input-type", "sql", "Specifies the type of input file: 'sql' or 'mysql-log'") + cmd.Flags().StringVar(s, "input-type", "sql", "Specifies the type of input file: 'sql', 'mysql-log' or 'vtgate-log'") } func configureLoader(inputType string, needsBindVars bool) (data.Loader, error) { diff --git a/go/data/vtgate_log_parse.go b/go/data/vtgate_log_parse.go index efd354d..e0ef595 100644 --- a/go/data/vtgate_log_parse.go +++ b/go/data/vtgate_log_parse.go @@ -20,6 +20,7 @@ import ( "bufio" "encoding/json" "fmt" + "hash/fnv" "os" "regexp" "strconv" @@ -42,11 +43,14 @@ type ( vtgateLogReaderState struct { logReaderState NeedsBindVars bool + uuidReg *regexp.Regexp } ) func (vll VtGateLogLoader) Load(fileName string) IteratorLoader { - reg := regexp.MustCompile(`\t"([^"]+)"\t(\{(?:[^{}]|(?:\{[^{}]*\}))*\}|"[^"]+")`) + reg := regexp.MustCompile(`\t"([^"]+)"\t(\{(?:[^{}]|\{[^{}]*})*}|"[^"]+")`) + uuidReg := regexp.MustCompile(`\t"([0-9a-fA-F\-]{36})"\t`) + fd, err := os.OpenFile(fileName, os.O_RDONLY, 0) if err != nil { return &errLoader{err: err} @@ -59,6 +63,7 @@ func (vll VtGateLogLoader) Load(fileName string) IteratorLoader { fd: fd, }, NeedsBindVars: vll.NeedsBindVars, + uuidReg: uuidReg, } } @@ -88,7 +93,7 @@ func (s *vtgateLogReaderState) Next() (Query, bool) { continue } line = strings.ReplaceAll(line, "\\n", "") - // Find the match + match := s.reg.FindStringSubmatch(line) if len(match) <= 2 { s.fail(fmt.Errorf("line %d: cannot parse log: %s", s.lineNumber, line)) @@ -96,17 +101,25 @@ func (s *vtgateLogReaderState) Next() (Query, bool) { } query := match[1] + + connectionID := s.extractSessionUUIDAsConnectionID(line) + if connectionID == -1 { + // something went wrong while extracting the connection ID + return Query{}, false + } + if !s.NeedsBindVars { return Query{ - Query: query, - Line: s.lineNumber, - Type: QueryT, + Query: query, + Line: s.lineNumber, + Type: QueryT, + ConnectionID: connectionID, }, true } - // If we care about bind variables (e.g. running 'trace') then we parse the query log - // output into bindVarsVtGate, we then transform it into something the Vitess library - // can understand (aka: map[string]*querypb.BindVariable), we then parse the query string + // If we care about bind variables (e.g., running 'trace'), then we parse the query log + // output into bindVarsVtGate, transform it into something the Vitess library + // can understand (map[string]*querypb.BindVariable), parse the query string, // and add the bind variables to it. bindVarsRaw := match[2] bvs, err := getBindVariables(bindVarsRaw, s.lineNumber) @@ -122,9 +135,10 @@ func (s *vtgateLogReaderState) Next() (Query, bool) { } return Query{ - Query: parsedQuery, - Line: s.lineNumber, - Type: QueryT, + Query: parsedQuery, + Line: s.lineNumber, + Type: QueryT, + ConnectionID: connectionID, }, true } @@ -133,6 +147,21 @@ func (s *vtgateLogReaderState) Next() (Query, bool) { return Query{}, false } +func (s *vtgateLogReaderState) extractSessionUUIDAsConnectionID(line string) int { + uuidMatch := s.uuidReg.FindStringSubmatch(line) + if len(uuidMatch) < 2 { + s.fail(fmt.Errorf("line %d: cannot extract session UUID: %s", s.lineNumber, line)) + return -1 + } + sessionUUID := uuidMatch[1] + + // Hash the session UUID using FNV-1a + h := fnv.New64a() + _, _ = h.Write([]byte(sessionUUID)) + connectionID := int(h.Sum64()) + return connectionID +} + func addBindVarsToQuery(query string, bvs map[string]*querypb.BindVariable) (string, error) { parser, err := sqlparser.New(sqlparser.Options{}) if err != nil { @@ -166,8 +195,18 @@ func getBindVariables(bindVarsRaw string, lineNumber int) (map[string]*querypb.B var val []byte switch { - case sqltypes.IsIntegral(bvType) || sqltypes.IsFloat(bvType): - val = []byte(strconv.FormatFloat(value.Value.(float64), 'f', -1, 64)) + case sqltypes.IsIntegral(bvType): + intVal, ok := value.Value.(float64) + if !ok { + return nil, fmt.Errorf("line %d: cannot parse integral bind variable", lineNumber) + } + val = strconv.AppendInt(nil, int64(intVal), 10) + case sqltypes.IsFloat(bvType): + floatVal, ok := value.Value.(float64) + if !ok { + return nil, fmt.Errorf("line %d: cannot parse float bind variable", lineNumber) + } + val = strconv.AppendFloat(nil, floatVal, 'f', -1, 64) case bvType == sqltypes.Tuple: // the query log of vtgate does not list all the values for a tuple // instead it lists the following: "v2": {"type": "TUPLE", "value": "2 items"} @@ -180,5 +219,5 @@ func getBindVariables(bindVarsRaw string, lineNumber int) (map[string]*querypb.B Value: val, } } - return bvProcessed, err + return bvProcessed, nil } diff --git a/go/data/vtgate_log_parse_test.go b/go/data/vtgate_log_parse_test.go index 6d4e2a6..1973819 100644 --- a/go/data/vtgate_log_parse_test.go +++ b/go/data/vtgate_log_parse_test.go @@ -17,6 +17,7 @@ limitations under the License. package data import ( + "fmt" "os" "strings" "testing" @@ -42,7 +43,7 @@ func TestParseVtGateQueryLog(t *testing.T) { var got []string for _, query := range gotQueries { - got = append(got, query.Query) + got = append(got, format(query)) } require.Equal(t, string(expect), strings.Join(got, "\n")) @@ -59,8 +60,12 @@ func TestParseVtGateQueryLogNoBindVars(t *testing.T) { var got []string for _, query := range gotQueries { - got = append(got, query.Query) + got = append(got, format(query)) } require.Equal(t, string(expect), strings.Join(got, "\n")) } + +func format(query Query) string { + return fmt.Sprintf("%d:%s", query.ConnectionID, query.Query) +} diff --git a/go/testdata/vtgate.query.log b/go/testdata/vtgate.query.log index 7c0d53c..168e336 100644 --- a/go/testdata/vtgate.query.log +++ b/go/testdata/vtgate.query.log @@ -6,13 +6,13 @@ Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.371324 2024-10-31 13 Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.384686 2024-10-31 13:55:59.387103 0.002417 0.000288 0.001545 0.000582 EXPLAIN "vexplain trace insert into customers(customer_id, customer_name, customer_pincode) values (:vtg1 /* INT64 */, :vtg2 /* VARCHAR */, :vtg3 /* INT64 */), (:vtg4 /* INT64 */, :vtg5 /* VARCHAR */, :vtg6 /* INT64 */), (:vtg7 /* INT64 */, :vtg8 /* VARCHAR */, :vtg9 /* INT64 */), (:vtg10 /* INT64 */, :vtg11 /* VARCHAR */, :vtg12 /* INT64 */), (:vtg13 /* INT64 */, :vtg14 /* VARCHAR */, :vtg15 /* INT64 */), (:vtg16 /* INT64 */, :vtg17 /* VARCHAR */, :vtg18 /* INT64 */), (:vtg19 /* INT64 */, :vtg20 /* VARCHAR */, :vtg21 /* INT64 */), (:vtg22 /* INT64 */, :vtg23 /* VARCHAR */, :vtg24 /* INT64 */), (:vtg25 /* INT64 */, :vtg26 /* VARCHAR */, :vtg27 /* INT64 */), (:vtg28 /* INT64 */, :vtg29 /* VARCHAR */, :vtg30 /* INT64 */), (:vtg31 /* INT64 */, :vtg32 /* VARCHAR */, :vtg33 /* INT64 */), (:vtg34 /* INT64 */, :vtg35 /* VARCHAR */, :vtg36 /* INT64 */), (:vtg37 /* INT64 */, :vtg38 /* VARCHAR */, :vtg39 /* INT64 */), (:vtg40 /* INT64 */, :vtg41 /* VARCHAR */, :vtg42 /* INT64 */), (:vtg43 /* INT64 */, :vtg44 /* VARCHAR */, :vtg45 /* INT64 */), (:vtg46 /* INT64 */, :vtg47 /* VARCHAR */, :vtg48 /* INT64 */), (:vtg49 /* INT64 */, :vtg50 /* VARCHAR */, :vtg51 /* INT64 */), (:vtg52 /* INT64 */, :vtg53 /* VARCHAR */, :vtg54 /* INT64 */), (:vtg55 /* INT64 */, :vtg56 /* VARCHAR */, :vtg57 /* INT64 */), (:vtg58 /* INT64 */, :vtg59 /* VARCHAR */, :vtg60 /* INT64 */), (:vtg61 /* INT64 */, :vtg62 /* VARCHAR */, :vtg63 /* INT64 */), (:vtg64 /* INT64 */, :vtg65 /* VARCHAR */, :vtg66 /* INT64 */), (:vtg67 /* INT64 */, :vtg68 /* VARCHAR */, :vtg69 /* INT64 */), (:vtg70 /* INT64 */, :vtg71 /* VARCHAR */, :vtg72 /* INT64 */), (:vtg73 /* INT64 */, :vtg74 /* VARCHAR */, :vtg75 /* INT64 */), (:vtg76 /* INT64 */, :vtg77 /* VARCHAR */, :vtg78 /* INT64 */), (:vtg79 /* INT64 */, :vtg80 /* VARCHAR */, :vtg81 /* INT64 */), (:vtg82 /* INT64 */, :vtg83 /* VARCHAR */, :vtg84 /* INT64 */), (:vtg85 /* INT64 */, :vtg86 /* VARCHAR */, :vtg87 /* INT64 */), (:vtg88 /* INT64 */, :vtg89 /* VARCHAR */, :vtg90 /* INT64 */)" {"vtg1": {"type": "INT64", "value": 1}, "vtg10": {"type": "INT64", "value": 4}, "vtg11": {"type": "VARCHAR", "value": "Bob"}, "vtg12": {"type": "INT64", "value": 110004}, "vtg13": {"type": "INT64", "value": 5}, "vtg14": {"type": "VARCHAR", "value": "Charlie"}, "vtg15": {"type": "INT64", "value": 110004}, "vtg16": {"type": "INT64", "value": 6}, "vtg17": {"type": "VARCHAR", "value": "David"}, "vtg18": {"type": "INT64", "value": 110006}, "vtg19": {"type": "INT64", "value": 7}, "vtg2": {"type": "VARCHAR", "value": "John Doe"}, "vtg20": {"type": "VARCHAR", "value": "Eve"}, "vtg21": {"type": "INT64", "value": 110007}, "vtg22": {"type": "INT64", "value": 8}, "vtg23": {"type": "VARCHAR", "value": "Frank"}, "vtg24": {"type": "INT64", "value": 110008}, "vtg25": {"type": "INT64", "value": 9}, "vtg26": {"type": "VARCHAR", "value": "Grace"}, "vtg27": {"type": "INT64", "value": 110009}, "vtg28": {"type": "INT64", "value": 10}, "vtg29": {"type": "VARCHAR", "value": "Heidi"}, "vtg3": {"type": "INT64", "value": 110001}, "vtg30": {"type": "INT64", "value": 110004}, "vtg31": {"type": "INT64", "value": 11}, "vtg32": {"type": "VARCHAR", "value": "Ivy"}, "vtg33": {"type": "INT64", "value": 110011}, "vtg34": {"type": "INT64", "value": 12}, "vtg35": {"type": "VARCHAR", "value": "Alice"}, "vtg36": {"type": "INT64", "value": 110005}, "vtg37": {"type": "INT64", "value": 13}, "vtg38": {"type": "VARCHAR", "value": "Bob"}, "vtg39": {"type": "INT64", "value": 110003}, "vtg4": {"type": "INT64", "value": 2}, "vtg40": {"type": "INT64", "value": 14}, "vtg41": {"type": "VARCHAR", "value": "Charlie"}, "vtg42": {"type": "INT64", "value": 110014}, "vtg43": {"type": "INT64", "value": 15}, "vtg44": {"type": "VARCHAR", "value": "David"}, "vtg45": {"type": "INT64", "value": 110015}, "vtg46": {"type": "INT64", "value": 16}, "vtg47": {"type": "VARCHAR", "value": "Frank"}, "vtg48": {"type": "INT64", "value": 110008}, "vtg49": {"type": "INT64", "value": 17}, "vtg5": {"type": "VARCHAR", "value": "Jane Doe"}, "vtg50": {"type": "VARCHAR", "value": "Grace"}, "vtg51": {"type": "INT64", "value": 110009}, "vtg52": {"type": "INT64", "value": 18}, "vtg53": {"type": "VARCHAR", "value": "Isaac"}, "vtg54": {"type": "INT64", "value": 110010}, "vtg55": {"type": "INT64", "value": 19}, "vtg56": {"type": "VARCHAR", "value": "Julia"}, "vtg57": {"type": "INT64", "value": 110011}, "vtg58": {"type": "INT64", "value": 20}, "vtg59": {"type": "VARCHAR", "value": "Kevin"}, "vtg6": {"type": "INT64", "value": 110002}, "vtg60": {"type": "INT64", "value": 110012}, "vtg61": {"type": "INT64", "value": 21}, "vtg62": {"type": "VARCHAR", "value": "Laura"}, "vtg63": {"type": "INT64", "value": 110013}, "vtg64": {"type": "INT64", "value": 22}, "vtg65": {"type": "VARCHAR", "value": "Michael"}, "vtg66": {"type": "INT64", "value": 110014}, "vtg67": {"type": "INT64", "value": 23}, "vtg68": {"type": "VARCHAR", "value": "Nina"}, "vtg69": {"type": "INT64", "value": 110015}, "vtg7": {"type": "INT64", "value": 3}, "vtg70": {"type": "INT64", "value": 24}, "vtg71": {"type": "VARCHAR", "value": "Oscar"}, "vtg72": {"type": "INT64", "value": 110001}, "vtg73": {"type": "INT64", "value": 25}, "vtg74": {"type": "VARCHAR", "value": "Patricia"}, "vtg75": {"type": "INT64", "value": 110002}, "vtg76": {"type": "INT64", "value": 26}, "vtg77": {"type": "VARCHAR", "value": "Quincy"}, "vtg78": {"type": "INT64", "value": 110003}, "vtg79": {"type": "INT64", "value": 27}, "vtg8": {"type": "VARCHAR", "value": "Alice"}, "vtg80": {"type": "VARCHAR", "value": "Rachel"}, "vtg81": {"type": "INT64", "value": 110004}, "vtg82": {"type": "INT64", "value": 28}, "vtg83": {"type": "VARCHAR", "value": "Samuel"}, "vtg84": {"type": "INT64", "value": 110005}, "vtg85": {"type": "INT64", "value": 29}, "vtg86": {"type": "VARCHAR", "value": "Tina"}, "vtg87": {"type": "INT64", "value": 110006}, "vtg88": {"type": "INT64", "value": 30}, "vtg89": {"type": "VARCHAR", "value": "Ulysses"}, "vtg9": {"type": "INT64", "value": 110003}, "vtg90": {"type": "INT64", "value": 110007}} 2 0 "" "PRIMARY" "2642245a-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers"] "mysqltest" 0.000000 0.000000 "" Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.387880 2024-10-31 13:55:59.399021 0.011141 0.002436 0.007479 0.001223 EXPLAIN "vexplain trace insert into orders(order_id, customer_id, order_date, order_amount) values (:vtg1 /* INT64 */, :vtg2 /* INT64 */, :vtg3 /* VARCHAR */, :vtg4 /* INT64 */), (:vtg5 /* INT64 */, :vtg6 /* INT64 */, :vtg7 /* VARCHAR */, :vtg8 /* INT64 */), (:vtg9 /* INT64 */, :vtg10 /* INT64 */, :vtg11 /* VARCHAR */, :vtg12 /* INT64 */), (:vtg13 /* INT64 */, :vtg14 /* INT64 */, :vtg15 /* VARCHAR */, :vtg16 /* INT64 */), (:vtg17 /* INT64 */, :vtg18 /* INT64 */, :vtg19 /* VARCHAR */, :vtg20 /* INT64 */), (:vtg21 /* INT64 */, :vtg22 /* INT64 */, :vtg23 /* VARCHAR */, :vtg24 /* INT64 */), (:vtg25 /* INT64 */, :vtg26 /* INT64 */, :vtg27 /* VARCHAR */, :vtg28 /* INT64 */), (:vtg29 /* INT64 */, :vtg30 /* INT64 */, :vtg31 /* VARCHAR */, :vtg32 /* INT64 */), (:vtg33 /* INT64 */, :vtg34 /* INT64 */, :vtg35 /* VARCHAR */, :vtg36 /* INT64 */), (:vtg37 /* INT64 */, :vtg38 /* INT64 */, :vtg39 /* VARCHAR */, :vtg40 /* INT64 */), (:vtg41 /* INT64 */, :vtg42 /* INT64 */, :vtg43 /* VARCHAR */, :vtg44 /* INT64 */), (:vtg45 /* INT64 */, :vtg46 /* INT64 */, :vtg47 /* VARCHAR */, :vtg48 /* INT64 */), (:vtg49 /* INT64 */, :vtg50 /* INT64 */, :vtg51 /* VARCHAR */, :vtg52 /* INT64 */), (:vtg53 /* INT64 */, :vtg54 /* INT64 */, :vtg55 /* VARCHAR */, :vtg56 /* INT64 */), (:vtg57 /* INT64 */, :vtg58 /* INT64 */, :vtg59 /* VARCHAR */, :vtg60 /* INT64 */), (:vtg61 /* INT64 */, :vtg62 /* INT64 */, :vtg63 /* VARCHAR */, :vtg64 /* INT64 */), (:vtg65 /* INT64 */, :vtg66 /* INT64 */, :vtg67 /* VARCHAR */, :vtg68 /* INT64 */), (:vtg69 /* INT64 */, :vtg70 /* INT64 */, :vtg71 /* VARCHAR */, :vtg72 /* INT64 */), (:vtg73 /* INT64 */, :vtg74 /* INT64 */, :vtg75 /* VARCHAR */, :vtg76 /* INT64 */), (:vtg77 /* INT64 */, :vtg78 /* INT64 */, :vtg79 /* VARCHAR */, :vtg80 /* INT64 */), (:vtg81 /* INT64 */, :vtg82 /* INT64 */, :vtg83 /* VARCHAR */, :vtg84 /* INT64 */), (:vtg85 /* INT64 */, :vtg86 /* INT64 */, :vtg87 /* VARCHAR */, :vtg88 /* INT64 */), (:vtg89 /* INT64 */, :vtg90 /* INT64 */, :vtg91 /* VARCHAR */, :vtg92 /* INT64 */), (:vtg93 /* INT64 */, :vtg94 /* INT64 */, :vtg95 /* VARCHAR */, :vtg96 /* INT64 */), (:vtg97 /* INT64 */, :vtg98 /* INT64 */, :vtg99 /* VARCHAR */, :vtg100 /* INT64 */), (:vtg101 /* INT64 */, :vtg102 /* INT64 */, :vtg103 /* VARCHAR */, :vtg104 /* INT64 */), (:vtg105 /* INT64 */, :vtg106 /* INT64 */, :vtg107 /* VARCHAR */, :vtg108 /* INT64 */), (:vtg109 /* INT64 */, :vtg110 /* INT64 */, :vtg111 /* VARCHAR */, :vtg112 /* INT64 */), (:vtg113 /* INT64 */, :vtg114 /* INT64 */, :vtg115 /* VARCHAR */, :vtg116 /* INT64 */), (:vtg117 /* INT64 */, :vtg118 /* INT64 */, :vtg119 /* VARCHAR */, :vtg120 /* INT64 */), (:vtg121 /* INT64 */, :vtg122 /* INT64 */, :vtg123 /* VARCHAR */, :vtg124 /* INT64 */), (:vtg125 /* INT64 */, :vtg126 /* INT64 */, :vtg127 /* VARCHAR */, :vtg128 /* INT64 */), (:vtg129 /* INT64 */, :vtg130 /* INT64 */, :vtg131 /* VARCHAR */, :vtg132 /* INT64 */), (:vtg133 /* INT64 */, :vtg134 /* INT64 */, :vtg135 /* VARCHAR */, :vtg136 /* INT64 */), (:vtg137 /* INT64 */, :vtg138 /* INT64 */, :vtg139 /* VARCHAR */, :vtg140 /* INT64 */), (:vtg141 /* INT64 */, :vtg142 /* INT64 */, :vtg143 /* VARCHAR */, :vtg144 /* INT64 */), (:vtg145 /* INT64 */, :vtg146 /* INT64 */, :vtg147 /* VARCHAR */, :vtg148 /* INT64 */), (:vtg149 /* INT64 */, :vtg150 /* INT64 */, :vtg151 /* VARCHAR */, :vtg152 /* INT64 */), (:vtg153 /* INT64 */, :vtg154 /* INT64 */, :vtg155 /* VARCHAR */, :vtg156 /* INT64 */), (:vtg157 /* INT64 */, :vtg158 /* INT64 */, :vtg159 /* VARCHAR */, :vtg160 /* INT64 */), (:vtg161 /* INT64 */, :vtg162 /* INT64 */, :vtg163 /* VARCHAR */, :vtg164 /* INT64 */), (:vtg165 /* INT64 */, :vtg166 /* INT64 */, :vtg167 /* VARCHAR */, :vtg168 /* INT64 */), (:vtg169 /* INT64 */, :vtg170 /* INT64 */, :vtg171 /* VARCHAR */, :vtg172 /* INT64 */), (:vtg173 /* INT64 */, :vtg174 /* INT64 */, :vtg175 /* VARCHAR */, :vtg176 /* INT64 */), (:vtg177 /* INT64 */, :vtg178 /* INT64 */, :vtg179 /* VARCHAR */, :vtg180 /* INT64 */), (:vtg181 /* INT64 */, :vtg182 /* INT64 */, :vtg183 /* VARCHAR */, :vtg184 /* INT64 */), (:vtg185 /* INT64 */, :vtg186 /* INT64 */, :vtg187 /* VARCHAR */, :vtg188 /* INT64 */), (:vtg189 /* INT64 */, :vtg190 /* INT64 */, :vtg191 /* VARCHAR */, :vtg192 /* INT64 */), (:vtg193 /* INT64 */, :vtg194 /* INT64 */, :vtg195 /* VARCHAR */, :vtg196 /* INT64 */), (:vtg197 /* INT64 */, :vtg198 /* INT64 */, :vtg199 /* VARCHAR */, :vtg200 /* INT64 */)" {"vtg1": {"type": "INT64", "value": 1}, "vtg10": {"type": "INT64", "value": 3}, "vtg100": {"type": "INT64", "value": 25000}, "vtg101": {"type": "INT64", "value": 26}, "vtg102": {"type": "INT64", "value": 9}, "vtg103": {"type": "VARCHAR", "value": "2020-01-26"}, "vtg104": {"type": "INT64", "value": 26000}, "vtg105": {"type": "INT64", "value": 27}, "vtg106": {"type": "INT64", "value": 10}, "vtg107": {"type": "VARCHAR", "value": "2020-01-27"}, "vtg108": {"type": "INT64", "value": 27000}, "vtg109": {"type": "INT64", "value": 28}, "vtg11": {"type": "VARCHAR", "value": "2020-01-03"}, "vtg110": {"type": "INT64", "value": 11}, "vtg111": {"type": "VARCHAR", "value": "2020-01-28"}, "vtg112": {"type": "INT64", "value": 28000}, "vtg113": {"type": "INT64", "value": 29}, "vtg114": {"type": "INT64", "value": 12}, "vtg115": {"type": "VARCHAR", "value": "2020-01-29"}, "vtg116": {"type": "INT64", "value": 29000}, "vtg117": {"type": "INT64", "value": 30}, "vtg118": {"type": "INT64", "value": 13}, "vtg119": {"type": "VARCHAR", "value": "2020-01-30"}, "vtg12": {"type": "INT64", "value": 3000}, "vtg120": {"type": "INT64", "value": 30000}, "vtg121": {"type": "INT64", "value": 31}, "vtg122": {"type": "INT64", "value": 14}, "vtg123": {"type": "VARCHAR", "value": "2020-01-31"}, "vtg124": {"type": "INT64", "value": 31000}, "vtg125": {"type": "INT64", "value": 32}, "vtg126": {"type": "INT64", "value": 15}, "vtg127": {"type": "VARCHAR", "value": "2020-02-01"}, "vtg128": {"type": "INT64", "value": 32000}, "vtg129": {"type": "INT64", "value": 33}, "vtg13": {"type": "INT64", "value": 4}, "vtg130": {"type": "INT64", "value": 16}, "vtg131": {"type": "VARCHAR", "value": "2020-02-02"}, "vtg132": {"type": "INT64", "value": 33000}, "vtg133": {"type": "INT64", "value": 34}, "vtg134": {"type": "INT64", "value": 17}, "vtg135": {"type": "VARCHAR", "value": "2020-02-03"}, "vtg136": {"type": "INT64", "value": 34000}, "vtg137": {"type": "INT64", "value": 35}, "vtg138": {"type": "INT64", "value": 18}, "vtg139": {"type": "VARCHAR", "value": "2020-02-04"}, "vtg14": {"type": "INT64", "value": 4}, "vtg140": {"type": "INT64", "value": 35000}, "vtg141": {"type": "INT64", "value": 36}, "vtg142": {"type": "INT64", "value": 19}, "vtg143": {"type": "VARCHAR", "value": "2020-02-05"}, "vtg144": {"type": "INT64", "value": 36000}, "vtg145": {"type": "INT64", "value": 37}, "vtg146": {"type": "INT64", "value": 20}, "vtg147": {"type": "VARCHAR", "value": "2020-02-06"}, "vtg148": {"type": "INT64", "value": 37000}, "vtg149": {"type": "INT64", "value": 38}, "vtg15": {"type": "VARCHAR", "value": "2020-01-04"}, "vtg150": {"type": "INT64", "value": 21}, "vtg151": {"type": "VARCHAR", "value": "2020-02-07"}, "vtg152": {"type": "INT64", "value": 38000}, "vtg153": {"type": "INT64", "value": 39}, "vtg154": {"type": "INT64", "value": 22}, "vtg155": {"type": "VARCHAR", "value": "2020-02-08"}, "vtg156": {"type": "INT64", "value": 39000}, "vtg157": {"type": "INT64", "value": 40}, "vtg158": {"type": "INT64", "value": 23}, "vtg159": {"type": "VARCHAR", "value": "2020-02-09"}, "vtg16": {"type": "INT64", "value": 4000}, "vtg160": {"type": "INT64", "value": 40000}, "vtg161": {"type": "INT64", "value": 41}, "vtg162": {"type": "INT64", "value": 24}, "vtg163": {"type": "VARCHAR", "value": "2020-02-10"}, "vtg164": {"type": "INT64", "value": 41000}, "vtg165": {"type": "INT64", "value": 42}, "vtg166": {"type": "INT64", "value": 25}, "vtg167": {"type": "VARCHAR", "value": "2020-02-11"}, "vtg168": {"type": "INT64", "value": 42000}, "vtg169": {"type": "INT64", "value": 43}, "vtg17": {"type": "INT64", "value": 5}, "vtg170": {"type": "INT64", "value": 26}, "vtg171": {"type": "VARCHAR", "value": "2020-02-12"}, "vtg172": {"type": "INT64", "value": 43000}, "vtg173": {"type": "INT64", "value": 44}, "vtg174": {"type": "INT64", "value": 27}, "vtg175": {"type": "VARCHAR", "value": "2020-02-13"}, "vtg176": {"type": "INT64", "value": 44000}, "vtg177": {"type": "INT64", "value": 45}, "vtg178": {"type": "INT64", "value": 28}, "vtg179": {"type": "VARCHAR", "value": "2020-02-14"}, "vtg18": {"type": "INT64", "value": 5}, "vtg180": {"type": "INT64", "value": 45000}, "vtg181": {"type": "INT64", "value": 46}, "vtg182": {"type": "INT64", "value": 29}, "vtg183": {"type": "VARCHAR", "value": "2020-02-15"}, "vtg184": {"type": "INT64", "value": 46000}, "vtg185": {"type": "INT64", "value": 47}, "vtg186": {"type": "INT64", "value": 30}, "vtg187": {"type": "VARCHAR", "value": "2020-02-16"}, "vtg188": {"type": "INT64", "value": 47000}, "vtg189": {"type": "INT64", "value": 48}, "vtg19": {"type": "VARCHAR", "value": "2020-01-05"}, "vtg190": {"type": "INT64", "value": 1}, "vtg191": {"type": "VARCHAR", "value": "2020-02-17"}, "vtg192": {"type": "INT64", "value": 48000}, "vtg193": {"type": "INT64", "value": 49}, "vtg194": {"type": "INT64", "value": 2}, "vtg195": {"type": "VARCHAR", "value": "2020-02-18"}, "vtg196": {"type": "INT64", "value": 49000}, "vtg197": {"type": "INT64", "value": 50}, "vtg198": {"type": "INT64", "value": 3}, "vtg199": {"type": "VARCHAR", "value": "2020-02-19"}, "vtg2": {"type": "INT64", "value": 1}, "vtg20": {"type": "INT64", "value": 5000}, "vtg200": {"type": "INT64", "value": 50000}, "vtg21": {"type": "INT64", "value": 6}, "vtg22": {"type": "INT64", "value": 6}, "vtg23": {"type": "VARCHAR", "value": "2020-01-06"}, "vtg24": {"type": "INT64", "value": 6000}, "vtg25": {"type": "INT64", "value": 7}, "vtg26": {"type": "INT64", "value": 7}, "vtg27": {"type": "VARCHAR", "value": "2020-01-07"}, "vtg28": {"type": "INT64", "value": 7000}, "vtg29": {"type": "INT64", "value": 8}, "vtg3": {"type": "VARCHAR", "value": "2020-01-01"}, "vtg30": {"type": "INT64", "value": 8}, "vtg31": {"type": "VARCHAR", "value": "2020-01-08"}, "vtg32": {"type": "INT64", "value": 8000}, "vtg33": {"type": "INT64", "value": 9}, "vtg34": {"type": "INT64", "value": 9}, "vtg35": {"type": "VARCHAR", "value": "2020-01-09"}, "vtg36": {"type": "INT64", "value": 9000}, "vtg37": {"type": "INT64", "value": 10}, "vtg38": {"type": "INT64", "value": 10}, "vtg39": {"type": "VARCHAR", "value": "2020-01-10"}, "vtg4": {"type": "INT64", "value": 1000}, "vtg40": {"type": "INT64", "value": 10000}, "vtg41": {"type": "INT64", "value": 11}, "vtg42": {"type": "INT64", "value": 11}, "vtg43": {"type": "VARCHAR", "value": "2020-01-11"}, "vtg44": {"type": "INT64", "value": 11000}, "vtg45": {"type": "INT64", "value": 12}, "vtg46": {"type": "INT64", "value": 12}, "vtg47": {"type": "VARCHAR", "value": "2020-01-12"}, "vtg48": {"type": "INT64", "value": 12000}, "vtg49": {"type": "INT64", "value": 13}, "vtg5": {"type": "INT64", "value": 2}, "vtg50": {"type": "INT64", "value": 13}, "vtg51": {"type": "VARCHAR", "value": "2020-01-13"}, "vtg52": {"type": "INT64", "value": 13000}, "vtg53": {"type": "INT64", "value": 14}, "vtg54": {"type": "INT64", "value": 14}, "vtg55": {"type": "VARCHAR", "value": "2020-01-14"}, "vtg56": {"type": "INT64", "value": 14000}, "vtg57": {"type": "INT64", "value": 15}, "vtg58": {"type": "INT64", "value": 15}, "vtg59": {"type": "VARCHAR", "value": "2020-01-15"}, "vtg6": {"type": "INT64", "value": 2}, "vtg60": {"type": "INT64", "value": 15000}, "vtg61": {"type": "INT64", "value": 16}, "vtg62": {"type": "INT64", "value": 16}, "vtg63": {"type": "VARCHAR", "value": "2020-01-16"}, "vtg64": {"type": "INT64", "value": 16000}, "vtg65": {"type": "INT64", "value": 17}, "vtg66": {"type": "INT64", "value": 17}, "vtg67": {"type": "VARCHAR", "value": "2020-01-17"}, "vtg68": {"type": "INT64", "value": 17000}, "vtg69": {"type": "INT64", "value": 18}, "vtg7": {"type": "VARCHAR", "value": "2020-01-02"}, "vtg70": {"type": "INT64", "value": 1}, "vtg71": {"type": "VARCHAR", "value": "2020-01-18"}, "vtg72": {"type": "INT64", "value": 18000}, "vtg73": {"type": "INT64", "value": 19}, "vtg74": {"type": "INT64", "value": 2}, "vtg75": {"type": "VARCHAR", "value": "2020-01-19"}, "vtg76": {"type": "INT64", "value": 19000}, "vtg77": {"type": "INT64", "value": 20}, "vtg78": {"type": "INT64", "value": 3}, "vtg79": {"type": "VARCHAR", "value": "2020-01-20"}, "vtg8": {"type": "INT64", "value": 2000}, "vtg80": {"type": "INT64", "value": 20000}, "vtg81": {"type": "INT64", "value": 21}, "vtg82": {"type": "INT64", "value": 4}, "vtg83": {"type": "VARCHAR", "value": "2020-01-21"}, "vtg84": {"type": "INT64", "value": 21000}, "vtg85": {"type": "INT64", "value": 22}, "vtg86": {"type": "INT64", "value": 5}, "vtg87": {"type": "VARCHAR", "value": "2020-01-22"}, "vtg88": {"type": "INT64", "value": 22000}, "vtg89": {"type": "INT64", "value": 23}, "vtg9": {"type": "INT64", "value": 3}, "vtg90": {"type": "INT64", "value": 6}, "vtg91": {"type": "VARCHAR", "value": "2020-01-23"}, "vtg92": {"type": "INT64", "value": 23000}, "vtg93": {"type": "INT64", "value": 24}, "vtg94": {"type": "INT64", "value": 7}, "vtg95": {"type": "VARCHAR", "value": "2020-01-24"}, "vtg96": {"type": "INT64", "value": 24000}, "vtg97": {"type": "INT64", "value": 25}, "vtg98": {"type": "INT64", "value": 8}, "vtg99": {"type": "VARCHAR", "value": "2020-01-25"}} 2 0 "" "PRIMARY" "2642245a-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.orders"] "mysqltest" 0.000000 0.000000 "" Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.399939 2024-10-31 13:55:59.406533 0.006594 0.005267 0.001326 0.000000 SELECT "select customer_id, customer_pincode from customers where customer_name = :customer_name /* VARCHAR */" {"customer_name": {"type": "VARCHAR", "value": "Alice"}} 2 0 "" "PRIMARY" "2642245a-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers"] "mysqltest" 0.000000 0.000000 "" -Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.448949 2024-10-31 13:55:59.449917 0.000968 0.000150 0.000816 0.000000 EXPLAIN "vexplain trace select customer_id, customer_pincode from customers where customer_name = :customer_name /* VARCHAR */" {"customer_name": {"type": "VARCHAR", "value": "Alice"}} 2 0 "" "PRIMARY" "2642245a-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers"] "mysqltest" 0.000000 0.000000 "" -Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.450074 2024-10-31 13:55:59.466978 0.016903 0.005573 0.011324 0.000000 SELECT "select c.customer_id, sum(o.order_amount) from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id" {} 42 0 "" "PRIMARY" "2642245a-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers","mysqltest.orders"] "mysqltest" 0.000000 0.000000 "" -Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.524263 2024-10-31 13:55:59.541357 0.017093 0.000398 0.016691 0.000000 EXPLAIN "vexplain trace select c.customer_id, sum(o.order_amount) from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id" {} 42 0 "" "PRIMARY" "2642245a-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers","mysqltest.orders"] "mysqltest" 0.000000 0.000000 "" -Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.541582 2024-10-31 13:55:59.547876 0.006294 0.000232 0.006060 0.000000 SELECT "select c.customer_id, p.area_name from customers as c join pincode_areas as p on c.customer_pincode = p.pincode" {} 32 0 "" "PRIMARY" "2642245a-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers","mysqltest.pincode_areas"] "mysqltest" 0.000000 0.000000 "" -Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.597428 2024-10-31 13:55:59.603448 0.006020 0.000225 0.005792 0.000000 EXPLAIN "vexplain trace select c.customer_id, p.area_name from customers as c join pincode_areas as p on c.customer_pincode = p.pincode" {} 32 0 "" "PRIMARY" "2642245a-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers","mysqltest.pincode_areas"] "mysqltest" 0.000000 0.000000 "" -Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.603608 2024-10-31 13:55:59.609379 0.005771 0.000677 0.005092 0.000000 SELECT "select distinct c.customer_id, c.customer_name from customers as c join orders as o on c.customer_id = o.customer_id where o.order_amount > (select avg(order_amount) from orders)" {} 29 0 "" "PRIMARY" "2642245a-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers","mysqltest.orders"] "mysqltest" 0.000000 0.000000 "" -Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.652759 2024-10-31 13:55:59.659110 0.006351 0.000426 0.005922 0.000000 EXPLAIN "vexplain trace select distinct c.customer_id, c.customer_name from customers as c join orders as o on c.customer_id = o.customer_id where o.order_amount > (select avg(order_amount) from orders)" {} 29 0 "" "PRIMARY" "2642245a-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers","mysqltest.orders"] "mysqltest" 0.000000 0.000000 "" +Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.448949 2024-10-31 13:55:59.449917 0.000968 0.000150 0.000816 0.000000 EXPLAIN "vexplain trace select customer_id, customer_pincode from customers where customer_name = :customer_name /* VARCHAR */" {"customer_name": {"type": "VARCHAR", "value": "Alice"}} 2 0 "" "PRIMARY" "2642245b-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers"] "mysqltest" 0.000000 0.000000 "" +Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.450074 2024-10-31 13:55:59.466978 0.016903 0.005573 0.011324 0.000000 SELECT "select c.customer_id, sum(o.order_amount) from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id" {} 42 0 "" "PRIMARY" "2642245b-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers","mysqltest.orders"] "mysqltest" 0.000000 0.000000 "" +Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.524263 2024-10-31 13:55:59.541357 0.017093 0.000398 0.016691 0.000000 EXPLAIN "vexplain trace select c.customer_id, sum(o.order_amount) from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id" {} 42 0 "" "PRIMARY" "2642245b-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers","mysqltest.orders"] "mysqltest" 0.000000 0.000000 "" +Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.541582 2024-10-31 13:55:59.547876 0.006294 0.000232 0.006060 0.000000 SELECT "select c.customer_id, p.area_name from customers as c join pincode_areas as p on c.customer_pincode = p.pincode" {} 32 0 "" "PRIMARY" "2642245b-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers","mysqltest.pincode_areas"] "mysqltest" 0.000000 0.000000 "" +Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.597428 2024-10-31 13:55:59.603448 0.006020 0.000225 0.005792 0.000000 EXPLAIN "vexplain trace select c.customer_id, p.area_name from customers as c join pincode_areas as p on c.customer_pincode = p.pincode" {} 32 0 "" "PRIMARY" "2642245b-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers","mysqltest.pincode_areas"] "mysqltest" 0.000000 0.000000 "" +Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.603608 2024-10-31 13:55:59.609379 0.005771 0.000677 0.005092 0.000000 SELECT "select distinct c.customer_id, c.customer_name from customers as c join orders as o on c.customer_id = o.customer_id where o.order_amount > (select avg(order_amount) from orders)" {} 29 0 "" "PRIMARY" "2642245b-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers","mysqltest.orders"] "mysqltest" 0.000000 0.000000 "" +Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.652759 2024-10-31 13:55:59.659110 0.006351 0.000426 0.005922 0.000000 EXPLAIN "vexplain trace select distinct c.customer_id, c.customer_name from customers as c join orders as o on c.customer_id = o.customer_id where o.order_amount > (select avg(order_amount) from orders)" {} 29 0 "" "PRIMARY" "2642245b-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers","mysqltest.orders"] "mysqltest" 0.000000 0.000000 "" Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.659252 2024-10-31 13:55:59.667417 0.008165 0.000224 0.007938 0.000000 SELECT "select c.customer_id, c.customer_name from customers as c left join orders as o on c.customer_id = o.customer_id where o.order_id is null" {} 62 0 "" "PRIMARY" "2642245a-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers","mysqltest.orders"] "mysqltest" 0.000000 0.000000 "" Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.715029 2024-10-31 13:55:59.723167 0.008137 0.000238 0.007896 0.000000 EXPLAIN "vexplain trace select c.customer_id, c.customer_name from customers as c left join orders as o on c.customer_id = o.customer_id where o.order_id is null" {} 62 0 "" "PRIMARY" "2642245a-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers","mysqltest.orders"] "mysqltest" 0.000000 0.000000 "" Execute 127.0.0.1:60335 'userData1' '' 2024-10-31 13:55:59.723373 2024-10-31 13:55:59.732395 0.009021 0.001151 0.007867 0.000000 SELECT "select c.customer_id, c.customer_name, sum(o.order_amount) as total_amount from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id, c.customer_name order by total_amount desc limit :vtg1 /* INT64 */" {"vtg1": {"type": "INT64", "value": 5}} 42 0 "" "PRIMARY" "2642245a-97c2-11ef-b9e9-321ca607f906" false ["mysqltest.customers","mysqltest.orders"] "mysqltest" 0.000000 0.000000 "" diff --git a/go/testdata/vtgate.query.log.parsed.bv.txt b/go/testdata/vtgate.query.log.parsed.bv.txt index 6325ad5..e2a2d0d 100644 --- a/go/testdata/vtgate.query.log.parsed.bv.txt +++ b/go/testdata/vtgate.query.log.parsed.bv.txt @@ -1,44 +1,44 @@ -create table customers ( +2883466137428011160:create table customers ( customer_id int, customer_name varchar(100), customer_pincode int, primary key (customer_id) ) -create table pincode_areas ( +2883466137428011160:create table pincode_areas ( pincode int, area_name varchar(100) ) -create table orders ( +2883466137428011160:create table orders ( order_id int, customer_id int, order_date date, order_amount double, primary key (order_id) ) -create table name_idx ( +2883466137428011160:create table name_idx ( `name` varchar(100), customer_id int, keyspace_id varbinary(16), primary key (`name`, customer_id) ) -vexplain trace insert into pincode_areas(pincode, area_name) values (110001, 'Connaught Place'), (110002, 'Lodhi Road'), (110003, 'Civil Lines'), (110004, 'Kashmere Gate'), (110005, 'Chandni Chowk'), (110006, 'Barakhamba Road'), (110007, 'Kamla Nagar'), (110008, 'Karol Bagh'), (110009, 'Paharganj'), (110010, 'Patel Nagar'), (110011, 'South Extension'), (110012, 'Lajpat Nagar'), (110013, 'Sarojini Nagar'), (110014, 'Malviya Nagar'), (110015, 'Saket') -vexplain trace insert into customers(customer_id, customer_name, customer_pincode) values (1, 'John Doe', 110001), (2, 'Jane Doe', 110002), (3, 'Alice', 110003), (4, 'Bob', 110004), (5, 'Charlie', 110004), (6, 'David', 110006), (7, 'Eve', 110007), (8, 'Frank', 110008), (9, 'Grace', 110009), (10, 'Heidi', 110004), (11, 'Ivy', 110011), (12, 'Alice', 110005), (13, 'Bob', 110003), (14, 'Charlie', 110014), (15, 'David', 110015), (16, 'Frank', 110008), (17, 'Grace', 110009), (18, 'Isaac', 110010), (19, 'Julia', 110011), (20, 'Kevin', 110012), (21, 'Laura', 110013), (22, 'Michael', 110014), (23, 'Nina', 110015), (24, 'Oscar', 110001), (25, 'Patricia', 110002), (26, 'Quincy', 110003), (27, 'Rachel', 110004), (28, 'Samuel', 110005), (29, 'Tina', 110006), (30, 'Ulysses', 110007) -vexplain trace insert into orders(order_id, customer_id, order_date, order_amount) values (1, 1, '2020-01-01', 1000), (2, 2, '2020-01-02', 2000), (3, 3, '2020-01-03', 3000), (4, 4, '2020-01-04', 4000), (5, 5, '2020-01-05', 5000), (6, 6, '2020-01-06', 6000), (7, 7, '2020-01-07', 7000), (8, 8, '2020-01-08', 8000), (9, 9, '2020-01-09', 9000), (10, 10, '2020-01-10', 10000), (11, 11, '2020-01-11', 11000), (12, 12, '2020-01-12', 12000), (13, 13, '2020-01-13', 13000), (14, 14, '2020-01-14', 14000), (15, 15, '2020-01-15', 15000), (16, 16, '2020-01-16', 16000), (17, 17, '2020-01-17', 17000), (18, 1, '2020-01-18', 18000), (19, 2, '2020-01-19', 19000), (20, 3, '2020-01-20', 20000), (21, 4, '2020-01-21', 21000), (22, 5, '2020-01-22', 22000), (23, 6, '2020-01-23', 23000), (24, 7, '2020-01-24', 24000), (25, 8, '2020-01-25', 25000), (26, 9, '2020-01-26', 26000), (27, 10, '2020-01-27', 27000), (28, 11, '2020-01-28', 28000), (29, 12, '2020-01-29', 29000), (30, 13, '2020-01-30', 30000), (31, 14, '2020-01-31', 31000), (32, 15, '2020-02-01', 32000), (33, 16, '2020-02-02', 33000), (34, 17, '2020-02-03', 34000), (35, 18, '2020-02-04', 35000), (36, 19, '2020-02-05', 36000), (37, 20, '2020-02-06', 37000), (38, 21, '2020-02-07', 38000), (39, 22, '2020-02-08', 39000), (40, 23, '2020-02-09', 40000), (41, 24, '2020-02-10', 41000), (42, 25, '2020-02-11', 42000), (43, 26, '2020-02-12', 43000), (44, 27, '2020-02-13', 44000), (45, 28, '2020-02-14', 45000), (46, 29, '2020-02-15', 46000), (47, 30, '2020-02-16', 47000), (48, 1, '2020-02-17', 48000), (49, 2, '2020-02-18', 49000), (50, 3, '2020-02-19', 50000) -select customer_id, customer_pincode from customers where customer_name = 'Alice' -vexplain trace select customer_id, customer_pincode from customers where customer_name = 'Alice' -select c.customer_id, sum(o.order_amount) from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id -vexplain trace select c.customer_id, sum(o.order_amount) from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id -select c.customer_id, p.area_name from customers as c join pincode_areas as p on c.customer_pincode = p.pincode -vexplain trace select c.customer_id, p.area_name from customers as c join pincode_areas as p on c.customer_pincode = p.pincode -select distinct c.customer_id, c.customer_name from customers as c join orders as o on c.customer_id = o.customer_id where o.order_amount > (select avg(order_amount) from orders) -vexplain trace select distinct c.customer_id, c.customer_name from customers as c join orders as o on c.customer_id = o.customer_id where o.order_amount > (select avg(order_amount) from orders) -select c.customer_id, c.customer_name from customers as c left join orders as o on c.customer_id = o.customer_id where o.order_id is null -vexplain trace select c.customer_id, c.customer_name from customers as c left join orders as o on c.customer_id = o.customer_id where o.order_id is null -select c.customer_id, c.customer_name, sum(o.order_amount) as total_amount from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id, c.customer_name order by total_amount desc limit 5 -vexplain trace select c.customer_id, c.customer_name, sum(o.order_amount) as total_amount from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id, c.customer_name order by total_amount desc limit 5 -select distinct c1.customer_id, c1.customer_name from customers as c1 join orders as o1 on c1.customer_id = o1.customer_id join orders as o2 on c1.customer_id = o2.customer_id where DATEDIFF(o2.order_date, o1.order_date) = 1 -vexplain trace select distinct c1.customer_id, c1.customer_name from customers as c1 join orders as o1 on c1.customer_id = o1.customer_id join orders as o2 on c1.customer_id = o2.customer_id where DATEDIFF(o2.order_date, o1.order_date) = 1 -select DATE_FORMAT(order_date, '%Y-%m') as `month`, count(distinct customer_id) as unique_customers, count(*) as total_orders, sum(order_amount) as total_sales from orders group by `month` order by `month` asc -vexplain trace select DATE_FORMAT(order_date, '%Y-%m') as `month`, count(distinct customer_id) as unique_customers, count(*) as total_orders, sum(order_amount) as total_sales from orders group by `month` order by `month` asc -select order_count, count(*) as customer_count from (select customer_id, count(*) as order_count from orders group by customer_id) as customer_orders group by order_count order by order_count asc -vexplain trace select order_count, count(*) as customer_count from (select customer_id, count(*) as order_count from orders group by customer_id) as customer_orders group by order_count order by order_count asc \ No newline at end of file +2883466137428011160:vexplain trace insert into pincode_areas(pincode, area_name) values (110001, 'Connaught Place'), (110002, 'Lodhi Road'), (110003, 'Civil Lines'), (110004, 'Kashmere Gate'), (110005, 'Chandni Chowk'), (110006, 'Barakhamba Road'), (110007, 'Kamla Nagar'), (110008, 'Karol Bagh'), (110009, 'Paharganj'), (110010, 'Patel Nagar'), (110011, 'South Extension'), (110012, 'Lajpat Nagar'), (110013, 'Sarojini Nagar'), (110014, 'Malviya Nagar'), (110015, 'Saket') +2883466137428011160:vexplain trace insert into customers(customer_id, customer_name, customer_pincode) values (1, 'John Doe', 110001), (2, 'Jane Doe', 110002), (3, 'Alice', 110003), (4, 'Bob', 110004), (5, 'Charlie', 110004), (6, 'David', 110006), (7, 'Eve', 110007), (8, 'Frank', 110008), (9, 'Grace', 110009), (10, 'Heidi', 110004), (11, 'Ivy', 110011), (12, 'Alice', 110005), (13, 'Bob', 110003), (14, 'Charlie', 110014), (15, 'David', 110015), (16, 'Frank', 110008), (17, 'Grace', 110009), (18, 'Isaac', 110010), (19, 'Julia', 110011), (20, 'Kevin', 110012), (21, 'Laura', 110013), (22, 'Michael', 110014), (23, 'Nina', 110015), (24, 'Oscar', 110001), (25, 'Patricia', 110002), (26, 'Quincy', 110003), (27, 'Rachel', 110004), (28, 'Samuel', 110005), (29, 'Tina', 110006), (30, 'Ulysses', 110007) +2883466137428011160:vexplain trace insert into orders(order_id, customer_id, order_date, order_amount) values (1, 1, '2020-01-01', 1000), (2, 2, '2020-01-02', 2000), (3, 3, '2020-01-03', 3000), (4, 4, '2020-01-04', 4000), (5, 5, '2020-01-05', 5000), (6, 6, '2020-01-06', 6000), (7, 7, '2020-01-07', 7000), (8, 8, '2020-01-08', 8000), (9, 9, '2020-01-09', 9000), (10, 10, '2020-01-10', 10000), (11, 11, '2020-01-11', 11000), (12, 12, '2020-01-12', 12000), (13, 13, '2020-01-13', 13000), (14, 14, '2020-01-14', 14000), (15, 15, '2020-01-15', 15000), (16, 16, '2020-01-16', 16000), (17, 17, '2020-01-17', 17000), (18, 1, '2020-01-18', 18000), (19, 2, '2020-01-19', 19000), (20, 3, '2020-01-20', 20000), (21, 4, '2020-01-21', 21000), (22, 5, '2020-01-22', 22000), (23, 6, '2020-01-23', 23000), (24, 7, '2020-01-24', 24000), (25, 8, '2020-01-25', 25000), (26, 9, '2020-01-26', 26000), (27, 10, '2020-01-27', 27000), (28, 11, '2020-01-28', 28000), (29, 12, '2020-01-29', 29000), (30, 13, '2020-01-30', 30000), (31, 14, '2020-01-31', 31000), (32, 15, '2020-02-01', 32000), (33, 16, '2020-02-02', 33000), (34, 17, '2020-02-03', 34000), (35, 18, '2020-02-04', 35000), (36, 19, '2020-02-05', 36000), (37, 20, '2020-02-06', 37000), (38, 21, '2020-02-07', 38000), (39, 22, '2020-02-08', 39000), (40, 23, '2020-02-09', 40000), (41, 24, '2020-02-10', 41000), (42, 25, '2020-02-11', 42000), (43, 26, '2020-02-12', 43000), (44, 27, '2020-02-13', 44000), (45, 28, '2020-02-14', 45000), (46, 29, '2020-02-15', 46000), (47, 30, '2020-02-16', 47000), (48, 1, '2020-02-17', 48000), (49, 2, '2020-02-18', 49000), (50, 3, '2020-02-19', 50000) +2883466137428011160:select customer_id, customer_pincode from customers where customer_name = 'Alice' +6110208129700697065:vexplain trace select customer_id, customer_pincode from customers where customer_name = 'Alice' +6110208129700697065:select c.customer_id, sum(o.order_amount) from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id +6110208129700697065:vexplain trace select c.customer_id, sum(o.order_amount) from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id +6110208129700697065:select c.customer_id, p.area_name from customers as c join pincode_areas as p on c.customer_pincode = p.pincode +6110208129700697065:vexplain trace select c.customer_id, p.area_name from customers as c join pincode_areas as p on c.customer_pincode = p.pincode +6110208129700697065:select distinct c.customer_id, c.customer_name from customers as c join orders as o on c.customer_id = o.customer_id where o.order_amount > (select avg(order_amount) from orders) +6110208129700697065:vexplain trace select distinct c.customer_id, c.customer_name from customers as c join orders as o on c.customer_id = o.customer_id where o.order_amount > (select avg(order_amount) from orders) +2883466137428011160:select c.customer_id, c.customer_name from customers as c left join orders as o on c.customer_id = o.customer_id where o.order_id is null +2883466137428011160:vexplain trace select c.customer_id, c.customer_name from customers as c left join orders as o on c.customer_id = o.customer_id where o.order_id is null +2883466137428011160:select c.customer_id, c.customer_name, sum(o.order_amount) as total_amount from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id, c.customer_name order by total_amount desc limit 5 +2883466137428011160:vexplain trace select c.customer_id, c.customer_name, sum(o.order_amount) as total_amount from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id, c.customer_name order by total_amount desc limit 5 +2883466137428011160:select distinct c1.customer_id, c1.customer_name from customers as c1 join orders as o1 on c1.customer_id = o1.customer_id join orders as o2 on c1.customer_id = o2.customer_id where DATEDIFF(o2.order_date, o1.order_date) = 1 +2883466137428011160:vexplain trace select distinct c1.customer_id, c1.customer_name from customers as c1 join orders as o1 on c1.customer_id = o1.customer_id join orders as o2 on c1.customer_id = o2.customer_id where DATEDIFF(o2.order_date, o1.order_date) = 1 +2883466137428011160:select DATE_FORMAT(order_date, '%Y-%m') as `month`, count(distinct customer_id) as unique_customers, count(*) as total_orders, sum(order_amount) as total_sales from orders group by `month` order by `month` asc +2883466137428011160:vexplain trace select DATE_FORMAT(order_date, '%Y-%m') as `month`, count(distinct customer_id) as unique_customers, count(*) as total_orders, sum(order_amount) as total_sales from orders group by `month` order by `month` asc +2883466137428011160:select order_count, count(*) as customer_count from (select customer_id, count(*) as order_count from orders group by customer_id) as customer_orders group by order_count order by order_count asc +2883466137428011160:vexplain trace select order_count, count(*) as customer_count from (select customer_id, count(*) as order_count from orders group by customer_id) as customer_orders group by order_count order by order_count asc \ No newline at end of file diff --git a/go/testdata/vtgate.query.log.parsed.txt b/go/testdata/vtgate.query.log.parsed.txt index 168ef38..0309998 100644 --- a/go/testdata/vtgate.query.log.parsed.txt +++ b/go/testdata/vtgate.query.log.parsed.txt @@ -1,25 +1,25 @@ -create table customers(customer_id int,customer_name varchar(100),customer_pincode int,primary key (customer_id)) -create table pincode_areas(pincode int,area_name varchar(100)) -create table orders(order_id int,customer_id int,order_date date,order_amount double,primary key (order_id)) -create table name_idx(name varchar(100),customer_id int,keyspace_id varbinary(16),primary key (name, customer_id)) -vexplain trace insert into pincode_areas(pincode, area_name) values (:vtg1 /* INT64 */, :vtg2 /* VARCHAR */), (:vtg3 /* INT64 */, :vtg4 /* VARCHAR */), (:vtg5 /* INT64 */, :vtg6 /* VARCHAR */), (:vtg7 /* INT64 */, :vtg8 /* VARCHAR */), (:vtg9 /* INT64 */, :vtg10 /* VARCHAR */), (:vtg11 /* INT64 */, :vtg12 /* VARCHAR */), (:vtg13 /* INT64 */, :vtg14 /* VARCHAR */), (:vtg15 /* INT64 */, :vtg16 /* VARCHAR */), (:vtg17 /* INT64 */, :vtg18 /* VARCHAR */), (:vtg19 /* INT64 */, :vtg20 /* VARCHAR */), (:vtg21 /* INT64 */, :vtg22 /* VARCHAR */), (:vtg23 /* INT64 */, :vtg24 /* VARCHAR */), (:vtg25 /* INT64 */, :vtg26 /* VARCHAR */), (:vtg27 /* INT64 */, :vtg28 /* VARCHAR */), (:vtg29 /* INT64 */, :vtg30 /* VARCHAR */) -vexplain trace insert into customers(customer_id, customer_name, customer_pincode) values (:vtg1 /* INT64 */, :vtg2 /* VARCHAR */, :vtg3 /* INT64 */), (:vtg4 /* INT64 */, :vtg5 /* VARCHAR */, :vtg6 /* INT64 */), (:vtg7 /* INT64 */, :vtg8 /* VARCHAR */, :vtg9 /* INT64 */), (:vtg10 /* INT64 */, :vtg11 /* VARCHAR */, :vtg12 /* INT64 */), (:vtg13 /* INT64 */, :vtg14 /* VARCHAR */, :vtg15 /* INT64 */), (:vtg16 /* INT64 */, :vtg17 /* VARCHAR */, :vtg18 /* INT64 */), (:vtg19 /* INT64 */, :vtg20 /* VARCHAR */, :vtg21 /* INT64 */), (:vtg22 /* INT64 */, :vtg23 /* VARCHAR */, :vtg24 /* INT64 */), (:vtg25 /* INT64 */, :vtg26 /* VARCHAR */, :vtg27 /* INT64 */), (:vtg28 /* INT64 */, :vtg29 /* VARCHAR */, :vtg30 /* INT64 */), (:vtg31 /* INT64 */, :vtg32 /* VARCHAR */, :vtg33 /* INT64 */), (:vtg34 /* INT64 */, :vtg35 /* VARCHAR */, :vtg36 /* INT64 */), (:vtg37 /* INT64 */, :vtg38 /* VARCHAR */, :vtg39 /* INT64 */), (:vtg40 /* INT64 */, :vtg41 /* VARCHAR */, :vtg42 /* INT64 */), (:vtg43 /* INT64 */, :vtg44 /* VARCHAR */, :vtg45 /* INT64 */), (:vtg46 /* INT64 */, :vtg47 /* VARCHAR */, :vtg48 /* INT64 */), (:vtg49 /* INT64 */, :vtg50 /* VARCHAR */, :vtg51 /* INT64 */), (:vtg52 /* INT64 */, :vtg53 /* VARCHAR */, :vtg54 /* INT64 */), (:vtg55 /* INT64 */, :vtg56 /* VARCHAR */, :vtg57 /* INT64 */), (:vtg58 /* INT64 */, :vtg59 /* VARCHAR */, :vtg60 /* INT64 */), (:vtg61 /* INT64 */, :vtg62 /* VARCHAR */, :vtg63 /* INT64 */), (:vtg64 /* INT64 */, :vtg65 /* VARCHAR */, :vtg66 /* INT64 */), (:vtg67 /* INT64 */, :vtg68 /* VARCHAR */, :vtg69 /* INT64 */), (:vtg70 /* INT64 */, :vtg71 /* VARCHAR */, :vtg72 /* INT64 */), (:vtg73 /* INT64 */, :vtg74 /* VARCHAR */, :vtg75 /* INT64 */), (:vtg76 /* INT64 */, :vtg77 /* VARCHAR */, :vtg78 /* INT64 */), (:vtg79 /* INT64 */, :vtg80 /* VARCHAR */, :vtg81 /* INT64 */), (:vtg82 /* INT64 */, :vtg83 /* VARCHAR */, :vtg84 /* INT64 */), (:vtg85 /* INT64 */, :vtg86 /* VARCHAR */, :vtg87 /* INT64 */), (:vtg88 /* INT64 */, :vtg89 /* VARCHAR */, :vtg90 /* INT64 */) -vexplain trace insert into orders(order_id, customer_id, order_date, order_amount) values (:vtg1 /* INT64 */, :vtg2 /* INT64 */, :vtg3 /* VARCHAR */, :vtg4 /* INT64 */), (:vtg5 /* INT64 */, :vtg6 /* INT64 */, :vtg7 /* VARCHAR */, :vtg8 /* INT64 */), (:vtg9 /* INT64 */, :vtg10 /* INT64 */, :vtg11 /* VARCHAR */, :vtg12 /* INT64 */), (:vtg13 /* INT64 */, :vtg14 /* INT64 */, :vtg15 /* VARCHAR */, :vtg16 /* INT64 */), (:vtg17 /* INT64 */, :vtg18 /* INT64 */, :vtg19 /* VARCHAR */, :vtg20 /* INT64 */), (:vtg21 /* INT64 */, :vtg22 /* INT64 */, :vtg23 /* VARCHAR */, :vtg24 /* INT64 */), (:vtg25 /* INT64 */, :vtg26 /* INT64 */, :vtg27 /* VARCHAR */, :vtg28 /* INT64 */), (:vtg29 /* INT64 */, :vtg30 /* INT64 */, :vtg31 /* VARCHAR */, :vtg32 /* INT64 */), (:vtg33 /* INT64 */, :vtg34 /* INT64 */, :vtg35 /* VARCHAR */, :vtg36 /* INT64 */), (:vtg37 /* INT64 */, :vtg38 /* INT64 */, :vtg39 /* VARCHAR */, :vtg40 /* INT64 */), (:vtg41 /* INT64 */, :vtg42 /* INT64 */, :vtg43 /* VARCHAR */, :vtg44 /* INT64 */), (:vtg45 /* INT64 */, :vtg46 /* INT64 */, :vtg47 /* VARCHAR */, :vtg48 /* INT64 */), (:vtg49 /* INT64 */, :vtg50 /* INT64 */, :vtg51 /* VARCHAR */, :vtg52 /* INT64 */), (:vtg53 /* INT64 */, :vtg54 /* INT64 */, :vtg55 /* VARCHAR */, :vtg56 /* INT64 */), (:vtg57 /* INT64 */, :vtg58 /* INT64 */, :vtg59 /* VARCHAR */, :vtg60 /* INT64 */), (:vtg61 /* INT64 */, :vtg62 /* INT64 */, :vtg63 /* VARCHAR */, :vtg64 /* INT64 */), (:vtg65 /* INT64 */, :vtg66 /* INT64 */, :vtg67 /* VARCHAR */, :vtg68 /* INT64 */), (:vtg69 /* INT64 */, :vtg70 /* INT64 */, :vtg71 /* VARCHAR */, :vtg72 /* INT64 */), (:vtg73 /* INT64 */, :vtg74 /* INT64 */, :vtg75 /* VARCHAR */, :vtg76 /* INT64 */), (:vtg77 /* INT64 */, :vtg78 /* INT64 */, :vtg79 /* VARCHAR */, :vtg80 /* INT64 */), (:vtg81 /* INT64 */, :vtg82 /* INT64 */, :vtg83 /* VARCHAR */, :vtg84 /* INT64 */), (:vtg85 /* INT64 */, :vtg86 /* INT64 */, :vtg87 /* VARCHAR */, :vtg88 /* INT64 */), (:vtg89 /* INT64 */, :vtg90 /* INT64 */, :vtg91 /* VARCHAR */, :vtg92 /* INT64 */), (:vtg93 /* INT64 */, :vtg94 /* INT64 */, :vtg95 /* VARCHAR */, :vtg96 /* INT64 */), (:vtg97 /* INT64 */, :vtg98 /* INT64 */, :vtg99 /* VARCHAR */, :vtg100 /* INT64 */), (:vtg101 /* INT64 */, :vtg102 /* INT64 */, :vtg103 /* VARCHAR */, :vtg104 /* INT64 */), (:vtg105 /* INT64 */, :vtg106 /* INT64 */, :vtg107 /* VARCHAR */, :vtg108 /* INT64 */), (:vtg109 /* INT64 */, :vtg110 /* INT64 */, :vtg111 /* VARCHAR */, :vtg112 /* INT64 */), (:vtg113 /* INT64 */, :vtg114 /* INT64 */, :vtg115 /* VARCHAR */, :vtg116 /* INT64 */), (:vtg117 /* INT64 */, :vtg118 /* INT64 */, :vtg119 /* VARCHAR */, :vtg120 /* INT64 */), (:vtg121 /* INT64 */, :vtg122 /* INT64 */, :vtg123 /* VARCHAR */, :vtg124 /* INT64 */), (:vtg125 /* INT64 */, :vtg126 /* INT64 */, :vtg127 /* VARCHAR */, :vtg128 /* INT64 */), (:vtg129 /* INT64 */, :vtg130 /* INT64 */, :vtg131 /* VARCHAR */, :vtg132 /* INT64 */), (:vtg133 /* INT64 */, :vtg134 /* INT64 */, :vtg135 /* VARCHAR */, :vtg136 /* INT64 */), (:vtg137 /* INT64 */, :vtg138 /* INT64 */, :vtg139 /* VARCHAR */, :vtg140 /* INT64 */), (:vtg141 /* INT64 */, :vtg142 /* INT64 */, :vtg143 /* VARCHAR */, :vtg144 /* INT64 */), (:vtg145 /* INT64 */, :vtg146 /* INT64 */, :vtg147 /* VARCHAR */, :vtg148 /* INT64 */), (:vtg149 /* INT64 */, :vtg150 /* INT64 */, :vtg151 /* VARCHAR */, :vtg152 /* INT64 */), (:vtg153 /* INT64 */, :vtg154 /* INT64 */, :vtg155 /* VARCHAR */, :vtg156 /* INT64 */), (:vtg157 /* INT64 */, :vtg158 /* INT64 */, :vtg159 /* VARCHAR */, :vtg160 /* INT64 */), (:vtg161 /* INT64 */, :vtg162 /* INT64 */, :vtg163 /* VARCHAR */, :vtg164 /* INT64 */), (:vtg165 /* INT64 */, :vtg166 /* INT64 */, :vtg167 /* VARCHAR */, :vtg168 /* INT64 */), (:vtg169 /* INT64 */, :vtg170 /* INT64 */, :vtg171 /* VARCHAR */, :vtg172 /* INT64 */), (:vtg173 /* INT64 */, :vtg174 /* INT64 */, :vtg175 /* VARCHAR */, :vtg176 /* INT64 */), (:vtg177 /* INT64 */, :vtg178 /* INT64 */, :vtg179 /* VARCHAR */, :vtg180 /* INT64 */), (:vtg181 /* INT64 */, :vtg182 /* INT64 */, :vtg183 /* VARCHAR */, :vtg184 /* INT64 */), (:vtg185 /* INT64 */, :vtg186 /* INT64 */, :vtg187 /* VARCHAR */, :vtg188 /* INT64 */), (:vtg189 /* INT64 */, :vtg190 /* INT64 */, :vtg191 /* VARCHAR */, :vtg192 /* INT64 */), (:vtg193 /* INT64 */, :vtg194 /* INT64 */, :vtg195 /* VARCHAR */, :vtg196 /* INT64 */), (:vtg197 /* INT64 */, :vtg198 /* INT64 */, :vtg199 /* VARCHAR */, :vtg200 /* INT64 */) -select customer_id, customer_pincode from customers where customer_name = :customer_name /* VARCHAR */ -vexplain trace select customer_id, customer_pincode from customers where customer_name = :customer_name /* VARCHAR */ -select c.customer_id, sum(o.order_amount) from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id -vexplain trace select c.customer_id, sum(o.order_amount) from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id -select c.customer_id, p.area_name from customers as c join pincode_areas as p on c.customer_pincode = p.pincode -vexplain trace select c.customer_id, p.area_name from customers as c join pincode_areas as p on c.customer_pincode = p.pincode -select distinct c.customer_id, c.customer_name from customers as c join orders as o on c.customer_id = o.customer_id where o.order_amount > (select avg(order_amount) from orders) -vexplain trace select distinct c.customer_id, c.customer_name from customers as c join orders as o on c.customer_id = o.customer_id where o.order_amount > (select avg(order_amount) from orders) -select c.customer_id, c.customer_name from customers as c left join orders as o on c.customer_id = o.customer_id where o.order_id is null -vexplain trace select c.customer_id, c.customer_name from customers as c left join orders as o on c.customer_id = o.customer_id where o.order_id is null -select c.customer_id, c.customer_name, sum(o.order_amount) as total_amount from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id, c.customer_name order by total_amount desc limit :vtg1 /* INT64 */ -vexplain trace select c.customer_id, c.customer_name, sum(o.order_amount) as total_amount from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id, c.customer_name order by total_amount desc limit :vtg1 /* INT64 */ -select distinct c1.customer_id, c1.customer_name from customers as c1 join orders as o1 on c1.customer_id = o1.customer_id join orders as o2 on c1.customer_id = o2.customer_id where DATEDIFF(o2.order_date, o1.order_date) = :vtg1 /* INT64 */ -vexplain trace select distinct c1.customer_id, c1.customer_name from customers as c1 join orders as o1 on c1.customer_id = o1.customer_id join orders as o2 on c1.customer_id = o2.customer_id where DATEDIFF(o2.order_date, o1.order_date) = :vtg1 /* INT64 */ -select DATE_FORMAT(order_date, :vtg1 /* VARCHAR */) as `month`, count(distinct customer_id) as unique_customers, count(*) as total_orders, sum(order_amount) as total_sales from orders group by `month` order by `month` asc -vexplain trace select DATE_FORMAT(order_date, :vtg1 /* VARCHAR */) as `month`, count(distinct customer_id) as unique_customers, count(*) as total_orders, sum(order_amount) as total_sales from orders group by `month` order by `month` asc -select order_count, count(*) as customer_count from (select customer_id, count(*) as order_count from orders group by customer_id) as customer_orders group by order_count order by order_count asc -vexplain trace select order_count, count(*) as customer_count from (select customer_id, count(*) as order_count from orders group by customer_id) as customer_orders group by order_count order by order_count asc \ No newline at end of file +2883466137428011160:create table customers(customer_id int,customer_name varchar(100),customer_pincode int,primary key (customer_id)) +2883466137428011160:create table pincode_areas(pincode int,area_name varchar(100)) +2883466137428011160:create table orders(order_id int,customer_id int,order_date date,order_amount double,primary key (order_id)) +2883466137428011160:create table name_idx(name varchar(100),customer_id int,keyspace_id varbinary(16),primary key (name, customer_id)) +2883466137428011160:vexplain trace insert into pincode_areas(pincode, area_name) values (:vtg1 /* INT64 */, :vtg2 /* VARCHAR */), (:vtg3 /* INT64 */, :vtg4 /* VARCHAR */), (:vtg5 /* INT64 */, :vtg6 /* VARCHAR */), (:vtg7 /* INT64 */, :vtg8 /* VARCHAR */), (:vtg9 /* INT64 */, :vtg10 /* VARCHAR */), (:vtg11 /* INT64 */, :vtg12 /* VARCHAR */), (:vtg13 /* INT64 */, :vtg14 /* VARCHAR */), (:vtg15 /* INT64 */, :vtg16 /* VARCHAR */), (:vtg17 /* INT64 */, :vtg18 /* VARCHAR */), (:vtg19 /* INT64 */, :vtg20 /* VARCHAR */), (:vtg21 /* INT64 */, :vtg22 /* VARCHAR */), (:vtg23 /* INT64 */, :vtg24 /* VARCHAR */), (:vtg25 /* INT64 */, :vtg26 /* VARCHAR */), (:vtg27 /* INT64 */, :vtg28 /* VARCHAR */), (:vtg29 /* INT64 */, :vtg30 /* VARCHAR */) +2883466137428011160:vexplain trace insert into customers(customer_id, customer_name, customer_pincode) values (:vtg1 /* INT64 */, :vtg2 /* VARCHAR */, :vtg3 /* INT64 */), (:vtg4 /* INT64 */, :vtg5 /* VARCHAR */, :vtg6 /* INT64 */), (:vtg7 /* INT64 */, :vtg8 /* VARCHAR */, :vtg9 /* INT64 */), (:vtg10 /* INT64 */, :vtg11 /* VARCHAR */, :vtg12 /* INT64 */), (:vtg13 /* INT64 */, :vtg14 /* VARCHAR */, :vtg15 /* INT64 */), (:vtg16 /* INT64 */, :vtg17 /* VARCHAR */, :vtg18 /* INT64 */), (:vtg19 /* INT64 */, :vtg20 /* VARCHAR */, :vtg21 /* INT64 */), (:vtg22 /* INT64 */, :vtg23 /* VARCHAR */, :vtg24 /* INT64 */), (:vtg25 /* INT64 */, :vtg26 /* VARCHAR */, :vtg27 /* INT64 */), (:vtg28 /* INT64 */, :vtg29 /* VARCHAR */, :vtg30 /* INT64 */), (:vtg31 /* INT64 */, :vtg32 /* VARCHAR */, :vtg33 /* INT64 */), (:vtg34 /* INT64 */, :vtg35 /* VARCHAR */, :vtg36 /* INT64 */), (:vtg37 /* INT64 */, :vtg38 /* VARCHAR */, :vtg39 /* INT64 */), (:vtg40 /* INT64 */, :vtg41 /* VARCHAR */, :vtg42 /* INT64 */), (:vtg43 /* INT64 */, :vtg44 /* VARCHAR */, :vtg45 /* INT64 */), (:vtg46 /* INT64 */, :vtg47 /* VARCHAR */, :vtg48 /* INT64 */), (:vtg49 /* INT64 */, :vtg50 /* VARCHAR */, :vtg51 /* INT64 */), (:vtg52 /* INT64 */, :vtg53 /* VARCHAR */, :vtg54 /* INT64 */), (:vtg55 /* INT64 */, :vtg56 /* VARCHAR */, :vtg57 /* INT64 */), (:vtg58 /* INT64 */, :vtg59 /* VARCHAR */, :vtg60 /* INT64 */), (:vtg61 /* INT64 */, :vtg62 /* VARCHAR */, :vtg63 /* INT64 */), (:vtg64 /* INT64 */, :vtg65 /* VARCHAR */, :vtg66 /* INT64 */), (:vtg67 /* INT64 */, :vtg68 /* VARCHAR */, :vtg69 /* INT64 */), (:vtg70 /* INT64 */, :vtg71 /* VARCHAR */, :vtg72 /* INT64 */), (:vtg73 /* INT64 */, :vtg74 /* VARCHAR */, :vtg75 /* INT64 */), (:vtg76 /* INT64 */, :vtg77 /* VARCHAR */, :vtg78 /* INT64 */), (:vtg79 /* INT64 */, :vtg80 /* VARCHAR */, :vtg81 /* INT64 */), (:vtg82 /* INT64 */, :vtg83 /* VARCHAR */, :vtg84 /* INT64 */), (:vtg85 /* INT64 */, :vtg86 /* VARCHAR */, :vtg87 /* INT64 */), (:vtg88 /* INT64 */, :vtg89 /* VARCHAR */, :vtg90 /* INT64 */) +2883466137428011160:vexplain trace insert into orders(order_id, customer_id, order_date, order_amount) values (:vtg1 /* INT64 */, :vtg2 /* INT64 */, :vtg3 /* VARCHAR */, :vtg4 /* INT64 */), (:vtg5 /* INT64 */, :vtg6 /* INT64 */, :vtg7 /* VARCHAR */, :vtg8 /* INT64 */), (:vtg9 /* INT64 */, :vtg10 /* INT64 */, :vtg11 /* VARCHAR */, :vtg12 /* INT64 */), (:vtg13 /* INT64 */, :vtg14 /* INT64 */, :vtg15 /* VARCHAR */, :vtg16 /* INT64 */), (:vtg17 /* INT64 */, :vtg18 /* INT64 */, :vtg19 /* VARCHAR */, :vtg20 /* INT64 */), (:vtg21 /* INT64 */, :vtg22 /* INT64 */, :vtg23 /* VARCHAR */, :vtg24 /* INT64 */), (:vtg25 /* INT64 */, :vtg26 /* INT64 */, :vtg27 /* VARCHAR */, :vtg28 /* INT64 */), (:vtg29 /* INT64 */, :vtg30 /* INT64 */, :vtg31 /* VARCHAR */, :vtg32 /* INT64 */), (:vtg33 /* INT64 */, :vtg34 /* INT64 */, :vtg35 /* VARCHAR */, :vtg36 /* INT64 */), (:vtg37 /* INT64 */, :vtg38 /* INT64 */, :vtg39 /* VARCHAR */, :vtg40 /* INT64 */), (:vtg41 /* INT64 */, :vtg42 /* INT64 */, :vtg43 /* VARCHAR */, :vtg44 /* INT64 */), (:vtg45 /* INT64 */, :vtg46 /* INT64 */, :vtg47 /* VARCHAR */, :vtg48 /* INT64 */), (:vtg49 /* INT64 */, :vtg50 /* INT64 */, :vtg51 /* VARCHAR */, :vtg52 /* INT64 */), (:vtg53 /* INT64 */, :vtg54 /* INT64 */, :vtg55 /* VARCHAR */, :vtg56 /* INT64 */), (:vtg57 /* INT64 */, :vtg58 /* INT64 */, :vtg59 /* VARCHAR */, :vtg60 /* INT64 */), (:vtg61 /* INT64 */, :vtg62 /* INT64 */, :vtg63 /* VARCHAR */, :vtg64 /* INT64 */), (:vtg65 /* INT64 */, :vtg66 /* INT64 */, :vtg67 /* VARCHAR */, :vtg68 /* INT64 */), (:vtg69 /* INT64 */, :vtg70 /* INT64 */, :vtg71 /* VARCHAR */, :vtg72 /* INT64 */), (:vtg73 /* INT64 */, :vtg74 /* INT64 */, :vtg75 /* VARCHAR */, :vtg76 /* INT64 */), (:vtg77 /* INT64 */, :vtg78 /* INT64 */, :vtg79 /* VARCHAR */, :vtg80 /* INT64 */), (:vtg81 /* INT64 */, :vtg82 /* INT64 */, :vtg83 /* VARCHAR */, :vtg84 /* INT64 */), (:vtg85 /* INT64 */, :vtg86 /* INT64 */, :vtg87 /* VARCHAR */, :vtg88 /* INT64 */), (:vtg89 /* INT64 */, :vtg90 /* INT64 */, :vtg91 /* VARCHAR */, :vtg92 /* INT64 */), (:vtg93 /* INT64 */, :vtg94 /* INT64 */, :vtg95 /* VARCHAR */, :vtg96 /* INT64 */), (:vtg97 /* INT64 */, :vtg98 /* INT64 */, :vtg99 /* VARCHAR */, :vtg100 /* INT64 */), (:vtg101 /* INT64 */, :vtg102 /* INT64 */, :vtg103 /* VARCHAR */, :vtg104 /* INT64 */), (:vtg105 /* INT64 */, :vtg106 /* INT64 */, :vtg107 /* VARCHAR */, :vtg108 /* INT64 */), (:vtg109 /* INT64 */, :vtg110 /* INT64 */, :vtg111 /* VARCHAR */, :vtg112 /* INT64 */), (:vtg113 /* INT64 */, :vtg114 /* INT64 */, :vtg115 /* VARCHAR */, :vtg116 /* INT64 */), (:vtg117 /* INT64 */, :vtg118 /* INT64 */, :vtg119 /* VARCHAR */, :vtg120 /* INT64 */), (:vtg121 /* INT64 */, :vtg122 /* INT64 */, :vtg123 /* VARCHAR */, :vtg124 /* INT64 */), (:vtg125 /* INT64 */, :vtg126 /* INT64 */, :vtg127 /* VARCHAR */, :vtg128 /* INT64 */), (:vtg129 /* INT64 */, :vtg130 /* INT64 */, :vtg131 /* VARCHAR */, :vtg132 /* INT64 */), (:vtg133 /* INT64 */, :vtg134 /* INT64 */, :vtg135 /* VARCHAR */, :vtg136 /* INT64 */), (:vtg137 /* INT64 */, :vtg138 /* INT64 */, :vtg139 /* VARCHAR */, :vtg140 /* INT64 */), (:vtg141 /* INT64 */, :vtg142 /* INT64 */, :vtg143 /* VARCHAR */, :vtg144 /* INT64 */), (:vtg145 /* INT64 */, :vtg146 /* INT64 */, :vtg147 /* VARCHAR */, :vtg148 /* INT64 */), (:vtg149 /* INT64 */, :vtg150 /* INT64 */, :vtg151 /* VARCHAR */, :vtg152 /* INT64 */), (:vtg153 /* INT64 */, :vtg154 /* INT64 */, :vtg155 /* VARCHAR */, :vtg156 /* INT64 */), (:vtg157 /* INT64 */, :vtg158 /* INT64 */, :vtg159 /* VARCHAR */, :vtg160 /* INT64 */), (:vtg161 /* INT64 */, :vtg162 /* INT64 */, :vtg163 /* VARCHAR */, :vtg164 /* INT64 */), (:vtg165 /* INT64 */, :vtg166 /* INT64 */, :vtg167 /* VARCHAR */, :vtg168 /* INT64 */), (:vtg169 /* INT64 */, :vtg170 /* INT64 */, :vtg171 /* VARCHAR */, :vtg172 /* INT64 */), (:vtg173 /* INT64 */, :vtg174 /* INT64 */, :vtg175 /* VARCHAR */, :vtg176 /* INT64 */), (:vtg177 /* INT64 */, :vtg178 /* INT64 */, :vtg179 /* VARCHAR */, :vtg180 /* INT64 */), (:vtg181 /* INT64 */, :vtg182 /* INT64 */, :vtg183 /* VARCHAR */, :vtg184 /* INT64 */), (:vtg185 /* INT64 */, :vtg186 /* INT64 */, :vtg187 /* VARCHAR */, :vtg188 /* INT64 */), (:vtg189 /* INT64 */, :vtg190 /* INT64 */, :vtg191 /* VARCHAR */, :vtg192 /* INT64 */), (:vtg193 /* INT64 */, :vtg194 /* INT64 */, :vtg195 /* VARCHAR */, :vtg196 /* INT64 */), (:vtg197 /* INT64 */, :vtg198 /* INT64 */, :vtg199 /* VARCHAR */, :vtg200 /* INT64 */) +2883466137428011160:select customer_id, customer_pincode from customers where customer_name = :customer_name /* VARCHAR */ +6110208129700697065:vexplain trace select customer_id, customer_pincode from customers where customer_name = :customer_name /* VARCHAR */ +6110208129700697065:select c.customer_id, sum(o.order_amount) from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id +6110208129700697065:vexplain trace select c.customer_id, sum(o.order_amount) from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id +6110208129700697065:select c.customer_id, p.area_name from customers as c join pincode_areas as p on c.customer_pincode = p.pincode +6110208129700697065:vexplain trace select c.customer_id, p.area_name from customers as c join pincode_areas as p on c.customer_pincode = p.pincode +6110208129700697065:select distinct c.customer_id, c.customer_name from customers as c join orders as o on c.customer_id = o.customer_id where o.order_amount > (select avg(order_amount) from orders) +6110208129700697065:vexplain trace select distinct c.customer_id, c.customer_name from customers as c join orders as o on c.customer_id = o.customer_id where o.order_amount > (select avg(order_amount) from orders) +2883466137428011160:select c.customer_id, c.customer_name from customers as c left join orders as o on c.customer_id = o.customer_id where o.order_id is null +2883466137428011160:vexplain trace select c.customer_id, c.customer_name from customers as c left join orders as o on c.customer_id = o.customer_id where o.order_id is null +2883466137428011160:select c.customer_id, c.customer_name, sum(o.order_amount) as total_amount from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id, c.customer_name order by total_amount desc limit :vtg1 /* INT64 */ +2883466137428011160:vexplain trace select c.customer_id, c.customer_name, sum(o.order_amount) as total_amount from customers as c join orders as o on c.customer_id = o.customer_id group by c.customer_id, c.customer_name order by total_amount desc limit :vtg1 /* INT64 */ +2883466137428011160:select distinct c1.customer_id, c1.customer_name from customers as c1 join orders as o1 on c1.customer_id = o1.customer_id join orders as o2 on c1.customer_id = o2.customer_id where DATEDIFF(o2.order_date, o1.order_date) = :vtg1 /* INT64 */ +2883466137428011160:vexplain trace select distinct c1.customer_id, c1.customer_name from customers as c1 join orders as o1 on c1.customer_id = o1.customer_id join orders as o2 on c1.customer_id = o2.customer_id where DATEDIFF(o2.order_date, o1.order_date) = :vtg1 /* INT64 */ +2883466137428011160:select DATE_FORMAT(order_date, :vtg1 /* VARCHAR */) as `month`, count(distinct customer_id) as unique_customers, count(*) as total_orders, sum(order_amount) as total_sales from orders group by `month` order by `month` asc +2883466137428011160:vexplain trace select DATE_FORMAT(order_date, :vtg1 /* VARCHAR */) as `month`, count(distinct customer_id) as unique_customers, count(*) as total_orders, sum(order_amount) as total_sales from orders group by `month` order by `month` asc +2883466137428011160:select order_count, count(*) as customer_count from (select customer_id, count(*) as order_count from orders group by customer_id) as customer_orders group by order_count order by order_count asc +2883466137428011160:vexplain trace select order_count, count(*) as customer_count from (select customer_id, count(*) as order_count from orders group by customer_id) as customer_orders group by order_count order by order_count asc \ No newline at end of file From 325719bd1273b9f9f1128e2a841c7aa7ba3003a7 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Wed, 20 Nov 2024 13:34:48 +0100 Subject: [PATCH 04/22] feat: beginning of transaction code --- CONTRIBUTING.md | 21 ------- README.md | 8 ++- go/transactions/transactions.go | 94 ++++++++++++++++++++++++++++ go/transactions/transactions_test.go | 17 +++++ 4 files changed, 118 insertions(+), 22 deletions(-) delete mode 100644 CONTRIBUTING.md create mode 100644 go/transactions/transactions.go create mode 100644 go/transactions/transactions_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index cecdfcd..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,21 +0,0 @@ -# How to contribute - -## Before you get started - -- Sign the CLA -- Read and observe our [Code of Conduct](https://github.com/pingcap/community/blob/master/CODE_OF_CONDUCT.md) - -## What you can contribute - -Anything! You can help by: - -- writing user document about how to use this framework -- triaging issues -- submitting new test cases -- fixing bugs of this test framework -- adding features that mysql test has but this implement does not -- ... - -## How to contact to the community - -Join us in the [tidbcommunity](https://join.slack.com/t/tidbcommunity/shared_invite/enQtNzc0MzI4ODExMDc4LWYwYmIzMjZkYzJiNDUxMmZlN2FiMGJkZjAyMzQ5NGU0NGY0NzI3NTYwMjAyNGQ1N2I2ZjAxNzc1OGUwYWM0NzE) slack workspace. diff --git a/README.md b/README.md index 26cff06..bcfec63 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,13 @@ We welcome contributions in the following areas: - Fixing bugs in the test framework - Adding features from the MySQL test framework that are missing in this implementation -For more details, see our [CONTRIBUTING.md](./CONTRIBUTING.md). +After cloning the repo, make sure to run + +```bash +make install-hooks +``` + +to install the pre-commit hooks. ## License diff --git a/go/transactions/transactions.go b/go/transactions/transactions.go new file mode 100644 index 0000000..eb4e004 --- /dev/null +++ b/go/transactions/transactions.go @@ -0,0 +1,94 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transactions + +import ( + "github.com/vitessio/vt/go/data" + "vitess.io/vitess/go/vt/sqlparser" +) + +type ( + Config struct { + FileName string + Loader data.Loader + } + + Connection struct { + // The connection ID + ID int + + buf []data.Query + + Autocommit bool + } + + TxSignature struct { + } +) + +func Run(cfg Config) { + // Figure out if autocommit is enabled + // If we see: + // 1. BEGIN we can assume autocommit is disabled + // 2. COMMIT and no BEGIN we can assume autocommit is enabled + // 3. ROLLBACK and no BEGIN we can assume autocommit is enabled + // 4. SET autocommit = 1/0 + count := 1000 + defaultAutocommit := true + loader := cfg.Loader.Load(cfg.FileName) + for { + count-- + if count == 0 { + // enough already. we'll assume autocommit is enabled because that is the default + break + } + query, kontinue := loader.Next() + if !kontinue { + break + } + + switch query.Type { + case data.Skip, data.Error, data.VExplain, data.Unknown: + panic("unexpected query type") + case data.Comment, data.CommentWithCommand, data.EmptyLine, data.WaitForAuthoritative, data.SkipIfBelowVersion: + // no-op for keys + case data.QueryT: + stmt, err := sqlparser.NewTestParser().Parse(query.Query) + if err != nil { + continue + } + switch stmt.(type) { + case *sqlparser.Begin: + defaultAutocommit = false + break + case *sqlparser.Commit: + break + } + } + } + err := loader.Close() + if err != nil { + panic(err.Error()) + } + + connections := map[int]*Connection{} + + loader = cfg.Loader.Load(cfg.FileName) + for { + + } +} diff --git a/go/transactions/transactions_test.go b/go/transactions/transactions_test.go new file mode 100644 index 0000000..ccd1d35 --- /dev/null +++ b/go/transactions/transactions_test.go @@ -0,0 +1,17 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transactions From b33e5d13865184b3bd3260c62d3a8d6d0b417db6 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Wed, 20 Nov 2024 14:02:45 +0100 Subject: [PATCH 05/22] Add logic for reading query log, batching queries related to a transaction, handle autocommit default/explicit setting. Queries sent to a channgel. No tests Signed-off-by: Rohit Nayak --- go/transactions/transactions.go | 66 +++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/go/transactions/transactions.go b/go/transactions/transactions.go index eb4e004..9004fd6 100644 --- a/go/transactions/transactions.go +++ b/go/transactions/transactions.go @@ -17,7 +17,10 @@ limitations under the License. package transactions import ( + "strings" + "github.com/vitessio/vt/go/data" + "vitess.io/vitess/go/vt/sqlparser" ) @@ -28,16 +31,12 @@ type ( } Connection struct { - // The connection ID - ID int - - buf []data.Query + Transaction []data.Query Autocommit bool } - TxSignature struct { - } + TxSignature struct{} ) func Run(cfg Config) { @@ -85,10 +84,63 @@ func Run(cfg Config) { panic(err.Error()) } - connections := map[int]*Connection{} + transactions := map[int]*Connection{} loader = cfg.Loader.Load(cfg.FileName) + ch := make(chan []data.Query, 1000) + for { + query, kontinue := loader.Next() + if !kontinue { + break + } + switch query.Type { + case data.Skip, data.Error, data.VExplain, data.Unknown: + panic("unexpected query type") + case data.Comment, data.CommentWithCommand, data.EmptyLine, data.WaitForAuthoritative, data.SkipIfBelowVersion: + // no-op for keys + case data.QueryT: + stmt, err := sqlparser.NewTestParser().Parse(query.Query) + if err != nil { + continue + } + switch stmt.(type) { + case *sqlparser.Begin: + case *sqlparser.Commit: + connection := transactions[query.ConnectionID] + ch <- connection.Transaction + connection.Transaction = nil + case *sqlparser.Set: + set := stmt.(*sqlparser.Set) + for _, expr := range set.Exprs { + if expr.Var.Name.Lowered() == "autocommit" { + val, ok := expr.Expr.(*sqlparser.Literal) + if !ok { + continue + } + val2 := strings.ToLower(val.Val) + if val2 == "1" || val2 == "on" || val2 == "true" { + transactions[query.ConnectionID].Autocommit = true + } else { + transactions[query.ConnectionID].Autocommit = false + } + } + } + default: + if sqlparser.IsDMLStatement(stmt) { + connection, ok := transactions[query.ConnectionID] + if !ok { + connection = &Connection{Autocommit: defaultAutocommit} + transactions[query.ConnectionID] = connection + } + if connection.Autocommit { + ch <- []data.Query{query} + } else { + connection.Transaction = append(connection.Transaction, query) + } + } + } + } } } From c9cd0b05bcdb8b699b847ec4cb89ceb529627dc2 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 20 Nov 2024 13:59:51 -0600 Subject: [PATCH 06/22] Add primitive logic to read and group transactions together Signed-off-by: Florent Poinsard --- go/data/query.go | 33 ++- go/data/query_log_parse.go | 4 +- go/data/query_log_parse_test.go | 6 +- go/data/typ.go | 4 +- go/data/vtgate_log_parse.go | 4 +- go/keys/keys.go | 33 +-- go/keys/keys_test.go | 4 +- go/keys/schemaInfo.go | 40 +-- go/keys/schemaInfo_test.go | 2 +- go/testdata/small-slow-query-log | 42 +++ go/testdata/small-slow-query-log.json | 28 ++ go/tester/tester.go | 2 +- go/transactions/transactions.go | 398 ++++++++++++++++++++------ go/transactions/transactions_test.go | 24 ++ 14 files changed, 476 insertions(+), 148 deletions(-) create mode 100644 go/testdata/small-slow-query-log create mode 100644 go/testdata/small-slow-query-log.json diff --git a/go/data/query.go b/go/data/query.go index 17d4c4e..3fce6e4 100644 --- a/go/data/query.go +++ b/go/data/query.go @@ -53,6 +53,37 @@ type ( } ) +// ForeachSQLQuery reads a query log file and calls the provided function for each normal SQL query in the log. +// If the query log contains directives, they will be read and queries will be skipped as necessary. +func ForeachSQLQuery(loader IteratorLoader, f func(Query) error) error { + skip := false + for { + query, kontinue := loader.Next() + if !kontinue { + break + } + + switch query.Type { + case Skip, Error, VExplain: + skip = true + case Unknown: + return fmt.Errorf("unknown command type: %s", query.Type) + case Comment, CommentWithCommand, EmptyLine, WaitForAuthoritative, SkipIfBelowVersion: + // no-op for keys + case SQLQuery: + if skip { + skip = false + continue + } + if err := f(query); err != nil { + return err + } + } + } + + return nil +} + // for a single query, it has some prefix. Prefix mapps to a query type. // e.g query_vertical maps to typ.Q_QUERY_VERTICAL func (q *Query) getQueryType(qu string) error { @@ -64,7 +95,7 @@ func (q *Query) getQueryType(qu string) error { if q.Type != CommentWithCommand { // A query that will sent to vitess q.Query = qu - q.Type = QueryT + q.Type = SQLQuery } else { log.WithFields(log.Fields{"line": q.Line, "command": q.FirstWord, "arguments": q.Query}).Error("invalid command") return fmt.Errorf("invalid command %s", q.FirstWord) diff --git a/go/data/query_log_parse.go b/go/data/query_log_parse.go index 23cec63..eddab64 100644 --- a/go/data/query_log_parse.go +++ b/go/data/query_log_parse.go @@ -147,7 +147,7 @@ func (s *mysqlLogReaderState) finalizeQuery() Query { query := Query{ Query: s.prevQuery, Line: s.queryStart, - Type: QueryT, + Type: SQLQuery, ConnectionID: s.prevConnectionID, } s.prevQuery = "" @@ -159,7 +159,7 @@ func (s *mysqlLogReaderState) processQuery(matches []string) Query { query := Query{ Query: s.prevQuery, Line: s.queryStart, - Type: QueryT, + Type: SQLQuery, ConnectionID: s.prevConnectionID, } s.prevQuery = "" diff --git a/go/data/query_log_parse_test.go b/go/data/query_log_parse_test.go index b7497d6..22a7f32 100644 --- a/go/data/query_log_parse_test.go +++ b/go/data/query_log_parse_test.go @@ -41,12 +41,12 @@ func TestSmallSnippet(t *testing.T) { { Query: "SET GLOBAL log_output = 'FILE'", Line: 4, - Type: QueryT, + Type: SQLQuery, ConnectionID: 32, }, { Query: "show databases", Line: 5, - Type: QueryT, + Type: SQLQuery, ConnectionID: 32, }, { Query: `UPDATE _vt.schema_migrations @@ -74,7 +74,7 @@ WHERE ) LIMIT 1`, Line: 6, - Type: QueryT, + Type: SQLQuery, ConnectionID: 24, }, } diff --git a/go/data/typ.go b/go/data/typ.go index 20b6282..7fdc6b5 100644 --- a/go/data/typ.go +++ b/go/data/typ.go @@ -21,7 +21,7 @@ import "strings" type CmdType int const ( - QueryT CmdType = iota + SQLQuery CmdType = iota Error Skip Unknown @@ -37,7 +37,7 @@ const ( ) var commandMap = map[string]CmdType{ //nolint:gochecknoglobals // this is instead of a const - "query": QueryT, + "query": SQLQuery, "error": Error, "skip": Skip, "skip_if_below_version": SkipIfBelowVersion, diff --git a/go/data/vtgate_log_parse.go b/go/data/vtgate_log_parse.go index e0ef595..ab89563 100644 --- a/go/data/vtgate_log_parse.go +++ b/go/data/vtgate_log_parse.go @@ -112,7 +112,7 @@ func (s *vtgateLogReaderState) Next() (Query, bool) { return Query{ Query: query, Line: s.lineNumber, - Type: QueryT, + Type: SQLQuery, ConnectionID: connectionID, }, true } @@ -137,7 +137,7 @@ func (s *vtgateLogReaderState) Next() (Query, bool) { return Query{ Query: parsedQuery, Line: s.lineNumber, - Type: QueryT, + Type: SQLQuery, ConnectionID: connectionID, }, true } diff --git a/go/keys/keys.go b/go/keys/keys.go index bc5b995..8a4116a 100644 --- a/go/keys/keys.go +++ b/go/keys/keys.go @@ -77,8 +77,8 @@ func Run(cfg Config) error { } func run(out io.Writer, cfg Config) error { - si := &schemaInfo{ - tables: make(map[string]columns), + si := &SchemaInfo{ + Tables: make(map[string]Columns), } ql := &queryList{ queries: make(map[string]*QueryAnalysisResult), @@ -86,28 +86,11 @@ func run(out io.Writer, cfg Config) error { } loader := cfg.Loader.Load(cfg.FileName) - skip := false - for { - query, kontinue := loader.Next() - if !kontinue { - break - } - switch query.Type { - case data.Skip, data.Error, data.VExplain: - skip = true - case data.Unknown: - return fmt.Errorf("unknown command type: %s", query.Type) - case data.Comment, data.CommentWithCommand, data.EmptyLine, data.WaitForAuthoritative, data.SkipIfBelowVersion: - // no-op for keys - case data.QueryT: - if skip { - skip = false - continue - } - process(query, si, ql) - } - } + _ = data.ForeachSQLQuery(loader, func(query data.Query) error { + process(query, si, ql) + return nil + }) closeErr := loader.Close() jsonWriteErr := ql.writeJSONTo(out) @@ -115,7 +98,7 @@ func run(out io.Writer, cfg Config) error { return errors.Join(closeErr, jsonWriteErr) } -func process(q data.Query, si *schemaInfo, ql *queryList) { +func process(q data.Query, si *SchemaInfo, ql *queryList) { ast, bv, err := sqlparser.NewTestParser().Parse2(q.Query) if err != nil { ql.addFailedQuery(q, err) @@ -130,7 +113,7 @@ func process(q data.Query, si *schemaInfo, ql *queryList) { } } -func (ql *queryList) processQuery(si *schemaInfo, ast sqlparser.Statement, q data.Query, bv sqlparser.BindVars) { +func (ql *queryList) processQuery(si *SchemaInfo, ast sqlparser.Statement, q data.Query, bv sqlparser.BindVars) { // handle panics defer func() { if r := recover(); r != nil { diff --git a/go/keys/keys_test.go b/go/keys/keys_test.go index f588634..e644aba 100644 --- a/go/keys/keys_test.go +++ b/go/keys/keys_test.go @@ -82,9 +82,9 @@ func TestKeys(t *testing.T) { func TestKeysNonAuthoritativeTable(t *testing.T) { q := data.Query{ Query: "select id from user where id = 20", - Type: data.QueryT, + Type: data.SQLQuery, } - si := &schemaInfo{} + si := &SchemaInfo{} ql := &queryList{ queries: make(map[string]*QueryAnalysisResult), failed: make(map[string]*QueryFailedResult), diff --git a/go/keys/schemaInfo.go b/go/keys/schemaInfo.go index 5416a8c..417901e 100644 --- a/go/keys/schemaInfo.go +++ b/go/keys/schemaInfo.go @@ -27,42 +27,42 @@ import ( "vitess.io/vitess/go/vt/vtgate/vindexes" ) -var _ semantics.SchemaInformation = (*schemaInfo)(nil) +var _ semantics.SchemaInformation = (*SchemaInfo)(nil) type ( - schemaInfo struct { - ksName string - tables map[string]columns + SchemaInfo struct { + KsName string + Tables map[string]Columns } - columns []vindexes.Column + Columns []vindexes.Column ) -func (s *schemaInfo) handleCreateTable(create *sqlparser.CreateTable) { - columns := make(columns, 0, len(create.TableSpec.Columns)) +func (s *SchemaInfo) handleCreateTable(create *sqlparser.CreateTable) { + columns := make(Columns, 0, len(create.TableSpec.Columns)) for _, col := range create.TableSpec.Columns { columns = append(columns, vindexes.Column{ Name: col.Name, Type: col.Type.SQLType(), }) } - s.tables[create.Table.Name.String()] = columns + s.Tables[create.Table.Name.String()] = columns } -func (s *schemaInfo) FindTableOrVindex(tablename sqlparser.TableName) (*vindexes.Table, vindexes.Vindex, string, topodata.TabletType, key.Destination, error) { +func (s *SchemaInfo) FindTableOrVindex(tablename sqlparser.TableName) (*vindexes.Table, vindexes.Vindex, string, topodata.TabletType, key.Destination, error) { var tbl *vindexes.Table ks := tablename.Qualifier.String() if ks == "" { - ks = s.ksName + ks = s.KsName } - if !tablename.Qualifier.NotEmpty() || tablename.Qualifier.String() == s.ksName { + if !tablename.Qualifier.NotEmpty() || tablename.Qualifier.String() == s.KsName { // This is a table from our keyspace. We should be able to find it - columns, found := s.tables[tablename.Name.String()] + columns, found := s.Tables[tablename.Name.String()] if found { tbl = &vindexes.Table{ Name: tablename.Name, - Keyspace: &vindexes.Keyspace{Name: s.ksName, Sharded: true}, + Keyspace: &vindexes.Keyspace{Name: s.KsName, Sharded: true}, Columns: columns, ColumnListAuthoritative: true, } @@ -81,30 +81,30 @@ func (s *schemaInfo) FindTableOrVindex(tablename sqlparser.TableName) (*vindexes return tbl, nil, ks, topodata.TabletType_REPLICA, nil, nil } -func (s *schemaInfo) ConnCollation() collations.ID { +func (s *SchemaInfo) ConnCollation() collations.ID { return collations.CollationBinaryID } -func (s *schemaInfo) Environment() *vtenv.Environment { +func (s *SchemaInfo) Environment() *vtenv.Environment { return vtenv.NewTestEnv() } -func (s *schemaInfo) ForeignKeyMode(string) (vschemapb.Keyspace_ForeignKeyMode, error) { +func (s *SchemaInfo) ForeignKeyMode(string) (vschemapb.Keyspace_ForeignKeyMode, error) { return vschemapb.Keyspace_unmanaged, nil } -func (s *schemaInfo) GetForeignKeyChecksState() *bool { +func (s *SchemaInfo) GetForeignKeyChecksState() *bool { return nil } -func (s *schemaInfo) KeyspaceError(string) error { +func (s *SchemaInfo) KeyspaceError(string) error { return nil } -func (s *schemaInfo) GetAggregateUDFs() []string { +func (s *SchemaInfo) GetAggregateUDFs() []string { return nil // TODO: maybe this should be a flag? } -func (s *schemaInfo) FindMirrorRule(sqlparser.TableName) (*vindexes.MirrorRule, error) { +func (s *SchemaInfo) FindMirrorRule(sqlparser.TableName) (*vindexes.MirrorRule, error) { return nil, nil } diff --git a/go/keys/schemaInfo_test.go b/go/keys/schemaInfo_test.go index 30ca4fb..77bfa8a 100644 --- a/go/keys/schemaInfo_test.go +++ b/go/keys/schemaInfo_test.go @@ -29,7 +29,7 @@ import ( func TestSchemaInfo(t *testing.T) { parser := sqlparser.NewTestParser() - si := &schemaInfo{tables: make(map[string]columns)} + si := &SchemaInfo{Tables: make(map[string]Columns)} ast, err := parser.Parse(`CREATE TABLE IF NOT EXISTS warehouse ( w_id INT NOT NULL, diff --git a/go/testdata/small-slow-query-log b/go/testdata/small-slow-query-log new file mode 100644 index 0000000..2464cd9 --- /dev/null +++ b/go/testdata/small-slow-query-log @@ -0,0 +1,42 @@ +/bin/mysqld, Version: 8.0.26 (Source distribution). started with: +Tcp port: 3306 Unix socket: /tmp/mysql.sock +# Time: 2023-08-01T12:00:01.852861Z +# User@Host: user[user] @ [XXX.XXX.XXX.XXX] Id: 779060 +# Query_time: 0.000043 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0 +SET timestamp=1690891201; +begin; +# Time: 2023-08-01T12:00:01.852861Z +# User@Host: user[user] @ [XXX.XXX.XXX.XXX] Id: 779060 +# Query_time: 0.000043 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0 +SET timestamp=1690891201; +update tblA set apa = 'toto' where foo = 12 and id = 43; +# Time: 2023-08-01T12:00:01.852861Z +# User@Host: user[user] @ [XXX.XXX.XXX.XXX] Id: 779060 +# Query_time: 0.000043 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0 +SET timestamp=1690891201; +update tblB set monkey = 'pippi' where bar = 12 and id = 44; +# Time: 2023-08-01T12:00:01.852861Z +# User@Host: user[user] @ [XXX.XXX.XXX.XXX] Id: 779060 +# Query_time: 0.000043 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0 +SET timestamp=1690891201; +commit; +# Time: 2023-08-01T12:00:01.852861Z +# User@Host: user[user] @ [XXX.XXX.XXX.XXX] Id: 779060 +# Query_time: 0.000043 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0 +SET timestamp=1690891201; +begin; +# Time: 2023-08-01T12:00:01.852861Z +# User@Host: user[user] @ [XXX.XXX.XXX.XXX] Id: 779060 +# Query_time: 0.000043 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0 +SET timestamp=1690891201; +update tblA set apa = 'toto' where foo = 43 and id = 5; +# Time: 2023-08-01T12:00:01.852861Z +# User@Host: user[user] @ [XXX.XXX.XXX.XXX] Id: 779060 +# Query_time: 0.000043 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0 +SET timestamp=1690891201; +update tblB set monkey = 'pippi' where bar = 43 and id = 16; +# Time: 2023-08-01T12:00:01.852861Z +# User@Host: user[user] @ [XXX.XXX.XXX.XXX] Id: 779060 +# Query_time: 0.000043 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0 +SET timestamp=1690891201; +commit; diff --git a/go/testdata/small-slow-query-log.json b/go/testdata/small-slow-query-log.json new file mode 100644 index 0000000..4254e3f --- /dev/null +++ b/go/testdata/small-slow-query-log.json @@ -0,0 +1,28 @@ +[ + { + "Queries": [ + "update tblA set apa = 'toto' where foo = :1 and id = :2", + "update tblB set monkey = 'pippi' where bar = :1 and id = :3" + ], + "Count": 0, + "Predicates": [ + "tblA.foo = :1", + "tblA.id = :2", + "tblB.bar = :1", + "tblB.id = :3" + ] + }, + { + "Queries": [ + "update tblA set apa = 'toto' where foo = :1 and id = :2", + "update tblB set monkey = 'pippi' where bar = :1 and id = :3" + ], + "Count": 0, + "Predicates": [ + "tblA.foo = :1", + "tblA.id = :2", + "tblB.bar = :1", + "tblB.id = :3" + ] + } +] diff --git a/go/tester/tester.go b/go/tester/tester.go index f654a0e..10ff819 100644 --- a/go/tester/tester.go +++ b/go/tester/tester.go @@ -189,7 +189,7 @@ func (t *Tester) handleQuery(q data.Query) { t.prepareVExplain(q.Query) case data.WaitForAuthoritative: t.waitAuthoritative(q.Query) - case data.QueryT: + case data.SQLQuery: if t.vexplain == "" { t.runQuery(q) return diff --git a/go/transactions/transactions.go b/go/transactions/transactions.go index 9004fd6..1c935ed 100644 --- a/go/transactions/transactions.go +++ b/go/transactions/transactions.go @@ -17,11 +17,18 @@ limitations under the License. package transactions import ( + "encoding/json" + "fmt" + "io" + "os" + "strconv" "strings" - "github.com/vitessio/vt/go/data" - "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vtgate/semantics" + + "github.com/vitessio/vt/go/data" + "github.com/vitessio/vt/go/keys" ) type ( @@ -31,15 +38,290 @@ type ( } Connection struct { - Transaction []data.Query + Transaction []sqlparser.Statement Autocommit bool } - TxSignature struct{} + TxSignature struct { + Queries []string + Count int + Predicates []predicateInfo + } + + predicateInfo struct { + Table string + Col string + Op sqlparser.ComparisonExprOperator + Val string + } ) +func (pi predicateInfo) String() string { + return fmt.Sprintf("%s.%s %s %s", pi.Table, pi.Col, pi.Op.ToString(), pi.Val) +} + +func (tx TxSignature) MarshalJSON() ([]byte, error) { + // Transform Predicates to an array of strings + predicateStrings := make([]string, len(tx.Predicates)) + for i, predicate := range tx.Predicates { + predicateStrings[i] = predicate.String() + } + + return json.Marshal(struct { + Queries []string + Count int + Predicates []string + }{ + Queries: tx.Queries, + Count: tx.Count, + Predicates: predicateStrings, + }) +} + func Run(cfg Config) { + run(os.Stdout, cfg) +} + +//nolint:funlen,gocognit,gocyclo,cyclop // this is dirty WIP +func run(out io.Writer, cfg Config) { + defaultAutocommit := GetAutocommitGuess(cfg) + transactions := map[int]*Connection{} + + loader := cfg.Loader.Load(cfg.FileName) + ch := make(chan []sqlparser.Statement, 1000) + + _ = data.ForeachSQLQuery(loader, func(query data.Query) error { + stmt, err := sqlparser.NewTestParser().Parse(query.Query) + if err != nil { + fmt.Println(err.Error()) + return nil + } + switch stmt := stmt.(type) { + case *sqlparser.Begin: + case *sqlparser.Commit: + connection := transactions[query.ConnectionID] + ch <- connection.Transaction + connection.Transaction = nil + case *sqlparser.Set: + for _, expr := range stmt.Exprs { + if expr.Var.Name.Lowered() == "autocommit" { + val, ok := expr.Expr.(*sqlparser.Literal) + if !ok { + continue + } + val2 := strings.ToLower(val.Val) + if val2 == "1" || val2 == "on" || val2 == "true" { + transactions[query.ConnectionID].Autocommit = true + } else { + transactions[query.ConnectionID].Autocommit = false + } + } + } + default: + if !sqlparser.IsDMLStatement(stmt) { + return nil + } + connection, ok := transactions[query.ConnectionID] + if !ok { + connection = &Connection{Autocommit: defaultAutocommit} + transactions[query.ConnectionID] = connection + } + if connection.Autocommit { + ch <- []sqlparser.Statement{stmt} + } else { + connection.Transaction = append(connection.Transaction, stmt) + } + } + return nil + }) + + // WIP: dummy data + // TODO: Use real schema information data with the 'vt schema' JSON output + si := &keys.SchemaInfo{ + Tables: map[string]keys.Columns{ + "tblA": { + {Name: sqlparser.NewIdentifierCI("apa")}, + {Name: sqlparser.NewIdentifierCI("foo")}, + {Name: sqlparser.NewIdentifierCI("id")}, + }, + "tblB": { + {Name: sqlparser.NewIdentifierCI("monkey")}, + {Name: sqlparser.NewIdentifierCI("bar")}, + {Name: sqlparser.NewIdentifierCI("id")}, + }, + "user": { + {Name: sqlparser.NewIdentifierCI("id")}, + {Name: sqlparser.NewIdentifierCI("name")}, + }, + "user_extra": { + {Name: sqlparser.NewIdentifierCI("user_id")}, + {Name: sqlparser.NewIdentifierCI("age")}, + }, + }, + } + + var txs []TxSignature +outer: + for { + select { + // TODO: when a transaction has the exact same signature, increment its usage count instead of adding a new one + case queries := <-ch: + var tx TxSignature + idToLiteral := make(map[string]int) + nextID := 1 + for _, query := range queries { + st, err := semantics.Analyze(query, "ks", si) + if err != nil { + panic(err) + } + + switch query := query.(type) { + case *sqlparser.Update: + // Step 0: + // We want to find all the predicates that can impact our vindex choice in the query. + // TODO: Implement more types of predicates, right now only comparisons with 1 column and 1 literal are handled. + // TODO: This whole step can actually be re-used for DELETE. + //nolint:nestif // this is dirty WIP + if query.Where != nil { + // Step 1: + // Find all predicates in the where clause that use a column and a literal + var predicates []predicateInfo + wheres := sqlparser.SplitAndExpression(nil, query.Where.Expr) + for _, where := range wheres { + if cmp, ok := where.(*sqlparser.ComparisonExpr); ok { + lhs, lhsOK := cmp.Left.(*sqlparser.ColName) + rhs, rhsOK := cmp.Right.(*sqlparser.ColName) + + if rhsStr := exprToString(cmp.Right); lhsOK && rhsStr != "" { + predicates = append(predicates, createPredicateInfo(st, lhs, cmp.Operator, rhsStr)) + } + + if lhsStr := exprToString(cmp.Left); rhsOK && lhsStr != "" { + switchedOp, ok := cmp.Operator.SwitchSides() + if ok { + predicates = append(predicates, createPredicateInfo(st, rhs, switchedOp, lhsStr)) + } + } + } + } + + // Step 2: + // Now that we have all the predicates, let's replace their literals with an ID + for i, predicate := range predicates { + id, ok := idToLiteral[predicate.Val] + if !ok { + idToLiteral[predicate.Val] = nextID + id = nextID + nextID++ + } + predicates[i].Val = fmt.Sprintf(":%d", id) + var foundOne bool + for _, txPred := range tx.Predicates { + if txPred == predicate { + foundOne = true + break + } + } + if !foundOne { + tx.Predicates = append(tx.Predicates, predicates[i]) + } + } + } + + // Step 3: + // Normalize the AST our own way: + // - Replace the value in SET by "v" + // - Replace the literals found in where clause comparisons by the corresponding ID we got earlier + normalizedAST := sqlparser.Rewrite(query, func(cursor *sqlparser.Cursor) bool { + switch node := cursor.Node().(type) { + case *sqlparser.SetExpr: + cursor.Replace(&sqlparser.SetExpr{ + Var: node.Var, + Expr: sqlparser.NewArgument("v"), + }) + case *sqlparser.Where: + var newWhere sqlparser.Where + wheres := sqlparser.SplitAndExpression(nil, query.Where.Expr) + for _, where := range wheres { + switch cmp := where.(type) { + case *sqlparser.ComparisonExpr: + lhs, lhsOK := cmp.Left.(*sqlparser.Literal) + rhs, rhsOK := cmp.Right.(*sqlparser.Literal) + if !lhsOK && !rhsOK || lhsOK && rhsOK { + newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr) + continue + } + + var newCmp sqlparser.ComparisonExpr + newCmp.Operator = cmp.Operator + if lhsOK { + id, ok := idToLiteral[lhs.Val] + if !ok { + panic("we must be able to find a corresponding id") + } + newCmp.Left = sqlparser.NewArgument(strconv.Itoa(id)) + newCmp.Right = cmp.Right + } else { + id, ok := idToLiteral[rhs.Val] + if !ok { + panic("we must be able to find a corresponding id") + } + newCmp.Right = sqlparser.NewArgument(strconv.Itoa(id)) + newCmp.Left = cmp.Left + } + newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr, &newCmp) + default: + newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr, where) + } + } + cursor.Replace(&newWhere) + } + return true + }, nil) + tx.Queries = append(tx.Queries, sqlparser.String(normalizedAST)) + default: + panic("not supported for now") + } + } + txs = append(txs, tx) + default: + break outer + } + } + + txsJSON, err := json.MarshalIndent(txs, "", " ") + if err != nil { + panic(err) + } + fmt.Fprintf(out, "%s\n", string(txsJSON)) +} + +func createPredicateInfo(st *semantics.SemTable, expr *sqlparser.ColName, op sqlparser.ComparisonExprOperator, value string) predicateInfo { + tableInfo, err := st.TableInfoForExpr(expr) + if err != nil { + panic(err) + } + table := tableInfo.GetVindexTable() + if table == nil { + panic("table not found") + } + return predicateInfo{ + Table: table.Name.String(), + Col: expr.Name.String(), + Op: op, + Val: value, + } +} + +func exprToString(expr sqlparser.Expr) string { + if v, ok := expr.(*sqlparser.Literal); ok { + return v.Val + } + return "" +} + +func GetAutocommitGuess(cfg Config) bool { // Figure out if autocommit is enabled // If we see: // 1. BEGIN we can assume autocommit is disabled @@ -49,98 +331,36 @@ func Run(cfg Config) { count := 1000 defaultAutocommit := true loader := cfg.Loader.Load(cfg.FileName) - for { + defer func() { + err := loader.Close() + if err != nil { + panic(err.Error()) + } + }() + _ = data.ForeachSQLQuery(loader, func(query data.Query) error { count-- if count == 0 { // enough already. we'll assume autocommit is enabled because that is the default - break - } - query, kontinue := loader.Next() - if !kontinue { - break + return io.EOF } - switch query.Type { - case data.Skip, data.Error, data.VExplain, data.Unknown: - panic("unexpected query type") - case data.Comment, data.CommentWithCommand, data.EmptyLine, data.WaitForAuthoritative, data.SkipIfBelowVersion: - // no-op for keys - case data.QueryT: - stmt, err := sqlparser.NewTestParser().Parse(query.Query) - if err != nil { - continue - } - switch stmt.(type) { - case *sqlparser.Begin: - defaultAutocommit = false - break - case *sqlparser.Commit: - break - } + stmt, err := sqlparser.NewTestParser().Parse(query.Query) + if err != nil { + fmt.Println(err.Error()) + return nil } - } - err := loader.Close() - if err != nil { - panic(err.Error()) - } - transactions := map[int]*Connection{} - - loader = cfg.Loader.Load(cfg.FileName) - ch := make(chan []data.Query, 1000) - - for { - query, kontinue := loader.Next() - if !kontinue { - break + switch stmt.(type) { + case *sqlparser.Begin: + // BEGIN seen, so autocommit is disabled + return io.EOF + case *sqlparser.Commit: + defaultAutocommit = false + // no BEGIN seen, so autocommit is disabled + return io.EOF } - switch query.Type { - case data.Skip, data.Error, data.VExplain, data.Unknown: - panic("unexpected query type") - case data.Comment, data.CommentWithCommand, data.EmptyLine, data.WaitForAuthoritative, data.SkipIfBelowVersion: - // no-op for keys - case data.QueryT: - stmt, err := sqlparser.NewTestParser().Parse(query.Query) - if err != nil { - continue - } - switch stmt.(type) { - case *sqlparser.Begin: - case *sqlparser.Commit: - connection := transactions[query.ConnectionID] - ch <- connection.Transaction - connection.Transaction = nil - case *sqlparser.Set: - set := stmt.(*sqlparser.Set) - for _, expr := range set.Exprs { - if expr.Var.Name.Lowered() == "autocommit" { - val, ok := expr.Expr.(*sqlparser.Literal) - if !ok { - continue - } - val2 := strings.ToLower(val.Val) - if val2 == "1" || val2 == "on" || val2 == "true" { - transactions[query.ConnectionID].Autocommit = true - } else { - transactions[query.ConnectionID].Autocommit = false - } - } - } - default: - if sqlparser.IsDMLStatement(stmt) { - connection, ok := transactions[query.ConnectionID] - if !ok { - connection = &Connection{Autocommit: defaultAutocommit} - transactions[query.ConnectionID] = connection - } - if connection.Autocommit { - ch <- []data.Query{query} - } else { - connection.Transaction = append(connection.Transaction, query) - } - } - } - } - } + return nil + }) + return defaultAutocommit } diff --git a/go/transactions/transactions_test.go b/go/transactions/transactions_test.go index ccd1d35..e0eb317 100644 --- a/go/transactions/transactions_test.go +++ b/go/transactions/transactions_test.go @@ -15,3 +15,27 @@ limitations under the License. */ package transactions + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/vitessio/vt/go/data" +) + +func TestRun(t *testing.T) { + sb := &strings.Builder{} + run(sb, Config{ + FileName: "../testdata/small-slow-query-log", + Loader: data.SlowQueryLogLoader{}, + }) + + out, err := os.ReadFile("../testdata/small-slow-query-log.json") + require.NoError(t, err) + + assert.Equal(t, string(out), sb.String()) +} From f52154b9bf4200f2ffd5d649d3d4c2341e3e8976 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Thu, 21 Nov 2024 15:11:18 +0100 Subject: [PATCH 07/22] more refactoring Signed-off-by: Andres Taylor --- go/testdata/small-slow-query-log.json | 28 -- .../small-slow-query-transactions.json | 28 ++ go/transactions/transactions.go | 440 ++++++++++-------- go/transactions/transactions_test.go | 9 +- 4 files changed, 281 insertions(+), 224 deletions(-) delete mode 100644 go/testdata/small-slow-query-log.json create mode 100644 go/testdata/small-slow-query-transactions.json diff --git a/go/testdata/small-slow-query-log.json b/go/testdata/small-slow-query-log.json deleted file mode 100644 index 4254e3f..0000000 --- a/go/testdata/small-slow-query-log.json +++ /dev/null @@ -1,28 +0,0 @@ -[ - { - "Queries": [ - "update tblA set apa = 'toto' where foo = :1 and id = :2", - "update tblB set monkey = 'pippi' where bar = :1 and id = :3" - ], - "Count": 0, - "Predicates": [ - "tblA.foo = :1", - "tblA.id = :2", - "tblB.bar = :1", - "tblB.id = :3" - ] - }, - { - "Queries": [ - "update tblA set apa = 'toto' where foo = :1 and id = :2", - "update tblB set monkey = 'pippi' where bar = :1 and id = :3" - ], - "Count": 0, - "Predicates": [ - "tblA.foo = :1", - "tblA.id = :2", - "tblB.bar = :1", - "tblB.id = :3" - ] - } -] diff --git a/go/testdata/small-slow-query-transactions.json b/go/testdata/small-slow-query-transactions.json new file mode 100644 index 0000000..5f3252e --- /dev/null +++ b/go/testdata/small-slow-query-transactions.json @@ -0,0 +1,28 @@ +[ + { + "Queries": [ + "update tblA set apa = 'toto' where foo = :0 and id = :1", + "update tblB set monkey = 'pippi' where bar = :0 and id = :2" + ], + "Count": 0, + "Predicates": [ + "tblA.foo = 0", + "tblA.id = 1", + "tblB.bar = 0", + "tblB.id = 2" + ] + }, + { + "Queries": [ + "update tblA set apa = 'toto' where foo = :0 and id = :1", + "update tblB set monkey = 'pippi' where bar = :0 and id = :2" + ], + "Count": 0, + "Predicates": [ + "tblA.foo = 0", + "tblA.id = 1", + "tblB.bar = 0", + "tblB.id = 2" + ] + } +] diff --git a/go/transactions/transactions.go b/go/transactions/transactions.go index 1c935ed..01e0f1f 100644 --- a/go/transactions/transactions.go +++ b/go/transactions/transactions.go @@ -23,6 +23,7 @@ import ( "os" "strconv" "strings" + "sync" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/semantics" @@ -53,12 +54,19 @@ type ( Table string Col string Op sqlparser.ComparisonExprOperator - Val string + Val int + } + + state struct { + parser *sqlparser.Parser + si *keys.SchemaInfo + mu sync.Mutex + txs []TxSignature } ) func (pi predicateInfo) String() string { - return fmt.Sprintf("%s.%s %s %s", pi.Table, pi.Col, pi.Op.ToString(), pi.Val) + return fmt.Sprintf("%s.%s %s %d", pi.Table, pi.Col, pi.Op.ToString(), pi.Val) } func (tx TxSignature) MarshalJSON() ([]byte, error) { @@ -80,52 +88,64 @@ func (tx TxSignature) MarshalJSON() ([]byte, error) { } func Run(cfg Config) { - run(os.Stdout, cfg) + s := &state{ + parser: sqlparser.NewTestParser(), + si: getFakeSchema(), + } + s.run(os.Stdout, cfg) } -//nolint:funlen,gocognit,gocyclo,cyclop // this is dirty WIP -func run(out io.Writer, cfg Config) { - defaultAutocommit := GetAutocommitGuess(cfg) - transactions := map[int]*Connection{} +func getAutocommitStatus(set *sqlparser.Set, oldState bool) bool { + for _, expr := range set.Exprs { + if expr.Var.Name.Lowered() == "autocommit" { + val, ok := expr.Expr.(*sqlparser.Literal) + if !ok { + continue + } + str := strings.ToLower(val.Val) + if str == "1" || str == "on" || str == "true" { + return true + } + return false + } + } + return oldState +} - loader := cfg.Loader.Load(cfg.FileName) - ch := make(chan []sqlparser.Statement, 1000) +func (s *state) parse(q string) sqlparser.Statement { + stmt, err := s.parser.Parse(q) + if err != nil { + return nil + } + return stmt +} + +func (s *state) startProducing(loader data.IteratorLoader, defaultAutocommit bool, ch chan<- []sqlparser.Statement) { + connections := map[int]*Connection{} _ = data.ForeachSQLQuery(loader, func(query data.Query) error { - stmt, err := sqlparser.NewTestParser().Parse(query.Query) - if err != nil { - fmt.Println(err.Error()) + stmt := s.parse(query.Query) + if stmt == nil { return nil } switch stmt := stmt.(type) { case *sqlparser.Begin: case *sqlparser.Commit: - connection := transactions[query.ConnectionID] + // Commit seen, so we can yield the queries in the transaction + connection := connections[query.ConnectionID] ch <- connection.Transaction connection.Transaction = nil case *sqlparser.Set: - for _, expr := range stmt.Exprs { - if expr.Var.Name.Lowered() == "autocommit" { - val, ok := expr.Expr.(*sqlparser.Literal) - if !ok { - continue - } - val2 := strings.ToLower(val.Val) - if val2 == "1" || val2 == "on" || val2 == "true" { - transactions[query.ConnectionID].Autocommit = true - } else { - transactions[query.ConnectionID].Autocommit = false - } - } - } + connection := connections[query.ConnectionID] + connection.Autocommit = getAutocommitStatus(stmt, connection.Autocommit) default: if !sqlparser.IsDMLStatement(stmt) { return nil } - connection, ok := transactions[query.ConnectionID] + connection, ok := connections[query.ConnectionID] if !ok { connection = &Connection{Autocommit: defaultAutocommit} - transactions[query.ConnectionID] = connection + connections[query.ConnectionID] = connection } if connection.Autocommit { ch <- []sqlparser.Statement{stmt} @@ -135,193 +155,198 @@ func run(out io.Writer, cfg Config) { } return nil }) +} - // WIP: dummy data - // TODO: Use real schema information data with the 'vt schema' JSON output - si := &keys.SchemaInfo{ - Tables: map[string]keys.Columns{ - "tblA": { - {Name: sqlparser.NewIdentifierCI("apa")}, - {Name: sqlparser.NewIdentifierCI("foo")}, - {Name: sqlparser.NewIdentifierCI("id")}, - }, - "tblB": { - {Name: sqlparser.NewIdentifierCI("monkey")}, - {Name: sqlparser.NewIdentifierCI("bar")}, - {Name: sqlparser.NewIdentifierCI("id")}, - }, - "user": { - {Name: sqlparser.NewIdentifierCI("id")}, - {Name: sqlparser.NewIdentifierCI("name")}, - }, - "user_extra": { - {Name: sqlparser.NewIdentifierCI("user_id")}, - {Name: sqlparser.NewIdentifierCI("age")}, - }, - }, +func exprToString(expr sqlparser.Expr) string { + if v, ok := expr.(*sqlparser.Literal); ok { + return v.Val } + return "" +} - var txs []TxSignature -outer: - for { - select { - // TODO: when a transaction has the exact same signature, increment its usage count instead of adding a new one - case queries := <-ch: - var tx TxSignature - idToLiteral := make(map[string]int) - nextID := 1 - for _, query := range queries { - st, err := semantics.Analyze(query, "ks", si) - if err != nil { - panic(err) - } +func createPredicateInfo( + st *semantics.SemTable, + expr *sqlparser.ColName, + op sqlparser.ComparisonExprOperator, + value string, + n *normalizer, +) predicateInfo { + tableInfo, err := st.TableInfoForExpr(expr) + if err != nil { + panic(err) + } + table := tableInfo.GetVindexTable() + if table == nil { + panic("table not found") + } + return predicateInfo{ + Table: table.Name.String(), + Col: expr.Name.String(), + Op: op, + Val: n.normalize(value), + } +} - switch query := query.(type) { - case *sqlparser.Update: - // Step 0: - // We want to find all the predicates that can impact our vindex choice in the query. - // TODO: Implement more types of predicates, right now only comparisons with 1 column and 1 literal are handled. - // TODO: This whole step can actually be re-used for DELETE. - //nolint:nestif // this is dirty WIP - if query.Where != nil { - // Step 1: - // Find all predicates in the where clause that use a column and a literal - var predicates []predicateInfo - wheres := sqlparser.SplitAndExpression(nil, query.Where.Expr) - for _, where := range wheres { - if cmp, ok := where.(*sqlparser.ComparisonExpr); ok { - lhs, lhsOK := cmp.Left.(*sqlparser.ColName) - rhs, rhsOK := cmp.Right.(*sqlparser.ColName) +type normalizer struct { + m map[string]int + next int +} - if rhsStr := exprToString(cmp.Right); lhsOK && rhsStr != "" { - predicates = append(predicates, createPredicateInfo(st, lhs, cmp.Operator, rhsStr)) - } +func (n *normalizer) normalize(s string) int { + v, ok := n.m[s] + if ok { + return v + } + id := n.next + n.m[s] = id + n.next++ + return id +} - if lhsStr := exprToString(cmp.Left); rhsOK && lhsStr != "" { - switchedOp, ok := cmp.Operator.SwitchSides() - if ok { - predicates = append(predicates, createPredicateInfo(st, rhs, switchedOp, lhsStr)) - } - } - } - } +func getPredicates(e sqlparser.Expr, st *semantics.SemTable, n *normalizer) (predicates []predicateInfo) { + for _, predicate := range sqlparser.SplitAndExpression(nil, e) { + cmp, ok := predicate.(*sqlparser.ComparisonExpr) + if !ok { + continue + } - // Step 2: - // Now that we have all the predicates, let's replace their literals with an ID - for i, predicate := range predicates { - id, ok := idToLiteral[predicate.Val] - if !ok { - idToLiteral[predicate.Val] = nextID - id = nextID - nextID++ - } - predicates[i].Val = fmt.Sprintf(":%d", id) - var foundOne bool - for _, txPred := range tx.Predicates { - if txPred == predicate { - foundOne = true - break - } - } - if !foundOne { - tx.Predicates = append(tx.Predicates, predicates[i]) + lhs, lhsOK := cmp.Left.(*sqlparser.ColName) + rhs, rhsOK := cmp.Right.(*sqlparser.ColName) + + if rhsStr := exprToString(cmp.Right); lhsOK && rhsStr != "" { + predicates = append(predicates, createPredicateInfo(st, lhs, cmp.Operator, rhsStr, n)) + } + + if lhsStr := exprToString(cmp.Left); rhsOK && lhsStr != "" { + switchedOp, ok := cmp.Operator.SwitchSides() + if ok { + predicates = append(predicates, createPredicateInfo(st, rhs, switchedOp, lhsStr, n)) + } + } + } + + return +} + +//nolint:gocognit // we are still not clean +func (s *state) consume(ch <-chan []sqlparser.Statement, wg *sync.WaitGroup) { + defer wg.Done() + for queries := range ch { + n := &normalizer{m: make(map[string]int)} + tx := TxSignature{} + for _, query := range queries { + st, err := semantics.Analyze(query, "ks", s.si) + if err != nil { + panic(err) + } + + switch query := query.(type) { + case *sqlparser.Update: + // Step 0: + // We want to find all the predicates that can impact our vindex choice in the query. + // TODO: Implement more types of predicates, right now only comparisons with 1 column and 1 literal are handled. + // TODO: This whole step can actually be re-used for DELETE. + if query.Where != nil { + // Step 1: + // Find all predicates in the where clause that use a column and a literal + predicates := getPredicates(query.Where.Expr, st, n) + + loop: + for i, predicate := range predicates { + for _, txPred := range tx.Predicates { + if txPred == predicate { + continue loop } } + + tx.Predicates = append(tx.Predicates, predicates[i]) } + } + + // Step 3: + // Normalize the AST our own way: + // - Replace the value in SET by "v" + // - Replace the literals found in where clause comparisons by the corresponding ID we got earlier + normalizedAST := sqlparser.Rewrite(query, func(cursor *sqlparser.Cursor) bool { + switch node := cursor.Node().(type) { + case *sqlparser.SetExpr: + cursor.Replace(&sqlparser.SetExpr{ + Var: node.Var, + Expr: sqlparser.NewArgument("v"), + }) + case *sqlparser.Where: + var newWhere sqlparser.Where + wheres := sqlparser.SplitAndExpression(nil, query.Where.Expr) + for _, where := range wheres { + switch cmp := where.(type) { + case *sqlparser.ComparisonExpr: + lhs, lhsOK := cmp.Left.(*sqlparser.Literal) + rhs, rhsOK := cmp.Right.(*sqlparser.Literal) + if !lhsOK && !rhsOK || lhsOK && rhsOK { + newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr) + continue + } - // Step 3: - // Normalize the AST our own way: - // - Replace the value in SET by "v" - // - Replace the literals found in where clause comparisons by the corresponding ID we got earlier - normalizedAST := sqlparser.Rewrite(query, func(cursor *sqlparser.Cursor) bool { - switch node := cursor.Node().(type) { - case *sqlparser.SetExpr: - cursor.Replace(&sqlparser.SetExpr{ - Var: node.Var, - Expr: sqlparser.NewArgument("v"), - }) - case *sqlparser.Where: - var newWhere sqlparser.Where - wheres := sqlparser.SplitAndExpression(nil, query.Where.Expr) - for _, where := range wheres { - switch cmp := where.(type) { - case *sqlparser.ComparisonExpr: - lhs, lhsOK := cmp.Left.(*sqlparser.Literal) - rhs, rhsOK := cmp.Right.(*sqlparser.Literal) - if !lhsOK && !rhsOK || lhsOK && rhsOK { - newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr) - continue - } - - var newCmp sqlparser.ComparisonExpr - newCmp.Operator = cmp.Operator - if lhsOK { - id, ok := idToLiteral[lhs.Val] - if !ok { - panic("we must be able to find a corresponding id") - } - newCmp.Left = sqlparser.NewArgument(strconv.Itoa(id)) - newCmp.Right = cmp.Right - } else { - id, ok := idToLiteral[rhs.Val] - if !ok { - panic("we must be able to find a corresponding id") - } - newCmp.Right = sqlparser.NewArgument(strconv.Itoa(id)) - newCmp.Left = cmp.Left - } - newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr, &newCmp) - default: - newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr, where) + var newCmp sqlparser.ComparisonExpr + newCmp.Operator = cmp.Operator + if lhsOK { + id := n.normalize(lhs.Val) + newCmp.Left = sqlparser.NewArgument(strconv.Itoa(id)) + newCmp.Right = cmp.Right + } else { + id := n.normalize(rhs.Val) + newCmp.Right = sqlparser.NewArgument(strconv.Itoa(id)) + newCmp.Left = cmp.Left } + newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr, &newCmp) + default: + newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr, where) } - cursor.Replace(&newWhere) } - return true - }, nil) - tx.Queries = append(tx.Queries, sqlparser.String(normalizedAST)) - default: - panic("not supported for now") - } + cursor.Replace(&newWhere) + } + return true + }, nil) + tx.Queries = append(tx.Queries, sqlparser.String(normalizedAST)) + default: + panic("not supported for now") } - txs = append(txs, tx) - default: - break outer } + s.addSignature(tx) } +} - txsJSON, err := json.MarshalIndent(txs, "", " ") - if err != nil { - panic(err) - } - fmt.Fprintf(out, "%s\n", string(txsJSON)) +func (s *state) addSignature(tx TxSignature) { + s.mu.Lock() + defer s.mu.Unlock() + s.txs = append(s.txs, tx) } -func createPredicateInfo(st *semantics.SemTable, expr *sqlparser.ColName, op sqlparser.ComparisonExprOperator, value string) predicateInfo { - tableInfo, err := st.TableInfoForExpr(expr) +func (s *state) run(out io.Writer, cfg Config) { + defaultAutocommit := s.getAutocommitGuess(cfg) + + loader := cfg.Loader.Load(cfg.FileName) + ch := make(chan []sqlparser.Statement, 1000) + + var wg sync.WaitGroup + wg.Add(1) + + go s.consume(ch, &wg) + go func() { + s.startProducing(loader, defaultAutocommit, ch) + close(ch) + }() + + wg.Wait() + + txsJSON, err := json.MarshalIndent(s.txs, "", " ") if err != nil { panic(err) } - table := tableInfo.GetVindexTable() - if table == nil { - panic("table not found") - } - return predicateInfo{ - Table: table.Name.String(), - Col: expr.Name.String(), - Op: op, - Val: value, - } + _, _ = fmt.Fprintf(out, "%s\n", string(txsJSON)) } -func exprToString(expr sqlparser.Expr) string { - if v, ok := expr.(*sqlparser.Literal); ok { - return v.Val - } - return "" -} - -func GetAutocommitGuess(cfg Config) bool { +func (s *state) getAutocommitGuess(cfg Config) bool { // Figure out if autocommit is enabled // If we see: // 1. BEGIN we can assume autocommit is disabled @@ -344,9 +369,8 @@ func GetAutocommitGuess(cfg Config) bool { return io.EOF } - stmt, err := sqlparser.NewTestParser().Parse(query.Query) - if err != nil { - fmt.Println(err.Error()) + stmt := s.parse(query.Query) + if stmt == nil { return nil } @@ -364,3 +388,31 @@ func GetAutocommitGuess(cfg Config) bool { }) return defaultAutocommit } + +func getFakeSchema() *keys.SchemaInfo { + // WIP: dummy data + // TODO: Use real schema information data with the 'vt schema' JSON output + si := &keys.SchemaInfo{ + Tables: map[string]keys.Columns{ + "tblA": { + {Name: sqlparser.NewIdentifierCI("apa")}, + {Name: sqlparser.NewIdentifierCI("foo")}, + {Name: sqlparser.NewIdentifierCI("id")}, + }, + "tblB": { + {Name: sqlparser.NewIdentifierCI("monkey")}, + {Name: sqlparser.NewIdentifierCI("bar")}, + {Name: sqlparser.NewIdentifierCI("id")}, + }, + "user": { + {Name: sqlparser.NewIdentifierCI("id")}, + {Name: sqlparser.NewIdentifierCI("name")}, + }, + "user_extra": { + {Name: sqlparser.NewIdentifierCI("user_id")}, + {Name: sqlparser.NewIdentifierCI("age")}, + }, + }, + } + return si +} diff --git a/go/transactions/transactions_test.go b/go/transactions/transactions_test.go index e0eb317..35661a6 100644 --- a/go/transactions/transactions_test.go +++ b/go/transactions/transactions_test.go @@ -23,18 +23,23 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/vt/sqlparser" "github.com/vitessio/vt/go/data" ) func TestRun(t *testing.T) { sb := &strings.Builder{} - run(sb, Config{ + s := &state{ + parser: sqlparser.NewTestParser(), + si: getFakeSchema(), + } + s.run(sb, Config{ FileName: "../testdata/small-slow-query-log", Loader: data.SlowQueryLogLoader{}, }) - out, err := os.ReadFile("../testdata/small-slow-query-log.json") + out, err := os.ReadFile("../testdata/small-slow-query-transactions.json") require.NoError(t, err) assert.Equal(t, string(out), sb.String()) From bac4e4eee1719ea954ad9a153813cfdcdd89cd7a Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 21 Nov 2024 08:33:58 -0600 Subject: [PATCH 08/22] Small refactor of consume Signed-off-by: Florent Poinsard --- .../small-slow-query-transactions.json | 8 +- go/transactions/transactions.go | 137 +++++++++--------- 2 files changed, 73 insertions(+), 72 deletions(-) diff --git a/go/testdata/small-slow-query-transactions.json b/go/testdata/small-slow-query-transactions.json index 5f3252e..ffc5f5d 100644 --- a/go/testdata/small-slow-query-transactions.json +++ b/go/testdata/small-slow-query-transactions.json @@ -1,8 +1,8 @@ [ { "Queries": [ - "update tblA set apa = 'toto' where foo = :0 and id = :1", - "update tblB set monkey = 'pippi' where bar = :0 and id = :2" + "update tblA set apa = :v where foo = :0 and id = :1", + "update tblB set monkey = :v where bar = :0 and id = :2" ], "Count": 0, "Predicates": [ @@ -14,8 +14,8 @@ }, { "Queries": [ - "update tblA set apa = 'toto' where foo = :0 and id = :1", - "update tblB set monkey = 'pippi' where bar = :0 and id = :2" + "update tblA set apa = :v where foo = :0 and id = :1", + "update tblB set monkey = :v where bar = :0 and id = :2" ], "Count": 0, "Predicates": [ diff --git a/go/transactions/transactions.go b/go/transactions/transactions.go index 01e0f1f..695aac7 100644 --- a/go/transactions/transactions.go +++ b/go/transactions/transactions.go @@ -69,7 +69,7 @@ func (pi predicateInfo) String() string { return fmt.Sprintf("%s.%s %s %d", pi.Table, pi.Col, pi.Op.ToString(), pi.Val) } -func (tx TxSignature) MarshalJSON() ([]byte, error) { +func (tx *TxSignature) MarshalJSON() ([]byte, error) { // Transform Predicates to an array of strings predicateStrings := make([]string, len(tx.Predicates)) for i, predicate := range tx.Predicates { @@ -228,7 +228,6 @@ func getPredicates(e sqlparser.Expr, st *semantics.SemTable, n *normalizer) (pre return } -//nolint:gocognit // we are still not clean func (s *state) consume(ch <-chan []sqlparser.Statement, wg *sync.WaitGroup) { defer wg.Done() for queries := range ch { @@ -242,72 +241,7 @@ func (s *state) consume(ch <-chan []sqlparser.Statement, wg *sync.WaitGroup) { switch query := query.(type) { case *sqlparser.Update: - // Step 0: - // We want to find all the predicates that can impact our vindex choice in the query. - // TODO: Implement more types of predicates, right now only comparisons with 1 column and 1 literal are handled. - // TODO: This whole step can actually be re-used for DELETE. - if query.Where != nil { - // Step 1: - // Find all predicates in the where clause that use a column and a literal - predicates := getPredicates(query.Where.Expr, st, n) - - loop: - for i, predicate := range predicates { - for _, txPred := range tx.Predicates { - if txPred == predicate { - continue loop - } - } - - tx.Predicates = append(tx.Predicates, predicates[i]) - } - } - - // Step 3: - // Normalize the AST our own way: - // - Replace the value in SET by "v" - // - Replace the literals found in where clause comparisons by the corresponding ID we got earlier - normalizedAST := sqlparser.Rewrite(query, func(cursor *sqlparser.Cursor) bool { - switch node := cursor.Node().(type) { - case *sqlparser.SetExpr: - cursor.Replace(&sqlparser.SetExpr{ - Var: node.Var, - Expr: sqlparser.NewArgument("v"), - }) - case *sqlparser.Where: - var newWhere sqlparser.Where - wheres := sqlparser.SplitAndExpression(nil, query.Where.Expr) - for _, where := range wheres { - switch cmp := where.(type) { - case *sqlparser.ComparisonExpr: - lhs, lhsOK := cmp.Left.(*sqlparser.Literal) - rhs, rhsOK := cmp.Right.(*sqlparser.Literal) - if !lhsOK && !rhsOK || lhsOK && rhsOK { - newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr) - continue - } - - var newCmp sqlparser.ComparisonExpr - newCmp.Operator = cmp.Operator - if lhsOK { - id := n.normalize(lhs.Val) - newCmp.Left = sqlparser.NewArgument(strconv.Itoa(id)) - newCmp.Right = cmp.Right - } else { - id := n.normalize(rhs.Val) - newCmp.Right = sqlparser.NewArgument(strconv.Itoa(id)) - newCmp.Left = cmp.Left - } - newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr, &newCmp) - default: - newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr, where) - } - } - cursor.Replace(&newWhere) - } - return true - }, nil) - tx.Queries = append(tx.Queries, sqlparser.String(normalizedAST)) + s.consumeUpdate(query, st, n, &tx) default: panic("not supported for now") } @@ -316,6 +250,73 @@ func (s *state) consume(ch <-chan []sqlparser.Statement, wg *sync.WaitGroup) { } } +func (tx *TxSignature) addPredicate(predicates []predicateInfo) { +loop: + for i, predicate := range predicates { + for _, txPred := range tx.Predicates { + if txPred == predicate { + continue loop + } + } + + tx.Predicates = append(tx.Predicates, predicates[i]) + } +} + +func (s *state) consumeUpdate(query *sqlparser.Update, st *semantics.SemTable, n *normalizer, tx *TxSignature) { + defer func() { + tx.Queries = append(tx.Queries, sqlparser.String(query)) + }() + + // Normalize the AST our own way: + // - Replace the value in SET by "v" + // - Replace the literals found in where clause comparisons by the corresponding ID we got earlier + for i, expr := range query.Exprs { + query.Exprs[i] = &sqlparser.UpdateExpr{ + Name: expr.Name, + Expr: sqlparser.NewArgument("v"), + } + } + + if query.Where == nil { + return + } + + // Find all predicates in the where clause that use a column and a literal + // TODO: Implement support for join predicates + tx.addPredicate(getPredicates(query.Where.Expr, st, n)) + + var newWhere sqlparser.Where + wheres := sqlparser.SplitAndExpression(nil, query.Where.Expr) + for _, where := range wheres { + switch cmp := where.(type) { + case *sqlparser.ComparisonExpr: + lhs, lhsOK := cmp.Left.(*sqlparser.Literal) + rhs, rhsOK := cmp.Right.(*sqlparser.Literal) + if !lhsOK && !rhsOK || lhsOK && rhsOK { + newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr) + continue + } + + var newCmp sqlparser.ComparisonExpr + newCmp.Operator = cmp.Operator + if lhsOK { + id := n.normalize(lhs.Val) + newCmp.Left = sqlparser.NewArgument(strconv.Itoa(id)) + newCmp.Right = cmp.Right + } else { + id := n.normalize(rhs.Val) + newCmp.Right = sqlparser.NewArgument(strconv.Itoa(id)) + newCmp.Left = cmp.Left + } + newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr, &newCmp) + default: + newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr, where) + } + } + query.Where = &newWhere +} + func (s *state) addSignature(tx TxSignature) { s.mu.Lock() defer s.mu.Unlock() From adf277eeae139efa632023d12a2259dd823f7c22 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 21 Nov 2024 08:39:38 -0600 Subject: [PATCH 09/22] Add txs to the cli Signed-off-by: Florent Poinsard --- go/cmd/root.go | 1 + go/cmd/transactions.go | 52 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 go/cmd/transactions.go diff --git a/go/cmd/root.go b/go/cmd/root.go index 3f0fe04..a79cde1 100644 --- a/go/cmd/root.go +++ b/go/cmd/root.go @@ -37,6 +37,7 @@ func Execute() { root.AddCommand(testerCmd()) root.AddCommand(tracerCmd()) root.AddCommand(keysCmd()) + root.AddCommand(transactionsCmd()) err := root.Execute() if err != nil { diff --git a/go/cmd/transactions.go b/go/cmd/transactions.go new file mode 100644 index 0000000..748409a --- /dev/null +++ b/go/cmd/transactions.go @@ -0,0 +1,52 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "github.com/spf13/cobra" + "github.com/vitessio/vt/go/transactions" +) + +func transactionsCmd() *cobra.Command { + var inputType string + + cmd := &cobra.Command{ + Use: "transactions ", + Aliases: []string{"txs"}, + Short: "Analyze transactions on a query log", + Example: "vt transactions file.log", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + cfg := transactions.Config{ + FileName: args[0], + } + + loader, err := configureLoader(inputType, false) + if err != nil { + return err + } + cfg.Loader = loader + + transactions.Run(cfg) + return nil + }, + } + + addInputTypeFlag(cmd, &inputType) + + return cmd +} From c6e6c355467c791431e933c5e15af49716fd9f56 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 21 Nov 2024 12:18:42 -0600 Subject: [PATCH 10/22] wip Signed-off-by: Florent Poinsard --- go/transactions/transactions.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/go/transactions/transactions.go b/go/transactions/transactions.go index 695aac7..7b016ad 100644 --- a/go/transactions/transactions.go +++ b/go/transactions/transactions.go @@ -21,6 +21,8 @@ import ( "fmt" "io" "os" + "slices" + "sort" "strconv" "strings" "sync" @@ -51,10 +53,11 @@ type ( } predicateInfo struct { - Table string - Col string - Op sqlparser.ComparisonExprOperator - Val int + Table string + Col string + Op sqlparser.ComparisonExprOperator + Val int + Signature string } state struct { @@ -65,8 +68,12 @@ type ( } ) -func (pi predicateInfo) String() string { - return fmt.Sprintf("%s.%s %s %d", pi.Table, pi.Col, pi.Op.ToString(), pi.Val) +func (pi *predicateInfo) String() string { + if pi.Signature != "" { + return pi.Signature + } + pi.Signature = fmt.Sprintf("%s.%s %s %d", pi.Table, pi.Col, pi.Op.ToString(), pi.Val) + return pi.Signature } func (tx *TxSignature) MarshalJSON() ([]byte, error) { @@ -320,6 +327,12 @@ func (s *state) consumeUpdate(query *sqlparser.Update, st *semantics.SemTable, n func (s *state) addSignature(tx TxSignature) { s.mu.Lock() defer s.mu.Unlock() + + slices.Sort(tx.Queries) + sort.Slice(tx.Predicates, func(i, j int) bool { + return tx.Predicates[i] + }) + s.txs = append(s.txs, tx) } From f983d933fd0d1af7afa91b9c7f2c645b8256a7da Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 22 Nov 2024 09:24:00 +0100 Subject: [PATCH 11/22] feat: find duplicates of txSignatures and merge them Signed-off-by: Andres Taylor --- .../small-slow-query-transactions.json | 15 +- go/transactions/transaction_signature.go | 178 ++++++++++++++++++ go/transactions/transaction_signature_test.go | 92 +++++++++ go/transactions/transactions.go | 73 +------ go/transactions/transactions_test.go | 1 + 5 files changed, 279 insertions(+), 80 deletions(-) create mode 100644 go/transactions/transaction_signature.go create mode 100644 go/transactions/transaction_signature_test.go diff --git a/go/testdata/small-slow-query-transactions.json b/go/testdata/small-slow-query-transactions.json index ffc5f5d..2a0ae7e 100644 --- a/go/testdata/small-slow-query-transactions.json +++ b/go/testdata/small-slow-query-transactions.json @@ -4,25 +4,12 @@ "update tblA set apa = :v where foo = :0 and id = :1", "update tblB set monkey = :v where bar = :0 and id = :2" ], - "Count": 0, "Predicates": [ "tblA.foo = 0", "tblA.id = 1", "tblB.bar = 0", "tblB.id = 2" - ] - }, - { - "Queries": [ - "update tblA set apa = :v where foo = :0 and id = :1", - "update tblB set monkey = :v where bar = :0 and id = :2" ], - "Count": 0, - "Predicates": [ - "tblA.foo = 0", - "tblA.id = 1", - "tblB.bar = 0", - "tblB.id = 2" - ] + "Count": 2 } ] diff --git a/go/transactions/transaction_signature.go b/go/transactions/transaction_signature.go new file mode 100644 index 0000000..db29ffb --- /dev/null +++ b/go/transactions/transaction_signature.go @@ -0,0 +1,178 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transactions + +import ( + "cmp" + "encoding/json" + "fmt" + "hash/fnv" + "sort" + + "vitess.io/vitess/go/vt/sqlparser" +) + +type ( + TxSignature struct { + Queries []string `json:"queries"` + Predicates []predicateInfo `json:"predicates"` + Count int `json:"count"` + } + + txSignatureMap struct { + data map[uint64][]*TxSignature + } + + predicateInfo struct { + Table string + Col string + Op sqlparser.ComparisonExprOperator + Val int + } +) + +func (pi predicateInfo) String() string { + return fmt.Sprintf("%s.%s %s %d", pi.Table, pi.Col, pi.Op.ToString(), pi.Val) +} + +func (pi predicateInfo) compareTo(b predicateInfo) int { + if pi.Table != b.Table { + return cmp.Compare(pi.Table, b.Table) + } + if pi.Col != b.Col { + return cmp.Compare(pi.Col, b.Col) + } + if pi.Op != b.Op { + return cmp.Compare(pi.Op, b.Op) + } + return cmp.Compare(pi.Val, b.Val) +} + +func (tx *TxSignature) MarshalJSON() ([]byte, error) { + // Transform Predicates to an array of strings + predicateStrings := make([]string, len(tx.Predicates)) + for i, predicate := range tx.Predicates { + predicateStrings[i] = predicate.String() + } + + return json.Marshal(struct { + Queries []string + Predicates []string + Count int + }{ + Queries: tx.Queries, + Predicates: predicateStrings, + Count: tx.Count, + }) +} + +func (tx *TxSignature) Hash64() uint64 { + hasher := fnv.New64a() + + for _, query := range tx.Queries { + _, _ = hasher.Write([]byte(query)) + _, _ = hasher.Write([]byte{0}) + } + + for _, pred := range tx.Predicates { + _, _ = hasher.Write([]byte(pred.String())) + _, _ = hasher.Write([]byte{0}) + } + + return hasher.Sum64() +} + +func (tx *TxSignature) addPredicate(predicates []predicateInfo) { + for _, predicate := range predicates { + index := sort.Search(len(tx.Predicates), func(i int) bool { + return tx.Predicates[i].compareTo(predicate) >= 0 + }) + + if index < len(tx.Predicates) && tx.Predicates[index].compareTo(predicate) == 0 { + continue // Predicate already exists; skip it + } + + // Insert the predicate at the correct position + tx.Predicates = append(tx.Predicates, predicate) // Expand the slice by one + copy(tx.Predicates[index+1:], tx.Predicates[index:]) // Shift elements to the right + tx.Predicates[index] = predicate // Place the new predicate + } +} + +func newTxSignatureMap() *txSignatureMap { + return &txSignatureMap{ + data: make(map[uint64][]*TxSignature), + } +} + +func (m *txSignatureMap) Add(tx *TxSignature) { + hash := tx.Hash64() + + bucket, exists := m.data[hash] + + // Check if the hash already exists + if !exists { + tx.Count = 1 + m.data[hash] = []*TxSignature{tx} + return + } + + // Iterate over the bucket to check for exact match + for _, existingTx := range bucket { + if tx.Equals(existingTx) { + existingTx.Count++ + return + } + } + + // No exact match found; append to the bucket + m.data[hash] = append(bucket, tx) +} + +func (tx *TxSignature) Equals(other *TxSignature) bool { + // Compare Queries + if len(tx.Queries) != len(other.Queries) { + return false + } + for i := range tx.Queries { + if tx.Queries[i] != other.Queries[i] { + return false + } + } + + // Compare Predicates + if len(tx.Predicates) != len(other.Predicates) { + return false + } + for i := range tx.Predicates { + if tx.Predicates[i] != other.Predicates[i] { + return false + } + } + + return true +} + +func (m *txSignatureMap) MarshalJSON() ([]byte, error) { + // Collect all TxSignatures into a slice + var signatures []*TxSignature + for _, bucket := range m.data { + signatures = append(signatures, bucket...) + } + + return json.Marshal(signatures) +} diff --git a/go/transactions/transaction_signature_test.go b/go/transactions/transaction_signature_test.go new file mode 100644 index 0000000..fde26d5 --- /dev/null +++ b/go/transactions/transaction_signature_test.go @@ -0,0 +1,92 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transactions + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/vt/sqlparser" +) + +func TestTxSignature_addPredicate(t *testing.T) { + tests := []struct { + name string + existing []predicateInfo + newOnes []predicateInfo + expectedResult []predicateInfo + }{ + { + name: "Add single predicate to empty list", + existing: []predicateInfo{}, + newOnes: []predicateInfo{ + {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, + }, + expectedResult: []predicateInfo{ + {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, + }, + }, + { + name: "Add one predicates, have one", + existing: []predicateInfo{ + {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, + }, + newOnes: []predicateInfo{ + {Table: "table", Col: "name", Op: sqlparser.LikeOp, Val: 2}, + }, + expectedResult: []predicateInfo{ + {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, + {Table: "table", Col: "name", Op: sqlparser.LikeOp, Val: 2}, + }, + }, + { + name: "Add one predicates, have one, reverse order", + existing: []predicateInfo{ + {Table: "table", Col: "name", Op: sqlparser.LikeOp, Val: 2}, + }, + newOnes: []predicateInfo{ + {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, + }, + expectedResult: []predicateInfo{ + {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, + {Table: "table", Col: "name", Op: sqlparser.LikeOp, Val: 2}, + }, + }, + { + name: "Add existing predicate", + existing: []predicateInfo{ + {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, + }, + newOnes: []predicateInfo{ + {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, + }, + expectedResult: []predicateInfo{ + {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tx := &TxSignature{ + Predicates: tt.existing, + } + tx.addPredicate(tt.newOnes) + assert.Equal(t, tt.expectedResult, tx.Predicates) + }) + } +} diff --git a/go/transactions/transactions.go b/go/transactions/transactions.go index 7b016ad..d6e3bdb 100644 --- a/go/transactions/transactions.go +++ b/go/transactions/transactions.go @@ -21,8 +21,6 @@ import ( "fmt" "io" "os" - "slices" - "sort" "strconv" "strings" "sync" @@ -46,58 +44,19 @@ type ( Autocommit bool } - TxSignature struct { - Queries []string - Count int - Predicates []predicateInfo - } - - predicateInfo struct { - Table string - Col string - Op sqlparser.ComparisonExprOperator - Val int - Signature string - } - state struct { parser *sqlparser.Parser si *keys.SchemaInfo mu sync.Mutex - txs []TxSignature + txs *txSignatureMap } ) -func (pi *predicateInfo) String() string { - if pi.Signature != "" { - return pi.Signature - } - pi.Signature = fmt.Sprintf("%s.%s %s %d", pi.Table, pi.Col, pi.Op.ToString(), pi.Val) - return pi.Signature -} - -func (tx *TxSignature) MarshalJSON() ([]byte, error) { - // Transform Predicates to an array of strings - predicateStrings := make([]string, len(tx.Predicates)) - for i, predicate := range tx.Predicates { - predicateStrings[i] = predicate.String() - } - - return json.Marshal(struct { - Queries []string - Count int - Predicates []string - }{ - Queries: tx.Queries, - Count: tx.Count, - Predicates: predicateStrings, - }) -} - func Run(cfg Config) { s := &state{ parser: sqlparser.NewTestParser(), si: getFakeSchema(), + txs: newTxSignatureMap(), } s.run(os.Stdout, cfg) } @@ -211,6 +170,7 @@ func (n *normalizer) normalize(s string) int { } func getPredicates(e sqlparser.Expr, st *semantics.SemTable, n *normalizer) (predicates []predicateInfo) { + // TODO: Implement support for join predicates for _, predicate := range sqlparser.SplitAndExpression(nil, e) { cmp, ok := predicate.(*sqlparser.ComparisonExpr) if !ok { @@ -239,7 +199,7 @@ func (s *state) consume(ch <-chan []sqlparser.Statement, wg *sync.WaitGroup) { defer wg.Done() for queries := range ch { n := &normalizer{m: make(map[string]int)} - tx := TxSignature{} + tx := &TxSignature{} for _, query := range queries { st, err := semantics.Analyze(query, "ks", s.si) if err != nil { @@ -248,7 +208,7 @@ func (s *state) consume(ch <-chan []sqlparser.Statement, wg *sync.WaitGroup) { switch query := query.(type) { case *sqlparser.Update: - s.consumeUpdate(query, st, n, &tx) + s.consumeUpdate(query, st, n, tx) default: panic("not supported for now") } @@ -257,19 +217,6 @@ func (s *state) consume(ch <-chan []sqlparser.Statement, wg *sync.WaitGroup) { } } -func (tx *TxSignature) addPredicate(predicates []predicateInfo) { -loop: - for i, predicate := range predicates { - for _, txPred := range tx.Predicates { - if txPred == predicate { - continue loop - } - } - - tx.Predicates = append(tx.Predicates, predicates[i]) - } -} - func (s *state) consumeUpdate(query *sqlparser.Update, st *semantics.SemTable, n *normalizer, tx *TxSignature) { defer func() { tx.Queries = append(tx.Queries, sqlparser.String(query)) @@ -290,7 +237,6 @@ func (s *state) consumeUpdate(query *sqlparser.Update, st *semantics.SemTable, n } // Find all predicates in the where clause that use a column and a literal - // TODO: Implement support for join predicates tx.addPredicate(getPredicates(query.Where.Expr, st, n)) var newWhere sqlparser.Where @@ -324,16 +270,11 @@ func (s *state) consumeUpdate(query *sqlparser.Update, st *semantics.SemTable, n query.Where = &newWhere } -func (s *state) addSignature(tx TxSignature) { +func (s *state) addSignature(tx *TxSignature) { s.mu.Lock() defer s.mu.Unlock() - slices.Sort(tx.Queries) - sort.Slice(tx.Predicates, func(i, j int) bool { - return tx.Predicates[i] - }) - - s.txs = append(s.txs, tx) + s.txs.Add(tx) } func (s *state) run(out io.Writer, cfg Config) { diff --git a/go/transactions/transactions_test.go b/go/transactions/transactions_test.go index 35661a6..ca21739 100644 --- a/go/transactions/transactions_test.go +++ b/go/transactions/transactions_test.go @@ -33,6 +33,7 @@ func TestRun(t *testing.T) { s := &state{ parser: sqlparser.NewTestParser(), si: getFakeSchema(), + txs: newTxSignatureMap(), } s.run(sb, Config{ FileName: "../testdata/small-slow-query-log", From e769a72d6a6f615e07d171ab0d4f145f56924965 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 22 Nov 2024 10:11:52 +0100 Subject: [PATCH 12/22] minor fixes Signed-off-by: Andres Taylor --- go/transactions/transaction_signature.go | 4 ++++ go/transactions/transactions.go | 24 +++++++++++++++--------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/go/transactions/transaction_signature.go b/go/transactions/transaction_signature.go index db29ffb..1aa63c6 100644 --- a/go/transactions/transaction_signature.go +++ b/go/transactions/transaction_signature.go @@ -174,5 +174,9 @@ func (m *txSignatureMap) MarshalJSON() ([]byte, error) { signatures = append(signatures, bucket...) } + sort.Slice(signatures, func(i, j int) bool { + return signatures[i].Count > signatures[j].Count + }) + return json.Marshal(signatures) } diff --git a/go/transactions/transactions.go b/go/transactions/transactions.go index d6e3bdb..02a2803 100644 --- a/go/transactions/transactions.go +++ b/go/transactions/transactions.go @@ -88,7 +88,14 @@ func (s *state) parse(q string) sqlparser.Statement { func (s *state) startProducing(loader data.IteratorLoader, defaultAutocommit bool, ch chan<- []sqlparser.Statement) { connections := map[int]*Connection{} - + getConn := func(id int) *Connection { + connection, ok := connections[id] + if !ok { + connection = &Connection{Autocommit: defaultAutocommit} + connections[id] = connection + } + return connection + } _ = data.ForeachSQLQuery(loader, func(query data.Query) error { stmt := s.parse(query.Query) if stmt == nil { @@ -98,21 +105,20 @@ func (s *state) startProducing(loader data.IteratorLoader, defaultAutocommit boo case *sqlparser.Begin: case *sqlparser.Commit: // Commit seen, so we can yield the queries in the transaction - connection := connections[query.ConnectionID] + connection := getConn(query.ConnectionID) + if connection.Transaction == nil { + return nil + } ch <- connection.Transaction connection.Transaction = nil case *sqlparser.Set: - connection := connections[query.ConnectionID] - connection.Autocommit = getAutocommitStatus(stmt, connection.Autocommit) + conn := getConn(query.ConnectionID) + conn.Autocommit = getAutocommitStatus(stmt, defaultAutocommit) default: if !sqlparser.IsDMLStatement(stmt) { return nil } - connection, ok := connections[query.ConnectionID] - if !ok { - connection = &Connection{Autocommit: defaultAutocommit} - connections[query.ConnectionID] = connection - } + connection := getConn(query.ConnectionID) if connection.Autocommit { ch <- []sqlparser.Statement{stmt} } else { From 0ad0d1262e4a9bdb161b96f23958b3adb9982f05 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 22 Nov 2024 10:30:43 +0100 Subject: [PATCH 13/22] feat: also analyze deletes Signed-off-by: Andres Taylor --- go/transactions/transactions.go | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/go/transactions/transactions.go b/go/transactions/transactions.go index 02a2803..b153793 100644 --- a/go/transactions/transactions.go +++ b/go/transactions/transactions.go @@ -215,8 +215,10 @@ func (s *state) consume(ch <-chan []sqlparser.Statement, wg *sync.WaitGroup) { switch query := query.(type) { case *sqlparser.Update: s.consumeUpdate(query, st, n, tx) + case *sqlparser.Delete: + s.consumeDelete(query, st, n, tx) default: - panic("not supported for now") + panic(fmt.Sprintf("not supported for now: %T", query)) } } s.addSignature(tx) @@ -244,11 +246,28 @@ func (s *state) consumeUpdate(query *sqlparser.Update, st *semantics.SemTable, n // Find all predicates in the where clause that use a column and a literal tx.addPredicate(getPredicates(query.Where.Expr, st, n)) + query.Where = normalizeWhere(query.Where, n) +} + +func (s *state) consumeDelete(del *sqlparser.Delete, st *semantics.SemTable, n *normalizer, tx *TxSignature) { + defer func() { + tx.Queries = append(tx.Queries, sqlparser.String(del)) + }() - var newWhere sqlparser.Where - wheres := sqlparser.SplitAndExpression(nil, query.Where.Expr) - for _, where := range wheres { - switch cmp := where.(type) { + if del.Where == nil { + return + } + + // Find all predicates in the where clause that use a column and a literal + tx.addPredicate(getPredicates(del.Where.Expr, st, n)) + del.Where = normalizeWhere(del.Where, n) +} + +func normalizeWhere(where *sqlparser.Where, n *normalizer) (newWhere *sqlparser.Where) { + newWhere = new(sqlparser.Where) + predicates := sqlparser.SplitAndExpression(nil, where.Expr) + for _, predicate := range predicates { + switch cmp := predicate.(type) { case *sqlparser.ComparisonExpr: lhs, lhsOK := cmp.Left.(*sqlparser.Literal) rhs, rhsOK := cmp.Right.(*sqlparser.Literal) @@ -270,10 +289,10 @@ func (s *state) consumeUpdate(query *sqlparser.Update, st *semantics.SemTable, n } newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr, &newCmp) default: - newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr, where) + newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr, predicate) } } - query.Where = &newWhere + return } func (s *state) addSignature(tx *TxSignature) { From 69f1c71b4dfd0aebeae69de6881ae45ee2bcdd56 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 22 Nov 2024 14:17:59 +0100 Subject: [PATCH 14/22] comments and remove uneccessary method Signed-off-by: Andres Taylor --- go/keys/schemaInfo.go | 4 ++++ go/transactions/transactions.go | 30 +--------------------------- go/transactions/transactions_test.go | 3 ++- 3 files changed, 7 insertions(+), 30 deletions(-) diff --git a/go/keys/schemaInfo.go b/go/keys/schemaInfo.go index 417901e..794e978 100644 --- a/go/keys/schemaInfo.go +++ b/go/keys/schemaInfo.go @@ -30,6 +30,10 @@ import ( var _ semantics.SchemaInformation = (*SchemaInfo)(nil) type ( + // SchemaInfo is a simple implementation of semantics.SchemaInformation + // It will claim that any table that is asked for is present in the schema, with no columns specified and authoratative columns set to false + // it has a createTableHandler that can be used to populate the schema if the + // query log contains the CREATE TABLE statements SchemaInfo struct { KsName string Tables map[string]Columns diff --git a/go/transactions/transactions.go b/go/transactions/transactions.go index b153793..ae5d5cf 100644 --- a/go/transactions/transactions.go +++ b/go/transactions/transactions.go @@ -55,7 +55,7 @@ type ( func Run(cfg Config) { s := &state{ parser: sqlparser.NewTestParser(), - si: getFakeSchema(), + si: &keys.SchemaInfo{}, txs: newTxSignatureMap(), } s.run(os.Stdout, cfg) @@ -368,31 +368,3 @@ func (s *state) getAutocommitGuess(cfg Config) bool { }) return defaultAutocommit } - -func getFakeSchema() *keys.SchemaInfo { - // WIP: dummy data - // TODO: Use real schema information data with the 'vt schema' JSON output - si := &keys.SchemaInfo{ - Tables: map[string]keys.Columns{ - "tblA": { - {Name: sqlparser.NewIdentifierCI("apa")}, - {Name: sqlparser.NewIdentifierCI("foo")}, - {Name: sqlparser.NewIdentifierCI("id")}, - }, - "tblB": { - {Name: sqlparser.NewIdentifierCI("monkey")}, - {Name: sqlparser.NewIdentifierCI("bar")}, - {Name: sqlparser.NewIdentifierCI("id")}, - }, - "user": { - {Name: sqlparser.NewIdentifierCI("id")}, - {Name: sqlparser.NewIdentifierCI("name")}, - }, - "user_extra": { - {Name: sqlparser.NewIdentifierCI("user_id")}, - {Name: sqlparser.NewIdentifierCI("age")}, - }, - }, - } - return si -} diff --git a/go/transactions/transactions_test.go b/go/transactions/transactions_test.go index ca21739..a3ddadd 100644 --- a/go/transactions/transactions_test.go +++ b/go/transactions/transactions_test.go @@ -26,13 +26,14 @@ import ( "vitess.io/vitess/go/vt/sqlparser" "github.com/vitessio/vt/go/data" + "github.com/vitessio/vt/go/keys" ) func TestRun(t *testing.T) { sb := &strings.Builder{} s := &state{ parser: sqlparser.NewTestParser(), - si: getFakeSchema(), + si: &keys.SchemaInfo{}, txs: newTxSignatureMap(), } s.run(sb, Config{ From f22056c1f3638735a1f7328d4adee4a756db9560 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 22 Nov 2024 14:25:55 +0100 Subject: [PATCH 15/22] feat: remove txs that only happened once Signed-off-by: Andres Taylor --- go/transactions/transaction_signature.go | 11 ++++++----- go/transactions/transactions.go | 2 -- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/go/transactions/transaction_signature.go b/go/transactions/transaction_signature.go index 1aa63c6..a23c29b 100644 --- a/go/transactions/transaction_signature.go +++ b/go/transactions/transaction_signature.go @@ -63,7 +63,6 @@ func (pi predicateInfo) compareTo(b predicateInfo) int { } func (tx *TxSignature) MarshalJSON() ([]byte, error) { - // Transform Predicates to an array of strings predicateStrings := make([]string, len(tx.Predicates)) for i, predicate := range tx.Predicates { predicateStrings[i] = predicate.String() @@ -144,7 +143,6 @@ func (m *txSignatureMap) Add(tx *TxSignature) { } func (tx *TxSignature) Equals(other *TxSignature) bool { - // Compare Queries if len(tx.Queries) != len(other.Queries) { return false } @@ -154,7 +152,6 @@ func (tx *TxSignature) Equals(other *TxSignature) bool { } } - // Compare Predicates if len(tx.Predicates) != len(other.Predicates) { return false } @@ -168,10 +165,14 @@ func (tx *TxSignature) Equals(other *TxSignature) bool { } func (m *txSignatureMap) MarshalJSON() ([]byte, error) { - // Collect all TxSignatures into a slice + // Collect all interesting TxSignatures into a slice var signatures []*TxSignature for _, bucket := range m.data { - signatures = append(signatures, bucket...) + for _, txSig := range bucket { + if txSig.Count > 1 { + signatures = append(signatures, txSig) + } + } } sort.Slice(signatures, func(i, j int) bool { diff --git a/go/transactions/transactions.go b/go/transactions/transactions.go index ae5d5cf..8400b5b 100644 --- a/go/transactions/transactions.go +++ b/go/transactions/transactions.go @@ -217,8 +217,6 @@ func (s *state) consume(ch <-chan []sqlparser.Statement, wg *sync.WaitGroup) { s.consumeUpdate(query, st, n, tx) case *sqlparser.Delete: s.consumeDelete(query, st, n, tx) - default: - panic(fmt.Sprintf("not supported for now: %T", query)) } } s.addSignature(tx) From 030f0255174921563da5cdfc206f70ba97c2f829 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 22 Nov 2024 15:41:53 +0100 Subject: [PATCH 16/22] feat: clean up signatures Signed-off-by: Andres Taylor --- .../small-slow-query-transactions.json | 14 ++--- go/transactions/transaction_signature.go | 51 +++++++++++++++++-- go/transactions/transactions.go | 38 ++++++++++---- 3 files changed, 81 insertions(+), 22 deletions(-) diff --git a/go/testdata/small-slow-query-transactions.json b/go/testdata/small-slow-query-transactions.json index 2a0ae7e..bd3ca7e 100644 --- a/go/testdata/small-slow-query-transactions.json +++ b/go/testdata/small-slow-query-transactions.json @@ -1,15 +1,15 @@ [ { - "Queries": [ - "update tblA set apa = :v where foo = :0 and id = :1", - "update tblB set monkey = :v where bar = :0 and id = :2" + "query-signatures": [ + "update tblA where foo = :0 and id = :1 set apa", + "update tblB where bar = :0 and id = :2 set monkey" ], - "Predicates": [ + "predicates": [ "tblA.foo = 0", - "tblA.id = 1", + "tblA.id = ?", "tblB.bar = 0", - "tblB.id = 2" + "tblB.id = ?" ], - "Count": 2 + "count": 2 } ] diff --git a/go/transactions/transaction_signature.go b/go/transactions/transaction_signature.go index a23c29b..d5fdd52 100644 --- a/go/transactions/transaction_signature.go +++ b/go/transactions/transaction_signature.go @@ -22,6 +22,7 @@ import ( "fmt" "hash/fnv" "sort" + "strconv" "vitess.io/vitess/go/vt/sqlparser" ) @@ -46,7 +47,11 @@ type ( ) func (pi predicateInfo) String() string { - return fmt.Sprintf("%s.%s %s %d", pi.Table, pi.Col, pi.Op.ToString(), pi.Val) + val := strconv.Itoa(pi.Val) + if pi.Val == -1 { + val = "?" + } + return fmt.Sprintf("%s.%s %s %s", pi.Table, pi.Col, pi.Op.ToString(), val) } func (pi predicateInfo) compareTo(b predicateInfo) int { @@ -69,9 +74,9 @@ func (tx *TxSignature) MarshalJSON() ([]byte, error) { } return json.Marshal(struct { - Queries []string - Predicates []string - Count int + Queries []string `json:"query-signatures"` + Predicates []string `json:"predicates"` + Count int `json:"count"` }{ Queries: tx.Queries, Predicates: predicateStrings, @@ -164,13 +169,49 @@ func (tx *TxSignature) Equals(other *TxSignature) bool { return true } +// CleanUp removes values that are only used once and replaces them with -1 +func (tx *TxSignature) CleanUp() *TxSignature { + newPredicates := make([]predicateInfo, 0, len(tx.Predicates)) + usedValues := make(map[int]int) + + // First let's count how many times each value is used + for _, pred := range tx.Predicates { + usedValues[pred.Val]++ + } + + // Now we replace values only used once with -1 + newCount := 0 + newValues := make(map[int]int) + for _, pred := range tx.Predicates { + if usedValues[pred.Val] == 1 { + pred.Val = -1 + } else { + newVal, found := newValues[pred.Val] + if !found { + // Assign a new value to this predicate + newVal = newCount + newCount++ + newValues[pred.Val] = newVal + } + pred.Val = newVal + } + newPredicates = append(newPredicates, pred) + } + + return &TxSignature{ + Queries: tx.Queries, + Predicates: newPredicates, + Count: tx.Count, + } +} + func (m *txSignatureMap) MarshalJSON() ([]byte, error) { // Collect all interesting TxSignatures into a slice var signatures []*TxSignature for _, bucket := range m.data { for _, txSig := range bucket { if txSig.Count > 1 { - signatures = append(signatures, txSig) + signatures = append(signatures, txSig.CleanUp()) } } } diff --git a/go/transactions/transactions.go b/go/transactions/transactions.go index 8400b5b..de35cd8 100644 --- a/go/transactions/transactions.go +++ b/go/transactions/transactions.go @@ -223,21 +223,39 @@ func (s *state) consume(ch <-chan []sqlparser.Statement, wg *sync.WaitGroup) { } } -func (s *state) consumeUpdate(query *sqlparser.Update, st *semantics.SemTable, n *normalizer, tx *TxSignature) { - defer func() { - tx.Queries = append(tx.Queries, sqlparser.String(query)) - }() +func querySignatureForUpd(query *sqlparser.Update) string { + buffer := sqlparser.NewTrackedBuffer(nil) + builder := buffer.Builder + builder.WriteString("update ") + + for i, tbl := range query.TableExprs { + tbl.Format(buffer) + if i < len(query.TableExprs)-1 { + buffer.WriteString(", ") + } + } + + if query.Where != nil { + query.Where.Format(buffer) + } + + builder.WriteString(" set ") - // Normalize the AST our own way: - // - Replace the value in SET by "v" - // - Replace the literals found in where clause comparisons by the corresponding ID we got earlier for i, expr := range query.Exprs { - query.Exprs[i] = &sqlparser.UpdateExpr{ - Name: expr.Name, - Expr: sqlparser.NewArgument("v"), + expr.Name.Format(buffer) + if i < len(query.Exprs)-1 { + buffer.WriteString(", ") } } + return builder.String() +} + +func (s *state) consumeUpdate(query *sqlparser.Update, st *semantics.SemTable, n *normalizer, tx *TxSignature) { + defer func() { + tx.Queries = append(tx.Queries, querySignatureForUpd(query)) + }() + if query.Where == nil { return } From 860e8612676a1c6ba8998a8357eb810c541c5094 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 22 Nov 2024 16:27:51 +0100 Subject: [PATCH 17/22] text: Add README.md for vt transactions Signed-off-by: Andres Taylor --- README.md | 6 +++ go/transactions/README.md | 86 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 go/transactions/README.md diff --git a/README.md b/README.md index bcfec63..0ac8249 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ The `vt` binary encapsulates several utility tools for Vitess, providing a compr ## Tools Included - **`vt test`**: A testing utility using the same test files as the [MySQL Test Framework](https://github.com/mysql/mysql-server/tree/8.0/mysql-test). It compares the results of identical queries executed on both MySQL and Vitess (vtgate), helping to ensure compatibility. - **`vt keys`**: A utility that analyzes query logs and provides information about queries, tables, joins, and column usage. +- **`vt transactions`**: A tool that analyzes query logs to identify transaction patterns and outputs a JSON report detailing these patterns. - **`vt trace`**: A tool that generates execution traces for queries without comparing against MySQL. It helps analyze query behavior and performance in Vitess environments. - **`vt summarize`**: A tool used to summarize or compare trace logs or key logs for easier human consumption. @@ -116,6 +117,11 @@ This command generates a `keys-log.json` file that contains a detailed analysis This command summarizes the key analysis, providing insight into which tables and columns are used across queries, and how frequently they are involved in filters, groupings, and joins. [Here](https://github.com/vitessio/vt/blob/main/go/summarize/testdata/keys-summary.md) is an example summary report. +## Transaction Analysis with vt transactions +The `vt transactions` command is designed to analyze query logs and identify patterns of transactional queries. +It processes the logs to find sequences of queries that form transactions and outputs a JSON report summarizing these patterns. +Read more about how to use and how to read the output in the [vt transactions documentation](./go/transactions/README.md). + ## Using `--backup-path` Flag The `--backup-path` flag allows `vt test` and `vt trace` to initialize tests from a database backup rather than an empty database. diff --git a/go/transactions/README.md b/go/transactions/README.md new file mode 100644 index 0000000..c38a670 --- /dev/null +++ b/go/transactions/README.md @@ -0,0 +1,86 @@ +# VT Transactions + +The vt transactions command is a sub-command of the vt toolset, designed to analyze query logs, identify transaction patterns, and produce a JSON report summarizing these patterns. +This tool is particularly useful for understanding complex transaction behaviors, optimizing database performance, choosing sharding strategy, and auditing transactional queries. + +## Usage + +The basic usage of vt transactions is: + +```bash +vt transactions querylog.log > report.json +``` + + * querylog.log: The input query log file. This can be in various formats, such as SQL files, slow query logs, MySQL general query logs, or VTGate query logs. + * report.json: The output JSON file containing the transaction patterns. + +### Supported Input Types + +`vt transactions` supports different input file formats through the --input-type flag: + * Default: Assumes the input is an SQL file or a slow query log. A SQL script would also fall under this category. + * MySQL General Query Log: Use --input-type=mysql-log for MySQL general query logs. + * VTGate Query Log: Use --input-type=vtgate-log for VTGate query logs. + +## Understanding the JSON Output + +The output JSON file contains an array of transaction patterns, each summarizing a set of queries that commonly occur together within transactions. Here’s a snippet of the JSON output: + +```json +{ + "query-signatures": [ + "update pos_reports where id = :0 set `csv`, `error`, intraday, pos_type, ...", + "update pos_date_requests where cache_key = :1 set cache_value" + ], + "predicates": [ + "pos_date_requests.cache_key = ?", + "pos_reports.id = ?" + ], + "count": 223 +} +``` + +### Fields Explanation + + * query-signatures: An array of generalized query patterns involved in the transaction. Placeholders like :0, :1, etc., represent variables in the queries. + * predicates: An array of predicates (conditions) extracted from the queries, generalized to identify patterns. + * count: The number of times this transaction pattern was observed in the logs. + +### Understanding predicates + +The predicates array lists the conditions used in the transactional queries, with variables generalized for pattern recognition. + * Shared Variables: If the same variable is used across different predicates within a transaction, it is assigned a numerical placeholder (e.g., 0, 1, 2). This indicates that the same variable or value is used in these predicates. + * Unique Variables: Variables that are unique to a single predicate are represented with a ?. + +### Example Explained + +Consider the following predicates array: + +```json +{ + "predicates": [ + "timesheets.day = ?", + "timesheets.craft_id = ?", + "timesheets.store_id = ?", + "dailies.day = 0", + "dailies.craft_id = 1", + "dailies.store_id = 2", + "tickets.day = 0", + "tickets.craft_id = 1", + "tickets.store_id = 2" + ] +} +``` + + * Shared Values: Predicates with the same value across different conditions are assigned numerical placeholders (0, 1, 2), indicating that the same variable or value is used in these predicates. + * For example, `dailies.craft_id = 1` and `tickets.craft_id = 1` share the same variable or value (represented as 1). + * Unique Values: Predicates used only once are represented with ?, indicating a unique or less significant variable in the pattern. + * For example, `timesheets.day = ?` represents a unique value for day. + +This numbering helps identify the relationships between different predicates in the transaction patterns and can be used to optimize queries or understand transaction scopes. + +## Practical Use Cases + + * Optimization: Identify frequently occurring transactions to optimize database performance. + * Sharding Strategy: When implementing horizontal sharding, it’s crucial to ensure that as many transactions as possible are confined to a single shard. The insights from vt transactions can help in choosing appropriate sharding keys for your tables to achieve this. + * Audit: Analyze transactional patterns for security audits or compliance checks. + * Debugging: Understand complex transaction behaviors during development or troubleshooting. From 6b3c7b1f7f8937a1ba9d448b6467e51d6dfe8fc0 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Mon, 25 Nov 2024 10:24:36 +0100 Subject: [PATCH 18/22] make pretty Signed-off-by: Andres Taylor --- go/cmd/transactions.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/cmd/transactions.go b/go/cmd/transactions.go index 748409a..b3aa62d 100644 --- a/go/cmd/transactions.go +++ b/go/cmd/transactions.go @@ -18,6 +18,7 @@ package cmd import ( "github.com/spf13/cobra" + "github.com/vitessio/vt/go/transactions" ) From 2781aa848c1f0dab3f9551058c4f6b3114dba7d4 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Mon, 25 Nov 2024 10:35:33 +0100 Subject: [PATCH 19/22] feat: add file type to json output Signed-off-by: Andres Taylor --- go/summarize/utils.go | 8 +++-- .../small-slow-query-transactions.json | 33 ++++++++++--------- go/transactions/transaction_signature.go | 7 +++- go/transactions/transactions_test.go | 3 ++ 4 files changed, 32 insertions(+), 19 deletions(-) diff --git a/go/summarize/utils.go b/go/summarize/utils.go index 670c4b5..2420887 100644 --- a/go/summarize/utils.go +++ b/go/summarize/utils.go @@ -30,12 +30,14 @@ const ( traceFile keysFile dbInfoFile + transactionFile ) var fileTypeMap = map[string]fileType{ //nolint:gochecknoglobals // this is instead of a const - "trace": traceFile, - "keys": keysFile, - "dbinfo": dbInfoFile, + "trace": traceFile, + "keys": keysFile, + "dbinfo": dbInfoFile, + "transactions": transactionFile, } // getFileType reads the first key-value pair from a JSON file and returns the type of the file diff --git a/go/testdata/small-slow-query-transactions.json b/go/testdata/small-slow-query-transactions.json index bd3ca7e..ce2669e 100644 --- a/go/testdata/small-slow-query-transactions.json +++ b/go/testdata/small-slow-query-transactions.json @@ -1,15 +1,18 @@ -[ - { - "query-signatures": [ - "update tblA where foo = :0 and id = :1 set apa", - "update tblB where bar = :0 and id = :2 set monkey" - ], - "predicates": [ - "tblA.foo = 0", - "tblA.id = ?", - "tblB.bar = 0", - "tblB.id = ?" - ], - "count": 2 - } -] +{ + "fileType": "transactions", + "signatures": [ + { + "query-signatures": [ + "update tblA where foo = :0 and id = :1 set apa", + "update tblB where bar = :0 and id = :2 set monkey" + ], + "predicates": [ + "tblA.foo = 0", + "tblA.id = ?", + "tblB.bar = 0", + "tblB.id = ?" + ], + "count": 2 + } + ] +} diff --git a/go/transactions/transaction_signature.go b/go/transactions/transaction_signature.go index d5fdd52..3af1248 100644 --- a/go/transactions/transaction_signature.go +++ b/go/transactions/transaction_signature.go @@ -220,5 +220,10 @@ func (m *txSignatureMap) MarshalJSON() ([]byte, error) { return signatures[i].Count > signatures[j].Count }) - return json.Marshal(signatures) + result := map[string]any{ + "fileType": "transactions", + "signatures": signatures, + } + + return json.Marshal(result) } diff --git a/go/transactions/transactions_test.go b/go/transactions/transactions_test.go index a3ddadd..06205e1 100644 --- a/go/transactions/transactions_test.go +++ b/go/transactions/transactions_test.go @@ -45,4 +45,7 @@ func TestRun(t *testing.T) { require.NoError(t, err) assert.Equal(t, string(out), sb.String()) + if t.Failed() { + _ = os.WriteFile("../testdata/expected/small-slow-query-transactions.json", []byte(sb.String()), 0o644) + } } From 3d8bf2b76358342e49f014982d3d9f577ba98a04 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 26 Nov 2024 08:42:44 +0100 Subject: [PATCH 20/22] feat: update the json output from tx analysis Signed-off-by: Andres Taylor --- .../small-slow-query-transactions.json | 54 ++++-- go/transactions/transaction_signature.go | 165 +++++++++--------- go/transactions/transaction_signature_test.go | 92 ---------- go/transactions/transactions.go | 110 ++++-------- 4 files changed, 163 insertions(+), 258 deletions(-) delete mode 100644 go/transactions/transaction_signature_test.go diff --git a/go/testdata/small-slow-query-transactions.json b/go/testdata/small-slow-query-transactions.json index ce2669e..2061b06 100644 --- a/go/testdata/small-slow-query-transactions.json +++ b/go/testdata/small-slow-query-transactions.json @@ -2,17 +2,51 @@ "fileType": "transactions", "signatures": [ { + "count": 2, "query-signatures": [ - "update tblA where foo = :0 and id = :1 set apa", - "update tblB where bar = :0 and id = :2 set monkey" - ], - "predicates": [ - "tblA.foo = 0", - "tblA.id = ?", - "tblB.bar = 0", - "tblB.id = ?" - ], - "count": 2 + { + "op": "update", + "affected_table": "tblA", + "updated_columns": [ + "apa" + ], + "predicates": [ + { + "table": "tblA", + "col": "foo", + "op": 0, + "val": 0 + }, + { + "table": "tblA", + "col": "id", + "op": 0, + "val": -1 + } + ] + }, + { + "op": "update", + "affected_table": "tblB", + "updated_columns": [ + "monkey" + ], + "predicates": [ + { + "table": "tblB", + "col": "bar", + "op": 0, + "val": 0 + }, + { + "table": "tblB", + "col": "id", + "op": 0, + "val": -1 + } + ] + } + ] } ] } diff --git a/go/transactions/transaction_signature.go b/go/transactions/transaction_signature.go index 3af1248..0bfc75b 100644 --- a/go/transactions/transaction_signature.go +++ b/go/transactions/transaction_signature.go @@ -17,9 +17,9 @@ limitations under the License. package transactions import ( - "cmp" "encoding/json" "fmt" + "hash" "hash/fnv" "sort" "strconv" @@ -29,9 +29,15 @@ import ( type ( TxSignature struct { - Queries []string `json:"queries"` - Predicates []predicateInfo `json:"predicates"` - Count int `json:"count"` + Count int `json:"count"` + Queries []TxQuery `json:"qqueries"` + } + + TxQuery struct { + Op string `json:"op"` + AffectedTable string `json:"affected_table"` + UpdatedColumns []string `json:"updated_columns,omitempty"` + Predicates []predicateInfo `json:"predicates,omitempty"` } txSignatureMap struct { @@ -39,10 +45,10 @@ type ( } predicateInfo struct { - Table string - Col string - Op sqlparser.ComparisonExprOperator - Val int + Table string `json:"table"` + Col string `json:"col"` + Op sqlparser.ComparisonExprOperator `json:"op"` + Val int `json:"val"` } ) @@ -54,33 +60,13 @@ func (pi predicateInfo) String() string { return fmt.Sprintf("%s.%s %s %s", pi.Table, pi.Col, pi.Op.ToString(), val) } -func (pi predicateInfo) compareTo(b predicateInfo) int { - if pi.Table != b.Table { - return cmp.Compare(pi.Table, b.Table) - } - if pi.Col != b.Col { - return cmp.Compare(pi.Col, b.Col) - } - if pi.Op != b.Op { - return cmp.Compare(pi.Op, b.Op) - } - return cmp.Compare(pi.Val, b.Val) -} - func (tx *TxSignature) MarshalJSON() ([]byte, error) { - predicateStrings := make([]string, len(tx.Predicates)) - for i, predicate := range tx.Predicates { - predicateStrings[i] = predicate.String() - } - return json.Marshal(struct { - Queries []string `json:"query-signatures"` - Predicates []string `json:"predicates"` - Count int `json:"count"` + Count int `json:"count"` + Queries []TxQuery `json:"query-signatures"` }{ - Queries: tx.Queries, - Predicates: predicateStrings, - Count: tx.Count, + Count: tx.Count, + Queries: tx.Queries, }) } @@ -88,35 +74,55 @@ func (tx *TxSignature) Hash64() uint64 { hasher := fnv.New64a() for _, query := range tx.Queries { - _, _ = hasher.Write([]byte(query)) - _, _ = hasher.Write([]byte{0}) - } - - for _, pred := range tx.Predicates { - _, _ = hasher.Write([]byte(pred.String())) - _, _ = hasher.Write([]byte{0}) + query.addToHash(hasher) } return hasher.Sum64() } -func (tx *TxSignature) addPredicate(predicates []predicateInfo) { - for _, predicate := range predicates { - index := sort.Search(len(tx.Predicates), func(i int) bool { - return tx.Predicates[i].compareTo(predicate) >= 0 - }) +func (tx TxQuery) addToHash(hash hash.Hash64) { + _, _ = hash.Write([]byte(tx.Op)) + _, _ = hash.Write([]byte{0}) + _, _ = hash.Write([]byte(tx.AffectedTable)) + _, _ = hash.Write([]byte{0}) - if index < len(tx.Predicates) && tx.Predicates[index].compareTo(predicate) == 0 { - continue // Predicate already exists; skip it - } + for _, col := range tx.UpdatedColumns { + _, _ = hash.Write([]byte(col)) + _, _ = hash.Write([]byte{0}) + } - // Insert the predicate at the correct position - tx.Predicates = append(tx.Predicates, predicate) // Expand the slice by one - copy(tx.Predicates[index+1:], tx.Predicates[index:]) // Shift elements to the right - tx.Predicates[index] = predicate // Place the new predicate + for _, pred := range tx.Predicates { + _, _ = hash.Write([]byte(pred.String())) + _, _ = hash.Write([]byte{0}) } } +func (tx TxQuery) Equals(other TxQuery) bool { + if tx.Op != other.Op { + return false + } + if tx.AffectedTable != other.AffectedTable { + return false + } + if len(tx.UpdatedColumns) != len(other.UpdatedColumns) { + return false + } + for i := range tx.UpdatedColumns { + if tx.UpdatedColumns[i] != other.UpdatedColumns[i] { + return false + } + } + if len(tx.Predicates) != len(other.Predicates) { + return false + } + for i := range tx.Predicates { + if tx.Predicates[i] != other.Predicates[i] { + return false + } + } + return true +} + func newTxSignatureMap() *txSignatureMap { return &txSignatureMap{ data: make(map[uint64][]*TxSignature), @@ -152,16 +158,7 @@ func (tx *TxSignature) Equals(other *TxSignature) bool { return false } for i := range tx.Queries { - if tx.Queries[i] != other.Queries[i] { - return false - } - } - - if len(tx.Predicates) != len(other.Predicates) { - return false - } - for i := range tx.Predicates { - if tx.Predicates[i] != other.Predicates[i] { + if !tx.Queries[i].Equals(other.Queries[i]) { return false } } @@ -171,37 +168,47 @@ func (tx *TxSignature) Equals(other *TxSignature) bool { // CleanUp removes values that are only used once and replaces them with -1 func (tx *TxSignature) CleanUp() *TxSignature { - newPredicates := make([]predicateInfo, 0, len(tx.Predicates)) usedValues := make(map[int]int) // First let's count how many times each value is used - for _, pred := range tx.Predicates { - usedValues[pred.Val]++ + for _, query := range tx.Queries { + for _, predicate := range query.Predicates { + usedValues[predicate.Val]++ + } } // Now we replace values only used once with -1 newCount := 0 newValues := make(map[int]int) - for _, pred := range tx.Predicates { - if usedValues[pred.Val] == 1 { - pred.Val = -1 - } else { - newVal, found := newValues[pred.Val] - if !found { - // Assign a new value to this predicate - newVal = newCount - newCount++ - newValues[pred.Val] = newVal + newQueries := make([]TxQuery, 0, len(tx.Queries)) + for _, query := range tx.Queries { + newPredicates := make([]predicateInfo, 0, len(query.Predicates)) + for _, predicate := range query.Predicates { + if usedValues[predicate.Val] == 1 { + predicate.Val = -1 + } else { + newVal, found := newValues[predicate.Val] + if !found { + // Assign a new value to this predicate + newVal = newCount + newCount++ + newValues[predicate.Val] = newVal + } + predicate.Val = newVal } - pred.Val = newVal + newPredicates = append(newPredicates, predicate) } - newPredicates = append(newPredicates, pred) + newQueries = append(newQueries, TxQuery{ + Op: query.Op, + AffectedTable: query.AffectedTable, + UpdatedColumns: query.UpdatedColumns, + Predicates: newPredicates, + }) } return &TxSignature{ - Queries: tx.Queries, - Predicates: newPredicates, - Count: tx.Count, + Queries: newQueries, + Count: tx.Count, } } diff --git a/go/transactions/transaction_signature_test.go b/go/transactions/transaction_signature_test.go deleted file mode 100644 index fde26d5..0000000 --- a/go/transactions/transaction_signature_test.go +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright 2024 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package transactions - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "vitess.io/vitess/go/vt/sqlparser" -) - -func TestTxSignature_addPredicate(t *testing.T) { - tests := []struct { - name string - existing []predicateInfo - newOnes []predicateInfo - expectedResult []predicateInfo - }{ - { - name: "Add single predicate to empty list", - existing: []predicateInfo{}, - newOnes: []predicateInfo{ - {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, - }, - expectedResult: []predicateInfo{ - {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, - }, - }, - { - name: "Add one predicates, have one", - existing: []predicateInfo{ - {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, - }, - newOnes: []predicateInfo{ - {Table: "table", Col: "name", Op: sqlparser.LikeOp, Val: 2}, - }, - expectedResult: []predicateInfo{ - {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, - {Table: "table", Col: "name", Op: sqlparser.LikeOp, Val: 2}, - }, - }, - { - name: "Add one predicates, have one, reverse order", - existing: []predicateInfo{ - {Table: "table", Col: "name", Op: sqlparser.LikeOp, Val: 2}, - }, - newOnes: []predicateInfo{ - {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, - }, - expectedResult: []predicateInfo{ - {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, - {Table: "table", Col: "name", Op: sqlparser.LikeOp, Val: 2}, - }, - }, - { - name: "Add existing predicate", - existing: []predicateInfo{ - {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, - }, - newOnes: []predicateInfo{ - {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, - }, - expectedResult: []predicateInfo{ - {Table: "table", Col: "id", Op: sqlparser.EqualOp, Val: 1}, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tx := &TxSignature{ - Predicates: tt.existing, - } - tx.addPredicate(tt.newOnes) - assert.Equal(t, tt.expectedResult, tx.Predicates) - }) - } -} diff --git a/go/transactions/transactions.go b/go/transactions/transactions.go index de35cd8..b6469e2 100644 --- a/go/transactions/transactions.go +++ b/go/transactions/transactions.go @@ -21,7 +21,6 @@ import ( "fmt" "io" "os" - "strconv" "strings" "sync" @@ -223,92 +222,46 @@ func (s *state) consume(ch <-chan []sqlparser.Statement, wg *sync.WaitGroup) { } } -func querySignatureForUpd(query *sqlparser.Update) string { - buffer := sqlparser.NewTrackedBuffer(nil) - builder := buffer.Builder - builder.WriteString("update ") - - for i, tbl := range query.TableExprs { - tbl.Format(buffer) - if i < len(query.TableExprs)-1 { - buffer.WriteString(", ") - } - } - +func (s *state) consumeUpdate(query *sqlparser.Update, st *semantics.SemTable, n *normalizer, tx *TxSignature) { + // Find all predicates in the where clause that use a column and a literal + var predicates []predicateInfo if query.Where != nil { - query.Where.Format(buffer) + predicates = getPredicates(query.Where.Expr, st, n) } - builder.WriteString(" set ") - - for i, expr := range query.Exprs { - expr.Name.Format(buffer) - if i < len(query.Exprs)-1 { - buffer.WriteString(", ") - } + updatedColumns := make([]string, 0, len(query.Exprs)) + for _, expr := range query.Exprs { + updatedColumns = append(updatedColumns, sqlparser.String(expr.Name.Name)) } - return builder.String() -} - -func (s *state) consumeUpdate(query *sqlparser.Update, st *semantics.SemTable, n *normalizer, tx *TxSignature) { - defer func() { - tx.Queries = append(tx.Queries, querySignatureForUpd(query)) - }() - - if query.Where == nil { - return + if len(query.TableExprs) != 1 { + // TODO: Implement support for multi-table updates + panic("multi-table updates not supported") } - // Find all predicates in the where clause that use a column and a literal - tx.addPredicate(getPredicates(query.Where.Expr, st, n)) - query.Where = normalizeWhere(query.Where, n) + tx.Queries = append(tx.Queries, TxQuery{ + Op: "update", + AffectedTable: sqlparser.String(query.TableExprs[0]), + UpdatedColumns: updatedColumns, + Predicates: predicates, + }) } func (s *state) consumeDelete(del *sqlparser.Delete, st *semantics.SemTable, n *normalizer, tx *TxSignature) { - defer func() { - tx.Queries = append(tx.Queries, sqlparser.String(del)) - }() - - if del.Where == nil { - return + var predicates []predicateInfo + if del.Where != nil { + predicates = getPredicates(del.Where.Expr, st, n) } - - // Find all predicates in the where clause that use a column and a literal - tx.addPredicate(getPredicates(del.Where.Expr, st, n)) - del.Where = normalizeWhere(del.Where, n) -} - -func normalizeWhere(where *sqlparser.Where, n *normalizer) (newWhere *sqlparser.Where) { - newWhere = new(sqlparser.Where) - predicates := sqlparser.SplitAndExpression(nil, where.Expr) - for _, predicate := range predicates { - switch cmp := predicate.(type) { - case *sqlparser.ComparisonExpr: - lhs, lhsOK := cmp.Left.(*sqlparser.Literal) - rhs, rhsOK := cmp.Right.(*sqlparser.Literal) - if !lhsOK && !rhsOK || lhsOK && rhsOK { - newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr) - continue - } - - var newCmp sqlparser.ComparisonExpr - newCmp.Operator = cmp.Operator - if lhsOK { - id := n.normalize(lhs.Val) - newCmp.Left = sqlparser.NewArgument(strconv.Itoa(id)) - newCmp.Right = cmp.Right - } else { - id := n.normalize(rhs.Val) - newCmp.Right = sqlparser.NewArgument(strconv.Itoa(id)) - newCmp.Left = cmp.Left - } - newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr, &newCmp) - default: - newWhere.Expr = sqlparser.AndExpressions(newWhere.Expr, predicate) - } + if len(del.TableExprs) != 1 { + // TODO: Implement support for multi-table deletes + panic("multi-table updates not supported") } - return + + tx.Queries = append(tx.Queries, TxQuery{ + Op: "delete", + AffectedTable: sqlparser.String(del.TableExprs[0]), + Predicates: predicates, + }) } func (s *state) addSignature(tx *TxSignature) { @@ -324,10 +277,13 @@ func (s *state) run(out io.Writer, cfg Config) { loader := cfg.Loader.Load(cfg.FileName) ch := make(chan []sqlparser.Statement, 1000) + noOfConsumers := 1 var wg sync.WaitGroup - wg.Add(1) + for range noOfConsumers { + wg.Add(1) + go s.consume(ch, &wg) + } - go s.consume(ch, &wg) go func() { s.startProducing(loader, defaultAutocommit, ch) close(ch) From 136969f9f7529e0471000bf55bfe287a80ca393b Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 26 Nov 2024 09:07:14 +0100 Subject: [PATCH 21/22] update README Signed-off-by: Andres Taylor --- go/transactions/README.md | 138 +++++++++++++++++++++++++++----------- 1 file changed, 100 insertions(+), 38 deletions(-) diff --git a/go/transactions/README.md b/go/transactions/README.md index c38a670..953d592 100644 --- a/go/transactions/README.md +++ b/go/transactions/README.md @@ -1,11 +1,12 @@ # VT Transactions -The vt transactions command is a sub-command of the vt toolset, designed to analyze query logs, identify transaction patterns, and produce a JSON report summarizing these patterns. -This tool is particularly useful for understanding complex transaction behaviors, optimizing database performance, choosing sharding strategy, and auditing transactional queries. +The `vt transactions` command is a sub-command of the `vt` toolset, designed to analyze query logs, identify transaction patterns, and produce a JSON report summarizing these patterns. This tool is particularly useful for understanding complex transaction behaviors, optimizing database performance, choosing a sharding strategy, and auditing transactional queries. + +Note: The JSON output generated by `vt transactions` is primarily intended for consumption by the `vt summarize` tool, which can aggregate multiple analysis reports into a human-readable summary. ## Usage -The basic usage of vt transactions is: +The basic usage of `vt transactions` is: ```bash vt transactions querylog.log > report.json @@ -27,60 +28,121 @@ The output JSON file contains an array of transaction patterns, each summarizing ```json { - "query-signatures": [ - "update pos_reports where id = :0 set `csv`, `error`, intraday, pos_type, ...", - "update pos_date_requests where cache_key = :1 set cache_value" - ], - "predicates": [ - "pos_date_requests.cache_key = ?", - "pos_reports.id = ?" - ], - "count": 223 + "fileType": "transactions", + "signatures": [ + { + "count": 2, + "query-signatures": [ + { + "op": "update", + "affected_table": "tblA", + "updated_columns": [ + "apa" + ], + "predicates": [ + { + "table": "tblA", + "col": "foo", + "op": 0, + "val": 0 + }, + { + "table": "tblA", + "col": "id", + "op": 0, + "val": -1 + } + ] + }, + { + "op": "update", + "affected_table": "tblB", + "updated_columns": [ + "monkey" + ], + "predicates": [ + { + "table": "tblB", + "col": "bar", + "op": 0, + "val": 0 + }, + { + "table": "tblB", + "col": "id", + "op": 0, + "val": -1 + } + ] + } + ] + } + ] } ``` ### Fields Explanation - * query-signatures: An array of generalized query patterns involved in the transaction. Placeholders like :0, :1, etc., represent variables in the queries. - * predicates: An array of predicates (conditions) extracted from the queries, generalized to identify patterns. - * count: The number of times this transaction pattern was observed in the logs. +The JSON output from `vt transactions` is structured to represent patterns of transactions found in your query logs. Here’s a breakdown of each field: + +#### Top-Level Fields + + * fileType: Indicates the type of the file. For outputs from `vt transactions`, this will be "transactions". + * signatures: An array where each element represents a unique transaction pattern detected in the logs. + +#### Inside Each Signature + +Each element in the signatures array is an object that summarizes a specific transaction pattern. It contains the following fields: + * count: The number of times this transaction pattern was observed. + * query-signatures: An array of queries that are part of this transaction pattern. Each query is represented in a generalized form to abstract away specific values and focus on the structure and relationships. -### Understanding predicates +#### Inside Each Query Signature -The predicates array lists the conditions used in the transactional queries, with variables generalized for pattern recognition. - * Shared Variables: If the same variable is used across different predicates within a transaction, it is assigned a numerical placeholder (e.g., 0, 1, 2). This indicates that the same variable or value is used in these predicates. - * Unique Variables: Variables that are unique to a single predicate are represented with a ?. +Each object in the query-signatures array represents a generalized query and includes: + * op: The operation type (e.g., "insert", "update", "delete"). + * affected_table: The table affected by the query. + * updated_columns: (Only for update operations) An array of column names that are updated by the query. + * predicates: An array of conditions (also known as predicates) used in the query’s WHERE clause. Each predicate abstracts the condition to focus on the pattern rather than specific values. Not all predicates are included in the query signature; only those that could be used by the planner to select if the transaction is a single shard or a distributed transaction. + +#### Inside Each Predicate + +Each predicate object in the predicates array includes: + * table: The name of the table referenced in the condition. + * col: The column name used in the condition. + * op: A code representing the comparison operator used in the condition. For example: + - 0 might represent the "=" operator. + - Other numbers might represent different operators like <, >, LIKE, etc. + * val: A generalized placeholder value used in the condition. Instead of showing specific values, placeholders are used to indicate where values are compared. Identical placeholders across different predicates suggest that the same variable or parameter is used. -1 is a special value that indicates a unique value used only by this predicate. ### Example Explained Consider the following predicates array: ```json -{ - "predicates": [ - "timesheets.day = ?", - "timesheets.craft_id = ?", - "timesheets.store_id = ?", - "dailies.day = 0", - "dailies.craft_id = 1", - "dailies.store_id = 2", - "tickets.day = 0", - "tickets.craft_id = 1", - "tickets.store_id = 2" - ] -} +"predicates": [ + { + "table": "tblA", + "col": "foo", + "op": 0, + "val": 0 + }, + { + "table": "tblA", + "col": "id", + "op": 0, + "val": -1 + } +] ``` - * Shared Values: Predicates with the same value across different conditions are assigned numerical placeholders (0, 1, 2), indicating that the same variable or value is used in these predicates. - * For example, `dailies.craft_id = 1` and `tickets.craft_id = 1` share the same variable or value (represented as 1). - * Unique Values: Predicates used only once are represented with ?, indicating a unique or less significant variable in the pattern. - * For example, `timesheets.day = ?` represents a unique value for day. + * The first predicate represents a condition on tblA.foo, using the operator code 0 (e.g., "="), with a generalized value 0. + * The second predicate represents a condition on tblA.id, also using the operator code 0, with a generalized value -1. That means that this value was only used by this predicate and not shared by any other queries in the transaction. -This numbering helps identify the relationships between different predicates in the transaction patterns and can be used to optimize queries or understand transaction scopes. +This numbering helps identify the relationships between different predicates in the transaction patterns and can be used to help guide choices in sharding strategies. ## Practical Use Cases * Optimization: Identify frequently occurring transactions to optimize database performance. - * Sharding Strategy: When implementing horizontal sharding, it’s crucial to ensure that as many transactions as possible are confined to a single shard. The insights from vt transactions can help in choosing appropriate sharding keys for your tables to achieve this. + * Sharding Strategy: When implementing horizontal sharding, it’s crucial to ensure that as many transactions as possible are confined to a single shard. The insights from `vt transactions` can help in choosing appropriate sharding keys for your tables to achieve this. * Audit: Analyze transactional patterns for security audits or compliance checks. * Debugging: Understand complex transaction behaviors during development or troubleshooting. From f5fffc211d6402d05d4915a803e198d42af3ea03 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 26 Nov 2024 12:31:22 +0100 Subject: [PATCH 22/22] test: added tests for autocommit setting Signed-off-by: Andres Taylor --- go/transactions/transactions_test.go | 50 ++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/go/transactions/transactions_test.go b/go/transactions/transactions_test.go index 06205e1..6dfb573 100644 --- a/go/transactions/transactions_test.go +++ b/go/transactions/transactions_test.go @@ -49,3 +49,53 @@ func TestRun(t *testing.T) { _ = os.WriteFile("../testdata/expected/small-slow-query-transactions.json", []byte(sb.String()), 0o644) } } + +func TestAutocommitSettings(t *testing.T) { + tests := []struct { + query string + expect bool + }{ + { + query: "set autocommit=1", + expect: true, + }, { + query: "set autocommit=0", + expect: false, + }, { + query: "set autocommit=on", + expect: true, + }, { + query: "set autocommit=off", + expect: false, + }, { + query: "set session autocommit=on", + expect: true, + }, { + query: "set session autocommit=off", + expect: false, + }, { + query: "set @@session.autocommit=1", + expect: true, + }, { + query: "set @@session.autocommit=0", + expect: false, + }, { + query: "set global autocommit = 1", + expect: true, + }, { + query: "set global autocommit = 0", + expect: false, + }, + } + + parser := sqlparser.NewTestParser() + for _, test := range tests { + t.Run(test.query, func(t *testing.T) { + stmt, err := parser.Parse(test.query) + require.NoError(t, err) + set, ok := stmt.(*sqlparser.Set) + require.True(t, ok) + assert.Equal(t, test.expect, getAutocommitStatus(set, false)) + }) + } +}