From f5bd8d7d6c7cfe9f0048f496f4114e036f021a91 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Sat, 23 Nov 2024 12:17:17 +0100 Subject: [PATCH 01/11] cleanup: make sure to follow linter rules Signed-off-by: Andres Taylor --- .golangci.yml | 1 + go/tools/inserts/create_tpch_inserts.go | 53 ++++++++++++++++--------- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index bc78c94..3c31b04 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -131,6 +131,7 @@ linters-settings: gosec: excludes: - G115 + - G404 inamedparam: # Skips check for interface methods with only a single parameter. diff --git a/go/tools/inserts/create_tpch_inserts.go b/go/tools/inserts/create_tpch_inserts.go index 6147b40..6793fea 100644 --- a/go/tools/inserts/create_tpch_inserts.go +++ b/go/tools/inserts/create_tpch_inserts.go @@ -18,7 +18,7 @@ package main import ( "fmt" - "math/rand" + rand "math/rand/v2" "strings" ) @@ -31,12 +31,22 @@ func main() { fmt.Print("Enter size factor: ") _, _ = fmt.Scan(&sizeFactor) - // Helper function to join values - joinValues := func(values []string) string { - return strings.Join(values, ",\n") - } + // Generate inserts for all tables + generateRegionInserts(sizeFactor) + generateNationInserts(sizeFactor) + generateSupplierInserts(sizeFactor) + generatePartInserts(sizeFactor) + generatePartsuppInserts(sizeFactor) + generateCustomerInserts(sizeFactor) + generateOrderInserts(sizeFactor) + generateLineitemInserts(sizeFactor) +} + +func joinValues(values []string) string { + return strings.Join(values, ",\n") +} - // Generate inserts for the region table +func generateRegionInserts(sizeFactor int) { fmt.Println("") var regionValues []string regionValues = append(regionValues, "(1, 'ASIA', 'Eastern Asia')") @@ -46,8 +56,9 @@ func main() { regionValues = append(regionValues, fmt.Sprintf("(%d, 'Region %d', 'Comment %d')", i, i, i)) } fmt.Printf("INSERT INTO region (R_REGIONKEY, R_NAME, R_COMMENT) VALUES\n%s;\n\n", joinValues(regionValues)) +} - // Generate inserts for the nation table, include 'JAPAN', 'INDIA', 'EGYPT', and 'MOZAMBIQUE' for the test queries +func generateNationInserts(sizeFactor int) { fmt.Println() var nationValues []string nationValues = append(nationValues, "(1, 'JAPAN', 1, 'Nation with advanced technology')") @@ -59,8 +70,9 @@ func main() { nationValues = append(nationValues, fmt.Sprintf("(%d, 'Nation %d', %d, 'Nation Comment %d')", i, i, regionKey, i)) } fmt.Printf("INSERT INTO nation (N_NATIONKEY, N_NAME, N_REGIONKEY, N_COMMENT) VALUES\n%s;\n\n", joinValues(nationValues)) +} - // Generate inserts for the supplier table +func generateSupplierInserts(sizeFactor int) { fmt.Println() var supplierValues []string supplierValues = append(supplierValues, "(1, 'Supplier A', '123 Square', 1, '86-123-4567', 5000.00, 'High quality steel')") @@ -69,48 +81,53 @@ func main() { supplierValues = append(supplierValues, fmt.Sprintf("(%d, 'Supplier %d', 'Address %d', %d, 'Phone %d', %d, 'Supplier Comment %d')", i, i, i, nationKey, i, 5000+i*100, i)) } fmt.Printf("INSERT INTO supplier (S_SUPPKEY, S_NAME, S_ADDRESS, S_NATIONKEY, S_PHONE, S_ACCTBAL, S_COMMENT) VALUES\n%s;\n\n", joinValues(supplierValues)) +} - // Generate inserts for the part table +func generatePartInserts(sizeFactor int) { fmt.Println() var partValues []string partValues = append(partValues, "(1, 'Part dimension', 'MFGR A', 'Brand#52', 'SMALL PLATED COPPER', 3, 'SM BOX', 45.00, 'Part with special dimensions')") partValues = append(partValues, "(2, 'Large Brush', 'MFGR B', 'Brand#34', 'LARGE BRUSHED COPPER', 12, 'LG BOX', 30.00, 'Brush for industrial use')") for i := 3; i <= sizeFactor*5; i++ { - partValues = append(partValues, fmt.Sprintf("(%d, 'Part %d', 'MFGR %d', 'Brand %d', 'Type %d', %d, 'Container %d', %.2f, 'Part Comment %d')", i, i, i, i, i, rand.Intn(100), i, float64(10+i*10), i)) + partValues = append(partValues, fmt.Sprintf("(%d, 'Part %d', 'MFGR %d', 'Brand %d', 'Type %d', %d, 'Container %d', %.2f, 'Part Comment %d')", i, i, i, i, i, rand.Int64N(100), i, float64(10+i*10), i)) } fmt.Printf("INSERT INTO part (P_PARTKEY, P_NAME, P_MFGR, P_BRAND, P_TYPE, P_SIZE, P_CONTAINER, P_RETAILPRICE, P_COMMENT) VALUES\n%s;\n\n", joinValues(partValues)) +} - // Generate inserts for the partsupp table +func generatePartsuppInserts(sizeFactor int) { fmt.Println() var partsuppValues []string for i := 1; i <= sizeFactor*10; i++ { partKey := (i-1)%5 + 1 suppKey := (i-1)%7 + 1 - partsuppValues = append(partsuppValues, fmt.Sprintf("(%d, %d, %d, %.2f, 'Partsupp Comment %d')", partKey, suppKey, rand.Intn(1000), float64(rand.Intn(2000))/100, i)) + partsuppValues = append(partsuppValues, fmt.Sprintf("(%d, %d, %d, %.2f, 'Partsupp Comment %d')", partKey, suppKey, rand.Int64N(1000), float64(rand.Int64N(2000))/100, i)) } fmt.Printf("INSERT INTO partsupp (PS_PARTKEY, PS_SUPPKEY, PS_AVAILQTY, PS_SUPPLYCOST, PS_COMMENT) VALUES\n%s;\n\n", joinValues(partsuppValues)) +} - // Generate inserts for the customer table, include 'AUTOMOBILE' segment for test queries +func generateCustomerInserts(sizeFactor int) { fmt.Println() var customerValues []string customerValues = append(customerValues, "(1, 'Customer A', '1234 Drive Lane', 1, '123-456-7890', 1000.00, 'AUTOMOBILE', 'Frequent automobile orders')") for i := 2; i <= sizeFactor*5; i++ { nationKey := (i-1)%12 + 1 - customerValues = append(customerValues, fmt.Sprintf("(%d, 'Customer %d', 'Address %d', %d, 'Phone %d', %.2f, 'Segment %d', 'Customer Comment %d')", i, i, i, nationKey, i, float64(rand.Intn(20000))/100, i%5, i)) + customerValues = append(customerValues, fmt.Sprintf("(%d, 'Customer %d', 'Address %d', %d, 'Phone %d', %.2f, 'Segment %d', 'Customer Comment %d')", i, i, i, nationKey, i, float64(rand.Int64N(20000))/100, i%5, i)) } fmt.Printf("INSERT INTO customer (C_CUSTKEY, C_NAME, C_ADDRESS, C_NATIONKEY, C_PHONE, C_ACCTBAL, C_MKTSEGMENT, C_COMMENT) VALUES\n%s;\n\n", joinValues(customerValues)) +} - // Generate inserts for the orders table +func generateOrderInserts(sizeFactor int) { fmt.Println() var orderValues []string orderValues = append(orderValues, "(1, 1, 'O', 15000.00, '1995-03-12', '1-URGENT', 'Clerk#0001', 1, 'Automobile related order')") for i := 2; i <= sizeFactor*5; i++ { custKey := (i-1)%5 + 1 - orderValues = append(orderValues, fmt.Sprintf("(%d, %d, 'O', %.2f, '1995-02-%02d', 'Priority %d', 'Clerk#%04d', %d, 'Order Comment %d')", i, custKey, float64(rand.Intn(50000)), rand.Intn(28)+1, rand.Intn(5)+1, i, rand.Intn(3)+1, i)) + orderValues = append(orderValues, fmt.Sprintf("(%d, %d, 'O', %.2f, '1995-02-%02d', 'Priority %d', 'Clerk#%04d', %d, 'Order Comment %d')", i, custKey, float64(rand.Int64N(50000)), rand.Int64N(28)+1, rand.Int64N(5)+1, i, rand.Int64N(3)+1, i)) } fmt.Printf("INSERT INTO orders (O_ORDERKEY, O_CUSTKEY, O_ORDERSTATUS, O_TOTALPRICE, O_ORDERDATE, O_ORDERPRIORITY, O_CLERK, O_SHIPPRIORITY, O_COMMENT) VALUES\n%s;\n\n", joinValues(orderValues)) +} - // Generate inserts for the lineitem table +func generateLineitemInserts(sizeFactor int) { fmt.Println() var lineitemValues []string lineitemValues = append(lineitemValues, "(1, 1, 1, 1, 20, 5000.00, 0.05, 0.10, 'R', 'O', '1995-03-15', '1995-03-14', '1995-03-16', 'DELIVER IN PERSON', 'AIR', 'Handle with care')") @@ -119,7 +136,7 @@ func main() { orderKey := (i-1)%5 + 1 partKey := (i-1)%5 + 1 suppKey := (i-1)%7 + 1 - lineitemValues = append(lineitemValues, fmt.Sprintf("(%d, %d, %d, %d, %d, %.2f, %.2f, %.2f, 'N', 'O', '1995-03-%02d', '1995-03-%02d', '1995-03-%02d', 'DELIVER IN PERSON', 'TRUCK', 'Lineitem Comment %d')", orderKey, partKey, suppKey, i, rand.Intn(100), float64(rand.Intn(20000)), float64(rand.Intn(20))/100, float64(rand.Intn(10))/100, rand.Intn(28)+1, rand.Intn(28)+2, rand.Intn(28)+3, i)) + lineitemValues = append(lineitemValues, fmt.Sprintf("(%d, %d, %d, %d, %d, %.2f, %.2f, %.2f, 'N', 'O', '1995-03-%02d', '1995-03-%02d', '1995-03-%02d', 'DELIVER IN PERSON', 'TRUCK', 'Lineitem Comment %d')", orderKey, partKey, suppKey, i, rand.Int64N(100), float64(rand.Int64N(20000)), float64(rand.Int64N(20))/100, float64(rand.Int64N(10))/100, rand.Int64N(28)+1, rand.Int64N(28)+2, rand.Int64N(28)+3, i)) } fmt.Printf("INSERT INTO lineitem (L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER, L_QUANTITY, L_EXTENDEDPRICE, L_DISCOUNT, L_TAX, L_RETURNFLAG, L_LINESTATUS, L_SHIPDATE, L_COMMITDATE, L_RECEIPTDATE, L_SHIPINSTRUCT, L_SHIPMODE, L_COMMENT) VALUES\n%s;\n\n", joinValues(lineitemValues)) } From fc165e2bc65ffbc6e132a9743c277ceb93cc9b01 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Sat, 23 Nov 2024 12:26:29 +0100 Subject: [PATCH 02/11] ci: make sure to run the linter Signed-off-by: Andres Taylor --- .github/workflows/ci.yml | 48 +++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c5b84a..e9f06d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,39 +7,65 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Checkout code + uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - name: Set up Go + uses: actions/setup-go@v5 with: go-version-file: './go.mod' - - run: go version + - name: Cache Go modules + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Install Dependencies + run: go mod download - - name: Go mod tidy + - name: Verify go.mod and go.sum run: | go mod tidy git diff --exit-code || (echo "go.mod or go.sum is not clean! Run 'go mod tidy' and commit the changes." && exit 1) - - name: Go Vet - run: go vet ./go/... + - name: Install golangci-lint + run: | + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | \ + sh -s -- -b $(go env GOPATH)/bin v1.54.2 + + - name: Install gofumpt + run: go install mvdan.cc/gofumpt@v0.5.0 + + - name: Install goimports-reviser + run: go install github.com/incu6us/goimports-reviser/v3@v3.3.0 + + - name: Display Go version + run: go version - name: Build run: make build - - name: Check formatting + - name: Check Formatting run: | make pretty git diff --exit-code || (echo "Code is not formatted correctly! Run 'make pretty' and commit the changes." && exit 1) + - name: Run golangci-lint + run: golangci-lint run ./... + - name: Install go-junit-report run: go install github.com/jstemmer/go-junit-report@latest - name: Run Tests and Convert to JUnit - run: go test -v ./go/... | go-junit-report > report.xml + run: go test -v ./... | go-junit-report > report.xml - name: Test Summary - if: true - uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 + uses: test-summary/action@v2.4 with: paths: "report.xml" - show: "fail" + show: "fail" \ No newline at end of file From c8a4375b5604c9578b5f1cada0754cf996f4d077 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Sat, 23 Nov 2024 12:31:34 +0100 Subject: [PATCH 03/11] refactor: make make lint happy Signed-off-by: Andres Taylor --- go/summarize/utils.go | 51 +++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/go/summarize/utils.go b/go/summarize/utils.go index 03d6e35..ae74095 100644 --- a/go/summarize/utils.go +++ b/go/summarize/utils.go @@ -32,7 +32,7 @@ const ( dbInfoFile ) -var fileTypeMap = map[string]fileType{ +var fileTypeMap = map[string]fileType{ //nolint:gochecknoglobals // this is instead of a const "trace": traceFile, "keys": keysFile, "dbinfo": dbInfoFile, @@ -43,7 +43,7 @@ var fileTypeMap = map[string]fileType{ func getFileType(filename string) (fileType, error) { file, err := os.Open(filename) if err != nil { - return unknownFile, errors.New(fmt.Sprintf("error opening file: %v", err)) + return unknownFile, fmt.Errorf("error opening file: %v", err) } defer file.Close() @@ -51,39 +51,38 @@ func getFileType(filename string) (fileType, error) { token, err := decoder.Token() if err != nil { - return unknownFile, errors.New(fmt.Sprintf("Error reading token: %v", err)) + return unknownFile, fmt.Errorf("error reading token: %v", err) } if delim, ok := token.(json.Delim); !ok || delim != '{' { - return unknownFile, errors.New(fmt.Sprintf("Expected start of object '{'")) + return unknownFile, errors.New("expected start of object '{'") } - for decoder.More() { - keyToken, err := decoder.Token() - if err != nil { - return unknownFile, errors.New(fmt.Sprintf("Error reading key token: %v", err)) - } + if !decoder.More() { + return unknownFile, nil + } - key, ok := keyToken.(string) - if !ok { - return unknownFile, errors.New(fmt.Sprintf("Expected key to be a string: %s", keyToken)) - } + keyToken, err := decoder.Token() + if err != nil { + return unknownFile, fmt.Errorf("error reading key token: %v", err) + } - valueToken, err := decoder.Token() - if err != nil { - return unknownFile, errors.New(fmt.Sprintf("Error reading value token: %v", err)) - } + key, ok := keyToken.(string) + if !ok { + return unknownFile, fmt.Errorf("expected key to be a string: %s", keyToken) + } - if key == "fileType" { - if fileType, ok := fileTypeMap[valueToken.(string)]; ok { - return fileType, nil - } else { - return unknownFile, errors.New(fmt.Sprintf("Unknown FileType: %s", valueToken)) - } - } else { - // Currently we expect the first key to be FileType, for optimization reasons - return unknownFile, nil + valueToken, err := decoder.Token() + if err != nil { + return unknownFile, fmt.Errorf("error reading value token: %v", err) + } + + if key == "fileType" { + if fileType, ok := fileTypeMap[valueToken.(string)]; ok { + return fileType, nil } + return unknownFile, fmt.Errorf("unknown FileType: %s", valueToken) } + return unknownFile, nil } From ddd72c4422496643d392bfabae8116ded5bb315c Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Sat, 23 Nov 2024 12:35:06 +0100 Subject: [PATCH 04/11] ci: use make lint in CI to make sure we get consistent results Signed-off-by: Andres Taylor --- .github/workflows/ci.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9f06d2..830226a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,17 +33,14 @@ jobs: go mod tidy git diff --exit-code || (echo "go.mod or go.sum is not clean! Run 'go mod tidy' and commit the changes." && exit 1) - - name: Install golangci-lint + - name: Install Tools run: | + go install mvdan.cc/gofumpt@v0.5.0 + go install github.com/incu6us/goimports-reviser/v3@v3.3.0 + go install github.com/jstemmer/go-junit-report@latest curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | \ sh -s -- -b $(go env GOPATH)/bin v1.54.2 - - name: Install gofumpt - run: go install mvdan.cc/gofumpt@v0.5.0 - - - name: Install goimports-reviser - run: go install github.com/incu6us/goimports-reviser/v3@v3.3.0 - - name: Display Go version run: go version @@ -55,11 +52,8 @@ jobs: make pretty git diff --exit-code || (echo "Code is not formatted correctly! Run 'make pretty' and commit the changes." && exit 1) - - name: Run golangci-lint - run: golangci-lint run ./... - - - name: Install go-junit-report - run: go install github.com/jstemmer/go-junit-report@latest + - name: Run Linting + run: make lint - name: Run Tests and Convert to JUnit run: go test -v ./... | go-junit-report > report.xml From 98836f7255b8d533ceeae442d4e0bcd5a180cf5d Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Mon, 25 Nov 2024 07:24:52 +0100 Subject: [PATCH 05/11] ci: upgrade linter version and align CI and Makefile Signed-off-by: Andres Taylor --- .github/workflows/ci.yml | 6 +----- Makefile | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 830226a..a5bd057 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,11 +35,7 @@ jobs: - name: Install Tools run: | - go install mvdan.cc/gofumpt@v0.5.0 - go install github.com/incu6us/goimports-reviser/v3@v3.3.0 - go install github.com/jstemmer/go-junit-report@latest - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | \ - sh -s -- -b $(go env GOPATH)/bin v1.54.2 + make install-tools - name: Display Go version run: go version diff --git a/Makefile b/Makefile index bc0b686..edf222a 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ GO := go REQUIRED_GO_VERSION := 1.23 -GOLANGCI_LINT_VERSION := v1.55.2 +GOLANGCI_LINT_VERSION := v1.62.0 # Version check check_version: From e3ab388eda8867e82613fc9680badd05c4743834 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Mon, 25 Nov 2024 07:45:03 +0100 Subject: [PATCH 06/11] Clean up code to follow linter rules --- .gitignore | 4 +++- go/data/vtgate_log_parse.go | 15 ++++++++++++--- go/summarize/reading.go | 6 +++++- go/summarize/summarize-keys.go | 23 +++++++++++++++-------- go/summarize/summarize-keys_test.go | 2 +- go/summarize/utils.go | 6 +++++- 6 files changed, 41 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 64d8c48..8bd809a 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,6 @@ errors/ /vt # Do not ignore anything inside the src/vitess-tester directory -!/go \ No newline at end of file +!/go + +go/testdata/expected/ diff --git a/go/data/vtgate_log_parse.go b/go/data/vtgate_log_parse.go index efd354d..92f2881 100644 --- a/go/data/vtgate_log_parse.go +++ b/go/data/vtgate_log_parse.go @@ -167,13 +167,22 @@ func getBindVariables(bindVarsRaw string, lineNumber int) (map[string]*querypb.B var val []byte switch { case sqltypes.IsIntegral(bvType) || sqltypes.IsFloat(bvType): - val = []byte(strconv.FormatFloat(value.Value.(float64), 'f', -1, 64)) + f, ok := value.Value.(float64) + if ok { + val = []byte(strconv.FormatFloat(f, 'f', -1, 64)) + } case bvType == sqltypes.Tuple: // the query log of vtgate does not list all the values for a tuple // instead it lists the following: "v2": {"type": "TUPLE", "value": "2 items"} return nil, fmt.Errorf("line %d: cannot parse tuple bind variables", lineNumber) - default: - val = []byte(value.Value.(string)) + } + if val == nil { + sval, ok := value.Value.(string) + if !ok { + return nil, fmt.Errorf("line %d: cannot parse bind variable value", lineNumber) + } + + val = []byte(sval) } bvProcessed[key] = &querypb.BindVariable{ Type: bvType, diff --git a/go/summarize/reading.go b/go/summarize/reading.go index b8fb0f0..1afa784 100644 --- a/go/summarize/reading.go +++ b/go/summarize/reading.go @@ -58,6 +58,10 @@ func getDecoderAndDelim(file *os.File) (*json.Decoder, json.Delim) { if err != nil { exit("Error reading json: " + err.Error()) } + delim, ok := val.(json.Delim) + if !ok { + exit("Error reading json: expected delimiter") + } // Reset the file pointer to the beginning _, err = file.Seek(0, io.SeekStart) @@ -65,7 +69,7 @@ func getDecoderAndDelim(file *os.File) (*json.Decoder, json.Delim) { exit("Error rewinding file: " + err.Error()) } decoder = json.NewDecoder(file) - return decoder, val.(json.Delim) + return decoder, delim } func readTracedQueryFile(decoder *json.Decoder, fileName string) readingSummary { diff --git a/go/summarize/summarize-keys.go b/go/summarize/summarize-keys.go index f1e807f..556054c 100644 --- a/go/summarize/summarize-keys.go +++ b/go/summarize/summarize-keys.go @@ -177,7 +177,7 @@ func printKeysSummary(out io.Writer, file readingSummary, now time.Time, hotMetr summary := summarizeKeysQueries(file.AnalysedQueries, metricReader, schemaInfo) renderHotQueries(md, summary.hotQueries, metricReader) - renderTableUsage(summary.tables, md) + renderTableUsage(summary.tables, md, schemaInfo != nil) renderTablesJoined(md, file.AnalysedQueries) renderFailures(md, summary.failures) @@ -268,7 +268,7 @@ func renderHotQueries(md *markdown.MarkDown, queries []keys.QueryAnalysisResult, } } -func renderTableUsage(tableSummaries []TableSummary, md *markdown.MarkDown) { +func renderTableUsage(tableSummaries []TableSummary, md *markdown.MarkDown, includeRowCount bool) { if len(tableSummaries) == 0 { return } @@ -278,7 +278,7 @@ func renderTableUsage(tableSummaries []TableSummary, md *markdown.MarkDown) { }) md.PrintHeader("Tables", 2) - renderTableOverview(md, tableSummaries) + renderTableOverview(md, tableSummaries, includeRowCount) md.PrintHeader("Column Usage", 3) for _, summary := range tableSummaries { @@ -286,16 +286,23 @@ func renderTableUsage(tableSummaries []TableSummary, md *markdown.MarkDown) { } } -func renderTableOverview(md *markdown.MarkDown, tableSummaries []TableSummary) { - headers := []string{"Table Name", "Reads", "Writes", "Number of Rows"} +func renderTableOverview(md *markdown.MarkDown, tableSummaries []TableSummary, includeRowCount bool) { + headers := []string{"Table Name", "Reads", "Writes"} + if includeRowCount { + headers = append(headers, "Number of Rows") + } var rows [][]string for _, summary := range tableSummaries { - rows = append(rows, []string{ + thisRow := []string{ summary.Table, strconv.Itoa(summary.ReadQueryCount), strconv.Itoa(summary.WriteQueryCount), - strconv.Itoa(summary.RowCount), - }) + } + if includeRowCount { + thisRow = append(thisRow, strconv.Itoa(summary.RowCount)) + } + + rows = append(rows, thisRow) } md.PrintTable(headers, rows) } diff --git a/go/summarize/summarize-keys_test.go b/go/summarize/summarize-keys_test.go index 03f50f2..d6b87a0 100644 --- a/go/summarize/summarize-keys_test.go +++ b/go/summarize/summarize-keys_test.go @@ -96,7 +96,7 @@ func TestSummarizeKeysWithHotnessFile(t *testing.T) { require.NoError(t, err) sb := &strings.Builder{} now := time.Date(2024, time.January, 1, 1, 2, 3, 0, time.UTC) - printKeysSummary(sb, file, now, metric, "../testdata/bigger_slow_query_schema_info.json") + printKeysSummary(sb, file, now, metric, "") expected, err := os.ReadFile(fmt.Sprintf("../testdata/bigger_slow_log_%s.md", metric)) require.NoError(t, err) assert.Equal(t, string(expected), sb.String()) diff --git a/go/summarize/utils.go b/go/summarize/utils.go index ae74095..670c4b5 100644 --- a/go/summarize/utils.go +++ b/go/summarize/utils.go @@ -78,7 +78,11 @@ func getFileType(filename string) (fileType, error) { } if key == "fileType" { - if fileType, ok := fileTypeMap[valueToken.(string)]; ok { + s, ok := valueToken.(string) + if !ok { + return unknownFile, fmt.Errorf("expected value to be a string: %s", valueToken) + } + if fileType, ok := fileTypeMap[s]; ok { return fileType, nil } return unknownFile, fmt.Errorf("unknown FileType: %s", valueToken) From db87c9a07d7112d41f9103aa8e5353aef2907bfd Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Mon, 25 Nov 2024 08:06:12 +0100 Subject: [PATCH 07/11] test: update expectation Signed-off-by: Andres Taylor --- go/summarize/utils_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/summarize/utils_test.go b/go/summarize/utils_test.go index 81209d0..f2c2d09 100644 --- a/go/summarize/utils_test.go +++ b/go/summarize/utils_test.go @@ -43,7 +43,7 @@ func TestGetFileType(t *testing.T) { { filename: "../testdata/mysql.query.log", expectedType: unknownFile, - expectedError: "Error reading token", + expectedError: "error reading token: invalid character '/' looking for beginning of value", }, } for _, tc := range testCases { From 03ebcbc37b0b611a8115bb1e585d647b4adbf2fb Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Mon, 25 Nov 2024 08:22:37 +0100 Subject: [PATCH 08/11] update makefile to check and install tools separately Signed-off-by: Andres Taylor --- Makefile | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index edf222a..a31ac2c 100644 --- a/Makefile +++ b/Makefile @@ -32,28 +32,33 @@ clean: rm -f vt # Pretty: formats the code using gofumpt and goimports-reviser -pretty: install-tools +pretty: check-tools @echo "Running formatting tools..." @gofumpt -l -w . >/dev/null 2>&1 || true @goimports-reviser -project-name $$(go list -m) -rm-unused -set-alias -format . >/dev/null 2>&1 || true -# Install tools: Checks if the required tools are installed, installs if missing +# Tools installation command install-tools: - @command -v gofumpt >/dev/null 2>&1 || { \ - echo "Installing gofumpt..."; \ - go install mvdan.cc/gofumpt@latest >/dev/null 2>&1; \ - } - @command -v goimports-reviser >/dev/null 2>&1 || { \ - echo "Installing goimports-reviser..."; \ - go install github.com/incu6us/goimports-reviser@latest >/dev/null 2>&1; \ - } - @command -v golangci-lint >/dev/null 2>&1 || { \ - echo "Installing golangci-lint..."; \ - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin $(GOLANGCI_LINT_VERSION); \ - } + @echo "Installing gofumpt..." + go install mvdan.cc/gofumpt@latest + + @echo "Installing goimports-reviser..." + go install github.com/incu6us/goimports-reviser/v3@latest + + @echo "Installing golangci-lint..." + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin $(GOLANGCI_LINT_VERSION) + + @echo "All tools installed successfully." + +# Ensure tools are available +check-tools: + @command -v gofumpt >/dev/null 2>&1 || { echo "gofumpt not found. Run 'make install-tools' to install it." >&2; exit 1; } + @command -v goimports-reviser >/dev/null 2>&1 || { echo "goimports-reviser not found. Run 'make install-tools' to install it." >&2; exit 1; } + @command -v golangci-lint >/dev/null 2>&1 || { echo "golangci_lint not found. Run 'make install-tools' to install it." >&2; exit 1; } + # Lint: runs golangci-lint -lint: install-tools +lint: check-tools @echo "Running golangci-lint..." @golangci-lint run --config .golangci.yml ./go/... From 4414d0a8269435e1554c8c5c63f7ba880e522e7b Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Mon, 25 Nov 2024 08:37:58 +0100 Subject: [PATCH 09/11] find GOPATH and use it Signed-off-by: Andres Taylor --- Makefile | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index a31ac2c..40bfdf6 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,20 @@ .PHONY: all build test tidy clean pretty install-tools lint install-hooks +.DEFAULT_GOAL := test_and_build -GO := go REQUIRED_GO_VERSION := 1.23 GOLANGCI_LINT_VERSION := v1.62.0 +# Determine the Go binary directory +GOBIN_DIR := $(or $(GOBIN), $(shell go env GOBIN)) +ifeq ($(GOBIN_DIR),) + GOBIN_DIR := $(shell go env GOPATH)/bin +endif + +test_and_build: test build + # Version check check_version: - @GO_VERSION=$$($(GO) version | awk '{print $$3}' | sed 's/go//'); \ + @GO_VERSION=$$(go version | awk '{print $$3}' | sed 's/go//'); \ MAJOR_VERSION=$$(echo $$GO_VERSION | cut -d. -f1); \ MINOR_VERSION=$$(echo $$GO_VERSION | cut -d. -f2); \ if [ "$$MAJOR_VERSION" -eq 1 ] && [ "$$MINOR_VERSION" -lt 23 ]; then \ @@ -19,16 +27,16 @@ check_version: default: check_version build build: - $(GO) build -o vt ./go/vt + go build -o vt ./go/vt test: - $(GO) test -count=1 ./go/... + go test -count=1 ./go/... tidy: - $(GO) mod tidy + go mod tidy clean: - $(GO) clean -i ./... + go clean -i ./... rm -f vt # Pretty: formats the code using gofumpt and goimports-reviser @@ -46,7 +54,8 @@ install-tools: go install github.com/incu6us/goimports-reviser/v3@latest @echo "Installing golangci-lint..." - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin $(GOLANGCI_LINT_VERSION) + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | \ + sh -s -- -b $(GOBIN_DIR) $(GOLANGCI_LINT_VERSION) @echo "All tools installed successfully." From c0e528aa727750a0b7554d18945a19e203ff010e Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Mon, 25 Nov 2024 08:42:01 +0100 Subject: [PATCH 10/11] restore junit-report installation Signed-off-by: Andres Taylor --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5bd057..ce26d0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,9 @@ jobs: - name: Run Linting run: make lint + - name: Install go-junit-report + run: go install github.com/jstemmer/go-junit-report@latest + - name: Run Tests and Convert to JUnit run: go test -v ./... | go-junit-report > report.xml From 8f059b7bafcb5024019855d474fbc789189ddf69 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Mon, 25 Nov 2024 10:01:44 +0100 Subject: [PATCH 11/11] make make pretty actually fix imports Signed-off-by: Andres Taylor --- Makefile | 4 ++-- git-hooks/pre-commit | 8 ++++---- go/summarize/utils_test.go | 1 - 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 40bfdf6..7562589 100644 --- a/Makefile +++ b/Makefile @@ -42,8 +42,8 @@ clean: # Pretty: formats the code using gofumpt and goimports-reviser pretty: check-tools @echo "Running formatting tools..." - @gofumpt -l -w . >/dev/null 2>&1 || true - @goimports-reviser -project-name $$(go list -m) -rm-unused -set-alias -format . >/dev/null 2>&1 || true + @gofumpt -w . >/dev/null 2>&1 + @goimports-reviser -recursive -project-name $$(go list -m) -rm-unused -set-alias ./go >/dev/null 2>&1 # Tools installation command install-tools: diff --git a/git-hooks/pre-commit b/git-hooks/pre-commit index 4c2b37d..83f8e1f 100755 --- a/git-hooks/pre-commit +++ b/git-hooks/pre-commit @@ -18,12 +18,12 @@ for file in $GO_FILES; do } done -# 2. Run goimports-reviser to verify formatting on changed files (without writing changes) -echo "Checking goimports-reviser formatting..." +# Check imports with goimports-reviser +echo "Checking goimports-reviser imports..." for file in $GO_FILES; do - goimports-reviser -project-name $(go list -m) -rm-unused -set-alias -format "$file" -diff >/dev/null 2>&1 + goimports-reviser -project-name $(go list -m) -rm-unused -set-alias "$file" >/dev/null 2>&1 if [[ $? -ne 0 ]]; then - echo "Error: Imports not correctly formatted in $file. Please run 'make pretty'." + echo "Error: Imports not correctly organized in $file. Please run 'make pretty'." exit 1 fi done diff --git a/go/summarize/utils_test.go b/go/summarize/utils_test.go index f2c2d09..5547fce 100644 --- a/go/summarize/utils_test.go +++ b/go/summarize/utils_test.go @@ -20,7 +20,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" )