Skip to content

Commit

Permalink
lightning: fix data error summary table format
Browse files Browse the repository at this point in the history 
Signed-off-by: lance6716 <[email protected]>
  • Loading branch information
@lance6716
lance6716 committed Dec 9, 2024
1 parent 28585c8 commit c5a9b10
Show file tree
Hide file tree
Showing 2 changed files with 53 additions and 22 deletions.
9 changes: 4 additions & 5 deletions br/pkg/lightning/errormanager/errormanager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -489,12 +489,11 @@ func (em *ErrorManager) Output() string {
t := table.NewWriter()
t.AppendHeader(table.Row{"#", "Error Type", "Error Count", "Error Data Table"})
t.SetColumnConfigs([]table.ColumnConfig{
{Name: "#", WidthMax: 6},
{Name: "Error Type", WidthMax: 20},
{Name: "Error Count", WidthMax: 12},
{Name: "Error Data Table", WidthMax: 42},
{Name: "#"},
{Name: "Error Type"},
{Name: "Error Count"},
{Name: "Error Data Table"},
})
t.SetAllowedRowLength(80)
t.SetRowPainter(func(row table.Row) text.Colors {
return text.Colors{text.FgRed}
})
Expand Down
66 changes: 49 additions & 17 deletions br/pkg/lightning/errormanager/errormanager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"io"
"math/rand"
"strconv"
"strings"
"testing"

"github.com/DATA-DOG/go-sqlmock"
Expand Down Expand Up @@ -55,7 +54,7 @@ func TestInit(t *testing.T) {
em.dupResolution = config.DupeResAlgRecord
mock.ExpectExec("CREATE SCHEMA IF NOT EXISTS `lightning_errors`;").
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("CREATE TABLE IF NOT EXISTS `lightning_errors`\\.conflict_error_v1_2.*").
mock.ExpectExec("CREATE TABLE IF NOT EXISTS `lightning_errors`\\.conflict_error_v1.*").
WillReturnResult(sqlmock.NewResult(2, 1))
err = em.Init(ctx)
require.NoError(t, err)
Expand All @@ -64,17 +63,17 @@ func TestInit(t *testing.T) {
em.remainingError.Type.Store(1)
mock.ExpectExec("CREATE SCHEMA IF NOT EXISTS `lightning_errors`;").
WillReturnResult(sqlmock.NewResult(3, 1))
mock.ExpectExec("CREATE TABLE IF NOT EXISTS `lightning_errors`\\.type_error_v2.*").
mock.ExpectExec("CREATE TABLE IF NOT EXISTS `lightning_errors`\\.type_error_v1.*").
WillReturnResult(sqlmock.NewResult(4, 1))
err = em.Init(ctx)
require.NoError(t, err)
em.dupResolution = config.DupeResAlgRecord
em.remainingError.Type.Store(1)
mock.ExpectExec("CREATE SCHEMA IF NOT EXISTS `lightning_errors`.*").
WillReturnResult(sqlmock.NewResult(5, 1))
mock.ExpectExec("CREATE TABLE IF NOT EXISTS `lightning_errors`\\.type_error_v2.*").
mock.ExpectExec("CREATE TABLE IF NOT EXISTS `lightning_errors`\\.type_error_v1.*").
WillReturnResult(sqlmock.NewResult(6, 1))
mock.ExpectExec("CREATE TABLE IF NOT EXISTS `lightning_errors`\\.conflict_error_v1_2.*").
mock.ExpectExec("CREATE TABLE IF NOT EXISTS `lightning_errors`\\.conflict_error_v1.*").
WillReturnResult(sqlmock.NewResult(7, 1))
err = em.Init(ctx)
require.NoError(t, err)
Expand Down Expand Up @@ -111,7 +110,7 @@ type mockRows struct {
}

func (r *mockRows) Columns() []string {
return []string{"id", "raw_handle", "raw_row"}
return []string{"_tidb_rowid", "raw_handle", "raw_row"}
}

func (r *mockRows) Close() error { return nil }
Expand All @@ -120,15 +119,15 @@ func (r *mockRows) Next(dest []driver.Value) error {
if r.start >= r.end {
return io.EOF
}
dest[0] = r.start // id
dest[0] = r.start // _tidb_rowid
dest[1] = []byte{} // raw_handle
dest[2] = []byte{} // raw_row
r.start++
return nil
}

func (c mockConn) QueryContext(_ context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
expectedQuery := "SELECT id, raw_handle, raw_row.*"
expectedQuery := "SELECT _tidb_rowid, raw_handle, raw_row.*"
if err := sqlmock.QueryMatcherRegexp.Match(expectedQuery, query); err != nil {
return &mockRows{}, nil
}
Expand Down Expand Up @@ -240,17 +239,28 @@ func TestErrorMgrErrorOutput(t *testing.T) {

em.remainingError.Syntax.Sub(1)
output = em.Output()
checkStr := strings.ReplaceAll(output, "\n", "")
expected := "Import Data Error Summary: +---+-------------+-------------+--------------------------------+| # | ERROR TYPE | ERROR COUNT | ERROR DATA TABLE |+---+-------------+-------------+--------------------------------+|\x1b[31m 1 \x1b[0m|\x1b[31m Data Syntax \x1b[0m|\x1b[31m 1 \x1b[0m|\x1b[31m `error_info`.`syntax_error_v2` \x1b[0m|+---+-------------+-------------+--------------------------------+"
require.Equal(t, expected, checkStr)
expected := "\n" +
"Import Data Error Summary: \n" +
"+---+-------------+-------------+--------------------------------+\n" +
"| # | ERROR TYPE | ERROR COUNT | ERROR DATA TABLE |\n" +
"+---+-------------+-------------+--------------------------------+\n" +
"|\x1b[31m 1 \x1b[0m|\x1b[31m Data Syntax \x1b[0m|\x1b[31m 1 \x1b[0m|\x1b[31m `error_info`.`syntax_error_v2` \x1b[0m|\n" +
"+---+-------------+-------------+--------------------------------+\n"
require.Equal(t, expected, output)

em.remainingError = cfg.App.MaxError
em.remainingError.Syntax.Sub(10)
em.remainingError.Type.Store(10)
output = em.Output()
checkStr = strings.ReplaceAll(output, "\n", "")
expected = "Import Data Error Summary: +---+-------------+-------------+--------------------------------+| # | ERROR TYPE | ERROR COUNT | ERROR DATA TABLE |+---+-------------+-------------+--------------------------------+|\x1b[31m 1 \x1b[0m|\x1b[31m Data Type \x1b[0m|\x1b[31m 90 \x1b[0m|\x1b[31m `error_info`.`type_error_v2` \x1b[0m||\x1b[31m 2 \x1b[0m|\x1b[31m Data Syntax \x1b[0m|\x1b[31m 10 \x1b[0m|\x1b[31m `error_info`.`syntax_error_v2` \x1b[0m|+---+-------------+-------------+--------------------------------+"
require.Equal(t, expected, checkStr)
expected = "\n" +
"Import Data Error Summary: \n" +
"+---+-------------+-------------+--------------------------------+\n" +
"| # | ERROR TYPE | ERROR COUNT | ERROR DATA TABLE |\n" +
"+---+-------------+-------------+--------------------------------+\n" +
"|\x1b[31m 1 \x1b[0m|\x1b[31m Data Type \x1b[0m|\x1b[31m 90 \x1b[0m|\x1b[31m `error_info`.`type_error_v2` \x1b[0m|\n" +
"|\x1b[31m 2 \x1b[0m|\x1b[31m Data Syntax \x1b[0m|\x1b[31m 10 \x1b[0m|\x1b[31m `error_info`.`syntax_error_v2` \x1b[0m|\n" +
"+---+-------------+-------------+--------------------------------+\n"
require.Equal(t, expected, output)

// change multiple keys
em.remainingError = cfg.App.MaxError
Expand All @@ -259,7 +269,29 @@ func TestErrorMgrErrorOutput(t *testing.T) {
em.remainingError.Type.Store(0)
em.remainingError.Conflict.Store(0)
output = em.Output()
checkStr = strings.ReplaceAll(output, "\n", "")
expected = "Import Data Error Summary: +---+---------------------+-------------+------------------------------------+| # | ERROR TYPE | ERROR COUNT | ERROR DATA TABLE |+---+---------------------+-------------+------------------------------------+|\x1b[31m 1 \x1b[0m|\x1b[31m Data Type \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`type_error_v2` \x1b[0m||\x1b[31m 2 \x1b[0m|\x1b[31m Data Syntax \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`syntax_error_v2` \x1b[0m||\x1b[31m 3 \x1b[0m|\x1b[31m Charset Error \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m \x1b[0m||\x1b[31m 4 \x1b[0m|\x1b[31m Unique Key Conflict \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`conflict_error_v1_2` \x1b[0m|+---+---------------------+-------------+------------------------------------+"
require.Equal(t, expected, checkStr)
expected = "\n" +
"Import Data Error Summary: \n" +
"+---+---------------------+-------------+------------------------------------+\n" +
"| # | ERROR TYPE | ERROR COUNT | ERROR DATA TABLE |\n" +
"+---+---------------------+-------------+------------------------------------+\n" +
"|\x1b[31m 1 \x1b[0m|\x1b[31m Data Type \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`type_error_v2` \x1b[0m|\n" +
"|\x1b[31m 2 \x1b[0m|\x1b[31m Data Syntax \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`syntax_error_v2` \x1b[0m|\n" +
"|\x1b[31m 3 \x1b[0m|\x1b[31m Charset Error \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m \x1b[0m|\n" +
"|\x1b[31m 4 \x1b[0m|\x1b[31m Unique Key Conflict \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `error_info`.`conflict_error_v1_2` \x1b[0m|\n" +
"+---+---------------------+-------------+------------------------------------+\n"
require.Equal(t, expected, output)

em.schemaEscaped = "`lightning_task_info`"
output = em.Output()
expected = "\n" +
"Import Data Error Summary: \n" +
"+---+---------------------+-------------+---------------------------------------------+\n" +
"| # | ERROR TYPE | ERROR COUNT | ERROR DATA TABLE |\n" +
"+---+---------------------+-------------+---------------------------------------------+\n" +
"|\x1b[31m 1 \x1b[0m|\x1b[31m Data Type \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `lightning_task_info`.`type_error_v2` \x1b[0m|\n" +
"|\x1b[31m 2 \x1b[0m|\x1b[31m Data Syntax \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `lightning_task_info`.`syntax_error_v2` \x1b[0m|\n" +
"|\x1b[31m 3 \x1b[0m|\x1b[31m Charset Error \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m \x1b[0m|\n" +
"|\x1b[31m 4 \x1b[0m|\x1b[31m Unique Key Conflict \x1b[0m|\x1b[31m 100 \x1b[0m|\x1b[31m `lightning_task_info`.`conflict_error_v1_2` \x1b[0m|\n" +
"+---+---------------------+-------------+---------------------------------------------+\n"
require.Equal(t, expected, output)
}

0 comments on commit c5a9b10

Please sign in to comment.