Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Column Migration: process migrations in batches #2506

Merged
merged 5 commits into from
Mar 11, 2025
+359 −286
Merged

Column Migration: process migrations in batches #2506

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3e809f9
materialize-sql: apply column migrations in batches for each table
mdibaiee Mar 10, 2025
3ff58f6
materialize-*: update column migration to process in batches
mdibaiee Mar 10, 2025
94d70d4
materialize-sql: s/variadic/slice in column migration instructions
mdibaiee Mar 10, 2025
6e27793
materialize-sql: migrate integer to number
mdibaiee Mar 11, 2025
ae53fc2
materialize-azure-fabric-warehouse: migrate integer to number
mdibaiee Mar 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions materialize-azure-fabric-warehouse/.snapshots/TestValidateAndApplyMigrations
View file
Original file line number Diff line number Diff line change
@@ -6,8 +6,10 @@ Base Initial Constraints:
{"Field":"flow_document","Type":2,"TypeString":"LOCATION_REQUIRED","Reason":"The root document must be materialized"}
{"Field":"flow_published_at","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
{"Field":"int64","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
{"Field":"int64ToNumber","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
{"Field":"intToNumber","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
{"Field":"intWidenedToJson","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
{"Field":"key","Type":2,"TypeString":"LOCATION_REQUIRED","Reason":"All Locations that are part of the collections key are required"}
{"Field":"key","Type":2,"TypeString":"LOCATION_REQUIRED","Reason":"The first collection key component is required to be included for standard updates"}
{"Field":"multiple","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This field is able to be materialized"}
{"Field":"nonScalarValue","Type":4,"TypeString":"FIELD_OPTIONAL","Reason":"Object fields may be materialized"}
{"Field":"nullValue","Type":5,"TypeString":"FIELD_FORBIDDEN","Reason":"Cannot materialize a field where the only possible type is 'null'"}
@@ -27,6 +29,8 @@ Migratable Changes Before Apply Schema:
{"Name":"flow_document","Nullable":"YES","Type":"varchar"}
{"Name":"flow_published_at","Nullable":"YES","Type":"datetime2"}
{"Name":"int64","Nullable":"YES","Type":"bigint"}
{"Name":"int64ToNumber","Nullable":"YES","Type":"decimal"}
{"Name":"intToNumber","Nullable":"YES","Type":"bigint"}
{"Name":"intWidenedToJson","Nullable":"YES","Type":"bigint"}
{"Name":"key","Nullable":"YES","Type":"varchar"}
{"Name":"multiple","Nullable":"YES","Type":"varchar"}
@@ -40,8 +44,8 @@ Migratable Changes Before Apply Schema:


Migratable Changes Before Apply Data:
key (NVARCHAR), _meta/flow_truncated (BIT), boolWidenedToJson (BIT), dateValue (DATE), datetimeValue (DATETIME2), flow_published_at (DATETIME2), int64 (BIGINT), intWidenedToJson (BIGINT), multiple (NVARCHAR), nonScalarValue (NVARCHAR), numericString (DECIMAL), optional (NVARCHAR), requiredNumeric (DECIMAL), scalarValue (NVARCHAR), stringWidenedToJson (NVARCHAR), timeValue (TIME), flow_document (NVARCHAR)
1, false, true, 2024-01-01T00:00:00Z, 2024-01-01T01:01:01.111111Z, 2024-09-13T01:01:01Z, 1, 999, <nil>, <nil>, 123, <nil>, 456, test, hello, 0001-01-01T01:01:01Z, {}
key (NVARCHAR), _meta/flow_truncated (BIT), boolWidenedToJson (BIT), dateValue (DATE), datetimeValue (DATETIME2), flow_published_at (DATETIME2), int64 (BIGINT), int64ToNumber (DECIMAL), intToNumber (BIGINT), intWidenedToJson (BIGINT), multiple (NVARCHAR), nonScalarValue (NVARCHAR), numericString (DECIMAL), optional (NVARCHAR), requiredNumeric (DECIMAL), scalarValue (NVARCHAR), stringWidenedToJson (NVARCHAR), timeValue (TIME), flow_document (NVARCHAR)
1, false, true, 2024-01-01T00:00:00Z, 2024-01-01T01:01:01.111111Z, 2024-09-13T01:01:01Z, 1, 10000000000000000000, 9223372036854775807, 999, <nil>, <nil>, 123, <nil>, 456, test, hello, 0001-01-01T01:01:01Z, {}

Migratable Changes Constraints:
{"Field":"_meta/flow_truncated","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
@@ -51,6 +55,8 @@ Migratable Changes Constraints:
{"Field":"flow_document","Type":1,"TypeString":"FIELD_REQUIRED","Reason":"This field is the document in the current materialization"}
{"Field":"flow_published_at","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
{"Field":"int64","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
{"Field":"int64ToNumber","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
{"Field":"intToNumber","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
{"Field":"intWidenedToJson","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
{"Field":"key","Type":1,"TypeString":"FIELD_REQUIRED","Reason":"This field is a key in the current materialization"}
{"Field":"multiple","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
@@ -72,6 +78,8 @@ Migratable Changes Applied Schema:
{"Name":"flow_document","Nullable":"YES","Type":"varchar"}
{"Name":"flow_published_at","Nullable":"YES","Type":"datetime2"}
{"Name":"int64","Nullable":"YES","Type":"decimal"}
{"Name":"int64ToNumber","Nullable":"YES","Type":"float"}
{"Name":"intToNumber","Nullable":"YES","Type":"float"}
{"Name":"intWidenedToJson","Nullable":"YES","Type":"varchar"}
{"Name":"key","Nullable":"YES","Type":"varchar"}
{"Name":"multiple","Nullable":"YES","Type":"varchar"}
@@ -85,6 +93,6 @@ Migratable Changes Applied Schema:


Migratable Changes Applied Data:
key (NVARCHAR), _meta/flow_truncated (BIT), boolWidenedToJson (NVARCHAR), dateValue (NVARCHAR), datetimeValue (NVARCHAR), flow_published_at (DATETIME2), int64 (DECIMAL), intWidenedToJson (NVARCHAR), multiple (NVARCHAR), nonScalarValue (NVARCHAR), numericString (NVARCHAR), optional (NVARCHAR), requiredNumeric (NVARCHAR), scalarValue (NVARCHAR), stringWidenedToJson (NVARCHAR), timeValue (NVARCHAR), flow_document (NVARCHAR)
1, false, true, 2024-01-01, 2024-01-01 01:01:01.111111, 2024-09-13T01:01:01Z, 1, 999, <nil>, <nil>, 123, <nil>, 456, test, hello, 01:01:01.000000, {}
key (NVARCHAR), _meta/flow_truncated (BIT), boolWidenedToJson (NVARCHAR), dateValue (NVARCHAR), datetimeValue (NVARCHAR), flow_published_at (DATETIME2), int64 (DECIMAL), int64ToNumber (FLOAT), intToNumber (FLOAT), intWidenedToJson (NVARCHAR), multiple (NVARCHAR), nonScalarValue (NVARCHAR), numericString (NVARCHAR), optional (NVARCHAR), requiredNumeric (NVARCHAR), scalarValue (NVARCHAR), stringWidenedToJson (NVARCHAR), timeValue (NVARCHAR), flow_document (NVARCHAR)
1, false, true, 2024-01-01, 2024-01-01 01:01:01.111111, 2024-09-13T01:01:01Z, 1, 1e+19, 9.223372036854776e+18, 999, <nil>, <nil>, 123, <nil>, 456, test, hello, 01:01:01.000000, {}

4 changes: 2 additions & 2 deletions materialize-azure-fabric-warehouse/sqlgen.go
View file
Original file line number Diff line number Diff line change
@@ -48,8 +48,8 @@ var dialect = func() sql.Dialect {

return sql.Dialect{
MigratableTypes: sql.MigrationSpecs{
"bigint": {sql.NewMigrationSpec([]string{"DECIMAL(38,0)", "VARCHAR(MAX)"})},
"decimal": {sql.NewMigrationSpec([]string{"VARCHAR(MAX)"})},
"bigint": {sql.NewMigrationSpec([]string{"FLOAT", "DECIMAL(38,0)", "VARCHAR(MAX)"})},
"decimal": {sql.NewMigrationSpec([]string{"FLOAT", "VARCHAR(MAX)"})},
"float": {sql.NewMigrationSpec([]string{"VARCHAR(MAX)"})},
"bit": {sql.NewMigrationSpec([]string{"VARCHAR(MAX)"}, sql.WithCastSQL(bitToStringCast))},
"date": {sql.NewMigrationSpec([]string{"VARCHAR(MAX)"})},
18 changes: 13 additions & 5 deletions materialize-bigquery/.snapshots/TestValidateAndApplyMigrations
View file
Original file line number Diff line number Diff line change
@@ -6,8 +6,10 @@ Base Initial Constraints:
{"Field":"flow_document","Type":2,"TypeString":"LOCATION_REQUIRED","Reason":"The root document must be materialized"}
{"Field":"flow_published_at","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
{"Field":"int64","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
{"Field":"int64ToNumber","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
{"Field":"intToNumber","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
{"Field":"intWidenedToJson","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
{"Field":"key","Type":2,"TypeString":"LOCATION_REQUIRED","Reason":"All Locations that are part of the collections key are required"}
{"Field":"key","Type":2,"TypeString":"LOCATION_REQUIRED","Reason":"The first collection key component is required to be included for standard updates"}
{"Field":"multiple","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This field is able to be materialized"}
{"Field":"nonScalarValue","Type":4,"TypeString":"FIELD_OPTIONAL","Reason":"Object fields may be materialized"}
{"Field":"nullValue","Type":5,"TypeString":"FIELD_FORBIDDEN","Reason":"Cannot materialize a field where the only possible type is 'null'"}
@@ -27,6 +29,8 @@ Migratable Changes Before Apply Schema:
{"Name":"flow_document","Nullable":"NO","Type":"JSON"}
{"Name":"flow_published_at","Nullable":"NO","Type":"TIMESTAMP"}
{"Name":"int64","Nullable":"YES","Type":"INT64"}
{"Name":"int64ToNumber","Nullable":"YES","Type":"BIGNUMERIC(38)"}
{"Name":"intToNumber","Nullable":"YES","Type":"INT64"}
{"Name":"intWidenedToJson","Nullable":"YES","Type":"INT64"}
{"Name":"key","Nullable":"NO","Type":"STRING"}
{"Name":"multiple","Nullable":"YES","Type":"JSON"}
@@ -40,8 +44,8 @@ Migratable Changes Before Apply Schema:


Migratable Changes Before Apply Data:
key (STRING), _meta_flow_truncated (BOOLEAN), boolWidenedToJson (BOOLEAN), dateValue (DATE), datetimeValue (TIMESTAMP), flow_published_at (TIMESTAMP), int64 (INTEGER), intWidenedToJson (INTEGER), multiple (JSON), nonScalarValue (JSON), numericString (BIGNUMERIC), optional (JSON), requiredNumeric (BIGNUMERIC), scalarValue (STRING), stringWidenedToJson (STRING), timeValue (STRING), flow_document (JSON)
1, false, true, 2024-01-01, 2024-01-01 01:01:01.111111 +0000 UTC, 2024-09-13 01:01:01 +0000 UTC, 1, 999, <nil>, <nil>, 12300000000000000000000000000000000000000/100000000000000000000000000000000000000, <nil>, 45600000000000000000000000000000000000000/100000000000000000000000000000000000000, test, hello, 01:01:01, {}
key (STRING), _meta_flow_truncated (BOOLEAN), boolWidenedToJson (BOOLEAN), dateValue (DATE), datetimeValue (TIMESTAMP), flow_published_at (TIMESTAMP), int64 (INTEGER), int64ToNumber (BIGNUMERIC), intToNumber (INTEGER), intWidenedToJson (INTEGER), multiple (JSON), nonScalarValue (JSON), numericString (BIGNUMERIC), optional (JSON), requiredNumeric (BIGNUMERIC), scalarValue (STRING), stringWidenedToJson (STRING), timeValue (STRING), flow_document (JSON)
1, false, true, 2024-01-01, 2024-01-01 01:01:01.111111 +0000 UTC, 2024-09-13 01:01:01 +0000 UTC, 1, 1000000000000000000000000000000000000000000000000000000000/100000000000000000000000000000000000000, 9223372036854775807, 999, <nil>, <nil>, 12300000000000000000000000000000000000000/100000000000000000000000000000000000000, <nil>, 45600000000000000000000000000000000000000/100000000000000000000000000000000000000, test, hello, 01:01:01, {}

Migratable Changes Constraints:
{"Field":"_meta/flow_truncated","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
@@ -51,6 +55,8 @@ Migratable Changes Constraints:
{"Field":"flow_document","Type":1,"TypeString":"FIELD_REQUIRED","Reason":"This field is the document in the current materialization"}
{"Field":"flow_published_at","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
{"Field":"int64","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
{"Field":"int64ToNumber","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
{"Field":"intToNumber","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
{"Field":"intWidenedToJson","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
{"Field":"key","Type":1,"TypeString":"FIELD_REQUIRED","Reason":"This field is a key in the current materialization"}
{"Field":"multiple","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
@@ -72,6 +78,8 @@ Migratable Changes Applied Schema:
{"Name":"flow_document","Nullable":"NO","Type":"JSON"}
{"Name":"flow_published_at","Nullable":"NO","Type":"TIMESTAMP"}
{"Name":"int64","Nullable":"YES","Type":"BIGNUMERIC(38)"}
{"Name":"int64ToNumber","Nullable":"YES","Type":"BIGNUMERIC(38)"}
{"Name":"intToNumber","Nullable":"YES","Type":"FLOAT64"}
{"Name":"intWidenedToJson","Nullable":"YES","Type":"JSON"}
{"Name":"key","Nullable":"NO","Type":"STRING"}
{"Name":"multiple","Nullable":"YES","Type":"JSON"}
@@ -85,6 +93,6 @@ Migratable Changes Applied Schema:


Migratable Changes Applied Data:
key (STRING), _meta_flow_truncated (BOOLEAN), flow_published_at (TIMESTAMP), multiple (JSON), nonScalarValue (JSON), optional (JSON), scalarValue (STRING), timeValue (STRING), flow_document (JSON), boolWidenedToJson (JSON), dateValue (STRING), datetimeValue (STRING), int64 (BIGNUMERIC), intWidenedToJson (JSON), numericString (STRING), requiredNumeric (STRING), stringWidenedToJson (JSON)
1, false, 2024-09-13 01:01:01 +0000 UTC, <nil>, <nil>, <nil>, test, 01:01:01, {}, true, 2024-01-01, 2024-01-01T01:01:01.111111Z, 100000000000000000000000000000000000000/100000000000000000000000000000000000000, 999, 123, 456, "hello"
key (STRING), _meta_flow_truncated (BOOLEAN), flow_published_at (TIMESTAMP), int64ToNumber (BIGNUMERIC), multiple (JSON), nonScalarValue (JSON), optional (JSON), scalarValue (STRING), timeValue (STRING), flow_document (JSON), boolWidenedToJson (JSON), dateValue (STRING), datetimeValue (STRING), int64 (BIGNUMERIC), intToNumber (FLOAT), intWidenedToJson (JSON), numericString (STRING), requiredNumeric (STRING), stringWidenedToJson (JSON)
1, false, 2024-09-13 01:01:01 +0000 UTC, 1000000000000000000000000000000000000000000000000000000000/100000000000000000000000000000000000000, <nil>, <nil>, <nil>, test, 01:01:01, {}, true, 2024-01-01, 2024-01-01T01:01:01.111111Z, 100000000000000000000000000000000000000/100000000000000000000000000000000000000, 9.223372036854776e+18, 999, 123, 456, "hello"

7 changes: 5 additions & 2 deletions materialize-bigquery/bigquery_test.go
View file
Original file line number Diff line number Diff line change
@@ -120,12 +120,15 @@ func TestValidateAndApplyMigrations(t *testing.T) {
keys = append(keys, testDialect.Identifier("flow_document"))
values = append(values, "JSON '{}'")

// bigquery does not support more than 6 fractional second precision, and will fail if we try
// to insert a value with 9
for i := range values {
// bigquery does not support more than 6 fractional second precision, and will fail if we try
// to insert a value with 9
if keys[i] == "datetimeValue" {
values[i] = "'2024-01-01 01:01:01.111111'"
}
if keys[i] == "int64ToNumber" {
values[i] = "BIGNUMERIC '" + values[i] + "'"
}
}

_, err = client.query(ctx, fmt.Sprintf(
50 changes: 10 additions & 40 deletions materialize-bigquery/client.go
View file
Original file line number Diff line number Diff line change
@@ -126,44 +126,16 @@ func (c *client) DeleteTable(ctx context.Context, path []string) (string, boiler
}

var columnMigrationSteps = []sql.ColumnMigrationStep{
func(dialect sql.Dialect, table sql.Table, migration sql.ColumnTypeMigration, tempColumnIdentifier string) (string, error) {
return fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s;",
table.Identifier,
tempColumnIdentifier,
// Always create these new columns as nullable
migration.NullableDDL,
), nil
},
func(dialect sql.Dialect, table sql.Table, migration sql.ColumnTypeMigration, tempColumnIdentifier string) (string, error) {
return fmt.Sprintf(
// The WHERE filter is required by some warehouses (bigquery)
"UPDATE %s SET %s = %s WHERE true;",
table.Identifier,
tempColumnIdentifier,
migration.CastSQL(migration),
), nil
},
func(dialect sql.Dialect, table sql.Table, migration sql.ColumnTypeMigration, _ string) (string, error) {
return fmt.Sprintf(
"ALTER TABLE %s DROP COLUMN %s;",
table.Identifier,
migration.Identifier,
), nil
},
func(dialect sql.Dialect, table sql.Table, migration sql.ColumnTypeMigration, tempColumnIdentifier string) (string, error) {
return fmt.Sprintf(
"ALTER TABLE %s RENAME COLUMN %s TO %s;",
table.Identifier,
tempColumnIdentifier,
migration.Identifier,
), nil
},
func(dialect sql.Dialect, table sql.Table, migration sql.ColumnTypeMigration, _ string) (string, error) {
sql.StdMigrationSteps[0],
sql.StdMigrationSteps[1],
sql.StdMigrationSteps[2],
sql.StdMigrationSteps[3],
func(dialect sql.Dialect, table sql.Table, instructions []sql.MigrationInstruction) ([]string, error) {
// BigQuery does not support making a column REQUIRED when it is NULLABLE
// TODO: do we prefer to backfill in these instances for BigQuery, or just continue
// with this no-op as-is?

return "", nil
return []string{}, nil
},
}

@@ -186,12 +158,10 @@ func (c *client) AlterTable(ctx context.Context, ta sql.TableAlter) (string, boi
}

if len(ta.ColumnTypeChanges) > 0 {
for _, m := range ta.ColumnTypeChanges {
if steps, err := sql.StdColumnTypeMigration(ctx, c.ep.Dialect, ta.Table, m, columnMigrationSteps...); err != nil {
return "", nil, fmt.Errorf("rendering column migration steps: %w", err)
} else {
stmts = append(stmts, steps...)
}
if steps, err := sql.StdColumnTypeMigrations(ctx, c.ep.Dialect, ta.Table, ta.ColumnTypeChanges, columnMigrationSteps...); err != nil {
return "", nil, fmt.Errorf("rendering column migration steps: %w", err)
} else {
stmts = append(stmts, steps...)
}
}

4 changes: 2 additions & 2 deletions materialize-bigquery/sqlgen.go
View file
Original file line number Diff line number Diff line change
@@ -109,9 +109,9 @@ func bqDialect(objAndArrayAsJson bool) sql.Dialect {
MigratableTypes: sql.MigrationSpecs{
"integer": {
sql.NewMigrationSpec([]string{"bignumeric(38,0)"}, sql.WithCastSQL(toBigNumericCast)),
sql.NewMigrationSpec([]string{"string"}),
sql.NewMigrationSpec([]string{"float64", "string"}),
},
"bignumeric": {sql.NewMigrationSpec([]string{"string"})},
"bignumeric": {sql.NewMigrationSpec([]string{"float64", "string"})},
"float": {sql.NewMigrationSpec([]string{"string"})},
"date": {sql.NewMigrationSpec([]string{"string"})},
"timestamp": {sql.NewMigrationSpec([]string{"string"}, sql.WithCastSQL(datetimeToStringCast))},
10 changes: 4 additions & 6 deletions materialize-cratedb/client.go
View file
Original file line number Diff line number Diff line change
@@ -153,12 +153,10 @@ func (c *client) AlterTable(ctx context.Context, ta sql.TableAlter) (string, boi
}

if len(ta.ColumnTypeChanges) > 0 {
for _, m := range ta.ColumnTypeChanges {
if steps, err := sql.StdColumnTypeMigration(ctx, crateDialect, ta.Table, m); err != nil {
return "", nil, fmt.Errorf("rendering column migration steps: %w", err)
} else {
stmts = append(stmts, steps...)
}
if steps, err := sql.StdColumnTypeMigrations(ctx, crateDialect, ta.Table, ta.ColumnTypeChanges); err != nil {
return "", nil, fmt.Errorf("rendering column migration steps: %w", err)
} else {
stmts = append(stmts, steps...)
}
}

Binary file modified materialize-databricks/.snapshots/TestValidateAndApplyMigrations
View file
Binary file not shown.
10 changes: 4 additions & 6 deletions materialize-databricks/client.go
View file
Original file line number Diff line number Diff line change
@@ -158,12 +158,10 @@ func (c *client) AlterTable(ctx context.Context, ta sql.TableAlter) (string, boi
}

if len(ta.ColumnTypeChanges) > 0 {
for _, m := range ta.ColumnTypeChanges {
if steps, err := sql.StdColumnTypeMigration(ctx, c.ep.Dialect, ta.Table, m); err != nil {
return "", nil, fmt.Errorf("rendering column migration steps: %w", err)
} else {
stmts = append(stmts, steps...)
}
if steps, err := sql.StdColumnTypeMigrations(ctx, c.ep.Dialect, ta.Table, ta.ColumnTypeChanges); err != nil {
return "", nil, fmt.Errorf("rendering column migration steps: %w", err)
} else {
stmts = append(stmts, steps...)
}
}

4 changes: 2 additions & 2 deletions materialize-databricks/sqlgen.go
View file
Original file line number Diff line number Diff line change
@@ -61,8 +61,8 @@ var databricksDialect = func() sql.Dialect {

return sql.Dialect{
MigratableTypes: sql.MigrationSpecs{
"decimal": {sql.NewMigrationSpec([]string{"string"})},
"long": {sql.NewMigrationSpec([]string{"numeric(38,0)", "string"})},
"decimal": {sql.NewMigrationSpec([]string{"double", "string"})},
"long": {sql.NewMigrationSpec([]string{"double", "numeric(38,0)", "string"})},
"double": {sql.NewMigrationSpec([]string{"string"})},
"timestamp": {sql.NewMigrationSpec([]string{"string"}, sql.WithCastSQL(datetimeToStringCast))},
"date": {sql.NewMigrationSpec([]string{"string"})},
Loading