Skip to content

Commit 5d31e3d

Browse files
mdibaieemdibaiee
committed
materialize-sql: migrate integer to number
1 parent 461b570 commit 5d31e3d

File tree

26 files changed

+81
-35
lines changed

26 files changed

+81
-35
lines changed

‎materialize-bigquery/.snapshots/TestValidateAndApplyMigrations

+12-4
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ Base Initial Constraints:
66
{"Field":"flow_document","Type":2,"TypeString":"LOCATION_REQUIRED","Reason":"The root document must be materialized"}
77
{"Field":"flow_published_at","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
88
{"Field":"int64","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
9+
{"Field":"int64ToNumber","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
10+
{"Field":"intToNumber","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
911
{"Field":"intWidenedToJson","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
1012
{"Field":"key","Type":2,"TypeString":"LOCATION_REQUIRED","Reason":"The first collection key component is required to be included for standard updates"}
1113
{"Field":"multiple","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This field is able to be materialized"}
@@ -27,6 +29,8 @@ Migratable Changes Before Apply Schema:
2729
{"Name":"flow_document","Nullable":"NO","Type":"JSON"}
2830
{"Name":"flow_published_at","Nullable":"NO","Type":"TIMESTAMP"}
2931
{"Name":"int64","Nullable":"YES","Type":"INT64"}
32+
{"Name":"int64ToNumber","Nullable":"YES","Type":"BIGNUMERIC(38)"}
33+
{"Name":"intToNumber","Nullable":"YES","Type":"INT64"}
3034
{"Name":"intWidenedToJson","Nullable":"YES","Type":"INT64"}
3135
{"Name":"key","Nullable":"NO","Type":"STRING"}
3236
{"Name":"multiple","Nullable":"YES","Type":"JSON"}
@@ -40,8 +44,8 @@ Migratable Changes Before Apply Schema:
4044

4145

4246
Migratable Changes Before Apply Data:
43-
key (STRING), _meta_flow_truncated (BOOLEAN), boolWidenedToJson (BOOLEAN), dateValue (DATE), datetimeValue (TIMESTAMP), flow_published_at (TIMESTAMP), int64 (INTEGER), intWidenedToJson (INTEGER), multiple (JSON), nonScalarValue (JSON), numericString (BIGNUMERIC), optional (JSON), requiredNumeric (BIGNUMERIC), scalarValue (STRING), stringWidenedToJson (STRING), timeValue (STRING), flow_document (JSON)
44-
1, false, true, 2024-01-01, 2024-01-01 01:01:01.111111 +0000 UTC, 2024-09-13 01:01:01 +0000 UTC, 1, 999, <nil>, <nil>, 12300000000000000000000000000000000000000/100000000000000000000000000000000000000, <nil>, 45600000000000000000000000000000000000000/100000000000000000000000000000000000000, test, hello, 01:01:01, {}
47+
key (STRING), _meta_flow_truncated (BOOLEAN), boolWidenedToJson (BOOLEAN), dateValue (DATE), datetimeValue (TIMESTAMP), flow_published_at (TIMESTAMP), int64 (INTEGER), int64ToNumber (BIGNUMERIC), intToNumber (INTEGER), intWidenedToJson (INTEGER), multiple (JSON), nonScalarValue (JSON), numericString (BIGNUMERIC), optional (JSON), requiredNumeric (BIGNUMERIC), scalarValue (STRING), stringWidenedToJson (STRING), timeValue (STRING), flow_document (JSON)
48+
1, false, true, 2024-01-01, 2024-01-01 01:01:01.111111 +0000 UTC, 2024-09-13 01:01:01 +0000 UTC, 1, 1000000000000000000000000000000000000000000000000000000000/100000000000000000000000000000000000000, 9223372036854775807, 999, <nil>, <nil>, 12300000000000000000000000000000000000000/100000000000000000000000000000000000000, <nil>, 45600000000000000000000000000000000000000/100000000000000000000000000000000000000, test, hello, 01:01:01, {}
4549

4650
Migratable Changes Constraints:
4751
{"Field":"_meta/flow_truncated","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
@@ -51,6 +55,8 @@ Migratable Changes Constraints:
5155
{"Field":"flow_document","Type":1,"TypeString":"FIELD_REQUIRED","Reason":"This field is the document in the current materialization"}
5256
{"Field":"flow_published_at","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
5357
{"Field":"int64","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
58+
{"Field":"int64ToNumber","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
59+
{"Field":"intToNumber","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
5460
{"Field":"intWidenedToJson","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
5561
{"Field":"key","Type":1,"TypeString":"FIELD_REQUIRED","Reason":"This field is a key in the current materialization"}
5662
{"Field":"multiple","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
@@ -72,6 +78,8 @@ Migratable Changes Applied Schema:
7278
{"Name":"flow_document","Nullable":"NO","Type":"JSON"}
7379
{"Name":"flow_published_at","Nullable":"NO","Type":"TIMESTAMP"}
7480
{"Name":"int64","Nullable":"YES","Type":"BIGNUMERIC(38)"}
81+
{"Name":"int64ToNumber","Nullable":"YES","Type":"BIGNUMERIC(38)"}
82+
{"Name":"intToNumber","Nullable":"YES","Type":"FLOAT64"}
7583
{"Name":"intWidenedToJson","Nullable":"YES","Type":"JSON"}
7684
{"Name":"key","Nullable":"NO","Type":"STRING"}
7785
{"Name":"multiple","Nullable":"YES","Type":"JSON"}
@@ -85,6 +93,6 @@ Migratable Changes Applied Schema:
8593

8694

8795
Migratable Changes Applied Data:
88-
key (STRING), _meta_flow_truncated (BOOLEAN), flow_published_at (TIMESTAMP), multiple (JSON), nonScalarValue (JSON), optional (JSON), scalarValue (STRING), timeValue (STRING), flow_document (JSON), boolWidenedToJson (JSON), dateValue (STRING), datetimeValue (STRING), int64 (BIGNUMERIC), intWidenedToJson (JSON), numericString (STRING), requiredNumeric (STRING), stringWidenedToJson (JSON)
89-
1, false, 2024-09-13 01:01:01 +0000 UTC, <nil>, <nil>, <nil>, test, 01:01:01, {}, true, 2024-01-01, 2024-01-01T01:01:01.111111Z, 100000000000000000000000000000000000000/100000000000000000000000000000000000000, 999, 123, 456, "hello"
96+
key (STRING), _meta_flow_truncated (BOOLEAN), flow_published_at (TIMESTAMP), int64ToNumber (BIGNUMERIC), multiple (JSON), nonScalarValue (JSON), optional (JSON), scalarValue (STRING), timeValue (STRING), flow_document (JSON), boolWidenedToJson (JSON), dateValue (STRING), datetimeValue (STRING), int64 (BIGNUMERIC), intToNumber (FLOAT), intWidenedToJson (JSON), numericString (STRING), requiredNumeric (STRING), stringWidenedToJson (JSON)
97+
1, false, 2024-09-13 01:01:01 +0000 UTC, 1000000000000000000000000000000000000000000000000000000000/100000000000000000000000000000000000000, <nil>, <nil>, <nil>, test, 01:01:01, {}, true, 2024-01-01, 2024-01-01T01:01:01.111111Z, 100000000000000000000000000000000000000/100000000000000000000000000000000000000, 9.223372036854776e+18, 999, 123, 456, "hello"
9098

‎materialize-bigquery/bigquery_test.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -120,12 +120,15 @@ func TestValidateAndApplyMigrations(t *testing.T) {
120120
keys = append(keys, testDialect.Identifier("flow_document"))
121121
values = append(values, "JSON '{}'")
122122

123-
// bigquery does not support more than 6 fractional second precision, and will fail if we try
124-
// to insert a value with 9
125123
for i := range values {
124+
// bigquery does not support more than 6 fractional second precision, and will fail if we try
125+
// to insert a value with 9
126126
if keys[i] == "datetimeValue" {
127127
values[i] = "'2024-01-01 01:01:01.111111'"
128128
}
129+
if keys[i] == "int64ToNumber" {
130+
values[i] = "BIGNUMERIC '" + values[i] + "'"
131+
}
129132
}
130133

131134
_, err = client.query(ctx, fmt.Sprintf(

‎materialize-bigquery/sqlgen.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -109,9 +109,9 @@ func bqDialect(objAndArrayAsJson bool) sql.Dialect {
109109
MigratableTypes: sql.MigrationSpecs{
110110
"integer": {
111111
sql.NewMigrationSpec([]string{"bignumeric(38,0)"}, sql.WithCastSQL(toBigNumericCast)),
112-
sql.NewMigrationSpec([]string{"string"}),
112+
sql.NewMigrationSpec([]string{"float64", "string"}),
113113
},
114-
"bignumeric": {sql.NewMigrationSpec([]string{"string"})},
114+
"bignumeric": {sql.NewMigrationSpec([]string{"float64", "string"})},
115115
"float": {sql.NewMigrationSpec([]string{"string"})},
116116
"date": {sql.NewMigrationSpec([]string{"string"})},
117117
"timestamp": {sql.NewMigrationSpec([]string{"string"}, sql.WithCastSQL(datetimeToStringCast))},

‎materialize-databricks/.snapshots/TestValidateAndApplyMigrations

893 Bytes
Binary file not shown.

‎materialize-databricks/sqlgen.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,8 @@ var databricksDialect = func() sql.Dialect {
6161

6262
return sql.Dialect{
6363
MigratableTypes: sql.MigrationSpecs{
64-
"decimal": {sql.NewMigrationSpec([]string{"string"})},
65-
"long": {sql.NewMigrationSpec([]string{"numeric(38,0)", "string"})},
64+
"decimal": {sql.NewMigrationSpec([]string{"double", "string"})},
65+
"long": {sql.NewMigrationSpec([]string{"double", "numeric(38,0)", "string"})},
6666
"double": {sql.NewMigrationSpec([]string{"string"})},
6767
"timestamp": {sql.NewMigrationSpec([]string{"string"}, sql.WithCastSQL(datetimeToStringCast))},
6868
"date": {sql.NewMigrationSpec([]string{"string"})},

‎materialize-motherduck/.snapshots/TestValidateAndApplyMigrations

+12-4
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ Base Initial Constraints:
66
{"Field":"flow_document","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The root document should usually be materialized"}
77
{"Field":"flow_published_at","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
88
{"Field":"int64","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
9+
{"Field":"int64ToNumber","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
10+
{"Field":"intToNumber","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
911
{"Field":"intWidenedToJson","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"The projection has a single scalar type"}
1012
{"Field":"key","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"All Locations that are part of the collections key are recommended"}
1113
{"Field":"multiple","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This field is able to be materialized"}
@@ -27,6 +29,8 @@ Migratable Changes Before Apply Schema:
2729
{"Name":"flow_document","Nullable":"NO","Type":"JSON"}
2830
{"Name":"flow_published_at","Nullable":"NO","Type":"TIMESTAMP WITH TIME ZONE"}
2931
{"Name":"int64","Nullable":"YES","Type":"BIGINT"}
32+
{"Name":"int64ToNumber","Nullable":"YES","Type":"HUGEINT"}
33+
{"Name":"intToNumber","Nullable":"YES","Type":"BIGINT"}
3034
{"Name":"intWidenedToJson","Nullable":"YES","Type":"BIGINT"}
3135
{"Name":"key","Nullable":"NO","Type":"VARCHAR"}
3236
{"Name":"multiple","Nullable":"YES","Type":"JSON"}
@@ -41,8 +45,8 @@ Migratable Changes Before Apply Schema:
4145

4246

4347
Migratable Changes Before Apply Data:
44-
key (VARCHAR), _meta/flow_truncated (BOOLEAN), boolWidenedToJson (BOOLEAN), dateValue (DATE), datetimeValue (TIMESTAMPTZ), flow_document (VARCHAR), flow_published_at (TIMESTAMPTZ), int64 (BIGINT), intWidenedToJson (BIGINT), multiple (VARCHAR), nonScalarValue (VARCHAR), numericString (HUGEINT), optional (VARCHAR), requiredNumeric (HUGEINT), scalarValue (VARCHAR), stringWidenedToJson (VARCHAR), timeValue (TIME), second_root (VARCHAR)
45-
1, false, true, 2024-01-01T00:00:00Z, 2024-01-01T01:01:01.111111Z, {}, 2024-09-13T01:01:01Z, 1, 999, <nil>, <nil>, 123, <nil>, 456, test, hello, 1970-01-01T01:01:01Z, {}
48+
key (VARCHAR), _meta/flow_truncated (BOOLEAN), boolWidenedToJson (BOOLEAN), dateValue (DATE), datetimeValue (TIMESTAMPTZ), flow_document (VARCHAR), flow_published_at (TIMESTAMPTZ), int64 (BIGINT), int64ToNumber (HUGEINT), intToNumber (BIGINT), intWidenedToJson (BIGINT), multiple (VARCHAR), nonScalarValue (VARCHAR), numericString (HUGEINT), optional (VARCHAR), requiredNumeric (HUGEINT), scalarValue (VARCHAR), stringWidenedToJson (VARCHAR), timeValue (TIME), second_root (VARCHAR)
49+
1, false, true, 2024-01-01T00:00:00Z, 2024-01-01T01:01:01.111111Z, {}, 2024-09-13T01:01:01Z, 1, 10000000000000000000, 9223372036854775807, 999, <nil>, <nil>, 123, <nil>, 456, test, hello, 1970-01-01T01:01:01Z, {}
4650

4751
Migratable Changes Constraints:
4852
{"Field":"_meta/flow_truncated","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
@@ -52,6 +56,8 @@ Migratable Changes Constraints:
5256
{"Field":"flow_document","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
5357
{"Field":"flow_published_at","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
5458
{"Field":"int64","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
59+
{"Field":"int64ToNumber","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
60+
{"Field":"intToNumber","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
5561
{"Field":"intWidenedToJson","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
5662
{"Field":"key","Type":1,"TypeString":"FIELD_REQUIRED","Reason":"This field is a key in the current materialization"}
5763
{"Field":"multiple","Type":3,"TypeString":"LOCATION_RECOMMENDED","Reason":"This location is part of the current materialization"}
@@ -73,6 +79,8 @@ Migratable Changes Applied Schema:
7379
{"Name":"flow_document","Nullable":"NO","Type":"JSON"}
7480
{"Name":"flow_published_at","Nullable":"NO","Type":"TIMESTAMP WITH TIME ZONE"}
7581
{"Name":"int64","Nullable":"YES","Type":"HUGEINT"}
82+
{"Name":"int64ToNumber","Nullable":"YES","Type":"DOUBLE"}
83+
{"Name":"intToNumber","Nullable":"YES","Type":"DOUBLE"}
7684
{"Name":"intWidenedToJson","Nullable":"YES","Type":"JSON"}
7785
{"Name":"key","Nullable":"NO","Type":"VARCHAR"}
7886
{"Name":"multiple","Nullable":"YES","Type":"JSON"}
@@ -87,6 +95,6 @@ Migratable Changes Applied Schema:
8795

8896

8997
Migratable Changes Applied Data:
90-
key (VARCHAR), _meta/flow_truncated (BOOLEAN), datetimeValue (TIMESTAMPTZ), flow_document (VARCHAR), flow_published_at (TIMESTAMPTZ), multiple (VARCHAR), nonScalarValue (VARCHAR), optional (VARCHAR), scalarValue (VARCHAR), second_root (VARCHAR), boolWidenedToJson (VARCHAR), dateValue (VARCHAR), int64 (HUGEINT), intWidenedToJson (VARCHAR), numericString (VARCHAR), requiredNumeric (VARCHAR), stringWidenedToJson (VARCHAR), timeValue (VARCHAR)
91-
1, false, 2024-01-01T01:01:01.111111Z, {}, 2024-09-13T01:01:01Z, <nil>, <nil>, <nil>, test, {}, true, 2024-01-01, 1, 999, 123, 456, "hello", 01:01:01
98+
key (VARCHAR), _meta/flow_truncated (BOOLEAN), datetimeValue (TIMESTAMPTZ), flow_document (VARCHAR), flow_published_at (TIMESTAMPTZ), multiple (VARCHAR), nonScalarValue (VARCHAR), optional (VARCHAR), scalarValue (VARCHAR), second_root (VARCHAR), boolWidenedToJson (VARCHAR), dateValue (VARCHAR), int64 (HUGEINT), int64ToNumber (DOUBLE), intToNumber (DOUBLE), intWidenedToJson (VARCHAR), numericString (VARCHAR), requiredNumeric (VARCHAR), stringWidenedToJson (VARCHAR), timeValue (VARCHAR)
99+
1, false, 2024-01-01T01:01:01.111111Z, {}, 2024-09-13T01:01:01Z, <nil>, <nil>, <nil>, test, {}, true, 2024-01-01, 1, 1e+19, 9.223372036854776e+18, 999, 123, 456, "hello", 01:01:01
92100

‎materialize-motherduck/sqlgen.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ var duckDialect = func() sql.Dialect {
4747
return sql.Dialect{
4848
MigratableTypes: sql.MigrationSpecs{
4949
"double": {sql.NewMigrationSpec([]string{"varchar"})},
50-
"bigint": {sql.NewMigrationSpec([]string{"hugeint", "varchar"})},
51-
"hugeint": {sql.NewMigrationSpec([]string{"varchar"})},
50+
"bigint": {sql.NewMigrationSpec([]string{"double", "hugeint", "varchar"})},
51+
"hugeint": {sql.NewMigrationSpec([]string{"double", "varchar"})},
5252
"date": {sql.NewMigrationSpec([]string{"varchar"})},
5353
"time": {sql.NewMigrationSpec([]string{"varchar"})},
5454
"*": {sql.NewMigrationSpec([]string{"json"}, sql.WithCastSQL(toJsonCast))},

0 commit comments

Comments
 (0)