Skip to content

Commit 18a38ab

Browse files
shubham-ybshubham-yb
committed
Merge branch 'main' into shubham/resumption
2 parents e8bd070 + fa542b7 commit 18a38ab

22 files changed

+264
-40
lines changed

‎migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql

+1
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,7 @@ CREATE TABLE public.locations (
386386

387387
CREATE TABLE image (title text, raster lo);
388388

389+
CREATE TABLE employees (id INT PRIMARY KEY, salary INT);
389390
-- create table with multirange data types
390391

391392
-- Create tables with primary keys directly

‎migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql

+6-1
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,11 @@ CREATE VIEW public.orders_view AS
3434
orders.xmin AS transaction_id
3535
FROM public.orders;
3636

37+
CREATE VIEW top_employees_view AS SELECT * FROM (
38+
SELECT * FROM employees
39+
ORDER BY salary DESC
40+
FETCH FIRST 2 ROWS WITH TIES
41+
) AS top_employees;
3742
CREATE VIEW public.my_films_view AS
3843
SELECT jt.* FROM
3944
my_films,
@@ -43,4 +48,4 @@ SELECT jt.* FROM
4348
kind text PATH '$.kind',
4449
NESTED PATH '$.films[*]' COLUMNS (
4550
title text FORMAT JSON PATH '$.title' OMIT QUOTES,
46-
director text PATH '$.director' KEEP QUOTES))) AS jt;
51+
director text PATH '$.director' KEEP QUOTES))) AS jt;

‎migtests/tests/analyze-schema/expected_issues.json

+10
Original file line numberDiff line numberDiff line change
@@ -1931,6 +1931,16 @@
19311931
"GH": "https://github.com/yugabyte/yugabyte-db/issues/25318",
19321932
"MinimumVersionsFixedIn": null
19331933
},
1934+
{
1935+
"IssueType": "unsupported_features",
1936+
"ObjectType": "VIEW",
1937+
"ObjectName": "top_employees_view",
1938+
"Reason": "FETCH .. WITH TIES",
1939+
"SqlStatement": "CREATE VIEW top_employees_view AS SELECT * FROM (\n\t\t\tSELECT * FROM employees\n\t\t\tORDER BY salary DESC\n\t\t\tFETCH FIRST 2 ROWS WITH TIES\n\t\t) AS top_employees;",
1940+
"Suggestion": "No workaround available right now",
1941+
"GH": "",
1942+
"MinimumVersionsFixedIn": null
1943+
},
19341944
{
19351945
"IssueType": "unsupported_datatypes",
19361946
"ObjectType": "TABLE",

‎migtests/tests/analyze-schema/summary.json

+6-5
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,10 @@
2626
},
2727
{
2828
"ObjectType": "TABLE",
29-
"TotalCount": 57,
29+
"TotalCount": 58,
3030
"InvalidCount": 49,
31-
"ObjectNames": "image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" },
31+
"ObjectNames": "employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" },
32+
3233
{
3334
"ObjectType": "INDEX",
3435
"TotalCount": 43,
@@ -50,9 +51,9 @@
5051
},
5152
{
5253
"ObjectType": "VIEW",
53-
"TotalCount": 6,
54-
"InvalidCount": 6,
55-
"ObjectNames": "public.my_films_view, v1, v2, test, public.orders_view, view_name"
54+
"TotalCount": 7,
55+
"InvalidCount": 7,
56+
"ObjectNames": "public.my_films_view, v1, v2, test, public.orders_view, view_name, top_employees_view"
5657
},
5758
{
5859
"ObjectType": "TRIGGER",

‎migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json

+54-10
Original file line numberDiff line numberDiff line change
@@ -38,15 +38,15 @@
3838
},
3939
{
4040
"ObjectType": "SEQUENCE",
41-
"TotalCount": 41,
41+
"TotalCount": 43,
4242
"InvalidCount": 0,
43-
"ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq"
43+
"ObjectNames": "public.employeesforview_id_seq, schema2.employeesforview_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq"
4444
},
4545
{
4646
"ObjectType": "TABLE",
47-
"TotalCount": 80,
47+
"TotalCount": 82,
4848
"InvalidCount": 35,
49-
"ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2"
49+
"ObjectNames": "public.employeesforview, schema2.employeesforview, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2"
5050
},
5151
{
5252
"ObjectType": "INDEX",
@@ -73,9 +73,9 @@
7373
},
7474
{
7575
"ObjectType": "VIEW",
76-
"TotalCount": 8,
77-
"InvalidCount": 4,
78-
"ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4, public.view_explicit_security_invoker"
76+
"TotalCount": 10,
77+
"InvalidCount": 6,
78+
"ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4, public.view_explicit_security_invoker, schema2.top_employees_view, public.top_employees_view"
7979
},
8080
{
8181
"ObjectType": "TRIGGER",
@@ -162,6 +162,7 @@
162162
"schema2.products",
163163
"schema2.foo",
164164
"schema2.Case_Sensitive_Columns",
165+
"schema2.employeesforview",
165166
"schema2.with_example1",
166167
"test_views.xyz_mview",
167168
"test_views.view_table2",
@@ -183,9 +184,10 @@
183184
"schema2.int_multirange_table",
184185
"schema2.numeric_multirange_table",
185186
"schema2.timestamp_multirange_table",
186-
"schema2.timestamptz_multirange_table"
187+
"schema2.timestamptz_multirange_table",
188+
"public.employeesforview"
187189
],
188-
"ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 86 objects (78 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.",
190+
"ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 88 objects (80 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.",
189191
"ShardedTables": [
190192
"public.combined_tbl",
191193
"public.citext_type",
@@ -642,6 +644,20 @@
642644
],
643645
"MinimumVersionsFixedIn": null
644646
},
647+
{
648+
"FeatureName": "FETCH .. WITH TIES Clause",
649+
"Objects": [
650+
{
651+
"ObjectName": "public.top_employees_view",
652+
"SqlStatement": "CREATE VIEW public.top_employees_view AS\n SELECT id,\n first_name,\n last_name,\n salary\n FROM ( SELECT employeesforview.id,\n employeesforview.first_name,\n employeesforview.last_name,\n employeesforview.salary\n FROM public.employeesforview\n ORDER BY employeesforview.salary DESC\n FETCH FIRST 2 ROWS WITH TIES) top_employees;"
653+
},
654+
{
655+
"ObjectName": "schema2.top_employees_view",
656+
"SqlStatement": "CREATE VIEW schema2.top_employees_view AS\n SELECT id,\n first_name,\n last_name,\n salary\n FROM ( SELECT employeesforview.id,\n employeesforview.first_name,\n employeesforview.last_name,\n employeesforview.salary\n FROM schema2.employeesforview\n ORDER BY employeesforview.salary DESC\n FETCH FIRST 2 ROWS WITH TIES) top_employees;"
657+
}
658+
],
659+
"MinimumVersionsFixedIn": null
660+
},
645661
{
646662
"FeatureName": "Security Invoker Views",
647663
"Objects": [
@@ -938,7 +954,7 @@
938954
{
939955
"SchemaName": "public",
940956
"ObjectName": "employees",
941-
"RowCount": 5,
957+
"RowCount": 10,
942958
"ColumnCount": 4,
943959
"Reads": 0,
944960
"Writes": 10,
@@ -2069,6 +2085,20 @@
20692085
"ParentTableName": "schema2.mixed_data_types_table1",
20702086
"SizeInBytes": 8192
20712087
},
2088+
{
2089+
"SchemaName": "public",
2090+
"ObjectName": "employeesforview",
2091+
"RowCount": 0,
2092+
"ColumnCount": 4,
2093+
"Reads": 0,
2094+
"Writes": 0,
2095+
"ReadsPerSecond": 0,
2096+
"WritesPerSecond": 0,
2097+
"IsIndex": false,
2098+
"ObjectType": "",
2099+
"ParentTableName": null,
2100+
"SizeInBytes": 0
2101+
},
20722102
{
20732103
"SchemaName": "public",
20742104
"ObjectName": "employees3",
@@ -2167,6 +2197,20 @@
21672197
"ParentTableName": null,
21682198
"SizeInBytes": 0
21692199
},
2200+
{
2201+
"SchemaName": "schema2",
2202+
"ObjectName": "employeesforview",
2203+
"RowCount": 0,
2204+
"ColumnCount": 4,
2205+
"Reads": 0,
2206+
"Writes": 0,
2207+
"ReadsPerSecond": 0,
2208+
"WritesPerSecond": 0,
2209+
"IsIndex": false,
2210+
"ObjectType": "",
2211+
"ParentTableName": null,
2212+
"SizeInBytes": 0
2213+
},
21702214
{
21712215
"SchemaName": "schema2",
21722216
"ObjectName": "bigint_multirange_table",

‎migtests/tests/pg/assessment-report-test/pg_assessment_report.sql

+14
Original file line numberDiff line numberDiff line change
@@ -393,6 +393,20 @@ BEGIN
393393
END;
394394
$$ LANGUAGE plpgsql;
395395

396+
-- for FETCH .. WITH TIES
397+
CREATE TABLE employeesForView (
398+
id SERIAL PRIMARY KEY,
399+
first_name VARCHAR(50) NOT NULL,
400+
last_name VARCHAR(50) NOT NULL,
401+
salary NUMERIC(10, 2) NOT NULL
402+
);
403+
404+
CREATE VIEW top_employees_view AS SELECT * FROM (
405+
SELECT * FROM employeesForView
406+
ORDER BY salary DESC
407+
FETCH FIRST 2 ROWS WITH TIES
408+
) AS top_employees;
409+
396410
-- SECURITY INVOKER VIEW
397411
CREATE TABLE public.employees (
398412
employee_id SERIAL PRIMARY KEY,

‎yb-voyager/cmd/assessMigrationCommand.go

+1
Original file line numberDiff line numberDiff line change
@@ -1023,6 +1023,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem
10231023
unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, queryissue.SYSTEM_COLUMNS_NAME, "", schemaAnalysisReport, false, ""))
10241024
unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", schemaAnalysisReport, false, ""))
10251025
unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, ""))
1026+
unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(FETCH_WITH_TIES_FEATURE, "", queryissue.FETCH_WITH_TIES, schemaAnalysisReport, false, ""))
10261027
unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_QUERY_FUNCTIONS_NAME, "", queryissue.JSON_QUERY_FUNCTION, schemaAnalysisReport, false, ""))
10271028
unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, ""))
10281029
unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, ""))

‎yb-voyager/cmd/constants.go

+1
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,7 @@ const (
217217
BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE = "BEFORE ROW triggers on Partitioned tables"
218218
PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE = "Primary / Unique key constraints on complex datatypes"
219219
REGEX_FUNCTIONS_FEATURE = "Regex Functions"
220+
FETCH_WITH_TIES_FEATURE = "FETCH .. WITH TIES Clause"
220221

221222
// Migration caveats
222223

‎yb-voyager/cmd/exportSchema.go

+7-8
Original file line numberDiff line numberDiff line change
@@ -90,10 +90,15 @@ func exportSchema() error {
9090
utils.PrintAndLog("Schema is not exported yet. Ignoring --start-clean flag.\n\n")
9191
}
9292
CreateMigrationProjectIfNotExists(source.DBType, exportDir)
93-
93+
err := retrieveMigrationUUID()
94+
if err != nil {
95+
log.Errorf("failed to get migration UUID: %v", err)
96+
return fmt.Errorf("failed to get migration UUID: %w", err)
97+
}
98+
9499
utils.PrintAndLog("export of schema for source type as '%s'\n", source.DBType)
95100
// Check connection with source database.
96-
err := source.DB().Connect()
101+
err = source.DB().Connect()
97102
if err != nil {
98103
log.Errorf("failed to connect to the source db: %s", err)
99104
return fmt.Errorf("failed to connect to the source db: %w", err)
@@ -153,12 +158,6 @@ func exportSchema() error {
153158
}
154159
}
155160

156-
err = retrieveMigrationUUID()
157-
if err != nil {
158-
log.Errorf("failed to get migration UUID: %v", err)
159-
return fmt.Errorf("failed to get migration UUID: %w", err)
160-
}
161-
162161
exportSchemaStartEvent := createExportSchemaStartedEvent()
163162
controlPlane.ExportSchemaStarted(&exportSchemaStartEvent)
164163

0 commit comments

Comments
 (0)