Make export-dir, source and target databases dynamic to enable parallel runs

.github/workflows/mysql-migtests.yml

@@ -137,6 +137,7 @@ jobs:
 
       # Placeholder for now so that a basic test can run
       - name: Create the live migration user
+        if: always()
         run: |
           mysql -uroot -proot -e 'GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'ybvoyager'@'127.0.0.1';'
 

migtests/lib/yb.py

@@ -52,7 +52,8 @@ def new_source_db():
 
 def verify_colocation(tgt, source_db_type):
 	print("Verifying the colocation of the tables")
-	json_file = "export-dir/assessment/reports/migration_assessment_report.json"
+	export_dir = os.getenv("EXPORT_DIR", "export-dir")
+	json_file = f"{export_dir}/assessment/reports/migration_assessment_report.json"
 
 	sharded_tables, colocated_tables = fetch_sharded_and_colocated_tables(json_file)
 

migtests/scripts/add-pk-from-alter-to-create

@@ -5,7 +5,7 @@ import os
 import re
 import shutil
 
-table_file_path = os.getenv('TEST_DIR')+'/export-dir/schema/tables/table.sql'
+table_file_path = os.getenv('EXPORT_DIR')+'/schema/tables/table.sql'
 
 #copy the table_file_path to table_file_path+'.old'
 src = table_file_path

migtests/scripts/functions.sh

@@ -134,7 +134,8 @@ grant_user_permission_oracle(){
 	*/
 	GRANT FLASHBACK ANY TABLE TO ybvoyager;
 EOF
-	run_sqlplus_as_sys ${db_name} "oracle-inputs.sql"	
+	run_sqlplus_as_sys ${db_name} "oracle-inputs.sql"
+	rm oracle-inputs.sql
 
 }
 
@@ -149,6 +150,8 @@ EOF
 	run_sqlplus_as_sys ${pdb_name} "create-pdb-tablespace.sql"
 	cp ${SCRIPTS}/oracle/live-grants.sql oracle-inputs.sql
 	run_sqlplus_as_sys ${cdb_name} "oracle-inputs.sql"
+	rm create-pdb-tablespace.sql
+	rm oracle-inputs.sql
 }
 
 grant_permissions_for_live_migration_pg() {
@@ -191,7 +194,7 @@ run_sqlplus_as_sys() {
 run_sqlplus_as_schema_owner() {
     db_name=$1
     sql=$2
-    conn_string="${SOURCE_DB_USER_SCHEMA_OWNER}/${SOURCE_DB_USER_SCHEMA_OWNER_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${db_name}"
+    conn_string="${SOURCE_DB_SCHEMA}/${SOURCE_DB_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${db_name}"
     echo exit | sqlplus -f "${conn_string}" @"${sql}"
 }
 
@@ -623,16 +626,6 @@ get_value_from_msr(){
   echo $val
 }
 
-create_ff_schema(){
-	db_name=$1
-
-	cat > create-ff-schema.sql << EOF
-	CREATE USER FF_SCHEMA IDENTIFIED BY "password";
-	GRANT all privileges to FF_SCHEMA;
-EOF
-	run_sqlplus_as_sys ${db_name} "create-ff-schema.sql"
-}
-
 set_replica_identity(){
 	db_schema=$1
     cat > alter_replica_identity.sql <<EOF
@@ -647,6 +640,7 @@ set_replica_identity(){
     END \$CUSTOM\$;
 EOF
     run_psql ${SOURCE_DB_NAME} "$(cat alter_replica_identity.sql)"
+	rm alter_replica_identity.sql
 }
 
 grant_permissions_for_live_migration() {
@@ -673,7 +667,15 @@ grant_permissions_for_live_migration() {
 setup_fallback_environment() {
 	if [ "${SOURCE_DB_TYPE}" = "oracle" ]; then
 		run_sqlplus_as_sys ${SOURCE_DB_NAME} ${SCRIPTS}/oracle/create_metadata_tables.sql
-		run_sqlplus_as_sys ${SOURCE_DB_NAME} ${SCRIPTS}/oracle/fall_back_prep.sql
+
+		TEMP_SCRIPT="/tmp/fall_back_prep.sql"
+
+		sed "s/TEST_SCHEMA/${SOURCE_DB_SCHEMA}/g" ${SCRIPTS}/oracle/fall_back_prep.sql > $TEMP_SCRIPT
+
+		run_sqlplus_as_sys ${SOURCE_DB_NAME} $TEMP_SCRIPT
+
+		# Clean up the temporary file after execution
+		rm -f $TEMP_SCRIPT
 		elif [ "${SOURCE_DB_TYPE}" = "postgresql" ]; then
 		conn_string="postgresql://${SOURCE_DB_ADMIN_USER}:${SOURCE_DB_ADMIN_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${SOURCE_DB_NAME}"
 		psql "${conn_string}" -v voyager_user="${SOURCE_DB_USER}" -v schema_list="${SOURCE_DB_SCHEMA}" -v replication_group='replication_group' -v is_live_migration=1 -v is_live_migration_fall_back=1 -f /opt/yb-voyager/guardrails-scripts/yb-voyager-pg-grant-migration-permissions.sql
@@ -1044,3 +1046,60 @@ cutover_to_target() {
 
     yb-voyager initiate cutover to target ${args} $*
 }
+
+create_source_db() {
+	source_db=$1
+	case ${SOURCE_DB_TYPE} in
+		postgresql)
+			run_psql postgres "DROP DATABASE IF EXISTS ${source_db};"
+			run_psql postgres "CREATE DATABASE ${source_db};"
+			;;
+		mysql)
+			run_mysql mysql "DROP DATABASE IF EXISTS ${source_db};"
+			run_mysql mysql "CREATE DATABASE ${source_db};"
+			;;
+		oracle)
+			cat > create-oracle-schema.sql << EOF
+			CREATE USER ${source_db} IDENTIFIED BY "password";
+			GRANT all privileges to ${source_db};
+EOF
+			run_sqlplus_as_sys ${SOURCE_DB_NAME} "create-oracle-schema.sql"
+			rm create-oracle-schema.sql
+			;;
+		*)
+			echo "ERROR: Source DB not created for ${SOURCE_DB_TYPE}"
+			exit 1
+			;;
+	esac
+}
+
+normalize_and_export_vars() {
+    local test_suffix=$1
+
+    # Normalize TEST_NAME
+	# Keeping the full name for PG and MySQL to test out large schema/export dir names
+    export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')"
+
+    # Set EXPORT_DIR
+    export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_${test_suffix}_export-dir"}
+    if [ -n "${SOURCE_DB_SSL_MODE}" ]; then
+        EXPORT_DIR="${EXPORT_DIR}_ssl"
+    fi
+
+    # Set database-specific variables
+    case "${SOURCE_DB_TYPE}" in
+        postgresql|mysql)
+            export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_${test_suffix}"}
+            ;;
+        oracle)
+            # Limit schema name to 10 characters for Oracle/Debezium due to 30 character limit
+			# Since test_suffix is the unique identifying factor, we need to add it post all the normalization 
+            export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_${test_suffix}"}
+            export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^}
+            ;;
+        *)
+            echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}"
+            exit 1
+            ;;
+    esac
+}

migtests/scripts/live-migration-fallb-run-test.sh

@@ -18,9 +18,7 @@ export REPO_ROOT="${PWD}"
 export SCRIPTS="${REPO_ROOT}/migtests/scripts"
 export TESTS_DIR="${REPO_ROOT}/migtests/tests"
 export TEST_DIR="${TESTS_DIR}/${TEST_NAME}"
-export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"}
 export QUEUE_SEGMENT_MAX_BYTES=400
-
 export PYTHONPATH="${REPO_ROOT}/migtests/lib"
 
 # Order of env.sh import matters.
@@ -37,10 +35,12 @@ else
 	source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh
 fi
 
-source ${SCRIPTS}/yugabytedb/env.sh
-
 source ${SCRIPTS}/functions.sh
 
+normalize_and_export_vars "fallb"
+
+source ${SCRIPTS}/yugabytedb/env.sh
+
 main() {
 
 	echo "Deleting the parent export-dir present in the test directory"
@@ -56,6 +56,10 @@ main() {
 	pushd ${TEST_DIR}
 
 	step "Initialise source database."
+	if [ "${SOURCE_DB_TYPE}" = "oracle" ]
+	then
+		create_source_db ${SOURCE_DB_SCHEMA}
+	fi
 	./init-db
 
 	step "Grant source database user permissions for live migration"	
@@ -169,7 +173,11 @@ main() {
 	import_schema --post-snapshot-import true --refresh-mviews=true
 
 	step "Run snapshot validations."
-	"${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'false' --fb_enabled 'true'
+	"${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'false' --fb_enabled 'true' || {
+			tail_log_file "yb-voyager-import-data.log"
+			tail_log_file "yb-voyager-export-data-from-source.log"
+			exit 1
+		} 
 
 	step "Inserting new events"
 	run_sql_file source_delta.sql
@@ -264,7 +272,7 @@ main() {
 
 	step "Clean up"
 	./cleanup-db
-	rm -rf "${EXPORT_DIR}/*"
+	rm -rf "${EXPORT_DIR}"
 	run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};"
 }
 

migtests/scripts/live-migration-fallf-run-test.sh

@@ -18,7 +18,6 @@ export REPO_ROOT="${PWD}"
 export SCRIPTS="${REPO_ROOT}/migtests/scripts"
 export TESTS_DIR="${REPO_ROOT}/migtests/tests"
 export TEST_DIR="${TESTS_DIR}/${TEST_NAME}"
-export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"}
 
 export PYTHONPATH="${REPO_ROOT}/migtests/lib"
 export PATH="${PATH}:/usr/lib/oracle/21/client64/bin"
@@ -38,12 +37,14 @@ else
     source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh
 fi
 
+source ${SCRIPTS}/functions.sh
+
+normalize_and_export_vars "fallf"
+
 source ${SCRIPTS}/${SOURCE_DB_TYPE}/ff_env.sh
 
 source ${SCRIPTS}/yugabytedb/env.sh
 
-source ${SCRIPTS}/functions.sh
-
 main() {
 
 	echo "Deleting the parent export-dir present in the test directory"
@@ -62,7 +63,9 @@ main() {
 
 	if [ "${SOURCE_DB_TYPE}" = "oracle" ]
 	then
-		create_ff_schema ${SOURCE_REPLICA_DB_NAME}
+		create_source_db ${SOURCE_DB_SCHEMA}
+		# TODO: Add dynamic Fall Forward schema creation. Currently using the same name for all tests.
+		create_source_db ${SOURCE_REPLICA_DB_SCHEMA}
 		run_sqlplus_as_sys ${SOURCE_REPLICA_DB_NAME} ${SCRIPTS}/oracle/create_metadata_tables.sql
 	fi
 	./init-db
@@ -206,7 +209,12 @@ main() {
 	import_schema --post-snapshot-import true --refresh-mviews true
 
 	step "Run snapshot validations."
-	"${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'true' --fb_enabled 'false'
+	"${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'true' --fb_enabled 'false' || {
+			tail_log_file "yb-voyager-import-data.log"
+			tail_log_file "yb-voyager-export-data-from-source.log"
+			tail_log_file "yb-voyager-import-data-to-source-replica.log"
+			exit 1
+		}
 
 	step "Inserting new events to source"
 	run_sql_file source_delta.sql
@@ -286,7 +294,7 @@ main() {
 
 	step "Clean up"
 	./cleanup-db
-	rm -rf "${EXPORT_DIR}/*"
+	rm -rf "${EXPORT_DIR}"
 	run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};"
 }
 

migtests/scripts/live-migration-run-test.sh

@@ -18,7 +18,6 @@ export REPO_ROOT="${PWD}"
 export SCRIPTS="${REPO_ROOT}/migtests/scripts"
 export TESTS_DIR="${REPO_ROOT}/migtests/tests"
 export TEST_DIR="${TESTS_DIR}/${TEST_NAME}"
-export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"}
 export QUEUE_SEGMENT_MAX_BYTES=400
 
 export PYTHONPATH="${REPO_ROOT}/migtests/lib"
@@ -37,10 +36,12 @@ else
 	source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh
 fi
 
-source ${SCRIPTS}/yugabytedb/env.sh
-
 source ${SCRIPTS}/functions.sh
 
+normalize_and_export_vars "live"
+
+source ${SCRIPTS}/yugabytedb/env.sh
+
 main() {
 
 	echo "Deleting the parent export-dir present in the test directory"
@@ -56,6 +57,10 @@ main() {
 	pushd ${TEST_DIR}
 
 	step "Initialise source database."
+	if [ "${SOURCE_DB_TYPE}" = "oracle" ]
+	then
+		create_source_db ${SOURCE_DB_SCHEMA}
+	fi
 	./init-db
 
 	step "Grant source database user permissions for live migration"
@@ -218,7 +223,7 @@ main() {
 
 	step "Clean up"
 	./cleanup-db
-	rm -rf "${EXPORT_DIR}/*"
+	rm -rf "${EXPORT_DIR}"
 	run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};"
 }
 

migtests/scripts/run-schema-migration.sh

@@ -139,7 +139,7 @@ main() {
 
 	step "Clean up"
 	./cleanup-db
-	rm -rf "${EXPORT_DIR}/*"
+	rm -rf "${EXPORT_DIR}"
 	run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};"
 }
 

migtests/scripts/run-test-export-data.sh

@@ -74,7 +74,7 @@ main() {
 
 	step "Clean up"
 	./cleanup-db
-	rm -rf "${EXPORT_DIR}/*"
+	rm -rf "${EXPORT_DIR}"
 	run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};"
 }
 

migtests/scripts/run-test.sh

@@ -17,7 +17,6 @@ export REPO_ROOT="${PWD}"
 export SCRIPTS="${REPO_ROOT}/migtests/scripts"
 export TESTS_DIR="${REPO_ROOT}/migtests/tests"
 export TEST_DIR="${TESTS_DIR}/${TEST_NAME}"
-export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"}
 
 export PYTHONPATH="${REPO_ROOT}/migtests/lib"
 
@@ -33,11 +32,16 @@ then
 else
 	source ${TEST_DIR}/env.sh
 fi
+
 source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh
-source ${SCRIPTS}/yugabytedb/env.sh
+
 
 source ${SCRIPTS}/functions.sh
 
+normalize_and_export_vars "offline"
+
+source ${SCRIPTS}/yugabytedb/env.sh
+
 main() {
 	echo "Deleting the parent export-dir present in the test directory"
 	rm -rf ${EXPORT_DIR}	
@@ -52,6 +56,18 @@ main() {
 	pushd ${TEST_DIR}
 
 	step "Initialise source database."
+	if [[ "${SKIP_DB_CREATION}" != "true" ]]; then
+	    if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then
+	        create_source_db "${SOURCE_DB_NAME}"
+	    elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then
+	        create_source_db "${SOURCE_DB_SCHEMA}"
+	    else
+	        echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}"
+	        exit 1
+	    fi
+	else
+	    echo "Skipping database creation as SKIP_DB_CREATION is set to true."
+	fi
 	./init-db
 
 	step "Grant source database user permissions"
@@ -190,7 +206,7 @@ main() {
 
 	step "Clean up"
 	./cleanup-db
-	rm -rf "${EXPORT_DIR}/*"
+	rm -rf "${EXPORT_DIR}"
 	run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};"
 }
 

migtests/scripts/run-validate-assessment-report.sh

@@ -94,7 +94,7 @@ main() {
 
 	step "Clean up"
 	./cleanup-db
-	rm -rf "${EXPORT_DIR}/*"
+	rm -rf "${EXPORT_DIR}"
 }
 
 main