fix(ci): fix iceberg-cdc integration_tests (risingwavelabs#13082)

Co-authored-by: ZENOTME <[email protected]>
zwang28 · Oct 27, 2023 · f60f5b6 · f60f5b6
1 parent 31d6dc0
commit f60f5b6
Show file tree

Hide file tree

Showing 8 changed files with 28 additions and 13 deletions.
diff --git a/...ion_tests/iceberg-cdc/docker-compose.yaml → ...tion_tests/iceberg-cdc/docker-compose.yml b/...ion_tests/iceberg-cdc/docker-compose.yaml → ...tion_tests/iceberg-cdc/docker-compose.yml
@@ -79,6 +79,15 @@ services:
       - "./mysql_prepare.sql:/mysql_prepare.sql"
     container_name: prepare_mysql
     restart: on-failure
+  datagen:
+    build: ../datagen
+    depends_on: [mysql]
+    command:
+      - /bin/sh
+      - -c
+      - /datagen --mode clickstream --qps 1 mysql --user mysqluser --password mysqlpw --host mysql --port 3306 --db mydb
+    container_name: datagen
+    restart: on-failure
 
   rest:
     image: tabulario/iceberg-rest:0.6.0

diff --git a/integration_tests/iceberg-cdc/python/Dockerfile b/integration_tests/iceberg-cdc/python/Dockerfile
@@ -0,0 +1,12 @@
+FROM python:3.8
+
+RUN curl -sSL https://install.python-poetry.org | python3 -
+
+WORKDIR /app
+
+COPY script /app
+
+RUN $HOME/.local/bin/poetry update
+
+CMD $HOME/.local/bin/poetry run python init.py;sleep 40;$HOME/.local/bin/poetry run python check.py
+
diff --git a/...gration_tests/iceberg-cdc/python/check.py → ..._tests/iceberg-cdc/python/script/check.py b/...gration_tests/iceberg-cdc/python/check.py → ..._tests/iceberg-cdc/python/script/check.py
diff --git a/...ation_tests/iceberg-cdc/python/config.ini → ...ests/iceberg-cdc/python/script/config.ini b/...ation_tests/iceberg-cdc/python/config.ini → ...ests/iceberg-cdc/python/script/config.ini
@@ -1,8 +1,8 @@
 [spark]
-url=sc://localhost:15002
+url=sc://spark:15002
 
 [risingwave]
 db=dev
 user=root
-host=127.0.0.1
+host=frontend-node-0
 port=4566
diff --git a/integration_tests/iceberg-cdc/python/init.py → ...n_tests/iceberg-cdc/python/script/init.py b/integration_tests/iceberg-cdc/python/init.py → ...n_tests/iceberg-cdc/python/script/init.py
diff --git a/...n_tests/iceberg-cdc/python/pyproject.toml → .../iceberg-cdc/python/script/pyproject.toml b/...n_tests/iceberg-cdc/python/pyproject.toml → .../iceberg-cdc/python/script/pyproject.toml
diff --git a/integration_tests/iceberg-cdc/run_test.sh b/integration_tests/iceberg-cdc/run_test.sh
@@ -6,14 +6,8 @@ docker-compose up -d --wait
 # To avoid exiting by unhealth, set it after start environment.
 set -ex
 
-# Generate data
-docker build -t iceberg-cdc-datagen ../datagen
-timeout 20 docker run --network=iceberg-cdc_default iceberg-cdc-datagen /datagen --mode clickstream --qps 1 mysql --user mysqluser --password mysqlpw --host mysql --port 3306 --db mydb &
+sleep 20
+docker stop datagen
 
-cd python
-poetry update --quiet
-# Init source, mv, and sink.
-poetry run python init.py
-# Wait for sink to be finished.
-sleep 40;
-poetry run python check.py
+docker build -t iceberg-cdc-python python
+docker run --network=iceberg-cdc_default iceberg-cdc-python
diff --git a/integration_tests/scripts/check_data.py b/integration_tests/scripts/check_data.py
@@ -54,7 +54,7 @@ def run_psql(sql):
 
 demo = sys.argv[1]
 upstream = sys.argv[2]  # mysql, postgres, etc. see scripts/integration_tests.sh
-if demo in ['docker', 'iceberg-sink','clickhouse-sink']:
+if demo in ['docker', 'iceberg-sink','clickhouse-sink', 'iceberg-cdc']:
     print('Skip for running test for `%s`' % demo)
     sys.exit(0)