chore: make risedev tasks more readable (risingwavelabs#10279)

Co-authored-by: Noel Kwan <[email protected]>

ci/Dockerfile

@@ -37,7 +37,7 @@ ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 
 # install build tools
 RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
-RUN cargo binstall -y --no-symlinks cargo-llvm-cov cargo-nextest cargo-udeps cargo-hakari cargo-sort cargo-make cargo-cache cargo-audit \
+RUN cargo binstall -y --no-symlinks cargo-llvm-cov cargo-nextest cargo-udeps cargo-hakari cargo-sort cargo-make@0.36.10 cargo-cache cargo-audit \
     && cargo install sccache --locked \
     && cargo install --git https://github.com/risinglightdb/sqllogictest-rs --rev dbadddb --bin sqllogictest --locked \
     && cargo cache -a \

ci/build-ci-image.sh

@@ -13,7 +13,7 @@ cat ../rust-toolchain
 # !!! CHANGE THIS WHEN YOU WANT TO BUMP CI IMAGE !!! #
 #          AND ALSO docker-compose.yml               #
 ######################################################
-export BUILD_ENV_VERSION=v20230531
+export BUILD_ENV_VERSION=v20230611
 
 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}"
 

ci/docker-compose.yml

@@ -70,7 +70,7 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230531
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230611
     depends_on:
       - mysql
       - db
@@ -79,20 +79,20 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230531
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230611
     depends_on:
       - mysql
       - db
     volumes:
       - ..:/risingwave
 
   rw-build-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230531
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230611
     volumes:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230531
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230611
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -103,7 +103,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230531
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230611
     depends_on:
       db:
         condition: service_healthy

ci/scripts/common.sh

@@ -10,8 +10,7 @@ export SCCACHE_REGION=us-east-2
 export SCCACHE_IDLE_TIMEOUT=0
 export CARGO_INCREMENTAL=0
 export CARGO_MAKE_PRINT_TIME_SUMMARY=true
-
-
+unset LANG
 if [ -n "${BUILDKITE_COMMIT:-}" ]; then
   export GIT_SHA=$BUILDKITE_COMMIT
 fi
@@ -82,7 +81,7 @@ function download_and_prepare_rw() {
   echo -e "\033[33mPrepare RiseDev dev cluster\033[0m"
 
   cargo make pre-start-dev
-  cargo make link-all-in-one-binaries
+  cargo make --allow-private link-all-in-one-binaries
 }
 
 # Arguments:

ci/scripts/pulsar-source-test.sh

@@ -39,7 +39,7 @@ cp ci/risedev-components.ci.env risedev-components.user.env
 
 echo "--- Prepare RiseDev dev cluster"
 cargo make pre-start-dev
-cargo make link-all-in-one-binaries
+cargo make --allow-private link-all-in-one-binaries
 
 echo "--- starting risingwave cluster with connector node"
 cargo make ci-start ci-1cn-1fe

risedev

@@ -1,21 +1,23 @@
 #!/usr/bin/env bash
 
 if [ -z "$(which cargo-binstall)" ]; then
-    cargo install cargo-quickinstall --locked
-    cargo quickinstall cargo-binstall
+    echo "Installing cargo-binstall..."
+    curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
 fi
 
 if [ -z "$(which cargo-make)" ]; then
     echo "Installing cargo-make..."
-    cargo binstall cargo-make --version "^0.36.6" --locked # >= 0.36.6, < 0.37.0
+    cargo binstall cargo-make --version "^0.36.10" --locked # >= 0.36.10, < 0.37.0
 fi
 
 touch risedev-components.user.env
 
 if [ $# -eq 0 ] || [ "$1" == "-h" ] || [ "$1" ==  "--help" ]; then
-    cargo make --list-all-steps
+    cargo make --list-all-steps --hide-uninteresting
     exit 0
 fi
 
-cargo make configure-if-not-configured
-cargo make "$@"
+cargo make --allow-private configure-if-not-configured
+# We marked many tasks as private, so we can have a more concise output when listing all tasks.
+# But we allow private tasks to be executed.
+cargo make --allow-private "$@"

src/frontend/planner_test/planner_test.toml

@@ -1,10 +1,3 @@
-[tasks.apply-planner-test]
-description = "Generate planner test data"
-script = '''
-echo "Deprecated."
-echo "Tip: use $(tput setaf 4)./risedev do-apply-planner-test$(tput sgr0) or alias $(tput setaf 4)./risedev dapt$(tput sgr0)."
-'''
-
 [tasks.do-apply-planner-test]
 description = "Apply planner test data"
 category = "RiseDev - Test"

src/java_binding/make-java-binding.toml

@@ -1,4 +1,5 @@
 [tasks.gen-java-binding-header]
+category = "RiseDev - Java Binding"
 description = "Generate the java binding C header file"
 script = '''
 #!/usr/bin/env bash
@@ -8,6 +9,7 @@ javac -h . -cp java-binding/src/main/java/ java-binding/src/main/java/com/rising
 '''
 
 [tasks.build-java-binding-rust]
+category = "RiseDev - Java Binding"
 description = "Build the java binding rust code"
 script = '''
 #!/usr/bin/env bash
@@ -16,6 +18,7 @@ cargo build -p risingwave_java_binding
 '''
 
 [tasks.build-java-binding-java]
+category = "RiseDev - Java Binding"
 description = "Build the java binding java code"
 script = '''
 #!/usr/bin/env bash
@@ -26,6 +29,7 @@ mvn dependency:copy-dependencies --no-transfer-progress --pl java-binding-integr
 '''
 
 [tasks.start-java-binding-demo-cluster]
+category = "RiseDev - Java Binding"
 description = "Start the RisingWave cluster for java binding demo"
 script = '''
 #!/usr/bin/env bash
@@ -38,6 +42,7 @@ ${RISINGWAVE_ROOT}/risedev d java-binding-demo
 '''
 
 [tasks.kill-java-binding-demo-cluster]
+category = "RiseDev - Java Binding"
 description = "Kill the RisingWave cluster for java binding demo"
 script = '''
 #!/usr/bin/env bash
@@ -50,6 +55,7 @@ ${RISINGWAVE_ROOT}/risedev k > /dev/null
 '''
 
 [tasks.ingest-data-and-run-java-binding]
+category = "RiseDev - Java Binding"
 description = "Ingest test data and run java binding demo"
 script = '''
 #!/usr/bin/env bash
@@ -69,6 +75,7 @@ bash ${RUST_JAVA_BINDING_ROOT}/run_demo.sh
 '''
 
 [tasks.run-java-binding-demo]
+category = "RiseDev - Java Binding"
 description = "Run the java binding demo"
 dependencies = [
     "build-java-binding-rust",
@@ -79,6 +86,7 @@ dependencies = [
 ]
 
 [tasks.run-java-binding-stream-chunk-demo]
+category = "RiseDev - Java Binding"
 description = "Run the java binding stream chunk demo"
 dependencies = [
     "build-java-binding-rust",
@@ -95,6 +103,4 @@ cd ${RISINGWAVE_ROOT}/java
 (cd ${RISINGWAVE_ROOT} && cargo run --bin data-chunk-payload-generator) | \
     java -cp "./java-binding-integration-test/target/dependency/*:./java-binding-integration-test/target/classes" \
      -Djava.library.path=${RISINGWAVE_ROOT}/target/debug  com.risingwave.java.binding.StreamChunkDemo
-
-
 '''

src/risedevtool/common.toml

@@ -37,6 +37,7 @@ exit 1
 '''
 
 [tasks.prepare]
+private = true
 category = "Misc"
 description = "Create .risingwave folder for storing temporary files and data"
 dependencies = ["prepare-dir", "check-risedev-configured"]

src/risedevtool/connector.toml

@@ -5,6 +5,7 @@ MAVEN_VERSION = "3.9.0"
 MAVEN_DOWNLOAD_PATH = "${PREFIX_TMP}/maven.tar.gz"
 
 [tasks.download-maven]
+private = true
 category = "RiseDev - Components"
 dependencies = ["prepare"]
 condition = { env_set = [ "ENABLE_BUILD_RW_CONNECTOR" ] }

src/risedevtool/etcd.toml

@@ -10,6 +10,7 @@ ETCD_DOWNLOAD_PATH_OTHER = "${PREFIX_TMP}/etcd.zip"
 ETCD_RELEASE = "etcd-${ETCD_VER}-${ETCD_SYSTEM}"
 
 [tasks.download-etcd]
+private = true
 category = "RiseDev - Components"
 dependencies = ["prepare"]
 condition = { env_set = [ "ENABLE_ETCD" ] }
@@ -37,7 +38,8 @@ mv "${PREFIX_TMP}/${ETCD_RELEASE}" "${PREFIX_BIN}/etcd"
 '''
 
 [tasks.clean-etcd-data]
-category = "RiseDev - Components"
+private = true
+category = "Misc"
 description = "Clean etcd data"
 condition = { env_set = [ "PREFIX_DATA" ] }
 script = '''

src/risedevtool/gcloud-pubsub.toml

@@ -8,6 +8,7 @@ GCLOUD_DOWNLOAD_TGZ = "https://dl.google.com/dl/cloudsdk/channels/rapid/download
 GCLOUD_SDK_DIR = "google-cloud-sdk"
 
 [tasks.download-pubsub]
+private = true
 category = "RiseDev - Components"
 dependencies = ["prepare"]
 condition = { env_set = [ "ENABLE_PUBSUB" ] }

src/risedevtool/grafana.toml

@@ -8,6 +8,7 @@ GRAFANA_RELEASE = "grafana-${GRAFANA_VERSION}"
 GRAFANA_DOWNLOAD_TAR_GZ = "https://dl.grafana.com/oss/release/${GRAFANA_RELEASE}.${GRAFANA_SYSTEM}.tar.gz"
 
 [tasks.download-grafana]
+private = true
 category = "RiseDev - Components"
 dependencies = ["prepare"]
 condition = { env_set = [ "ENABLE_PROMETHEUS_GRAFANA" ] }

src/risedevtool/jaeger.toml

@@ -8,6 +8,7 @@ JAEGER_RELEASE = "jaeger-${JAEGER_VERSION}-${JAEGER_SYSTEM}"
 JAEGER_DOWNLOAD_TAR_GZ = "https://github.com/jaegertracing/jaeger/releases/download/v${JAEGER_VERSION}/${JAEGER_RELEASE}.tar.gz"
 
 [tasks.download-jaeger]
+private = true
 category = "RiseDev - Components"
 dependencies = ["prepare"]
 condition = { env_set = [ "ENABLE_COMPUTE_TRACING" ] }

src/risedevtool/kafka.toml

@@ -8,6 +8,7 @@ KAFKA_RELEASE = "kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}"
 KAFKA_DOWNLOAD_TGZ = "https://dlcdn.apache.org/kafka/${KAFKA_VERSION}/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz"
 
 [tasks.download-kafka]
+private = true
 category = "RiseDev - Components"
 dependencies = ["prepare"]
 condition = { env_set = [ "ENABLE_KAFKA" ] }

src/risedevtool/minio.toml

@@ -6,6 +6,7 @@ MCLI_DOWNLOAD_BIN = "https://dl.min.io/client/mc/release/${MINIO_SYSTEM}/mc"
 MINIO_DOWNLOAD_BIN = "https://dl.min.io/server/minio/release/${MINIO_SYSTEM}/minio"
 
 [tasks.download-minio]
+private = true
 category = "RiseDev - Components"
 dependencies = ["prepare"]
 condition = { env_set = [ "ENABLE_MINIO" ] }
@@ -25,6 +26,7 @@ mv "${PREFIX_TMP}/minio" "${PREFIX_BIN}/minio"
 '''
 
 [tasks.download-mcli]
+private = true
 category = "RiseDev - Components"
 dependencies = ["prepare"]
 condition = { env_set = [ "ENABLE_MINIO" ] }

src/risedevtool/prometheus.toml

@@ -9,6 +9,7 @@ PROMETHEUS_RELEASE = "prometheus-${PROMETHEUS_VERSION}.${PROMETHEUS_SYSTEM}"
 PROMETHEUS_DOWNLOAD_TAR_GZ = "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/${PROMETHEUS_RELEASE}.tar.gz"
 
 [tasks.download-prometheus]
+private = true
 category = "RiseDev - Components"
 dependencies = ["prepare"]
 condition = { env_set = [ "ENABLE_PROMETHEUS_GRAFANA" ] }

src/risedevtool/redis.toml

@@ -7,6 +7,7 @@ REDIS_RELEASE = "redis-${REDIS_VERSION}"
 REDIS_DOWNLOAD_URL = "https://github.com/redis/redis/archive/${REDIS_VERSION}.tar.gz"
 
 [tasks.download-redis]
+private = true
 category = "RiseDev - Components"
 dependencies = ["prepare"]
 condition = { env_set = [ "ENABLE_REDIS" ] }

src/risedevtool/risedev-components.toml

@@ -1,4 +1,5 @@
 [tasks.configure-if-not-configured]
+private = true
 condition = { env_not_set = [ "RISEDEV_CONFIGURED" ] }
 script = '''
 #!/usr/bin/env bash
@@ -8,6 +9,8 @@ cargo run --bin risedev-config -- -f "${ENV_PATH}" default
 '''
 
 [tasks.configure]
+category = "RiseDev - Build"
+description = "Configure what components to download/build"
 script = '''
 #!/usr/bin/env bash
 set -e

src/sqlparser/test_runner/sqlparser_test.toml

@@ -8,6 +8,7 @@ cargo run --bin parser-test-apply
 '''
 
 [tasks.apply-parser-test]
+private = true
 description = "Generate parser test data"
 dependencies = [
   "update-parser-test"

src/stream/tests/integration_tests/integration_test.toml

@@ -1,5 +1,5 @@
 [tasks.do-apply-stream-integration-test]
-description = "Apply stream integration test output snapshots."
+description = "Apply stream integration test output snapshots"
 category = "RiseDev - Test"
 dependencies = ["install-nextest"]
 script = '''