build(docker): reorganize Dockerfile to be more cacheable & add regis…

…try build cache (risingwavelabs#12193)

.dockerignore

@@ -48,3 +48,6 @@ risedev-components.user.env
 riselab-components.user.env
 
 .git/
+
+Dockerfile
+.dockerignore

ci/scripts/docker.sh

@@ -7,9 +7,26 @@ ghcraddr="ghcr.io/risingwavelabs/risingwave"
 dockerhubaddr="risingwavelabs/risingwave"
 arch="$(uname -m)"
 
+echo "--- ghcr login"
+echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
+
+echo "--- dockerhub login"
+echo "$DOCKER_TOKEN" | docker login -u "risingwavelabs" --password-stdin
+
 # Build RisingWave docker image ${BUILDKITE_COMMIT}-${arch}
 echo "--- docker build and tag"
-docker build -f docker/Dockerfile --build-arg "GIT_SHA=${BUILDKITE_COMMIT}" -t "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" --target risingwave .
+docker buildx create \
+  --name container \
+  --driver=docker-container
+
+docker buildx build -f docker/Dockerfile \
+  --build-arg "GIT_SHA=${BUILDKITE_COMMIT}" -t "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" \
+  --progress plain \
+  --builder=container \
+  --load \
+  --cache-to "type=registry,ref=ghcr.io/risingwavelabs/risingwave-build-cache:${arch}" \
+  --cache-from "type=registry,ref=ghcr.io/risingwavelabs/risingwave-build-cache:${arch}" \
+  .
 
 echo "--- check the image can start correctly"
 container_id=$(docker run -d "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" playground)
@@ -25,12 +42,6 @@ fi
 echo "--- docker images"
 docker images
 
-echo "--- ghcr login"
-echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
-
-echo "--- dockerhub login"
-echo "$DOCKER_TOKEN" | docker login -u "risingwavelabs" --password-stdin
-
 echo "--- docker push to ghcr"
 docker push "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}"
 

docker/Dockerfile

@@ -17,32 +17,56 @@ COPY ./proto  /risingwave/proto
 
 RUN cd /risingwave/dashboard && npm i && npm run build-static && rm -rf node_modules
 
-FROM base AS builder
+FROM base AS rust-base
 
-RUN apt-get update && apt-get -y install make cmake protobuf-compiler curl bash lld maven unzip
+RUN apt-get -y install make cmake protobuf-compiler curl bash lld unzip
 
 SHELL ["/bin/bash", "-c"]
 
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
+ENV PATH /root/.cargo/bin/:$PATH
+ENV CARGO_INCREMENTAL=0
 
-RUN mkdir -p /risingwave
+COPY rust-toolchain rust-toolchain
 
-WORKDIR /risingwave
+# We need to add the `rustfmt` dependency, otherwise `risingwave_pb` will not compile
+RUN rustup self update \
+  && rustup set profile minimal \
+  && rustup show \
+  && rustup component add rustfmt
 
-COPY ./ /risingwave
 
-ENV PATH /root/.cargo/bin/:$PATH
+# TODO: cargo-chef doesn't work well now, because we update Cargo.lock very often.
+# We may consider sccache instead.
 
-ENV IN_CONTAINER=1
+# RUN cargo install --git https://github.com/xxchan/cargo-chef cargo-chef --locked --rev 11f9fed
 
+# FROM rust-base AS rust-planner
+
+# RUN mkdir -p /risingwave
+# WORKDIR /risingwave
+# COPY ./ /risingwave
+
+# RUN cargo chef prepare --recipe-path recipe.json
+
+# FROM rust-base AS rust-builder
+
+# RUN mkdir -p /risingwave
+# WORKDIR /risingwave
+
+# COPY --from=rust-planner /risingwave/recipe.json recipe.json
+
+# # Build dependencies - this can be cached if the dependencies don't change
+# RUN cargo chef cook --release --recipe-path recipe.json
+
+FROM rust-base AS rust-builder
+
+# Build application
 ARG GIT_SHA
 ENV GIT_SHA=$GIT_SHA
 
-# We need to add the `rustfmt` dependency, otherwise `risingwave_pb` will not compile
-RUN rustup self update \
-  && rustup set profile minimal \
-  && rustup show \
-  && rustup component add rustfmt
+COPY ./ /risingwave
+WORKDIR /risingwave
 
 RUN cargo fetch && \
   cargo build -p risingwave_cmd_all --release --features "rw-static-link" && \
@@ -53,9 +77,39 @@ RUN cargo fetch && \
   chmod +x /risingwave/bin/jeprof && \
   mkdir -p /risingwave/lib && cargo clean
 
-RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true -Djava.binding.release=true && \
-    mkdir -p /risingwave/bin/connector-node && \
-    tar -zxvf /risingwave/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz -C /risingwave/bin/connector-node
+FROM base AS java-planner
+
+RUN mkdir -p /risingwave
+WORKDIR /risingwave
+
+COPY java /risingwave/java/
+
+# Move java/**/pom.xml to poms/**/pom.xml
+RUN find . -name pom.xml -exec bash -c 'mkdir -p poms/$(dirname {}); mv {} poms/{}' \;
+
+# We use rust-maven-plugin to build java-binding. So it's FROM rust-base
+FROM rust-base AS java-builder
+
+RUN apt-get -y install maven
+
+RUN mkdir -p /risingwave
+WORKDIR /risingwave/java
+
+# 1. copy only poms
+COPY --from=java-planner /risingwave/poms /risingwave/java/
+
+# 2. start downloading dependencies
+RUN mvn dependency:go-offline --fail-never
+
+# 3. add all source code and start compiling
+# TODO: only add java related code so that changing rust code won't recompile java code
+# Currently java-binding depends on the workspace Cargo.toml, which depends on the whole rust codebase
+# Besides, rust-maven-plugin sets --target-dir, so the dependencies are built twice. How to dedup?
+COPY ./ /risingwave
+
+RUN mvn -B package -Dmaven.test.skip=true -Djava.binding.release=true && \
+  mkdir -p /risingwave/bin/connector-node && \
+  tar -zxvf /risingwave/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz -C /risingwave/bin/connector-node
 
 FROM base AS risingwave
 
@@ -66,18 +120,19 @@ RUN apt-get -y install gdb \
 
 RUN mkdir -p /risingwave/bin/connector-node && mkdir -p /risingwave/lib
 
-COPY --from=builder /risingwave/bin/risingwave /risingwave/bin/risingwave
-COPY --from=builder /risingwave/bin/risingwave.dwp /risingwave/bin/risingwave.dwp
-COPY --from=builder /risingwave/bin/connector-node /risingwave/bin/connector-node
+COPY --from=rust-builder /risingwave/bin/risingwave /risingwave/bin/risingwave
+COPY --from=rust-builder /risingwave/bin/risingwave.dwp /risingwave/bin/risingwave.dwp
+COPY --from=java-builder /risingwave/bin/connector-node /risingwave/bin/connector-node
 COPY --from=dashboard-builder /risingwave/dashboard/out /risingwave/ui
-COPY --from=builder /risingwave/bin/jeprof /usr/local/bin/jeprof
+COPY --from=rust-builder /risingwave/bin/jeprof /usr/local/bin/jeprof
 
 # Set default playground mode to docker-playground profile
 ENV PLAYGROUND_PROFILE docker-playground
 # Set default dashboard UI to local path instead of github proxy
 ENV RW_DASHBOARD_UI_PATH /risingwave/ui
 # Set default connector libs path
 ENV CONNECTOR_LIBS_PATH /risingwave/bin/connector-node/libs
+ENV IN_CONTAINER=1
 
 ENTRYPOINT [ "/risingwave/bin/risingwave" ]
 CMD [ "playground" ]