-
-
Notifications
You must be signed in to change notification settings - Fork 54
Expand file tree
/
Copy pathDockerfile
More file actions
138 lines (120 loc) · 6.61 KB
/
Copy pathDockerfile
File metadata and controls
138 lines (120 loc) · 6.61 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
# Cross-compiling multi-arch build with a durable cargo-chef dependency layer.
#
# The builder runs on the NATIVE build platform (`--platform=$BUILDPLATFORM`)
# and cross-compiles to the requested target arch. Previously the builder ran
# under QEMU for linux/arm64, which emulated the *entire* Rust compile and took
# ~2h per release. Cross-compiling on the native runner brings arm64 back to
# minutes; only the tiny runtime layer (ca-certificates) still touches QEMU.
#
# cargo-chef splits the expensive external-crate compile (aws-lc-sys, chromium/
# CDP, tokio, rustls, ...) into its own `cacher` stage keyed ONLY on the Cargo
# manifests (recipe.json). A source-only change leaves recipe.json byte-
# identical -> the cook is reused and only the workspace crates recompile.
#
# DURABILITY: prod (rolling-deploy.sh) bakes the `cacher` stage as a TAGGED
# image `crw-api-deps:<fp>` and overrides DEPS_STAGE to it, so `builder` runs
# `FROM crw-api-deps:<fp>`. Tagged image layers are referenced (not dangling)
# and are NOT build cache, so the nightly `docker image prune -f` +
# `docker builder prune --keep-storage 1GB` cannot evict them. CI leaves
# DEPS_STAGE=cacher (cooks inline; gha mode=max caches the cacher layer).
# Global ARG (before the first FROM) so it is usable in the `builder` FROM line.
ARG DEPS_STAGE=cacher
# Which workspace binaries to cook + build. Default = all three (the public
# open-core image ships crw + crw-server + crw-mcp). Prod overrides it to just
# `-p crw-server --features cdp`: the engine container only runs crw-server
# (CMD), so building crw-cli + crw-mcp there is wasted work — dropping them
# removes two of three thin-LTO links (faster + far less CPU contention with the
# live engine). MUST be identical in the cook and the build (fingerprint match).
ARG CARGO_PKGS="-p crw-server --features cdp -p crw-mcp -p crw-cli"
# Cap on compile parallelism. cargo reads CARGO_BUILD_JOBS natively (BuildKit
# injects a declared ARG into the RUN env), so no -j flag is needed. "default" =
# all cores (CI); prod passes a number below the core count so a build never
# starves the live engine of CPU. Must NOT be empty — cargo rejects an empty
# value ("could not parse ``").
ARG CARGO_BUILD_JOBS=default
# ---- shared toolchain base --------------------------------------------------
FROM --platform=$BUILDPLATFORM rust:1.97-bookworm@sha256:606f3248aa86ce49e0b98d9e0bbffde042adeb18982320f97bcc218615de1c99 AS chef
# Provided automatically by buildx: amd64 | arm64.
ARG TARGETARCH
WORKDIR /app
# Rust target + (arm64) cross linker toolchain; record the target triple.
RUN set -eux; \
case "$TARGETARCH" in \
amd64) RUST_TARGET=x86_64-unknown-linux-gnu ;; \
arm64) RUST_TARGET=aarch64-unknown-linux-gnu; \
apt-get update; \
# crossbuild-essential-arm64 = the aarch64 gcc/g++ AND the target
# libc dev headers (libc6-dev-arm64-cross). The bare cross gcc
# alone lacks sys/types.h etc., which broke aws-lc-sys's C build.
apt-get install -y --no-install-recommends crossbuild-essential-arm64; \
rm -rf /var/lib/apt/lists/* ;; \
*) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 1 ;; \
esac; \
rustup target add "$RUST_TARGET"; \
echo "$RUST_TARGET" > /rust_target
# cargo-chef as a host (BUILDPLATFORM) tool. Pinned so the install layer is
# deterministic; must be a release that parses this edition-2024 / resolver-2
# workspace (verify `cargo chef prepare` succeeds before merge — see PR checklist).
RUN cargo install cargo-chef --locked --version 0.1.77
# Shared build env — MUST be identical in `cacher` (cook) and `builder` (final
# compile) or the cooked deps get a different cargo fingerprint and recompile.
# Baked into `chef` so both inherit it verbatim (and so the tagged deps image
# carries it too).
# - Linker for the aarch64 cross target (ignored on native amd64).
# - Workspace release profile is fat LTO + codegen-units=1, whose final link
# of crw-server (aws-lc-sys + full graph) needs several GB and OOM-killed
# the build (#90). thin LTO + 16 CGUs: far lower peak memory, faster link,
# negligible runtime difference.
ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \
CARGO_PROFILE_RELEASE_LTO=thin \
CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
# ---- planner: derive the dependency recipe from the manifests only ----------
FROM chef AS planner
COPY . .
RUN cargo chef prepare --recipe-path /recipe.json
# ---- cacher: compile ONLY the dependency graph (the durable, taggable layer) -
# Keyed solely on /recipe.json; SAME --target/--features/-p set (CARGO_PKGS) as
# the real build so the cooked artifacts fingerprint-match and are reused.
FROM chef AS cacher
ARG CARGO_PKGS
ARG CARGO_BUILD_JOBS
COPY --from=planner /recipe.json /recipe.json
# $CARGO_PKGS is intentionally unquoted so it word-splits into cargo args.
# Parallelism comes from the CARGO_BUILD_JOBS env (injected from the ARG).
RUN set -eux; \
RUST_TARGET="$(cat /rust_target)"; \
cargo chef cook --release --target "$RUST_TARGET" \
$CARGO_PKGS \
--recipe-path /recipe.json
# ---- builder: compile the workspace crates on top of the cooked deps --------
# DEPS_STAGE defaults to the in-tree `cacher` (CI / any plain `docker build`
# cooks deps inline). Prod overrides it to the pinned crw-api-deps:<fp> image.
FROM ${DEPS_STAGE} AS builder
ARG CARGO_PKGS
ARG CARGO_BUILD_JOBS
COPY . .
# Build the CARGO_PKGS set, then copy whatever binaries were produced (the set
# is variable: all three by default, only crw-server in prod).
RUN set -eux; \
RUST_TARGET="$(cat /rust_target)"; \
cargo build --release --target "$RUST_TARGET" \
$CARGO_PKGS; \
mkdir -p /out; \
for b in crw crw-server crw-mcp; do \
f="target/${RUST_TARGET}/release/$b"; \
if [ -f "$f" ]; then cp "$f" /out/; fi; \
done; \
test -f /out/crw-server # crw-server is required in every build
# ---- runtime (unchanged) ----------------------------------------------------
FROM debian:bookworm-slim@sha256:7b140f374b289a7c2befc338f42ebe6441b7ea838a042bbd5acbfca6ec875818
RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/*
# Copy whatever binaries the builder produced (all three for the public image;
# just crw-server for the prod engine build). A directory copy tolerates the
# variable set — no per-binary COPY that would fail when a binary was skipped.
COPY --from=builder /out/ /usr/local/bin/
COPY config.default.toml /app/config.default.toml
COPY config.docker.toml /app/config.docker.toml
WORKDIR /app
LABEL io.modelcontextprotocol.server.name="io.github.us/crw"
EXPOSE 3000
CMD ["crw-server"]