Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,44 @@ RUN curl -LsSf https://astral.sh/uv/install.sh | sh \
&& ~/.local/bin/uv pip install --python /opt/sglang-env/bin/python "sglang==${SGLANG_VERSION}"

RUN /opt/sglang-env/bin/python -c "import sglang; print(sglang.__version__)" > /opt/sglang-env/version

# --- Diffusers variant ---
FROM llamacpp AS diffusers

ARG DIFFUSERS_VERSION=0.31.0
ARG TORCH_VERSION=2.5.1

USER root

RUN apt update && apt install -y \
python3 python3-venv python3-dev \
curl ca-certificates build-essential \
&& rm -rf /var/lib/apt/lists/*

RUN mkdir -p /opt/diffusers-env && chown -R modelrunner:modelrunner /opt/diffusers-env

USER modelrunner

# Install uv and diffusers as modelrunner user
RUN curl -LsSf https://astral.sh/uv/install.sh | sh \
&& ~/.local/bin/uv venv --python /usr/bin/python3 /opt/diffusers-env \
&& ~/.local/bin/uv pip install --python /opt/diffusers-env/bin/python \
"diffusers==${DIFFUSERS_VERSION}" \
"torch==${TORCH_VERSION}" \
"transformers" \
"accelerate" \
"safetensors" \
"fastapi" \
"uvicorn[standard]" \
"pillow"

# Determine the Python site-packages directory dynamically and copy the Python server code
RUN PYTHON_SITE_PACKAGES=$(/opt/diffusers-env/bin/python -c "import site; print(site.getsitepackages()[0])") && \
mkdir -p "$(dirname "$PYTHON_SITE_PACKAGES")" && \
cp -r python/diffusers_server "$PYTHON_SITE_PACKAGES/"

RUN /opt/diffusers-env/bin/python -c "import diffusers; print(diffusers.__version__)" > /opt/diffusers-env/version

FROM llamacpp AS final-llamacpp
# Copy the built binary from builder
COPY --from=builder /app/model-runner /app/model-runner
Expand All @@ -158,3 +196,7 @@ COPY --from=builder /app/model-runner /app/model-runner
FROM sglang AS final-sglang
# Copy the built binary from builder-sglang (without vLLM)
COPY --from=builder-sglang /app/model-runner /app/model-runner

FROM diffusers AS final-diffusers
# Copy the built binary from builder
COPY --from=builder /app/model-runner /app/model-runner
2 changes: 1 addition & 1 deletion backends_vllm_stub.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
)

func initVLLMBackend(log *logrus.Logger, modelManager *models.Manager) (inference.Backend, error) {
return nil, nil
return nil, nil // VLLM backend is disabled
}

func registerVLLMBackend(backends map[string]inference.Backend, backend inference.Backend) {
Expand Down
Loading