Containerfile.llama-server

ARG LLAMA_SERVER_IMAGE_VERSION="server-cuda13-b9894@sha256:9b61117f56447863261f6ae0d15c7d330f219b9fbddeac1ccd336f6fc2b31e81"
ARG MODEL_HF_REPO="unsloth/Qwen3.5-4B-GGUF"
ARG MODEL_HF_FILE="Qwen3.5-4B-Q4_K_M.gguf"


FROM ghcr.io/ggml-org/llama.cpp:${LLAMA_SERVER_IMAGE_VERSION} AS llama_cpp


FROM cgr.dev/chainguard/wolfi-base AS huggingface
ARG MODEL_HF_REPO
ARG MODEL_HF_FILE

RUN apk add --no-cache curl bash python3
ENV HF_HOME=/hf
ENV HF_CLI_BIN_DIR=/usr/bin
RUN curl -LsSf https://hf.co/cli/install.sh | bash
RUN --mount=type=secret,id=HF_TOKEN \
    ls -l /run/secrets && \
    cat /run/secrets/HF_TOKEN | wc -c
RUN --mount=type=secret,id=HF_TOKEN \
    HF_TOKEN="$(cat /run/secrets/HF_TOKEN)" \
    hf download \
        --local-dir "/models/$MODEL_HF_REPO" \
        "$MODEL_HF_REPO" "$MODEL_HF_FILE"


FROM localhost/grate-base:latest
ARG LLAMA_URL="https://github.com/ggml-org/llama.cpp/releases/download/b9864/llama-b9864-bin-ubuntu-vulkan-x64.tar.gz"
ARG MODEL_HF_REPO
ARG MODEL_HF_FILE

USER root

RUN apk add --no-cache ca-certificates libstdc++ libgcc zlib
COPY --from=llama_cpp /usr/local/cuda/lib64 /usr/local/cuda/lib64
COPY --from=llama_cpp /usr/local/cuda/compat /usr/local/cuda/compat
ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/cuda/compat
ENV PATH=/usr/local/cuda/bin:${PATH}

RUN apk add --no-cache curl libgomp
COPY --from=llama_cpp /app /app
RUN chown -R grate:grate /app
ENV LD_LIBRARY_PATH=/app:${LD_LIBRARY_PATH}

COPY llama-server.sh /usr/bin
COPY --from=huggingface /models /usr/share/llama/models

USER grate:grate

VOLUME /models
EXPOSE 8080 8443

ENV LLAMA_BACKEND="cuda"
ENV LLAMA_MODEL_PATH="/models/${MODEL_HF_REPO}/${MODEL_HF_FILE}"
ENTRYPOINT ["/usr/bin/nono-wrapper.sh", "--gpu", "/usr/bin/llama-server.sh"]
CMD ["-c", "51200", "-ngl", "20"]