longbow-quarrel/Dockerfile.cuda at main · 23skdu/longbow-quarrel · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
# =============================================================================
# Longbow-Quarrel - NVIDIA CUDA Build
# =============================================================================
# This Dockerfile builds Quarrel with CUDA acceleration for NVIDIA GPUs.
# Requires NVIDIA Container Toolkit for GPU access.
#
# Build:    docker build -f Dockerfile.cuda -t longbow-quarrel:cuda .
# Run:      docker run --gpus all -it longbow-quarrel:cuda --model /data/model.gguf
# =============================================================================

# -----------------------------------------------------------------------------
# Build Stage: CUDA Compilation
# -----------------------------------------------------------------------------
FROM nvidia/cuda:12.4.0-devel-ubuntu22.04 AS cuda-builder

# Install build dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
    golang-go \
    git \
    build-essential \
    pkg-config \
    wget \
    libcudnn9-dev-cuda-12 \
    && rm -rf /var/lib/apt/lists/*

# Install CUDA stub library for linking
RUN echo "INPUT(-lcudart)" > /usr/lib/x86_64-linux-gnu/libcuda.a && \
    ln -sf /usr/lib/x86_64-linux-gnu/libcuda.a /usr/lib/x86_64-linux-gnu/libcuda.so

# Install a more recent Go version
RUN wget -q https://go.dev/dl/go1.26.3.linux-amd64.tar.gz && \
    rm -rf /usr/local/go && \
    tar -C /usr/local -xzf go1.26.3.linux-amd64.tar.gz && \
    rm go1.26.3.linux-amd64.tar.gz

ENV PATH=/usr/local/go/bin:$PATH

# Set environment variables for CUDA
ENV CUDA_PATH=/usr/local/cuda \
    PATH=/usr/local/cuda/bin:$PATH \
    GO111MODULE=on

ENV CGO_LDFLAGS="-L/usr/local/cuda/lib64 -L/usr/lib/x86_64-linux-gnu -lcudart -lcublas -lcudnn" \
    CGO_CFLAGS="-I/usr/local/cuda/include"

WORKDIR /app

# Copy go mod files
COPY go.mod go.sum ./
RUN go mod download

# Copy source code
COPY . .

# Build with CUDA support
RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -tags cuda \
    -o quarrel-cuda ./cmd/quarrel

# -----------------------------------------------------------------------------
# Runtime Stage: Minimal CUDA Runtime
# -----------------------------------------------------------------------------
FROM nvidia/cuda:12.4.0-runtime-ubuntu22.04

# Install cuDNN runtime
RUN apt-get update && apt-get install -y --no-install-recommends \
    libcudnn9-cuda-12 \
    && rm -rf /var/lib/apt/lists/*

WORKDIR /app

# Copy CUDA library stubs for runtime linking
COPY --from=cuda-builder /usr/local/cuda/lib64/libcudart.so* /usr/local/cuda/lib64/
COPY --from=cuda-builder /usr/local/cuda/lib64/libcuda.so* /usr/lib/x86_64-linux-gnu/

# Copy the CUDA binary and kernels
COPY --from=cuda-builder /app/quarrel-cuda ./quarrel
COPY --from=cuda-builder /app/internal/device/libcuda_kernels.so /usr/local/lib/libcuda_kernels.so

# Set library path
ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/cuda/lib64:/usr/lib/x86_64-linux-gnu

# Set default command
CMD ["./quarrel", "--help"]

# =============================================================================
# Alternative: Multi-stage with CPU fallback
# =============================================================================
# For deployments that need both CUDA and CPU backends, use:
#
# FROM nvidia/cuda:12.4-devel-ubuntu22.04 AS builder
# ... (same as above) ...
# RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -tags cuda \
#     -o quarrel ./cmd/simple
#
# FROM ubuntu:22.04
# RUN apt-get install -y --no-install-recommends libcuda1 libcublas-12-4
# COPY --from=builder /app/quarrel .
# CMD ["./quarrel"]
#
# =============================================================================
# GPU Requirements
# =============================================================================
# - NVIDIA Driver: 550.54.14+ (for CUDA 12.4)
# - NVIDIA Container Toolkit: v1.16.0+
# - Supported GPUs: Volta, Turing, Ampere, Ada Lovelace, Hopper
#
# Run with GPU access:
#   docker run --gpus all -v /path/to/models:/data longbow-quarrel:cuda \
#       --model /data/smollm2.gguf --prompt "Hello"
#
# =============================================================================