aikit/.github/workflows/test-packager-inference.yaml at 18ee1e135bb017c498b877f7742b86996bc83f41 · kaito-project/aikit · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
name: test-packager-inference

on:
  workflow_dispatch:
  push:
    paths-ignore:
      - '**.md'
      - 'website/**'
  pull_request:
    paths-ignore:
      - '**.md'
      - 'website/**'

permissions: read-all

jobs:
  build-and-infer:
    runs-on: ubuntu-latest-16-cores
    timeout-minutes: 60
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@58077d3c7e43986b6b15fba718e8ea69e387dfcc # v2.15.1
        with:
          egress-policy: audit
          allowed-endpoints: >
            auth.docker.io:443
            github.com:443
            *.githubusercontent.com:443
            proxy.golang.org:443
            registry-1.docker.io:443
            sum.golang.org:443
            *.ubuntu.com:80
            security.ubuntu.com:80
            ghcr.io:443
            huggingface.co:443

      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup buildx with host network access
        run: |
          docker buildx create --use --name host-network --driver-opt network=host || true
          docker buildx inspect --bootstrap

      - name: Start local registry
        run: docker run -d -p 5000:5000 --name registry registry:2

      - name: Build & push frontend syntax image to local registry
        run: |
          set -euo pipefail
          docker buildx build . -t localhost:5000/aikit:local --push --provenance=false --progress plain

      - name: Build raw modelpack OCI layout (huggingface source)
        run: |
          set -euo pipefail
          mkdir -p modelpack-out
          docker buildx build . \
            --build-arg BUILDKIT_SYNTAX=localhost:5000/aikit:local \
            --target packager/modelpack \
            --build-arg source=huggingface://MaziyarPanahi/Llama-3.2-1B-Instruct-GGUF/Llama-3.2-1B-Instruct.Q4_K_M.gguf \
            --build-arg name=llama-3.2-1b-instruct \
            --build-arg layer_packaging=raw \
            --output type=local,dest=modelpack-out
          layout_root=modelpack-out
          if [ ! -f "$layout_root/index.json" ] && [ -f modelpack-out/layout/index.json ]; then
            layout_root=modelpack-out/layout
          fi
          test -f "$layout_root/index.json"
          echo 'Modelpack layout:'
          find modelpack-out -maxdepth 2 -type f | head -50

      - name: Push modelpack to local registry
        run: |
          set -euo pipefail
          layout_root=modelpack-out
          if [ ! -f "$layout_root/index.json" ] && [ -f modelpack-out/layout/index.json ]; then
            layout_root=modelpack-out/layout
          fi
          skopeo copy --dest-tls-verify=false oci:$layout_root docker://localhost:5000/aikit/llama-3.2-1b-instruct:modelpack
          skopeo inspect --tls-verify=false --raw docker://localhost:5000/aikit/llama-3.2-1b-instruct:modelpack > pushed-manifest.json
          sha=sha256:$(sha256sum pushed-manifest.json | awk '{print $1}')
          echo "Pushed manifest digest: $sha"

      - name: Prepare inference aikitfile
        run: |
          # Start from existing test spec and adjust:
          sed '1s|.*|#syntax=localhost:5000/aikit:local|' test/aikitfile-llama.yaml > inference-aikitfile.yaml
          # Replace the source line with oci reference; keep rest of templates/config
          sed -i "s|^\( *source: \).*|    source: oci://localhost:5000/aikit/llama-3.2-1b-instruct:modelpack|" inference-aikitfile.yaml
          # Optionally shrink context size for faster test (reduce to 512)
          sed -i 's/context_size: 8192/context_size: 512/' inference-aikitfile.yaml
          # Ensure parameters.model is set explicitly (some specs may vary)
          sed -i 's|^\( *model: \).*|      model: Llama-3.2-1B-Instruct.Q4_K_M.gguf|' inference-aikitfile.yaml
          grep -n 'oci://localhost:5000/aikit/llama-3.2-1b-instruct:modelpack' inference-aikitfile.yaml
          sed -n '1,120p' inference-aikitfile.yaml

      - name: Build inference image from aikit spec
        run: |
          set -euo pipefail
          docker buildx build . \
            -f inference-aikitfile.yaml \
            --tag aikit-infer:latest \
            --load
          echo 'Inference container image built and loaded into local Docker daemon.'

      - name: Run container (local server) and perform inference
        run: |
          set -euo pipefail
          docker run -d --name aikit-infer -p 8080:8080 aikit-infer:latest
          result=$(curl --fail --retry 15 --retry-all-errors http://127.0.0.1:8080/v1/chat/completions -H 'Content-Type: application/json' -d '{"model":"llama-3.2-1b-instruct","messages":[{"role":"user","content":"explain kubernetes in a sentence"}]}')
          echo "$result"
          choices=$(echo "$result" | jq '.choices')
          if [ -z "$choices" ] || [ "$choices" = "null" ]; then
            echo 'No choices in response'; docker logs aikit-infer; docker kill aikit-infer; exit 1; fi

      - name: save logs
        if: always()
        run: docker logs aikit-infer > /tmp/aikit-infer.log || true

      - name: cleanup container
        if: always()
        run: docker rm -f aikit-infer || true

      - name: publish inference artifacts
        if: always()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: inference-logs
          path: /tmp/aikit-infer.log