aimengpt/docker-compose-gguf.yml at master · aietal/aimengpt · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
version: '3.9'

networks:
  net:
    driver: bridge

services:
  chroma-server:
    image: server
    build:
      context: ../chroma
      dockerfile: Dockerfile
    volumes:
      - ../chroma:/chroma
      - index_data:/index_data
    command: uvicorn chromadb.app:app --reload --workers 1 --host 0.0.0.0 --port 8000 --log-config log_config.yml
    environment:
      - IS_PERSISTENT=TRUE
    ports:
      - 8000:8000
    networks:
      - net

  aimengpt-api:
    # Pin to llama-cpp-python 0.1.80 with GGUF support
    image: ghcr.io/abetlen/llama-cpp-python:latest@sha256:de0fd227f348b5e43d4b5b7300f1344e712c14132914d1332182e9ecfde502b2
    restart: on-failure
    volumes:
      - './models:/models'
      - './api:/api'
    ports:
      - 3001:8000
    environment:
      MODEL: '/models/${MODEL_NAME:-code-llama-2-7b-chat.gguf}'
      MODEL_DOWNLOAD_URL: '${MODEL_DOWNLOAD_URL:-https://huggingface.co/TheBloke/CodeLlama-7B-Instruct-GGUF/resolve/main/codellama-7b-instruct.Q4_K_M.gguf}'
      N_GQA: '${N_GQA:-1}'
      USE_MLOCK: 1
    cap_add:
      - IPC_LOCK
    command: '/bin/sh /api/run.sh'
    networks:
      - net

  aimengpt-ui:
    build:
      context: ./ui
      dockerfile: Dockerfile
    ports:
      - 3000:3000
    restart: on-failure
    environment:
      - 'OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXX'
      - 'OPENAI_API_HOST=http://aimengpt-api:8000'
      - 'DEFAULT_MODEL=/models/${MODEL_NAME:-llama-2-7b-chat.bin}'
      - 'WAIT_HOSTS=aimengpt-api:8000'
      - 'WAIT_TIMEOUT=${WAIT_TIMEOUT:-3600}'
    networks:
      - net

volumes:
  index_data:
    driver: local
  backups:
    driver: local