rag-doc-example/valohai.yaml at main · valohai/rag-doc-example · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
- step:
    name: create-embeddings
    image: ghcr.io/astral-sh/uv:python3.11-bookworm-slim
    command: uv run rag-doctor create-database {parameters}
    parameters:
      - name: content_column_index
        description: Index of the document content column in the CSV files
        type: integer
        default: 0
      - name: source_column_index
        description: Index of the source link column in the CSV files
        type: integer
        default: 1
      - name: header_row_skip
        description: Number of initial rows to skip in the CSV files i.e. the header rows
        type: integer
        default: 0
    inputs:
      - name: documentation_csv
        description: CSV file(s) containing the technical documentation
        default: https://valohai-examples.s3.amazonaws.com/rag-doc/huggingface-docs.csv

- step:
    name: do-query
    image: ghcr.io/astral-sh/uv:python3.11-bookworm-slim
    command: uv run rag-doctor query {parameters}
    parameters:
      - name: question
        description: The question(s) to ask about the documentation
        type: string
        multiple: repeat
      - name: provider
        description: LLM provider to use (openai or anthropic)
        type: string
        default: openai
        choices: [openai, anthropic]
    inputs:
      - name: embedding_db
        description: Zip archive containing the Qdrant vector database for the embeddings

- step:
    name: evaluate-rag
    image: ghcr.io/astral-sh/uv:python3.11-bookworm-slim
    command: uv run rag-doctor evaluate {parameters}
    inputs:
      - name: responses
        description: JSON file(s) containing generated responses to evaluate
      - name: gold_standards
        description: CSV file containing ground truth answers for evaluation
        default: https://valohai-examples.s3.eu-west-1.amazonaws.com/rag-doc/rag_ground_truth.csv

- endpoint:
    name: ask
    image: ghcr.io/astral-sh/uv:python3.11-bookworm-slim
    port: 8000
    server-command: uv run --group server rag-doctor serve --host 0.0.0.0 --database_dir ./qdrant_data --provider openai
    files:
      - name: embedding_db
        path: qdrant_data/embeddings.zip
        description: Zip archive containing the Qdrant vector database for the embeddings

- pipeline:
    name: assistant-pipeline
    nodes:
      - name: embeddings
        type: execution
        step: create-embeddings
      - name: manual-evaluation
        type: execution
        step: do-query
        override:
          parameters:
            - name: question
              default: ["who is Michael Jordan?", "what is a dataset?", "hello", "how to create a model?"]
    edges:
      - [embeddings.output.*, manual-evaluation.input.embedding_db]


- pipeline:
    name: assistant-pipeline-with-deployment
    nodes:
      - name: embeddings
        type: execution
        step: create-embeddings
      - name: manual-evaluation
        type: execution
        step: do-query
        override:
          parameters:
            - name: question
              default: ["who is Michael Jordan?", "what is a dataset?", "hello", "how to create a model?"]
      - name: deploy
        type: deployment
        deployment: public
        endpoints: [ask]
        actions:
          - when: node-starting
            then: require-approval
    edges:
      - [embeddings.output.*, manual-evaluation.input.embedding_db]
      - [embeddings.output.*, deploy.file.ask.embedding_db]

- pipeline:
    name: rag-evaluation-pipeline
    nodes:
      - name: embeddings
        type: execution
        step: create-embeddings
      - name: generate-responses
        type: execution
        step: do-query
        override:
          parameters:
            - name: question
              default: ["what is a dataset?", "how to create a model?", "What is an Inference Endpoint?", "How do I create an Inference Endpoint?", "How can I choose a metric for my task?"]
      - name: evaluate
        type: execution
        step: evaluate-rag
    edges:
      - [embeddings.output.*, generate-responses.input.embedding_db]
      - [generate-responses.output.*, evaluate.input.responses]