llama-cookbook/end-to-end-use-cases/coding/text2sql/eval/llama_eval.sh at 4bb7faa35c7779d296b7db1e0d9c2b02be0d20dc · meta-llama/llama-cookbook · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
eval_path='../data/dev_20240627/dev.json'
db_root_path='../data/dev_20240627/dev_databases/'
ground_truth_path='../data/'

# Llama models on Together
#YOUR_API_KEY='YOUR_TOGETHER_API_KEY'
#model='meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo'
#model='meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo'
#model='meta-llama/Llama-3.3-70B-Instruct-Turbo'
#model='meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8'
#model='meta-llama/Llama-4-Scout-17B-16E-Instruct'

# Llama models on Llama API
YOUR_API_KEY='YOUR_LLAMA_API_KEY'
model='Llama-3.3-8B-Instruct'
#model='Llama-3.3-70B-Instruct'
#model='Llama-4-Maverick-17B-128E-Instruct-FP8'
#model='Llama-4-Scout-17B-16E-Instruct-FP8'

# Llama model on Hugging Face Hub
# https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct
# YOUR_API_KEY='huggingface'
# model='meta-llama/Llama-3.1-8B-Instruct'

# Fine-tuned Llama models locally
#YOUR_API_KEY='finetuned'
#model='../fine_tuning/llama31-8b-text2sql-fine-tuned'

data_output_path="./output/$model/"

echo "Text2SQL using $model"
python3 -u llama_text2sql.py --db_root_path ${db_root_path} --api_key ${YOUR_API_KEY} \
--model ${model} --eval_path ${eval_path} --data_output_path ${data_output_path}

# Check if llama_text2sql.py exited successfully
if [ $? -eq 0 ]; then
    echo "llama_text2sql.py completed successfully. Proceeding with evaluation..."
    python3 -u text2sql_eval.py --db_root_path ${db_root_path} --predicted_sql_path ${data_output_path} \
    --ground_truth_path ${ground_truth_path} \
    --diff_json_path ${eval_path}

    echo "Done evaluating $model."

else
    echo "Error: llama_text2sql.py failed with exit code $?. Skipping evaluation."
    exit 1
fi