Skip to content

Commit 7d1ccb2

Browse files
guangy10guangy10
committed
Create unified script and workflow for llama-fast models validationin
1 parent bf2ccad commit 7d1ccb2

File tree

8 files changed

+399
-34
lines changed

8 files changed

+399
-34
lines changed

‎.ci/scripts/convert_checkpoint.sh

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
set -eu
10+
11+
function convert_checkpoint() {
12+
local MODEL_REPO="$1"
13+
local CHECKPOINT_NAME="${MODEL_REPO##*/}"
14+
15+
if [[ $CHECKPOINT_NAME == *"stories15M"* || $CHECKPOINT_NAME == *"stories42M"* || $CHECKPOINT_NAME == *"stories110M"* ]]; then
16+
# We need this to make the workflow unique for all models because convert_hf_checkpoint will always convert the checkpoint to model.pth
17+
pushd "checkpoints/${MODEL_REPO}"
18+
if [ ! -f "model.pth" ]; then
19+
mv "$CHECKPOINT_NAME.pt" "model.pth"
20+
fi
21+
popd
22+
return 0
23+
fi
24+
25+
if [ -f "checkpoints/$MODEL_REPO/model.pth" ]; then
26+
echo "Converted checkpoint already exists. Skipping conversion for $MODEL_REPO."
27+
return 0
28+
fi
29+
echo "Convert Huggingface checkpoint for $MODEL_REPO"
30+
python scripts/convert_hf_checkpoint.py --checkpoint-dir "checkpoints/$MODEL_REPO"
31+
}
32+
33+
34+
convert_checkpoint $1

‎.ci/scripts/gather_test_models.py

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
#!/usr/bin/env python
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
import itertools
9+
import json
10+
import os
11+
from typing import Any
12+
13+
14+
MODEL_REPOS = {
15+
"tinyllamas/stories15M": "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.pt,https://github.com/karpathy/llama2.c/raw/master/tokenizer.model,https://github.com/karpathy/llama2.c/raw/master/tokenizer.bin",
16+
# "tinyllamas/stories42M": "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories42M.pt,https://github.com/karpathy/llama2.c/raw/master/tokenizer.model,https://github.com/karpathy/llama2.c/raw/master/tokenizer.bin",
17+
"tinyllamas/stories110M": "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories110M.pt,https://github.com/karpathy/llama2.c/raw/master/tokenizer.model,https://github.com/karpathy/llama2.c/raw/master/tokenizer.bin",
18+
}
19+
20+
JOB_RUNNERS = {
21+
"32-core-ubuntu": "linux x86",
22+
# "macos-13": "macos x86", # not working for ExecuTorch yet
23+
"macos-14": "macos M1",
24+
}
25+
26+
27+
def set_output(name: str, val: Any) -> None:
28+
"""
29+
Set the GitHb output so that it can be accessed by other jobs
30+
"""
31+
print(f"Setting {val} to GitHub output")
32+
33+
if os.getenv("GITHUB_OUTPUT"):
34+
with open(str(os.getenv("GITHUB_OUTPUT")), "a") as env:
35+
print(f"{name}={val}", file=env)
36+
else:
37+
print(f"::set-output name={name}::{val}")
38+
39+
40+
def export_models_for_ci() -> dict[str, dict]:
41+
"""
42+
This gathers all the models that we want to test on GitHub OSS CI
43+
"""
44+
45+
# This is the JSON syntax for configuration matrix used by GitHub
46+
# https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
47+
models = {"include": []}
48+
49+
for repo_name, runner in itertools.product(
50+
MODEL_REPOS.keys(),
51+
JOB_RUNNERS.keys(),
52+
):
53+
record = {
54+
"repo_name": repo_name,
55+
"resources": MODEL_REPOS[repo_name],
56+
"runner": runner,
57+
"platform": JOB_RUNNERS[runner],
58+
"timeout": 90,
59+
}
60+
61+
models["include"].append(record)
62+
63+
set_output("models", json.dumps(models))
64+
65+
66+
if __name__ == "__main__":
67+
export_models_for_ci()

‎.ci/scripts/validate.sh

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
set -u
10+
11+
function generate_eager_model_output() {
12+
local CHECKPOINT_PATH="$1"
13+
local TARGET_DEVICE="${2:-cpu}"
14+
local MODEL_DIR="${CHECKPOINT_PATH%/*}"
15+
local MODEL_NAME=$(basename "$CHECKPOINT_PATH" | sed 's/\.[^.]*$//')
16+
echo "Run inference with eager model for $MODEL_NAME"
17+
python -W ignore generate.py --checkpoint-path "$CHECKPOINT_PATH" --prompt "$PROMPT" --device "$TARGET_DEVICE" > "$MODEL_DIR/output_eager"
18+
cat "$MODEL_DIR/output_eager"
19+
}
20+
21+
function generate_compiled_model_output() {
22+
local CHECKPOINT_PATH="$1"
23+
local TARGET_DEVICE="${2:-cpu}"
24+
local MODEL_DIR="${CHECKPOINT_PATH%/*}"
25+
local MODEL_NAME=$(basename "$CHECKPOINT_PATH" | sed 's/\.[^.]*$//')
26+
echo ""############### Run inference with torch.compile for $MODEL_NAME "###############"
27+
python -W ignore generate.py --compile --checkpoint-path "$CHECKPOINT_PATH" --prompt "$PROMPT" --device "$TARGET_DEVICE" > "$MODEL_DIR/output_compiled"
28+
cat "$MODEL_DIR/output_compiled"
29+
}
30+
31+
function generate_aoti_model_output() {
32+
local CHECKPOINT_PATH="$1"
33+
local TARGET_DEVICE="${2:-cpu}"
34+
local MODEL_DIR="${CHECKPOINT_PATH%/*}"
35+
local MODEL_NAME=$(basename "$CHECKPOINT_PATH" | sed 's/\.[^.]*$//')
36+
echo ""############### Run inference with AOTInductor for $MODEL_NAME "###############"
37+
python -W ignore export.py --checkpoint-path "$CHECKPOINT_PATH" --output-dso-path "${MODEL_DIR}/${MODEL_NAME}.so" --device "$TARGET_DEVICE"
38+
python -W ignore generate.py --checkpoint-path "$CHECKPOINT_PATH" --dso-path "$MODEL_DIR/${MODEL_NAME}.so" --prompt "$PROMPT" > "$MODEL_DIR/output_aoti"
39+
cat "$MODEL_DIR/output_aoti"
40+
}
41+
42+
function generate_executorch_model_output() {
43+
local CHECKPOINT_PATH="$1"
44+
local TARGET_DEVICE="${2:-cpu}"
45+
local MODEL_DIR="${CHECKPOINT_PATH%/*}"
46+
local MODEL_NAME=$(basename "$CHECKPOINT_PATH" | sed 's/\.[^.]*$//')
47+
echo ""############### Run inference with ExecuTorch using XNNPACK for $MODEL_NAME "###############"
48+
python -W ignore export.py --checkpoint-path "$CHECKPOINT_PATH" --output-pte-path "$MODEL_DIR/${MODEL_NAME}.pte" -d "fp32"
49+
python -W ignore generate.py --checkpoint-path "$CHECKPOINT_PATH" --prompt "$PROMPT" --device "$TARGET_DEVICE" --pte-path "$MODEL_DIR/${MODEL_NAME}.pte" > "$MODEL_DIR/output_et"
50+
cat "$MODEL_DIR/output_et"
51+
}
52+
53+
54+
CHECKPOINT_PATH="$1"
55+
TARGET_DEVICE="${2:-cpu}"
56+
PROMPT="Hello, my name is"
57+
58+
generate_compiled_model_output $CHECKPOINT_PATH $TARGET_DEVICE
59+
generate_aoti_model_output $CHECKPOINT_PATH $TARGET_DEVICE
60+
generate_executorch_model_output $CHECKPOINT_PATH $TARGET_DEVICE

‎.ci/scripts/wget_checkpoint.sh

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -exu
9+
10+
MODEL_REPO="$1"
11+
RESOURCES_STRING="$2"
12+
CHECKPOINT_NAME="${MODEL_REPO##*/}"
13+
14+
pushd "${LLAMA_FAST_ROOT}" || exit
15+
16+
# Create the directory for the checkpoint
17+
mkdir -p "checkpoints/${MODEL_REPO}"
18+
cd "checkpoints/${MODEL_REPO}" || exit
19+
20+
# Download all resources
21+
IFS=',' # Set the field separator to comma
22+
for resource in $RESOURCES_STRING; do
23+
echo "Downloading: $resource"
24+
if ! wget "$resource" 2>&1; then
25+
echo "Error: Failed to download $resource" >&2
26+
exit 1
27+
fi
28+
done
29+
30+
popd || exit

‎.github/workflows/pull.yml

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: pull
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
workflow_dispatch:
9+
10+
jobs:
11+
gather-models:
12+
runs-on: ubuntu-22.04
13+
outputs:
14+
models: ${{ steps.gather-models.outputs.models }}
15+
steps:
16+
- uses: actions/checkout@v3
17+
with:
18+
submodules: 'false'
19+
- uses: actions/setup-python@v4
20+
with:
21+
python-version: '3.11'
22+
- name: Extract the list of models to test
23+
id: gather-models
24+
run: |
25+
set -eux
26+
PYTHONPATH="${PWD}" python .ci/scripts/gather_test_models.py
27+
test-cpu:
28+
name: test-cpu (${{ matrix.platform }}, ${{ matrix.repo_name }})
29+
needs: gather-models
30+
strategy:
31+
matrix: ${{ fromJSON(needs.gather-models.outputs.models) }}
32+
fail-fast: false
33+
runs-on: ${{ matrix.runner }}
34+
env:
35+
LLAMA_FAST_ROOT: ${{ github.workspace }}
36+
REPO_NAME: ${{ matrix.repo_name }}
37+
ENABKE_ET_PYBIND: ${{ matrix.runner == 'macos-14' && 'false' || 'true' }}
38+
steps:
39+
- name: Checkout repo
40+
uses: actions/checkout@v3
41+
- name: Setup Python
42+
uses: actions/setup-python@v4
43+
with:
44+
python-version: '3.11'
45+
- name: Print machine info
46+
run: |
47+
echo "$(uname -a)"
48+
- name: Install dependencies
49+
run: |
50+
bash ${LLAMA_FAST_ROOT}/scripts/install_et.sh $ENABKE_ET_PYBIND
51+
- name: Download checkpoints
52+
run: |
53+
bash ${LLAMA_FAST_ROOT}/.ci/scripts/wget_checkpoint.sh ${{ matrix.repo_name }} "${{ matrix.resources }}"
54+
- name: Run validation
55+
run: |
56+
pushd ${LLAMA_FAST_ROOT}
57+
export CHECKPOINT_PATH=${LLAMA_FAST_ROOT}/checkpoints/${REPO_NAME}/model.pth
58+
bash ${LLAMA_FAST_ROOT}/.ci/scripts/convert_checkpoint.sh ${REPO_NAME}
59+
bash ${LLAMA_FAST_ROOT}/.ci/scripts/validate.sh ${CHECKPOINT_PATH}

‎.github/workflows/runner_et.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ jobs:
3737
pip install -r requirements.txt
3838
3939
export LLAMA_FAST_ROOT=${PWD}
40-
export ET_NO_PYBIND=1
41-
./scripts/install_et.sh
40+
export ENABLE_ET_PYBIND=false
41+
./scripts/install_et.sh $ENABLE_ET_PYBIND
4242
cmake -S ./runner-et -B build/cmake-out -G Ninja
4343
cmake --build ./build/cmake-out
4444
- name: Download checkpoints

‎scripts/install_et.sh

+63-32
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,63 @@
1-
cd ${LLAMA_FAST_ROOT}
2-
echo "Inside: $LLAMA_FAST_ROOT"
3-
4-
echo "Cloning executorch to ${LLAMA_FAST_ROOT}/build/src"
5-
rm -rf ${LLAMA_FAST_ROOT}/build
6-
mkdir -p ${LLAMA_FAST_ROOT}/build/src
7-
cd ${LLAMA_FAST_ROOT}/build/src
8-
git clone https://github.com/pytorch/executorch.git
9-
cd executorch
10-
echo "Install executorch: submodule update"
11-
git submodule sync
12-
git submodule update --init
13-
14-
echo "Applying fixes"
15-
cp ${LLAMA_FAST_ROOT}/scripts/fixes_et/module.cpp ${LLAMA_FAST_ROOT}/build/src/executorch/extension/module/module.cpp # ET uses non-standard C++ that does not compile in GCC
16-
cp ${LLAMA_FAST_ROOT}/scripts/fixes_et/managed_tensor.h ${LLAMA_FAST_ROOT}/build/src/executorch/extension/runner_util/managed_tensor.h # ET is missing headers for vector/memory. This causes downstream issues when building runner-et.
17-
18-
echo "Building and installing python libraries"
19-
if [ -n "${ET_NO_PYBIND}" ]; then
20-
echo "Not installing pybind"
21-
./install_requirements.sh
22-
else
23-
echo "Installing pybind"
24-
./install_requirements.sh --pybind xnnpack
25-
fi
26-
27-
echo "Building and installing C++ libraries"
28-
echo "Inside: ${PWD}"
29-
mkdir cmake-out
30-
cmake -DCMAKE_BUILD_TYPE=Release -DEXECUTORCH_BUILD_OPTIMIZED=ON -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON -DEXECUTORCH_BUILD_XNNPACK=ON -S . -B cmake-out -G Ninja
31-
cmake --build cmake-out
32-
cmake --install cmake-out --prefix ${LLAMA_FAST_ROOT}/build/install
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -exu
9+
10+
install_pip_dependencies() {
11+
echo "Intalling common pip packages"
12+
13+
pip install wheel
14+
pip install cmake
15+
pip install ninja
16+
pip install zstd
17+
pushd ${LLAMA_FAST_ROOT}
18+
pip install -r ./requirements.txt
19+
popd
20+
}
21+
22+
install_executorch() {
23+
echo "Cloning executorch to ${LLAMA_FAST_ROOT}/build/src"
24+
rm -rf ${LLAMA_FAST_ROOT}/build
25+
mkdir -p ${LLAMA_FAST_ROOT}/build/src
26+
pushd ${LLAMA_FAST_ROOT}/build/src
27+
git clone https://github.com/pytorch/executorch.git
28+
cd executorch
29+
echo "Install executorch: submodule update"
30+
git submodule sync
31+
git submodule update --init
32+
33+
echo "Applying fixes"
34+
cp ${LLAMA_FAST_ROOT}/scripts/fixes_et/module.cpp ${LLAMA_FAST_ROOT}/build/src/executorch/extension/module/module.cpp # ET uses non-standard C++ that does not compile in GCC
35+
cp ${LLAMA_FAST_ROOT}/scripts/fixes_et/managed_tensor.h ${LLAMA_FAST_ROOT}/build/src/executorch/extension/runner_util/managed_tensor.h # ET is missing headers for vector/memory. This causes downstream issues when building runner-et.
36+
37+
echo "Building and installing python libraries"
38+
echo "Building and installing python libraries"
39+
if [ "${ENABLE_ET_PYBIND}" = false ]; then
40+
echo "Not installing pybind"
41+
bash ./install_requirements.sh
42+
else
43+
echo "Installing pybind"
44+
bash ./install_requirements.sh --pybind xnnpack
45+
fi
46+
pip list
47+
48+
echo "Building and installing C++ libraries"
49+
echo "Inside: ${PWD}"
50+
mkdir cmake-out
51+
cmake -DCMAKE_BUILD_TYPE=Release -DEXECUTORCH_BUILD_OPTIMIZED=ON -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON -DEXECUTORCH_BUILD_XNNPACK=ON -S . -B cmake-out -G Ninja
52+
cmake --build cmake-out
53+
cmake --install cmake-out --prefix ${LLAMA_FAST_ROOT}/build/install
54+
popd
55+
}
56+
57+
58+
ENABLE_ET_PYBIND="${1:-true}"
59+
60+
pushd ${LLAMA_FAST_ROOT}
61+
install_pip_dependencies
62+
install_executorch $ENABLE_ET_PYBIND
63+
popd

0 commit comments

Comments
 (0)