Skip to content

Commit af37621

Browse files
ZePan110ZePan110
committed
Fix
Signed-off-by: ZePan110 <[email protected]>
1 parent c79f310 commit af37621

File tree

1 file changed

+139
-138
lines changed

1 file changed

+139
-138
lines changed

‎.github/workflows/_run-docker-compose.yml‎

Lines changed: 139 additions & 138 deletions
Original file line numberDiff line numberDiff line change
@@ -69,17 +69,19 @@ jobs:
6969
- name: Get test matrix
7070
shell: bash
7171
id: test-case-matrix
72+
env:
73+
HARDWARE: ${{ inputs.hardware }}
7274
run: |
7375
example_l=$(echo ${{ inputs.example }} | tr '[:upper:]' '[:lower:]')
7476
cd ${{ github.workspace }}/${{ inputs.example }}/tests
7577
run_test_cases=""
7678
77-
if [[ "${{ inputs.hardware }}" == "gaudi"* ]]; then
79+
if [[ "$HARDWARE}" == "gaudi"* ]]; then
7880
hardware="gaudi"
79-
elif [[ "${{ inputs.hardware }}" == "xeon"* ]]; then
81+
elif [[ "$HARDWARE" == "xeon"* ]]; then
8082
hardware="xeon"
8183
else
82-
hardware="${{ inputs.hardware }}"
84+
hardware="$HARDWARE"
8385
fi
8486
default_test_case=$(find . -type f -name "test_compose_on_$hardware.sh" | cut -d/ -f2)
8587
if [ "$default_test_case" ]; then run_test_cases="$default_test_case"; fi
@@ -109,138 +111,137 @@ jobs:
109111
# if [ -z "$run_test_cases" ] && [[ $(printf '%s\n' "${changed_files[@]}" | grep ${{ inputs.example }} | grep /tests/) ]]; then
110112
# run_test_cases=$other_test_cases
111113
# fi
112-
echo "${{ inputs.hardware }}"
113-
#
114-
# if [[ "${{ hardware }}" == "xeon"* ]]; then
115-
# if [ -f "${{ github.workspace }}/${{ inputs.example }}/tests/test_ui_on_xeon.sh" ]; then
116-
# run_test_cases="$run_test_cases test_ui_on_xeon.sh";
117-
# fi
118-
# fi
119-
# test_cases=$(echo $run_test_cases | tr ' ' '\n' | sort -u | jq -R '.' | jq -sc '.')
120-
# echo "test_cases=$test_cases"
121-
# echo "test_cases=$test_cases" >> $GITHUB_OUTPUT
122-
123-
# compose-test:
124-
# needs: [get-test-case]
125-
# if: ${{ needs.get-test-case.outputs.test_cases != '[""]' }}
126-
# strategy:
127-
# matrix:
128-
# test_case: ${{ fromJSON(needs.get-test-case.outputs.test_cases) }}
129-
# fail-fast: false
130-
# runs-on: ${{ inputs.hardware }}
131-
# continue-on-error: true
132-
# steps:
133-
# - name: Clean up Working Directory
134-
# run: |
135-
# sudo rm -rf ${{github.workspace}}/* || true
136-
137-
# echo "Cleaning up containers using ports..."
138-
# cid=$(docker ps --format '{{.Names}} : {{.Ports}}' | grep -v ' : $' | grep -v 0.0.0.0:5000 | awk -F' : ' '{print $1}')
139-
# if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
140-
# docker system prune -f
141-
142-
# echo "Cleaning up images ..."
143-
# docker images --filter reference="*/*/*:latest" -q | xargs -r docker rmi && sleep 1s
144-
# docker images --filter reference="*/*:ci" -q | xargs -r docker rmi && sleep 1s
145-
# docker images --filter reference="*:5000/*/*" -q | xargs -r docker rmi && sleep 1s
146-
# docker images --filter reference="opea/comps-base" -q | xargs -r docker rmi && sleep 1s
147-
# docker images
148-
149-
# - name: Checkout out Repo
150-
# uses: actions/checkout@v4
151-
# with:
152-
# ref: ${{ needs.get-test-case.outputs.CHECKOUT_REF }}
153-
# fetch-depth: 0
154-
155-
# - name: Clean up container before test
156-
# shell: bash
157-
# run: |
158-
# docker ps
159-
# cd ${{ github.workspace }}/${{ inputs.example }}
160-
# export test_case=${{ matrix.test_case }}
161-
# export hardware=${{ inputs.hardware }}
162-
# bash ${{ github.workspace }}/.github/workflows/scripts/docker_compose_clean_up.sh "containers"
163-
# bash ${{ github.workspace }}/.github/workflows/scripts/docker_compose_clean_up.sh "ports"
164-
# docker ps
165-
166-
# - name: Log in DockerHub
167-
# uses: docker/[email protected]
168-
# with:
169-
# username: ${{ secrets.DOCKERHUB_USER }}
170-
# password: ${{ secrets.DOCKERHUB_TOKEN }}
171-
172-
# - name: Run test
173-
# shell: bash
174-
# env:
175-
# HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HF_TOKEN }}
176-
# HF_TOKEN: ${{ secrets.HF_TOKEN }}
177-
# GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
178-
# GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
179-
# PINECONE_KEY: ${{ secrets.PINECONE_KEY }}
180-
# PINECONE_KEY_LANGCHAIN_TEST: ${{ secrets.PINECONE_KEY_LANGCHAIN_TEST }}
181-
# SDK_BASE_URL: ${{ secrets.SDK_BASE_URL }}
182-
# SERVING_TOKEN: ${{ secrets.SERVING_TOKEN }}
183-
# OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
184-
# FINNHUB_API_KEY: ${{ secrets.FINNHUB_API_KEY }}
185-
# FINANCIAL_DATASETS_API_KEY: ${{ secrets.FINANCIAL_DATASETS_API_KEY }}
186-
# TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }}
187-
# IMAGE_REPO: ${{ inputs.registry }}
188-
# IMAGE_TAG: ${{ inputs.tag }}
189-
# opea_branch: ${{ inputs.opea_branch }}
190-
# example: ${{ inputs.example }}
191-
# hardware: ${{ inputs.hardware }}
192-
# test_case: ${{ matrix.test_case }}
193-
# use_model_cache: ${{ inputs.use_model_cache }}
194-
# run: |
195-
# cd ${{ github.workspace }}/$example/tests
196-
# if [[ "$IMAGE_REPO" == "" ]]; then export IMAGE_REPO="${OPEA_IMAGE_REPO}opea"; fi
197-
# if [[ "$use_model_cache" == "true" ]]; then
198-
# if [ -d "/data2/hf_model" ]; then
199-
# export model_cache="/data2/hf_model"
200-
# else
201-
# echo "Model cache directory /data2/hf_model does not exist"
202-
# export model_cache="$HOME/.cache/huggingface/hub"
203-
# fi
204-
# if [[ "$test_case" == *"rocm"* ]]; then
205-
# export model_cache="/var/lib/GenAI/data"
206-
# fi
207-
# fi
208-
# if [ -f "${test_case}" ]; then timeout 60m bash "${test_case}"; else echo "Test script {${test_case}} not found, skip test!"; fi
209-
210-
# - name: Clean up container after test
211-
# if: always()
212-
# run: |
213-
# set -x
214-
215-
# echo "Cleaning up containers using ports..."
216-
# cid=$(docker ps --format '{{.Names}} : {{.Ports}}' | grep -v ' : $' | grep -v 0.0.0.0:5000 | awk -F' : ' '{print $1}')
217-
# if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
218-
219-
# echo "Cleaning up images ..."
220-
# df -h
221-
# sleep 1
222-
# docker system df
223-
# sleep 1
224-
# if [[ "${{ inputs.hardware }}" == "xeon"* ]]; then
225-
# docker system prune -a -f
226-
# else
227-
# docker images --filter reference="*/*/*:latest" -q | xargs -r docker rmi && sleep 1s
228-
# docker images --filter reference="*/*:ci" -q | xargs -r docker rmi && sleep 1s
229-
# docker images --filter reference="*:5000/*/*" -q | xargs -r docker rmi && sleep 1s
230-
# docker images --filter reference="opea/comps-base" -q | xargs -r docker rmi && sleep 1s
231-
# docker system prune -f
232-
# fi
233-
# sleep 5
234-
# docker images
235-
# sleep 1
236-
# df -h
237-
# sleep 1
238-
# docker system df
239-
# sleep 1
240-
241-
# - name: Publish pipeline artifact
242-
# if: ${{ !cancelled() }}
243-
# uses: actions/upload-artifact@v4
244-
# with:
245-
# name: ${{ inputs.hardware }}_${{ inputs.example }}_${{ matrix.test_case }}
246-
# path: ${{ github.workspace }}/${{ inputs.example }}/tests/*.log
114+
115+
if [[ "$hardware" == "xeon"* ]]; then
116+
if [ -f "${{ github.workspace }}/${{ inputs.example }}/tests/test_ui_on_xeon.sh" ]; then
117+
run_test_cases="$run_test_cases test_ui_on_xeon.sh";
118+
fi
119+
fi
120+
test_cases=$(echo $run_test_cases | tr ' ' '\n' | sort -u | jq -R '.' | jq -sc '.')
121+
echo "test_cases=$test_cases"
122+
echo "test_cases=$test_cases" >> $GITHUB_OUTPUT
123+
124+
compose-test:
125+
needs: [get-test-case]
126+
if: ${{ needs.get-test-case.outputs.test_cases != '[""]' }}
127+
strategy:
128+
matrix:
129+
test_case: ${{ fromJSON(needs.get-test-case.outputs.test_cases) }}
130+
fail-fast: false
131+
runs-on: ${{ inputs.hardware }}
132+
continue-on-error: true
133+
steps:
134+
- name: Clean up Working Directory
135+
run: |
136+
sudo rm -rf ${{github.workspace}}/* || true
137+
138+
echo "Cleaning up containers using ports..."
139+
cid=$(docker ps --format '{{.Names}} : {{.Ports}}' | grep -v ' : $' | grep -v 0.0.0.0:5000 | awk -F' : ' '{print $1}')
140+
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
141+
docker system prune -f
142+
143+
echo "Cleaning up images ..."
144+
docker images --filter reference="*/*/*:latest" -q | xargs -r docker rmi && sleep 1s
145+
docker images --filter reference="*/*:ci" -q | xargs -r docker rmi && sleep 1s
146+
docker images --filter reference="*:5000/*/*" -q | xargs -r docker rmi && sleep 1s
147+
docker images --filter reference="opea/comps-base" -q | xargs -r docker rmi && sleep 1s
148+
docker images
149+
150+
- name: Checkout out Repo
151+
uses: actions/checkout@v4
152+
with:
153+
ref: ${{ needs.get-test-case.outputs.CHECKOUT_REF }}
154+
fetch-depth: 0
155+
156+
- name: Clean up container before test
157+
shell: bash
158+
run: |
159+
docker ps
160+
cd ${{ github.workspace }}/${{ inputs.example }}
161+
export test_case=${{ matrix.test_case }}
162+
export hardware=${{ inputs.hardware }}
163+
bash ${{ github.workspace }}/.github/workflows/scripts/docker_compose_clean_up.sh "containers"
164+
bash ${{ github.workspace }}/.github/workflows/scripts/docker_compose_clean_up.sh "ports"
165+
docker ps
166+
167+
- name: Log in DockerHub
168+
uses: docker/[email protected]
169+
with:
170+
username: ${{ secrets.DOCKERHUB_USER }}
171+
password: ${{ secrets.DOCKERHUB_TOKEN }}
172+
173+
- name: Run test
174+
shell: bash
175+
env:
176+
HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HF_TOKEN }}
177+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
178+
GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
179+
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
180+
PINECONE_KEY: ${{ secrets.PINECONE_KEY }}
181+
PINECONE_KEY_LANGCHAIN_TEST: ${{ secrets.PINECONE_KEY_LANGCHAIN_TEST }}
182+
SDK_BASE_URL: ${{ secrets.SDK_BASE_URL }}
183+
SERVING_TOKEN: ${{ secrets.SERVING_TOKEN }}
184+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
185+
FINNHUB_API_KEY: ${{ secrets.FINNHUB_API_KEY }}
186+
FINANCIAL_DATASETS_API_KEY: ${{ secrets.FINANCIAL_DATASETS_API_KEY }}
187+
TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }}
188+
IMAGE_REPO: ${{ inputs.registry }}
189+
IMAGE_TAG: ${{ inputs.tag }}
190+
opea_branch: ${{ inputs.opea_branch }}
191+
example: ${{ inputs.example }}
192+
hardware: ${{ inputs.hardware }}
193+
test_case: ${{ matrix.test_case }}
194+
use_model_cache: ${{ inputs.use_model_cache }}
195+
run: |
196+
cd ${{ github.workspace }}/$example/tests
197+
if [[ "$IMAGE_REPO" == "" ]]; then export IMAGE_REPO="${OPEA_IMAGE_REPO}opea"; fi
198+
if [[ "$use_model_cache" == "true" ]]; then
199+
if [ -d "/data2/hf_model" ]; then
200+
export model_cache="/data2/hf_model"
201+
else
202+
echo "Model cache directory /data2/hf_model does not exist"
203+
export model_cache="$HOME/.cache/huggingface/hub"
204+
fi
205+
if [[ "$test_case" == *"rocm"* ]]; then
206+
export model_cache="/var/lib/GenAI/data"
207+
fi
208+
fi
209+
if [ -f "${test_case}" ]; then timeout 60m bash "${test_case}"; else echo "Test script {${test_case}} not found, skip test!"; fi
210+
211+
- name: Clean up container after test
212+
if: always()
213+
run: |
214+
set -x
215+
216+
echo "Cleaning up containers using ports..."
217+
cid=$(docker ps --format '{{.Names}} : {{.Ports}}' | grep -v ' : $' | grep -v 0.0.0.0:5000 | awk -F' : ' '{print $1}')
218+
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
219+
220+
echo "Cleaning up images ..."
221+
df -h
222+
sleep 1
223+
docker system df
224+
sleep 1
225+
if [[ "${{ inputs.hardware }}" == "xeon"* ]]; then
226+
docker system prune -a -f
227+
else
228+
docker images --filter reference="*/*/*:latest" -q | xargs -r docker rmi && sleep 1s
229+
docker images --filter reference="*/*:ci" -q | xargs -r docker rmi && sleep 1s
230+
docker images --filter reference="*:5000/*/*" -q | xargs -r docker rmi && sleep 1s
231+
docker images --filter reference="opea/comps-base" -q | xargs -r docker rmi && sleep 1s
232+
docker system prune -f
233+
fi
234+
sleep 5
235+
docker images
236+
sleep 1
237+
df -h
238+
sleep 1
239+
docker system df
240+
sleep 1
241+
242+
- name: Publish pipeline artifact
243+
if: ${{ !cancelled() }}
244+
uses: actions/upload-artifact@v4
245+
with:
246+
name: ${{ inputs.hardware }}_${{ inputs.example }}_${{ matrix.test_case }}
247+
path: ${{ github.workspace }}/${{ inputs.example }}/tests/*.log

0 commit comments

Comments
 (0)