Skip to content
This repository was archived by the owner on Oct 25, 2024. It is now read-only.

Commit 49336d3

Browse files
lvliang-intellvliang-intel
authored
Enable NeuralChat Unit Test process (#195)
1 parent d3206e6 commit 49336d3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+380
-453
lines changed

‎.github/workflows/cpp-graph-test.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ jobs:
3737
uses: actions/checkout@v3
3838
with:
3939
submodules: "recursive"
40+
fetch-tags: true
4041

4142
- name: Env build
4243
run: |

‎.github/workflows/deploy-test.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ jobs:
6565
uses: actions/checkout@v3
6666
with:
6767
submodules: "recursive"
68+
fetch-tags: true
6869
# We need this because GitHub needs to clone the branch to pipeline
6970
- name: Docker Build
7071
run: |

‎.github/workflows/llm-test.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ jobs:
4141
uses: actions/checkout@v3
4242
with:
4343
submodules: "recursive"
44+
fetch-tags: true
4445

4546
- name: Env build
4647
run: |

‎.github/workflows/optimize-test.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ jobs:
6262
uses: actions/checkout@v3
6363
with:
6464
submodules: "recursive"
65+
fetch-tags: true
6566
# We need this because GitHub needs to clone the branch to pipeline
6667
- name: Docker Build
6768
run: |

‎.github/workflows/script/SparseLibCI/run_sparse_lib.sh

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
11
#!/bin/bash
2-
export COVERAGE_RCFILE="/intel-extension-for-transformers/.github/workflows/script/unitTest/coverage/.coveragerc"
3-
42
output_log_dir="/intel-extension-for-transformers/benchmark_log"
53
WORKING_DIR="/intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime/deprecated"
64
mkdir ${output_log_dir}

‎.github/workflows/script/formatScan/pylint.sh

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ $BOLD_YELLOW && echo "---------------- run python setup.py sdist bdist_wheel ---
1212
export PYTHONPATH=`pwd`
1313
#$BOLD_YELLOW && echo "---------------- pip install binary -------------" && $RESET
1414
#pip install dist/intel_extension_for_transformers*.whl
15-
1615
pip list
1716

1817
cd /intel-extension-for-transformers/intel_extension_for_transformers/neural_chat/

‎.github/workflows/script/unitTest/coverage/.coveragerc

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎.github/workflows/script/unitTest/coverage/.engine-coveragerc

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[run]
2+
branch = True
3+
4+
[report]
5+
omit =
6+
*/**/fake*yaml
7+
*/intel_extension_for_transformers/neural_chat/**
8+
*/intel_extension_for_transformers/transformers/**
9+
*/intel_extension_for_transformers/llm/evaluation/**
10+
*/intel_extension_for_transformers/llm/finetuning/**
11+
*/intel_extension_for_transformers/llm/inference/**
12+
*/intel_extension_for_transformers/llm/quantization/**
13+
exclude_lines =
14+
pragma: no cover

‎.github/workflows/script/unitTest/coverage/.neural-chat-coveragerc

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
[run]
2+
branch = True
3+
4+
[report]
5+
omit =
6+
*/**/fake*yaml
7+
*/**/fake.py
8+
*/intel_extension_for_transformers/llm/amp/**
9+
*/intel_extension_for_transformers/llm/evaluation/**
10+
*/intel_extension_for_transformers/llm/quantization/**
11+
*/intel_extension_for_transformers/llm/library/**
12+
*/intel_extension_for_transformers/llm/operator/**
13+
*/intel_extension_for_transformers/llm/runtime/**
14+
*/intel_extension_for_transformers/transformers/**
15+
exclude_lines =
16+
pragma: no cover
17+
raise NotImplementedError
18+
raise TypeError
19+
if self.device == "gpu":
20+
if device == "gpu":
21+
except ImportError:
22+
except Exception as e:
23+
onnx_version < ONNX18_VERSION
24+
onnx_version >= ONNX18_VERSION

‎.github/workflows/script/unitTest/coverage/calc_coverage.sh

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,18 +87,22 @@ function check_coverage_status() {
8787
$BOLD_RED && echo "Unit Test failed with ${item} coverage decrease ${decrease}%" && $RESET
8888
done
8989
$BOLD_RED && echo "compare coverage to give detail info" && $RESET
90-
bash -x compare_coverage.sh ${coverage_compare} ${coverage_log_pr}/coverage.log ${coverage_log_base}/coverage.log "FAILED" ${coverage_PR_lines_rate} ${coverage_base_lines_rate} ${coverage_PR_branches_rate} ${coverage_base_branches_rate}
90+
bash compare_coverage.sh ${coverage_compare} ${coverage_log_pr}/coverage.log ${coverage_log_base}/coverage.log "FAILED" ${coverage_PR_lines_rate} ${coverage_base_lines_rate} ${coverage_PR_branches_rate} ${coverage_base_branches_rate}
9191
exit 1
9292
else
9393
$BOLD_GREEN && echo "Unit Test success with coverage lines: ${coverage_PR_lines_rate}%, branches: ${coverage_PR_branches_rate}%" && $RESET
9494
$BOLD_GREEN && echo "compare coverage to give detail info" && $RESET
95-
bash -x compare_coverage.sh ${coverage_compare} ${coverage_log_pr}/coverage.log ${coverage_log_base}/coverage.log "SUCCESS" ${coverage_PR_lines_rate} ${coverage_base_lines_rate} ${coverage_PR_branches_rate} ${coverage_base_branches_rate}
95+
bash compare_coverage.sh ${coverage_compare} ${coverage_log_pr}/coverage.log ${coverage_log_base}/coverage.log "SUCCESS" ${coverage_PR_lines_rate} ${coverage_base_lines_rate} ${coverage_PR_branches_rate} ${coverage_base_branches_rate}
9696
fi
9797
}
9898

9999
function main() {
100100
compare_coverage
101101
check_coverage_status
102+
$BOLD_BLUE && echo "PR lines coverage: $lines_PR_covered/$lines_PR_valid ($coverage_PR_lines_rate%)" && $RESET
103+
$BOLD_BLUE && echo "PR branches coverage: $branches_PR_covered/$branches_PR_valid ($coverage_PR_branches_rate%)" && $RESET
104+
$BOLD_BLUE && echo "BASE lines coverage: $lines_base_covered/$lines_base_valid ($coverage_base_lines_rate%)" && $RESET
105+
$BOLD_BLUE && echo "BASE branches coverage: $branches_base_covered/$branches_base_valid ($coverage_base_branches_rate%)" && $RESET
102106
}
103107

104108
main

‎.github/workflows/script/unitTest/run_unit_test_engine.sh

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22
source /intel-extension-for-transformers/.github/workflows/script/change_color.sh
3-
export COVERAGE_RCFILE="/intel-extension-for-transformers/.github/workflows/script/unitTest/coverage/.coveragerc"
3+
export COVERAGE_RCFILE="/intel-extension-for-transformers/.github/workflows/script/unitTest/coverage/.engine-coveragerc"
44
LOG_DIR=/log_dir
55
mkdir -p ${LOG_DIR}
66
WORKING_DIR="/intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime/deprecated"
@@ -86,7 +86,13 @@ function gtest() {
8686

8787
function main() {
8888
bash /intel-extension-for-transformers/.github/workflows/script/unitTest/env_setup.sh
89-
echo "test on ${test_name}"
89+
cd ${WORKING_DIR}/test/pytest || exit 1
90+
if [ -f "requirements.txt" ]; then
91+
python -m pip install --default-timeout=100 -r requirements.txt
92+
pip list
93+
else
94+
echo "Not found requirements.txt file."
95+
fi
9096
if [[ $test_name == "PR-test" ]]; then
9197
pytest "${LOG_DIR}/coverage_pr"
9298
gtest

‎.github/workflows/script/unitTest/run_unit_test_neuralchat.sh

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/bin/bash
2+
source /intel-extension-for-transformers/.github/workflows/script/change_color.sh
3+
export COVERAGE_RCFILE="/intel-extension-for-transformers/.github/workflows/script/unitTest/coverage/.neural-chat-coveragerc"
4+
LOG_DIR=/log_dir
5+
mkdir -p ${LOG_DIR}
6+
WORKING_DIR="/intel-extension-for-transformers/intel_extension_for_transformers/neural_chat/tests"
7+
# get parameters
8+
PATTERN='[-a-zA-Z0-9_]*='
9+
PERF_STABLE_CHECK=true
10+
11+
for i in "$@"; do
12+
case $i in
13+
--test_name=*)
14+
test_name=`echo $i | sed "s/${PATTERN}//"`;;
15+
*)
16+
echo "Parameter $i not recognized."; exit 1;;
17+
esac
18+
done
19+
20+
function pytest() {
21+
local coverage_log_dir=$1
22+
mkdir -p ${coverage_log_dir}
23+
24+
cd ${WORKING_DIR} || exit 1
25+
JOB_NAME=unit_test
26+
ut_log_name=${LOG_DIR}/${JOB_NAME}.log
27+
export GLOG_minloglevel=2
28+
29+
itrex_path=$(python -c 'import intel_extension_for_transformers; import os; print(os.path.dirname(intel_extension_for_transformers.__file__))')
30+
find . -name "test*.py" | sed 's,\.\/,coverage run --source='"${itrex_path}"' --append ,g' | sed 's/$/ --verbose/' >run.sh
31+
coverage erase
32+
33+
# run UT
34+
$BOLD_YELLOW && echo "cat run.sh..." && $RESET
35+
cat run.sh | tee ${ut_log_name}
36+
$BOLD_YELLOW && echo "------UT start-------" && $RESET
37+
bash run.sh 2>&1 | tee -a ${ut_log_name}
38+
$BOLD_YELLOW && echo "------UT end -------" && $RESET
39+
40+
# run coverage report
41+
coverage report -m --rcfile=${COVERAGE_RCFILE} | tee ${coverage_log_dir}/coverage.log
42+
coverage html -d ${coverage_log_dir}/htmlcov --rcfile=${COVERAGE_RCFILE}
43+
coverage xml -o ${coverage_log_dir}/coverage.xml --rcfile=${COVERAGE_RCFILE}
44+
45+
# check UT status
46+
if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ]; then
47+
$BOLD_RED && echo "Find errors in UT test, please check the output..." && $RESET
48+
exit 1
49+
fi
50+
$BOLD_GREEN && echo "UT finished successfully! " && $RESET
51+
}
52+
53+
function main() {
54+
bash /intel-extension-for-transformers/.github/workflows/script/unitTest/env_setup.sh
55+
wget http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb
56+
dpkg -i libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb
57+
cd ${WORKING_DIR} || exit 1
58+
if [ -f "requirements.txt" ]; then
59+
python -m pip install --default-timeout=100 -r requirements.txt
60+
pip list
61+
else
62+
echo "Not found requirements.txt file."
63+
fi
64+
echo "test on ${test_name}"
65+
if [[ $test_name == "PR-test" ]]; then
66+
pytest "${LOG_DIR}/coverage_pr"
67+
elif [[ $test_name == "baseline" ]]; then
68+
pytest "${LOG_DIR}/coverage_base"
69+
fi
70+
}
71+
main

‎.github/workflows/unit-test-engine.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ jobs:
5656
with:
5757
submodules: "recursive"
5858
ref: ${{ matrix.test_branch }}
59+
fetch-tags: true
5960

6061
- name: Docker Build
6162
run: |

‎.github/workflows/unit-test-kernel.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ jobs:
4747
uses: actions/checkout@v3
4848
with:
4949
submodules: "recursive"
50+
fetch-tags: true
5051

5152
- name: Docker Build
5253
run: |

‎.github/workflows/unit-test-neuralchat.yml

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
name: NeuralChat Unit Test
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
paths:
7+
- intel_extension_for_transformers/neural_chat/tests/**
8+
- .github/workflows/unit-test-neuralchat.yml
9+
- .github/workflows/script/unitTest/**
10+
workflow_dispatch:
11+
12+
# If there is a new commit, the previous jobs will be canceled
13+
concurrency:
14+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
15+
cancel-in-progress: true
16+
17+
env:
18+
DOCKER_CONFIG_NAME: "commonDockerConfig"
19+
REPO_NAME: "intel-extension-for-transformers"
20+
REPO_TAG: "py38"
21+
DOCKER_FILE_NAME: "devel"
22+
CONTAINER_NAME: "utTest"
23+
EXTRA_CONTAINER_NAME: "modelTest"
24+
25+
jobs:
26+
unit-test:
27+
runs-on: [self-hosted, Linux, X64, itrex-node]
28+
strategy:
29+
matrix:
30+
include:
31+
- test_branch: ${{ github.ref }}
32+
test_name: "PR-test"
33+
- test_branch: "main"
34+
test_name: "baseline"
35+
steps:
36+
- name: Docker Clean Up
37+
run: |
38+
docker ps -a
39+
if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then
40+
docker start ${{ env.CONTAINER_NAME }}
41+
echo "remove left files through container ..."
42+
docker exec ${{ env.CONTAINER_NAME }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
43+
fi
44+
if [[ $(docker ps -a | grep -i '${{ env.EXTRA_CONTAINER_NAME }}'$) ]]; then
45+
docker start ${{ env.EXTRA_CONTAINER_NAME }}
46+
echo "remove left files through container ..."
47+
docker exec ${{ env.EXTRA_CONTAINER_NAME }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
48+
fi
49+
50+
- name: Checkout out Repo
51+
uses: actions/checkout@v3
52+
with:
53+
submodules: "recursive"
54+
ref: ${{ matrix.test_branch }}
55+
fetch-tags: true
56+
57+
- name: Docker Build
58+
run: |
59+
docker build -f ${{ github.workspace }}/.github/workflows/docker/${{ env.DOCKER_FILE_NAME }}.dockerfile -t ${{ env.REPO_NAME }}:${{ env.REPO_TAG }} .
60+
61+
- name: Docker Run
62+
run: |
63+
if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then
64+
docker stop ${{ env.CONTAINER_NAME }}
65+
docker rm -vf ${{ env.CONTAINER_NAME }} || true
66+
fi
67+
docker run -dit --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} -v /dev/shm:/dev/shm \
68+
-v ${{ github.workspace }}:/intel-extension-for-transformers \
69+
${{ env.REPO_NAME }}:${{ env.REPO_TAG }}
70+
71+
- name: Env build
72+
run: |
73+
docker exec ${{ env.CONTAINER_NAME }} \
74+
bash /intel-extension-for-transformers/.github/workflows/script/prepare_env.sh
75+
76+
- name: Binary build
77+
run: |
78+
docker exec ${{ env.CONTAINER_NAME }} \
79+
bash -c "cd /intel-extension-for-transformers/.github/workflows/script \
80+
&& bash install_binary.sh \
81+
&& pip install intel_extension_for_pytorch wget sentencepiece \
82+
&& pip install transformers>=4.32.0 \
83+
&& pip install peft torch num2words speechbrain paddlepaddle \
84+
&& pip install paddlespeech==1.4.1 shortuuid gptcache evaluate \
85+
&& pip install fschat pydub python-multipart PyPDF2 langchain \
86+
&& pip install python-docx scikit-learn farm-haystack librosa beautifulsoup4 \
87+
&& pip install InstructorEmbedding chromadb pydantic fastapi starlette \
88+
&& pip install yacs uvicorn optimum optimum[habana] \
89+
&& pip install sentence_transformers unstructured markdown rouge_score \
90+
&& pip install --upgrade accelerate"
91+
92+
- name: Run UT
93+
run: |
94+
docker exec ${{ env.CONTAINER_NAME }} \
95+
bash -c "cd /intel-extension-for-transformers/.github/workflows/script/unitTest \
96+
&& bash run_unit_test_neuralchat.sh --test_name=${{ matrix.test_name }}"
97+
98+
- name: Collect log
99+
if: ${{ !cancelled() }}
100+
run: |
101+
docker exec ${{ env.CONTAINER_NAME }} \
102+
bash -c "cd /intel-extension-for-transformers && \
103+
mv /log_dir . "
104+
105+
- name: Publish pipeline artifact
106+
uses: actions/upload-artifact@v3
107+
if: ${{ !cancelled() }}
108+
with:
109+
name: "UnitTest${{ matrix.test_name }}"
110+
path: ${{ github.workspace }}/log_dir
111+
112+
113+
Genreate-Report:
114+
runs-on: itrex-node-spell
115+
needs: [unit-test]
116+
steps:
117+
- name: Checkout out Repo
118+
uses: actions/checkout@v3
119+
with:
120+
submodules: "recursive"
121+
122+
- name: Download UT PR Log
123+
uses: actions/download-artifact@v3
124+
with:
125+
path: ${{ github.workspace }}/log_dir
126+
127+
- name: Display structure of downloaded files
128+
run: cd ${{ github.workspace }}/log_dir && ls -R
129+
130+
- name: Calculte coverage
131+
run: |
132+
cd ${{ github.workspace }}/.github/workflows/script/unitTest/coverage
133+
/usr/bin/bash calc_coverage.sh ${{ github.workspace }}/log_dir
134+
135+
- name: Publish pipeline artifact
136+
uses: actions/upload-artifact@v3
137+
if: ${{ !cancelled() }}
138+
with:
139+
name: Neural Chat Unit Test
140+
path: ${{ github.workspace }}/log_dir
141+
retention-days: 5

‎.github/workflows/unit-test-optimize.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ jobs:
5656
with:
5757
submodules: "recursive"
5858
ref: ${{ matrix.test_branch }}
59+
fetch-tags: true
5960

6061
- name: Docker Build
6162
run: |
@@ -131,7 +132,7 @@ jobs:
131132
uses: actions/upload-artifact@v3
132133
if: ${{ !cancelled() }}
133134
with:
134-
name: Engine Unit Test
135+
name: Optimize Unit Test
135136
path: ${{ github.workspace }}/log_dir
136137
retention-days: 5
137138

‎intel_extension_for_transformers/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,4 @@
1818
try:
1919
from ._version import __version__ # load _version file generated by setuptools_scm
2020
except ModuleNotFoundError:
21-
__version__ = "0.0.0"
21+
__version__ = "1.1"

0 commit comments

Comments
 (0)