|
5 | 5 | # service: service path name, like 'agent_langchain', 'asr_whisper' |
6 | 6 | # hardware: 'intel_cpu', 'intel_hpu', ... |
7 | 7 |
|
8 | | -set -xe |
| 8 | +set -e |
9 | 9 | cd $WORKSPACE |
10 | 10 | changed_files_full=$changed_files_full |
11 | 11 | run_matrix="{\"include\":[" |
12 | 12 |
|
13 | 13 | # add test services when comps code change |
14 | 14 | function find_test_1() { |
15 | | - local pre_service=$1 |
| 15 | + local pre_service_path=$1 |
16 | 16 | local n=$2 |
17 | 17 | local all_service=$3 |
18 | 18 |
|
19 | | - common_file_change=$(printf '%s\n' "${changed_files[@]}"| grep ${pre_service} | cut -d'/' -f$n | grep -E '*.py' | grep -vE '__init__.py|version.py' | sort -u) || true |
| 19 | + common_file_change=$(printf '%s\n' "${changed_files[@]}"| grep ${pre_service_path} | cut -d'/' -f$n | grep -E '*.py' | grep -vE '__init__.py|version.py' | sort -u) || true |
20 | 20 | if [ "$common_file_change" ] || [ "$all_service" = "true" ]; then |
21 | 21 | # if common files changed, run all services |
22 | | - services=$(ls ${pre_service} | cut -d'/' -f$n | grep -vE '*.md|*.py|*.sh|*.yaml|*.yml|*.pdf' | sort -u) || true |
| 22 | + services=$(ls ${pre_service_path} | cut -d'/' -f$n | grep -vE '*.md|*.py|*.sh|*.yaml|*.yml|*.pdf' | sort -u) || true |
23 | 23 | all_service="true" |
24 | 24 | else |
25 | 25 | # if specific service files changed, only run the specific service |
26 | | - services=$(printf '%s\n' "${changed_files[@]}"| grep ${pre_service} | cut -d'/' -f$n | grep -vE '*.py|*.sh|*.yaml|*.yml|*.pdf' | sort -u) || true |
| 26 | + services=$(printf '%s\n' "${changed_files[@]}"| grep ${pre_service_path} | cut -d'/' -f$n | grep -vE '*.py|*.sh|*.yaml|*.yml|*.pdf' | sort -u) || true |
27 | 27 | fi |
28 | 28 |
|
29 | 29 | for service in ${services}; do |
30 | | - service=$pre_service/$service |
31 | | - if [[ $(ls ${service} | grep -E "Dockerfile*") ]]; then |
32 | | - service_name=$(echo $service | tr '/' '_' | cut -c7-) # comps/dataprep/redis/langchain -> dataprep_redis_langchain |
33 | | - default_service_script_path=$(find ./tests -type f -name test_${service_name}.sh) || true |
34 | | - if [ "$default_service_script_path" ]; then |
35 | | - run_matrix="${run_matrix}{\"service\":\"${service_name}\",\"hardware\":\"intel_cpu\"}," |
| 30 | + service_path=$pre_service_path/$service |
| 31 | + if [[ $(ls ${service_path} | grep -E "Dockerfile*") ]]; then |
| 32 | + if [[ $(ls ${service_path} | grep "integrations") ]]; then |
| 33 | + # new org with `src` and `integrations` folder |
| 34 | + run_all_interation="false" |
| 35 | + service_name=$(echo $service_path | sed 's:/src::' | tr '/' '_' | cut -c7-) # comps/retrievers/src/redis/langchain -> retrievers_redis_langchain |
| 36 | + common_file_change_insight=$(printf '%s\n' "${changed_files[@]}"| grep ${service_path} | grep -vE 'integrations' | sort -u) || true |
| 37 | + if [ "$common_file_change_insight" ]; then |
| 38 | + # if common file changed, run all integrations |
| 39 | + run_all_interation="true" |
| 40 | + fi |
| 41 | + if [ "$run_all_interation" = "false" ]; then |
| 42 | + changed_integrations=$(printf '%s\n' "${changed_files[@]}"| grep ${service_path} | grep -E 'integrations' | cut -d'/' -f$((n+2)) | cut -d'.' -f1 | sort -u) || true |
| 43 | + for integration in ${changed_integrations}; do |
| 44 | + # Accurate matching test scripts |
| 45 | + # find_test=$(find ./tests -type f \( -name test_${service_name}_${integration}.sh -o -name test_${service_name}_${integration}_on_*.sh \)) || true |
| 46 | + # Fuzzy matching test scripts, for example, llms/src/text-generation/integrations/opea.py match several tests. |
| 47 | + find_test=$(find ./tests -type f -name test_${service_name}_${integration}*.sh) || true |
| 48 | + if [ "$find_test" ]; then |
| 49 | + fill_in_matrix "$find_test" |
| 50 | + else |
| 51 | + run_all_interation="true" |
| 52 | + break |
| 53 | + fi |
| 54 | + done |
| 55 | + fi |
| 56 | + if [ "$run_all_interation" = "true" ]; then |
| 57 | + find_test=$(find ./tests -type f -name test_${service_name}*.sh) || true |
| 58 | + if [ "$find_test" ]; then |
| 59 | + fill_in_matrix "$find_test" |
| 60 | + fi |
| 61 | + fi |
| 62 | + else |
| 63 | + # old org without 'src' folder |
| 64 | + service_name=$(echo $service_path | tr '/' '_' | cut -c7-) # comps/retrievers/redis/langchain -> retrievers_redis_langchain |
| 65 | + find_test=$(find ./tests -type f -name test_${service_name}*.sh) || true |
| 66 | + if [ "$find_test" ]; then |
| 67 | + fill_in_matrix "$find_test" |
| 68 | + fi |
36 | 69 | fi |
37 | | - other_service_script_path=$(find ./tests -type f -name test_${service_name}_on_*.sh) || true |
38 | | - for script in ${other_service_script_path}; do |
39 | | - _service=$(echo $script | cut -d'/' -f4 | cut -d'.' -f1 | cut -c6-) |
40 | | - hardware=${_service#*_on_} |
41 | | - run_matrix="${run_matrix}{\"service\":\"${service_name}\",\"hardware\":\"${hardware}\"}," |
42 | | - done |
43 | 70 | else |
44 | | - find_test_1 $service $((n+1)) $all_service |
| 71 | + find_test_1 $service_path $((n+1)) $all_service |
45 | 72 | fi |
46 | 73 | done |
47 | 74 | } |
48 | 75 |
|
| 76 | +function fill_in_matrix() { |
| 77 | + find_test=$1 |
| 78 | + for test in ${find_test}; do |
| 79 | + _service=$(echo $test | cut -d'/' -f4 | cut -d'.' -f1 | cut -c6-) |
| 80 | + _fill_in_matrix $_service |
| 81 | + done |
| 82 | +} |
| 83 | + |
| 84 | +function _fill_in_matrix() { |
| 85 | + _service=$1 |
| 86 | + if [ $(echo ${_service} | grep -c "_on_") == 0 ]; then |
| 87 | + service=${_service} |
| 88 | + hardware="intel_cpu" |
| 89 | + else |
| 90 | + hardware=${_service#*_on_} |
| 91 | + fi |
| 92 | + echo "service=${_service}, hardware=${hardware}" |
| 93 | + if [[ $(echo ${run_matrix} | grep -c "{\"service\":\"${_service}\",\"hardware\":\"${hardware}\"},") == 0 ]]; then |
| 94 | + run_matrix="${run_matrix}{\"service\":\"${_service}\",\"hardware\":\"${hardware}\"}," |
| 95 | + echo "------------------ add one service ------------------" |
| 96 | + fi |
| 97 | + sleep 1s |
| 98 | +} |
| 99 | + |
49 | 100 | # add test case when test scripts code change |
50 | 101 | function find_test_2() { |
51 | 102 | test_files=$(printf '%s\n' "${changed_files[@]}" | grep -E "*.sh") || true |
52 | 103 | for test_file in ${test_files}; do |
53 | | - _service=$(echo $test_file | cut -d'/' -f3 | cut -d'.' -f1 | cut -c6-) |
54 | | - if [ $(echo ${_service} | grep -c "_on_") == 0 ]; then |
55 | | - service=${_service} |
56 | | - hardware="intel_cpu" |
57 | | - else |
58 | | - service=${_service%_on_*} |
59 | | - hardware=${_service#*_on_} |
60 | | - fi |
61 | | - if [[ $(echo ${run_matrix} | grep -c "{\"service\":\"${service}\",\"hardware\":\"${hardware}\"},") == 0 ]]; then |
62 | | - run_matrix="${run_matrix}{\"service\":\"${service}\",\"hardware\":\"${hardware}\"}," |
| 104 | + if [ -f $test_file ]; then |
| 105 | + _service=$(echo $test_file | cut -d'/' -f3 | cut -d'.' -f1 | cut -c6-) |
| 106 | + _fill_in_matrix $_service |
63 | 107 | fi |
64 | 108 | done |
65 | 109 | } |
66 | 110 |
|
67 | 111 | function main() { |
68 | 112 |
|
69 | | - changed_files=$(printf '%s\n' "${changed_files_full[@]}" | grep 'comps/' | grep -vE '*.md|comps/cores') || true |
| 113 | + changed_files=$(printf '%s\n' "${changed_files_full[@]}" | grep 'comps/' | grep -vE '*.md|comps/cores|comps/3rd_parties|deployment|*.yaml') || true |
| 114 | + echo "===========start find_test_1============" |
| 115 | + echo "changed_files=${changed_files}" |
70 | 116 | find_test_1 "comps" 2 false |
71 | 117 | sleep 1s |
| 118 | + echo "run_matrix=${run_matrix}" |
72 | 119 | echo "===========finish find_test_1============" |
73 | 120 |
|
74 | 121 | changed_files=$(printf '%s\n' "${changed_files_full[@]}" | grep 'tests/' | grep -vE '*.md|*.txt|tests/cores') || true |
| 122 | + echo "===========start find_test_2============" |
| 123 | + echo "changed_files=${changed_files}" |
75 | 124 | find_test_2 |
76 | 125 | sleep 1s |
| 126 | + echo "run_matrix=${run_matrix}" |
77 | 127 | echo "===========finish find_test_2============" |
78 | 128 |
|
79 | 129 | run_matrix=$run_matrix"]}" |
80 | | - echo "run_matrix=${run_matrix}" |
81 | 130 | echo "run_matrix=${run_matrix}" >> $GITHUB_OUTPUT |
82 | 131 | } |
83 | 132 |
|
|
0 commit comments