Skip to content

Commit 9d6d7b8

Browse files
ZePan110pre-commit-ci[bot]
ZePan110
and
pre-commit-ci[bot]
authored
Check image and service names in compose.yaml (opea-project#951)
* WIP Signed-off-by: ZePan110 <[email protected]> * Chack image and service names in compose.yaml Signed-off-by: ZePan110 <[email protected]> * merge pr-check-duplicated-image.yml to pr-dockerfile-path-scan.yaml Signed-off-by: ZePan110 <[email protected]> * Remove .github/workflows/pr-check-duplicated-image.yml Signed-off-by: ZePan110 <[email protected]> * Unblocking txt files from push-image-build.yml Signed-off-by: ZePan110 <[email protected]> * Fix name error Signed-off-by: ZePan110 <[email protected]> * Split pr-dockerfile-path-scan.yaml to .github/workflows/pr-dockerfile-path-scan.yaml and .github/workflows/pr-link-path-scan.yaml and change the mask of .github/workflows/push-image-build.yml Signed-off-by: ZePan110 <[email protected]> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: ZePan110 <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
1 parent 9a0d91a commit 9d6d7b8

File tree

7 files changed

+230
-159
lines changed

7 files changed

+230
-159
lines changed

‎.github/workflows/docker/compose/guardrails-compose.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,11 @@ services:
2323
build:
2424
dockerfile: comps/guardrails/wildguard/langchain/Dockerfile
2525
image: ${REGISTRY:-opea}/guardrails-wildguard:${TAG:-latest}
26-
guardrails-pii-detection-predictionguard:
26+
guardrails-pii-predictionguard:
2727
build:
2828
dockerfile: comps/guardrails/pii_detection/predictionguard/Dockerfile
2929
image: ${REGISTRY:-opea}/guardrails-pii-predictionguard:${TAG:-latest}
30-
guardrails-toxicity-detection-predictionguard:
30+
guardrails-toxicity-predictionguard:
3131
build:
3232
dockerfile: comps/guardrails/toxicity_detection/predictionguard/Dockerfile
3333
image: ${REGISTRY:-opea}/guardrails-toxicity-predictionguard:${TAG:-latest}

‎.github/workflows/docker/compose/llms-compose.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ services:
5454
build:
5555
dockerfile: comps/llms/text-generation/vllm/llama_index/Dockerfile
5656
image: ${REGISTRY:-opea}/llm-vllm-llamaindex:${TAG:-latest}
57-
llm-predictionguard:
57+
llm-textgen-predictionguard:
5858
build:
5959
dockerfile: comps/llms/text-generation/predictionguard/Dockerfile
6060
image: ${REGISTRY:-opea}/llm-textgen-predictionguard:${TAG:-latest}

‎.github/workflows/pr-check-duplicated-image.yml‎

Lines changed: 0 additions & 40 deletions
This file was deleted.

‎.github/workflows/pr-dockerfile-path-scan.yaml‎

Lines changed: 27 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Copyright (C) 2024 Intel Corporation
22
# SPDX-License-Identifier: Apache-2.0
33

4-
name: File Change Warning
4+
name: Compose file and dockerfile path checking
55

66
on:
77
pull_request:
@@ -151,140 +151,51 @@ jobs:
151151
exit 1
152152
fi
153153
154-
check-the-validity-of-hyperlinks-in-README:
154+
check-image-and-service-names-in-compose-yaml:
155155
runs-on: ubuntu-latest
156156
steps:
157157
- name: Clean Up Working Directory
158158
run: sudo rm -rf ${{github.workspace}}/*
159159

160-
- name: Checkout Repo GenAIComps
160+
- name: Checkout Repo GenAIExamples
161161
uses: actions/checkout@v4
162-
with:
163-
fetch-depth: 0
164162

165-
- name: Check the Validity of Hyperlinks
166-
# ignore_links=("https://platform.openai.com/docs/api-reference/fine-tuning"
167-
# "https://platform.openai.com/docs/api-reference/"
168-
# "https://openai.com/index/whisper/"
169-
# "https://platform.openai.com/docs/api-reference/chat/create")
163+
- name: Check name agreement in compose.yaml
170164
run: |
165+
pip install ruamel.yaml
171166
cd ${{github.workspace}}
172-
fail="FALSE"
173-
merged_commit=$(git log -1 --format='%H')
174-
changed_files="$(git diff --name-status --diff-filter=ARM ${{ github.event.pull_request.base.sha }} ${merged_commit} | awk '/\.md$/ {print $NF}')"
175-
if [ -n "$changed_files" ]; then
176-
for changed_file in $changed_files; do
177-
echo $changed_file
178-
url_lines=$(grep -H -Eo '\]\(http[s]?://[^)]+\)' "$changed_file" | grep -Ev 'GenAIComps/blob/main') || true
179-
if [ -n "$url_lines" ]; then
180-
for url_line in $url_lines; do
181-
url=$(echo "$url_line"|cut -d '(' -f2 | cut -d ')' -f1|sed 's/\.git$//')
182-
path=$(echo "$url_line"|cut -d':' -f1 | cut -d'/' -f2-)
183-
if [[ "$url" == "https://platform.openai.com/docs/api-reference/"* || "https://www.docker.com/get-started" == "$url" || "https://openai.com/index/whisper/" == "$url" ]]; then
184-
echo "Link "$url" from ${{github.workspace}}/$path need to be verified by a real person."
185-
else
186-
response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url")
187-
if [ "$response" -ne 200 ]; then
188-
echo "**********Validation failed, try again**********"
189-
response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url")
190-
if [ "$response_retry" -eq 200 ]; then
191-
echo "*****Retry successfully*****"
192-
else
193-
echo "Invalid link from ${{github.workspace}}/$path: $url"
194-
fail="TRUE"
195-
fi
196-
fi
197-
fi
198-
done
199-
fi
200-
done
201-
else
202-
echo "No changed .md file."
203-
fi
204-
205-
if [[ "$fail" == "TRUE" ]]; then
167+
consistency="TRUE"
168+
yamls=$(find .github/workflows/docker/compose/ -name '*-compose.yaml')
169+
for build_yaml in $yamls; do
170+
message=$(python3 .github/workflows/scripts/check-name-agreement.py "$build_yaml")
171+
if [[ "$message" != *"consistent"* ]]; then
172+
consistency="FALSE"
173+
echo "Inconsistent service name and image name found in file $build_yaml."
174+
echo "$message"
175+
fi
176+
done
177+
if [[ "$consistency" == "FALSE" ]]; then
178+
echo "Please ensure that the service and image names are consistent in build.yaml, otherwise we cannot guarantee that your image will be published correctly."
206179
exit 1
207-
else
208-
echo "All hyperlinks are valid."
209180
fi
210181
shell: bash
211182

212-
check-the-validity-of-relative-path:
183+
check-duplicated-image:
213184
runs-on: ubuntu-latest
214185
steps:
215-
- name: Clean up Working Directory
186+
- name: Clean Up Working Directory
216187
run: sudo rm -rf ${{github.workspace}}/*
217188

218-
- name: Checkout Repo GenAIComps
189+
- name: Checkout Repo
219190
uses: actions/checkout@v4
220-
with:
221-
fetch-depth: 0
222191

223-
- name: Checking Relative Path Validity
192+
- name: Check all the docker image build files
224193
run: |
194+
pip install PyYAML
225195
cd ${{github.workspace}}
226-
fail="FALSE"
227-
repo_name=${{ github.event.pull_request.head.repo.full_name }}
228-
if [ "$(echo "$repo_name"|cut -d'/' -f1)" != "opea-project" ]; then
229-
owner=$(echo "${{ github.event.pull_request.head.repo.full_name }}" |cut -d'/' -f1)
230-
branch="https://github.com/$owner/GenAIComps/tree/${{ github.event.pull_request.head.ref }}"
231-
else
232-
branch="https://github.com/opea-project/GenAIComps/blob/${{ github.event.pull_request.head.ref }}"
233-
fi
234-
link_head="https://github.com/opea-project/GenAIComps/blob/main"
235-
236-
merged_commit=$(git log -1 --format='%H')
237-
changed_files="$(git diff --name-status --diff-filter=ARM ${{ github.event.pull_request.base.sha }} ${merged_commit} | awk '/\.md$/ {print $NF}')"
238-
png_lines=$(grep -Eo '\]\([^)]+\)' --include='*.md' -r .|grep -Ev 'http')
239-
if [ -n "$png_lines" ]; then
240-
for png_line in $png_lines; do
241-
refer_path=$(echo "$png_line"|cut -d':' -f1 | cut -d'/' -f2-)
242-
png_path=$(echo "$png_line"|cut -d '(' -f2 | cut -d ')' -f1)
243-
244-
if [[ "${png_path:0:1}" == "/" ]]; then
245-
check_path=$png_path
246-
elif [[ "$png_path" == *#* ]]; then
247-
relative_path=$(echo "$png_path" | cut -d '#' -f1)
248-
if [ -n "$relative_path" ]; then
249-
check_path=$(dirname "$refer_path")/$relative_path
250-
png_path=$(echo "$png_path" | awk -F'#' '{print "#" $2}')
251-
else
252-
check_path=$refer_path
253-
fi
254-
else
255-
check_path=$(dirname "$refer_path")/$png_path
256-
fi
257-
258-
if [ -e "$check_path" ]; then
259-
real_path=$(realpath $check_path)
260-
if [[ "$png_line" == *#* ]]; then
261-
if [ -n "changed_files" ] && echo "$changed_files" | grep -q "^${refer_path}$"; then
262-
url_dev=$branch$(echo "$real_path" | sed 's|.*/GenAIComps||')$png_path
263-
response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url_dev")
264-
if [ "$response" -ne 200 ]; then
265-
echo "**********Validation failed, try again**********"
266-
response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url_dev")
267-
if [ "$response_retry" -eq 200 ]; then
268-
echo "*****Retry successfully*****"
269-
else
270-
echo "Invalid path from ${{github.workspace}}/$refer_path: $png_path, link: $url_dev"
271-
fail="TRUE"
272-
fi
273-
else
274-
echo "Validation succeed $png_line"
275-
fi
276-
fi
277-
fi
278-
else
279-
echo "$check_path does not exist"
280-
fail="TRUE"
281-
fi
282-
done
283-
fi
284-
285-
if [[ "$fail" == "TRUE" ]]; then
286-
exit 1
287-
else
288-
echo "All hyperlinks are valid."
289-
fi
196+
build_files=""
197+
for f in `find .github/workflows/docker/compose/ -name '*.yaml'`; do
198+
build_files="$build_files $f"
199+
done
200+
python3 .github/workflows/scripts/check_duplicated_image.py $build_files
290201
shell: bash

‎.github/workflows/pr-link-path-scan.yaml‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
name: Check hyperlinks and relative path validity
5+
6+
on:
7+
pull_request:
8+
branches: [main]
9+
types: [opened, reopened, ready_for_review, synchronize]
10+
11+
# If there is a new commit, the previous jobs will be canceled
12+
concurrency:
13+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
14+
cancel-in-progress: true
15+
16+
jobs:
17+
check-the-validity-of-hyperlinks-in-README:
18+
runs-on: ubuntu-latest
19+
steps:
20+
- name: Clean Up Working Directory
21+
run: sudo rm -rf ${{github.workspace}}/*
22+
23+
- name: Checkout Repo GenAIComps
24+
uses: actions/checkout@v4
25+
with:
26+
fetch-depth: 0
27+
28+
- name: Check the Validity of Hyperlinks
29+
# ignore_links=("https://platform.openai.com/docs/api-reference/fine-tuning"
30+
# "https://platform.openai.com/docs/api-reference/"
31+
# "https://openai.com/index/whisper/"
32+
# "https://platform.openai.com/docs/api-reference/chat/create")
33+
run: |
34+
cd ${{github.workspace}}
35+
fail="FALSE"
36+
merged_commit=$(git log -1 --format='%H')
37+
changed_files="$(git diff --name-status --diff-filter=ARM ${{ github.event.pull_request.base.sha }} ${merged_commit} | awk '/\.md$/ {print $NF}')"
38+
if [ -n "$changed_files" ]; then
39+
for changed_file in $changed_files; do
40+
echo $changed_file
41+
url_lines=$(grep -H -Eo '\]\(http[s]?://[^)]+\)' "$changed_file" | grep -Ev 'GenAIComps/blob/main') || true
42+
if [ -n "$url_lines" ]; then
43+
for url_line in $url_lines; do
44+
url=$(echo "$url_line"|cut -d '(' -f2 | cut -d ')' -f1|sed 's/\.git$//')
45+
path=$(echo "$url_line"|cut -d':' -f1 | cut -d'/' -f2-)
46+
if [[ "$url" == "https://platform.openai.com/docs/api-reference/"* || "https://www.docker.com/get-started" == "$url" || "https://openai.com/index/whisper/" == "$url" ]]; then
47+
echo "Link "$url" from ${{github.workspace}}/$path need to be verified by a real person."
48+
else
49+
response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url")
50+
if [ "$response" -ne 200 ]; then
51+
echo "**********Validation failed, try again**********"
52+
response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url")
53+
if [ "$response_retry" -eq 200 ]; then
54+
echo "*****Retry successfully*****"
55+
else
56+
echo "Invalid link from ${{github.workspace}}/$path: $url"
57+
fail="TRUE"
58+
fi
59+
fi
60+
fi
61+
done
62+
fi
63+
done
64+
else
65+
echo "No changed .md file."
66+
fi
67+
68+
if [[ "$fail" == "TRUE" ]]; then
69+
exit 1
70+
else
71+
echo "All hyperlinks are valid."
72+
fi
73+
shell: bash
74+
75+
check-the-validity-of-relative-path:
76+
runs-on: ubuntu-latest
77+
steps:
78+
- name: Clean up Working Directory
79+
run: sudo rm -rf ${{github.workspace}}/*
80+
81+
- name: Checkout Repo GenAIComps
82+
uses: actions/checkout@v4
83+
with:
84+
fetch-depth: 0
85+
86+
- name: Checking Relative Path Validity
87+
run: |
88+
cd ${{github.workspace}}
89+
fail="FALSE"
90+
repo_name=${{ github.event.pull_request.head.repo.full_name }}
91+
if [ "$(echo "$repo_name"|cut -d'/' -f1)" != "opea-project" ]; then
92+
owner=$(echo "${{ github.event.pull_request.head.repo.full_name }}" |cut -d'/' -f1)
93+
branch="https://github.com/$owner/GenAIComps/tree/${{ github.event.pull_request.head.ref }}"
94+
else
95+
branch="https://github.com/opea-project/GenAIComps/blob/${{ github.event.pull_request.head.ref }}"
96+
fi
97+
link_head="https://github.com/opea-project/GenAIComps/blob/main"
98+
99+
merged_commit=$(git log -1 --format='%H')
100+
changed_files="$(git diff --name-status --diff-filter=ARM ${{ github.event.pull_request.base.sha }} ${merged_commit} | awk '/\.md$/ {print $NF}')"
101+
png_lines=$(grep -Eo '\]\([^)]+\)' --include='*.md' -r .|grep -Ev 'http')
102+
if [ -n "$png_lines" ]; then
103+
for png_line in $png_lines; do
104+
refer_path=$(echo "$png_line"|cut -d':' -f1 | cut -d'/' -f2-)
105+
png_path=$(echo "$png_line"|cut -d '(' -f2 | cut -d ')' -f1)
106+
107+
if [[ "${png_path:0:1}" == "/" ]]; then
108+
check_path=$png_path
109+
elif [[ "$png_path" == *#* ]]; then
110+
relative_path=$(echo "$png_path" | cut -d '#' -f1)
111+
if [ -n "$relative_path" ]; then
112+
check_path=$(dirname "$refer_path")/$relative_path
113+
png_path=$(echo "$png_path" | awk -F'#' '{print "#" $2}')
114+
else
115+
check_path=$refer_path
116+
fi
117+
else
118+
check_path=$(dirname "$refer_path")/$png_path
119+
fi
120+
121+
if [ -e "$check_path" ]; then
122+
real_path=$(realpath $check_path)
123+
if [[ "$png_line" == *#* ]]; then
124+
if [ -n "changed_files" ] && echo "$changed_files" | grep -q "^${refer_path}$"; then
125+
url_dev=$branch$(echo "$real_path" | sed 's|.*/GenAIComps||')$png_path
126+
response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url_dev")
127+
if [ "$response" -ne 200 ]; then
128+
echo "**********Validation failed, try again**********"
129+
response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url_dev")
130+
if [ "$response_retry" -eq 200 ]; then
131+
echo "*****Retry successfully*****"
132+
else
133+
echo "Invalid path from ${{github.workspace}}/$refer_path: $png_path, link: $url_dev"
134+
fail="TRUE"
135+
fi
136+
else
137+
echo "Validation succeed $png_line"
138+
fi
139+
fi
140+
fi
141+
else
142+
echo "$check_path does not exist"
143+
fail="TRUE"
144+
fi
145+
done
146+
fi
147+
148+
if [[ "$fail" == "TRUE" ]]; then
149+
exit 1
150+
else
151+
echo "All hyperlinks are valid."
152+
fi
153+
shell: bash

0 commit comments

Comments
 (0)