Skip to content

Commit b12ca80

Browse files
c1truac1trua
committed
ci(ops): add OSS presign diagnostics
1 parent 94dddad commit b12ca80

1 file changed

Lines changed: 240 additions & 12 deletions

File tree

‎.github/workflows/test-ops.yml‎

Lines changed: 240 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,22 @@ on:
1111
- mailpit_latest_code
1212
- compose_ps
1313
- api_logs
14+
- api_logs_by_request_id
15+
- api_recent_errors
16+
- api_runtime_env
17+
- oss_config
18+
- ecs_ram_role_metadata
19+
- oss_network_probe
20+
- presign_debug_bundle
1421
api_log_lines:
15-
description: Number of API log lines for api_logs
22+
description: Number of API log lines for log actions
1623
required: false
17-
default: "120"
24+
default: "1000"
25+
type: string
26+
request_id:
27+
description: Optional request_id for api_logs_by_request_id or presign_debug_bundle
28+
required: false
29+
default: ""
1830
type: string
1931

2032
permissions:
@@ -55,6 +67,7 @@ jobs:
5567
env:
5668
ACTION: ${{ inputs.action }}
5769
API_LOG_LINES: ${{ inputs.api_log_lines }}
70+
REQUEST_ID: ${{ inputs.request_id }}
5871
TEST_SERVER_HOST: ${{ secrets.TEST_SERVER_HOST }}
5972
TEST_SERVER_USERNAME: ${{ secrets.TEST_SERVER_USERNAME }}
6073
run: |
@@ -66,13 +79,19 @@ jobs:
6679
exit 1
6780
;;
6881
esac
69-
if [ "$API_LOG_LINES" -lt 1 ] || [ "$API_LOG_LINES" -gt 500 ]; then
70-
echo "api_log_lines must be between 1 and 500" >&2
82+
if [ "$API_LOG_LINES" -lt 1 ] || [ "$API_LOG_LINES" -gt 5000 ]; then
83+
echo "api_log_lines must be between 1 and 5000" >&2
7184
exit 1
7285
fi
86+
case "$REQUEST_ID" in
87+
*[!A-Za-z0-9_.:-]*)
88+
echo "request_id contains unsupported characters" >&2
89+
exit 1
90+
;;
91+
esac
7392
7493
ssh -i ~/.ssh/id_test_server "$TEST_SERVER_USERNAME@$TEST_SERVER_HOST" \
75-
"ACTION=$ACTION API_LOG_LINES=$API_LOG_LINES /bin/sh -s" <<'REMOTE'
94+
"ACTION=$ACTION API_LOG_LINES=$API_LOG_LINES REQUEST_ID=$REQUEST_ID /bin/sh -s" <<'REMOTE'
7695
set -eu
7796
cd /opt/cixing
7897
@@ -84,6 +103,183 @@ jobs:
84103
$DOCKER compose --env-file /opt/cixing/deploy.env -f /opt/cixing/docker-compose.yml "$@"
85104
}
86105
106+
print_section() {
107+
printf '\n== %s ==\n' "$1"
108+
}
109+
110+
collect_api_logs() {
111+
logs_file="$(mktemp)"
112+
if compose logs --no-color --tail="$API_LOG_LINES" api > "$logs_file"; then
113+
return 0
114+
fi
115+
status="$?"
116+
rm -f "$logs_file"
117+
return "$status"
118+
}
119+
120+
cleanup_api_logs() {
121+
if [ "${logs_file:-}" != "" ]; then
122+
rm -f "$logs_file"
123+
fi
124+
}
125+
126+
print_api_logs() {
127+
collect_api_logs
128+
grep -v '"path":"/healthz"' "$logs_file" || true
129+
cleanup_api_logs
130+
}
131+
132+
print_api_logs_by_request_id() {
133+
if [ -z "$REQUEST_ID" ]; then
134+
echo "request_id is required for this action" >&2
135+
exit 1
136+
fi
137+
collect_api_logs
138+
if ! grep -F "$REQUEST_ID" "$logs_file"; then
139+
echo "No API log lines found for request_id=$REQUEST_ID in the latest $API_LOG_LINES lines" >&2
140+
fi
141+
cleanup_api_logs
142+
}
143+
144+
print_api_recent_errors() {
145+
collect_api_logs
146+
grep -E '"status":5[0-9][0-9]' "$logs_file" || echo "No 5xx API access logs found in the latest $API_LOG_LINES lines"
147+
cleanup_api_logs
148+
}
149+
150+
print_api_runtime_env() {
151+
api_container_id="$(compose ps -q api || true)"
152+
if [ -z "$api_container_id" ]; then
153+
echo "API container not found" >&2
154+
return 0
155+
fi
156+
$DOCKER inspect --format 'image={{.Config.Image}} state={{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}}' "$api_container_id"
157+
echo "selected_env:"
158+
$DOCKER inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$api_container_id" \
159+
| grep -E '^(CONFIG_FILE=|OSS_|ALIYUN_|ALIBABA_)' \
160+
| sed -E \
161+
-e 's/([^=]*(SECRET|TOKEN|PASSWORD)[^=]*)=.*/\1=<redacted>/' \
162+
-e 's/(OSS_ACCESS_KEY_ID)=.*/\1=<redacted>/' \
163+
|| true
164+
}
165+
166+
get_oss_value() {
167+
key="$1"
168+
awk -v key="$key" '
169+
/^oss:/ { in_oss=1; next }
170+
in_oss && /^[^[:space:]][^:]*:/ { exit }
171+
in_oss {
172+
line=$0
173+
sub(/^[[:space:]]+/, "", line)
174+
if (index(line, key ":") == 1) {
175+
sub(/^[^:]+:[[:space:]]*/, "", line)
176+
gsub(/^"/, "", line)
177+
gsub(/"$/, "", line)
178+
print line
179+
exit
180+
}
181+
}
182+
' /opt/cixing/config/config.yaml
183+
}
184+
185+
print_oss_config() {
186+
if [ ! -f /opt/cixing/config/config.yaml ]; then
187+
echo "/opt/cixing/config/config.yaml not found" >&2
188+
exit 1
189+
fi
190+
ls -l /opt/cixing/config/config.yaml /opt/cixing/deploy.env /opt/cixing/docker-compose.yml 2>/dev/null || true
191+
awk '
192+
/^oss:/ { in_oss=1; print; next }
193+
in_oss && /^[^[:space:]][^:]*:/ { exit }
194+
in_oss {
195+
line=$0
196+
if (line ~ /^[[:space:]]+access_key_id:/) {
197+
sub(/:.*/, ": <redacted>", line)
198+
}
199+
if (line ~ /^[[:space:]]+access_key_secret:/) {
200+
sub(/:.*/, ": <redacted>", line)
201+
}
202+
if (line ~ /^[[:space:]]+assume_role_external_id:/) {
203+
sub(/:.*/, ": <redacted>", line)
204+
}
205+
print line
206+
}
207+
' /opt/cixing/config/config.yaml
208+
}
209+
210+
metadata_get() {
211+
path="$1"
212+
token="$(curl -fsS -X PUT --connect-timeout 2 --max-time 4 \
213+
-H "X-aliyun-ecs-metadata-token-ttl-seconds: 60" \
214+
http://100.100.100.200/latest/api/token 2>/dev/null || true)"
215+
if [ -n "$token" ]; then
216+
curl -fsS --connect-timeout 2 --max-time 4 \
217+
-H "X-aliyun-ecs-metadata-token: $token" \
218+
"http://100.100.100.200$path"
219+
else
220+
curl -fsS --connect-timeout 2 --max-time 4 "http://100.100.100.200$path"
221+
fi
222+
}
223+
224+
print_ecs_ram_role_metadata() {
225+
configured_role="$(get_oss_value ecs_role_name)"
226+
echo "configured_ecs_role_name=${configured_role:-<empty>}"
227+
228+
roles="$(metadata_get /latest/meta-data/ram/security-credentials/ 2>/dev/null || true)"
229+
if [ -z "$roles" ]; then
230+
echo "No ECS RAM role metadata returned from 100.100.100.200" >&2
231+
return 0
232+
fi
233+
234+
echo "metadata_roles:"
235+
printf '%s\n' "$roles"
236+
237+
role="$configured_role"
238+
if [ -z "$role" ]; then
239+
role="$(printf '%s\n' "$roles" | head -n 1)"
240+
fi
241+
if [ -z "$role" ]; then
242+
echo "No role name available for credential metadata lookup" >&2
243+
return 0
244+
fi
245+
246+
echo "metadata_credentials_for=$role"
247+
metadata_get "/latest/meta-data/ram/security-credentials/$role" 2>/dev/null \
248+
| sed -E \
249+
-e 's/"AccessKeyId"[[:space:]]*:[[:space:]]*"[^"]+"/"AccessKeyId":"<redacted>"/g' \
250+
-e 's/"AccessKeySecret"[[:space:]]*:[[:space:]]*"[^"]+"/"AccessKeySecret":"<redacted>"/g' \
251+
-e 's/"SecurityToken"[[:space:]]*:[[:space:]]*"[^"]+"/"SecurityToken":"<redacted>"/g' \
252+
|| echo "Failed to fetch credential metadata for role=$role" >&2
253+
}
254+
255+
probe_url() {
256+
label="$1"
257+
url="$2"
258+
if [ -z "$url" ]; then
259+
echo "$label=<empty>"
260+
return 0
261+
fi
262+
case "$url" in
263+
http://*|https://*) ;;
264+
*) url="https://$url" ;;
265+
esac
266+
printf '%s=%s ' "$label" "$url"
267+
if curl -sS -I -o /dev/null --connect-timeout 5 --max-time 10 \
268+
-w 'http_code=%{http_code} remote_ip=%{remote_ip} time_total=%{time_total}\n' "$url"; then
269+
return 0
270+
fi
271+
echo "connection_failed"
272+
}
273+
274+
print_oss_network_probe() {
275+
sts_endpoint="$(get_oss_value assume_role_sts_endpoint)"
276+
public_endpoint="$(get_oss_value public_endpoint)"
277+
internal_endpoint="$(get_oss_value internal_endpoint)"
278+
probe_url "assume_role_sts_endpoint" "$sts_endpoint"
279+
probe_url "public_endpoint" "$public_endpoint"
280+
probe_url "internal_endpoint" "$internal_endpoint"
281+
}
282+
87283
case "$ACTION" in
88284
mailpit_latest_code)
89285
raw="$(curl -fsS http://127.0.0.1:8025/api/v1/message/latest/raw)"
@@ -98,15 +294,47 @@ jobs:
98294
compose ps
99295
;;
100296
api_logs)
101-
logs_file="$(mktemp)"
102-
if compose logs --tail="$API_LOG_LINES" api > "$logs_file"; then
103-
grep -v '"path":"/healthz"' "$logs_file" || true
104-
rm -f "$logs_file"
297+
print_api_logs
298+
;;
299+
api_logs_by_request_id)
300+
print_api_logs_by_request_id
301+
;;
302+
api_recent_errors)
303+
print_api_recent_errors
304+
;;
305+
api_runtime_env)
306+
print_api_runtime_env
307+
;;
308+
oss_config)
309+
print_oss_config
310+
;;
311+
ecs_ram_role_metadata)
312+
print_ecs_ram_role_metadata
313+
;;
314+
oss_network_probe)
315+
print_oss_network_probe
316+
;;
317+
presign_debug_bundle)
318+
print_section "Server Time"
319+
date -u
320+
TZ=Asia/Hong_Kong date
321+
print_section "Compose Status"
322+
compose ps
323+
print_section "API Runtime Env"
324+
print_api_runtime_env
325+
print_section "API Logs"
326+
if [ -n "$REQUEST_ID" ]; then
327+
print_api_logs_by_request_id
105328
else
106-
status="$?"
107-
rm -f "$logs_file"
108-
exit "$status"
329+
print_api_recent_errors
330+
print_api_logs
109331
fi
332+
print_section "OSS Config"
333+
print_oss_config
334+
print_section "ECS RAM Role Metadata"
335+
print_ecs_ram_role_metadata
336+
print_section "OSS Network Probe"
337+
print_oss_network_probe
110338
;;
111339
*)
112340
echo "Unsupported action: $ACTION" >&2

0 commit comments

Comments
 (0)