Skip to content

Commit 6a99d43

Browse files
GhostbabyGhostbaby
authored
Merge pull request #5 from cen-ngc5139/ghostbaby
feat: enhance Makefile for E2E testing and update Docker CI workflow
2 parents f89f0b9 + 615932a commit 6a99d43

32 files changed

+35502
-2
lines changed

‎.github/workflows/docker-image.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: Docker Image CI
22

33
on:
44
push:
5-
branches: ["main", "ghostbaby"]
5+
branches: ["main"]
66

77
jobs:
88
build:

‎.github/workflows/push.yml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
name: Push request
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- main
7+
- ghostbaby
8+
9+
jobs:
10+
e2e-build:
11+
runs-on: ubuntu-22.04
12+
steps:
13+
- name: Checkout
14+
uses: actions/checkout@main
15+
- name: Install dependencies
16+
run: |
17+
sudo apt-get update && sudo apt-get install -y \
18+
golang-1.22 \
19+
git \
20+
make \
21+
gcc \
22+
clang \
23+
llvm \
24+
&& mkdir -p ./internal/binary && sudo rm -rf /var/lib/apt/lists/*
25+
- name: Build binary
26+
run: |
27+
make build
28+
- uses: actions/upload-artifact@master
29+
with:
30+
name: shepherd
31+
path: |
32+
./cmd/shepherd
33+
- uses: actions/upload-artifact@master
34+
with:
35+
name: config.yaml
36+
path: |
37+
./cmd/config.yaml
38+
e2e-6_10:
39+
needs: e2e-build
40+
uses: ./.github/workflows/reusable-workflow.yml
41+
with:
42+
kernel: "6.10"
43+
e2e-6_6:
44+
needs: e2e-build
45+
uses: ./.github/workflows/reusable-workflow.yml
46+
with:
47+
kernel: "6.6"
48+
e2e-6_1:
49+
needs: e2e-build
50+
uses: ./.github/workflows/reusable-workflow.yml
51+
with:
52+
kernel: "6.1"
53+
e2e-5_15:
54+
needs: e2e-build
55+
uses: ./.github/workflows/reusable-workflow.yml
56+
with:
57+
kernel: "5.15"
58+
e2e-5_10:
59+
needs: e2e-build
60+
uses: ./.github/workflows/reusable-workflow.yml
61+
with:
62+
kernel: "5.10"
63+
e2e-5_8:
64+
needs: e2e-build
65+
uses: ./.github/workflows/reusable-workflow.yml
66+
with:
67+
kernel: "5.8"
68+
e2e-5_4:
69+
needs: e2e-build
70+
uses: ./.github/workflows/reusable-workflow.yml
71+
with:
72+
kernel: "5.4"

‎.github/workflows/reusable-workflow.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
name: reusable jobs
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
kernel:
7+
description: 'Kernel'
8+
required: true
9+
type: string
10+
11+
jobs:
12+
e2e-kernel:
13+
runs-on: ubuntu-22.04
14+
steps:
15+
- name: Checkout
16+
uses: actions/checkout@main
17+
- name: Install qemu & friends
18+
run: |
19+
sudo apt-get update
20+
sudo apt-get install -y qemu-utils qemu-system-x86 sshpass
21+
- uses: actions/download-artifact@master
22+
with:
23+
name: shepherd
24+
path: ./cmd/
25+
- uses: actions/download-artifact@master
26+
with:
27+
name: config.yaml
28+
path: ./cmd/
29+
- name: Run e2e tests on kernel ${{ inputs.kernel }}
30+
run: |
31+
KERNEL=${{ inputs.kernel }} make e2e

‎Makefile‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,53 @@ image:
2828
docker buildx create --use
2929
docker buildx build --platform linux/amd64 -t ghostbaby/shepherd:v0.0.1-amd64 --push .
3030
docker buildx build --platform linux/arm64 -t ghostbaby/shepherd:v0.0.1-arm64 --push .
31+
32+
.ONESHELL:
33+
prepare_e2e_filesystem:
34+
cd ./tests/e2e/vm/filesystem
35+
# build filesystem image and store as tar archive
36+
DOCKER_BUILDKIT=1 docker build --output "type=tar,dest=filesystem.tar" .
37+
# convert tar to qcow2 image
38+
sudo virt-make-fs --format=qcow2 --size=+100M filesystem.tar filesystem-large.qcow2
39+
# reduce size of image
40+
qemu-img convert filesystem-large.qcow2 -O qcow2 filesystem.qcow2
41+
# reduce size by packing
42+
zip filesystem.zip filesystem.qcow2
43+
# remove unnecessary files
44+
rm -f filesystem-large.qcow2 filesystem.qcow2 filesystem.tar
45+
46+
.ONESHELL:
47+
start_qemu:
48+
cd ./tests/e2e/vm/filesystem
49+
rm -f filesystem.qcow2 filesystem-diff.qcow2
50+
unzip ./filesystem.zip
51+
sudo qemu-img create -f qcow2 -b filesystem.qcow2 -F qcow2 filesystem-diff.qcow2
52+
PWD=$(pwd)
53+
sudo qemu-system-x86_64 \
54+
-cpu host \
55+
-m 4G \
56+
-smp 4 \
57+
-kernel ${PWD}/tests/e2e/vm/kernels/${KERNEL}/bzImage \
58+
-append "console=ttyS0 root=/dev/sda rw" \
59+
-drive file="${PWD}/tests/e2e/vm/filesystem/filesystem-diff.qcow2,format=qcow2" \
60+
-net nic -net user,hostfwd=tcp::10022-:22,hostfwd=tcp::16676-:6676,hostfwd=tcp::10443-:443 \
61+
-enable-kvm \
62+
-pidfile qemu.pid \
63+
-nographic &
64+
65+
.ONESHELL:
66+
prepare_e2e: start_qemu
67+
while ! nc -z 127.0.0.1 10022 ; do echo "waiting for ssh"; sleep 1; done
68+
sshpass -p root scp -o 'StrictHostKeyChecking no' -P 10022 ./cmd/shepherd [email protected]:/root/shepherd
69+
sshpass -p root scp -o 'StrictHostKeyChecking no' -P 10022 ./cmd/config.yaml [email protected]:/root/config.yaml
70+
sshpass -p root ssh -p 10022 [email protected] 'chmod 0655 /root/shepherd && systemctl start shepherd.service'
71+
while ! sshpass -p root ssh -p 10022 [email protected] 'systemctl is-active shepherd.service' ; do echo "waiting for shepherd service"; sleep 1; done
72+
73+
.ONESHELL:
74+
e2e: prepare_e2e
75+
ifconfig
76+
RC=$$?
77+
pwd
78+
sshpass -p root ssh -p 10022 [email protected] 'ls /root && ls /root/log/'
79+
sudo cat ./tests/e2e/vm/filesystem/qemu.pid | sudo xargs kill
80+
exit $$RC

‎internal/output/sched_delay.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package output
22

33
import (
44
"context"
5+
"fmt"
56
"os"
67

78
"github.com/ClickHouse/clickhouse-go/v2"
@@ -31,7 +32,7 @@ func ProcessSchedDelay(coll *ebpf.Collection, ctx context.Context, cfg config.Co
3132
log.Fatalf("failed to connect to clickhouse: %v", err)
3233
}
3334

34-
// defer conn.Close()
35+
defer conn.Close()
3536

3637
// 准备批量插入语句
3738
batch, err := conn.PrepareBatch(ctx, `
@@ -61,6 +62,8 @@ func ProcessSchedDelay(coll *ebpf.Collection, ctx context.Context, cfg config.Co
6162
continue
6263
}
6364

65+
fmt.Println(event)
66+
6467
batch, count, err = insertSchedMetrics(ctx, conn, batch, event, count)
6568
if err != nil {
6669
log.Errorf("failed to insert sched metrics: %v", err)

‎tests/e2e/README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
### e2e tests for various Linux kernels
2+
3+
- e2e tests module uses `qemu` emulation tool to emulate various linux kernels, see target `start_qemu` in [Makefile](../../Makefile)
4+
- `qemu` uses precompiled linux kernels, see [kernel compilation README](./vm/kernels/README.md)
5+
- `qemu` uses prepared filesystem based on [Dockerfile](./vm/filesystem/Dockerfile)

‎tests/e2e/resources/fields_bytes.json‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
{
2+
"edges_fields": [
3+
{
4+
"field_name": "id",
5+
"type": "string"
6+
},
7+
{
8+
"field_name": "source",
9+
"type": "string"
10+
},
11+
{
12+
"field_name": "target",
13+
"type": "string"
14+
},
15+
{
16+
"field_name": "mainStat",
17+
"type": "string"
18+
},
19+
{
20+
"field_name": "secondaryStat",
21+
"type": "string"
22+
}
23+
],
24+
"nodes_fields": [
25+
{
26+
"field_name": "id",
27+
"type": "string"
28+
},
29+
{
30+
"field_name": "title",
31+
"type": "string"
32+
},
33+
{
34+
"field_name": "subTitle",
35+
"type": "string"
36+
},
37+
{
38+
"field_name": "mainStat",
39+
"displayName": "Bytes received ",
40+
"type": "string"
41+
},
42+
{
43+
"field_name": "secondaryStat",
44+
"displayName": "Bytes responded ",
45+
"type": "string"
46+
},
47+
{
48+
"color": "blue",
49+
"field_name": "arc__1",
50+
"type": "number",
51+
"displayName": "Bytes received"
52+
},
53+
{
54+
"color": "yellow",
55+
"field_name": "arc__2",
56+
"type": "number",
57+
"displayName": "Bytes responded"
58+
}
59+
]
60+
}

‎tests/e2e/resources/fields_connection.json‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
{
2+
"edges_fields": [
3+
{
4+
"field_name": "id",
5+
"type": "string"
6+
},
7+
{
8+
"field_name": "source",
9+
"type": "string"
10+
},
11+
{
12+
"field_name": "target",
13+
"type": "string"
14+
},
15+
{
16+
"field_name": "mainStat",
17+
"type": "string"
18+
},
19+
{
20+
"field_name": "secondaryStat",
21+
"type": "string"
22+
}
23+
],
24+
"nodes_fields": [
25+
{
26+
"field_name": "id",
27+
"type": "string"
28+
},
29+
{
30+
"field_name": "title",
31+
"type": "string"
32+
},
33+
{
34+
"field_name": "subTitle",
35+
"type": "string"
36+
},
37+
{
38+
"field_name": "mainStat",
39+
"displayName": "All connections ",
40+
"type": "string"
41+
},
42+
{
43+
"field_name": "secondaryStat",
44+
"displayName": "Persistent connections ",
45+
"type": "string"
46+
},
47+
{
48+
"color": "green",
49+
"field_name": "arc__1",
50+
"type": "number",
51+
"displayName": "Persistent connections"
52+
},
53+
{
54+
"color": "red",
55+
"field_name": "arc__2",
56+
"type": "number",
57+
"displayName": "Short-lived connections"
58+
}
59+
]
60+
}

‎tests/e2e/resources/fields_duration.json‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
{
2+
"edges_fields": [
3+
{
4+
"field_name": "id",
5+
"type": "string"
6+
},
7+
{
8+
"field_name": "source",
9+
"type": "string"
10+
},
11+
{
12+
"field_name": "target",
13+
"type": "string"
14+
},
15+
{
16+
"field_name": "mainStat",
17+
"type": "string"
18+
},
19+
{
20+
"field_name": "secondaryStat",
21+
"type": "string"
22+
}
23+
],
24+
"nodes_fields": [
25+
{
26+
"field_name": "id",
27+
"type": "string"
28+
},
29+
{
30+
"field_name": "title",
31+
"type": "string"
32+
},
33+
{
34+
"field_name": "subTitle",
35+
"type": "string"
36+
},
37+
{
38+
"field_name": "mainStat",
39+
"displayName": "Average duration ",
40+
"type": "string"
41+
},
42+
{
43+
"field_name": "secondaryStat",
44+
"displayName": "Max duration ",
45+
"type": "string"
46+
},
47+
{
48+
"color": "purple",
49+
"field_name": "arc__1",
50+
"type": "number",
51+
"displayName": "Average duration"
52+
},
53+
{
54+
"color": "white",
55+
"field_name": "arc__2",
56+
"type": "number",
57+
"displayName": "Max duration"
58+
}
59+
]
60+
}

0 commit comments

Comments
 (0)