Skip to content

Commit cfcefcf

Browse files
authored
CI: testing frame order and fps (realsenseai#331)
1 parent 60e95ab commit cfcefcf

6 files changed

Lines changed: 205 additions & 8 deletions

File tree

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException
2+
3+
pipeline {
4+
agent { label 'rs-orin-01.realsenseai.com' }
5+
6+
options {
7+
timestamps()
8+
timeout(time: 30, unit: 'MINUTES')
9+
}
10+
11+
parameters {
12+
booleanParam(
13+
name: 'REBOOT',
14+
description: 'When set the artifacts will be installed to target agent and the agent rebooted',
15+
defaultValue: false,
16+
)
17+
string(
18+
name: 'ARTIFACTS',
19+
description: 'Jenkins job for artifacts to be installed on target agent',
20+
defaultValue: "D4xx_Kernel_Module_Jetson_JP6"
21+
)
22+
string(
23+
name: 'BUILD',
24+
description: 'Build number for artifacts. Leave empty for last successful'
25+
)
26+
}
27+
28+
stages {
29+
stage('Get artifacts') {
30+
when {
31+
expression { params.REBOOT == true }
32+
}
33+
steps {
34+
script {
35+
def buildSelector
36+
if (params.BUILD?.trim()) {
37+
buildSelector = specific(params.BUILD)
38+
} else {
39+
buildSelector = lastSuccessful()
40+
}
41+
42+
copyArtifacts filter: '**/*.tar.bz2',
43+
projectName: params.ARTIFACTS,
44+
flatten: true,
45+
selector: buildSelector,
46+
target: 'artifacts/'
47+
}
48+
}
49+
}
50+
stage('Install artifacts') {
51+
when {
52+
expression { params.REBOOT == true }
53+
}
54+
steps {
55+
sh """#!/bin/sh
56+
rm -rf lib/modules
57+
tar -xf artifacts/rootfs.tar.bz2
58+
# external script on agent to install artifacts
59+
sudo install.tegra.artifacts.sh
60+
"""
61+
}
62+
post {
63+
success {
64+
sh 'nohup sudo reboot &'
65+
}
66+
}
67+
}
68+
stage('Pytest') {
69+
steps {
70+
sh """#!/bin/sh
71+
pytest --tb=no -s test
72+
"""
73+
}
74+
}
75+
}
76+
}

test/install.tegra.artifacts.sh

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# add jenkins user to sudo group
5+
# add the following line to /etc/sudoers for jenkins user, here nvidia
6+
# nvidia ALL=(root) NOPASSWD: /sbin/reboot, /sbin/install.tegra.artifacts.sh
7+
8+
RELEASE=$(ls lib/modules)
9+
cp -R lib/modules/* /lib/modules
10+
[[ -f /boot/Image ]] && rm /boot/Image
11+
[[ -f /boot/initrd.img ]] && rm /boot/initrd.img
12+
cp boot/Image /boot/Image-$RELEASE
13+
ln -s /boot/Image-$RELEASE /boot/Image
14+
update-initramfs -uk $RELEASE
15+
ln -s /boot/initrd.img-$RELEASE /boot/initrd.img
16+
cp boot/*.dtbo /boot/
17+

test/pytest.ini

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[pytest]
2+
markers =
3+
d457: realsense D457 camera
4+

test/run_ci.py

100644100755
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ def run_test(cmd):
4646
timeout=200,
4747
check=True )
4848
except Exception as e:
49-
print( "Exception occurred.")
49+
print("Exception occurred: {}".format( e ))
5050

5151

5252
def run_tests_on_d457():
@@ -73,6 +73,7 @@ def run_tests_on_d457():
7373
except getopt.GetoptError as err:
7474
print( err )
7575
usage()
76+
sys.exit ( 1 )
7677

7778
for opt, arg in opts:
7879
if opt in ('-h', '--help'):

test/test_fps.py

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
import subprocess
2+
import pytest
3+
import re
4+
5+
@pytest.mark.d457
6+
@pytest.mark.parametrize("frames", {150})
7+
@pytest.mark.parametrize("device", {'0', '2'})
8+
def test_fps(device, frames):
9+
try:
10+
print(f"\nDevice: {device}")
11+
formats = get_formats(device)
12+
for w, h in formats:
13+
print(f"Format: {w}x{h}")
14+
for FPS in formats[(w, h)]:
15+
cmd = [ "v4l2-ctl",
16+
f"-d{device}",
17+
f"--set-fmt-video=width={w},height={h}",
18+
]
19+
subprocess.check_call(cmd)
20+
print(f"FPS/{FPS}:", end=' ')
21+
cmd = [ "v4l2-ctl",
22+
f"-d{device}",
23+
"-p",
24+
f"{FPS}",
25+
]
26+
subprocess.check_call(cmd, stdout=subprocess.DEVNULL)
27+
cmd = [ "v4l2-ctl",
28+
f"-d{device}",
29+
"--stream-mmap",
30+
"--stream-count",
31+
f"{frames}",
32+
"--verbose",
33+
]
34+
timeout = 4.0 * frames / FPS
35+
output = subprocess.run(cmd,
36+
check=True,
37+
text=True,
38+
capture_output=True,
39+
timeout=timeout).stderr.splitlines()
40+
last = None
41+
skip = True # skip first FPS measurement
42+
kpi = 5 # [%]
43+
count = 0
44+
for line in output:
45+
m = re.search(r"cap dqbuf:.*seq:\s*(\d*) bytesused:", line)
46+
if m:
47+
count += 1
48+
frame = int(m.group(1))
49+
# print(f"{frame}", end='')
50+
if last:
51+
assert frame > last, f"Repeated frame: {frame}"
52+
assert (frame - last) < 3, f"Frames dropped between: {last} and {frame}"
53+
m = re.search(r"delta:\s*(\d+\.\d+) ms", line)
54+
if m:
55+
fps = 1000 / float(m.group(1))
56+
# print(f"/{fps:.2f}", end='')
57+
if not skip:
58+
assert fps > FPS * (1 - kpi/100), f"FPS too low: {fps:.2f}/{FPS}"
59+
assert fps < FPS * (1 + kpi/100), f"FPS too high: {fps:.2f}/{FPS}"
60+
else:
61+
# print('?', end='')
62+
skip = False
63+
# print(end=',')
64+
last = frame
65+
print()
66+
assert last, "No frames arrived"
67+
assert count == frames, f"Missing frames: {count} < {frames}"
68+
except subprocess.TimeoutExpired:
69+
assert False, "No frames arrived"
70+
71+
def get_formats(device):
72+
cmd = [ "v4l2-ctl",
73+
"-d" + device,
74+
"--list-formats-ext",
75+
]
76+
output = subprocess.run(cmd,
77+
check=True,
78+
text=True,
79+
capture_output=True
80+
).stdout.splitlines()
81+
formats = {}
82+
last = None
83+
for line in output:
84+
m = re.search(r"\s*Size: Discrete\s*(\d+)x(\d+)", line)
85+
if m:
86+
w = int(m.group(1))
87+
h = int(m.group(2))
88+
last = (w, h)
89+
if not last in formats:
90+
formats[last] = set()
91+
continue
92+
m = re.search(r"\s*Interval: Discrete.*\((\d+\.\d+)\s+fps\)", line)
93+
if m:
94+
fps = float(m.group(1))
95+
if last:
96+
formats[last].add(fps)
97+
return formats

test/test_fw_version.py

100644100755
Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,16 @@
55
@pytest.mark.parametrize("device", {'0'})
66
def test_fw_version(device):
77
try:
8-
result = subprocess.check_call(["v4l2-ctl", "-d"+device, "-C", "fw_version"])
8+
key = "fw_version"
9+
result = subprocess.check_call(["v4l2-ctl", "-d"+device, "-C", key])
910
assert result == 0
1011

11-
std_output = subprocess.check_output(["v4l2-ctl", "-d"+device, "-C", "fw_version"])
12-
assert "fw version: " in std_output, "Couldn't fetch FW version"
12+
std_output = subprocess.check_output(["v4l2-ctl", "-d"+device, "-C", key])
13+
key += ": "
14+
assert key in std_output.decode(), "Couldn't fetch FW version"
1315

1416
# Remove the 'fw version: ' string from std output
15-
fw_version = int(std_output.replace("fw version: ", ""))
17+
fw_version = int(std_output.decode().replace(key, ""))
1618

1719
fw_version_str = str(fw_version>>24 & 0xFF) + "." + str(fw_version>>16 & 0xFF) + "." + str(fw_version>>8 & 0xFF) + "." + str(fw_version & 0xFF)
1820
print ("fw_version:", fw_version_str)
@@ -21,14 +23,14 @@ def test_fw_version(device):
2123
assert fw_version == (fw_version & 0x05FFFFFF), "Expected FW version is 5.x.x.x, but received {}".format(fw_version_str)
2224

2325
# Get DFU device name
24-
dfu_device = subprocess.check_output(["ls", "/sys/class/d4xx-class/"])
26+
dfu_device = subprocess.check_output(["ls", "/sys/class/d4xx-class/"]).decode()
2527
assert "d4xx-dfu-" in dfu_device, "D4xx DFU device not found"
2628

2729
# Get FW version from DFU device info
28-
dfu_device_info = subprocess.check_output(["cat", "/dev/"+dfu_device.strip()])
30+
dfu_device_info = subprocess.check_output(["cat", "/dev/"+dfu_device.strip()]).decode()
2931

3032
# Check whether the DFU info also has same FW version
3133
assert fw_version_str in dfu_device_info, "FW versions read through v4l2-ctl utility and DFU device info doesn't match"
3234

3335
except Exception as e:
34-
assert False, e
36+
assert False, "Exception caught during test: {}".format(e)

0 commit comments

Comments
 (0)