Skip to content

Commit 35486c6

Browse files
AtomicFSAtomicFS
committed
ci: add fwci test
- add fwci configuration files - add CI/CD job for triggering the FWCI testing suite Signed-off-by: AtomicFS <vojtech.vesely@9elements.com>
1 parent 9f38dca commit 35486c6

16 files changed

Lines changed: 743 additions & 1 deletion

File tree

‎.firmwareci/.jinja2_templates/defaults.j2‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
## Can be changed depending on the FWCI tester device used
2+
{%- set RPI_HOSTNAME = "fwci-dutctl-tester" %}
3+
{%- set RPI_FQDN = RPI_HOSTNAME ~ ".sec.9e.network" %}
4+
## Do not change values below, unless you know what you are doing
5+
{%- set PKG_NAME = "dutctl" %}
6+
{%- set DUTCTL_CONFIG_FILE = "/etc/dutctl.yaml" %}
7+
{%- set SYSTEMD_FILE = "dutagent.service" %}
8+
{%- set DUTAGENT_PORT = "2024" %}
9+
{%- set DUTCTL_ARGS = '"-s", "' ~ RPI_HOSTNAME ~ ':' ~ DUTAGENT_PORT ~ '"' %}
10+
{%- set DUTCTL_ARGS_STR = '-s ' ~ RPI_HOSTNAME ~ ':' ~ DUTAGENT_PORT %}
11+
## Values below should not change
12+
{%- set GPIO_PIN_TEST_OLIMEX = "21" %}
13+
{%- set FAKE_SERIAL_INPUT = "/home/oscar/ttyS1" %}
14+
{%- set FAKE_SERIAL_OUTPUT = "/home/oscar/ttyS2" %}

‎.firmwareci/.jinja2_templates/defaults.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
transport_oscar: &transport_oscar
2+
proto: ssh
3+
options:
4+
host: "[[attributes.Host]]"
5+
user: oscar
6+
identity_file: "/root/.ssh/fwci"
7+
# TODO: The ssh keys (identity_file) are hard-coded in the FWCI docker at the moment
8+
9+
transport_root: &transport_root
10+
proto: ssh
11+
options:
12+
host: "[[attributes.Host]]"
13+
user: root
14+
identity_file: "/root/.ssh/fwci"
15+
# TODO: The ssh keys (identity_file) are hard-coded in the FWCI docker at the moment

‎.firmwareci/.jinja2_templates/post.yaml.j2‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
post-stage:
2+
# ===========================================
3+
# Delete the files copied over for the test
4+
# ===========================================
5+
- cmd: shell
6+
name: Stop remotelab-server systemd service
7+
transport: *transport_root
8+
parameters:
9+
command: "systemctl stop {{ defaults.SYSTEMD_FILE }} || true"
10+
11+
- cmd: shell
12+
name: Uninstall the package
13+
transport: *transport_root
14+
parameters:
15+
command: "apt remove --assume-yes {{ defaults.PKG_NAME }} || true"
16+
17+
- cmd: shell
18+
name: Delete testing configuration file
19+
transport: *transport_root
20+
parameters:
21+
command: "rm {{ defaults.DUTCTL_CONFIG_FILE }} || true"
22+
23+
- cmd: shell
24+
name: Delete package
25+
transport: *transport_root
26+
parameters:
27+
command: "rm {{ defaults.PKG_NAME }}.deb || true"
28+
29+
- cmd: shell
30+
name: Shutdown fake serial
31+
transport: *transport_oscar
32+
parameters:
33+
command: "pkill socat || true"

‎.firmwareci/.jinja2_templates/pre.yaml.j2‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
pre-stage:
2+
# Fix permissions
3+
- cmd: cmd
4+
name: chmod dutctl
5+
transport:
6+
proto: local
7+
parameters:
8+
executable: chmod
9+
args: ["755", "[[input.DUTCTL]]"]
10+
11+
# ========================
12+
# Copy files over to RPI
13+
# ========================
14+
- cmd: copy
15+
name: Copy over {{ defaults.PKG_NAME }} debian package
16+
transport: *transport_root
17+
parameters:
18+
source: "[[input.REMOTE_PKG]]"
19+
destination: "{{ defaults.PKG_NAME }}.deb"
20+
21+
- cmd: cmd
22+
name: Install {{ defaults.PKG_NAME }} package
23+
transport: *transport_root
24+
parameters:
25+
executable: dpkg
26+
args: ["-i", "{{ defaults.PKG_NAME }}.deb"]
27+
28+
- cmd: copy
29+
name: Copy over test config file
30+
transport: *transport_root
31+
parameters:
32+
source: "[[input.CONFIG]]"
33+
destination: {{ defaults.DUTCTL_CONFIG_FILE }}
34+
35+
- cmd: cmd
36+
name: chmod config file
37+
transport: *transport_root
38+
parameters:
39+
executable: chmod
40+
args: ["644", {{ defaults.DUTCTL_CONFIG_FILE }}]
41+
42+
# ==============
43+
# Start server
44+
# ==============
45+
- cmd: cmd
46+
name: Reload systemctl daemon
47+
transport: *transport_root
48+
parameters:
49+
executable: systemctl
50+
args: ["daemon-reload"]
51+
52+
- cmd: cmd
53+
name: Start systemd service
54+
transport: *transport_root
55+
parameters:
56+
executable: systemctl
57+
args: ["start", {{ defaults.SYSTEMD_FILE }}]
58+
59+
- cmd: shell
60+
name: Status systemd service
61+
transport: *transport_root
62+
parameters:
63+
command: "systemctl status --no-pager {{ defaults.SYSTEMD_FILE }}"
64+
expect:
65+
- regex: "Active: active \\(running\\)"
66+
67+
# ==========================
68+
# Test dutctl help command
69+
# ==========================
70+
- cmd: cmd
71+
name: dutctl help
72+
transport:
73+
proto: local
74+
parameters:
75+
executable: "[[input.DUTCTL]]"
76+
args: ["--help"]

‎.firmwareci/README.md‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
# FirmwareCI configuration for dutctl testing
2+
3+
This directory contains FirmwareCI configurations and tests.
4+
5+
For detailed information on FirmwareCI, please refer to the [official documentation](https://docs.firmware-ci.com/).
6+
7+
8+
## Requirements
9+
10+
- Parametric and re-usable testing
11+
- Easy to expand
12+
- Reliable
13+
- Can be used in CI/CD to verify each pull request
14+
15+
16+
## Design decisions
17+
18+
### Jinja templating
19+
We want to test multiple features (flashing, power control, serial, etc). And to keep everything nicely organised we need to have each feature-specific test as it's own test.
20+
21+
However, the setup and tear-down of the tests is rather complex:
22+
- copy over the compiled binaries
23+
- copy over configuration files
24+
- spin up the server (agent)
25+
- execute the test
26+
- shut-down the server
27+
- delete the copied binaries
28+
- delete configuration files
29+
30+
In case of the serial feature, it also means spinning-up "fake" / "dummy" serial that can be used for the testing.
31+
32+
It would be stupid to copy-paste all this setup into each test file, especially because keeping all of them in sync would be a nightmare.
33+
34+
For this reason I have decided to use [jinja2 templates](https://jinja.palletsprojects.com/en/stable/api/) to automate this menial and error-prone work. To make it even simpler, I have also included [Taskfile](https://taskfile.dev/docs/guide) which has already all the jobs written there, so to generate all of the templates is as easy as simply calling `task jinja2:templates`.
35+
36+
Thanks to jinja templating, and how the tests are structured, tests are parametric and easy to expand.
37+
38+
39+
### Debian packaging magic
40+
As you might notice, instead of simply copying over the compiled binaries (`dutctl` / `dutagent` / ...) we use Debian packages.
41+
42+
The biggest motivator here to do this was to accommodate for setup and tear-down, and the differences between tested versions (future-proofing). This way, we leverage the power of package manager to make sure that the cleanup (uninstalling) of old file is complete and that no files are left behind! This makes sure that the environment is always pristine.
43+
44+
Another huge advantage is, that with this approach, that run-time dependencies are also handled by package manager. Because of this, the test hardware (Raspberry Pi) has minimal (if any) setup required for use in the FWCI tests. Just install package, test and uninstall! Easy!
45+
46+
47+
## Project structure
48+
49+
The most important files / directories are:
50+
- `.firmwareci/Taskfile.yml`
51+
- `.firmwareci/.jinja2_templates/`
52+
- `.firmwareci/workflows/`
53+
54+
### Taskfile
55+
The `.firmwareci/Taskfile.yml` is there to help and automate some mundane and repetitive jobs.
56+
57+
Contains task `jinja2:templates` which finds all jinja tempaltes in `.firmwareci/workflows/` and `.firmwareci/duts/` and runs a tempalting engine on them.
58+
59+
The list of the templates is created dynamically with the shell command:
60+
```bash
61+
find './workflows/' './duts/' -type f -name '*.yaml.j2'
62+
```
63+
64+
This list is then iterated over in a for-loop, such as that it takes `./duts/dut-rpi-fti-tester/dut.yaml.j2` as input, and produces `./duts/dut-rpi-fti-tester/dut.yaml`.
65+
66+
67+
### jinja2_templates
68+
The `.firmwareci/.jinja2_templates` contains the "common" building-blocks that are shared across templates. The `defaults.j2` and `defaults.yaml` are for storing some basics (they are intentionally separate):
69+
- `defaults.j2`
70+
- this is pure jinja2 syntax, and is imported into template with
71+
```j2
72+
{%- import ".jinja2_templates/defaults.j2" as defaults %}
73+
```
74+
- import means that the variables will become accessible, and can then be accessed in the template with:
75+
```j2
76+
{{ defaults.DUTCTL_ARGS }}
77+
```
78+
- `defaults.yaml`
79+
- this is pure YAML file to be `include`d into a template with
80+
```j2
81+
{% include ".jinja2_templates/defaults.yaml" with context %}
82+
```
83+
- essentially this is to just simply render the content of the file, ignoring any variable assignments or macros within it
84+
- technically speaking it could be renamed to `defaults.yaml.j2`, but it does not contain any jinja2 code and renaming it would have no effect
85+
- `pre.yaml.j2` and `post.yaml.j2`
86+
- for pre-stage and post-stage respectively
87+
- they are copy-pasted into each test (they are `include`d in tests, meaning that their content will be rendered)
88+
89+
90+
### workflows
91+
The `workflows` directory is where the tests live.
92+
93+
### Systemd service
94+
The `dutagent.service` file is a systemd service file to start a dutagent on dutworker.
95+
96+
At the moment it is only used for firmware-ci testing.
97+
98+
99+
## Adding completely new test
100+
To add a completely new tests from scratch, create a new `.yaml.j2` file in `.firmwareci/workflows/workflow-rpi-dutctl-tester/`. I would recommend to copy-paste some existing test (for example the `dummy-modules.yaml.j2` is rather simple) and use it as basis.
101+
102+
Then update the test according to your needs.
103+
104+
Run `task fwci:jinja2:templates` and it will automatically generate a complete tests. That is it. It is as simple as that.
105+
106+
Next, I recommend to run `fwci fwci:validate`.
107+
108+
As the last step, add all new files (even those jinja2 generated) into git commit. This is because FirmwareCI cannot pre-process the tests on it's own, which is why we have to include even the "generated code".

‎.firmwareci/Taskfile.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
---
2+
version: "3"
3+
4+
tasks:
5+
jinja2:templates:
6+
desc: Run jinja2 on template files
7+
cmds:
8+
- task: jinja2:find-and-process-all-tempaltes
9+
10+
fwci:validate:
11+
desc: Validate FWCI files
12+
cmds:
13+
- fwci validate
14+
15+
#================
16+
# Internal tasks
17+
#================
18+
19+
jinja2:find-and-process-all-tempaltes:
20+
desc: Run jinja2 on jinja2 templates
21+
internal: true
22+
label: 'jinja2-tests-{{.REPO}}'
23+
vars:
24+
WORKFLOWS:
25+
sh: find './workflows/' './duts/' -type f -name '*.yaml.j2'
26+
sources:
27+
- .jinja2_templates/*
28+
- duts/**/*.yaml.j2
29+
- workflows/workflow-*/defaults.j2
30+
- workflows/workflow-*/**/*.yaml.j2
31+
cmds:
32+
- for:
33+
var: WORKFLOWS
34+
task: jinja2:run-on-file
35+
vars:
36+
INPUTFILE:
37+
sh: echo "{{.ITEM}}"
38+
OUTPUTFILE:
39+
sh: echo "{{.ITEM}}" | sed -E 's/\.j2$//g'
40+
41+
jinja2:run-on-file:
42+
desc: Run jinja2 on a file
43+
internal: true
44+
cmds:
45+
- pwd
46+
- echo 'INPUTFILE = {{.INPUTFILE}}'
47+
- echo 'OUTPUTFILE = {{.OUTPUTFILE}}'
48+
- j2 --format env -o '{{.OUTPUTFILE}}' '{{.INPUTFILE}}'
49+
requires:
50+
vars: ["INPUTFILE", "OUTPUTFILE"]

‎.firmwareci/duts/dut-rpi-dutctl-tester/dut.yaml‎

Lines changed: 6 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.firmwareci/duts/dut-rpi-dutctl-tester/dut.yaml.j2‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
# Device-under-Test Configuration for RaspberryPi acting as dutctl tester
3+
{%- import ".jinja2_templates/defaults.j2" as defaults %}
4+
name: {{ defaults.RPI_HOSTNAME }}
5+
label: {{ defaults.RPI_HOSTNAME }}
6+
attributes:
7+
Host: "{{ defaults.RPI_FQDN }}"

‎.firmwareci/storage/.gitkeep‎

Whitespace-only changes.

0 commit comments

Comments
 (0)