Skip to content

Commit 1f0b96a

Browse files
mcgrofmcgrof
committed
fio: add mock test framework for isolated unit testing
Introduce a mock test framework to validate specific algorithms and calculations in isolation. This allows testing numerical precision, edge cases, and algorithmic correctness without requiring the full FIO infrastructure. Features: - TAP (Test Anything Protocol) output for CI integration - Isolated testing of specific calculations - Clear documentation of test purposes and goals - Simple make target: 'make mock-tests' The first test validates the numerical precision improvements in steady state latency calculations, demonstrating how the framework can catch subtle precision and overflow issues. Structure: - mock-tests/lib/tap.h: TAP output support - mock-tests/tests/: Individual test programs - mock-tests/Makefile: Build system - mock-tests/README.md: Comprehensive documentation This framework complements the existing integration tests by focusing on unit-level validation of critical calculations. Generated-by: Claude AI Signed-off-by: Luis Chamberlain <[email protected]>
1 parent 9e3cb57 commit 1f0b96a

File tree

5 files changed

+562
-0
lines changed

5 files changed

+562
-0
lines changed

‎Makefile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -646,6 +646,7 @@ clean: FORCE
646646
@rm -f .depend $(FIO_OBJS) $(GFIO_OBJS) $(OBJS) $(T_OBJS) $(UT_OBJS) $(PROGS) $(T_PROGS) $(T_TEST_PROGS) core.* core gfio unittests/unittest FIO-VERSION-FILE *.[do] lib/*.d oslib/*.[do] crc/*.d engines/*.[do] engines/*.so profiles/*.[do] t/*.[do] t/*/*.[do] unittests/*.[do] unittests/*/*.[do] config-host.mak config-host.h y.tab.[ch] lex.yy.c exp/*.[do] lexer.h
647647
@rm -f t/fio-btrace2fio t/io_uring t/read-to-pipe-async
648648
@rm -rf doc/output
649+
@$(MAKE) -C mock-tests clean
649650

650651
distclean: clean FORCE
651652
@rm -f cscope.out fio.pdf fio_generate_plots.pdf fio2gnuplot.pdf fiologparser_hist.pdf
@@ -665,6 +666,10 @@ doc: tools/plot/fio2gnuplot.1
665666
test: fio
666667
./fio --minimal --thread --exitall_on_error --runtime=1s --name=nulltest --ioengine=null --rw=randrw --iodepth=2 --norandommap --random_generator=tausworthe64 --size=16T --name=verifyfstest --filename=fiotestfile.tmp --unlink=1 --rw=write --verify=crc32c --verify_state_save=0 --size=16K
667668

669+
670+
mock-tests:
671+
$(MAKE) -C mock-tests test
672+
668673
fulltest:
669674
sudo modprobe null_blk && \
670675
if [ ! -e /usr/include/libzbc/zbc.h ]; then \

‎mock-tests/Makefile‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# Makefile for FIO mock tests
2+
#
3+
# These tests validate specific algorithmic improvements and edge cases
4+
# using isolated mock implementations.
5+
6+
CC ?= gcc
7+
CFLAGS = -Wall -Wextra -O2 -g -I. -I.. -lm
8+
TEST_DIR = tests
9+
LIB_DIR = lib
10+
BUILD_DIR = build
11+
12+
# List of test programs
13+
TESTS = test_latency_precision
14+
15+
# Build paths
16+
TEST_SRCS = $(addprefix $(TEST_DIR)/, $(addsuffix .c, $(TESTS)))
17+
TEST_BINS = $(addprefix $(BUILD_DIR)/, $(TESTS))
18+
19+
# TAP test runner
20+
TAP_RUNNER = prove
21+
22+
.PHONY: all clean test help
23+
24+
all: $(BUILD_DIR) $(TEST_BINS)
25+
26+
$(BUILD_DIR):
27+
@mkdir -p $(BUILD_DIR)
28+
29+
$(BUILD_DIR)/%: $(TEST_DIR)/%.c $(LIB_DIR)/tap.h
30+
$(CC) $(CFLAGS) -o $@ $<
31+
32+
test: all
33+
@echo "Running FIO mock tests..."
34+
@echo "========================="
35+
@failed=0; \
36+
for test in $(TEST_BINS); do \
37+
echo "Running $$test..."; \
38+
./$$test; \
39+
if [ $$? -ne 0 ]; then \
40+
failed=$$((failed + 1)); \
41+
fi; \
42+
echo; \
43+
done; \
44+
if [ $$failed -gt 0 ]; then \
45+
echo "FAILED: $$failed test(s) failed"; \
46+
exit 1; \
47+
else \
48+
echo "SUCCESS: All tests passed"; \
49+
fi
50+
51+
# Run tests with TAP harness if available
52+
test-tap: all
53+
@if command -v $(TAP_RUNNER) >/dev/null 2>&1; then \
54+
$(TAP_RUNNER) -v $(TEST_BINS); \
55+
else \
56+
echo "TAP runner '$(TAP_RUNNER)' not found, running tests directly..."; \
57+
$(MAKE) test; \
58+
fi
59+
60+
# Run a specific test
61+
test-%: $(BUILD_DIR)/%
62+
./$(BUILD_DIR)/$*
63+
64+
clean:
65+
rm -rf $(BUILD_DIR)
66+
67+
help:
68+
@echo "FIO Mock Tests"
69+
@echo "=============="
70+
@echo ""
71+
@echo "Available targets:"
72+
@echo " make all - Build all tests"
73+
@echo " make test - Run all tests"
74+
@echo " make test-tap - Run tests with TAP harness (if available)"
75+
@echo " make test-NAME - Run specific test (e.g., make test-latency_precision)"
76+
@echo " make clean - Remove build artifacts"
77+
@echo " make help - Show this help message"
78+
@echo ""
79+
@echo "Available tests:"
80+
@for test in $(TESTS); do echo " - $$test"; done

‎mock-tests/README.md‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
# FIO Mock Tests
2+
3+
## Overview
4+
5+
The FIO mock test suite provides isolated unit testing for specific algorithms,
6+
calculations, and edge cases within FIO. These tests use mock implementations
7+
to validate correctness without requiring the full FIO infrastructure.
8+
9+
## Purpose and Goals
10+
11+
### Why Mock Tests?
12+
13+
1. **Isolation**: Test specific algorithms without full system dependencies
14+
2. **Precision**: Validate numerical calculations and edge cases precisely
15+
3. **Speed**: Run quickly without I/O operations or system calls
16+
4. **Clarity**: Each test focuses on a single aspect with clear documentation
17+
5. **Regression Prevention**: Catch subtle bugs in mathematical operations
18+
19+
### What Mock Tests Are NOT
20+
21+
- Not integration tests (use `make test` for that)
22+
- Not performance benchmarks (use FIO itself)
23+
- Not I/O path testing (requires real FIO execution)
24+
25+
## Structure
26+
27+
```
28+
mock-tests/
29+
├── lib/ # Common test infrastructure
30+
│ └── tap.h # TAP (Test Anything Protocol) output support
31+
├── tests/ # Individual test programs
32+
│ └── test_*.c # Test source files
33+
├── build/ # Build artifacts (created by make)
34+
└── Makefile # Build system for mock tests
35+
```
36+
37+
## Running Tests
38+
39+
### Run all mock tests:
40+
```bash
41+
make mock-tests
42+
```
43+
44+
### Run tests from the mock-tests directory:
45+
```bash
46+
cd mock-tests
47+
make test # Run all tests
48+
make test-tap # Run with TAP harness (if prove is installed)
49+
make test-latency_precision # Run specific test
50+
```
51+
52+
### Clean build artifacts:
53+
```bash
54+
make clean # From mock-tests directory
55+
# or
56+
make clean # From main FIO directory (cleans everything)
57+
```
58+
59+
## TAP Output Format
60+
61+
Tests produce TAP (Test Anything Protocol) output for easy parsing:
62+
63+
```
64+
TAP version 13
65+
1..12
66+
ok 1 - Microsecond latency: 123456000 == 123456000
67+
ok 2 - Millisecond latency: 1234567890000 == 1234567890000
68+
not ok 3 - Some failing test
69+
# All tests passed
70+
```
71+
72+
This format is understood by many test harnesses and CI systems.
73+
74+
## Writing New Mock Tests
75+
76+
### 1. Create test file in `tests/`:
77+
78+
```c
79+
#include "../lib/tap.h"
80+
81+
int main(void) {
82+
tap_init();
83+
tap_plan(3); // Number of tests
84+
85+
tap_ok(1 == 1, "Basic equality");
86+
tap_ok(2 + 2 == 4, "Addition works");
87+
tap_skip("Not implemented yet");
88+
89+
return tap_done();
90+
}
91+
```
92+
93+
### 2. Add to Makefile:
94+
95+
Edit `mock-tests/Makefile` and add your test name to the `TESTS` variable.
96+
97+
### 3. Document your test:
98+
99+
Each test should have a comprehensive header comment explaining:
100+
- Purpose of the test
101+
- Background on what's being tested
102+
- Why this test matters
103+
- What specific cases are covered
104+
105+
## Available Tests
106+
107+
### test_latency_precision
108+
109+
**Purpose**: Validates numerical precision improvements in steady state latency calculations.
110+
111+
**Background**: When calculating total latency from mean and sample count, large values
112+
can cause precision loss or overflow. This test validates the improvement from:
113+
```c
114+
// Before: potential precision loss
115+
total = (uint64_t)(mean * samples);
116+
117+
// After: explicit double precision
118+
total = (uint64_t)(mean * (double)samples);
119+
```
120+
121+
**Test Cases**:
122+
- Normal operating ranges (microseconds to seconds)
123+
- Edge cases near uint64_t overflow
124+
- Zero sample defensive programming
125+
- Precision in accumulation across threads
126+
- Fractional nanosecond preservation
127+
128+
## Design Principles
129+
130+
1. **Isolation**: Mock only what's needed, test one thing at a time
131+
2. **Clarity**: Clear test names and diagnostic messages
132+
3. **Coverage**: Test normal cases, edge cases, and error conditions
133+
4. **Documentation**: Explain WHY each test exists
134+
5. **Reproducibility**: Deterministic tests with no random elements
135+
136+
## Integration with CI
137+
138+
The TAP output format makes these tests easy to integrate with CI systems:
139+
140+
```bash
141+
# In CI script
142+
make mock-tests || exit 1
143+
```
144+
145+
Or with TAP parsing for better reports:
146+
147+
```bash
148+
prove -v mock-tests/build/*
149+
```
150+
151+
## Future Enhancements
152+
153+
Potential areas for expansion:
154+
- Mock tests for parsing algorithms
155+
- Edge case validation for statistical calculations
156+
- Overflow detection in various calculations
157+
- Precision validation for other numerical operations
158+
159+
## Contributing
160+
161+
When adding new mock tests:
162+
1. Follow the existing patterns
163+
2. Document thoroughly
164+
3. Use meaningful test descriptions
165+
4. Include both positive and negative test cases
166+
5. Test edge cases and boundary conditions

‎mock-tests/lib/tap.h‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
/*
2+
* TAP (Test Anything Protocol) output support for FIO mock tests
3+
*
4+
* This provides a simple TAP output format for automated testing.
5+
* TAP is a simple text-based protocol for test results that can be
6+
* consumed by various test harnesses.
7+
*
8+
* Format:
9+
* TAP version 13
10+
* 1..N
11+
* ok 1 - test description
12+
* not ok 2 - test description
13+
* # diagnostic message
14+
*/
15+
16+
#ifndef FIO_MOCK_TAP_H
17+
#define FIO_MOCK_TAP_H
18+
19+
#include <stdio.h>
20+
#include <stdarg.h>
21+
#include <stdbool.h>
22+
23+
static int tap_test_count = 0;
24+
static int tap_failures = 0;
25+
static bool tap_planned = false;
26+
27+
/* Initialize TAP output */
28+
static inline void tap_init(void) {
29+
printf("TAP version 13\n");
30+
tap_test_count = 0;
31+
tap_failures = 0;
32+
tap_planned = false;
33+
}
34+
35+
/* Plan the number of tests */
36+
static inline void tap_plan(int n) {
37+
printf("1..%d\n", n);
38+
tap_planned = true;
39+
}
40+
41+
/* Report a test result */
42+
static inline void tap_ok(bool condition, const char *fmt, ...) {
43+
va_list args;
44+
tap_test_count++;
45+
46+
if (condition) {
47+
printf("ok %d - ", tap_test_count);
48+
} else {
49+
printf("not ok %d - ", tap_test_count);
50+
tap_failures++;
51+
}
52+
53+
va_start(args, fmt);
54+
vprintf(fmt, args);
55+
va_end(args);
56+
printf("\n");
57+
}
58+
59+
/* Skip a test */
60+
static inline void tap_skip(const char *reason, ...) {
61+
va_list args;
62+
tap_test_count++;
63+
64+
printf("ok %d # SKIP ", tap_test_count);
65+
va_start(args, reason);
66+
vprintf(reason, args);
67+
va_end(args);
68+
printf("\n");
69+
}
70+
71+
/* Output a diagnostic message */
72+
static inline void tap_diag(const char *fmt, ...) {
73+
va_list args;
74+
printf("# ");
75+
va_start(args, fmt);
76+
vprintf(fmt, args);
77+
va_end(args);
78+
printf("\n");
79+
}
80+
81+
/* Check if a value is within tolerance */
82+
static inline bool tap_within_tolerance(double actual, double expected, double tolerance) {
83+
double diff = actual - expected;
84+
if (diff < 0) diff = -diff;
85+
return diff <= tolerance;
86+
}
87+
88+
/* Finish TAP output and return exit code */
89+
static inline int tap_done(void) {
90+
if (!tap_planned) {
91+
printf("1..%d\n", tap_test_count);
92+
}
93+
94+
if (tap_failures > 0) {
95+
tap_diag("Failed %d/%d tests", tap_failures, tap_test_count);
96+
return 1;
97+
}
98+
99+
tap_diag("All tests passed");
100+
return 0;
101+
}
102+
103+
#endif /* FIO_MOCK_TAP_H */

0 commit comments

Comments
 (0)