Skip to content

Commit 1d8bb4c

Browse files
scottopellscottopell
committed
Initial creation of netflow v5 payload generation
1 parent 3c2c850 commit 1d8bb4c

File tree

5 files changed

+693
-0
lines changed

5 files changed

+693
-0
lines changed

‎examples/netflow_v5.yaml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Example configuration for NetFlow v5 load testing
2+
# This generates NetFlow v5 packets and sends them over UDP
3+
4+
version: "1"
5+
6+
target: null # No target needed for generators
7+
8+
generator:
9+
udp:
10+
seed: [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]
11+
addr: "127.0.0.1:9995" # NetFlow collector address
12+
variant:
13+
net_flow_v5:
14+
# Number of flow records per packet (1-30 to stay within MTU)
15+
flows_per_packet:
16+
min: 5
17+
max: 20
18+
19+
# Source IP range (10.0.0.0/8 network)
20+
src_ip_range:
21+
min: 167772161 # 10.0.0.1 in u32
22+
max: 184549374 # 10.255.255.254 in u32
23+
24+
# Destination IP range (192.168.0.0/16 network)
25+
dst_ip_range:
26+
min: 3232235521 # 192.168.1.1 in u32
27+
max: 3232301054 # 192.168.255.254 in u32
28+
29+
# Port ranges
30+
src_port_range:
31+
min: 1024
32+
max: 65535
33+
dst_port_range:
34+
min: 1
35+
max: 65535
36+
37+
# Flow metrics
38+
packet_count_range:
39+
min: 1
40+
max: 1000
41+
byte_count_range:
42+
min: 64
43+
max: 150000
44+
flow_duration_range:
45+
min: 1000 # 1 second
46+
max: 300000 # 5 minutes
47+
48+
# Interface and AS ranges
49+
interface_range:
50+
min: 1
51+
max: 10
52+
as_number_range:
53+
min: 100
54+
max: 65000
55+
56+
# Protocol distribution (weights)
57+
protocol_weights:
58+
tcp: 70 # 70% TCP traffic
59+
udp: 25 # 25% UDP traffic
60+
icmp: 3 # 3% ICMP traffic
61+
other: 2 # 2% other protocols
62+
63+
# Router identification
64+
engine_type: 1
65+
engine_id: 0
66+
67+
# Traffic generation settings
68+
bytes_per_second: "1MB" # Rate limit
69+
maximum_block_size: "1400B" # Stay within MTU
70+
maximum_prebuild_cache_size_bytes: "100MB"
71+
72+
# Throttling (optional)
73+
throttle:
74+
stable: {} # Stable rate throttling
75+
76+
# Observability configuration
77+
observer:
78+
prometheus:
79+
addr: "127.0.0.1:9090"
80+
81+
# General settings
82+
general:
83+
id: "netflow-v5-generator"

‎lading_payload/src/NETFLOW_README.md‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
# NetFlow v5 Support for Lading
2+
3+
This implementation adds NetFlow v5 packet generation support to the lading load testing framework.
4+
5+
## Overview
6+
7+
NetFlow v5 is a network protocol developed by Cisco for collecting IP traffic information. This implementation generates realistic NetFlow v5 export packets that can be used to load test NetFlow collectors.
8+
9+
## Features
10+
11+
- **Standards Compliant**: Generates proper NetFlow v5 packets following Cisco's specification
12+
- **Configurable Data Ranges**: All packet fields can be configured with ranges for realistic data generation
13+
- **Protocol Distribution**: Weighted distribution of TCP, UDP, ICMP, and other protocols
14+
- **Fixed Random Seed**: Uses deterministic random generation for reproducible test data
15+
- **UDP Transport**: Sends packets over UDP (standard for NetFlow exports)
16+
- **MTU Aware**: Respects packet size limits to stay within network MTU
17+
18+
## Packet Structure
19+
20+
Each NetFlow v5 packet contains:
21+
- **Header** (24 bytes): Version, flow count, timestamps, sequence numbers, engine info
22+
- **Flow Records** (48 bytes each): Source/destination IPs, ports, byte/packet counts, timing, protocols, AS numbers
23+
24+
## Configuration
25+
26+
The NetFlow v5 payload type supports extensive configuration:
27+
28+
### Basic Example
29+
```yaml
30+
generator:
31+
udp:
32+
addr: "127.0.0.1:9995"
33+
variant:
34+
net_flow_v5: {} # Uses default configuration
35+
```
36+
37+
### Full Configuration Example
38+
```yaml
39+
generator:
40+
udp:
41+
variant:
42+
net_flow_v5:
43+
flows_per_packet:
44+
min: 1
45+
max: 30
46+
src_ip_range:
47+
min: 167772161 # 10.0.0.1
48+
max: 184549374 # 10.255.255.254
49+
dst_ip_range:
50+
min: 3232235521 # 192.168.1.1
51+
max: 3232301054 # 192.168.255.254
52+
protocol_weights:
53+
tcp: 70
54+
udp: 25
55+
icmp: 3
56+
other: 2
57+
```
58+
59+
## Configuration Options
60+
61+
| Field | Type | Description | Default |
62+
|-------|------|-------------|---------|
63+
| `flows_per_packet` | Range<u16> | Number of flow records per packet | 1-30 |
64+
| `src_ip_range` | Range<u32> | Source IP addresses as u32 | 10.0.0.0/8 |
65+
| `dst_ip_range` | Range<u32> | Destination IP addresses as u32 | 192.168.0.0/16 |
66+
| `src_port_range` | Range<u16> | Source port numbers | 1024-65535 |
67+
| `dst_port_range` | Range<u16> | Destination port numbers | 1-65535 |
68+
| `packet_count_range` | Range<u32> | Packets per flow | 1-10000 |
69+
| `byte_count_range` | Range<u32> | Bytes per flow | 64-1500000 |
70+
| `flow_duration_range` | Range<u32> | Flow duration in milliseconds | 1s-1h |
71+
| `interface_range` | Range<u16> | Interface indices | 1-254 |
72+
| `as_number_range` | Range<u16> | BGP AS numbers | 1-65535 |
73+
| `protocol_weights` | ProtocolWeights | Protocol distribution weights | TCP:70, UDP:25, ICMP:3, Other:2 |
74+
| `engine_type` | u8 | Flow switching engine type | 0 |
75+
| `engine_id` | u8 | Flow switching engine ID | 0 |
76+
77+
## IP Address Configuration
78+
79+
IP addresses are configured as u32 values in network byte order. Helper conversions:
80+
81+
```bash
82+
# Convert IP to u32
83+
python3 -c "import socket, struct; print(struct.unpack('!I', socket.inet_aton('10.0.0.1'))[0])"
84+
85+
# Convert u32 to IP
86+
python3 -c "import socket, struct; print(socket.inet_ntoa(struct.pack('!I', 167772161)))"
87+
```
88+
89+
## Load Testing Scenarios
90+
91+
### Single Router Simulation
92+
```yaml
93+
generator:
94+
udp:
95+
addr: "127.0.0.1:9995"
96+
variant:
97+
net_flow_v5:
98+
engine_id: 1
99+
bytes_per_second: "10MB"
100+
```
101+
102+
### Multiple Router Simulation
103+
Deploy multiple generator instances with different engine_id values to simulate multiple routers.
104+
105+
### Burst Testing
106+
```yaml
107+
generator:
108+
udp:
109+
variant:
110+
net_flow_v5:
111+
flows_per_packet:
112+
min: 25
113+
max: 30 # Maximum flows per packet
114+
bytes_per_second: "50MB" # High rate for burst testing
115+
```
116+
117+
## Usage
118+
119+
1. **Start a NetFlow collector** (or use netcat for testing):
120+
```bash
121+
nc -ul 9995 # Listen on UDP port 9995
122+
```
123+
124+
2. **Run lading with NetFlow configuration**:
125+
```bash
126+
cargo run -- --config netflow_v5_example.yaml
127+
```
128+
129+
3. **Monitor metrics** at http://localhost:9090 (if Prometheus observer is configured)
130+
131+
## Metrics
132+
133+
The generator emits standard UDP generator metrics:
134+
- `bytes_written`: Total bytes sent
135+
- `packets_sent`: Total packets sent
136+
- `request_failure`: Failed send attempts
137+
- `connection_failure`: Socket bind failures
138+
- `bytes_per_second`: Configured transmission rate
139+
140+
## Implementation Details
141+
142+
- Uses deterministic random generation with configurable seed
143+
- Generates realistic flow timing relationships
144+
- Handles protocol-specific fields (TCP flags, port usage)
145+
- Maintains sequence numbers across packets
146+
- Respects MTU limits (max 30 flows per packet = 1464 bytes)
147+
- Network byte order encoding for all multi-byte fields
148+
149+
## Testing
150+
151+
Run NetFlow-specific tests:
152+
```bash
153+
cd lading_payload && cargo test netflow
154+
```
155+
156+
## Future Enhancements
157+
158+
Potential areas for expansion:
159+
- NetFlow v9 template-based format
160+
- IPFIX support
161+
- IPv6 flow records
162+
- Flow sampling configuration
163+
- Custom field templates

‎lading_payload/src/block.rs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,20 @@ impl Cache {
343343
let span = span!(Level::INFO, "fixed", payload = "otel-metrics");
344344
let _guard = span.enter();
345345

346+
construct_block_cache_inner(rng, &mut pyld, maximum_block_bytes, total_bytes.get())?
347+
}
348+
crate::Config::NetFlowV5(config) => {
349+
match config.valid() {
350+
Ok(()) => (),
351+
Err(e) => {
352+
warn!("Invalid NetFlowV5 configuration: {}", e);
353+
return Err(Error::InvalidConfig(e));
354+
}
355+
}
356+
let mut pyld = crate::NetFlowV5::new(*config, &mut rng)?;
357+
let span = span!(Level::INFO, "fixed", payload = "netflow-v5");
358+
let _guard = span.enter();
359+
346360
construct_block_cache_inner(rng, &mut pyld, maximum_block_bytes, total_bytes.get())?
347361
}
348362
};

‎lading_payload/src/lib.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ pub use datadog_logs::DatadogLog;
3636
pub use dogstatsd::DogStatsD;
3737
pub use fluent::Fluent;
3838
pub use json::Json;
39+
pub use netflow::NetFlowV5;
3940
pub use opentelemetry_log::OpentelemetryLogs;
4041
pub use opentelemetry_metric::OpentelemetryMetrics;
4142
pub use opentelemetry_trace::OpentelemetryTraces;
@@ -51,6 +52,7 @@ pub mod datadog_logs;
5152
pub mod dogstatsd;
5253
pub mod fluent;
5354
pub mod json;
55+
pub mod netflow;
5456
pub mod opentelemetry_log;
5557
pub mod opentelemetry_metric;
5658
pub mod opentelemetry_trace;
@@ -171,6 +173,8 @@ pub enum Config {
171173
DogStatsD(crate::dogstatsd::Config),
172174
/// Generates `TraceAgent` payloads in JSON format
173175
TraceAgent(Encoding),
176+
/// Generates NetFlow v5 packets
177+
NetFlowV5(crate::netflow::Config),
174178
}
175179

176180
#[derive(Debug)]
@@ -189,6 +193,7 @@ pub(crate) enum Payload {
189193
OtelMetrics(OpentelemetryMetrics),
190194
DogStatsdD(DogStatsD),
191195
TraceAgent(TraceAgent),
196+
NetFlowV5(NetFlowV5),
192197
}
193198

194199
impl Serialize for Payload {
@@ -211,6 +216,7 @@ impl Serialize for Payload {
211216
Payload::OtelMetrics(ser) => ser.to_bytes(rng, max_bytes, writer),
212217
Payload::DogStatsdD(ser) => ser.to_bytes(rng, max_bytes, writer),
213218
Payload::TraceAgent(ser) => ser.to_bytes(rng, max_bytes, writer),
219+
Payload::NetFlowV5(ser) => ser.to_bytes(rng, max_bytes, writer),
214220
}
215221
}
216222

0 commit comments

Comments
 (0)