Skip to content

Commit d5eaa85

Browse files
author
stack
committed
Add new tests
1 parent 9274b15 commit d5eaa85

File tree

3 files changed

+476
-0
lines changed

3 files changed

+476
-0
lines changed

‎fdbclient/tests/continuous_stress_test.sh‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Continuous stress test: Run batches of 10 concurrent s3client_test.sh instances
4+
# for 5 minutes, restarting immediately when batches complete
5+
#
6+
set -euo pipefail
7+
8+
BUILD_DIR="/Users/stack/build_output"
9+
CONCURRENT_INSTANCES=10
10+
TEST_DURATION_MINUTES=5
11+
TEST_DURATION_SECONDS=$((TEST_DURATION_MINUTES * 60))
12+
13+
# Statistics tracking
14+
TOTAL_BATCHES=0
15+
TOTAL_INSTANCES=0
16+
TOTAL_SUCCESSFUL=0
17+
TOTAL_FAILED=0
18+
TOTAL_PORT_CONFLICTS=0
19+
START_TIME=$(date +%s)
20+
END_TIME=$((START_TIME + TEST_DURATION_SECONDS))
21+
22+
echo "$(date -Iseconds) Starting continuous stress test for $TEST_DURATION_MINUTES minutes"
23+
echo "$(date -Iseconds) Each batch: $CONCURRENT_INSTANCES concurrent instances"
24+
echo "$(date -Iseconds) Will run until: $(date -Iseconds -r $END_TIME)"
25+
26+
# Function to run a single batch of concurrent tests
27+
run_batch() {
28+
local batch_num="$1"
29+
local batch_start_time=$(date +%s)
30+
31+
echo "$(date -Iseconds) ===== BATCH $batch_num: Starting $CONCURRENT_INSTANCES concurrent instances ====="
32+
33+
# Array to track PIDs for this batch
34+
local PIDS=()
35+
local batch_successful=0
36+
local batch_failed=0
37+
local batch_port_conflicts=0
38+
39+
# Start concurrent instances
40+
for i in $(seq 1 $CONCURRENT_INSTANCES); do
41+
local instance_id="${batch_num}_${i}"
42+
SCRATCH_DIR=$(mktemp -d "/tmp/stress_${instance_id}.XXXXXX")
43+
44+
# Run s3client_test.sh in background
45+
(
46+
cd fdbclient/tests
47+
if ./s3client_test.sh "$BUILD_DIR" "$SCRATCH_DIR" > "/tmp/stress_${instance_id}.log" 2>&1; then
48+
echo "$(date -Iseconds) Instance ${instance_id} SUCCESS"
49+
else
50+
echo "$(date -Iseconds) Instance ${instance_id} FAILED"
51+
fi
52+
) &
53+
54+
PIDS+=($!)
55+
56+
# Very brief stagger to increase port conflicts
57+
sleep 0.1
58+
done
59+
60+
echo "$(date -Iseconds) Batch $batch_num: All instances started, waiting for completion..."
61+
62+
# Wait for all instances in this batch
63+
for i in "${!PIDS[@]}"; do
64+
local pid=${PIDS[$i]}
65+
local instance_id="${batch_num}_$((i+1))"
66+
67+
if wait "$pid"; then
68+
((batch_successful++))
69+
echo "$(date -Iseconds) Instance ${instance_id} (PID $pid) completed successfully"
70+
else
71+
((batch_failed++))
72+
echo "$(date -Iseconds) Instance ${instance_id} (PID $pid) FAILED"
73+
fi
74+
done
75+
76+
# Check for port conflicts in this batch
77+
for i in $(seq 1 $CONCURRENT_INSTANCES); do
78+
local instance_id="${batch_num}_${i}"
79+
if [[ -f "/tmp/stress_${instance_id}.log" ]] && grep -q "Port.*already in use\|trying next port" "/tmp/stress_${instance_id}.log"; then
80+
((batch_port_conflicts++))
81+
fi
82+
done
83+
84+
local batch_end_time=$(date +%s)
85+
local batch_duration=$((batch_end_time - batch_start_time))
86+
87+
# Update global statistics
88+
((TOTAL_BATCHES++))
89+
TOTAL_INSTANCES=$((TOTAL_INSTANCES + CONCURRENT_INSTANCES))
90+
TOTAL_SUCCESSFUL=$((TOTAL_SUCCESSFUL + batch_successful))
91+
TOTAL_FAILED=$((TOTAL_FAILED + batch_failed))
92+
TOTAL_PORT_CONFLICTS=$((TOTAL_PORT_CONFLICTS + batch_port_conflicts))
93+
94+
echo "$(date -Iseconds) Batch $batch_num completed in ${batch_duration}s: ${batch_successful}/${CONCURRENT_INSTANCES} successful, $batch_port_conflicts port conflicts"
95+
96+
# Check for orphaned processes
97+
local orphaned_count=0
98+
if pgrep -f "mocks3server\|MockS3Server" >/dev/null 2>&1; then
99+
orphaned_count=$(pgrep -f "mocks3server\|MockS3Server" | wc -l)
100+
echo "$(date -Iseconds) WARNING: $orphaned_count orphaned MockS3Server processes detected!"
101+
# Kill orphaned processes
102+
pkill -f "mocks3server\|MockS3Server" 2>/dev/null || true
103+
fi
104+
105+
# Print running statistics
106+
local current_time=$(date +%s)
107+
local elapsed_time=$((current_time - START_TIME))
108+
local remaining_time=$((END_TIME - current_time))
109+
110+
echo "$(date -Iseconds) === RUNNING STATS (${elapsed_time}s elapsed, ${remaining_time}s remaining) ==="
111+
echo " Batches: $TOTAL_BATCHES"
112+
echo " Total instances: $TOTAL_INSTANCES"
113+
echo " Successful: $TOTAL_SUCCESSFUL ($((TOTAL_SUCCESSFUL * 100 / TOTAL_INSTANCES))%)"
114+
echo " Failed: $TOTAL_FAILED"
115+
echo " Port conflicts: $TOTAL_PORT_CONFLICTS"
116+
echo " Success rate: $((TOTAL_SUCCESSFUL * 100 / TOTAL_INSTANCES))%"
117+
echo ""
118+
}
119+
120+
# Main stress test loop
121+
batch_counter=1
122+
while [[ $(date +%s) -lt $END_TIME ]]; do
123+
# Check if we have enough time for another batch (estimate 30 seconds per batch)
124+
current_time=$(date +%s)
125+
remaining_time=$((END_TIME - current_time))
126+
127+
if [[ $remaining_time -lt 30 ]]; then
128+
echo "$(date -Iseconds) Less than 30 seconds remaining, ending stress test"
129+
break
130+
fi
131+
132+
run_batch "$batch_counter"
133+
((batch_counter++))
134+
135+
# Brief pause between batches to avoid overwhelming the system
136+
sleep 2
137+
done
138+
139+
# Final statistics
140+
echo "$(date -Iseconds) ===== FINAL STRESS TEST RESULTS ====="
141+
echo "Test duration: $TEST_DURATION_MINUTES minutes"
142+
echo "Total batches completed: $TOTAL_BATCHES"
143+
echo "Total instances run: $TOTAL_INSTANCES"
144+
echo "Total successful: $TOTAL_SUCCESSFUL"
145+
echo "Total failed: $TOTAL_FAILED"
146+
echo "Total port conflicts detected: $TOTAL_PORT_CONFLICTS"
147+
echo "Overall success rate: $((TOTAL_SUCCESSFUL * 100 / TOTAL_INSTANCES))%"
148+
echo "Average instances per batch: $((TOTAL_INSTANCES / TOTAL_BATCHES))"
149+
echo "Port conflicts per batch: $((TOTAL_PORT_CONFLICTS / TOTAL_BATCHES))"
150+
151+
# Final cleanup check
152+
if pgrep -f "mocks3server\|MockS3Server" >/dev/null 2>&1; then
153+
orphaned_count=$(pgrep -f "mocks3server\|MockS3Server" | wc -l)
154+
echo "WARNING: $orphaned_count orphaned processes remain"
155+
pkill -f "mocks3server\|MockS3Server" 2>/dev/null || true
156+
else
157+
echo "✅ Perfect cleanup - no orphaned processes"
158+
fi
159+
160+
echo "$(date -Iseconds) Continuous stress test completed!"
161+
162+
# Provide summary of what logs to check
163+
echo ""
164+
echo "Log files created: /tmp/stress_*.log"
165+
echo "To check individual results: grep 'PASSED\\|FAILED' /tmp/stress_*.log"
166+
echo "To check port conflicts: grep 'Port.*already in use' /tmp/stress_*.log"

‎fdbclient/tests/simple_continuous_test.sh‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Simple continuous test: Run 10 concurrent s3client tests repeatedly for 5 minutes
4+
#
5+
set -euo pipefail
6+
7+
BUILD_DIR="/Users/stack/build_output"
8+
CONCURRENT_INSTANCES=10
9+
DURATION_MINUTES=5
10+
DURATION_SECONDS=$((DURATION_MINUTES * 60))
11+
12+
echo "$(date -Iseconds) Starting simple continuous test for $DURATION_MINUTES minutes"
13+
echo "$(date -Iseconds) Running $CONCURRENT_INSTANCES concurrent instances per batch"
14+
15+
START_TIME=$(date +%s)
16+
END_TIME=$((START_TIME + DURATION_SECONDS))
17+
BATCH_NUM=1
18+
TOTAL_TESTS=0
19+
TOTAL_SUCCESS=0
20+
TOTAL_FAILED=0
21+
22+
while [[ $(date +%s) -lt $END_TIME ]]; do
23+
echo ""
24+
echo "$(date -Iseconds) ===== BATCH $BATCH_NUM ====="
25+
26+
# Start concurrent instances
27+
PIDS=()
28+
for i in $(seq 1 $CONCURRENT_INSTANCES); do
29+
SCRATCH_DIR=$(mktemp -d "/tmp/continuous_${BATCH_NUM}_${i}.XXXXXX")
30+
31+
(
32+
cd fdbclient/tests
33+
./s3client_test.sh "$BUILD_DIR" "$SCRATCH_DIR" > "/tmp/continuous_${BATCH_NUM}_${i}.log" 2>&1
34+
) &
35+
36+
PIDS+=($!)
37+
echo "$(date -Iseconds) Started instance ${BATCH_NUM}_${i} (PID ${PIDS[$((i-1))]})"
38+
39+
# Brief stagger
40+
sleep 0.2
41+
done
42+
43+
echo "$(date -Iseconds) Batch $BATCH_NUM: All $CONCURRENT_INSTANCES instances started, waiting..."
44+
45+
# Wait for this batch to complete
46+
BATCH_SUCCESS=0
47+
BATCH_FAILED=0
48+
49+
for i in "${!PIDS[@]}"; do
50+
pid=${PIDS[$i]}
51+
if wait "$pid"; then
52+
((BATCH_SUCCESS++))
53+
((TOTAL_SUCCESS++))
54+
else
55+
((BATCH_FAILED++))
56+
((TOTAL_FAILED++))
57+
fi
58+
((TOTAL_TESTS++))
59+
done
60+
61+
# Check port conflicts
62+
PORT_CONFLICTS=0
63+
for i in $(seq 1 $CONCURRENT_INSTANCES); do
64+
if grep -q "Port.*already in use\|trying next port" "/tmp/continuous_${BATCH_NUM}_${i}.log" 2>/dev/null; then
65+
((PORT_CONFLICTS++))
66+
fi
67+
done
68+
69+
echo "$(date -Iseconds) Batch $BATCH_NUM completed: $BATCH_SUCCESS success, $BATCH_FAILED failed, $PORT_CONFLICTS port conflicts"
70+
71+
# Show some port conflict examples
72+
if [[ $PORT_CONFLICTS -gt 0 ]]; then
73+
echo "$(date -Iseconds) Port conflict examples:"
74+
for i in $(seq 1 $CONCURRENT_INSTANCES); do
75+
if grep -q "MockS3Server ready on port" "/tmp/continuous_${BATCH_NUM}_${i}.log" 2>/dev/null; then
76+
grep "MockS3Server ready on port" "/tmp/continuous_${BATCH_NUM}_${i}.log" | head -1
77+
break
78+
fi
79+
done
80+
fi
81+
82+
# Running stats
83+
ELAPSED=$(($(date +%s) - START_TIME))
84+
REMAINING=$((END_TIME - $(date +%s)))
85+
SUCCESS_RATE=$((TOTAL_SUCCESS * 100 / TOTAL_TESTS))
86+
87+
echo "$(date -Iseconds) Running stats: $TOTAL_TESTS tests, $TOTAL_SUCCESS success ($SUCCESS_RATE%), ${ELAPSED}s elapsed, ${REMAINING}s remaining"
88+
89+
((BATCH_NUM++))
90+
91+
# Check if we have time for another batch (need at least 15 seconds)
92+
if [[ $REMAINING -lt 15 ]]; then
93+
echo "$(date -Iseconds) Less than 15 seconds remaining, ending test"
94+
break
95+
fi
96+
97+
# Brief pause between batches
98+
sleep 1
99+
done
100+
101+
echo ""
102+
echo "$(date -Iseconds) ===== FINAL RESULTS ====="
103+
echo "Duration: $DURATION_MINUTES minutes"
104+
echo "Batches completed: $((BATCH_NUM - 1))"
105+
echo "Total tests: $TOTAL_TESTS"
106+
echo "Total successful: $TOTAL_SUCCESS"
107+
echo "Total failed: $TOTAL_FAILED"
108+
echo "Success rate: $((TOTAL_SUCCESS * 100 / TOTAL_TESTS))%"
109+
110+
# Check cleanup
111+
if pgrep -f "mocks3server\|MockS3Server" >/dev/null 2>&1; then
112+
ORPHANED=$(pgrep -f "mocks3server\|MockS3Server" | wc -l)
113+
echo "WARNING: $ORPHANED orphaned processes found"
114+
pkill -f "mocks3server\|MockS3Server" 2>/dev/null || true
115+
else
116+
echo "✅ Perfect cleanup - no orphaned processes"
117+
fi
118+
119+
echo "$(date -Iseconds) Continuous test completed!"
120+
echo ""
121+
echo "Check logs: ls /tmp/continuous_*.log"
122+
echo "Check port conflicts: grep -l 'Port.*already in use' /tmp/continuous_*.log | wc -l"

0 commit comments

Comments
 (0)