Skip to content

Commit 25021a1

Browse files
JoeOakhartNavaclaude
JoeOakhartNava
and
claude
committed
feat(8a5b-e07b): GREEN — implement close benchmark mode in ticket-benchmark.sh
Story db66-0242: benchmark confirms ticket close under 10s with 200 tickets. Adds --mode=close that seeds realistic population (mixed types, statuses, deps, archived) and times full ticket transition close path. Uses reducer --exclude-archived for accurate non-archived count. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent abf6b0d commit 25021a1

File tree

2 files changed

+176
-6
lines changed

2 files changed

+176
-6
lines changed

‎.test-index‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,4 +189,4 @@ plugins/dso/agents/red-test-writer.md: tests/agents/test-red-test-writer.sh
189189
plugins/dso/agents/red-test-evaluator.md: tests/agents/test-red-test-evaluator.sh
190190
tests/lib/markdown_helpers.py:tests/docs/test_extract_section_deduplication.py
191191
plugins/dso/scripts/ticket-unblock.py: tests/scripts/test_ticket_unblock.py
192-
plugins/dso/scripts/ticket-benchmark.sh: tests/scripts/test-ticket-benchmark.sh [test_close_benchmark_under_threshold]
192+
plugins/dso/scripts/ticket-benchmark.sh: tests/scripts/test-ticket-benchmark.sh

‎plugins/dso/scripts/ticket-benchmark.sh‎

Lines changed: 175 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,34 @@
11
#!/usr/bin/env bash
22
# plugins/dso/scripts/ticket-benchmark.sh
3-
# Benchmark the ticket list command against a seeded ticket system.
3+
# Benchmark the ticket list or close command against a seeded ticket system.
44
#
5-
# Usage: ticket-benchmark.sh [-n <count>] [--threshold <seconds>]
5+
# Usage: ticket-benchmark.sh [-n <count>] [--threshold <seconds>] [--mode=list|close]
66
# -n <count> Number of tickets to seed (default: 300; 0 = use existing repo tickets)
77
# --threshold <secs> Max acceptable wall-clock time in seconds
8-
# Defaults: 3s for n<=300, 10s for n<=1000, 30s for n>1000
8+
# list defaults: 3s for n<=300, 10s for n<=1000, 30s for n>1000
9+
# close default: 10s
10+
# --mode=list|close Benchmark mode (default: list for backward compatibility)
11+
# list: measures ticket list wall-clock time
12+
# close: seeds a mixed population, measures ticket transition open->closed
913
#
1014
# When run without -n (or with -n 0) inside a repo that already has a ticket
1115
# system initialized, benchmarks the existing tickets. Otherwise creates a
1216
# temporary git repo, seeds N tickets, and benchmarks that.
1317
#
14-
# Output: "Elapsed: X.XXs for N tickets" to stdout.
18+
# Output: "Elapsed: X.XXs for N tickets" to stdout (list mode)
19+
# "Elapsed: X.XXs for closing ticket with N non-archived tickets in tracker" (close mode)
1520
# Exit 0 if elapsed < threshold, exit 1 if elapsed >= threshold.
1621
set -euo pipefail
1722

1823
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
1924
LIST_SCRIPT="$SCRIPT_DIR/ticket-list.sh"
25+
TICKET_SCRIPT="$SCRIPT_DIR/ticket"
2026
REDUCER="$SCRIPT_DIR/ticket-reducer.py"
2127

2228
# ── Parse arguments ──────────────────────────────────────────────────────────
2329
seed_count=0
2430
threshold=""
31+
mode="list"
2532

2633
while [[ $# -gt 0 ]]; do
2734
case "$1" in
@@ -33,16 +40,31 @@ while [[ $# -gt 0 ]]; do
3340
threshold="${2:?'--threshold requires a seconds argument'}"
3441
shift 2
3542
;;
43+
--mode=*)
44+
mode="${1#--mode=}"
45+
shift
46+
;;
47+
--mode)
48+
mode="${2:?'--mode requires list or close'}"
49+
shift 2
50+
;;
3651
*)
3752
echo "Error: unknown argument '$1'" >&2
38-
echo "Usage: ticket-benchmark.sh [-n <count>] [--threshold <seconds>]" >&2
53+
echo "Usage: ticket-benchmark.sh [-n <count>] [--threshold <seconds>] [--mode=list|close]" >&2
3954
exit 2
4055
;;
4156
esac
4257
done
4358

59+
# Validate mode
60+
if [[ "$mode" != "list" && "$mode" != "close" ]]; then
61+
echo "Error: --mode must be 'list' or 'close', got '$mode'" >&2
62+
exit 2
63+
fi
64+
4465
# ── Determine working mode ──────────────────────────────────────────────────
4566
_tmp_dir=""
67+
_close_repo=""
4668
_cleanup() {
4769
if [[ -n "$_tmp_dir" ]] && [[ -d "$_tmp_dir" ]]; then
4870
rm -rf "$_tmp_dir"
@@ -52,6 +74,154 @@ trap _cleanup EXIT
5274

5375
tracker_dir=""
5476

77+
# ── Close mode: seed a mixed population and benchmark ticket transition ───────
78+
if [[ "$mode" == "close" ]]; then
79+
if [[ "$seed_count" -gt 0 ]]; then
80+
# Create a temporary git repo with full ticket system for close benchmark
81+
_tmp_dir=$(mktemp -d)
82+
_close_repo="$_tmp_dir/repo"
83+
84+
git init -q -b main "$_close_repo"
85+
git -C "$_close_repo" config user.email "benchmark@test.com"
86+
git -C "$_close_repo" config user.name "Benchmark"
87+
echo "init" > "$_close_repo/README.md"
88+
git -C "$_close_repo" add -A
89+
git -C "$_close_repo" commit -q -m "init"
90+
91+
# Initialize ticket system in temp repo
92+
(cd "$_close_repo" && bash "$TICKET_SCRIPT" init >/dev/null 2>&1) || {
93+
echo "Error: failed to initialize ticket system in temp repo" >&2
94+
exit 1
95+
}
96+
97+
tracker_dir="$_close_repo/.tickets-tracker"
98+
99+
# Seed a realistic mixed population:
100+
# - 3 epics (open)
101+
# - 10 stories (in_progress) as children of first epic
102+
# - remaining tasks as open standalone
103+
# - 50 archived (closed) tasks
104+
# - 15+ dependency links between task pairs
105+
106+
local_epic_count=3
107+
local_story_count=10
108+
# Non-archived count: epics + stories + tasks = seed_count
109+
local_archived_count=50
110+
local_task_count=$(( seed_count - local_epic_count - local_story_count - local_archived_count ))
111+
if [[ "$local_task_count" -lt 1 ]]; then
112+
local_task_count=1
113+
fi
114+
local_link_count=15
115+
116+
first_epic_id=""
117+
118+
# Create epics
119+
for (( i = 1; i <= local_epic_count; i++ )); do
120+
eid=$(cd "$_close_repo" && bash "$TICKET_SCRIPT" create epic "Benchmark epic $i" 2>/dev/null) || true
121+
if [[ $i -eq 1 ]]; then first_epic_id="$eid"; fi
122+
done
123+
124+
# Create stories as children of first epic (transition to in_progress)
125+
if [[ -n "$first_epic_id" ]]; then
126+
for (( i = 1; i <= local_story_count; i++ )); do
127+
sid=$(cd "$_close_repo" && bash "$TICKET_SCRIPT" create story "Benchmark story $i" "$first_epic_id" 2>/dev/null) || true
128+
if [[ -n "$sid" ]]; then
129+
(cd "$_close_repo" && bash "$TICKET_SCRIPT" transition "$sid" open in_progress >/dev/null 2>/dev/null) || true
130+
fi
131+
done
132+
fi
133+
134+
# Create standalone open tasks and collect IDs for linking
135+
task_ids=()
136+
for (( i = 1; i <= local_task_count; i++ )); do
137+
tid=$(cd "$_close_repo" && bash "$TICKET_SCRIPT" create task "Benchmark task $i" 2>/dev/null) || true
138+
if [[ -n "$tid" ]]; then task_ids+=("$tid"); fi
139+
done
140+
141+
# Create archived (closed) tasks
142+
for (( i = 1; i <= local_archived_count; i++ )); do
143+
aid=$(cd "$_close_repo" && bash "$TICKET_SCRIPT" create task "Archived task $i" 2>/dev/null) || true
144+
if [[ -n "$aid" ]]; then
145+
(cd "$_close_repo" && bash "$TICKET_SCRIPT" transition "$aid" open closed --reason="Fixed: benchmark seed" >/dev/null 2>/dev/null) || true
146+
fi
147+
done
148+
149+
# Add dependency links between task pairs
150+
links_added=0
151+
pair_count="${#task_ids[@]}"
152+
for (( i = 0; i < pair_count - 1 && links_added < local_link_count; i += 2 )); do
153+
src="${task_ids[$i]}"
154+
tgt="${task_ids[$((i+1))]}"
155+
if [[ -n "$src" ]] && [[ -n "$tgt" ]]; then
156+
(cd "$_close_repo" && bash "$TICKET_SCRIPT" link "$src" "$tgt" depends_on >/dev/null 2>/dev/null) || true
157+
(( links_added++ )) || true
158+
fi
159+
done
160+
161+
# Count non-archived tickets via reducer (authoritative source for archived state)
162+
non_archived_count=$(python3 "$REDUCER" --batch --exclude-archived "$tracker_dir" 2>/dev/null | python3 -c 'import json,sys; print(len(json.loads(sys.stdin.read())))' 2>/dev/null) || non_archived_count="?"
163+
else
164+
# Use existing repo
165+
if [[ -n "${TICKETS_TRACKER_DIR:-}" ]]; then
166+
tracker_dir="$TICKETS_TRACKER_DIR"
167+
_close_repo="$(dirname "$tracker_dir")"
168+
else
169+
repo_root="$(git rev-parse --show-toplevel 2>/dev/null)" || {
170+
echo "Error: not inside a git repository and -n not specified" >&2
171+
exit 1
172+
}
173+
tracker_dir="$repo_root/.tickets-tracker"
174+
_close_repo="$repo_root"
175+
fi
176+
177+
if [[ ! -d "$tracker_dir" ]]; then
178+
echo "Error: ticket system not initialized at $tracker_dir" >&2
179+
exit 1
180+
fi
181+
182+
# Count non-archived tickets via reducer (consistent with seeded mode)
183+
non_archived_count=$(python3 "$REDUCER" --batch --exclude-archived "$tracker_dir" 2>/dev/null | python3 -c 'import json,sys; print(len(json.loads(sys.stdin.read())))' 2>/dev/null) || non_archived_count="?"
184+
fi
185+
186+
# Apply default threshold for close mode
187+
if [[ -z "$threshold" ]]; then
188+
threshold="10"
189+
fi
190+
191+
# Create the target ticket: a simple task in open status with no children
192+
target_id=$(cd "$_close_repo" && bash "$TICKET_SCRIPT" create task "Target close benchmark task" 2>/dev/null) || {
193+
echo "Error: failed to create target ticket for close benchmark" >&2
194+
exit 1
195+
}
196+
197+
if [[ -z "$target_id" ]]; then
198+
echo "Error: ticket create returned empty ID" >&2
199+
exit 1
200+
fi
201+
202+
# Measure wall-clock time of full ticket transition open->closed
203+
start_time=$(python3 -c "import time; print(f'{time.time():.6f}')")
204+
205+
(cd "$_close_repo" && bash "$TICKET_SCRIPT" transition "$target_id" open closed --reason="Fixed: benchmark" >/dev/null 2>/dev/null)
206+
207+
end_time=$(python3 -c "import time; print(f'{time.time():.6f}')")
208+
209+
elapsed=$(python3 -c "print(f'{float(\"$end_time\") - float(\"$start_time\"):.2f}')")
210+
211+
echo "Elapsed: ${elapsed}s for closing ticket with $non_archived_count non-archived tickets in tracker"
212+
213+
# Threshold check
214+
over=$(python3 -c "print('1' if float('$elapsed') >= float('$threshold') else '0')")
215+
216+
if [[ "$over" == "1" ]]; then
217+
echo "FAIL: ${elapsed}s >= threshold ${threshold}s" >&2
218+
exit 1
219+
fi
220+
221+
exit 0
222+
fi
223+
224+
# ── List mode: seed and benchmark ticket list ─────────────────────────────────
55225
if [[ "$seed_count" -gt 0 ]]; then
56226
# Create a temporary git repo and seed tickets
57227
_tmp_dir=$(mktemp -d)

0 commit comments

Comments
 (0)