workflow.sh

#!/usr/bin/env bash
# Local monorepo workflow: Rust core → SDK artifacts → main app (src/) (no registry publish).
#
# The main app (./src/cyt/) uses cyt-indexer-sdk from sdk/python via pyproject.toml:
#   [tool.uv.sources]
#   cyt-indexer-sdk = { path = "sdk/python", editable = true }
# Production installs (pip install clear-your-tools) pull cyt-indexer-sdk from PyPI instead.
#
# Usage:
#   ./scripts/local/dev/workflow.sh [--short|--silent] [--serial] [--xdist] [--runtime] <command> [args...]
#
# Options:
#   --short | --silent   Only print error/warning lines (hide info/success noise)
#   --serial             Force legacy serial test path (debug / low-RAM machines)
#   --xdist              Enable pytest -n auto inside each Python unit shard (parallel gate only)
#   --runtime            Enable runtime tests that may spawn cyt hook daemon or launch proxy
#
# Commands:
#   Core (Rust):
#     core-rust | rust     parallel rust gate (or serial cargo test with --serial)
#     indexer [subcmd]     cyt-indexer build tools|skills / retrieve (see help)
#
#   SDKs:
#     sdk-python           maturin develop --release + pytest tests/unit + verify
#     sdk-rust-release     cyt-indexer python+node release build + fossa-deps cargo entries
#     sdk-verify           verify sdk/python install + native import
#     sdk-typescript       npm ci, build, test (sdk/typescript)
#     sdk-c                cmake build + ctest (sdk/c)
#     sdk-go               build C FFI + go test (sdk/go)
#     sdk-all              all SDK targets above
#
#   Main app (src/):
#     app-setup | setup    uv sync workspace (editable sdk/python via pyproject.toml)
#     app-verify           verify main app (src/) re-exports local cyt-indexer-sdk
#     app-test | test      parallel python gate (or serial pytest with --serial)
#     app-test-runtime     pytest runtime category (may spawn hook daemon / launch proxy)
#     app-build | build-wheels
#                          uv build clear-your-tools wheel/sdist
#     app-all              app-setup → app-verify → app-test → app-build
#
#   Other:
#     proxy [args...]      verify + uv run src/cyt/cli/app.py proxy ...
#     simulate-registry    isolated venv: install built wheels + cargo/npm dry-run checks
#     ci                   app-setup → app-verify → ast-grep → import checks → ruff → pytest categories → app-build
#     all                  parallel monorepo gate (rust → sdk c/go/ts → py; use --serial for legacy)
#
# Examples:
#   ./scripts/local/dev/workflow.sh all
#   ./scripts/local/dev/workflow.sh --silent sdk-go
#   ./scripts/local/dev/workflow.sh sdk-go
#   ./scripts/local/dev/workflow.sh proxy --port 8834
#   KEEP_SIM_DIR=1 ./scripts/local/dev/workflow.sh simulate-registry
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# shellcheck disable=SC1091
source "${SCRIPT_DIR}/helpers.sh"
export SHORTEN_ROOT="${CYT_REPO_ROOT}"

CYT_LOCAL_DEV_SHORT="${CYT_LOCAL_DEV_SHORT:-}"
CYT_LOCAL_DEV_RUNTIME="${CYT_LOCAL_DEV_RUNTIME:-}"
CYT_WORKFLOW_SERIAL="${CYT_WORKFLOW_SERIAL:-}"
CYT_WORKFLOW_XDIST="${CYT_WORKFLOW_XDIST:-}"
LOCAL_DEV_ARGS=()
while (($#)); do
	case "$1" in
	--short | --silent)
		CYT_LOCAL_DEV_SHORT=1
		shift
		;;
	--serial)
		CYT_WORKFLOW_SERIAL=1
		shift
		;;
	--xdist)
		CYT_WORKFLOW_XDIST=1
		shift
		;;
	--runtime)
		CYT_LOCAL_DEV_RUNTIME=1
		shift
		;;
	*)
		LOCAL_DEV_ARGS+=("$1")
		shift
		;;
	esac
done
export CYT_LOCAL_DEV_SHORT
export CYT_WORKFLOW_SERIAL
export CYT_WORKFLOW_XDIST
if [[ -n "${CYT_LOCAL_DEV_RUNTIME}" ]]; then
	export CYT_RUN_RUNTIME_TESTS=1
else
	unset CYT_RUN_RUNTIME_TESTS
fi

usage() {
	sed -n '2,46p' "$0" | sed 's/^# \{0,1\}//'
}

_cyt_local_dev_main() {
	local cmd="${1:-}"
	shift || true

	case "${cmd}" in
	core-rust | rust)
		require_repo_root
		cyt_prek_parallel_gate rust-only
		;;
	indexer)
		require_repo_root
		require_cmd jq
		sub="${1:-all}"
		shift || true
		case "${sub}" in
		build)
			target="${1:-tools}"
			shift || true
			case "${target}" in
			tools)
				cyt_indexer_build_catalog
				;;
			skills)
				cyt_indexer_build_skills "$@"
				;;
			*)
				die "unknown build target: ${target} (try: tools, skills)"
				;;
			esac
			;;
		survivors)
			cyt_indexer_extract_survivors
			;;
		retrieve)
			target="${1:-tools}"
			if [[ "${target}" == --* ]]; then
				target="tools"
			else
				shift || true
			fi
			case "${target}" in
			tools)
				cyt_indexer_retrieve "$@"
				;;
			skills)
				cyt_indexer_retrieve_skills "$@"
				;;
			*)
				die "unknown retrieve target: ${target} (try: tools, skills)"
				;;
			esac
			;;
		all)
			cyt_indexer_all "$@"
			;;
		-h | --help | help)
			cat <<EOF
Usage: ./scripts/local/dev/workflow.sh indexer [build|survivors|retrieve|all] [args...]

  build tools   jq '.body.tools' debug/full_example.json -> cyt-indexer build tools -> .catalog/
  build skills  cyt-indexer build skills --skills DIR [--output DIR]
  survivors     jq rerank json/md -> .catalog/survivors.json (scores as numbers)
  retrieve tools   cyt-indexer retrieve tools with default policies (score filter off for rerank survivors)
  retrieve skills  cyt-indexer retrieve skills --catalog DIR --doc-id ID --query TYPE --output FILE
  all           build tools + survivors + retrieve (default)

Retrieve defaults:
  --system-policy prune_optional
  --mcp-policy prune_all
  --tool-policy AskUserQuestion=always_include

Override via env:
  CYT_CATALOG_DIR  CYT_EXAMPLE_JSON  CYT_SURVIVORS_JSON  CYT_RETRIEVE_OUT
  CYT_INDEXER_SYSTEM_POLICY  CYT_INDEXER_MCP_POLICY
  CYT_INDEXER_TOOL_POLICIES='AskUserQuestion=always_include Agent=prune_optional'

Examples:
  ./scripts/local/dev/workflow.sh indexer
  ./scripts/local/dev/workflow.sh indexer build
  ./scripts/local/dev/workflow.sh indexer build skills --skills ~/.claude/skills --output ./.catalog
  ./scripts/local/dev/workflow.sh indexer retrieve --tool-policy Bash=always_include
  ./scripts/local/dev/workflow.sh indexer retrieve skills --catalog ./.catalog --doc-id lean-ctx__skill --query content --line_num 15
EOF
			;;
		*)
			die "unknown indexer subcommand: ${sub} (try: build, survivors, retrieve, all)"
			;;
		esac
		;;
	sdk-python)
		require_repo_root
		cyt_build_sdk_python
		cyt_verify_sdk_python
		;;
	sdk-rust-release)
		require_repo_root
		cyt_build_sdk_rust_release
		;;
	sdk-verify)
		require_repo_root
		cyt_verify_sdk_python
		;;
	sdk-typescript)
		require_repo_root
		cyt_build_sdk_typescript
		;;
	sdk-c)
		require_repo_root
		cyt_build_sdk_c
		;;
	sdk-go)
		require_repo_root
		cyt_build_sdk_go
		;;
	sdk-all)
		require_repo_root
		cyt_section "SDK: Python"
		cyt_build_sdk_python
		cyt_verify_sdk_python
		cyt_section "SDK: C"
		cyt_build_sdk_c
		cyt_section "SDK: Go"
		cyt_build_sdk_go
		cyt_section "SDK: TypeScript"
		cyt_build_sdk_typescript
		;;
	app-setup | setup)
		require_repo_root
		cyt_sync_app
		;;
	app-verify)
		require_repo_root
		cyt_verify_app_python
		;;
	verify)
		require_repo_root
		cyt_verify_sdk_python
		cyt_verify_app_python
		;;
	app-test | test)
		require_repo_root
		if [[ "${CYT_WORKFLOW_SERIAL:-}" == 1 ]]; then
			cyt_test_app
		else
			cyt_prek_parallel_gate py-only
		fi
		;;
	app-test-runtime)
		require_repo_root
		cyt_test_app_runtime
		;;
	app-build | build-wheels)
		require_repo_root
		cyt_sync_app
		cyt_verify_app_python
		cyt_build_app_wheel
		;;
	app-all)
		require_repo_root
		cyt_build_all_app
		;;
	proxy)
		require_repo_root
		cyt_run_proxy "$@"
		;;
	simulate-registry)
		require_repo_root
		require_cmd uv
		require_cmd cargo
		require_cmd npm

		local sim_label="simulate-registry" sim_total=10

		cyt_step "${sim_label}" 1 "${sim_total}" "sync app"
		cyt_sync_app
		cyt_step_done "${sim_label}"

		cyt_step "${sim_label}" 2 "${sim_total}" "build SDK Python (maturin + pytest)"
		# Prek-loop runs pytest-sdk-python / local-dev-sdk-python before this hook.
		# On Windows, maturin develop cannot overwrite _native.pyd while pytest still
		# holds the DLL; step 5 (uv build) compiles Rust again for the wheel smoke test.
		if [[ -f "${CYT_REPO_ROOT}/sdk/python/src/cyt_indexer/_native.pyd" ]] ||
			[[ -f "${CYT_REPO_ROOT}/sdk/python/src/cyt_indexer/_native.so" ]]; then
			SKIP_MATURIN_DEVELOP=1 cyt_build_sdk_python
		else
			cyt_build_sdk_python
		fi
		cyt_step_done "${sim_label}"

		cyt_step "${sim_label}" 3 "${sim_total}" "build Rust indexer (cargo build/test + catalog smoke)"
		cyt_build_rust
		cyt_step_done "${sim_label}"

		cyt_step "${sim_label}" 4 "${sim_total}" "prepare isolated install workspace"
		SIM_DIR="${CYT_SIM_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/cyt-local-dev.XXXXXX")}"
		SIM_DIR="$(cd "${SIM_DIR}" && pwd -P)"
		KEEP_SIM_DIR="${KEEP_SIM_DIR:-}"
		trap '[[ -n "${KEEP_SIM_DIR}" ]] || rm -rf "${SIM_DIR}"' EXIT
		mkdir -p "${SIM_DIR}/dist-sdk" "${SIM_DIR}/dist-app" "${SIM_DIR}/npm-pack"
		cyt_step_done "${sim_label}"

		cyt_step "${sim_label}" 5 "${sim_total}" "build cyt-indexer-sdk wheel"
		cyt_run chunk_run_without_worktree_patches "${CYT_REPO_ROOT}" \
			bash -c "cd \"${CYT_REPO_ROOT}/sdk/python\" && uv build -o \"${SIM_DIR}/dist-sdk\""
		cyt_step_done "${sim_label}"

		cyt_step "${sim_label}" 6 "${sim_total}" "build clear-your-tools wheel"
		cyt_run bash -c "cd \"${CYT_REPO_ROOT}\" && uv build -o \"${SIM_DIR}/dist-app\""
		cyt_step_done "${sim_label}"

		cyt_step "${sim_label}" 7 "${sim_total}" "cargo publish --dry-run"
		cyt_cargo_publish_dry_run
		cyt_step_done "${sim_label}"

		cyt_step "${sim_label}" 8 "${sim_total}" "npm pack TypeScript SDK"
		cyt_run bash -c "cd \"${CYT_REPO_ROOT}/sdk/typescript\" && npm ci && npm run build && npm pack --pack-destination \"${SIM_DIR}/npm-pack\""
		cyt_step_done "${sim_label}"

		cyt_step "${sim_label}" 9 "${sim_total}" "install wheels in isolated venv"
		SIM_VENV="${SIM_DIR}/venv"
		cyt_run uv venv "${SIM_VENV}"
		SIM_PY="$(cyt_venv_python "${SIM_VENV}")"
		SDK_WHL=("${SIM_DIR}"/dist-sdk/cyt_indexer_sdk-*.whl)
		APP_WHL=("${SIM_DIR}"/dist-app/clear_your_tools-*.whl)
		[[ -f "${SDK_WHL[0]}" ]] || die "SDK wheel not found under ${SIM_DIR}/dist-sdk"
		[[ -f "${APP_WHL[0]}" ]] || die "app wheel not found under ${SIM_DIR}/dist-app"
		cyt_run uv pip install --python "${SIM_PY}" "${SDK_WHL[0]}"
		# Avoid `path.whl[all]` on Windows: uv wrongly parses `[all]` in file URLs (C:/Users/... → C:~/...).
		cyt_run uv pip install --python "${SIM_PY}" "${APP_WHL[0]}"
		cyt_step_done "${sim_label}"

		cyt_step "${sim_label}" 10 "${sim_total}" "smoke imports"
		cyt_run "${SIM_PY}" - <<'PY'
from importlib import metadata

from cyt_indexer._native import build_catalog_index as native_build
from cyt_indexer.build import build_catalog_index as sdk_build
from cyt.indexer.build import build_catalog_index as app_build
import cyt

assert callable(native_build)
assert sdk_build is app_build
assert metadata.version("cyt-indexer-sdk") == metadata.version("clear-your-tools")
print("OK: isolated wheel install")
print("  cyt version:", getattr(cyt, "__version__", "?"))
print("  cyt-indexer-sdk:", metadata.version("cyt-indexer-sdk"))
PY

		cyt_step_done "${sim_label}"

		if [[ -z "${CYT_LOCAL_DEV_SHORT:-}" ]]; then
			info "simulate-registry done (${SIM_DIR})"
		fi
		if [[ -n "${KEEP_SIM_DIR}" ]]; then
			trap - EXIT
			info "KEEP_SIM_DIR=1 — directory kept"
		fi
		;;
	all)
		require_repo_root
		cyt_run_all
		if [[ -n "${CYT_RUN_RUNTIME_TESTS:-}" ]]; then
			cyt_test_app_runtime
		fi
		info "all done"
		;;
	ci)
		require_repo_root
		cyt_section "CI"
		cyt_verify_dependency_pins
		cyt_sync_app
		cyt_verify_app_python
		cyt_verify_sdk_import
		if command -v ast-grep >/dev/null 2>&1; then
			info "ast-grep scan"
			cyt_run ast-grep scan src/ sdk/
		else
			info "skip ast-grep (not on PATH)"
		fi
		cyt_run uv run python scripts/pre-commit-hooks/check_agent_imports.py
		cyt_run uv run python scripts/pre-commit-hooks/check_cyt_client_imports.py
		if command -v ruff >/dev/null 2>&1 || [[ -x "${CYT_VENV_BIN}/ruff" ]]; then
			info "ruff check"
			cyt_run uv run ruff check src/cyt src/tests
		else
			info "skip ruff (not on PATH)"
		fi
		if [[ "${CYT_WORKFLOW_SERIAL:-}" == 1 ]]; then
			cyt_test_app_python
			cyt_build_app_wheel
		else
			cyt_prek_parallel_gate py-only
		fi
		;;
	"" | -h | --help | help)
		usage
		;;
	*)
		if [[ -n "${CYT_LOCAL_DEV_SHORT:-}" ]]; then
			die "unknown command: ${cmd}"
		fi
		echo "unknown command: ${cmd}" >&2
		echo >&2
		usage >&2
		return 1
		;;
	esac
}

if [[ -n "${CYT_LOCAL_DEV_SHORT}" ]]; then
	_cyt_local_dev_main "${LOCAL_DEV_ARGS[@]}" 2>&1 | "${SCRIPT_DIR}/../../lib/shorten-paths.sh" | cyt_filter_short_logs
else
	_cyt_local_dev_main "${LOCAL_DEV_ARGS[@]}" 2>&1 | "${SCRIPT_DIR}/../../lib/shorten-paths.sh"
fi
exit "${PIPESTATUS[0]}"