Skip to content

Commit 8dbd4b8

Browse files
lucarliglucarlig
committed
feat: add benchmark suite and ui
Signed-off-by: lucarlig <luca.carlig@ibm.com>
1 parent 88d2100 commit 8dbd4b8

32 files changed

+9579
-0
lines changed

‎Makefile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7057,6 +7057,12 @@ profile-compare:
70577057
--current $(PROFILE_DIR)/mcp_calls_profile.prof \
70587058
--output $(REPORTS_DIR)/profile-comparison.json
70597059

7060+
# help: benchmark - Open the interactive benchmark launcher
7061+
.PHONY: benchmark
7062+
benchmark:
7063+
@echo "Starting benchmark console (first run may compile; wait for TUI)..."
7064+
cargo run --manifest-path tools_rust/benchmark_console/Cargo.toml --
7065+
70607066
.PHONY: async-validate
70617067
async-validate:
70627068
@echo "✅ Validating async code patterns..."

‎benchmarks/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""Benchmark packages."""

‎benchmarks/contextforge/Containerfile‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
# syntax=docker/dockerfile:1.7
2+
3+
ARG PYTHON_VERSION=3.12
4+
ARG ENABLE_RUST=false
5+
ARG ENABLE_PROFILING=false
6+
7+
FROM quay.io/pypa/manylinux2014:2026.03.06-3 AS rust-builder-base
8+
ARG ENABLE_RUST
9+
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
10+
ENV CARGO_HOME=/usr/local/cargo
11+
ENV RUSTUP_HOME=/usr/local/rustup
12+
ENV PATH="${CARGO_HOME}/bin:${PATH}"
13+
WORKDIR /build
14+
15+
RUN if [ "$ENABLE_RUST" != "true" ]; then \
16+
mkdir -p /build/rust-wheels; \
17+
exit 0; \
18+
fi
19+
20+
RUN if [ "$ENABLE_RUST" = "true" ]; then \
21+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable; \
22+
fi
23+
24+
COPY plugins_rust/ /build/plugins_rust/
25+
26+
RUN if [ "$ENABLE_RUST" = "true" ]; then \
27+
if [ -f "${CARGO_HOME}/env" ]; then . "${CARGO_HOME}/env"; fi \
28+
&& mkdir -p /build/rust-wheels \
29+
&& /opt/python/cp312-cp312/bin/python -m pip install --upgrade pip maturin \
30+
&& for plugin_dir in /build/plugins_rust/*/; do \
31+
if [ -f "$plugin_dir/Cargo.toml" ] && [ -f "$plugin_dir/pyproject.toml" ]; then \
32+
/opt/python/cp312-cp312/bin/maturin build --release --compatibility manylinux2014 --manifest-path "$plugin_dir/Cargo.toml" --out /build/rust-wheels || exit 1; \
33+
fi; \
34+
done; \
35+
fi
36+
37+
FROM rust-builder-base AS rust-builder
38+
39+
FROM registry.access.redhat.com/ubi10/ubi:10.1-1772441712 AS builder
40+
SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
41+
42+
ARG PYTHON_VERSION
43+
ARG ENABLE_RUST=false
44+
ARG ENABLE_PROFILING=false
45+
ARG GRPC_PYTHON_BUILD_SYSTEM_OPENSSL='False'
46+
47+
RUN dnf upgrade -y \
48+
&& dnf install -y \
49+
python${PYTHON_VERSION} \
50+
python${PYTHON_VERSION}-devel \
51+
binutils \
52+
openssl-devel \
53+
gcc \
54+
gcc-c++ \
55+
postgresql-devel \
56+
curl \
57+
&& update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
58+
&& dnf clean all
59+
60+
WORKDIR /app
61+
62+
RUN if [ "$(uname -m)" = "s390x" ] || [ "$(uname -m)" = "ppc64le" ]; then \
63+
echo "export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL='True'" > /etc/profile.d/use-openssl.sh; \
64+
else \
65+
echo "export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL='False'" > /etc/profile.d/use-openssl.sh; \
66+
fi \
67+
&& chmod 644 /etc/profile.d/use-openssl.sh
68+
69+
COPY pyproject.toml /app/
70+
COPY --from=rust-builder /build/rust-wheels/ /tmp/rust-wheels/
71+
72+
RUN . /etc/profile.d/use-openssl.sh \
73+
&& python3 -m venv /app/.venv \
74+
&& /app/.venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel pdm uv \
75+
&& /app/.venv/bin/uv pip install ".[redis,postgres,mysql,observability,granian]" \
76+
&& if [ "$ENABLE_RUST" = "true" ] && ls /tmp/rust-wheels/*.whl >/dev/null 2>&1; then \
77+
/app/.venv/bin/python3 -m pip install /tmp/rust-wheels/*.whl; \
78+
fi \
79+
&& if [ "$ENABLE_PROFILING" = "true" ]; then \
80+
/app/.venv/bin/pip install --no-cache-dir "memray>=1.17.0" "py-spy>=0.4.0"; \
81+
fi \
82+
&& /app/.venv/bin/pip uninstall --yes uv pip setuptools wheel pdm \
83+
&& rm -rf /tmp/rust-wheels /root/.cache /var/cache/dnf \
84+
&& rm -rf /app/*.egg-info /app/build /app/dist /app/.eggs /app/.venv/share/python-wheels
85+
86+
COPY run-gunicorn.sh run-granian.sh run.sh /app/
87+
COPY gunicorn.config.py /app/
88+
COPY mcpgateway/ /app/mcpgateway/
89+
COPY plugins/ /app/plugins/
90+
COPY benchmarks/contextforge/ /app/benchmarks/contextforge/
91+
COPY mcp-catalog.yml /app/
92+
93+
RUN cp /app/benchmarks/contextforge/docker-entrypoint.sh /app/docker-entrypoint.sh \
94+
&& chmod +x /app/run-gunicorn.sh /app/run-granian.sh /app/docker-entrypoint.sh /app/run.sh /app/benchmarks/contextforge/run-uvicorn.sh \
95+
&& python3 -OO -m compileall -q /app/.venv /app/mcpgateway /app/plugins /app/benchmarks/contextforge 2>/dev/null || true \
96+
&& find /app -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true \
97+
&& chown -R 1001:0 /app \
98+
&& chmod -R g=u /app
99+
100+
FROM registry.access.redhat.com/ubi10/ubi-minimal:10.1-1772441549 AS runtime
101+
102+
ARG PYTHON_VERSION=3.12
103+
ARG ENABLE_PROFILING=false
104+
105+
RUN microdnf upgrade -y --nodocs --setopt=install_weak_deps=0 \
106+
&& microdnf install -y --nodocs --setopt=install_weak_deps=0 \
107+
python${PYTHON_VERSION} \
108+
ca-certificates \
109+
procps-ng \
110+
shadow-utils \
111+
&& if [ "$ENABLE_PROFILING" = "true" ]; then \
112+
microdnf install -y --nodocs --setopt=install_weak_deps=0 gdb; \
113+
fi \
114+
&& microdnf clean all \
115+
&& rm -rf /var/cache/yum \
116+
&& ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python3 \
117+
&& useradd --uid 1001 --gid 0 --home-dir /app --shell /sbin/nologin --no-create-home --comment app app
118+
119+
COPY --from=builder --chown=1001:0 /app /app
120+
121+
ENV PATH="/app/.venv/bin:${PATH}" \
122+
PYTHONDONTWRITEBYTECODE=1 \
123+
PYTHONUNBUFFERED=1 \
124+
PYTHONHASHSEED=random \
125+
PIP_NO_CACHE_DIR=1 \
126+
PIP_DISABLE_PIP_VERSION_CHECK=1
127+
128+
WORKDIR /app
129+
EXPOSE 4444
130+
USER 1001
131+
132+
HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
133+
CMD ["python3", "-c", "import httpx,sys;sys.exit(0 if httpx.get('http://localhost:4444/health',timeout=5).status_code==200 else 1)"]
134+
135+
ENTRYPOINT ["/app/docker-entrypoint.sh"]

‎benchmarks/contextforge/__init__.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
"""ContextForge benchmark bundle."""
2+
3+
from .runner import main
4+
5+
__all__ = ["main"]

‎benchmarks/contextforge/__main__.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
"""Module entrypoint for the self-contained benchmark bundle."""
2+
3+
from __future__ import annotations
4+
5+
from .runner import main
6+
7+
8+
if __name__ == "__main__":
9+
raise SystemExit(main())

‎benchmarks/contextforge/docker-entrypoint.sh‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
HTTP_SERVER="${HTTP_SERVER:-gunicorn}"
6+
7+
case "${HTTP_SERVER}" in
8+
granian)
9+
echo "Starting ContextForge benchmark image with Granian..."
10+
exec ./run-granian.sh "$@"
11+
;;
12+
gunicorn)
13+
echo "Starting ContextForge benchmark image with Gunicorn..."
14+
exec ./run-gunicorn.sh "$@"
15+
;;
16+
uvicorn)
17+
echo "Starting ContextForge benchmark image with Uvicorn..."
18+
exec ./benchmarks/contextforge/run-uvicorn.sh "$@"
19+
;;
20+
*)
21+
echo "ERROR: Unknown HTTP_SERVER value: ${HTTP_SERVER}"
22+
echo "Valid options: granian, gunicorn, uvicorn"
23+
exit 1
24+
;;
25+
esac

‎benchmarks/contextforge/locust/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""Benchmark Locust drivers."""

0 commit comments

Comments
 (0)