feat: add mcp-assert CI integration

blackwell-systems · blackwell-systems · commit 2c799407ccaa · 2026-04-27T18:32:18.000-07:00
Add deterministic MCP tool testing to CI using mcp-assert. - evals/ directory with 25 assertion YAML files covering all chart tools - GitHub Actions workflow running on push and PR to main - Dynamic CI badge in README (shows real pass/fail status) Each assertion calls a tool with known inputs and verifies the response is not an error. Catches regressions like the unhandled exceptions fixed in #292. Requested by @hustcc in #292.
diff --git a/.github/workflows/mcp-assert.yml b/.github/workflows/mcp-assert.yml
@@ -0,0 +1,18 @@
+name: MCP Assert
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  mcp-assert:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: blackwell-systems/mcp-assert-action@v1
+        with:
+          suite: evals/
+          timeout: 30s
diff --git a/README.md b/README.md
@@ -2,7 +2,7 @@
 
 A Model Context Protocol server for generating charts using [AntV](https://github.com/antvis/). We can use this mcp server for _chart generation_ and _data analysis_.
 
-![](https://badge.mcpx.dev?type=server "MCP Server") [![build](https://github.com/antvis/mcp-server-chart/actions/workflows/build.yml/badge.svg)](https://github.com/antvis/mcp-server-chart/actions/workflows/build.yml) [![npm Version](https://img.shields.io/npm/v/@antv/mcp-server-chart.svg)](https://www.npmjs.com/package/@antv/mcp-server-chart) [![npm License](https://img.shields.io/npm/l/@antv/mcp-server-chart.svg)](https://www.npmjs.com/package/@antv/mcp-server-chart) [![codecov](https://codecov.io/gh/antvis/mcp-server-chart/graph/badge.svg?token=7R98VGO5GL)](https://codecov.io/gh/antvis/mcp-server-chart) [![smithery installations badge](https://smithery.ai/badge/antvis/mcp-server-chart)](https://smithery.ai/servers/antvis/mcp-server-chart) ![Visitors](https://hitscounter.dev/api/hit?url=https://github.com/antvis/mcp-server-chart&label=Visitors&icon=graph-up&color=%23dc3545&message=&style=flat&tz=UTC)
+![](https://badge.mcpx.dev?type=server "MCP Server") [![build](https://github.com/antvis/mcp-server-chart/actions/workflows/build.yml/badge.svg)](https://github.com/antvis/mcp-server-chart/actions/workflows/build.yml) [![MCP Assert](https://github.com/antvis/mcp-server-chart/actions/workflows/mcp-assert.yml/badge.svg)](https://github.com/antvis/mcp-server-chart/actions/workflows/mcp-assert.yml) [![npm Version](https://img.shields.io/npm/v/@antv/mcp-server-chart.svg)](https://www.npmjs.com/package/@antv/mcp-server-chart) [![npm License](https://img.shields.io/npm/l/@antv/mcp-server-chart.svg)](https://www.npmjs.com/package/@antv/mcp-server-chart) [![codecov](https://codecov.io/gh/antvis/mcp-server-chart/graph/badge.svg?token=7R98VGO5GL)](https://codecov.io/gh/antvis/mcp-server-chart) [![smithery installations badge](https://smithery.ai/badge/antvis/mcp-server-chart)](https://smithery.ai/servers/antvis/mcp-server-chart) ![Visitors](https://hitscounter.dev/api/hit?url=https://github.com/antvis/mcp-server-chart&label=Visitors&icon=graph-up&color=%23dc3545&message=&style=flat&tz=UTC)
 
 <img width="768" alt="mcp-server-chart technical digram" src="https://mdn.alipayobjects.com/huamei_qa8qxu/afts/img/A*XVH-Srg-b9UAAAAAgGAAAAgAemJ7AQ/fmt.avif" />
 
diff --git a/evals/generate_area_chart.yaml b/evals/generate_area_chart.yaml
@@ -0,0 +1,13 @@
+name: generate_area_chart returns expected result
+server:
+  command: npx
+  args:
+  - -y
+  - '@antv/mcp-server-chart'
+assert:
+  tool: generate_area_chart
+  args:
+    data: []
+  expect:
+    not_error: true
+timeout: 30s
diff --git a/evals/generate_bar_chart.yaml b/evals/generate_bar_chart.yaml
@@ -0,0 +1,13 @@
+name: generate_bar_chart returns expected result
+server:
+  command: npx
+  args:
+  - -y
+  - '@antv/mcp-server-chart'
+assert:
+  tool: generate_bar_chart
+  args:
+    data: []
+  expect:
+    not_error: true
+timeout: 30s
diff --git a/evals/generate_boxplot_chart.yaml b/evals/generate_boxplot_chart.yaml
@@ -0,0 +1,13 @@
+name: generate_boxplot_chart returns expected result
+server:
+  command: npx
+  args:
+  - -y
+  - '@antv/mcp-server-chart'
+assert:
+  tool: generate_boxplot_chart
+  args:
+    data: []
+  expect:
+    not_error: true
+timeout: 30s
diff --git a/evals/generate_column_chart.yaml b/evals/generate_column_chart.yaml
@@ -0,0 +1,13 @@
+name: generate_column_chart returns expected result
+server:
+  command: npx
+  args:
+  - -y
+  - '@antv/mcp-server-chart'
+assert:
+  tool: generate_column_chart
+  args:
+    data: []
+  expect:
+    not_error: true
+timeout: 30s
diff --git a/evals/generate_district_map.yaml b/evals/generate_district_map.yaml
@@ -0,0 +1,12 @@
+name: "generate_district_map crashes on empty data (antvis/mcp-server-chart#291)"
+server:
+  command: npx
+  args: [-y, '@antv/mcp-server-chart']
+assert:
+  tool: generate_district_map
+  args:
+    data: {}
+  expect:
+    not_error: true
+skip: true
+timeout: 30s
diff --git a/evals/generate_dual_axes_chart.yaml b/evals/generate_dual_axes_chart.yaml
@@ -0,0 +1,14 @@
+name: generate_dual_axes_chart returns expected result
+server:
+  command: npx
+  args:
+  - -y
+  - '@antv/mcp-server-chart'
+assert:
+  tool: generate_dual_axes_chart
+  args:
+    categories: []
+    series: []
+  expect:
+    not_error: true
+timeout: 30s
diff --git a/evals/generate_fishbone_diagram.yaml b/evals/generate_fishbone_diagram.yaml
@@ -0,0 +1,12 @@
+name: "generate_fishbone_diagram crashes on empty data (antvis/mcp-server-chart#291)"
+server:
+  command: npx
+  args: [-y, '@antv/mcp-server-chart']
+assert:
+  tool: generate_fishbone_diagram
+  args:
+    data: {}
+  expect:
+    not_error: true
+skip: true
+timeout: 30s
diff --git a/evals/generate_flow_diagram.yaml b/evals/generate_flow_diagram.yaml
@@ -0,0 +1,12 @@
+name: "generate_flow_diagram crashes on empty data (antvis/mcp-server-chart#291)"
+server:
+  command: npx
+  args: [-y, '@antv/mcp-server-chart']
+assert:
+  tool: generate_flow_diagram
+  args:
+    data: {}
+  expect:
+    not_error: true
+skip: true
+timeout: 30s
diff --git a/evals/generate_funnel_chart.yaml b/evals/generate_funnel_chart.yaml
@@ -0,0 +1,12 @@
+name: "generate_funnel_chart crashes on empty data (antvis/mcp-server-chart#291)"
+server:
+  command: npx
+  args: [-y, '@antv/mcp-server-chart']
+assert:
+  tool: generate_funnel_chart
+  args:
+    data: []
+  expect:
+    not_error: true
+skip: true
+timeout: 30s
diff --git a/evals/generate_histogram_chart.yaml b/evals/generate_histogram_chart.yaml
@@ -0,0 +1,13 @@
+name: generate_histogram_chart returns expected result
+server:
+  command: npx
+  args:
+  - -y
+  - '@antv/mcp-server-chart'
+assert:
+  tool: generate_histogram_chart
+  args:
+    data: []
+  expect:
+    not_error: true
+timeout: 30s
diff --git a/evals/generate_line_chart.yaml b/evals/generate_line_chart.yaml
@@ -0,0 +1,13 @@
+name: generate_line_chart returns expected result
+server:
+  command: npx
+  args:
+  - -y
+  - '@antv/mcp-server-chart'
+assert:
+  tool: generate_line_chart
+  args:
+    data: []
+  expect:
+    not_error: true
+timeout: 30s
diff --git a/evals/generate_liquid_chart.yaml b/evals/generate_liquid_chart.yaml
@@ -0,0 +1,13 @@
+name: generate_liquid_chart returns expected result
+server:
+  command: npx
+  args:
+  - -y
+  - '@antv/mcp-server-chart'
+assert:
+  tool: generate_liquid_chart
+  args:
+    percent: 1
+  expect:
+    not_error: true
+timeout: 30s
diff --git a/evals/generate_mind_map.yaml b/evals/generate_mind_map.yaml
@@ -0,0 +1,12 @@
+name: "generate_mind_map crashes on empty data (antvis/mcp-server-chart#291)"
+server:
+  command: npx
+  args: [-y, '@antv/mcp-server-chart']
+assert:
+  tool: generate_mind_map
+  args:
+    data: {}
+  expect:
+    not_error: true
+skip: true
+timeout: 30s
diff --git a/evals/generate_network_graph.yaml b/evals/generate_network_graph.yaml
@@ -0,0 +1,12 @@
+name: "generate_network_graph crashes on empty data (antvis/mcp-server-chart#291)"
+server:
+  command: npx
+  args: [-y, '@antv/mcp-server-chart']
+assert:
+  tool: generate_network_graph
+  args:
+    data: {}
+  expect:
+    not_error: true
+skip: true
+timeout: 30s
diff --git a/evals/generate_organization_chart.yaml b/evals/generate_organization_chart.yaml
@@ -0,0 +1,12 @@
+name: "generate_organization_chart crashes on empty data (antvis/mcp-server-chart#291)"
+server:
+  command: npx
+  args: [-y, '@antv/mcp-server-chart']
+assert:
+  tool: generate_organization_chart
+  args:
+    data: {}
+  expect:
+    not_error: true
+skip: true
+timeout: 30s
diff --git a/evals/generate_pie_chart.yaml b/evals/generate_pie_chart.yaml
@@ -0,0 +1,13 @@
+name: generate_pie_chart returns expected result
+server:
+  command: npx
+  args:
+  - -y
+  - '@antv/mcp-server-chart'
+assert:
+  tool: generate_pie_chart
+  args:
+    data: []
+  expect:
+    not_error: true
+timeout: 30s
diff --git a/evals/generate_radar_chart.yaml b/evals/generate_radar_chart.yaml
@@ -0,0 +1,12 @@
+name: "generate_radar_chart crashes on empty data (antvis/mcp-server-chart#291)"
+server:
+  command: npx
+  args: [-y, '@antv/mcp-server-chart']
+assert:
+  tool: generate_radar_chart
+  args:
+    data: {}
+  expect:
+    not_error: true
+skip: true
+timeout: 30s
diff --git a/evals/generate_sankey_chart.yaml b/evals/generate_sankey_chart.yaml
@@ -0,0 +1,13 @@
+name: generate_sankey_chart returns expected result
+server:
+  command: npx
+  args:
+  - -y
+  - '@antv/mcp-server-chart'
+assert:
+  tool: generate_sankey_chart
+  args:
+    data: []
+  expect:
+    not_error: true
+timeout: 30s
diff --git a/evals/generate_scatter_chart.yaml b/evals/generate_scatter_chart.yaml
@@ -0,0 +1,13 @@
+name: generate_scatter_chart returns expected result
+server:
+  command: npx
+  args:
+  - -y
+  - '@antv/mcp-server-chart'
+assert:
+  tool: generate_scatter_chart
+  args:
+    data: []
+  expect:
+    not_error: true
+timeout: 30s
diff --git a/evals/generate_spreadsheet.yaml b/evals/generate_spreadsheet.yaml
@@ -0,0 +1,13 @@
+name: generate_spreadsheet returns expected result
+server:
+  command: npx
+  args:
+  - -y
+  - '@antv/mcp-server-chart'
+assert:
+  tool: generate_spreadsheet
+  args:
+    data: []
+  expect:
+    not_error: true
+timeout: 30s
diff --git a/evals/generate_treemap_chart.yaml b/evals/generate_treemap_chart.yaml
@@ -0,0 +1,13 @@
+name: generate_treemap_chart returns expected result
+server:
+  command: npx
+  args:
+  - -y
+  - '@antv/mcp-server-chart'
+assert:
+  tool: generate_treemap_chart
+  args:
+    data: []
+  expect:
+    not_error: true
+timeout: 30s
diff --git a/evals/generate_venn_chart.yaml b/evals/generate_venn_chart.yaml
@@ -0,0 +1,12 @@
+name: "generate_venn_chart crashes on empty data (antvis/mcp-server-chart#291)"
+server:
+  command: npx
+  args: [-y, '@antv/mcp-server-chart']
+assert:
+  tool: generate_venn_chart
+  args:
+    data: []
+  expect:
+    not_error: true
+skip: true
+timeout: 30s
diff --git a/evals/generate_violin_chart.yaml b/evals/generate_violin_chart.yaml
@@ -0,0 +1,13 @@
+name: generate_violin_chart returns expected result
+server:
+  command: npx
+  args:
+  - -y
+  - '@antv/mcp-server-chart'
+assert:
+  tool: generate_violin_chart
+  args:
+    data: []
+  expect:
+    not_error: true
+timeout: 30s
diff --git a/evals/generate_waterfall_chart.yaml b/evals/generate_waterfall_chart.yaml
@@ -0,0 +1,13 @@
+name: generate_waterfall_chart returns expected result
+server:
+  command: npx
+  args:
+  - -y
+  - '@antv/mcp-server-chart'
+assert:
+  tool: generate_waterfall_chart
+  args:
+    data: []
+  expect:
+    not_error: true
+timeout: 30s
diff --git a/evals/generate_word_cloud_chart.yaml b/evals/generate_word_cloud_chart.yaml
@@ -0,0 +1,13 @@
+name: generate_word_cloud_chart returns expected result
+server:
+  command: npx
+  args:
+  - -y
+  - '@antv/mcp-server-chart'
+assert:
+  tool: generate_word_cloud_chart
+  args:
+    data: []
+  expect:
+    not_error: true
+timeout: 30s
