Skip to content

Commit ea3af18

Browse files
authored
Merge pull request #95 from Azure-Samples/image-mcp
Add MCP server with image search and carousel display
2 parents 3de3406 + 2e7305d commit ea3af18

280 files changed

Lines changed: 1188 additions & 31 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
---
2+
name: convert-heic-to-jpeg
3+
description: Convert HEIC or HEIF images to JPEG with `uv run` using inline script dependencies. Use this when new phone photos are added under `pictures/` and need to be converted before indexing or upload.
4+
argument-hint: "[path] [--quality 90] [--overwrite]"
5+
---
6+
7+
# Convert HEIC To JPEG
8+
9+
Use this skill when you need to convert `.heic` or `.heif` images into `.jpg` files for local development workflows.
10+
11+
The script for this skill lives at [./convert_heic_to_jpeg.py](./convert_heic_to_jpeg.py) and is designed to be run with `uv run`.
12+
13+
## What this skill does
14+
15+
- Converts a single HEIC or HEIF file to JPEG.
16+
- Recursively scans a directory for HEIC or HEIF files.
17+
- Writes JPEG files next to the source images by default.
18+
- Can write output into a separate directory.
19+
- Skips existing JPEG files unless `--overwrite` is passed.
20+
21+
## How to run it
22+
23+
Run the script with `uv run` so the inline PEP 723 dependencies are installed automatically:
24+
25+
```bash
26+
uv run .agents/skills/convert-heic-to-jpeg/convert_heic_to_jpeg.py pictures
27+
```
28+
29+
Convert one file:
30+
31+
```bash
32+
uv run .agents/skills/convert-heic-to-jpeg/convert_heic_to_jpeg.py pictures/clothes/IMG_3233.HEIC
33+
```
34+
35+
Convert into a separate output directory:
36+
37+
```bash
38+
uv run .agents/skills/convert-heic-to-jpeg/convert_heic_to_jpeg.py pictures --output-dir converted-pictures
39+
```
40+
41+
Overwrite existing JPEG files:
42+
43+
```bash
44+
uv run .agents/skills/convert-heic-to-jpeg/convert_heic_to_jpeg.py pictures --overwrite
45+
```
46+
47+
## Recommended Workflow
48+
49+
1. Point the script at `pictures/` or a subdirectory such as `pictures/clothes/`.
50+
2. Review the summary output to confirm how many files were converted or skipped.
51+
3. Re-run with `--overwrite` only if you intentionally want to replace existing JPEG outputs.
52+
53+
## Notes
54+
55+
- This skill is stored in `.agents/skills`, which VS Code documents as a default project skill location.
56+
- The script depends on Pillow and `pillow-heif`, declared inline in the script so they can be resolved by `uv run`.
Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
# /// script
2+
# requires-python = ">=3.11"
3+
# dependencies = [
4+
# "pillow>=11.0.0",
5+
# "pillow-heif>=0.22.0",
6+
# ]
7+
# ///
8+
9+
"""Convert HEIC and HEIF images to JPEG format."""
10+
11+
from __future__ import annotations
12+
13+
import argparse
14+
from dataclasses import dataclass
15+
from pathlib import Path
16+
17+
from PIL import Image
18+
import pillow_heif
19+
20+
# Register HEIF support with Pillow before opening files.
21+
pillow_heif.register_heif_opener()
22+
23+
SUPPORTED_EXTENSIONS = {".heic", ".heif"}
24+
25+
26+
@dataclass(frozen=True)
27+
class ConversionResult:
28+
"""Capture the outcome for a single source image."""
29+
30+
source_path: Path
31+
output_path: Path
32+
status: str
33+
34+
35+
def is_heif_file(path: Path) -> bool:
36+
"""Return whether the path has a supported HEIF extension."""
37+
return path.suffix.lower() in SUPPORTED_EXTENSIONS
38+
39+
40+
def build_output_path(
41+
source_path: Path, input_root: Path, output_dir: Path | None
42+
) -> Path:
43+
"""Build the destination JPEG path for a source HEIF image."""
44+
if output_dir is None:
45+
return source_path.with_suffix(".jpg")
46+
47+
if input_root.is_file():
48+
return output_dir / f"{source_path.stem}.jpg"
49+
50+
relative_path = source_path.relative_to(input_root).with_suffix(".jpg")
51+
return output_dir / relative_path
52+
53+
54+
def save_as_jpeg(source_path: Path, output_path: Path, quality: int) -> None:
55+
"""Open a HEIF image and save it as JPEG."""
56+
output_path.parent.mkdir(parents=True, exist_ok=True)
57+
58+
with Image.open(source_path) as image:
59+
if image.mode not in ("RGB", "L"):
60+
image = image.convert("RGB")
61+
image.save(output_path, "JPEG", quality=quality)
62+
63+
64+
def convert_heic_to_jpeg(
65+
source_path: Path,
66+
input_root: Path,
67+
output_dir: Path | None = None,
68+
quality: int = 90,
69+
overwrite: bool = False,
70+
) -> ConversionResult:
71+
"""Convert one HEIF image to JPEG."""
72+
output_path = build_output_path(source_path, input_root, output_dir)
73+
74+
if output_path.exists() and not overwrite:
75+
return ConversionResult(
76+
source_path=source_path, output_path=output_path, status="skipped"
77+
)
78+
79+
save_as_jpeg(source_path, output_path, quality)
80+
return ConversionResult(
81+
source_path=source_path, output_path=output_path, status="converted"
82+
)
83+
84+
85+
def gather_source_files(input_path: Path) -> list[Path]:
86+
"""Return all HEIF images under the given file or directory."""
87+
if input_path.is_file():
88+
if not is_heif_file(input_path):
89+
raise ValueError(f"Unsupported input file: {input_path}")
90+
return [input_path]
91+
92+
return sorted(
93+
path for path in input_path.rglob("*") if path.is_file() and is_heif_file(path)
94+
)
95+
96+
97+
def parse_args() -> argparse.Namespace:
98+
"""Parse command-line arguments."""
99+
parser = argparse.ArgumentParser(description="Convert HEIC or HEIF images to JPEG")
100+
parser.add_argument("input", type=Path, help="Input HEIC/HEIF file or directory")
101+
parser.add_argument(
102+
"-o",
103+
"--output-dir",
104+
type=Path,
105+
default=None,
106+
help="Optional output directory for JPEG files",
107+
)
108+
parser.add_argument(
109+
"-q",
110+
"--quality",
111+
type=int,
112+
default=90,
113+
help="JPEG quality from 1 to 100 (default: 90)",
114+
)
115+
parser.add_argument(
116+
"--overwrite",
117+
action="store_true",
118+
help="Replace existing JPEG files instead of skipping them",
119+
)
120+
return parser.parse_args()
121+
122+
123+
def main() -> int:
124+
"""Run the CLI conversion workflow."""
125+
args = parse_args()
126+
127+
if not args.input.exists():
128+
raise FileNotFoundError(f"Input path does not exist: {args.input}")
129+
130+
if not 1 <= args.quality <= 100:
131+
raise ValueError("Quality must be between 1 and 100")
132+
133+
source_files = gather_source_files(args.input)
134+
if not source_files:
135+
print("No HEIC or HEIF files found.")
136+
return 0
137+
138+
converted_count = 0
139+
skipped_count = 0
140+
141+
for source_path in source_files:
142+
result = convert_heic_to_jpeg(
143+
source_path=source_path,
144+
input_root=args.input,
145+
output_dir=args.output_dir,
146+
quality=args.quality,
147+
overwrite=args.overwrite,
148+
)
149+
print(f"{result.status:9} {result.source_path} -> {result.output_path}")
150+
if result.status == "converted":
151+
converted_count += 1
152+
else:
153+
skipped_count += 1
154+
155+
print(
156+
f"Summary: converted={converted_count} skipped={skipped_count} total={len(source_files)}"
157+
)
158+
return 0
159+
160+
161+
if __name__ == "__main__":
162+
raise SystemExit(main())

.github/workflows/python.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
fail-fast: false
1515
matrix:
1616
os: ["ubuntu-latest"]
17-
python_version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
17+
python_version: ["3.10", "3.11", "3.12", "3.13"]
1818
steps:
1919
- uses: actions/checkout@v5
2020
- name: Install uv

.vscode/mcp.json

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"servers": {
3+
"image-search-local": {
4+
"type": "http",
5+
"url": "http://localhost:8001/mcp"
6+
},
7+
"image-search-azure": {
8+
"type": "http",
9+
"url": "https://<your-mcp-container-app>/mcp"
10+
}
11+
}
12+
}

README.md

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@ urlFragment: image-search-aisearch
1919

2020
# Image search with Azure AI Search
2121

22-
This project creates a simple image search application using Azure AI Search. The application allows you to search for images with a textual query, and searches using a multimodal embedding from the Azure AI Vision API. The frontend is built with TypeScript/React and the backend is built with Python/Quart.
22+
This project creates an image search web application and an [MCP (Model Context Protocol)](https://modelcontextprotocol.io) server, both backed by [Azure AI Search](https://learn.microsoft.com/azure/search/search-what-is-azure-search). They allow you to search for images using natural language queries, leveraging multimodal embeddings from the Azure AI Vision API and image descriptions generated by the Azure OpenAI Service.
23+
24+
The web app frontend is built with TypeScript/React and the backend with Python/Quart. The MCP server is built with Python/FastMCP, and includes two tools that can be used by any MCP client, such as VS Code GitHub Copilot. The `image_search` tool returns a list of search results with thumbnail previews and descriptions, and the `display_image_files` tool renders images using [MCP apps](https://apps.extensions.modelcontextprotocol.io/api/).
2325

2426
This project uses the sample nature data set from [Vision Studio](https://learn.microsoft.com/azure/ai-services/computer-vision/overview-vision-studio).
2527

@@ -56,7 +58,7 @@ either by deleting the resource group in the Portal or running `azd down`.
5658
First install the required tools:
5759

5860
* [Azure Developer CLI](https://aka.ms/azure-dev/install)
59-
* [Python 3.9, 3.10, or 3.11](https://www.python.org/downloads/)
61+
* [Python 3.10 or later](https://www.python.org/downloads/)
6062
* **Important**: Python and the pip package manager must be in the path in Windows for the setup scripts to work.
6163
* **Important**: Ensure you can run `python --version` from console. On Ubuntu, you might need to run `sudo apt install python-is-python3` to link `python` to `python3`.
6264
* [Node.js 14+](https://nodejs.org/en/download/)
@@ -75,18 +77,55 @@ Then bring down the project code:
7577

7678
Execute the following command, if you don't have any pre-existing Azure services and want to start from a fresh deployment.
7779

78-
1. Run `azd up` - This will provision Azure resources and deploy this sample to those resources, including building the search index based on the files found in the `./data` folder.
80+
1. Run `azd up` - This will provision Azure resources and deploy this sample to those resources, including building the search index based on the files found under `pictures/`.
7981
* **Important**: Beware that the resources created by this command will incur immediate costs, primarily from the AI Search resource. These resources may accrue costs even if you interrupt the command before it is fully executed. You can run `azd down` or delete the resources manually to avoid unnecessary spending.
8082
1. After the application has been successfully deployed you will see a URL printed to the console. Click that URL to interact with the application in your browser.
8183

84+
## How the indexer works
85+
86+
When `setup_search_service.py` runs (automatically during `azd up`, or manually), it sets up an Azure AI Search indexing pipeline:
87+
88+
1. **Blob upload** – Images from the `pictures/` folder are uploaded to an Azure Blob Storage container. Only files placed directly under `pictures/` are uploaded.
89+
2. **Data source** – A Search data source is configured to point at that blob container.
90+
3. **Skillset** – Two skills are applied to each image:
91+
- [`VisionVectorizeSkill`](https://learn.microsoft.com/azure/search/cognitive-search-skill-vision-vectorize) generates a 1024-dimensional multimodal embedding using Azure AI Vision.
92+
- [`ChatCompletionSkill`](https://learn.microsoft.com/azure/search/chat-completion-skill-example-usage) (optional) produces a natural-language description of each image, stored as `verbalized_image`.
93+
4. **Index** – Results are written to a Search index with an `embedding` vector field backed by an HNSW algorithm, plus a built-in AI Vision vectorizer so that query text is embedded at search time without extra code.
94+
5. **Indexer** – Runs the pipeline: blob → normalized images → skills → index projections. Each image within a blob becomes its own indexed document.
95+
96+
At query time, the search service vectorizes the text query using the same AI Vision model and performs an approximate nearest-neighbor search over the stored embeddings.
97+
8298
## Running locally
8399

84100
You can only run locally **after** having successfully run the `azd up` command. If you haven't yet, follow the steps in [Azure deployment](#azure-deployment) above.
85101

102+
### Web app
103+
86104
1. Run `azd auth login`
87105
2. Change dir to `app` and run `./start.ps1` or `./start.sh` depending on your OS.
88106
3. Open a browser and navigate to `http://localhost:50505`
89107

108+
### MCP server
109+
110+
1. Run `azd auth login`
111+
2. Run `python app/backend/mcp_server.py`
112+
3. The server starts on `http://localhost:8001`. Add it to your MCP client configuration (e.g. VS Code or Claude Desktop) pointing at that URL.
113+
114+
To connect to a deployed MCP server instead, update `.vscode/mcp.json` with your deployed Container App hostname and append `/mcp` (for example, using the `SERVICE_MCP_URI` output from `azd up`).
115+
116+
## Adding new images
117+
118+
To add new images to the search index after the initial deployment:
119+
120+
1. Place new JPEG image files directly under the `pictures/` folder. Convert to JPEG first if needed.
121+
2. Run the setup script inside the Python environment to upload and re-index:
122+
123+
```bash
124+
python ./app/backend/setup_search_service.py
125+
```
126+
127+
The script skips blobs that already exist, so only new files will be uploaded. The indexer will then vectorize and index the new images.
128+
90129
## Clean up
91130

92131
To clean up all the resources created by this sample:

app/backend/Dockerfile.mcp

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# ------------------- Stage 0: Base Stage ------------------------------
2+
FROM python:3.11-alpine AS base
3+
4+
WORKDIR /code
5+
6+
# Install tini, a tiny init for containers
7+
RUN apk add --update --no-cache tini
8+
9+
# Install required packages for cryptography package
10+
# https://cryptography.io/en/latest/installation/#building-cryptography-on-linux
11+
RUN apk add gcc musl-dev python3-dev libffi-dev openssl-dev cargo pkgconfig jpeg-dev zlib-dev libwebp-dev
12+
13+
# ------------------- Stage 1: Build Stage ------------------------------
14+
FROM base AS build
15+
16+
COPY requirements.txt .
17+
18+
RUN pip3 install -r requirements.txt
19+
20+
COPY . .
21+
# ------------------- Stage 2: Final Stage ------------------------------
22+
FROM base AS final
23+
24+
RUN addgroup -S app && adduser -S app -G app
25+
26+
COPY --from=build --chown=app:app /usr/local/lib/python3.11 /usr/local/lib/python3.11
27+
COPY --from=build --chown=app:app /usr/local/bin /usr/local/bin
28+
COPY --from=build --chown=app:app /code /code
29+
30+
USER app
31+
32+
EXPOSE 8001
33+
34+
ENTRYPOINT ["tini", "python", "mcp_server.py"]

app/backend/app.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,9 @@ async def search():
5858
search_text=None,
5959
top=size,
6060
vector_queries=[
61-
VectorizableTextQuery(k=size, fields="embedding", text=search_text)
61+
VectorizableTextQuery(
62+
k_nearest_neighbors=size, fields="embedding", text=search_text
63+
)
6264
],
6365
select="metadata_storage_path",
6466
)

0 commit comments

Comments
 (0)