From 071875e40f64d6d11dde43e27e4e970c10769145 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 08:14:50 +0000 Subject: [PATCH 01/13] Build Creer v0.1: FastAPI backend + VS Code extension Scaffold AI repo generation end-to-end: planner/generator API with path validation, TypeScript extension command to write files into the workspace with optional git init, and a PLAN.md for v0.2 priorities. Co-authored-by: Sanath S Patil --- .gitignore | 9 + PLAN.md | 26 +++ README.md | 73 +++++++ backend/.env.example | 2 + backend/.gitignore | 6 + backend/app/__init__.py | 1 + backend/app/generator.py | 49 +++++ backend/app/planner.py | 58 +++++ backend/app/templates.py | 17 ++ backend/app/validator.py | 31 +++ backend/config.py | 7 + backend/main.py | 36 ++++ backend/requirements.txt | 5 + extension/.gitignore | 4 + extension/.vscode/launch.json | 13 ++ extension/.vscode/tasks.json | 11 + extension/package-lock.json | 392 ++++++++++++++++++++++++++++++++++ extension/package.json | 53 +++++ extension/src/extension.ts | 118 ++++++++++ extension/tsconfig.json | 16 ++ 20 files changed, 927 insertions(+) create mode 100644 .gitignore create mode 100644 PLAN.md create mode 100644 README.md create mode 100644 backend/.env.example create mode 100644 backend/.gitignore create mode 100644 backend/app/__init__.py create mode 100644 backend/app/generator.py create mode 100644 backend/app/planner.py create mode 100644 backend/app/templates.py create mode 100644 backend/app/validator.py create mode 100644 backend/config.py create mode 100644 backend/main.py create mode 100644 backend/requirements.txt create mode 100644 extension/.gitignore create mode 100644 extension/.vscode/launch.json create mode 100644 extension/.vscode/tasks.json create mode 100644 extension/package-lock.json create mode 100644 extension/package.json create mode 100644 extension/src/extension.ts create mode 100644 extension/tsconfig.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..25ec95c --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.DS_Store +*.log +.env +**/venv/ +**/__pycache__/ +**/node_modules/ +**/out/ +*.pyc +.pytest_cache/ diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..7561215 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,26 @@ +# Creer — Final Plan (post v0.1) + +v0.1 delivers the foundation: FastAPI planner/generator + VS Code command that writes a generated repo into the workspace (optional git init). + +## v0.2 priorities + +1. **Preview before writing** — show planned file tree / diffs and require confirm before disk writes. +2. **GitHub repo creation** — create remote repo and push the scaffolded project. +3. **Curated templates** — expand `backend/app/templates.py` into a real template system (stack presets + AI fill). +4. **Overwrite protection** — stronger conflict detection per-file (not only folder-level). +5. **Chat command `/creer`** — invoke scaffolding from chat / agent surface. + +## Stretch (v0.3+) + +- Production-grade validation layer (schema, path sandbox, content size limits) +- Streaming generation progress to the extension UI +- Local/offline model backends +- Open-source README / LICENSE / CI templates baked into every scaffold + +## Non-goals (keep out of early versions) + +- Multi-agent orchestration +- Memory graphs +- Overengineered plugin frameworks + +Stay power-focused: idea → plan → files → workspace. diff --git a/README.md b/README.md new file mode 100644 index 0000000..8629a9b --- /dev/null +++ b/README.md @@ -0,0 +1,73 @@ +# Creer + +AI-powered repo scaffolding inside your workspace. + +Describe an idea → Creer plans a clean structure → writes files into your VS Code workspace → optionally initializes git. + +## Architecture + +``` +creer/ +├── backend/ # Python FastAPI AI engine +└── extension/ # VS Code extension +``` + +## Prerequisites + +- Python 3.10+ +- Node.js 18+ +- OpenAI API key +- VS Code / Cursor + +## Backend (v0.1) + +```bash +cd backend +python -m venv venv +source venv/bin/activate # Windows: venv\Scripts\activate +pip install -r requirements.txt +cp .env.example .env +# set OPENAI_API_KEY in .env +uvicorn main:app --reload --port 8000 +``` + +Health check: + +```bash +curl http://localhost:8000/health +``` + +Generate: + +```bash +curl -X POST http://localhost:8000/generate \ + -H "Content-Type: application/json" \ + -d '{"idea":"Build a FastAPI todo app"}' +``` + +## Extension (v0.1) + +```bash +cd extension +npm install +npm run compile +``` + +In VS Code / Cursor: + +1. Open the `extension/` folder +2. Press **F5** (Run Extension) +3. In the Extension Development Host, open a workspace folder +4. Command Palette → **Creer: Create New Repo** +5. Enter an idea + +Settings: + +| Setting | Default | Description | +|---|---|---| +| `creer.backendUrl` | `http://localhost:8000` | Backend base URL | +| `creer.initGit` | `true` | Run `git init` + initial commit after scaffolding | + +## License + +MIT diff --git a/backend/.env.example b/backend/.env.example new file mode 100644 index 0000000..fb03904 --- /dev/null +++ b/backend/.env.example @@ -0,0 +1,2 @@ +OPENAI_API_KEY=sk-your-key-here +CREER_MODEL=gpt-4o-mini diff --git a/backend/.gitignore b/backend/.gitignore new file mode 100644 index 0000000..7d49b78 --- /dev/null +++ b/backend/.gitignore @@ -0,0 +1,6 @@ +.env +venv/ +__pycache__/ +*.pyc +.pytest_cache/ +.mypy_cache/ diff --git a/backend/app/__init__.py b/backend/app/__init__.py new file mode 100644 index 0000000..1c9c4d8 --- /dev/null +++ b/backend/app/__init__.py @@ -0,0 +1 @@ +"""Creer backend package.""" diff --git a/backend/app/generator.py b/backend/app/generator.py new file mode 100644 index 0000000..2d1823a --- /dev/null +++ b/backend/app/generator.py @@ -0,0 +1,49 @@ +from openai import OpenAI +from config import OPENAI_API_KEY, MODEL + +_client: OpenAI | None = None + + +def _get_client() -> OpenAI: + global _client + if _client is None: + if not OPENAI_API_KEY: + raise ValueError("OPENAI_API_KEY is not set") + _client = OpenAI(api_key=OPENAI_API_KEY) + return _client + + +def generate_files(plan: dict) -> dict[str, str]: + """Generate file contents one-by-one from a project plan.""" + files_output: dict[str, str] = {} + + for file_path in plan["files"]: + prompt = f""" +Generate the full content for file: {file_path} + +Project Stack: {plan["stack"]} +Project Name: {plan["project_name"]} + +Follow best practices. +Return ONLY the file content — no markdown fences, no explanation. +""" + + response = _get_client().chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=0.3, + ) + + content = response.choices[0].message.content or "" + # Strip accidental markdown fences + if content.startswith("```"): + lines = content.splitlines() + if lines and lines[0].startswith("```"): + lines = lines[1:] + if lines and lines[-1].strip() == "```": + lines = lines[:-1] + content = "\n".join(lines) + + files_output[file_path] = content + + return files_output diff --git a/backend/app/planner.py b/backend/app/planner.py new file mode 100644 index 0000000..e5bb6e3 --- /dev/null +++ b/backend/app/planner.py @@ -0,0 +1,58 @@ +from openai import OpenAI +import json +import re +from config import OPENAI_API_KEY, MODEL + +_client: OpenAI | None = None + + +def _get_client() -> OpenAI: + global _client + if _client is None: + if not OPENAI_API_KEY: + raise ValueError("OPENAI_API_KEY is not set") + _client = OpenAI(api_key=OPENAI_API_KEY) + return _client + + +def _parse_json(content: str) -> dict: + """Parse model JSON, tolerating optional markdown fences.""" + text = content.strip() + fence = re.search(r"```(?:json)?\s*([\s\S]*?)```", text) + if fence: + text = fence.group(1).strip() + return json.loads(text) + + +def plan_project(idea: str) -> dict: + prompt = f""" +You are a senior software architect. + +Convert the following idea into a clean project structure. + +Return ONLY valid JSON (no markdown): +{{ + "project_name": "...", + "stack": "...", + "files": ["path/file.py", ...] +}} + +Rules: +- project_name must be a valid folder name (lowercase, hyphens ok) +- files should be a focused, production-ready starter set (typically 5–15 files) +- include README.md and a dependency manifest appropriate for the stack + +Idea: {idea} +""" + + response = _get_client().chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=0.2, + response_format={"type": "json_object"}, + ) + + plan = _parse_json(response.choices[0].message.content) + if not isinstance(plan.get("files"), list) or not plan.get("project_name"): + raise ValueError("Planner returned an invalid plan structure") + return plan diff --git a/backend/app/templates.py b/backend/app/templates.py new file mode 100644 index 0000000..0f0fb5b --- /dev/null +++ b/backend/app/templates.py @@ -0,0 +1,17 @@ +"""Curated starter templates (stub for v0.2 template system).""" + +TEMPLATES: dict[str, dict] = { + "fastapi-minimal": { + "stack": "FastAPI + Uvicorn", + "files": [ + "main.py", + "requirements.txt", + "README.md", + ".env.example", + ], + }, +} + + +def get_template(name: str) -> dict | None: + return TEMPLATES.get(name) diff --git a/backend/app/validator.py b/backend/app/validator.py new file mode 100644 index 0000000..7226e74 --- /dev/null +++ b/backend/app/validator.py @@ -0,0 +1,31 @@ +"""Lightweight plan/file validation for v0.1.""" + +import re + +SAFE_NAME = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,63}$") +SAFE_PATH = re.compile(r"^(?!/)(?!.*(?:^|/)\.\.(?:/|$))[a-zA-Z0-9._/-]+$") + + +def validate_plan(plan: dict) -> None: + name = plan.get("project_name") + files = plan.get("files") + + if not isinstance(name, str) or not SAFE_NAME.match(name): + raise ValueError(f"Invalid project_name: {name!r}") + + if not isinstance(files, list) or not files: + raise ValueError("Plan must include a non-empty files list") + + for path in files: + if not isinstance(path, str) or not SAFE_PATH.match(path): + raise ValueError(f"Unsafe or invalid file path: {path!r}") + + +def validate_files(files: dict) -> None: + if not isinstance(files, dict) or not files: + raise ValueError("Generated files must be a non-empty mapping") + for path, content in files.items(): + if not isinstance(path, str) or not SAFE_PATH.match(path): + raise ValueError(f"Unsafe or invalid file path: {path!r}") + if not isinstance(content, str): + raise ValueError(f"File content for {path!r} must be a string") diff --git a/backend/config.py b/backend/config.py new file mode 100644 index 0000000..0bac399 --- /dev/null +++ b/backend/config.py @@ -0,0 +1,7 @@ +import os +from dotenv import load_dotenv + +load_dotenv() + +OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") +MODEL = os.getenv("CREER_MODEL", "gpt-4o-mini") diff --git a/backend/main.py b/backend/main.py new file mode 100644 index 0000000..ffd246c --- /dev/null +++ b/backend/main.py @@ -0,0 +1,36 @@ +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel, Field + +from app.planner import plan_project +from app.generator import generate_files +from app.validator import validate_plan, validate_files + +app = FastAPI(title="Creer", version="0.1.0") + + +class ProjectRequest(BaseModel): + idea: str = Field(..., min_length=3, max_length=4000) + + +@app.get("/health") +def health(): + return {"status": "ok", "version": "0.1.0"} + + +@app.post("/generate") +def generate_project(request: ProjectRequest): + try: + plan = plan_project(request.idea) + validate_plan(plan) + files = generate_files(plan) + validate_files(files) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException(status_code=500, detail=f"Generation failed: {exc}") from exc + + return { + "project_name": plan["project_name"], + "stack": plan.get("stack"), + "files": files, + } diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..67fc149 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,5 @@ +fastapi>=0.115.0 +uvicorn[standard]>=0.32.0 +openai>=1.55.0 +python-dotenv>=1.0.0 +pydantic>=2.9.0 diff --git a/extension/.gitignore b/extension/.gitignore new file mode 100644 index 0000000..e6b3087 --- /dev/null +++ b/extension/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +out/ +*.vsix +.vscode-test/ diff --git a/extension/.vscode/launch.json b/extension/.vscode/launch.json new file mode 100644 index 0000000..a142310 --- /dev/null +++ b/extension/.vscode/launch.json @@ -0,0 +1,13 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Run Extension", + "type": "extensionHost", + "request": "launch", + "args": ["--extensionDevelopmentPath=${workspaceFolder}"], + "outFiles": ["${workspaceFolder}/out/**/*.js"], + "preLaunchTask": "npm: compile" + } + ] +} diff --git a/extension/.vscode/tasks.json b/extension/.vscode/tasks.json new file mode 100644 index 0000000..c8e2ec3 --- /dev/null +++ b/extension/.vscode/tasks.json @@ -0,0 +1,11 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "type": "npm", + "script": "compile", + "group": "build", + "problemMatcher": ["$tsc"] + } + ] +} diff --git a/extension/package-lock.json b/extension/package-lock.json new file mode 100644 index 0000000..54caa3e --- /dev/null +++ b/extension/package-lock.json @@ -0,0 +1,392 @@ +{ + "name": "creer", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "creer", + "version": "0.1.0", + "dependencies": { + "axios": "^1.7.9" + }, + "devDependencies": { + "@types/node": "^20.17.0", + "@types/vscode": "^1.85.0", + "typescript": "^5.6.3" + }, + "engines": { + "vscode": "^1.85.0" + } + }, + "node_modules/@types/node": { + "version": "20.19.43", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz", + "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/vscode": { + "version": "1.125.0", + "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.125.0.tgz", + "integrity": "sha512-0icm/ZQAaism87P0ekHqi4/Ju9du+Tm0RUW+y7vqRsxY2cY0FNRX1nAnaW7nT6npPt2tfHiheZ55Zm9UhqonFA==", + "dev": true, + "license": "MIT" + }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.19.0.tgz", + "integrity": "sha512-ht/iuYZXEjFxLH/Hkezgd7m6JKlHHXEUSneaDz8uZe1Gj5QZtCnpyDsckvAiEnT89OEbCLmnte4R4sn7P0EKFw==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.16.0", + "form-data": "^4.0.6", + "https-proxy-agent": "^5.0.1", + "proxy-from-env": "^2.1.0" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/follow-redirects": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.4", + "mime-types": "^2.1.35" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/proxy-from-env": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", + "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/extension/package.json b/extension/package.json new file mode 100644 index 0000000..5f1db08 --- /dev/null +++ b/extension/package.json @@ -0,0 +1,53 @@ +{ + "name": "creer", + "displayName": "Creer", + "description": "AI-powered repo scaffolding inside your workspace", + "version": "0.1.0", + "publisher": "creer", + "engines": { + "vscode": "^1.85.0" + }, + "categories": [ + "Other", + "Snippets" + ], + "activationEvents": [], + "main": "./out/extension.js", + "contributes": { + "commands": [ + { + "command": "creer.createRepo", + "title": "Creer: Create New Repo" + } + ], + "configuration": { + "title": "Creer", + "properties": { + "creer.backendUrl": { + "type": "string", + "default": "http://localhost:8000", + "description": "Creer backend base URL" + }, + "creer.initGit": { + "type": "boolean", + "default": true, + "description": "Initialize a git repo after scaffolding" + } + } + } + }, + "scripts": { + "vscode:prepublish": "npm run compile", + "compile": "tsc -p ./", + "watch": "tsc -watch -p ./", + "lint": "tsc --noEmit -p ./" + }, + "dependencies": { + "axios": "^1.7.9" + }, + "devDependencies": { + "@types/node": "^20.17.0", + "@types/vscode": "^1.85.0", + "typescript": "^5.6.3" + } +} diff --git a/extension/src/extension.ts b/extension/src/extension.ts new file mode 100644 index 0000000..8184c79 --- /dev/null +++ b/extension/src/extension.ts @@ -0,0 +1,118 @@ +import * as vscode from 'vscode'; +import axios from 'axios'; +import * as fs from 'fs'; +import * as path from 'path'; +import { exec } from 'child_process'; +import { promisify } from 'util'; + +const execAsync = promisify(exec); + +interface GenerateResponse { + project_name: string; + stack?: string; + files: Record; +} + +async function initGit(projectPath: string): Promise { + await execAsync('git init', { cwd: projectPath }); + await execAsync('git add .', { cwd: projectPath }); + try { + await execAsync('git commit -m "Initial commit"', { cwd: projectPath }); + } catch { + // Commit can fail if git user.name/email are unset — still leave the repo initialized. + } +} + +export function activate(context: vscode.ExtensionContext) { + const disposable = vscode.commands.registerCommand('creer.createRepo', async () => { + const idea = await vscode.window.showInputBox({ + prompt: 'Describe the project you want to create', + placeHolder: 'Build a FastAPI todo app', + ignoreFocusOut: true, + }); + + if (!idea?.trim()) { + return; + } + + const workspaceFolders = vscode.workspace.workspaceFolders; + if (!workspaceFolders) { + vscode.window.showErrorMessage('Open a workspace folder first.'); + return; + } + + const config = vscode.workspace.getConfiguration('creer'); + const backendUrl = (config.get('backendUrl') || 'http://localhost:8000').replace(/\/$/, ''); + const shouldInitGit = config.get('initGit') ?? true; + + const rootPath = workspaceFolders[0].uri.fsPath; + + try { + const response = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'Creer: generating project…', + cancellable: false, + }, + async () => + axios.post( + `${backendUrl}/generate`, + { idea: idea.trim() }, + { timeout: 300_000 } + ) + ); + + const projectName = response.data.project_name; + const files = response.data.files; + + if (!projectName || !files || typeof files !== 'object') { + vscode.window.showErrorMessage('Creer backend returned an invalid response.'); + return; + } + + const projectPath = path.join(rootPath, projectName); + + if (fs.existsSync(projectPath)) { + const overwrite = await vscode.window.showWarningMessage( + `Folder "${projectName}" already exists. Overwrite files?`, + { modal: true }, + 'Overwrite' + ); + if (overwrite !== 'Overwrite') { + return; + } + } + + fs.mkdirSync(projectPath, { recursive: true }); + + for (const filePath of Object.keys(files)) { + const fullPath = path.join(projectPath, filePath); + fs.mkdirSync(path.dirname(fullPath), { recursive: true }); + fs.writeFileSync(fullPath, files[filePath], 'utf8'); + } + + if (shouldInitGit) { + try { + await initGit(projectPath); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + vscode.window.showWarningMessage(`Project created, but git init failed: ${message}`); + } + } + + vscode.window.showInformationMessage(`Project ${projectName} created.`); + } catch (err) { + if (axios.isAxiosError(err)) { + const detail = err.response?.data?.detail || err.message; + vscode.window.showErrorMessage(`Creer generation failed: ${detail}`); + return; + } + const message = err instanceof Error ? err.message : String(err); + vscode.window.showErrorMessage(`Creer failed: ${message}`); + } + }); + + context.subscriptions.push(disposable); +} + +export function deactivate() {} diff --git a/extension/tsconfig.json b/extension/tsconfig.json new file mode 100644 index 0000000..80604a6 --- /dev/null +++ b/extension/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "ES2022", + "lib": ["ES2022"], + "outDir": "out", + "rootDir": "src", + "sourceMap": true, + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src"], + "exclude": ["node_modules", "out"] +} From 0bc414db3143492cd7d8e9019e9c710f4305f856 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 08:23:00 +0000 Subject: [PATCH 02/13] Implement Creer v0.2: templates, preview, GitHub, overwrite, chat Backend: curated templates, /plan + /templates, stronger validation, optional GitHub repo create API. Extension: plan preview before write, per-file overwrite protection, GitHub push flow, /creer chat entry and @creer participant, with path-safe writes and shell-safe git exec. Co-authored-by: Sanath S Patil --- PLAN.md | 20 +-- README.md | 75 ++++++++- backend/app/github.py | 63 ++++++++ backend/app/planner.py | 75 ++++++++- backend/app/templates.py | 109 ++++++++++++- backend/app/validator.py | 89 +++++++++- backend/main.py | 120 +++++++++++++- backend/requirements.txt | 1 + extension/package.json | 35 +++- extension/src/api.ts | 116 +++++++++++++ extension/src/extension.ts | 143 +++++----------- extension/src/git.ts | 78 +++++++++ extension/src/preview.ts | 59 +++++++ extension/src/scaffold.ts | 314 ++++++++++++++++++++++++++++++++++++ extension/src/writeFiles.ts | 139 ++++++++++++++++ 15 files changed, 1297 insertions(+), 139 deletions(-) create mode 100644 backend/app/github.py create mode 100644 extension/src/api.ts create mode 100644 extension/src/git.ts create mode 100644 extension/src/preview.ts create mode 100644 extension/src/scaffold.ts create mode 100644 extension/src/writeFiles.ts diff --git a/PLAN.md b/PLAN.md index 7561215..69ee028 100644 --- a/PLAN.md +++ b/PLAN.md @@ -1,21 +1,23 @@ # Creer — Final Plan (post v0.1) -v0.1 delivers the foundation: FastAPI planner/generator + VS Code command that writes a generated repo into the workspace (optional git init). +v0.1 delivered the foundation: FastAPI planner/generator + VS Code command that writes a generated repo into the workspace (optional git init). -## v0.2 priorities +## v0.2 — done -1. **Preview before writing** — show planned file tree / diffs and require confirm before disk writes. -2. **GitHub repo creation** — create remote repo and push the scaffolded project. -3. **Curated templates** — expand `backend/app/templates.py` into a real template system (stack presets + AI fill). -4. **Overwrite protection** — stronger conflict detection per-file (not only folder-level). -5. **Chat command `/creer`** — invoke scaffolding from chat / agent surface. +1. **Preview before writing** — plan preview markdown + confirm before generate/write (`creer.previewBeforeWrite`). +2. **GitHub repo creation** — `POST /github/create-repo` + extension remote add/push (`creer.createGitHubRepo`, token settings). +3. **Curated templates** — `GET /templates` + template-anchored `/plan` & `/generate` (`backend/app/templates.py`). +4. **Overwrite protection** — per-file conflict detection with overwrite / skip / cancel. +5. **Chat command `/creer`** — `creer.createRepoFromChat` + `@creer` chat participant (feature-detected). -## Stretch (v0.3+) +Also in v0.2: modular extension layout (`api` / `scaffold` / `git` / `writeFiles` / `preview`), backend path/content validation, lazy OpenAI clients, extension path sandbox on write. + +## v0.3 priorities (next) -- Production-grade validation layer (schema, path sandbox, content size limits) - Streaming generation progress to the extension UI - Local/offline model backends - Open-source README / LICENSE / CI templates baked into every scaffold +- Hardening: avoid putting GitHub tokens on `git push` argv (credential helper / askpass); SecretStorage instead of plaintext `creer.githubToken` setting ## Non-goals (keep out of early versions) diff --git a/README.md b/README.md index 8629a9b..e45dd37 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ AI-powered repo scaffolding inside your workspace. -Describe an idea → Creer plans a clean structure → writes files into your VS Code workspace → optionally initializes git. +Describe an idea → Creer plans a clean structure → optionally preview/confirm → writes files into your VS Code workspace → optionally initializes git and creates a GitHub remote. + +**Current version: 0.2.0** ## Architecture @@ -16,10 +18,10 @@ creer/ - Python 3.10+ - Node.js 18+ -- OpenAI API key +- OpenAI API key (required for AI planning/generation; template-only planning can run without it) - VS Code / Cursor -## Backend (v0.1) +## Backend (v0.2) ```bash cd backend @@ -31,12 +33,58 @@ cp .env.example .env uvicorn main:app --reload --port 8000 ``` +### Endpoints + +| Method | Path | Description | +|---|---|---| +| `GET` | `/health` | Liveness + version (`0.2.0`) | +| `GET` | `/templates` | List curated starter templates | +| `POST` | `/plan` | Return a project plan (no file contents) | +| `POST` | `/generate` | Plan (or accept a plan) and generate file contents | +| `POST` | `/github/create-repo` | Create a GitHub repo for the authenticated user | + +#### `POST /plan` + +```json +{ "idea": "Build a FastAPI todo app", "template_id": "fastapi-minimal" } +``` + +`template_id` is optional. Response includes `project_name`, `stack`, `files`, and optionally `template_id` / `description`. + +#### `POST /generate` + +```json +{ + "idea": "Build a FastAPI todo app", + "template_id": "fastapi-minimal", + "plan": null +} +``` + +Omit `plan` to plan then generate, or pass a prior `/plan` body to skip re-planning. Response includes `project_name`, `stack`, and `files` (path → content map). + +#### `POST /github/create-repo` + +Prefer `Authorization: Bearer `; `body.token` is also accepted. + +```json +{ "name": "my-app", "private": true, "description": "optional" } +``` + +Returns `html_url`, `clone_url`, `full_name`. + Health check: ```bash curl http://localhost:8000/health ``` +List templates: + +```bash +curl http://localhost:8000/templates +``` + Generate: ```bash @@ -45,7 +93,7 @@ curl -X POST http://localhost:8000/generate \ -d '{"idea":"Build a FastAPI todo app"}' ``` -## Extension (v0.1) +## Extension (v0.2) ```bash cd extension @@ -58,15 +106,28 @@ In VS Code / Cursor: 1. Open the `extension/` folder 2. Press **F5** (Run Extension) 3. In the Extension Development Host, open a workspace folder -4. Command Palette → **Creer: Create New Repo** -5. Enter an idea +4. Command Palette → **Creer: Create New Repo** (or **Creer: Create from Chat Prompt**) +5. Enter an idea, pick a template or AI plan, confirm the preview, and write files + +### Commands + +| Command | Title | +|---|---| +| `creer.createRepo` | Creer: Create New Repo | +| `creer.createRepoFromChat` | Creer: Create from Chat Prompt | + +Chat: `@creer ` (when the host supports chat participants) or run **Create from Chat Prompt** with a `/creer …` style input. -Settings: +### Settings | Setting | Default | Description | |---|---|---| | `creer.backendUrl` | `http://localhost:8000` | Backend base URL | | `creer.initGit` | `true` | Run `git init` + initial commit after scaffolding | +| `creer.previewBeforeWrite` | `true` | Show plan preview and confirm before generating/writing | +| `creer.createGitHubRepo` | `false` | After scaffolding, create a GitHub remote repository | +| `creer.githubPrivate` | `true` | Create GitHub repositories as private | +| `creer.githubToken` | `""` | GitHub PAT for create/push (prompted if empty when needed) | ## License diff --git a/backend/app/github.py b/backend/app/github.py new file mode 100644 index 0000000..0dee3b2 --- /dev/null +++ b/backend/app/github.py @@ -0,0 +1,63 @@ +"""Optional GitHub REST helper for creating repositories.""" + +from __future__ import annotations + +import httpx + + +def create_github_repo( + token: str, + name: str, + private: bool = True, + description: str = "", +) -> dict: + """ + Create a GitHub repository for the authenticated user. + + Returns dict with html_url, clone_url, and full_name. + Raises ValueError on auth/API failures. + """ + if not token or not isinstance(token, str): + raise ValueError("GitHub token is required") + if not name or not isinstance(name, str): + raise ValueError("Repository name is required") + + headers = { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "Creer/0.2", + } + payload = { + "name": name, + "private": private, + "description": description or "", + "auto_init": False, + } + + try: + with httpx.Client(timeout=30.0) as client: + resp = client.post( + "https://api.github.com/user/repos", + headers=headers, + json=payload, + ) + except httpx.HTTPError as exc: + raise ValueError(f"GitHub request failed: {exc}") from exc + + if resp.status_code == 401: + raise ValueError("GitHub authentication failed — check your token") + if resp.status_code == 403: + raise ValueError("GitHub forbidden — token may lack repo scope") + if resp.status_code == 422: + detail = resp.json().get("message", resp.text) if resp.headers.get("content-type", "").startswith("application/json") else resp.text + raise ValueError(f"GitHub rejected repo creation: {detail}") + if resp.status_code >= 400: + raise ValueError(f"GitHub API error ({resp.status_code}): {resp.text[:300]}") + + data = resp.json() + return { + "html_url": data.get("html_url", ""), + "clone_url": data.get("clone_url", ""), + "full_name": data.get("full_name", ""), + } diff --git a/backend/app/planner.py b/backend/app/planner.py index e5bb6e3..6cd8a62 100644 --- a/backend/app/planner.py +++ b/backend/app/planner.py @@ -1,7 +1,14 @@ -from openai import OpenAI +"""Project planner — AI-driven or template-based.""" + +from __future__ import annotations + import json import re -from config import OPENAI_API_KEY, MODEL + +from openai import OpenAI + +from config import MODEL, OPENAI_API_KEY +from app.templates import apply_template, get_template, slugify _client: OpenAI | None = None @@ -24,7 +31,7 @@ def _parse_json(content: str) -> dict: return json.loads(text) -def plan_project(idea: str) -> dict: +def _ai_plan(idea: str) -> dict: prompt = f""" You are a senior software architect. @@ -38,8 +45,8 @@ def plan_project(idea: str) -> dict: }} Rules: -- project_name must be a valid folder name (lowercase, hyphens ok) -- files should be a focused, production-ready starter set (typically 5–15 files) +- project_name must be a valid folder name (lowercase, hyphens ok, no spaces) +- files should be a focused, production-ready starter set (typically 5–15 files, max 40) - include README.md and a dependency manifest appropriate for the stack Idea: {idea} @@ -56,3 +63,61 @@ def plan_project(idea: str) -> dict: if not isinstance(plan.get("files"), list) or not plan.get("project_name"): raise ValueError("Planner returned an invalid plan structure") return plan + + +def _ai_name_project(idea: str, template: dict) -> str: + """Optionally ask the model for a short project name; fall back to slugify.""" + try: + prompt = f""" +Given this project idea and stack, return ONLY valid JSON: +{{"project_name": "short-kebab-case-name"}} + +Rules: +- lowercase, hyphens ok, no spaces +- 2–40 characters preferred +- must be a valid folder name + +Idea: {idea} +Stack: {template.get("stack", "")} +Template: {template.get("name", "")} +""" + response = _get_client().chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=0.2, + response_format={"type": "json_object"}, + ) + data = _parse_json(response.choices[0].message.content) + name = data.get("project_name") + if isinstance(name, str) and name.strip(): + return slugify(name) + except Exception: + pass + return slugify(idea) + + +def plan_project(idea: str, template_id: str | None = None) -> dict: + """ + Build a project plan from an idea, optionally anchored to a curated template. + + When template_id is set: + - files and stack come from the template + - project_name is derived deterministically via slugify (no API key required) + - if OPENAI_API_KEY is present, AI may refine project_name only + + Without template_id: full AI planning (requires OPENAI_API_KEY). + """ + if template_id: + tmpl = get_template(template_id) + if tmpl is None: + raise ValueError(f"Unknown template_id: {template_id!r}") + + plan = apply_template(template_id, idea) + + # Optionally refine name with AI when a key is available + if OPENAI_API_KEY: + plan["project_name"] = _ai_name_project(idea, tmpl) + + return plan + + return _ai_plan(idea) diff --git a/backend/app/templates.py b/backend/app/templates.py index 0f0fb5b..0475b3c 100644 --- a/backend/app/templates.py +++ b/backend/app/templates.py @@ -1,17 +1,120 @@ -"""Curated starter templates (stub for v0.2 template system).""" +"""Curated starter templates for Creer v0.2.""" + +from __future__ import annotations + +import re TEMPLATES: dict[str, dict] = { "fastapi-minimal": { + "id": "fastapi-minimal", + "name": "FastAPI Minimal", + "description": "Minimal FastAPI API with uvicorn, health check, and env config.", "stack": "FastAPI + Uvicorn", "files": [ "main.py", "requirements.txt", "README.md", ".env.example", + ".gitignore", + ], + }, + "express-api": { + "id": "express-api", + "name": "Express API", + "description": "Node.js Express REST API with a basic router and scripts.", + "stack": "Node.js + Express", + "files": [ + "package.json", + "src/index.js", + "src/routes/health.js", + "README.md", + ".gitignore", + ".env.example", + ], + }, + "nextjs-app": { + "id": "nextjs-app", + "name": "Next.js App", + "description": "Next.js App Router starter with a home page and package manifest.", + "stack": "Next.js + React", + "files": [ + "package.json", + "next.config.js", + "tsconfig.json", + "app/layout.tsx", + "app/page.tsx", + "app/globals.css", + "README.md", + ".gitignore", + ], + }, + "python-cli": { + "id": "python-cli", + "name": "Python CLI", + "description": "Python command-line tool with argparse entrypoint and packaging basics.", + "stack": "Python CLI", + "files": [ + "pyproject.toml", + "src/__init__.py", + "src/cli.py", + "src/__main__.py", + "README.md", + ".gitignore", + ], + }, + "static-site": { + "id": "static-site", + "name": "Static Site", + "description": "Simple static HTML/CSS/JS site ready to open in a browser.", + "stack": "HTML + CSS + JavaScript", + "files": [ + "index.html", + "styles.css", + "script.js", + "README.md", ], }, } -def get_template(name: str) -> dict | None: - return TEMPLATES.get(name) +def slugify(text: str) -> str: + """Derive a filesystem-safe project name from free text.""" + slug = text.strip().lower() + slug = re.sub(r"[^a-z0-9]+", "-", slug) + slug = slug.strip("-") + slug = re.sub(r"-{2,}", "-", slug) + if not slug: + slug = "project" + return slug[:64] + + +def list_templates() -> list[dict]: + """Return all curated templates as a list of dicts.""" + return [dict(t) for t in TEMPLATES.values()] + + +def get_template(template_id: str) -> dict | None: + """Look up a template by id.""" + tmpl = TEMPLATES.get(template_id) + return dict(tmpl) if tmpl else None + + +def apply_template(template_id: str, idea: str) -> dict: + """ + Build a plan from a curated template. + + Uses the template's files and stack. Derives project_name by slugifying + the idea (deterministic — no API key required). + """ + tmpl = get_template(template_id) + if tmpl is None: + raise ValueError(f"Unknown template_id: {template_id!r}") + + project_name = slugify(idea) + return { + "project_name": project_name, + "stack": tmpl["stack"], + "files": list(tmpl["files"]), + "template_id": tmpl["id"], + "description": tmpl.get("description", ""), + } diff --git a/backend/app/validator.py b/backend/app/validator.py index 7226e74..d6978e8 100644 --- a/backend/app/validator.py +++ b/backend/app/validator.py @@ -1,31 +1,108 @@ -"""Lightweight plan/file validation for v0.1.""" +"""Production-grade plan/file validation for Creer v0.2.""" + +from __future__ import annotations import re +# project_name: lowercase start, alphanumerics/hyphen/underscore/dot, no spaces SAFE_NAME = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,63}$") + +# Relative paths only: no leading slash, no .. segments, safe chars SAFE_PATH = re.compile(r"^(?!/)(?!.*(?:^|/)\.\.(?:/|$))[a-zA-Z0-9._/-]+$") +# Windows drive / UNC-ish prefixes +WINDOWS_DRIVE = re.compile(r"^[a-zA-Z]:") + +MAX_FILES = 40 +MAX_CONTENT_BYTES_PER_FILE = 200_000 +MAX_TOTAL_CONTENT_BYTES = 2_000_000 + + +def _reject_unsafe_path(path: str, *, label: str = "file path") -> None: + if not isinstance(path, str): + raise ValueError(f"Invalid {label}: must be a string, got {type(path).__name__}") + + if "\x00" in path: + raise ValueError(f"Unsafe {label}: contains null byte: {path!r}") + + if path.startswith("/") or path.startswith("\\"): + raise ValueError(f"Unsafe {label}: absolute paths are not allowed: {path!r}") + + if WINDOWS_DRIVE.match(path) or path.startswith("\\\\"): + raise ValueError(f"Unsafe {label}: Windows drive/UNC paths are not allowed: {path!r}") + + # Normalize separators for .. checks + normalized = path.replace("\\", "/") + parts = normalized.split("/") + if ".." in parts or any(p == ".." for p in parts): + raise ValueError(f"Unsafe {label}: path traversal ('..') is not allowed: {path!r}") + + if not SAFE_PATH.match(path): + raise ValueError(f"Unsafe or invalid {label}: {path!r}") + def validate_plan(plan: dict) -> None: + if not isinstance(plan, dict): + raise ValueError("Plan must be a dictionary") + name = plan.get("project_name") files = plan.get("files") - if not isinstance(name, str) or not SAFE_NAME.match(name): + if not isinstance(name, str) or not name: raise ValueError(f"Invalid project_name: {name!r}") + if " " in name: + raise ValueError(f"Invalid project_name: spaces are not allowed: {name!r}") + + if "\x00" in name: + raise ValueError(f"Invalid project_name: contains null byte: {name!r}") + + if not SAFE_NAME.match(name): + raise ValueError( + f"Invalid project_name: must be 1–64 chars, start with alphanumeric, " + f"and contain only [a-zA-Z0-9._-]: {name!r}" + ) + if not isinstance(files, list) or not files: raise ValueError("Plan must include a non-empty files list") + if len(files) > MAX_FILES: + raise ValueError(f"Plan exceeds max file count ({MAX_FILES}): got {len(files)}") + + seen: set[str] = set() for path in files: - if not isinstance(path, str) or not SAFE_PATH.match(path): - raise ValueError(f"Unsafe or invalid file path: {path!r}") + _reject_unsafe_path(path) + if path in seen: + raise ValueError(f"Duplicate file path in plan: {path!r}") + seen.add(path) def validate_files(files: dict) -> None: if not isinstance(files, dict) or not files: raise ValueError("Generated files must be a non-empty mapping") + + if len(files) > MAX_FILES: + raise ValueError(f"Generated files exceed max file count ({MAX_FILES}): got {len(files)}") + + total_bytes = 0 for path, content in files.items(): - if not isinstance(path, str) or not SAFE_PATH.match(path): - raise ValueError(f"Unsafe or invalid file path: {path!r}") + _reject_unsafe_path(path) + if not isinstance(content, str): raise ValueError(f"File content for {path!r} must be a string") + + if "\x00" in content: + raise ValueError(f"File content for {path!r} contains null bytes") + + size = len(content.encode("utf-8")) + if size > MAX_CONTENT_BYTES_PER_FILE: + raise ValueError( + f"File {path!r} exceeds max content size " + f"({MAX_CONTENT_BYTES_PER_FILE} bytes): {size} bytes" + ) + total_bytes += size + + if total_bytes > MAX_TOTAL_CONTENT_BYTES: + raise ValueError( + f"Total content exceeds max ({MAX_TOTAL_CONTENT_BYTES} bytes): {total_bytes} bytes" + ) diff --git a/backend/main.py b/backend/main.py index ffd246c..379c3fb 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,26 +1,93 @@ -from fastapi import FastAPI, HTTPException +from fastapi import FastAPI, Header, HTTPException from pydantic import BaseModel, Field from app.planner import plan_project from app.generator import generate_files from app.validator import validate_plan, validate_files +from app.templates import list_templates, get_template +from app.github import create_github_repo -app = FastAPI(title="Creer", version="0.1.0") +VERSION = "0.2.0" +app = FastAPI(title="Creer", version=VERSION) -class ProjectRequest(BaseModel): + +class PlanRequest(BaseModel): + idea: str = Field(..., min_length=3, max_length=4000) + template_id: str | None = None + + +class PlanBody(BaseModel): + project_name: str + stack: str | None = None + files: list[str] + template_id: str | None = None + description: str | None = None + + +class GenerateRequest(BaseModel): idea: str = Field(..., min_length=3, max_length=4000) + template_id: str | None = None + plan: PlanBody | None = None + + +class GitHubCreateRepoRequest(BaseModel): + name: str = Field(..., min_length=1, max_length=100) + private: bool = True + description: str = "" + token: str | None = None @app.get("/health") def health(): - return {"status": "ok", "version": "0.1.0"} + return {"status": "ok", "version": VERSION} + + +@app.get("/templates") +def templates(): + return {"templates": list_templates()} + + +@app.post("/plan") +def plan_only(request: PlanRequest): + """Return a project plan without generating file contents.""" + try: + if request.template_id and get_template(request.template_id) is None: + raise ValueError(f"Unknown template_id: {request.template_id!r}") + plan = plan_project(request.idea, template_id=request.template_id) + validate_plan(plan) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException(status_code=500, detail=f"Planning failed: {exc}") from exc + + result = { + "project_name": plan["project_name"], + "stack": plan.get("stack"), + "files": plan["files"], + } + if plan.get("template_id"): + result["template_id"] = plan["template_id"] + if plan.get("description"): + result["description"] = plan["description"] + return result @app.post("/generate") -def generate_project(request: ProjectRequest): +def generate_project(request: GenerateRequest): try: - plan = plan_project(request.idea) + if request.plan is not None: + plan = request.plan.model_dump() + # Drop nulls except keep stack as empty string for the generator prompt + plan = {k: v for k, v in plan.items() if v is not None} + plan.setdefault("stack", "") + if request.template_id and "template_id" not in plan: + plan["template_id"] = request.template_id + else: + if request.template_id and get_template(request.template_id) is None: + raise ValueError(f"Unknown template_id: {request.template_id!r}") + plan = plan_project(request.idea, template_id=request.template_id) + validate_plan(plan) files = generate_files(plan) validate_files(files) @@ -29,8 +96,47 @@ def generate_project(request: ProjectRequest): except Exception as exc: raise HTTPException(status_code=500, detail=f"Generation failed: {exc}") from exc - return { + result = { "project_name": plan["project_name"], "stack": plan.get("stack"), "files": files, } + if plan.get("template_id"): + result["template_id"] = plan["template_id"] + return result + + +@app.post("/github/create-repo") +def github_create_repo( + request: GitHubCreateRepoRequest, + authorization: str | None = Header(default=None), +): + """Create a GitHub repo. Prefer Authorization: Bearer ; body.token also accepted.""" + token = None + if authorization: + parts = authorization.split(" ", 1) + if len(parts) == 2 and parts[0].lower() == "bearer": + token = parts[1].strip() + else: + token = authorization.strip() + if not token: + token = request.token + if not token: + raise HTTPException( + status_code=401, + detail="GitHub token required via Authorization: Bearer or body.token", + ) + + try: + result = create_github_repo( + token=token, + name=request.name, + private=request.private, + description=request.description, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException(status_code=500, detail=f"GitHub create failed: {exc}") from exc + + return result diff --git a/backend/requirements.txt b/backend/requirements.txt index 67fc149..f5c2db6 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -3,3 +3,4 @@ uvicorn[standard]>=0.32.0 openai>=1.55.0 python-dotenv>=1.0.0 pydantic>=2.9.0 +httpx>=0.27.0 diff --git a/extension/package.json b/extension/package.json index 5f1db08..a69c1b7 100644 --- a/extension/package.json +++ b/extension/package.json @@ -2,7 +2,7 @@ "name": "creer", "displayName": "Creer", "description": "AI-powered repo scaffolding inside your workspace", - "version": "0.1.0", + "version": "0.2.0", "publisher": "creer", "engines": { "vscode": "^1.85.0" @@ -18,6 +18,19 @@ { "command": "creer.createRepo", "title": "Creer: Create New Repo" + }, + { + "command": "creer.createRepoFromChat", + "title": "Creer: Create from Chat Prompt" + } + ], + "chatParticipants": [ + { + "id": "creer.participant", + "fullName": "Creer", + "name": "creer", + "description": "Scaffold a repo with Creer", + "isSticky": false } ], "configuration": { @@ -32,6 +45,26 @@ "type": "boolean", "default": true, "description": "Initialize a git repo after scaffolding" + }, + "creer.githubToken": { + "type": "string", + "default": "", + "description": "GitHub personal access token for creating/pushing repositories" + }, + "creer.githubPrivate": { + "type": "boolean", + "default": true, + "description": "Create GitHub repositories as private" + }, + "creer.createGitHubRepo": { + "type": "boolean", + "default": false, + "description": "After scaffolding, create a GitHub remote repository" + }, + "creer.previewBeforeWrite": { + "type": "boolean", + "default": true, + "description": "Show a plan preview and confirm before generating and writing files" } } } diff --git a/extension/src/api.ts b/extension/src/api.ts new file mode 100644 index 0000000..5df587c --- /dev/null +++ b/extension/src/api.ts @@ -0,0 +1,116 @@ +import axios, { AxiosError } from 'axios'; +import * as vscode from 'vscode'; + +export interface Template { + id: string; + name: string; + description: string; + stack: string; + files: string[]; +} + +export interface PlanResponse { + project_name: string; + stack?: string; + files: string[]; + template_id?: string; + description?: string; +} + +export interface GenerateResponse { + project_name: string; + stack?: string; + files: Record; + template_id?: string; +} + +export interface GitHubCreateRepoResponse { + html_url: string; + clone_url: string; + full_name: string; +} + +function getBackendUrl(): string { + const config = vscode.workspace.getConfiguration('creer'); + return (config.get('backendUrl') || 'http://localhost:8000').replace(/\/$/, ''); +} + +export function formatAxiosError(err: unknown, fallback: string): string { + if (axios.isAxiosError(err)) { + const ax = err as AxiosError<{ detail?: string }>; + const detail = ax.response?.data?.detail; + if (typeof detail === 'string' && detail.trim()) { + return detail; + } + if (ax.message) { + return ax.message; + } + } + if (err instanceof Error) { + return err.message; + } + return fallback; +} + +export async function fetchTemplates(): Promise { + const backendUrl = getBackendUrl(); + const response = await axios.get<{ templates: Template[] }>(`${backendUrl}/templates`, { + timeout: 30_000, + }); + return response.data.templates ?? []; +} + +export async function postPlan(idea: string, templateId?: string): Promise { + const backendUrl = getBackendUrl(); + const body: { idea: string; template_id?: string } = { idea }; + if (templateId) { + body.template_id = templateId; + } + const response = await axios.post(`${backendUrl}/plan`, body, { + timeout: 300_000, + }); + return response.data; +} + +export async function postGenerate( + idea: string, + options?: { templateId?: string; plan?: PlanResponse } +): Promise { + const backendUrl = getBackendUrl(); + const body: { + idea: string; + template_id?: string; + plan?: PlanResponse; + } = { idea }; + if (options?.templateId) { + body.template_id = options.templateId; + } + if (options?.plan) { + body.plan = options.plan; + } + const response = await axios.post(`${backendUrl}/generate`, body, { + timeout: 300_000, + }); + return response.data; +} + +export async function createGitHubRepo( + token: string, + name: string, + options?: { private?: boolean; description?: string } +): Promise { + const backendUrl = getBackendUrl(); + const isPrivate = options?.private ?? true; + const description = options?.description ?? ''; + const response = await axios.post( + `${backendUrl}/github/create-repo`, + { name, private: isPrivate, description }, + { + timeout: 60_000, + headers: { + Authorization: `Bearer ${token}`, + }, + } + ); + return response.data; +} diff --git a/extension/src/extension.ts b/extension/src/extension.ts index 8184c79..07addbd 100644 --- a/extension/src/extension.ts +++ b/extension/src/extension.ts @@ -1,118 +1,59 @@ import * as vscode from 'vscode'; -import axios from 'axios'; -import * as fs from 'fs'; -import * as path from 'path'; -import { exec } from 'child_process'; -import { promisify } from 'util'; - -const execAsync = promisify(exec); - -interface GenerateResponse { - project_name: string; - stack?: string; - files: Record; -} - -async function initGit(projectPath: string): Promise { - await execAsync('git init', { cwd: projectPath }); - await execAsync('git add .', { cwd: projectPath }); - try { - await execAsync('git commit -m "Initial commit"', { cwd: projectPath }); - } catch { - // Commit can fail if git user.name/email are unset — still leave the repo initialized. - } -} +import { runScaffoldFlow } from './scaffold'; export function activate(context: vscode.ExtensionContext) { - const disposable = vscode.commands.registerCommand('creer.createRepo', async () => { - const idea = await vscode.window.showInputBox({ - prompt: 'Describe the project you want to create', - placeHolder: 'Build a FastAPI todo app', - ignoreFocusOut: true, - }); - - if (!idea?.trim()) { - return; - } + const createRepo = vscode.commands.registerCommand('creer.createRepo', async () => { + await runScaffoldFlow({ fromChat: false }); + }); - const workspaceFolders = vscode.workspace.workspaceFolders; - if (!workspaceFolders) { - vscode.window.showErrorMessage('Open a workspace folder first.'); - return; + const createRepoFromChat = vscode.commands.registerCommand( + 'creer.createRepoFromChat', + async (idea?: string) => { + const initial = typeof idea === 'string' ? idea : undefined; + await runScaffoldFlow({ idea: initial, fromChat: true }); } + ); - const config = vscode.workspace.getConfiguration('creer'); - const backendUrl = (config.get('backendUrl') || 'http://localhost:8000').replace(/\/$/, ''); - const shouldInitGit = config.get('initGit') ?? true; - - const rootPath = workspaceFolders[0].uri.fsPath; - - try { - const response = await vscode.window.withProgress( - { - location: vscode.ProgressLocation.Notification, - title: 'Creer: generating project…', - cancellable: false, - }, - async () => - axios.post( - `${backendUrl}/generate`, - { idea: idea.trim() }, - { timeout: 300_000 } - ) - ); - - const projectName = response.data.project_name; - const files = response.data.files; + context.subscriptions.push(createRepo, createRepoFromChat); + registerChatParticipant(context); +} - if (!projectName || !files || typeof files !== 'object') { - vscode.window.showErrorMessage('Creer backend returned an invalid response.'); - return; - } +function registerChatParticipant(context: vscode.ExtensionContext): void { + // Runtime feature-detect: chat API may be missing on older VS Code hosts. + const create = + typeof vscode.chat?.createChatParticipant === 'function' + ? vscode.chat.createChatParticipant.bind(vscode.chat) + : undefined; - const projectPath = path.join(rootPath, projectName); + if (!create) { + return; + } - if (fs.existsSync(projectPath)) { - const overwrite = await vscode.window.showWarningMessage( - `Folder "${projectName}" already exists. Overwrite files?`, - { modal: true }, - 'Overwrite' - ); - if (overwrite !== 'Overwrite') { + try { + const participant = create( + 'creer.participant', + async (request, _context, stream, _token) => { + const idea = (request.prompt || '').trim(); + if (!idea) { + stream.markdown( + 'Provide an idea after `@creer`, for example: `@creer Build a FastAPI todo app`.\n\n' + + 'You can also run **Creer: Create from Chat Prompt** and type `/creer …`.' + ); return; } - } - fs.mkdirSync(projectPath, { recursive: true }); - - for (const filePath of Object.keys(files)) { - const fullPath = path.join(projectPath, filePath); - fs.mkdirSync(path.dirname(fullPath), { recursive: true }); - fs.writeFileSync(fullPath, files[filePath], 'utf8'); - } - - if (shouldInitGit) { - try { - await initGit(projectPath); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - vscode.window.showWarningMessage(`Project created, but git init failed: ${message}`); - } + stream.markdown( + `Scaffolding with Creer: **${idea}**…\n\n` + + 'Follow the prompts to pick a template, preview the plan, and confirm.' + ); + await runScaffoldFlow({ idea, fromChat: true }); } + ); - vscode.window.showInformationMessage(`Project ${projectName} created.`); - } catch (err) { - if (axios.isAxiosError(err)) { - const detail = err.response?.data?.detail || err.message; - vscode.window.showErrorMessage(`Creer generation failed: ${detail}`); - return; - } - const message = err instanceof Error ? err.message : String(err); - vscode.window.showErrorMessage(`Creer failed: ${message}`); - } - }); - - context.subscriptions.push(disposable); + context.subscriptions.push(participant); + } catch { + // Hosts without chat support — ignore registration failure. + } } export function deactivate() {} diff --git a/extension/src/git.ts b/extension/src/git.ts new file mode 100644 index 0000000..a111c69 --- /dev/null +++ b/extension/src/git.ts @@ -0,0 +1,78 @@ +import { execFile } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import { promisify } from 'util'; + +const execFileAsync = promisify(execFile); + +async function git(cwd: string, args: string[]): Promise { + await execFileAsync('git', args, { cwd }); +} + +export async function isGitRepo(projectPath: string): Promise { + return fs.existsSync(path.join(projectPath, '.git')); +} + +export async function initGit(projectPath: string): Promise { + await git(projectPath, ['init']); + await git(projectPath, ['add', '.']); + try { + await git(projectPath, ['commit', '-m', 'Initial commit']); + } catch { + // Commit can fail if git user.name/email are unset — still leave the repo initialized. + } +} + +export async function ensureGitRepo(projectPath: string): Promise { + if (!(await isGitRepo(projectPath))) { + await initGit(projectPath); + } +} + +export async function addRemoteAndPush( + projectPath: string, + cloneUrl: string, + token?: string +): Promise { + // Prefer authenticated HTTPS URL when a token is available so push works non-interactively. + // Args are passed via execFile (no shell) so tokens/URLs cannot inject commands. + let pushUrl = cloneUrl; + if (token && cloneUrl.startsWith('https://')) { + pushUrl = cloneUrl.replace( + 'https://', + `https://x-access-token:${encodeURIComponent(token)}@` + ); + } + + try { + await git(projectPath, ['remote', 'remove', 'origin']); + } catch { + // No existing origin — fine. + } + + await git(projectPath, ['remote', 'add', 'origin', cloneUrl]); + + // Ensure we have a branch name for -u push. + try { + await git(projectPath, ['rev-parse', '--verify', 'HEAD']); + } catch { + // No commits yet — stage and commit if possible. + await git(projectPath, ['add', '.']); + try { + await git(projectPath, ['commit', '-m', 'Initial commit']); + } catch { + throw new Error('Cannot push: repository has no commits (set git user.name/email).'); + } + } + + // Push using authenticated URL without rewriting the stored remote (keep clone_url clean). + try { + await git(projectPath, ['push', '-u', pushUrl, 'HEAD']); + } catch { + // Fallback: try main explicitly + await git(projectPath, ['push', '-u', pushUrl, 'HEAD:main']); + } + + // Keep origin pointing at the clean clone URL (without embedded token). + await git(projectPath, ['remote', 'set-url', 'origin', cloneUrl]); +} diff --git a/extension/src/preview.ts b/extension/src/preview.ts new file mode 100644 index 0000000..3449c80 --- /dev/null +++ b/extension/src/preview.ts @@ -0,0 +1,59 @@ +import * as vscode from 'vscode'; +import type { PlanResponse } from './api'; + +function buildFileTreeMarkdown(files: string[]): string { + const sorted = [...files].sort((a, b) => a.localeCompare(b)); + return sorted.map((f) => `- \`${f}\``).join('\n'); +} + +export function buildPlanPreviewMarkdown(plan: PlanResponse, idea: string): string { + const stack = plan.stack?.trim() || '(unspecified)'; + const templateLine = plan.template_id + ? `\n**Template:** \`${plan.template_id}\`\n` + : '\n**Template:** AI plan (no template)\n'; + const description = plan.description?.trim() + ? `\n**Description:** ${plan.description.trim()}\n` + : ''; + + return [ + `# Creer plan preview`, + '', + `**Idea:** ${idea}`, + '', + `**Project:** \`${plan.project_name}\``, + '', + `**Stack:** ${stack}`, + templateLine, + description, + `**Files (${plan.files.length}):**`, + '', + buildFileTreeMarkdown(plan.files), + '', + '---', + '', + '_Confirm generation to write these files into your workspace._', + '', + ].join('\n'); +} + +/** + * Open an untitled markdown preview document and ask the user to confirm generation. + * Returns true if the user confirms, false if they cancel. + */ +export async function showPlanPreviewAndConfirm(plan: PlanResponse, idea: string): Promise { + const markdown = buildPlanPreviewMarkdown(plan, idea); + const doc = await vscode.workspace.openTextDocument({ + content: markdown, + language: 'markdown', + }); + await vscode.window.showTextDocument(doc, { preview: true, preserveFocus: false }); + + const choice = await vscode.window.showInformationMessage( + `Creer plan ready: ${plan.project_name} (${plan.files.length} files). Generate & write files?`, + { modal: true }, + 'Generate & write files', + 'Cancel' + ); + + return choice === 'Generate & write files'; +} diff --git a/extension/src/scaffold.ts b/extension/src/scaffold.ts new file mode 100644 index 0000000..5e1eacf --- /dev/null +++ b/extension/src/scaffold.ts @@ -0,0 +1,314 @@ +import * as path from 'path'; +import * as vscode from 'vscode'; +import { + createGitHubRepo, + fetchTemplates, + formatAxiosError, + postGenerate, + postPlan, + type PlanResponse, + type Template, +} from './api'; +import { addRemoteAndPush, ensureGitRepo, initGit } from './git'; +import { showPlanPreviewAndConfirm } from './preview'; +import { + assertSafeProjectName, + findConflicts, + resolveConflicts, + writeProjectFiles, +} from './writeFiles'; + +export interface ScaffoldOptions { + /** Pre-filled idea (e.g. from chat). If omitted, prompts the user. */ + idea?: string; + /** Use chat-style InputBox placeholder (/creer …). */ + fromChat?: boolean; +} + +function stripCreerPrefix(raw: string): string { + return raw.replace(/^\s*\/creer\b\s*/i, '').trim(); +} + +async function promptForIdea(fromChat: boolean, initial?: string): Promise { + if (initial?.trim()) { + const stripped = stripCreerPrefix(initial); + if (stripped) { + return stripped; + } + } + + const value = await vscode.window.showInputBox({ + prompt: fromChat + ? 'Enter a /creer prompt describing the project to scaffold' + : 'Describe the project you want to create', + placeHolder: fromChat + ? '/creer Build a FastAPI todo' + : 'Build a FastAPI todo app', + value: initial?.trim() || undefined, + ignoreFocusOut: true, + }); + + if (!value?.trim()) { + return undefined; + } + return stripCreerPrefix(value); +} + +async function pickTemplate(templates: Template[]): Promise { + // null = cancelled; undefined = AI plan (no template); string = template id + const items: Array = [ + { + label: 'AI plan (no template)', + description: 'Let Creer choose the stack and file layout', + templateId: undefined, + }, + ...templates.map((t) => ({ + label: t.name, + description: t.stack, + detail: t.description, + templateId: t.id, + })), + ]; + + const picked = await vscode.window.showQuickPick(items, { + placeHolder: 'Select a template or AI plan', + ignoreFocusOut: true, + matchOnDescription: true, + matchOnDetail: true, + }); + + if (!picked) { + return null; + } + return picked.templateId; +} + +async function resolveGitHubToken(): Promise { + const config = vscode.workspace.getConfiguration('creer'); + const configured = (config.get('githubToken') || '').trim(); + if (configured) { + return configured; + } + + const token = await vscode.window.showInputBox({ + prompt: 'GitHub personal access token (repo scope)', + placeHolder: 'ghp_…', + password: true, + ignoreFocusOut: true, + }); + return token?.trim() || undefined; +} + +async function maybeCreateGitHubRemote( + projectPath: string, + projectName: string, + idea: string +): Promise { + const config = vscode.workspace.getConfiguration('creer'); + const autoCreate = config.get('createGitHubRepo') ?? false; + const isPrivate = config.get('githubPrivate') ?? true; + + let shouldCreate = autoCreate; + if (!shouldCreate) { + const answer = await vscode.window.showInformationMessage( + 'Create GitHub repository?', + 'Yes', + 'No' + ); + shouldCreate = answer === 'Yes'; + } + + if (!shouldCreate) { + return; + } + + const token = await resolveGitHubToken(); + if (!token) { + vscode.window.showWarningMessage('GitHub token required to create a repository. Skipped.'); + return; + } + + try { + const repo = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'Creer: creating GitHub repository…', + cancellable: false, + }, + async () => + createGitHubRepo(token, projectName, { + private: isPrivate, + description: idea.slice(0, 200), + }) + ); + + await ensureGitRepo(projectPath); + await addRemoteAndPush(projectPath, repo.clone_url, token); + + const open = await vscode.window.showInformationMessage( + `GitHub repo created: ${repo.html_url}`, + 'Open' + ); + if (open === 'Open') { + await vscode.env.openExternal(vscode.Uri.parse(repo.html_url)); + } + } catch (err) { + const message = formatAxiosError(err, 'GitHub create/push failed'); + vscode.window.showWarningMessage(`Project created, but GitHub step failed: ${message}`); + } +} + +/** + * Shared scaffold flow used by createRepo, createRepoFromChat, and the chat participant. + */ +export async function runScaffoldFlow(options: ScaffoldOptions = {}): Promise { + const fromChat = options.fromChat ?? false; + const idea = await promptForIdea(fromChat, options.idea); + if (!idea) { + return; + } + + const workspaceFolders = vscode.workspace.workspaceFolders; + if (!workspaceFolders) { + vscode.window.showErrorMessage('Open a workspace folder first.'); + return; + } + + const config = vscode.workspace.getConfiguration('creer'); + const shouldInitGit = config.get('initGit') ?? true; + const previewBeforeWrite = config.get('previewBeforeWrite') ?? true; + const rootPath = workspaceFolders[0].uri.fsPath; + + try { + // 1) Templates + let templates: Template[] = []; + try { + templates = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'Creer: loading templates…', + cancellable: false, + }, + () => fetchTemplates() + ); + } catch (err) { + const message = formatAxiosError(err, 'Failed to load templates'); + vscode.window.showWarningMessage( + `Could not load templates (${message}). Continuing with AI plan only.` + ); + } + + const templatePick = await pickTemplate(templates); + if (templatePick === null) { + return; + } + const templateId = templatePick; // string | undefined + + // 2) Plan + const plan: PlanResponse = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'Creer: planning project…', + cancellable: false, + }, + () => postPlan(idea, templateId) + ); + + if (!plan.project_name || !Array.isArray(plan.files)) { + vscode.window.showErrorMessage('Creer backend returned an invalid plan.'); + return; + } + + // Ensure template_id is on the plan when selected + if (templateId && !plan.template_id) { + plan.template_id = templateId; + } + + // 3) Preview / confirm + if (previewBeforeWrite) { + const confirmed = await showPlanPreviewAndConfirm(plan, idea); + if (!confirmed) { + return; + } + } + + // 4) Generate + const generated = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'Creer: generating project…', + cancellable: false, + }, + () => + postGenerate(idea, { + templateId: plan.template_id ?? templateId, + plan, + }) + ); + + const projectName = generated.project_name || plan.project_name; + const files = generated.files; + + if (!projectName || !files || typeof files !== 'object') { + vscode.window.showErrorMessage('Creer backend returned an invalid generate response.'); + return; + } + + try { + assertSafeProjectName(projectName); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + vscode.window.showErrorMessage(`Creer refused unsafe project name: ${message}`); + return; + } + + const projectPath = path.join(rootPath, projectName); + // Ensure the resolved project folder stays under the workspace root + const resolvedRoot = path.resolve(rootPath); + const resolvedProject = path.resolve(projectPath); + const rootPrefix = resolvedRoot.endsWith(path.sep) + ? resolvedRoot + : resolvedRoot + path.sep; + if (resolvedProject !== resolvedRoot && !resolvedProject.startsWith(rootPrefix)) { + vscode.window.showErrorMessage( + `Creer refused project path outside workspace: ${projectName}` + ); + return; + } + + // 5) Conflict resolution + const conflicts = findConflicts(projectPath, files); + const resolution = await resolveConflicts(conflicts); + if (resolution === 'cancel') { + return; + } + + // 6) Write + const { written, skipped } = writeProjectFiles(projectPath, files, resolution); + if (written === 0 && skipped === 0) { + vscode.window.showWarningMessage('No files were written.'); + return; + } + + // 7) Git init + if (shouldInitGit) { + try { + await initGit(projectPath); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + vscode.window.showWarningMessage(`Project created, but git init failed: ${message}`); + } + } + + // 8) GitHub remote (optional) + await maybeCreateGitHubRemote(projectPath, projectName, idea); + + const skipNote = skipped > 0 ? ` (${skipped} existing skipped)` : ''; + vscode.window.showInformationMessage( + `Project ${projectName} created at ${projectPath}${skipNote}.` + ); + } catch (err) { + const message = formatAxiosError(err, 'Creer failed'); + vscode.window.showErrorMessage(`Creer failed: ${message}`); + } +} diff --git a/extension/src/writeFiles.ts b/extension/src/writeFiles.ts new file mode 100644 index 0000000..af5f6c9 --- /dev/null +++ b/extension/src/writeFiles.ts @@ -0,0 +1,139 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import * as vscode from 'vscode'; + +export type ConflictResolution = 'overwrite' | 'skip' | 'cancel'; + +/** Safe project folder name: alphanumeric start, then [a-zA-Z0-9._-], max 64. */ +const SAFE_PROJECT_NAME = /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,63}$/; + +/** + * Validate project_name before joining under the workspace root. + * Rejects path separators and `..` so the folder cannot escape the workspace. + */ +export function assertSafeProjectName(name: string): void { + if (typeof name !== 'string' || !name) { + throw new Error(`Invalid project name: ${JSON.stringify(name)}`); + } + if (name.includes('\0') || name.includes('/') || name.includes('\\')) { + throw new Error(`Unsafe project name: ${JSON.stringify(name)}`); + } + // Reject `.` / `..` as the whole name (path-like); substring `foo..bar` is allowed by backend SAFE_NAME + if (name === '.' || name === '..') { + throw new Error(`Unsafe project name: ${JSON.stringify(name)}`); + } + if (!SAFE_PROJECT_NAME.test(name)) { + throw new Error( + `Invalid project name (use 1–64 chars, alphanumeric start, [a-zA-Z0-9._-]): ${JSON.stringify(name)}` + ); + } +} + +/** + * Resolve a relative path under projectPath and ensure it cannot escape the project root. + * Rejects absolute paths, null bytes, and `..` traversal (defense in depth vs backend). + */ +export function resolveSafeProjectPath(projectPath: string, relativePath: string): string { + if (typeof relativePath !== 'string' || !relativePath) { + throw new Error(`Invalid file path: ${JSON.stringify(relativePath)}`); + } + if (relativePath.includes('\0')) { + throw new Error(`Unsafe file path (null byte): ${JSON.stringify(relativePath)}`); + } + if (path.isAbsolute(relativePath)) { + throw new Error(`Absolute file paths are not allowed: ${JSON.stringify(relativePath)}`); + } + // Windows drive / UNC when running on win32; also reject drive-like prefixes on any OS + if (/^[a-zA-Z]:/.test(relativePath) || relativePath.startsWith('\\\\')) { + throw new Error(`Windows drive/UNC paths are not allowed: ${JSON.stringify(relativePath)}`); + } + + const normalizedSep = relativePath.replace(/\\/g, '/'); + const segments = normalizedSep.split('/'); + if (segments.some((s) => s === '..')) { + throw new Error(`Path traversal ('..') is not allowed: ${JSON.stringify(relativePath)}`); + } + if (segments.some((s) => s === '')) { + // Leading/trailing/double slashes → empty segment + throw new Error(`Invalid file path: ${JSON.stringify(relativePath)}`); + } + + const root = path.resolve(projectPath); + const fullPath = path.resolve(root, ...segments); + const prefix = root.endsWith(path.sep) ? root : root + path.sep; + if (fullPath !== root && !fullPath.startsWith(prefix)) { + throw new Error(`Path escapes project root: ${JSON.stringify(relativePath)}`); + } + return fullPath; +} + +export function findConflicts( + projectPath: string, + files: Record +): string[] { + const conflicts: string[] = []; + for (const relativePath of Object.keys(files)) { + const fullPath = resolveSafeProjectPath(projectPath, relativePath); + if (fs.existsSync(fullPath) && fs.statSync(fullPath).isFile()) { + conflicts.push(relativePath); + } + } + return conflicts.sort((a, b) => a.localeCompare(b)); +} + +export async function resolveConflicts(conflicts: string[]): Promise { + if (conflicts.length === 0) { + return 'overwrite'; + } + + const preview = conflicts.slice(0, 10); + const extra = conflicts.length > 10 ? `\n…and ${conflicts.length - 10} more` : ''; + const list = preview.map((p) => `• ${p}`).join('\n'); + + const choice = await vscode.window.showWarningMessage( + `${conflicts.length} file(s) already exist under the project folder:\n${list}${extra}`, + { modal: true }, + 'Overwrite all', + 'Skip existing', + 'Cancel' + ); + + if (choice === 'Overwrite all') { + return 'overwrite'; + } + if (choice === 'Skip existing') { + return 'skip'; + } + return 'cancel'; +} + +export function writeProjectFiles( + projectPath: string, + files: Record, + resolution: ConflictResolution +): { written: number; skipped: number } { + if (resolution === 'cancel') { + return { written: 0, skipped: 0 }; + } + + fs.mkdirSync(projectPath, { recursive: true }); + + let written = 0; + let skipped = 0; + + for (const relativePath of Object.keys(files)) { + const fullPath = resolveSafeProjectPath(projectPath, relativePath); + const exists = fs.existsSync(fullPath) && fs.statSync(fullPath).isFile(); + + if (exists && resolution === 'skip') { + skipped += 1; + continue; + } + + fs.mkdirSync(path.dirname(fullPath), { recursive: true }); + fs.writeFileSync(fullPath, files[relativePath], 'utf8'); + written += 1; + } + + return { written, skipped }; +} From 0da405176d328cc1753e3902e9a0bab9042e9b9c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 08:30:51 +0000 Subject: [PATCH 03/13] Implement Creer v0.3: streaming, offline backends, bake-ins, token hardening Add SSE /generate/stream with per-file progress, OpenAI-compatible base URL and CREER_OFFLINE stubs, LICENSE/CI bake-ins, SecretStorage for GitHub tokens, and GIT_ASKPASS push so tokens never appear on argv or remote URLs. Co-authored-by: Sanath S Patil --- PLAN.md | 16 +- README.md | 64 +++++++- backend/.env.example | 11 ++ backend/app/bakeins.py | 157 +++++++++++++++++++ backend/app/generator.py | 267 ++++++++++++++++++++++++++++---- backend/app/llm.py | 30 ++++ backend/app/planner.py | 39 +++-- backend/app/validator.py | 3 +- backend/config.py | 2 + backend/main.py | 109 +++++++++++-- extension/package.json | 18 ++- extension/src/api.ts | 11 ++ extension/src/extension.ts | 29 +++- extension/src/git.ts | 98 +++++++++--- extension/src/scaffold.ts | 117 ++++++++++---- extension/src/secrets.ts | 91 +++++++++++ extension/src/streamGenerate.ts | 224 +++++++++++++++++++++++++++ 17 files changed, 1154 insertions(+), 132 deletions(-) create mode 100644 backend/app/bakeins.py create mode 100644 backend/app/llm.py create mode 100644 extension/src/secrets.ts create mode 100644 extension/src/streamGenerate.ts diff --git a/PLAN.md b/PLAN.md index 69ee028..5b26714 100644 --- a/PLAN.md +++ b/PLAN.md @@ -12,12 +12,18 @@ v0.1 delivered the foundation: FastAPI planner/generator + VS Code command that Also in v0.2: modular extension layout (`api` / `scaffold` / `git` / `writeFiles` / `preview`), backend path/content validation, lazy OpenAI clients, extension path sandbox on write. -## v0.3 priorities (next) +## v0.3 — done -- Streaming generation progress to the extension UI -- Local/offline model backends -- Open-source README / LICENSE / CI templates baked into every scaffold -- Hardening: avoid putting GitHub tokens on `git push` argv (credential helper / askpass); SecretStorage instead of plaintext `creer.githubToken` setting +1. **Streaming generation** — `POST /generate/stream` (SSE) + extension `creer.useStreaming` with per-file progress and fallback to `/generate`. +2. **Local / offline backends** — `OPENAI_BASE_URL` for OpenAI-compatible servers; `CREER_OFFLINE` for template/stub-only generation. +3. **Open-source bake-ins** — every scaffold gets `LICENSE` (MIT), optional default `README.md`, and `.github/workflows/ci.yml` when missing (`backend/app/bakeins.py`). +4. **Hardening** — `GIT_ASKPASS` for push (no token on argv/URL); GitHub token in SecretStorage (`creer.setGitHubToken` / `creer.clearGitHubToken`) with deprecated settings fallback. + +## v0.4 (optional next) + +- Cancellation for long streaming generates +- Richer bake-in / template composition (user-selectable license, CI presets) +- Light telemetry-free quality gates on generated trees ## Non-goals (keep out of early versions) diff --git a/README.md b/README.md index e45dd37..453c8c1 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ AI-powered repo scaffolding inside your workspace. Describe an idea → Creer plans a clean structure → optionally preview/confirm → writes files into your VS Code workspace → optionally initializes git and creates a GitHub remote. -**Current version: 0.2.0** +**Current version: 0.3.0** ## Architecture @@ -18,10 +18,10 @@ creer/ - Python 3.10+ - Node.js 18+ -- OpenAI API key (required for AI planning/generation; template-only planning can run without it) +- OpenAI API key **or** a local OpenAI-compatible server (`OPENAI_BASE_URL`), unless using offline template mode (`CREER_OFFLINE=1`) - VS Code / Cursor -## Backend (v0.2) +## Backend (v0.3) ```bash cd backend @@ -29,18 +29,28 @@ python -m venv venv source venv/bin/activate # Windows: venv\Scripts\activate pip install -r requirements.txt cp .env.example .env -# set OPENAI_API_KEY in .env +# set OPENAI_API_KEY and/or OPENAI_BASE_URL in .env uvicorn main:app --reload --port 8000 ``` +### Environment + +| Variable | Description | +|---|---| +| `OPENAI_API_KEY` | OpenAI API key (optional if using a local server via `OPENAI_BASE_URL`, or `CREER_OFFLINE=1`) | +| `OPENAI_BASE_URL` | Optional OpenAI-compatible base URL for local/offline backends (Ollama, LM Studio, vLLM, etc.). Example: `http://127.0.0.1:11434/v1` | +| `CREER_MODEL` | Model name (default `gpt-4o-mini`) | +| `CREER_OFFLINE` | Set to `1`, `true`, or `yes` for template-only / stub generation — never calls the LLM. Offline planning requires a `template_id`. | + ### Endpoints | Method | Path | Description | |---|---|---| -| `GET` | `/health` | Liveness + version (`0.2.0`) | +| `GET` | `/health` | Liveness + version (`0.3.0`), offline flag, base URL status | | `GET` | `/templates` | List curated starter templates | | `POST` | `/plan` | Return a project plan (no file contents) | | `POST` | `/generate` | Plan (or accept a plan) and generate file contents | +| `POST` | `/generate/stream` | Same as `/generate`, streamed as SSE with per-file progress | | `POST` | `/github/create-repo` | Create a GitHub repo for the authenticated user | #### `POST /plan` @@ -49,7 +59,7 @@ uvicorn main:app --reload --port 8000 { "idea": "Build a FastAPI todo app", "template_id": "fastapi-minimal" } ``` -`template_id` is optional. Response includes `project_name`, `stack`, `files`, and optionally `template_id` / `description`. +`template_id` is optional (required when `CREER_OFFLINE=1`). Response includes `project_name`, `stack`, `files`, and optionally `template_id` / `description`. #### `POST /generate` @@ -63,6 +73,32 @@ uvicorn main:app --reload --port 8000 Omit `plan` to plan then generate, or pass a prior `/plan` body to skip re-planning. Response includes `project_name`, `stack`, and `files` (path → content map). +Every scaffold is merged with **bake-ins** after generation: + +- `LICENSE` — MIT (always ensured if missing) +- `README.md` — only if the plan did not already produce one +- `.github/workflows/ci.yml` — stack-heuristic CI workflow if missing + +#### `POST /generate/stream` + +Same request body as `/generate`. Response is `text/event-stream` with JSON payloads on `data:` lines: + +| Event | Fields | Meaning | +|---|---|---| +| `start` | `project_name`, `total`, `stack` | Generation started | +| `file` | `index`, `total`, `path`, `status` (`generating` \| `done`), optional `bytes` | Per-file progress | +| `done` | `project_name`, `stack`, `files`, optional `template_id` | Final file map (includes bake-ins) | +| `error` | `detail` | Failure | + +Example: + +```bash +curl -N -X POST http://localhost:8000/generate/stream \ + -H "Content-Type: application/json" \ + -H "Accept: text/event-stream" \ + -d '{"idea":"Build a FastAPI todo app","template_id":"fastapi-minimal"}' +``` + #### `POST /github/create-repo` Prefer `Authorization: Bearer `; `body.token` is also accepted. @@ -93,7 +129,7 @@ curl -X POST http://localhost:8000/generate \ -d '{"idea":"Build a FastAPI todo app"}' ``` -## Extension (v0.2) +## Extension (v0.3) ```bash cd extension @@ -115,6 +151,8 @@ In VS Code / Cursor: |---|---| | `creer.createRepo` | Creer: Create New Repo | | `creer.createRepoFromChat` | Creer: Create from Chat Prompt | +| `creer.setGitHubToken` | Creer: Set GitHub Token (SecretStorage) | +| `creer.clearGitHubToken` | Creer: Clear GitHub Token | Chat: `@creer ` (when the host supports chat participants) or run **Create from Chat Prompt** with a `/creer …` style input. @@ -125,9 +163,19 @@ Chat: `@creer ` (when the host supports chat participants) or run **Create | `creer.backendUrl` | `http://localhost:8000` | Backend base URL | | `creer.initGit` | `true` | Run `git init` + initial commit after scaffolding | | `creer.previewBeforeWrite` | `true` | Show plan preview and confirm before generating/writing | +| `creer.useStreaming` | `true` | Use SSE `/generate/stream` with per-file progress; falls back to `/generate` on failure | | `creer.createGitHubRepo` | `false` | After scaffolding, create a GitHub remote repository | | `creer.githubPrivate` | `true` | Create GitHub repositories as private | -| `creer.githubToken` | `""` | GitHub PAT for create/push (prompted if empty when needed) | +| `creer.githubToken` | `""` | **Deprecated.** Prefer SecretStorage via **Creer: Set GitHub Token**. Used only as a fallback when SecretStorage is empty. | + +### GitHub token (SecretStorage) + +Tokens are stored in VS Code **SecretStorage**, not in plaintext settings: + +1. **Creer: Set GitHub Token** — prompt and store +2. **Creer: Clear GitHub Token** — remove from SecretStorage + +When creating/pushing a GitHub repo, Creer resolves the token as: SecretStorage → deprecated `creer.githubToken` setting → one-time prompt (optionally save). Push uses `GIT_ASKPASS` so the token is never embedded in the remote URL or git argv. ## License diff --git a/backend/.env.example b/backend/.env.example index fb03904..30d816e 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -1,2 +1,13 @@ +# OpenAI API key (required unless CREER_OFFLINE=1 or using a local server that ignores it) OPENAI_API_KEY=sk-your-key-here + +# Optional OpenAI-compatible base URL for local/offline backends (Ollama, LM Studio, vLLM, etc.) +# Example: http://127.0.0.1:11434/v1 +OPENAI_BASE_URL= + +# Model name (OpenAI or local-compatible) CREER_MODEL=gpt-4o-mini + +# Template-only / stub generation — never calls the LLM +# Set to 1, true, or yes to enable +CREER_OFFLINE= diff --git a/backend/app/bakeins.py b/backend/app/bakeins.py new file mode 100644 index 0000000..7f28d0e --- /dev/null +++ b/backend/app/bakeins.py @@ -0,0 +1,157 @@ +"""Open-source bake-ins merged into every scaffold (LICENSE, README, CI).""" + +from __future__ import annotations + + +def _mit_license(project_name: str) -> str: + holder = project_name.strip() if project_name and project_name.strip() else "Creer Scaffold" + return f"""MIT License + +Copyright (c) 2026 {holder} + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + + +def _default_readme(plan: dict) -> str: + name = plan.get("project_name") or "project" + stack = plan.get("stack") or "" + desc = plan.get("description") or f"Scaffolded by Creer for {name}." + lines = [ + f"# {name}", + "", + desc, + "", + ] + if stack: + lines.extend([f"**Stack:** {stack}", ""]) + lines.extend( + [ + "## Getting started", + "", + "This project was generated with [Creer](https://github.com/).", + "Install dependencies and follow stack-specific docs in this repo.", + "", + ] + ) + return "\n".join(lines) + + +def _ci_workflow(plan: dict) -> str: + stack = (plan.get("stack") or "").lower() + paths = " ".join(plan.get("files") or []).lower() + blob = f"{stack} {paths}" + + is_python = any( + tok in blob + for tok in ("python", "fastapi", "uvicorn", "pytest", "requirements.txt", "pyproject.toml") + ) + is_node = any( + tok in blob + for tok in ("node", "express", "next", "npm", "package.json", "react") + ) + + if is_python: + return """name: CI + +on: + push: + branches: [main, master] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f pyproject.toml ]; then pip install -e ".[dev]" || pip install -e .; fi + pip install pytest + - name: Run tests + run: pytest -q || echo "No tests yet — scaffold CI ok" +""" + + if is_node: + return """name: CI + +on: + push: + branches: [main, master] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: npm + - name: Install + run: npm ci || npm install + - name: Test + run: npm test --if-present +""" + + return """name: CI + +on: + push: + branches: [main, master] + pull_request: + +jobs: + scaffold: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Scaffold CI + run: echo "Creer scaffold CI — add stack-specific checks as the project grows" +""" + + +def apply_bakeins(plan: dict, files: dict[str, str]) -> dict[str, str]: + """ + Merge open-source bake-ins into generated files. + + - LICENSE is always ensured (MIT, 2026). + - README.md is created only if missing (never overwrite AI/template README). + - .github/workflows/ci.yml is added if missing (stack heuristics). + """ + out = dict(files) + project_name = plan.get("project_name") or "Creer Scaffold" + + if "LICENSE" not in out: + out["LICENSE"] = _mit_license(project_name) + + if "README.md" not in out: + out["README.md"] = _default_readme(plan) + + ci_path = ".github/workflows/ci.yml" + if ci_path not in out: + out[ci_path] = _ci_workflow(plan) + + return out diff --git a/backend/app/generator.py b/backend/app/generator.py index 2d1823a..db78c02 100644 --- a/backend/app/generator.py +++ b/backend/app/generator.py @@ -1,49 +1,256 @@ -from openai import OpenAI -from config import OPENAI_API_KEY, MODEL +"""File content generation — LLM-driven or offline stubs.""" -_client: OpenAI | None = None +from __future__ import annotations +import json +from collections.abc import Iterator +from typing import Any -def _get_client() -> OpenAI: - global _client - if _client is None: - if not OPENAI_API_KEY: - raise ValueError("OPENAI_API_KEY is not set") - _client = OpenAI(api_key=OPENAI_API_KEY) - return _client +from config import CREER_OFFLINE, MODEL, OPENAI_API_KEY, OPENAI_BASE_URL +from app.llm import _get_client -def generate_files(plan: dict) -> dict[str, str]: - """Generate file contents one-by-one from a project plan.""" - files_output: dict[str, str] = {} +def _use_offline(plan: dict) -> bool: + """True when CREER_OFFLINE, or when no LLM endpoint is configured and a plan exists.""" + if CREER_OFFLINE: + return True + # Local OpenAI-compatible servers (Ollama, etc.) may omit a real API key. + if OPENAI_BASE_URL: + return False + if not OPENAI_API_KEY and plan.get("files"): + return True + return False + + +def _stub_content(file_path: str, plan: dict) -> str: + """Deterministic stub content for offline / template-only generation.""" + name = plan.get("project_name") or "project" + stack = plan.get("stack") or "" + desc = plan.get("description") or f"Scaffolded project: {name}." + base = file_path.rsplit("/", 1)[-1] + base_lower = base.lower() + ext = base_lower.rsplit(".", 1)[-1] if "." in base_lower else "" + + if base_lower == "readme.md": + lines = [f"# {name}", "", desc, ""] + if stack: + lines.extend([f"**Stack:** {stack}", ""]) + lines.extend( + [ + "## Status", + "", + "Generated in offline / template-only mode. Replace stubs with real implementation.", + "", + ] + ) + return "\n".join(lines) + + if base_lower == "requirements.txt": + if "fastapi" in stack.lower() or "uvicorn" in stack.lower(): + return "fastapi>=0.115.0\nuvicorn[standard]>=0.32.0\npython-dotenv>=1.0.0\n" + return "# TODO: add dependencies\n" + + if base_lower == "package.json": + pkg = { + "name": name, + "version": "0.1.0", + "private": True, + "scripts": { + "start": "node src/index.js", + "test": "echo \"No tests yet\" && exit 0", + }, + "dependencies": {}, + } + if "express" in stack.lower(): + pkg["dependencies"]["express"] = "^4.21.0" + pkg["scripts"]["start"] = "node src/index.js" + if "next" in stack.lower(): + pkg["dependencies"]["next"] = "^14.2.0" + pkg["dependencies"]["react"] = "^18.3.0" + pkg["dependencies"]["react-dom"] = "^18.3.0" + pkg["scripts"] = { + "dev": "next dev", + "build": "next build", + "start": "next start", + "test": "echo \"No tests yet\" && exit 0", + } + return json.dumps(pkg, indent=2) + "\n" + + if base_lower == "pyproject.toml": + return f"""[project] +name = "{name}" +version = "0.1.0" +description = "{desc.replace(chr(34), "'")}" +requires-python = ">=3.10" +dependencies = [] + +[project.scripts] +{name} = "src.cli:main" +""" + + if base_lower in (".gitignore",): + return ( + "__pycache__/\n*.py[cod]\n.venv/\nvenv/\n.env\n" + "node_modules/\ndist/\nbuild/\n.DS_Store\n" + ) + + if base_lower in (".env.example",): + return "# Example environment variables\n# KEY=value\n" + + if base_lower == "tsconfig.json": + return json.dumps( + { + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": True, + "skipLibCheck": True, + "strict": True, + "module": "esnext", + "moduleResolution": "bundler", + "jsx": "preserve", + "noEmit": True, + "incremental": True, + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], + "exclude": ["node_modules"], + }, + indent=2, + ) + "\n" + + if base_lower == "next.config.js": + return "/** @type {import('next').NextConfig} */\nconst nextConfig = {};\nmodule.exports = nextConfig;\n" + + if ext in ("py",): + if base_lower == "main.py" and "fastapi" in stack.lower(): + return ( + '"""Application entrypoint."""\n\n' + "from fastapi import FastAPI\n\n" + f'app = FastAPI(title="{name}")\n\n\n' + '@app.get("/health")\n' + "def health():\n" + ' return {"status": "ok"}\n\n\n' + "# TODO: implement\n" + ) + return f'"""{file_path}"""\n\n# TODO: implement\n' + + if ext in ("js", "mjs", "cjs"): + return f"// {file_path}\n// TODO: implement\n" + + if ext in ("ts", "tsx"): + return f"// {file_path}\n// TODO: implement\n" - for file_path in plan["files"]: - prompt = f""" + if ext == "css": + return f"/* {file_path} */\n/* TODO: implement */\n" + + if ext in ("html", "htm"): + return ( + "\n" + '\n' + "\n" + ' \n' + f" {name}\n" + ' \n' + "\n" + "\n" + f"

{name}

\n" + " \n" + ' \n' + "\n" + "\n" + ) + + if ext in ("yml", "yaml"): + return f"# {file_path}\n# TODO: implement\n" + + if ext == "json": + return "{}\n" + + if ext == "toml": + return f"# {file_path}\n# TODO: implement\n" + + # Default: comment-style stub + return f"# TODO: implement ({file_path})\n" + + +def _strip_fences(content: str) -> str: + if content.startswith("```"): + lines = content.splitlines() + if lines and lines[0].startswith("```"): + lines = lines[1:] + if lines and lines[-1].strip() == "```": + lines = lines[:-1] + content = "\n".join(lines) + return content + + +def _generate_one_llm(file_path: str, plan: dict) -> str: + prompt = f""" Generate the full content for file: {file_path} -Project Stack: {plan["stack"]} -Project Name: {plan["project_name"]} +Project Stack: {plan.get("stack", "")} +Project Name: {plan.get("project_name", "")} Follow best practices. Return ONLY the file content — no markdown fences, no explanation. """ + response = _get_client().chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=0.3, + ) + content = response.choices[0].message.content or "" + return _strip_fences(content) - response = _get_client().chat.completions.create( - model=MODEL, - messages=[{"role": "user", "content": prompt}], - temperature=0.3, + +def generate_files_iter(plan: dict) -> Iterator[tuple[dict[str, Any], dict[str, str]]]: + """ + Yield (event_dict, partial_files) progress while generating. + + Events: + - file / generating + - file / done (with bytes) + Does not yield start/done/error — caller owns those. + """ + files_output: dict[str, str] = {} + file_list = list(plan["files"]) + total = len(file_list) + offline = _use_offline(plan) + + for index, file_path in enumerate(file_list, start=1): + yield ( + { + "event": "file", + "index": index, + "total": total, + "path": file_path, + "status": "generating", + }, + dict(files_output), ) - content = response.choices[0].message.content or "" - # Strip accidental markdown fences - if content.startswith("```"): - lines = content.splitlines() - if lines and lines[0].startswith("```"): - lines = lines[1:] - if lines and lines[-1].strip() == "```": - lines = lines[:-1] - content = "\n".join(lines) + if offline: + content = _stub_content(file_path, plan) + else: + content = _generate_one_llm(file_path, plan) files_output[file_path] = content + yield ( + { + "event": "file", + "index": index, + "total": total, + "path": file_path, + "status": "done", + "bytes": len(content.encode("utf-8")), + }, + dict(files_output), + ) + +def generate_files(plan: dict) -> dict[str, str]: + """Generate file contents one-by-one from a project plan.""" + files_output: dict[str, str] = {} + for _event, partial in generate_files_iter(plan): + files_output = partial return files_output diff --git a/backend/app/llm.py b/backend/app/llm.py new file mode 100644 index 0000000..e12a328 --- /dev/null +++ b/backend/app/llm.py @@ -0,0 +1,30 @@ +"""Shared OpenAI-compatible client for Creer.""" + +from __future__ import annotations + +from openai import OpenAI + +from config import OPENAI_API_KEY, OPENAI_BASE_URL + +_client: OpenAI | None = None + + +def _get_client() -> OpenAI: + """Lazy OpenAI client. Passes base_url when OPENAI_BASE_URL is set (Ollama, etc.).""" + global _client + if _client is None: + if not OPENAI_API_KEY and not OPENAI_BASE_URL: + raise ValueError( + "OPENAI_API_KEY is not set (or set OPENAI_BASE_URL for a local OpenAI-compatible server)" + ) + kwargs: dict = {"api_key": OPENAI_API_KEY or "not-needed"} + if OPENAI_BASE_URL: + kwargs["base_url"] = OPENAI_BASE_URL + _client = OpenAI(**kwargs) + return _client + + +def reset_client() -> None: + """Clear the cached client (useful in tests).""" + global _client + _client = None diff --git a/backend/app/planner.py b/backend/app/planner.py index 6cd8a62..9922a07 100644 --- a/backend/app/planner.py +++ b/backend/app/planner.py @@ -5,21 +5,14 @@ import json import re -from openai import OpenAI - -from config import MODEL, OPENAI_API_KEY +from config import CREER_OFFLINE, MODEL, OPENAI_API_KEY, OPENAI_BASE_URL +from app.llm import _get_client from app.templates import apply_template, get_template, slugify -_client: OpenAI | None = None - -def _get_client() -> OpenAI: - global _client - if _client is None: - if not OPENAI_API_KEY: - raise ValueError("OPENAI_API_KEY is not set") - _client = OpenAI(api_key=OPENAI_API_KEY) - return _client +def _llm_configured() -> bool: + """True when an OpenAI key or OpenAI-compatible base URL is available.""" + return bool(OPENAI_API_KEY or OPENAI_BASE_URL) def _parse_json(content: str) -> dict: @@ -103,10 +96,17 @@ def plan_project(idea: str, template_id: str | None = None) -> dict: When template_id is set: - files and stack come from the template - project_name is derived deterministically via slugify (no API key required) - - if OPENAI_API_KEY is present, AI may refine project_name only + - if an LLM is configured and not offline, AI may refine project_name only - Without template_id: full AI planning (requires OPENAI_API_KEY). + Without template_id: full AI planning (requires OPENAI_API_KEY or OPENAI_BASE_URL), + unless offline — offline without template_id raises ValueError. """ + if CREER_OFFLINE and not template_id: + raise ValueError( + "Offline mode requires template_id. Pass a curated template_id " + "(see GET /templates) — AI planning is disabled when CREER_OFFLINE is set." + ) + if template_id: tmpl = get_template(template_id) if tmpl is None: @@ -114,10 +114,17 @@ def plan_project(idea: str, template_id: str | None = None) -> dict: plan = apply_template(template_id, idea) - # Optionally refine name with AI when a key is available - if OPENAI_API_KEY: + # Optionally refine name with AI when an LLM endpoint is available and not offline + if _llm_configured() and not CREER_OFFLINE: plan["project_name"] = _ai_name_project(idea, tmpl) return plan + if not _llm_configured() and not CREER_OFFLINE: + # No LLM endpoint and no template — cannot plan with AI + raise ValueError( + "OPENAI_API_KEY (or OPENAI_BASE_URL) is not set. Provide a template_id " + "for template-only planning, or set CREER_OFFLINE=1 with a template_id." + ) + return _ai_plan(idea) diff --git a/backend/app/validator.py b/backend/app/validator.py index d6978e8..f5fcdae 100644 --- a/backend/app/validator.py +++ b/backend/app/validator.py @@ -13,7 +13,8 @@ # Windows drive / UNC-ish prefixes WINDOWS_DRIVE = re.compile(r"^[a-zA-Z]:") -MAX_FILES = 40 +# Plan file cap; generated set may also include bake-ins (LICENSE, CI, optional README). +MAX_FILES = 45 MAX_CONTENT_BYTES_PER_FILE = 200_000 MAX_TOTAL_CONTENT_BYTES = 2_000_000 diff --git a/backend/config.py b/backend/config.py index 0bac399..2089ca1 100644 --- a/backend/config.py +++ b/backend/config.py @@ -4,4 +4,6 @@ load_dotenv() OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") +OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL") # e.g. http://127.0.0.1:11434/v1 for Ollama MODEL = os.getenv("CREER_MODEL", "gpt-4o-mini") +CREER_OFFLINE = os.getenv("CREER_OFFLINE", "").lower() in ("1", "true", "yes") diff --git a/backend/main.py b/backend/main.py index 379c3fb..daf2b71 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,13 +1,23 @@ +"""Creer FastAPI application — plan, generate, stream, GitHub helpers.""" + +from __future__ import annotations + +import json +from collections.abc import Iterator + from fastapi import FastAPI, Header, HTTPException +from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field +from config import CREER_OFFLINE, MODEL, OPENAI_BASE_URL +from app.bakeins import apply_bakeins from app.planner import plan_project -from app.generator import generate_files +from app.generator import generate_files, generate_files_iter from app.validator import validate_plan, validate_files from app.templates import list_templates, get_template from app.github import create_github_repo -VERSION = "0.2.0" +VERSION = "0.3.0" app = FastAPI(title="Creer", version=VERSION) @@ -38,9 +48,36 @@ class GitHubCreateRepoRequest(BaseModel): token: str | None = None +def _resolve_plan(request: GenerateRequest) -> dict: + """Resolve a validated plan from GenerateRequest (shared by sync + stream).""" + if request.plan is not None: + plan = request.plan.model_dump() + plan = {k: v for k, v in plan.items() if v is not None} + plan.setdefault("stack", "") + if request.template_id and "template_id" not in plan: + plan["template_id"] = request.template_id + else: + if request.template_id and get_template(request.template_id) is None: + raise ValueError(f"Unknown template_id: {request.template_id!r}") + plan = plan_project(request.idea, template_id=request.template_id) + + validate_plan(plan) + return plan + + +def _sse(data: dict) -> str: + return f"data: {json.dumps(data, ensure_ascii=False)}\n\n" + + @app.get("/health") def health(): - return {"status": "ok", "version": VERSION} + return { + "status": "ok", + "version": VERSION, + "offline": CREER_OFFLINE, + "base_url_set": bool(OPENAI_BASE_URL), + "model": MODEL, + } @app.get("/templates") @@ -76,20 +113,9 @@ def plan_only(request: PlanRequest): @app.post("/generate") def generate_project(request: GenerateRequest): try: - if request.plan is not None: - plan = request.plan.model_dump() - # Drop nulls except keep stack as empty string for the generator prompt - plan = {k: v for k, v in plan.items() if v is not None} - plan.setdefault("stack", "") - if request.template_id and "template_id" not in plan: - plan["template_id"] = request.template_id - else: - if request.template_id and get_template(request.template_id) is None: - raise ValueError(f"Unknown template_id: {request.template_id!r}") - plan = plan_project(request.idea, template_id=request.template_id) - - validate_plan(plan) + plan = _resolve_plan(request) files = generate_files(plan) + files = apply_bakeins(plan, files) validate_files(files) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc @@ -106,6 +132,57 @@ def generate_project(request: GenerateRequest): return result +@app.post("/generate/stream") +def generate_project_stream(request: GenerateRequest): + """Stream generation progress as Server-Sent Events (JSON data lines).""" + + def event_stream() -> Iterator[str]: + plan: dict | None = None + try: + plan = _resolve_plan(request) + file_list = list(plan["files"]) + yield _sse( + { + "event": "start", + "project_name": plan["project_name"], + "total": len(file_list), + "stack": plan.get("stack") or "", + } + ) + + files: dict[str, str] = {} + for event, partial in generate_files_iter(plan): + files = partial + yield _sse(event) + + files = apply_bakeins(plan, files) + validate_files(files) + + done: dict = { + "event": "done", + "project_name": plan["project_name"], + "files": files, + "stack": plan.get("stack") or "", + } + if plan.get("template_id"): + done["template_id"] = plan["template_id"] + yield _sse(done) + except ValueError as exc: + yield _sse({"event": "error", "detail": str(exc)}) + except Exception as exc: + yield _sse({"event": "error", "detail": f"Generation failed: {exc}"}) + + return StreamingResponse( + event_stream(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) + + @app.post("/github/create-repo") def github_create_repo( request: GitHubCreateRepoRequest, diff --git a/extension/package.json b/extension/package.json index a69c1b7..09fe82e 100644 --- a/extension/package.json +++ b/extension/package.json @@ -2,7 +2,7 @@ "name": "creer", "displayName": "Creer", "description": "AI-powered repo scaffolding inside your workspace", - "version": "0.2.0", + "version": "0.3.0", "publisher": "creer", "engines": { "vscode": "^1.85.0" @@ -22,6 +22,14 @@ { "command": "creer.createRepoFromChat", "title": "Creer: Create from Chat Prompt" + }, + { + "command": "creer.setGitHubToken", + "title": "Creer: Set GitHub Token" + }, + { + "command": "creer.clearGitHubToken", + "title": "Creer: Clear GitHub Token" } ], "chatParticipants": [ @@ -49,7 +57,8 @@ "creer.githubToken": { "type": "string", "default": "", - "description": "GitHub personal access token for creating/pushing repositories" + "description": "DEPRECATED: Prefer SecretStorage via 'Creer: Set GitHub Token'. Used only as a fallback when SecretStorage is empty.", + "deprecationMessage": "Use 'Creer: Set GitHub Token' (SecretStorage) instead of storing the token in settings." }, "creer.githubPrivate": { "type": "boolean", @@ -65,6 +74,11 @@ "type": "boolean", "default": true, "description": "Show a plan preview and confirm before generating and writing files" + }, + "creer.useStreaming": { + "type": "boolean", + "default": true, + "description": "Use SSE streaming (/generate/stream) with per-file progress; falls back to /generate on failure" } } } diff --git a/extension/src/api.ts b/extension/src/api.ts index 5df587c..f2dd6bf 100644 --- a/extension/src/api.ts +++ b/extension/src/api.ts @@ -94,6 +94,17 @@ export async function postGenerate( return response.data; } +/** Re-export streaming generate for callers that import from api. */ +export { streamGenerate } from './streamGenerate'; +export type { + StreamProgressEvent, + StreamGenerateOptions, + StreamStartEvent, + StreamFileEvent, + StreamDoneEvent, + StreamErrorEvent, +} from './streamGenerate'; + export async function createGitHubRepo( token: string, name: string, diff --git a/extension/src/extension.ts b/extension/src/extension.ts index 07addbd..abac31a 100644 --- a/extension/src/extension.ts +++ b/extension/src/extension.ts @@ -1,20 +1,41 @@ import * as vscode from 'vscode'; import { runScaffoldFlow } from './scaffold'; +import { clearGitHubToken, setGitHubToken } from './secrets'; export function activate(context: vscode.ExtensionContext) { const createRepo = vscode.commands.registerCommand('creer.createRepo', async () => { - await runScaffoldFlow({ fromChat: false }); + await runScaffoldFlow({ context, fromChat: false }); }); const createRepoFromChat = vscode.commands.registerCommand( 'creer.createRepoFromChat', async (idea?: string) => { const initial = typeof idea === 'string' ? idea : undefined; - await runScaffoldFlow({ idea: initial, fromChat: true }); + await runScaffoldFlow({ context, idea: initial, fromChat: true }); } ); - context.subscriptions.push(createRepo, createRepoFromChat); + const setToken = vscode.commands.registerCommand('creer.setGitHubToken', async () => { + const token = await vscode.window.showInputBox({ + prompt: 'GitHub personal access token (repo scope) — stored in SecretStorage', + placeHolder: 'ghp_…', + password: true, + ignoreFocusOut: true, + }); + const trimmed = token?.trim(); + if (!trimmed) { + return; + } + await setGitHubToken(context, trimmed); + void vscode.window.showInformationMessage('Creer: GitHub token saved to SecretStorage.'); + }); + + const clearToken = vscode.commands.registerCommand('creer.clearGitHubToken', async () => { + await clearGitHubToken(context); + void vscode.window.showInformationMessage('Creer: GitHub token cleared from SecretStorage.'); + }); + + context.subscriptions.push(createRepo, createRepoFromChat, setToken, clearToken); registerChatParticipant(context); } @@ -46,7 +67,7 @@ function registerChatParticipant(context: vscode.ExtensionContext): void { `Scaffolding with Creer: **${idea}**…\n\n` + 'Follow the prompts to pick a template, preview the plan, and confirm.' ); - await runScaffoldFlow({ idea, fromChat: true }); + await runScaffoldFlow({ context, idea, fromChat: true }); } ); diff --git a/extension/src/git.ts b/extension/src/git.ts index a111c69..daf7643 100644 --- a/extension/src/git.ts +++ b/extension/src/git.ts @@ -1,12 +1,20 @@ import { execFile } from 'child_process'; import * as fs from 'fs'; +import * as os from 'os'; import * as path from 'path'; import { promisify } from 'util'; const execFileAsync = promisify(execFile); -async function git(cwd: string, args: string[]): Promise { - await execFileAsync('git', args, { cwd }); +async function git( + cwd: string, + args: string[], + env?: NodeJS.ProcessEnv +): Promise { + await execFileAsync('git', args, { + cwd, + env: env ?? process.env, + }); } export async function isGitRepo(projectPath: string): Promise { @@ -29,34 +37,64 @@ export async function ensureGitRepo(projectPath: string): Promise { } } +/** + * Write a temporary GIT_ASKPASS helper that reads the token from CREER_GITHUB_TOKEN. + * Never put the token in the remote URL or argv. + */ +function writeAskpassScript(): string { + const isWin = process.platform === 'win32'; + const askpassPath = path.join( + os.tmpdir(), + `creer-askpass-${process.pid}-${Date.now()}${isWin ? '.cmd' : '.sh'}` + ); + + if (isWin) { + // Git asks with prompts containing "Username" / "Password". + const script = [ + '@echo off', + 'setlocal EnableExtensions', + 'echo(%* | findstr /I "Username" >nul', + 'if not errorlevel 1 (', + ' echo x-access-token', + ') else (', + ' echo(%CREER_GITHUB_TOKEN%', + ')', + '', + ].join('\r\n'); + fs.writeFileSync(askpassPath, script, { encoding: 'utf8' }); + } else { + const script = [ + '#!/bin/sh', + 'case "$1" in', + ' *[Uu]sername*) printf "%s" "x-access-token" ;;', + ' *) printf "%s" "$CREER_GITHUB_TOKEN" ;;', + 'esac', + '', + ].join('\n'); + fs.writeFileSync(askpassPath, script, { encoding: 'utf8', mode: 0o700 }); + } + + return askpassPath; +} + export async function addRemoteAndPush( projectPath: string, cloneUrl: string, token?: string ): Promise { - // Prefer authenticated HTTPS URL when a token is available so push works non-interactively. - // Args are passed via execFile (no shell) so tokens/URLs cannot inject commands. - let pushUrl = cloneUrl; - if (token && cloneUrl.startsWith('https://')) { - pushUrl = cloneUrl.replace( - 'https://', - `https://x-access-token:${encodeURIComponent(token)}@` - ); - } - try { await git(projectPath, ['remote', 'remove', 'origin']); } catch { // No existing origin — fine. } + // Always store a clean clone URL (no embedded credentials). await git(projectPath, ['remote', 'add', 'origin', cloneUrl]); // Ensure we have a branch name for -u push. try { await git(projectPath, ['rev-parse', '--verify', 'HEAD']); } catch { - // No commits yet — stage and commit if possible. await git(projectPath, ['add', '.']); try { await git(projectPath, ['commit', '-m', 'Initial commit']); @@ -65,14 +103,36 @@ export async function addRemoteAndPush( } } - // Push using authenticated URL without rewriting the stored remote (keep clone_url clean). + let askpassPath: string | undefined; try { - await git(projectPath, ['push', '-u', pushUrl, 'HEAD']); - } catch { - // Fallback: try main explicitly - await git(projectPath, ['push', '-u', pushUrl, 'HEAD:main']); + const pushEnv: NodeJS.ProcessEnv = { ...process.env }; + + if (token) { + askpassPath = writeAskpassScript(); + pushEnv.GIT_ASKPASS = askpassPath; + pushEnv.GIT_TERMINAL_PROMPT = '0'; + pushEnv.CREER_GITHUB_TOKEN = token; + // Prefer askpass over interactive prompts / GUI helpers for this child only. + pushEnv.SSH_ASKPASS = askpassPath; + pushEnv.SSH_ASKPASS_REQUIRE = 'never'; + } + + // Push to the clean clone URL — credentials come from askpass/env only. + try { + await git(projectPath, ['push', '-u', cloneUrl, 'HEAD'], pushEnv); + } catch { + await git(projectPath, ['push', '-u', cloneUrl, 'HEAD:main'], pushEnv); + } + } finally { + if (askpassPath) { + try { + fs.unlinkSync(askpassPath); + } catch { + // Best-effort cleanup. + } + } } - // Keep origin pointing at the clean clone URL (without embedded token). + // Keep origin pointing at the clean clone URL. await git(projectPath, ['remote', 'set-url', 'origin', cloneUrl]); } diff --git a/extension/src/scaffold.ts b/extension/src/scaffold.ts index 5e1eacf..a9d1128 100644 --- a/extension/src/scaffold.ts +++ b/extension/src/scaffold.ts @@ -6,11 +6,14 @@ import { formatAxiosError, postGenerate, postPlan, + type GenerateResponse, type PlanResponse, type Template, } from './api'; import { addRemoteAndPush, ensureGitRepo, initGit } from './git'; import { showPlanPreviewAndConfirm } from './preview'; +import { resolveGitHubToken } from './secrets'; +import { streamGenerate, type StreamProgressEvent } from './streamGenerate'; import { assertSafeProjectName, findConflicts, @@ -19,6 +22,8 @@ import { } from './writeFiles'; export interface ScaffoldOptions { + /** Extension context (required for SecretStorage). */ + context: vscode.ExtensionContext; /** Pre-filled idea (e.g. from chat). If omitted, prompts the user. */ idea?: string; /** Use chat-style InputBox placeholder (/creer …). */ @@ -83,23 +88,8 @@ async function pickTemplate(templates: Template[]): Promise { - const config = vscode.workspace.getConfiguration('creer'); - const configured = (config.get('githubToken') || '').trim(); - if (configured) { - return configured; - } - - const token = await vscode.window.showInputBox({ - prompt: 'GitHub personal access token (repo scope)', - placeHolder: 'ghp_…', - password: true, - ignoreFocusOut: true, - }); - return token?.trim() || undefined; -} - async function maybeCreateGitHubRemote( + context: vscode.ExtensionContext, projectPath: string, projectName: string, idea: string @@ -122,7 +112,7 @@ async function maybeCreateGitHubRemote( return; } - const token = await resolveGitHubToken(); + const token = await resolveGitHubToken(context); if (!token) { vscode.window.showWarningMessage('GitHub token required to create a repository. Skipped.'); return; @@ -158,10 +148,80 @@ async function maybeCreateGitHubRemote( } } +function reportStreamProgress( + progress: vscode.Progress<{ message?: string; increment?: number }>, + ev: StreamProgressEvent +): void { + if (ev.event === 'start') { + progress.report({ message: `Starting ${ev.project_name} (${ev.total} files)…` }); + return; + } + if (ev.event === 'file') { + const status = ev.status === 'done' ? 'done' : 'generating'; + progress.report({ + message: `[${ev.index}/${ev.total}] ${ev.path} (${status})`, + }); + } +} + +async function generateWithOptionalStream( + idea: string, + plan: PlanResponse, + templateId: string | undefined, + useStreaming: boolean +): Promise { + const genOpts = { + templateId: plan.template_id ?? templateId, + plan, + }; + + if (!useStreaming) { + return vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'Creer: generating project…', + cancellable: false, + }, + () => postGenerate(idea, genOpts) + ); + } + + try { + return await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'Creer: generating project…', + cancellable: false, + }, + async (progress) => + streamGenerate({ + idea, + templateId: genOpts.templateId, + plan: genOpts.plan, + onProgress: (ev) => reportStreamProgress(progress, ev), + }) + ); + } catch (err) { + const message = formatAxiosError(err, 'Streaming generate failed'); + vscode.window.showWarningMessage( + `Streaming failed (${message}); falling back to non-streaming generate.` + ); + return vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'Creer: generating project (fallback)…', + cancellable: false, + }, + () => postGenerate(idea, genOpts) + ); + } +} + /** * Shared scaffold flow used by createRepo, createRepoFromChat, and the chat participant. */ -export async function runScaffoldFlow(options: ScaffoldOptions = {}): Promise { +export async function runScaffoldFlow(options: ScaffoldOptions): Promise { + const { context } = options; const fromChat = options.fromChat ?? false; const idea = await promptForIdea(fromChat, options.idea); if (!idea) { @@ -177,6 +237,7 @@ export async function runScaffoldFlow(options: ScaffoldOptions = {}): Promise('initGit') ?? true; const previewBeforeWrite = config.get('previewBeforeWrite') ?? true; + const useStreaming = config.get('useStreaming') ?? true; const rootPath = workspaceFolders[0].uri.fsPath; try { @@ -232,18 +293,12 @@ export async function runScaffoldFlow(options: ScaffoldOptions = {}): Promise - postGenerate(idea, { - templateId: plan.template_id ?? templateId, - plan, - }) + // 4) Generate (streaming with progress when enabled) + const generated = await generateWithOptionalStream( + idea, + plan, + templateId, + useStreaming ); const projectName = generated.project_name || plan.project_name; @@ -301,7 +356,7 @@ export async function runScaffoldFlow(options: ScaffoldOptions = {}): Promise 0 ? ` (${skipped} existing skipped)` : ''; vscode.window.showInformationMessage( diff --git a/extension/src/secrets.ts b/extension/src/secrets.ts new file mode 100644 index 0000000..a1341fd --- /dev/null +++ b/extension/src/secrets.ts @@ -0,0 +1,91 @@ +import * as vscode from 'vscode'; + +/** SecretStorage key for the GitHub personal access token. */ +export const GITHUB_TOKEN_SECRET_KEY = 'creer.githubToken'; + +/** + * Read GitHub token: SecretStorage first, then deprecated config fallback. + */ +export async function getGitHubToken( + context: vscode.ExtensionContext +): Promise { + const secret = (await context.secrets.get(GITHUB_TOKEN_SECRET_KEY))?.trim(); + if (secret) { + return secret; + } + + // Migration path: plaintext setting (deprecated) + const fromConfig = ( + vscode.workspace.getConfiguration('creer').get('githubToken') || '' + ).trim(); + return fromConfig || undefined; +} + +export async function setGitHubToken( + context: vscode.ExtensionContext, + token: string +): Promise { + await context.secrets.store(GITHUB_TOKEN_SECRET_KEY, token.trim()); +} + +export async function clearGitHubToken(context: vscode.ExtensionContext): Promise { + await context.secrets.delete(GITHUB_TOKEN_SECRET_KEY); +} + +/** + * Prompt for a token and optionally persist it in SecretStorage. + */ +export async function promptAndStoreGitHubToken( + context: vscode.ExtensionContext +): Promise { + const token = await vscode.window.showInputBox({ + prompt: 'GitHub personal access token (repo scope)', + placeHolder: 'ghp_…', + password: true, + ignoreFocusOut: true, + }); + + const trimmed = token?.trim(); + if (!trimmed) { + return undefined; + } + + const saveChoice = await vscode.window.showQuickPick( + [ + { + label: 'Save to SecretStorage (recommended)', + description: 'Stored securely; preferred over settings.json', + id: 'save', + }, + { + label: 'Use once (do not save)', + description: 'Token is only used for this operation', + id: 'once', + }, + ], + { + placeHolder: 'Save GitHub token?', + ignoreFocusOut: true, + } + ); + + if (saveChoice?.id === 'save') { + await setGitHubToken(context, trimmed); + void vscode.window.showInformationMessage('Creer: GitHub token saved to SecretStorage.'); + } + + return trimmed; +} + +/** + * Resolve token: SecretStorage → config fallback → prompt (offer save). + */ +export async function resolveGitHubToken( + context: vscode.ExtensionContext +): Promise { + const existing = await getGitHubToken(context); + if (existing) { + return existing; + } + return promptAndStoreGitHubToken(context); +} diff --git a/extension/src/streamGenerate.ts b/extension/src/streamGenerate.ts new file mode 100644 index 0000000..522832a --- /dev/null +++ b/extension/src/streamGenerate.ts @@ -0,0 +1,224 @@ +import * as http from 'http'; +import * as https from 'https'; +import { URL } from 'url'; +import * as vscode from 'vscode'; +import type { GenerateResponse, PlanResponse } from './api'; + +export type StreamStartEvent = { + event: 'start'; + project_name: string; + total: number; + stack: string; +}; + +export type StreamFileEvent = { + event: 'file'; + index: number; + total: number; + path: string; + status: 'generating' | 'done'; + bytes?: number; +}; + +export type StreamDoneEvent = { + event: 'done'; + project_name: string; + stack: string; + files: Record; + template_id?: string; +}; + +export type StreamErrorEvent = { + event: 'error'; + detail: string; +}; + +export type StreamProgressEvent = + | StreamStartEvent + | StreamFileEvent + | StreamDoneEvent + | StreamErrorEvent; + +export interface StreamGenerateOptions { + idea: string; + templateId?: string; + plan?: PlanResponse; + onProgress?: (event: StreamProgressEvent) => void; +} + +function getBackendUrl(): string { + const config = vscode.workspace.getConfiguration('creer'); + return (config.get('backendUrl') || 'http://localhost:8000').replace(/\/$/, ''); +} + +function parseSseChunk( + buffer: string, + onEvent: (data: StreamProgressEvent) => void +): string { + // Normalize CRLF so event separators are always `\n\n` (proxies / Windows). + let remaining = buffer.replace(/\r\n/g, '\n'); + for (;;) { + const sep = remaining.indexOf('\n\n'); + if (sep === -1) { + break; + } + const rawEvent = remaining.slice(0, sep); + remaining = remaining.slice(sep + 2); + + for (const line of rawEvent.split('\n')) { + const trimmed = line.trim(); + if (!trimmed.startsWith('data:')) { + continue; + } + const payload = trimmed.slice(5).trim(); + if (!payload || payload === '[DONE]') { + continue; + } + try { + const parsed = JSON.parse(payload) as StreamProgressEvent; + if (parsed && typeof parsed === 'object' && 'event' in parsed) { + onEvent(parsed); + } + } catch { + // Ignore malformed JSON fragments; wait for a complete event. + } + } + } + return remaining; +} + +/** + * POST /generate/stream and parse SSE (`data: {...}\\n\\n`). + * Uses Node http/https so streaming works in the VS Code extension CommonJS host. + */ +export function streamGenerate(options: StreamGenerateOptions): Promise { + const backendUrl = getBackendUrl(); + const url = new URL(`${backendUrl}/generate/stream`); + const body: { + idea: string; + template_id?: string; + plan?: PlanResponse; + } = { idea: options.idea }; + if (options.templateId) { + body.template_id = options.templateId; + } + if (options.plan) { + body.plan = options.plan; + } + + const payload = JSON.stringify(body); + const lib = url.protocol === 'https:' ? https : http; + + return new Promise((resolve, reject) => { + let settled = false; + let doneResult: GenerateResponse | undefined; + let buffer = ''; + + const fail = (err: Error) => { + if (settled) { + return; + } + settled = true; + reject(err); + }; + + const succeed = (result: GenerateResponse) => { + if (settled) { + return; + } + settled = true; + resolve(result); + }; + + let req: http.ClientRequest; + + const handleEvent = (ev: StreamProgressEvent) => { + options.onProgress?.(ev); + + if (ev.event === 'error') { + // Stop reading further events; destroy the socket so we do not hang. + try { + req.destroy(); + } catch { + // ignore + } + fail(new Error(ev.detail || 'Streaming generation failed')); + return; + } + + if (ev.event === 'done') { + doneResult = { + project_name: ev.project_name, + stack: ev.stack, + files: ev.files, + }; + if (ev.template_id) { + doneResult.template_id = ev.template_id; + } + } + }; + + req = lib.request( + { + protocol: url.protocol, + hostname: url.hostname, + port: url.port || (url.protocol === 'https:' ? 443 : 80), + path: `${url.pathname}${url.search}`, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Accept: 'text/event-stream', + 'Content-Length': Buffer.byteLength(payload), + Connection: 'keep-alive', + }, + timeout: 600_000, + }, + (res) => { + const status = res.statusCode ?? 0; + if (status < 200 || status >= 300) { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => { + const text = Buffer.concat(chunks).toString('utf8'); + let detail = text; + try { + const parsed = JSON.parse(text) as { detail?: string }; + if (typeof parsed.detail === 'string') { + detail = parsed.detail; + } + } catch { + // keep raw text + } + fail(new Error(detail || `HTTP ${status} from /generate/stream`)); + }); + return; + } + + res.setEncoding('utf8'); + res.on('data', (chunk: string) => { + buffer = parseSseChunk(buffer + chunk, handleEvent); + }); + res.on('end', () => { + if (buffer.trim()) { + parseSseChunk(buffer + '\n\n', handleEvent); + } + if (doneResult) { + succeed(doneResult); + } else if (!settled) { + fail(new Error('Stream ended without a done event')); + } + }); + res.on('error', (err) => fail(err instanceof Error ? err : new Error(String(err)))); + } + ); + + req.on('timeout', () => { + req.destroy(); + fail(new Error('Streaming generation timed out')); + }); + req.on('error', (err) => fail(err instanceof Error ? err : new Error(String(err)))); + + req.write(payload); + req.end(); + }); +} From 557db3ce0a02542c7c24691c888c05c4dcc9de28 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 08:36:15 +0000 Subject: [PATCH 04/13] Implement Creer v0.4: cancel, selectable bake-ins, quality gates Add stream job_id cancellation, license/CI bake-in options with GET /bakeins, telemetry-free quality gates, and extension AbortSignal progress cancel plus bake-in QuickPicks. Co-authored-by: Sanath S Patil --- PLAN.md | 34 ++--- README.md | 149 +++++++------------- backend/app/bakeins.py | 144 ++++++++++++++++---- backend/app/generator.py | 37 ++++- backend/app/jobs.py | 76 +++++++++++ backend/app/quality.py | 102 ++++++++++++++ backend/main.py | 128 +++++++++++++++++- extension/package.json | 19 ++- extension/src/api.ts | 68 +++++++++- extension/src/scaffold.ts | 232 +++++++++++++++++++++++++++++--- extension/src/streamGenerate.ts | 170 +++++++++++++++++++++-- 11 files changed, 972 insertions(+), 187 deletions(-) create mode 100644 backend/app/jobs.py create mode 100644 backend/app/quality.py diff --git a/PLAN.md b/PLAN.md index 5b26714..0b64265 100644 --- a/PLAN.md +++ b/PLAN.md @@ -1,31 +1,35 @@ -# Creer — Final Plan (post v0.1) +# Creer — Final Plan v0.1 delivered the foundation: FastAPI planner/generator + VS Code command that writes a generated repo into the workspace (optional git init). ## v0.2 — done 1. **Preview before writing** — plan preview markdown + confirm before generate/write (`creer.previewBeforeWrite`). -2. **GitHub repo creation** — `POST /github/create-repo` + extension remote add/push (`creer.createGitHubRepo`, token settings). -3. **Curated templates** — `GET /templates` + template-anchored `/plan` & `/generate` (`backend/app/templates.py`). +2. **GitHub repo creation** — `POST /github/create-repo` + extension remote add/push. +3. **Curated templates** — `GET /templates` + template-anchored `/plan` & `/generate`. 4. **Overwrite protection** — per-file conflict detection with overwrite / skip / cancel. -5. **Chat command `/creer`** — `creer.createRepoFromChat` + `@creer` chat participant (feature-detected). - -Also in v0.2: modular extension layout (`api` / `scaffold` / `git` / `writeFiles` / `preview`), backend path/content validation, lazy OpenAI clients, extension path sandbox on write. +5. **Chat command `/creer`** — `creer.createRepoFromChat` + `@creer` chat participant. ## v0.3 — done -1. **Streaming generation** — `POST /generate/stream` (SSE) + extension `creer.useStreaming` with per-file progress and fallback to `/generate`. -2. **Local / offline backends** — `OPENAI_BASE_URL` for OpenAI-compatible servers; `CREER_OFFLINE` for template/stub-only generation. -3. **Open-source bake-ins** — every scaffold gets `LICENSE` (MIT), optional default `README.md`, and `.github/workflows/ci.yml` when missing (`backend/app/bakeins.py`). -4. **Hardening** — `GIT_ASKPASS` for push (no token on argv/URL); GitHub token in SecretStorage (`creer.setGitHubToken` / `creer.clearGitHubToken`) with deprecated settings fallback. +1. **Streaming generation** — `POST /generate/stream` (SSE) + extension progress UI. +2. **Local / offline backends** — `OPENAI_BASE_URL`, `CREER_OFFLINE`. +3. **Open-source bake-ins** — LICENSE / README / CI via `bakeins.py`. +4. **Hardening** — `GIT_ASKPASS` + SecretStorage for GitHub tokens. + +## v0.4 — done + +1. **Cancellation** — `job_id` on stream + `POST /generate/cancel`; extension AbortSignal + cancellable progress. +2. **Selectable bake-ins** — license (`mit` / `apache-2.0` / `none`) and CI presets (`auto` / `python` / `node` / `none`); `GET /bakeins`. +3. **Quality gates** — telemetry-free tree checks (`quality` on generate/done; `POST /quality`). -## v0.4 (optional next) +## v0.5 (optional next) -- Cancellation for long streaming generates -- Richer bake-in / template composition (user-selectable license, CI presets) -- Light telemetry-free quality gates on generated trees +- Diff preview of generated file contents before write +- Multi-root workspace targeting +- Template packs as installable JSON/YAML -## Non-goals (keep out of early versions) +## Non-goals - Multi-agent orchestration - Memory graphs diff --git a/README.md b/README.md index 453c8c1..2af926d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ AI-powered repo scaffolding inside your workspace. Describe an idea → Creer plans a clean structure → optionally preview/confirm → writes files into your VS Code workspace → optionally initializes git and creates a GitHub remote. -**Current version: 0.3.0** +**Current version: 0.4.0** ## Architecture @@ -21,7 +21,7 @@ creer/ - OpenAI API key **or** a local OpenAI-compatible server (`OPENAI_BASE_URL`), unless using offline template mode (`CREER_OFFLINE=1`) - VS Code / Cursor -## Backend (v0.3) +## Backend (v0.4) ```bash cd backend @@ -37,99 +37,61 @@ uvicorn main:app --reload --port 8000 | Variable | Description | |---|---| -| `OPENAI_API_KEY` | OpenAI API key (optional if using a local server via `OPENAI_BASE_URL`, or `CREER_OFFLINE=1`) | -| `OPENAI_BASE_URL` | Optional OpenAI-compatible base URL for local/offline backends (Ollama, LM Studio, vLLM, etc.). Example: `http://127.0.0.1:11434/v1` | +| `OPENAI_API_KEY` | OpenAI API key (optional if using `OPENAI_BASE_URL` or `CREER_OFFLINE=1`) | +| `OPENAI_BASE_URL` | OpenAI-compatible base URL (Ollama, LM Studio, etc.). Example: `http://127.0.0.1:11434/v1` | | `CREER_MODEL` | Model name (default `gpt-4o-mini`) | -| `CREER_OFFLINE` | Set to `1`, `true`, or `yes` for template-only / stub generation — never calls the LLM. Offline planning requires a `template_id`. | +| `CREER_OFFLINE` | `1` / `true` / `yes` for template-only stubs (planning requires `template_id`) | ### Endpoints | Method | Path | Description | |---|---|---| -| `GET` | `/health` | Liveness + version (`0.3.0`), offline flag, base URL status | -| `GET` | `/templates` | List curated starter templates | -| `POST` | `/plan` | Return a project plan (no file contents) | -| `POST` | `/generate` | Plan (or accept a plan) and generate file contents | -| `POST` | `/generate/stream` | Same as `/generate`, streamed as SSE with per-file progress | -| `POST` | `/github/create-repo` | Create a GitHub repo for the authenticated user | - -#### `POST /plan` - -```json -{ "idea": "Build a FastAPI todo app", "template_id": "fastapi-minimal" } -``` - -`template_id` is optional (required when `CREER_OFFLINE=1`). Response includes `project_name`, `stack`, `files`, and optionally `template_id` / `description`. - -#### `POST /generate` +| `GET` | `/health` | Version `0.4.0`, offline flag, model | +| `GET` | `/templates` | Curated starter templates | +| `GET` | `/bakeins` | License + CI bake-in options | +| `POST` | `/plan` | Plan only (no file contents) | +| `POST` | `/generate` | Generate files (+ bake-ins + `quality`) | +| `POST` | `/generate/stream` | SSE progress; `start` includes `job_id` | +| `POST` | `/generate/cancel` | Cancel by `job_id` | +| `POST` | `/quality` | Dry-run quality gates | +| `POST` | `/github/create-repo` | Create GitHub repo | + +#### Generate body (sync or stream) ```json { "idea": "Build a FastAPI todo app", "template_id": "fastapi-minimal", - "plan": null + "plan": null, + "job_id": null, + "bakeins": { + "license": "mit", + "ci": "auto", + "include_readme": true + } } ``` -Omit `plan` to plan then generate, or pass a prior `/plan` body to skip re-planning. Response includes `project_name`, `stack`, and `files` (path → content map). - -Every scaffold is merged with **bake-ins** after generation: - -- `LICENSE` — MIT (always ensured if missing) -- `README.md` — only if the plan did not already produce one -- `.github/workflows/ci.yml` — stack-heuristic CI workflow if missing +`bakeins.license`: `mit` | `apache-2.0` | `none` +`bakeins.ci`: `auto` | `python` | `node` | `none` -#### `POST /generate/stream` +#### Stream events -Same request body as `/generate`. Response is `text/event-stream` with JSON payloads on `data:` lines: - -| Event | Fields | Meaning | -|---|---|---| -| `start` | `project_name`, `total`, `stack` | Generation started | -| `file` | `index`, `total`, `path`, `status` (`generating` \| `done`), optional `bytes` | Per-file progress | -| `done` | `project_name`, `stack`, `files`, optional `template_id` | Final file map (includes bake-ins) | -| `error` | `detail` | Failure | - -Example: +| Event | Meaning | +|---|---| +| `start` | Includes `job_id`, `total`, `project_name` | +| `file` | Per-file `generating` / `done` | +| `done` | Full `files` map + `quality` | +| `cancelled` | Stopped via `/generate/cancel` | +| `error` | Failure detail | ```bash curl -N -X POST http://localhost:8000/generate/stream \ -H "Content-Type: application/json" \ - -H "Accept: text/event-stream" \ - -d '{"idea":"Build a FastAPI todo app","template_id":"fastapi-minimal"}' + -d '{"idea":"todo api","template_id":"fastapi-minimal"}' ``` -#### `POST /github/create-repo` - -Prefer `Authorization: Bearer `; `body.token` is also accepted. - -```json -{ "name": "my-app", "private": true, "description": "optional" } -``` - -Returns `html_url`, `clone_url`, `full_name`. - -Health check: - -```bash -curl http://localhost:8000/health -``` - -List templates: - -```bash -curl http://localhost:8000/templates -``` - -Generate: - -```bash -curl -X POST http://localhost:8000/generate \ - -H "Content-Type: application/json" \ - -d '{"idea":"Build a FastAPI todo app"}' -``` - -## Extension (v0.3) +## Extension (v0.4) ```bash cd extension @@ -137,13 +99,10 @@ npm install npm run compile ``` -In VS Code / Cursor: - -1. Open the `extension/` folder -2. Press **F5** (Run Extension) -3. In the Extension Development Host, open a workspace folder -4. Command Palette → **Creer: Create New Repo** (or **Creer: Create from Chat Prompt**) -5. Enter an idea, pick a template or AI plan, confirm the preview, and write files +1. Open `extension/` → **F5** +2. Open a workspace folder +3. **Creer: Create New Repo** +4. Pick template → license/CI → confirm preview → generate (cancellable while streaming) ### Commands @@ -151,31 +110,25 @@ In VS Code / Cursor: |---|---| | `creer.createRepo` | Creer: Create New Repo | | `creer.createRepoFromChat` | Creer: Create from Chat Prompt | -| `creer.setGitHubToken` | Creer: Set GitHub Token (SecretStorage) | +| `creer.setGitHubToken` | Creer: Set GitHub Token | | `creer.clearGitHubToken` | Creer: Clear GitHub Token | -Chat: `@creer ` (when the host supports chat participants) or run **Create from Chat Prompt** with a `/creer …` style input. - ### Settings | Setting | Default | Description | |---|---|---| | `creer.backendUrl` | `http://localhost:8000` | Backend base URL | -| `creer.initGit` | `true` | Run `git init` + initial commit after scaffolding | -| `creer.previewBeforeWrite` | `true` | Show plan preview and confirm before generating/writing | -| `creer.useStreaming` | `true` | Use SSE `/generate/stream` with per-file progress; falls back to `/generate` on failure | -| `creer.createGitHubRepo` | `false` | After scaffolding, create a GitHub remote repository | -| `creer.githubPrivate` | `true` | Create GitHub repositories as private | -| `creer.githubToken` | `""` | **Deprecated.** Prefer SecretStorage via **Creer: Set GitHub Token**. Used only as a fallback when SecretStorage is empty. | - -### GitHub token (SecretStorage) - -Tokens are stored in VS Code **SecretStorage**, not in plaintext settings: - -1. **Creer: Set GitHub Token** — prompt and store -2. **Creer: Clear GitHub Token** — remove from SecretStorage - -When creating/pushing a GitHub repo, Creer resolves the token as: SecretStorage → deprecated `creer.githubToken` setting → one-time prompt (optionally save). Push uses `GIT_ASKPASS` so the token is never embedded in the remote URL or git argv. +| `creer.initGit` | `true` | `git init` + initial commit | +| `creer.previewBeforeWrite` | `true` | Confirm plan before generate | +| `creer.useStreaming` | `true` | SSE progress (cancellable) | +| `creer.promptBakeins` | `true` | QuickPick license/CI each run | +| `creer.license` | `mit` | Used when not prompting | +| `creer.ciPreset` | `auto` | Used when not prompting | +| `creer.createGitHubRepo` | `false` | Create GitHub remote after scaffold | +| `creer.githubPrivate` | `true` | Private GitHub repos | +| `creer.githubToken` | `""` | **Deprecated** — use SecretStorage | + +GitHub push uses `GIT_ASKPASS` (token never on argv/URL). ## License diff --git a/backend/app/bakeins.py b/backend/app/bakeins.py index 7f28d0e..c4e1ae9 100644 --- a/backend/app/bakeins.py +++ b/backend/app/bakeins.py @@ -1,7 +1,46 @@ -"""Open-source bake-ins merged into every scaffold (LICENSE, README, CI).""" +"""Open-source bake-ins merged into scaffolds (LICENSE, README, CI).""" from __future__ import annotations +from typing import Any, Literal + +LicenseId = Literal["mit", "apache-2.0", "none"] +CiId = Literal["auto", "python", "node", "none"] + + +def list_bakein_options() -> dict[str, list[dict[str, str]]]: + return { + "licenses": [ + {"id": "mit", "name": "MIT"}, + {"id": "apache-2.0", "name": "Apache 2.0"}, + {"id": "none", "name": "No license"}, + ], + "ci": [ + {"id": "auto", "name": "Auto-detect"}, + {"id": "python", "name": "Python"}, + {"id": "node", "name": "Node"}, + {"id": "none", "name": "No CI"}, + ], + } + + +def _normalize_options(options: dict[str, Any] | None) -> dict[str, Any]: + opts = options or {} + license_id = opts.get("license") or "mit" + ci_id = opts.get("ci") or "auto" + include_readme = opts.get("include_readme") + if include_readme is None: + include_readme = True + if license_id not in ("mit", "apache-2.0", "none"): + license_id = "mit" + if ci_id not in ("auto", "python", "node", "none"): + ci_id = "auto" + return { + "license": license_id, + "ci": ci_id, + "include_readme": bool(include_readme), + } + def _mit_license(project_name: str) -> str: holder = project_name.strip() if project_name and project_name.strip() else "Creer Scaffold" @@ -29,6 +68,28 @@ def _mit_license(project_name: str) -> str: """ +def _apache_license(project_name: str) -> str: + holder = project_name.strip() if project_name and project_name.strip() else "Creer Scaffold" + return f"""Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +Copyright 2026 {holder} + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + + def _default_readme(plan: dict) -> str: name = plan.get("project_name") or "project" stack = plan.get("stack") or "" @@ -45,7 +106,7 @@ def _default_readme(plan: dict) -> str: [ "## Getting started", "", - "This project was generated with [Creer](https://github.com/).", + "This project was generated with Creer.", "Install dependencies and follow stack-specific docs in this repo.", "", ] @@ -53,22 +114,8 @@ def _default_readme(plan: dict) -> str: return "\n".join(lines) -def _ci_workflow(plan: dict) -> str: - stack = (plan.get("stack") or "").lower() - paths = " ".join(plan.get("files") or []).lower() - blob = f"{stack} {paths}" - - is_python = any( - tok in blob - for tok in ("python", "fastapi", "uvicorn", "pytest", "requirements.txt", "pyproject.toml") - ) - is_node = any( - tok in blob - for tok in ("node", "express", "next", "npm", "package.json", "react") - ) - - if is_python: - return """name: CI +def _ci_python() -> str: + return """name: CI on: push: @@ -93,8 +140,9 @@ def _ci_workflow(plan: dict) -> str: run: pytest -q || echo "No tests yet — scaffold CI ok" """ - if is_node: - return """name: CI + +def _ci_node() -> str: + return """name: CI on: push: @@ -116,6 +164,8 @@ def _ci_workflow(plan: dict) -> str: run: npm test --if-present """ + +def _ci_generic() -> str: return """name: CI on: @@ -133,25 +183,63 @@ def _ci_workflow(plan: dict) -> str: """ -def apply_bakeins(plan: dict, files: dict[str, str]) -> dict[str, str]: +def _ci_workflow(plan: dict, ci_id: str) -> str | None: + if ci_id == "none": + return None + if ci_id == "python": + return _ci_python() + if ci_id == "node": + return _ci_node() + + # auto + stack = (plan.get("stack") or "").lower() + paths = " ".join(plan.get("files") or []).lower() + blob = f"{stack} {paths}" + + is_python = any( + tok in blob + for tok in ("python", "fastapi", "uvicorn", "pytest", "requirements.txt", "pyproject.toml") + ) + is_node = any( + tok in blob for tok in ("node", "express", "next", "npm", "package.json", "react") + ) + + if is_python: + return _ci_python() + if is_node: + return _ci_node() + return _ci_generic() + + +def apply_bakeins( + plan: dict, + files: dict[str, str], + options: dict[str, Any] | None = None, +) -> dict[str, str]: """ Merge open-source bake-ins into generated files. - - LICENSE is always ensured (MIT, 2026). - - README.md is created only if missing (never overwrite AI/template README). - - .github/workflows/ci.yml is added if missing (stack heuristics). + - LICENSE per options.license (mit | apache-2.0 | none); never overwrite existing + - README.md created only if missing and include_readme + - .github/workflows/ci.yml per options.ci when missing """ + opts = _normalize_options(options) out = dict(files) project_name = plan.get("project_name") or "Creer Scaffold" - if "LICENSE" not in out: - out["LICENSE"] = _mit_license(project_name) + if opts["license"] != "none" and "LICENSE" not in out: + if opts["license"] == "apache-2.0": + out["LICENSE"] = _apache_license(project_name) + else: + out["LICENSE"] = _mit_license(project_name) - if "README.md" not in out: + if opts["include_readme"] and "README.md" not in out: out["README.md"] = _default_readme(plan) ci_path = ".github/workflows/ci.yml" if ci_path not in out: - out[ci_path] = _ci_workflow(plan) + ci_body = _ci_workflow(plan, opts["ci"]) + if ci_body is not None: + out[ci_path] = ci_body return out diff --git a/backend/app/generator.py b/backend/app/generator.py index db78c02..7dc8c59 100644 --- a/backend/app/generator.py +++ b/backend/app/generator.py @@ -3,7 +3,7 @@ from __future__ import annotations import json -from collections.abc import Iterator +from collections.abc import Callable, Iterator from typing import Any from config import CREER_OFFLINE, MODEL, OPENAI_API_KEY, OPENAI_BASE_URL @@ -203,13 +203,17 @@ def _generate_one_llm(file_path: str, plan: dict) -> str: return _strip_fences(content) -def generate_files_iter(plan: dict) -> Iterator[tuple[dict[str, Any], dict[str, str]]]: +def generate_files_iter( + plan: dict, + should_cancel: Callable[[], bool] | None = None, +) -> Iterator[tuple[dict[str, Any], dict[str, str]]]: """ Yield (event_dict, partial_files) progress while generating. Events: - file / generating - file / done (with bytes) + - cancelled (when should_cancel returns True) Does not yield start/done/error — caller owns those. """ files_output: dict[str, str] = {} @@ -218,6 +222,16 @@ def generate_files_iter(plan: dict) -> Iterator[tuple[dict[str, Any], dict[str, offline = _use_offline(plan) for index, file_path in enumerate(file_list, start=1): + if should_cancel and should_cancel(): + yield ( + { + "event": "cancelled", + "detail": "Cancelled by user", + }, + dict(files_output), + ) + return + yield ( { "event": "file", @@ -229,6 +243,16 @@ def generate_files_iter(plan: dict) -> Iterator[tuple[dict[str, Any], dict[str, dict(files_output), ) + if should_cancel and should_cancel(): + yield ( + { + "event": "cancelled", + "detail": "Cancelled by user", + }, + dict(files_output), + ) + return + if offline: content = _stub_content(file_path, plan) else: @@ -248,9 +272,14 @@ def generate_files_iter(plan: dict) -> Iterator[tuple[dict[str, Any], dict[str, ) -def generate_files(plan: dict) -> dict[str, str]: +def generate_files( + plan: dict, + should_cancel: Callable[[], bool] | None = None, +) -> dict[str, str]: """Generate file contents one-by-one from a project plan.""" files_output: dict[str, str] = {} - for _event, partial in generate_files_iter(plan): + for event, partial in generate_files_iter(plan, should_cancel=should_cancel): files_output = partial + if event.get("event") == "cancelled": + raise ValueError(event.get("detail") or "Cancelled by user") return files_output diff --git a/backend/app/jobs.py b/backend/app/jobs.py new file mode 100644 index 0000000..07c569c --- /dev/null +++ b/backend/app/jobs.py @@ -0,0 +1,76 @@ +"""In-memory generation job registry for stream cancellation.""" + +from __future__ import annotations + +import threading +import time +import uuid +from dataclasses import dataclass, field + +# Jobs older than this are eligible for cleanup. +_TTL_SECONDS = 30 * 60 + + +@dataclass +class _Job: + cancelled: bool = False + created_at: float = field(default_factory=time.time) + + +_lock = threading.Lock() +_jobs: dict[str, _Job] = {} + + +def _purge_expired(now: float | None = None) -> None: + now = now if now is not None else time.time() + expired = [jid for jid, job in _jobs.items() if now - job.created_at > _TTL_SECONDS] + for jid in expired: + del _jobs[jid] + + +def create_job(job_id: str | None = None) -> str: + """Register a new job (or ensure an existing id) and return its id. + + If the id was already cancelled (cancel-before-start), keep it cancelled. + """ + jid = (job_id or "").strip() or uuid.uuid4().hex + with _lock: + _purge_expired() + existing = _jobs.get(jid) + if existing is not None: + # Refresh TTL but preserve cancelled flag. + existing.created_at = time.time() + return jid + _jobs[jid] = _Job() + return jid + + +def is_cancelled(job_id: str | None) -> bool: + if not job_id: + return False + with _lock: + job = _jobs.get(job_id) + return bool(job and job.cancelled) + + +def cancel_job(job_id: str) -> bool: + """Mark a job cancelled. Returns True if the job existed (or was created as cancelled).""" + jid = (job_id or "").strip() + if not jid: + return False + with _lock: + _purge_expired() + job = _jobs.get(jid) + if job is None: + # Allow cancel-before-start races: register already-cancelled. + _jobs[jid] = _Job(cancelled=True) + return True + job.cancelled = True + return True + + +def finish_job(job_id: str | None) -> None: + if not job_id: + return + with _lock: + _jobs.pop(job_id, None) diff --git a/backend/app/quality.py b/backend/app/quality.py new file mode 100644 index 0000000..be6f534 --- /dev/null +++ b/backend/app/quality.py @@ -0,0 +1,102 @@ +"""Telemetry-free quality gates for generated project trees.""" + +from __future__ import annotations + +from typing import Any + +MANIFEST_NAMES = { + "requirements.txt", + "pyproject.toml", + "package.json", + "Pipfile", + "go.mod", + "Cargo.toml", + "composer.json", +} + + +def run_quality_gates( + plan: dict, + files: dict[str, str], + *, + expect_license: bool = True, +) -> list[dict[str, Any]]: + """ + Return light static issues for a generated tree. + + Severities: error | warning | info + """ + issues: list[dict[str, Any]] = [] + + paths = list(files.keys()) + seen: set[str] = set() + for path in paths: + if path in seen: + issues.append( + { + "code": "path_duplicate", + "severity": "error", + "path": path, + "message": f"Duplicate path in generated tree: {path}", + } + ) + seen.add(path) + + for path, content in files.items(): + if not isinstance(content, str) or not content.strip(): + issues.append( + { + "code": "empty_file", + "severity": "warning", + "path": path, + "message": f"File is empty or whitespace-only: {path}", + } + ) + + if "README.md" not in files and "README" not in files: + issues.append( + { + "code": "missing_readme", + "severity": "warning", + "message": "No README.md in generated tree", + } + ) + + if expect_license and "LICENSE" not in files: + issues.append( + { + "code": "missing_license", + "severity": "warning", + "message": "LICENSE missing after bake-ins", + } + ) + + lower_names = {p.split("/")[-1] for p in files} + if not (lower_names & MANIFEST_NAMES): + issues.append( + { + "code": "no_manifest", + "severity": "info", + "message": "No dependency manifest (requirements.txt / package.json / …)", + } + ) + + # Plan vs files: warn if plan listed paths that weren't generated + planned = plan.get("files") or [] + if isinstance(planned, list): + missing = [p for p in planned if isinstance(p, str) and p not in files] + for path in missing[:20]: + issues.append( + { + "code": "missing_planned_file", + "severity": "warning", + "path": path, + "message": f"Planned file was not generated: {path}", + } + ) + + return issues + + +def has_errors(issues: list[dict[str, Any]]) -> bool: + return any(i.get("severity") == "error" for i in issues) diff --git a/backend/main.py b/backend/main.py index daf2b71..10d7017 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,23 +1,26 @@ -"""Creer FastAPI application — plan, generate, stream, GitHub helpers.""" +"""Creer FastAPI application — plan, generate, stream, cancel, GitHub helpers.""" from __future__ import annotations import json from collections.abc import Iterator +from typing import Any, Literal from fastapi import FastAPI, Header, HTTPException from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field from config import CREER_OFFLINE, MODEL, OPENAI_BASE_URL -from app.bakeins import apply_bakeins +from app.bakeins import apply_bakeins, list_bakein_options from app.planner import plan_project from app.generator import generate_files, generate_files_iter from app.validator import validate_plan, validate_files from app.templates import list_templates, get_template from app.github import create_github_repo +from app.jobs import cancel_job, create_job, finish_job, is_cancelled +from app.quality import has_errors, run_quality_gates -VERSION = "0.3.0" +VERSION = "0.4.0" app = FastAPI(title="Creer", version=VERSION) @@ -35,10 +38,28 @@ class PlanBody(BaseModel): description: str | None = None +class BakeinOptions(BaseModel): + license: Literal["mit", "apache-2.0", "none"] = "mit" + ci: Literal["auto", "python", "node", "none"] = "auto" + include_readme: bool = True + + class GenerateRequest(BaseModel): idea: str = Field(..., min_length=3, max_length=4000) template_id: str | None = None plan: PlanBody | None = None + job_id: str | None = None + bakeins: BakeinOptions | None = None + + +class CancelRequest(BaseModel): + job_id: str = Field(..., min_length=1) + + +class QualityRequest(BaseModel): + plan: PlanBody | None = None + files: dict[str, str] + bakeins: BakeinOptions | None = None class GitHubCreateRepoRequest(BaseModel): @@ -65,6 +86,16 @@ def _resolve_plan(request: GenerateRequest) -> dict: return plan +def _bakein_dict(bakeins: BakeinOptions | None) -> dict[str, Any] | None: + return bakeins.model_dump() if bakeins is not None else None + + +def _expect_license(bakeins: BakeinOptions | None) -> bool: + if bakeins is None: + return True + return bakeins.license != "none" + + def _sse(data: dict) -> str: return f"data: {json.dumps(data, ensure_ascii=False)}\n\n" @@ -85,6 +116,11 @@ def templates(): return {"templates": list_templates()} +@app.get("/bakeins") +def bakeins(): + return list_bakein_options() + + @app.post("/plan") def plan_only(request: PlanRequest): """Return a project plan without generating file contents.""" @@ -112,20 +148,35 @@ def plan_only(request: PlanRequest): @app.post("/generate") def generate_project(request: GenerateRequest): + job_id = create_job(request.job_id) try: + if is_cancelled(job_id): + raise ValueError("Cancelled by user") plan = _resolve_plan(request) - files = generate_files(plan) - files = apply_bakeins(plan, files) + files = generate_files(plan, should_cancel=lambda: is_cancelled(job_id)) + files = apply_bakeins(plan, files, _bakein_dict(request.bakeins)) validate_files(files) + quality = run_quality_gates( + plan, files, expect_license=_expect_license(request.bakeins) + ) + if has_errors(quality): + detail = "; ".join( + f"{i.get('code')}: {i.get('message')}" for i in quality if i.get("severity") == "error" + ) + raise ValueError(f"Quality gate failed: {detail}") except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc except Exception as exc: raise HTTPException(status_code=500, detail=f"Generation failed: {exc}") from exc + finally: + finish_job(job_id) result = { "project_name": plan["project_name"], "stack": plan.get("stack"), "files": files, + "quality": quality, + "job_id": job_id, } if plan.get("template_id"): result["template_id"] = plan["template_id"] @@ -135,15 +186,27 @@ def generate_project(request: GenerateRequest): @app.post("/generate/stream") def generate_project_stream(request: GenerateRequest): """Stream generation progress as Server-Sent Events (JSON data lines).""" + job_id = create_job(request.job_id) def event_stream() -> Iterator[str]: plan: dict | None = None try: + if is_cancelled(job_id): + yield _sse( + { + "event": "cancelled", + "job_id": job_id, + "detail": "Cancelled by user", + } + ) + return + plan = _resolve_plan(request) file_list = list(plan["files"]) yield _sse( { "event": "start", + "job_id": job_id, "project_name": plan["project_name"], "total": len(file_list), "stack": plan.get("stack") or "", @@ -151,18 +214,47 @@ def event_stream() -> Iterator[str]: ) files: dict[str, str] = {} - for event, partial in generate_files_iter(plan): + cancelled = False + for event, partial in generate_files_iter( + plan, should_cancel=lambda: is_cancelled(job_id) + ): files = partial + if event.get("event") == "cancelled": + cancelled = True + yield _sse( + { + "event": "cancelled", + "job_id": job_id, + "detail": event.get("detail") or "Cancelled by user", + } + ) + break yield _sse(event) - files = apply_bakeins(plan, files) + if cancelled: + return + + files = apply_bakeins(plan, files, _bakein_dict(request.bakeins)) validate_files(files) + quality = run_quality_gates( + plan, files, expect_license=_expect_license(request.bakeins) + ) + if has_errors(quality): + detail = "; ".join( + f"{i.get('code')}: {i.get('message')}" + for i in quality + if i.get("severity") == "error" + ) + yield _sse({"event": "error", "detail": f"Quality gate failed: {detail}", "quality": quality}) + return done: dict = { "event": "done", + "job_id": job_id, "project_name": plan["project_name"], "files": files, "stack": plan.get("stack") or "", + "quality": quality, } if plan.get("template_id"): done["template_id"] = plan["template_id"] @@ -171,6 +263,8 @@ def event_stream() -> Iterator[str]: yield _sse({"event": "error", "detail": str(exc)}) except Exception as exc: yield _sse({"event": "error", "detail": f"Generation failed: {exc}"}) + finally: + finish_job(job_id) return StreamingResponse( event_stream(), @@ -183,6 +277,26 @@ def event_stream() -> Iterator[str]: ) +@app.post("/generate/cancel") +def generate_cancel(request: CancelRequest): + ok = cancel_job(request.job_id) + if not ok: + raise HTTPException(status_code=400, detail="Invalid job_id") + return {"cancelled": True, "job_id": request.job_id} + + +@app.post("/quality") +def quality_check(request: QualityRequest): + plan = request.plan.model_dump() if request.plan is not None else {"files": list(request.files.keys())} + plan.setdefault("files", list(request.files.keys())) + quality = run_quality_gates( + plan, + request.files, + expect_license=_expect_license(request.bakeins), + ) + return {"quality": quality, "ok": not has_errors(quality)} + + @app.post("/github/create-repo") def github_create_repo( request: GitHubCreateRepoRequest, diff --git a/extension/package.json b/extension/package.json index 09fe82e..d412ff4 100644 --- a/extension/package.json +++ b/extension/package.json @@ -2,7 +2,7 @@ "name": "creer", "displayName": "Creer", "description": "AI-powered repo scaffolding inside your workspace", - "version": "0.3.0", + "version": "0.4.0", "publisher": "creer", "engines": { "vscode": "^1.85.0" @@ -79,6 +79,23 @@ "type": "boolean", "default": true, "description": "Use SSE streaming (/generate/stream) with per-file progress; falls back to /generate on failure" + }, + "creer.license": { + "type": "string", + "default": "mit", + "enum": ["mit", "apache-2.0", "none"], + "description": "License bake-in for generated projects" + }, + "creer.ciPreset": { + "type": "string", + "default": "auto", + "enum": ["auto", "python", "node", "none"], + "description": "CI workflow bake-in preset for generated projects" + }, + "creer.promptBakeins": { + "type": "boolean", + "default": true, + "description": "When true, QuickPick license and CI each run; when false, use creer.license and creer.ciPreset" } } } diff --git a/extension/src/api.ts b/extension/src/api.ts index f2dd6bf..49f4182 100644 --- a/extension/src/api.ts +++ b/extension/src/api.ts @@ -17,11 +17,38 @@ export interface PlanResponse { description?: string; } +export type LicenseBakein = 'mit' | 'apache-2.0' | 'none'; +export type CiBakein = 'auto' | 'python' | 'node' | 'none'; + +export interface BakeinOptions { + license: LicenseBakein; + ci: CiBakein; + include_readme?: boolean; +} + +export interface BakeinChoice { + id: string; + name: string; +} + +export interface BakeinsResponse { + licenses: BakeinChoice[]; + ci: BakeinChoice[]; +} + +export interface QualityIssue { + code: string; + severity: string; + message: string; + path?: string; +} + export interface GenerateResponse { project_name: string; stack?: string; files: Record; template_id?: string; + quality?: QualityIssue[]; } export interface GitHubCreateRepoResponse { @@ -30,6 +57,13 @@ export interface GitHubCreateRepoResponse { full_name: string; } +export interface GenerateOptions { + templateId?: string; + plan?: PlanResponse; + jobId?: string; + bakeins?: BakeinOptions; +} + function getBackendUrl(): string { const config = vscode.workspace.getConfiguration('creer'); return (config.get('backendUrl') || 'http://localhost:8000').replace(/\/$/, ''); @@ -60,6 +94,27 @@ export async function fetchTemplates(): Promise { return response.data.templates ?? []; } +export async function fetchBakeins(): Promise { + const backendUrl = getBackendUrl(); + const response = await axios.get(`${backendUrl}/bakeins`, { + timeout: 30_000, + }); + return { + licenses: response.data.licenses ?? [], + ci: response.data.ci ?? [], + }; +} + +export async function postCancel(jobId: string): Promise<{ cancelled: true }> { + const backendUrl = getBackendUrl(); + const response = await axios.post<{ cancelled: true }>( + `${backendUrl}/generate/cancel`, + { job_id: jobId }, + { timeout: 30_000 } + ); + return response.data; +} + export async function postPlan(idea: string, templateId?: string): Promise { const backendUrl = getBackendUrl(); const body: { idea: string; template_id?: string } = { idea }; @@ -74,13 +129,15 @@ export async function postPlan(idea: string, templateId?: string): Promise { const backendUrl = getBackendUrl(); const body: { idea: string; template_id?: string; plan?: PlanResponse; + job_id?: string; + bakeins?: BakeinOptions; } = { idea }; if (options?.templateId) { body.template_id = options.templateId; @@ -88,6 +145,12 @@ export async function postGenerate( if (options?.plan) { body.plan = options.plan; } + if (options?.jobId) { + body.job_id = options.jobId; + } + if (options?.bakeins) { + body.bakeins = options.bakeins; + } const response = await axios.post(`${backendUrl}/generate`, body, { timeout: 300_000, }); @@ -95,7 +158,7 @@ export async function postGenerate( } /** Re-export streaming generate for callers that import from api. */ -export { streamGenerate } from './streamGenerate'; +export { streamGenerate, CancelledError, isCancellationError } from './streamGenerate'; export type { StreamProgressEvent, StreamGenerateOptions, @@ -103,6 +166,7 @@ export type { StreamFileEvent, StreamDoneEvent, StreamErrorEvent, + StreamCancelledEvent, } from './streamGenerate'; export async function createGitHubRepo( diff --git a/extension/src/scaffold.ts b/extension/src/scaffold.ts index a9d1128..5ed6fc4 100644 --- a/extension/src/scaffold.ts +++ b/extension/src/scaffold.ts @@ -2,18 +2,27 @@ import * as path from 'path'; import * as vscode from 'vscode'; import { createGitHubRepo, + fetchBakeins, fetchTemplates, formatAxiosError, postGenerate, postPlan, + type BakeinOptions, + type CiBakein, type GenerateResponse, + type LicenseBakein, type PlanResponse, + type QualityIssue, type Template, } from './api'; import { addRemoteAndPush, ensureGitRepo, initGit } from './git'; import { showPlanPreviewAndConfirm } from './preview'; import { resolveGitHubToken } from './secrets'; -import { streamGenerate, type StreamProgressEvent } from './streamGenerate'; +import { + isCancellationError, + streamGenerate, + type StreamProgressEvent, +} from './streamGenerate'; import { assertSafeProjectName, findConflicts, @@ -30,6 +39,19 @@ export interface ScaffoldOptions { fromChat?: boolean; } +const LICENSE_FALLBACK: Array<{ id: LicenseBakein; name: string }> = [ + { id: 'mit', name: 'MIT' }, + { id: 'apache-2.0', name: 'Apache-2.0' }, + { id: 'none', name: 'None' }, +]; + +const CI_FALLBACK: Array<{ id: CiBakein; name: string }> = [ + { id: 'auto', name: 'Auto (detect from stack)' }, + { id: 'python', name: 'Python' }, + { id: 'node', name: 'Node' }, + { id: 'none', name: 'None' }, +]; + function stripCreerPrefix(raw: string): string { return raw.replace(/^\s*\/creer\b\s*/i, '').trim(); } @@ -88,6 +110,90 @@ async function pickTemplate(templates: Template[]): Promise { + const config = vscode.workspace.getConfiguration('creer'); + const promptBakeins = config.get('promptBakeins') ?? true; + const settingsLicense = config.get('license') ?? 'mit'; + const settingsCi = config.get('ciPreset') ?? 'auto'; + + const defaultLicense: LicenseBakein = isLicenseBakein(settingsLicense) + ? settingsLicense + : 'mit'; + const defaultCi: CiBakein = isCiBakein(settingsCi) ? settingsCi : 'auto'; + + if (!promptBakeins) { + return { license: defaultLicense, ci: defaultCi }; + } + + let licenses: Array<{ id: string; name: string }> = LICENSE_FALLBACK.map((l) => ({ + id: l.id, + name: l.name, + })); + let ciOptions: Array<{ id: string; name: string }> = CI_FALLBACK.map((c) => ({ + id: c.id, + name: c.name, + })); + + try { + const remote = await fetchBakeins(); + if (remote.licenses.length > 0) { + licenses = remote.licenses; + } + if (remote.ci.length > 0) { + ciOptions = remote.ci; + } + } catch { + // Use built-in fallbacks when /bakeins is unavailable. + } + + const licenseItems: Array = licenses.map( + (l) => ({ + label: l.name, + description: l.id, + value: l.id, + }) + ); + const licensePick = await vscode.window.showQuickPick(licenseItems, { + placeHolder: `Select a license (default: ${defaultLicense})`, + ignoreFocusOut: true, + matchOnDescription: true, + }); + if (!licensePick) { + return null; + } + + const ciItems: Array = ciOptions.map((c) => ({ + label: c.name, + description: c.id, + value: c.id, + })); + const ciPick = await vscode.window.showQuickPick(ciItems, { + placeHolder: `Select a CI preset (default: ${defaultCi})`, + ignoreFocusOut: true, + matchOnDescription: true, + }); + if (!ciPick) { + return null; + } + + const license = isLicenseBakein(licensePick.value) ? licensePick.value : defaultLicense; + const ci = isCiBakein(ciPick.value) ? ciPick.value : defaultCi; + return { license, ci }; +} + async function maybeCreateGitHubRemote( context: vscode.ExtensionContext, projectPath: string, @@ -153,7 +259,10 @@ function reportStreamProgress( ev: StreamProgressEvent ): void { if (ev.event === 'start') { - progress.report({ message: `Starting ${ev.project_name} (${ev.total} files)…` }); + const jobNote = ev.job_id ? ` · job ${ev.job_id}` : ''; + progress.report({ + message: `Starting ${ev.project_name} (${ev.total} files)${jobNote}…`, + }); return; } if (ev.event === 'file') { @@ -164,15 +273,58 @@ function reportStreamProgress( } } +function reportQualityIssues(quality: QualityIssue[] | undefined): void { + if (!quality || quality.length === 0) { + return; + } + + const errors = quality.filter((q) => { + const s = (q.severity || '').toLowerCase(); + return s === 'error' || s === 'critical' || s === 'fatal'; + }); + const warnings = quality.filter((q) => { + const s = (q.severity || '').toLowerCase(); + return s === 'warning' || s === 'warn'; + }); + const other = quality.length - errors.length - warnings.length; + + if (errors.length > 0) { + const sample = errors + .slice(0, 3) + .map((e) => (e.path ? `${e.code} (${e.path})` : e.code)) + .join(', '); + const more = errors.length > 3 ? ` (+${errors.length - 3} more)` : ''; + void vscode.window.showErrorMessage( + `Creer quality gates reported ${errors.length} error(s)` + + (warnings.length ? `, ${warnings.length} warning(s)` : '') + + `: ${sample}${more}` + ); + return; + } + + const parts: string[] = []; + if (warnings.length) { + parts.push(`${warnings.length} warning(s)`); + } + if (other > 0) { + parts.push(`${other} other issue(s)`); + } + void vscode.window.showWarningMessage( + `Creer quality gates: ${parts.join(', ') || `${quality.length} issue(s)`}.` + ); +} + async function generateWithOptionalStream( idea: string, plan: PlanResponse, templateId: string | undefined, - useStreaming: boolean + useStreaming: boolean, + bakeins: BakeinOptions ): Promise { const genOpts = { templateId: plan.template_id ?? templateId, plan, + bakeins, }; if (!useStreaming) { @@ -191,17 +343,31 @@ async function generateWithOptionalStream( { location: vscode.ProgressLocation.Notification, title: 'Creer: generating project…', - cancellable: false, + cancellable: true, }, - async (progress) => - streamGenerate({ - idea, - templateId: genOpts.templateId, - plan: genOpts.plan, - onProgress: (ev) => reportStreamProgress(progress, ev), - }) + async (progress, cancellationToken) => { + const controller = new AbortController(); + const sub = cancellationToken.onCancellationRequested(() => { + controller.abort(); + }); + try { + return await streamGenerate({ + idea, + templateId: genOpts.templateId, + plan: genOpts.plan, + bakeins: genOpts.bakeins, + signal: controller.signal, + onProgress: (ev) => reportStreamProgress(progress, ev), + }); + } finally { + sub.dispose(); + } + } ); } catch (err) { + if (isCancellationError(err)) { + throw err; + } const message = formatAxiosError(err, 'Streaming generate failed'); vscode.window.showWarningMessage( `Streaming failed (${message}); falling back to non-streaming generate.` @@ -293,13 +459,31 @@ export async function runScaffoldFlow(options: ScaffoldOptions): Promise { } } - // 4) Generate (streaming with progress when enabled) - const generated = await generateWithOptionalStream( - idea, - plan, - templateId, - useStreaming - ); + // 4) Bake-ins (license / CI) + const bakeins = await resolveBakeins(); + if (!bakeins) { + return; + } + + // 5) Generate (streaming with cancellable progress when enabled) + let generated: GenerateResponse; + try { + generated = await generateWithOptionalStream( + idea, + plan, + templateId, + useStreaming, + bakeins + ); + } catch (err) { + if (isCancellationError(err)) { + void vscode.window.showInformationMessage('Creer generation cancelled.'); + return; + } + throw err; + } + + reportQualityIssues(generated.quality); const projectName = generated.project_name || plan.project_name; const files = generated.files; @@ -331,21 +515,21 @@ export async function runScaffoldFlow(options: ScaffoldOptions): Promise { return; } - // 5) Conflict resolution + // 6) Conflict resolution const conflicts = findConflicts(projectPath, files); const resolution = await resolveConflicts(conflicts); if (resolution === 'cancel') { return; } - // 6) Write + // 7) Write const { written, skipped } = writeProjectFiles(projectPath, files, resolution); if (written === 0 && skipped === 0) { vscode.window.showWarningMessage('No files were written.'); return; } - // 7) Git init + // 8) Git init if (shouldInitGit) { try { await initGit(projectPath); @@ -355,7 +539,7 @@ export async function runScaffoldFlow(options: ScaffoldOptions): Promise { } } - // 8) GitHub remote (optional) + // 9) GitHub remote (optional) await maybeCreateGitHubRemote(context, projectPath, projectName, idea); const skipNote = skipped > 0 ? ` (${skipped} existing skipped)` : ''; @@ -363,6 +547,10 @@ export async function runScaffoldFlow(options: ScaffoldOptions): Promise { `Project ${projectName} created at ${projectPath}${skipNote}.` ); } catch (err) { + if (isCancellationError(err)) { + void vscode.window.showInformationMessage('Creer generation cancelled.'); + return; + } const message = formatAxiosError(err, 'Creer failed'); vscode.window.showErrorMessage(`Creer failed: ${message}`); } diff --git a/extension/src/streamGenerate.ts b/extension/src/streamGenerate.ts index 522832a..01a260e 100644 --- a/extension/src/streamGenerate.ts +++ b/extension/src/streamGenerate.ts @@ -1,14 +1,16 @@ +import axios from 'axios'; import * as http from 'http'; import * as https from 'https'; import { URL } from 'url'; import * as vscode from 'vscode'; -import type { GenerateResponse, PlanResponse } from './api'; +import type { BakeinOptions, GenerateResponse, PlanResponse, QualityIssue } from './api'; export type StreamStartEvent = { event: 'start'; project_name: string; total: number; stack: string; + job_id?: string; }; export type StreamFileEvent = { @@ -26,6 +28,7 @@ export type StreamDoneEvent = { stack: string; files: Record; template_id?: string; + quality?: QualityIssue[]; }; export type StreamErrorEvent = { @@ -33,19 +36,55 @@ export type StreamErrorEvent = { detail: string; }; +export type StreamCancelledEvent = { + event: 'cancelled'; + job_id: string; + detail: string; +}; + export type StreamProgressEvent = | StreamStartEvent | StreamFileEvent | StreamDoneEvent - | StreamErrorEvent; + | StreamErrorEvent + | StreamCancelledEvent; export interface StreamGenerateOptions { idea: string; templateId?: string; plan?: PlanResponse; + bakeins?: BakeinOptions; + jobId?: string; + signal?: AbortSignal; + onJobId?: (jobId: string) => void; onProgress?: (event: StreamProgressEvent) => void; } +/** Thrown when the user or AbortSignal cancels streaming generation. */ +export class CancelledError extends Error { + readonly jobId?: string; + + constructor(message = 'Generation cancelled', jobId?: string) { + super(message); + this.name = 'AbortError'; + this.jobId = jobId; + Object.setPrototypeOf(this, new.target.prototype); + } +} + +export function isCancellationError(err: unknown): boolean { + if (!err || typeof err !== 'object') { + return false; + } + const e = err as { name?: string; code?: string }; + return ( + err instanceof CancelledError || + e.name === 'AbortError' || + e.name === 'CancelledError' || + e.code === 'ERR_CANCELED' + ); +} + function getBackendUrl(): string { const config = vscode.workspace.getConfiguration('creer'); return (config.get('backendUrl') || 'http://localhost:8000').replace(/\/$/, ''); @@ -90,6 +129,7 @@ function parseSseChunk( /** * POST /generate/stream and parse SSE (`data: {...}\\n\\n`). * Uses Node http/https so streaming works in the VS Code extension CommonJS host. + * Supports AbortSignal: posts /generate/cancel with job_id (when known) and destroys the request. */ export function streamGenerate(options: StreamGenerateOptions): Promise { const backendUrl = getBackendUrl(); @@ -98,6 +138,8 @@ export function streamGenerate(options: StreamGenerateOptions): Promise { if (settled) { return; } settled = true; + cleanupAbort(); reject(err); }; @@ -127,21 +179,87 @@ export function streamGenerate(options: StreamGenerateOptions): Promise { + try { + req.destroy(); + } catch { + // ignore + } + }; + + const requestBackendCancel = () => { + if (cancelPosted || !jobId) { + return; + } + cancelPosted = true; + // Inline cancel call to avoid a circular import with api.ts. + void axios + .post( + `${backendUrl}/generate/cancel`, + { job_id: jobId }, + { timeout: 30_000 } + ) + .catch(() => { + // Best-effort; local abort still tears down the HTTP stream. + }); + }; + + const abortNow = () => { + requestBackendCancel(); + destroyRequest(); + fail(new CancelledError('Generation cancelled', jobId)); + }; + + const onAbort = () => { + abortNow(); + }; + + const cleanupAbort = () => { + if (options.signal) { + options.signal.removeEventListener('abort', onAbort); + } + }; + + if (options.signal?.aborted) { + // No request yet — reject immediately (and cancel if we already have a job id). + requestBackendCancel(); + fail(new CancelledError('Generation cancelled', jobId)); + return; + } const handleEvent = (ev: StreamProgressEvent) => { options.onProgress?.(ev); + if (ev.event === 'start') { + if (ev.job_id) { + jobId = ev.job_id; + options.onJobId?.(ev.job_id); + // Race: user may have aborted between request start and start event. + if (options.signal?.aborted) { + abortNow(); + } + } + return; + } + + if (ev.event === 'cancelled') { + destroyRequest(); + fail( + new CancelledError( + ev.detail || 'Generation cancelled', + ev.job_id || jobId + ) + ); + return; + } + if (ev.event === 'error') { // Stop reading further events; destroy the socket so we do not hang. - try { - req.destroy(); - } catch { - // ignore - } + destroyRequest(); fail(new Error(ev.detail || 'Streaming generation failed')); return; } @@ -155,6 +273,9 @@ export function streamGenerate(options: StreamGenerateOptions): Promise fail(err instanceof Error ? err : new Error(String(err)))); + res.on('error', (err) => { + if (settled) { + return; + } + // Destroy after abort often surfaces as a socket error — treat as cancel. + if (options.signal?.aborted) { + fail(new CancelledError('Generation cancelled', jobId)); + return; + } + fail(err instanceof Error ? err : new Error(String(err))); + }); } ); @@ -216,7 +347,26 @@ export function streamGenerate(options: StreamGenerateOptions): Promise fail(err instanceof Error ? err : new Error(String(err)))); + req.on('error', (err) => { + if (settled) { + return; + } + if (options.signal?.aborted || isCancellationError(err)) { + fail(new CancelledError('Generation cancelled', jobId)); + return; + } + // Node often emits Error with code after destroy(); ignore if we already aborted. + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ECONNRESET' && cancelPosted) { + fail(new CancelledError('Generation cancelled', jobId)); + return; + } + fail(err instanceof Error ? err : new Error(String(err))); + }); + + if (options.signal) { + options.signal.addEventListener('abort', onAbort, { once: true }); + } req.write(payload); req.end(); From 4d5163cf2e6fb95cc84a60c67e6fa8fcd8d72cd5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 08:42:35 +0000 Subject: [PATCH 05/13] Implement Creer v0.5: content diff preview, packs, multi-root Add installable JSON/YAML template packs with /packs API and pack_id planning, extension multi-root workspace picker, and generated-content diff preview before writing files to disk. Co-authored-by: Sanath S Patil --- README.md | 110 +++++-------- backend/.env.example | 4 + backend/app/packs.py | 179 ++++++++++++++++++++ backend/app/planner.py | 48 ++++-- backend/config.py | 2 + backend/main.py | 57 ++++++- backend/packs/express-ts.yaml | 13 ++ backend/packs/fastapi-crud.json | 17 ++ backend/packs/python-lib.json | 15 ++ backend/requirements.txt | 1 + extension/package.json | 12 +- extension/src/api.ts | 40 ++++- extension/src/contentPreview.ts | 279 ++++++++++++++++++++++++++++++++ extension/src/preview.ts | 11 +- extension/src/scaffold.ts | 161 ++++++++++++++---- extension/src/streamGenerate.ts | 10 +- extension/src/workspace.ts | 44 +++++ 17 files changed, 880 insertions(+), 123 deletions(-) create mode 100644 backend/app/packs.py create mode 100644 backend/packs/express-ts.yaml create mode 100644 backend/packs/fastapi-crud.json create mode 100644 backend/packs/python-lib.json create mode 100644 extension/src/contentPreview.ts create mode 100644 extension/src/workspace.ts diff --git a/README.md b/README.md index 2af926d..7d92e59 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,13 @@ AI-powered repo scaffolding inside your workspace. -Describe an idea → Creer plans a clean structure → optionally preview/confirm → writes files into your VS Code workspace → optionally initializes git and creates a GitHub remote. - -**Current version: 0.4.0** +**Current version: 0.5.0** ## Architecture ``` creer/ -├── backend/ # Python FastAPI AI engine +├── backend/ # Python FastAPI AI engine (+ packs/) └── extension/ # VS Code extension ``` @@ -18,18 +16,16 @@ creer/ - Python 3.10+ - Node.js 18+ -- OpenAI API key **or** a local OpenAI-compatible server (`OPENAI_BASE_URL`), unless using offline template mode (`CREER_OFFLINE=1`) +- OpenAI API key **or** `OPENAI_BASE_URL` **or** `CREER_OFFLINE=1` (with template/pack) - VS Code / Cursor -## Backend (v0.4) +## Backend (v0.5) ```bash cd backend -python -m venv venv -source venv/bin/activate # Windows: venv\Scripts\activate +python -m venv venv && source venv/bin/activate pip install -r requirements.txt cp .env.example .env -# set OPENAI_API_KEY and/or OPENAI_BASE_URL in .env uvicorn main:app --reload --port 8000 ``` @@ -37,72 +33,54 @@ uvicorn main:app --reload --port 8000 | Variable | Description | |---|---| -| `OPENAI_API_KEY` | OpenAI API key (optional if using `OPENAI_BASE_URL` or `CREER_OFFLINE=1`) | -| `OPENAI_BASE_URL` | OpenAI-compatible base URL (Ollama, LM Studio, etc.). Example: `http://127.0.0.1:11434/v1` | +| `OPENAI_API_KEY` | OpenAI API key | +| `OPENAI_BASE_URL` | OpenAI-compatible base URL (Ollama, etc.) | | `CREER_MODEL` | Model name (default `gpt-4o-mini`) | -| `CREER_OFFLINE` | `1` / `true` / `yes` for template-only stubs (planning requires `template_id`) | +| `CREER_OFFLINE` | Template/pack-only stubs | +| `CREER_PACKS_DIR` | Extra packs directory (overrides same ids) | ### Endpoints | Method | Path | Description | |---|---|---| -| `GET` | `/health` | Version `0.4.0`, offline flag, model | -| `GET` | `/templates` | Curated starter templates | -| `GET` | `/bakeins` | License + CI bake-in options | -| `POST` | `/plan` | Plan only (no file contents) | -| `POST` | `/generate` | Generate files (+ bake-ins + `quality`) | -| `POST` | `/generate/stream` | SSE progress; `start` includes `job_id` | -| `POST` | `/generate/cancel` | Cancel by `job_id` | -| `POST` | `/quality` | Dry-run quality gates | +| `GET` | `/health` | Version `0.5.0`, packs_count | +| `GET` | `/templates` | Built-in templates | +| `GET` | `/packs` | Installable JSON/YAML packs | +| `GET` | `/packs/{id}` | Single pack | +| `GET` | `/bakeins` | License/CI options | +| `POST` | `/plan` | Plan (`template_id` **or** `pack_id`) | +| `POST` | `/generate` | Generate + bake-ins + quality | +| `POST` | `/generate/stream` | SSE progress (`job_id`) | +| `POST` | `/generate/cancel` | Cancel job | +| `POST` | `/quality` | Dry-run gates | | `POST` | `/github/create-repo` | Create GitHub repo | -#### Generate body (sync or stream) +### Packs + +Drop `.json` / `.yaml` files into `backend/packs/` (or `CREER_PACKS_DIR`): ```json { - "idea": "Build a FastAPI todo app", - "template_id": "fastapi-minimal", - "plan": null, - "job_id": null, - "bakeins": { - "license": "mit", - "ci": "auto", - "include_readme": true - } + "id": "fastapi-crud", + "name": "FastAPI CRUD", + "description": "CRUD API starter", + "stack": "FastAPI + Uvicorn", + "version": "1.0.0", + "files": ["main.py", "requirements.txt", "README.md"] } ``` -`bakeins.license`: `mit` | `apache-2.0` | `none` -`bakeins.ci`: `auto` | `python` | `node` | `none` +Shipped examples: `fastapi-crud`, `express-ts`, `python-lib`. -#### Stream events - -| Event | Meaning | -|---|---| -| `start` | Includes `job_id`, `total`, `project_name` | -| `file` | Per-file `generating` / `done` | -| `done` | Full `files` map + `quality` | -| `cancelled` | Stopped via `/generate/cancel` | -| `error` | Failure detail | +## Extension (v0.5) ```bash -curl -N -X POST http://localhost:8000/generate/stream \ - -H "Content-Type: application/json" \ - -d '{"idea":"todo api","template_id":"fastapi-minimal"}' +cd extension && npm install && npm run compile ``` -## Extension (v0.4) - -```bash -cd extension -npm install -npm run compile -``` +Flow: idea → template/pack → bake-ins → plan preview → generate (cancellable) → **content diff preview** → write → optional git/GitHub. -1. Open `extension/` → **F5** -2. Open a workspace folder -3. **Creer: Create New Repo** -4. Pick template → license/CI → confirm preview → generate (cancellable while streaming) +Multi-root: QuickPick workspace folder (or `creer.defaultWorkspaceFolder`). ### Commands @@ -117,18 +95,16 @@ npm run compile | Setting | Default | Description | |---|---|---| -| `creer.backendUrl` | `http://localhost:8000` | Backend base URL | -| `creer.initGit` | `true` | `git init` + initial commit | -| `creer.previewBeforeWrite` | `true` | Confirm plan before generate | -| `creer.useStreaming` | `true` | SSE progress (cancellable) | -| `creer.promptBakeins` | `true` | QuickPick license/CI each run | -| `creer.license` | `mit` | Used when not prompting | -| `creer.ciPreset` | `auto` | Used when not prompting | -| `creer.createGitHubRepo` | `false` | Create GitHub remote after scaffold | -| `creer.githubPrivate` | `true` | Private GitHub repos | -| `creer.githubToken` | `""` | **Deprecated** — use SecretStorage | - -GitHub push uses `GIT_ASKPASS` (token never on argv/URL). +| `creer.backendUrl` | `http://localhost:8000` | Backend URL | +| `creer.contentPreview` | `true` | Diff/content preview before write | +| `creer.defaultWorkspaceFolder` | `""` | Multi-root folder name/path hint | +| `creer.previewBeforeWrite` | `true` | Plan tree confirm before generate | +| `creer.useStreaming` | `true` | SSE progress | +| `creer.promptBakeins` | `true` | QuickPick license/CI | +| `creer.license` / `creer.ciPreset` | `mit` / `auto` | Defaults when not prompting | +| `creer.initGit` | `true` | git init + commit | +| `creer.createGitHubRepo` | `false` | Create remote | +| `creer.githubPrivate` | `true` | Private repos | ## License diff --git a/backend/.env.example b/backend/.env.example index 30d816e..bb3c577 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -11,3 +11,7 @@ CREER_MODEL=gpt-4o-mini # Template-only / stub generation — never calls the LLM # Set to 1, true, or yes to enable CREER_OFFLINE= + +# Optional extra packs directory (JSON/YAML). Merged with backend/packs; +# user packs override built-in packs on the same id. +# CREER_PACKS_DIR=/path/to/my-packs diff --git a/backend/app/packs.py b/backend/app/packs.py new file mode 100644 index 0000000..f3a06c8 --- /dev/null +++ b/backend/app/packs.py @@ -0,0 +1,179 @@ +"""Installable template packs (JSON/YAML) for Creer v0.5.""" + +from __future__ import annotations + +import json +import os +import re +from pathlib import Path + +import yaml + +from app.templates import slugify +from app.validator import _reject_unsafe_path + +# backend/packs — resolved relative to backend root (parent of app/) +_BACKEND_ROOT = Path(__file__).resolve().parent.parent +PACKS_DIR = _BACKEND_ROOT / "packs" + +_PACK_FILE_SUFFIXES = (".json", ".yaml", ".yml") +_SAFE_ID = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,63}$") + + +def _extra_packs_dir() -> Path | None: + """Optional user packs directory via CREER_PACKS_DIR.""" + raw = os.getenv("CREER_PACKS_DIR", "").strip() + if not raw: + return None + return Path(raw).expanduser().resolve() + + +def load_pack_file(path: Path | str) -> dict: + """ + Load and validate a pack from a JSON or YAML file. + + Required: id, name, non-empty files list with safe relative paths. + Optional: description, stack, version. + """ + path = Path(path) + if not path.is_file(): + raise ValueError(f"Pack file not found: {path}") + + suffix = path.suffix.lower() + try: + text = path.read_text(encoding="utf-8") + except OSError as exc: + raise ValueError(f"Cannot read pack file {path}: {exc}") from exc + + if suffix == ".json": + try: + data = json.loads(text) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON in pack {path}: {exc}") from exc + elif suffix in (".yaml", ".yml"): + try: + data = yaml.safe_load(text) + except yaml.YAMLError as exc: + raise ValueError(f"Invalid YAML in pack {path}: {exc}") from exc + else: + raise ValueError(f"Unsupported pack file type: {path.suffix!r}") + + if not isinstance(data, dict): + raise ValueError(f"Pack must be a mapping/object: {path}") + + pack_id = data.get("id") + name = data.get("name") + files = data.get("files") + + if not isinstance(pack_id, str) or not pack_id.strip(): + raise ValueError(f"Pack missing valid id: {path}") + pack_id = pack_id.strip() + if not _SAFE_ID.match(pack_id): + raise ValueError(f"Invalid pack id {pack_id!r} in {path}") + + if not isinstance(name, str) or not name.strip(): + raise ValueError(f"Pack missing valid name: {path}") + + if not isinstance(files, list) or not files: + raise ValueError(f"Pack must include a non-empty files list: {path}") + + validated_files: list[str] = [] + seen: set[str] = set() + for entry in files: + if not isinstance(entry, str) or not entry.strip(): + raise ValueError(f"Invalid file path in pack {path}: {entry!r}") + fpath = entry.strip().replace("\\", "/") + _reject_unsafe_path(fpath, label="pack file path") + if fpath in seen: + raise ValueError(f"Duplicate file path in pack {path}: {fpath!r}") + seen.add(fpath) + validated_files.append(fpath) + + version = data.get("version", "1.0.0") + if version is not None and not isinstance(version, str): + raise ValueError(f"Pack version must be a string: {path}") + + stack = data.get("stack", "") + if stack is not None and not isinstance(stack, str): + raise ValueError(f"Pack stack must be a string: {path}") + + description = data.get("description", "") + if description is not None and not isinstance(description, str): + raise ValueError(f"Pack description must be a string: {path}") + + return { + "id": pack_id, + "name": name.strip(), + "description": (description or "").strip(), + "stack": (stack or "").strip(), + "version": (version or "1.0.0").strip(), + "files": validated_files, + } + + +def _load_packs_from_dir(directory: Path) -> dict[str, dict]: + """Load all valid pack files from a directory (non-recursive).""" + result: dict[str, dict] = {} + if not directory.is_dir(): + return result + + for path in sorted(directory.iterdir()): + if not path.is_file(): + continue + if path.suffix.lower() not in _PACK_FILE_SUFFIXES: + continue + try: + pack = load_pack_file(path) + except ValueError: + # Skip invalid packs rather than failing the whole listing + continue + result[pack["id"]] = pack + return result + + +def _all_packs() -> dict[str, dict]: + """ + Merge built-in packs with optional CREER_PACKS_DIR. + + User packs override built-in packs on id collision. + """ + packs = _load_packs_from_dir(PACKS_DIR) + extra = _extra_packs_dir() + if extra is not None: + packs.update(_load_packs_from_dir(extra)) + return packs + + +def list_packs() -> list[dict]: + """Return all available packs (built-in + user), sorted by id.""" + packs = _all_packs() + return [dict(packs[k]) for k in sorted(packs.keys())] + + +def get_pack(pack_id: str) -> dict | None: + """Look up a pack by id.""" + if not pack_id: + return None + pack = _all_packs().get(pack_id) + return dict(pack) if pack else None + + +def apply_pack(pack_id: str, idea: str) -> dict: + """ + Build a plan from an installable pack. + + Uses the pack's files and stack. Derives project_name by slugifying + the idea (deterministic — no API key required). + """ + pack = get_pack(pack_id) + if pack is None: + raise ValueError(f"Unknown pack_id: {pack_id!r}") + + project_name = slugify(idea) + return { + "project_name": project_name, + "stack": pack["stack"], + "files": list(pack["files"]), + "pack_id": pack["id"], + "description": pack.get("description", ""), + } diff --git a/backend/app/planner.py b/backend/app/planner.py index 9922a07..8e225b6 100644 --- a/backend/app/planner.py +++ b/backend/app/planner.py @@ -7,6 +7,7 @@ from config import CREER_OFFLINE, MODEL, OPENAI_API_KEY, OPENAI_BASE_URL from app.llm import _get_client +from app.packs import apply_pack, get_pack from app.templates import apply_template, get_template, slugify @@ -89,24 +90,46 @@ def _ai_name_project(idea: str, template: dict) -> str: return slugify(idea) -def plan_project(idea: str, template_id: str | None = None) -> dict: +def plan_project( + idea: str, + template_id: str | None = None, + pack_id: str | None = None, +) -> dict: """ - Build a project plan from an idea, optionally anchored to a curated template. + Build a project plan from an idea, optionally anchored to a pack or template. - When template_id is set: - - files and stack come from the template + pack_id and template_id are mutually exclusive — both set raises ValueError. + + When pack_id or template_id is set: + - files and stack come from the pack/template - project_name is derived deterministically via slugify (no API key required) - if an LLM is configured and not offline, AI may refine project_name only - Without template_id: full AI planning (requires OPENAI_API_KEY or OPENAI_BASE_URL), - unless offline — offline without template_id raises ValueError. + Without either: full AI planning (requires OPENAI_API_KEY or OPENAI_BASE_URL), + unless offline — offline without pack_id or template_id raises ValueError. """ - if CREER_OFFLINE and not template_id: + if pack_id and template_id: + raise ValueError("Provide pack_id or template_id, not both") + + if CREER_OFFLINE and not pack_id and not template_id: raise ValueError( - "Offline mode requires template_id. Pass a curated template_id " - "(see GET /templates) — AI planning is disabled when CREER_OFFLINE is set." + "Offline mode requires pack_id or template_id. Pass a pack_id " + "(see GET /packs) or template_id (see GET /templates) — AI planning " + "is disabled when CREER_OFFLINE is set." ) + if pack_id: + pack = get_pack(pack_id) + if pack is None: + raise ValueError(f"Unknown pack_id: {pack_id!r}") + + plan = apply_pack(pack_id, idea) + + if _llm_configured() and not CREER_OFFLINE: + plan["project_name"] = _ai_name_project(idea, pack) + + return plan + if template_id: tmpl = get_template(template_id) if tmpl is None: @@ -121,10 +144,11 @@ def plan_project(idea: str, template_id: str | None = None) -> dict: return plan if not _llm_configured() and not CREER_OFFLINE: - # No LLM endpoint and no template — cannot plan with AI + # No LLM endpoint and no pack/template — cannot plan with AI raise ValueError( - "OPENAI_API_KEY (or OPENAI_BASE_URL) is not set. Provide a template_id " - "for template-only planning, or set CREER_OFFLINE=1 with a template_id." + "OPENAI_API_KEY (or OPENAI_BASE_URL) is not set. Provide a pack_id " + "or template_id for pack/template-only planning, or set " + "CREER_OFFLINE=1 with a pack_id or template_id." ) return _ai_plan(idea) diff --git a/backend/config.py b/backend/config.py index 2089ca1..7845039 100644 --- a/backend/config.py +++ b/backend/config.py @@ -7,3 +7,5 @@ OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL") # e.g. http://127.0.0.1:11434/v1 for Ollama MODEL = os.getenv("CREER_MODEL", "gpt-4o-mini") CREER_OFFLINE = os.getenv("CREER_OFFLINE", "").lower() in ("1", "true", "yes") +# Optional extra packs directory (merged with backend/packs; user overrides on id collision) +CREER_PACKS_DIR = os.getenv("CREER_PACKS_DIR", "").strip() or None diff --git a/backend/main.py b/backend/main.py index 10d7017..8b98684 100644 --- a/backend/main.py +++ b/backend/main.py @@ -16,11 +16,12 @@ from app.generator import generate_files, generate_files_iter from app.validator import validate_plan, validate_files from app.templates import list_templates, get_template +from app.packs import list_packs, get_pack from app.github import create_github_repo from app.jobs import cancel_job, create_job, finish_job, is_cancelled from app.quality import has_errors, run_quality_gates -VERSION = "0.4.0" +VERSION = "0.5.0" app = FastAPI(title="Creer", version=VERSION) @@ -28,6 +29,7 @@ class PlanRequest(BaseModel): idea: str = Field(..., min_length=3, max_length=4000) template_id: str | None = None + pack_id: str | None = None class PlanBody(BaseModel): @@ -35,6 +37,7 @@ class PlanBody(BaseModel): stack: str | None = None files: list[str] template_id: str | None = None + pack_id: str | None = None description: str | None = None @@ -47,6 +50,7 @@ class BakeinOptions(BaseModel): class GenerateRequest(BaseModel): idea: str = Field(..., min_length=3, max_length=4000) template_id: str | None = None + pack_id: str | None = None plan: PlanBody | None = None job_id: str | None = None bakeins: BakeinOptions | None = None @@ -69,18 +73,38 @@ class GitHubCreateRepoRequest(BaseModel): token: str | None = None +def _check_pack_template_exclusive( + pack_id: str | None, template_id: str | None +) -> None: + if pack_id and template_id: + raise ValueError("Provide pack_id or template_id, not both") + + def _resolve_plan(request: GenerateRequest) -> dict: """Resolve a validated plan from GenerateRequest (shared by sync + stream).""" + _check_pack_template_exclusive(request.pack_id, request.template_id) + if request.plan is not None: plan = request.plan.model_dump() plan = {k: v for k, v in plan.items() if v is not None} plan.setdefault("stack", "") + _check_pack_template_exclusive(plan.get("pack_id"), plan.get("template_id")) + if request.pack_id and "pack_id" not in plan: + plan["pack_id"] = request.pack_id if request.template_id and "template_id" not in plan: plan["template_id"] = request.template_id + # Re-check after merging top-level ids into plan + _check_pack_template_exclusive(plan.get("pack_id"), plan.get("template_id")) else: + if request.pack_id and get_pack(request.pack_id) is None: + raise ValueError(f"Unknown pack_id: {request.pack_id!r}") if request.template_id and get_template(request.template_id) is None: raise ValueError(f"Unknown template_id: {request.template_id!r}") - plan = plan_project(request.idea, template_id=request.template_id) + plan = plan_project( + request.idea, + template_id=request.template_id, + pack_id=request.pack_id, + ) validate_plan(plan) return plan @@ -108,6 +132,7 @@ def health(): "offline": CREER_OFFLINE, "base_url_set": bool(OPENAI_BASE_URL), "model": MODEL, + "packs_count": len(list_packs()), } @@ -116,6 +141,19 @@ def templates(): return {"templates": list_templates()} +@app.get("/packs") +def packs(): + return {"packs": list_packs()} + + +@app.get("/packs/{pack_id}") +def pack_detail(pack_id: str): + pack = get_pack(pack_id) + if pack is None: + raise HTTPException(status_code=404, detail=f"Unknown pack_id: {pack_id!r}") + return pack + + @app.get("/bakeins") def bakeins(): return list_bakein_options() @@ -125,9 +163,16 @@ def bakeins(): def plan_only(request: PlanRequest): """Return a project plan without generating file contents.""" try: + _check_pack_template_exclusive(request.pack_id, request.template_id) + if request.pack_id and get_pack(request.pack_id) is None: + raise ValueError(f"Unknown pack_id: {request.pack_id!r}") if request.template_id and get_template(request.template_id) is None: raise ValueError(f"Unknown template_id: {request.template_id!r}") - plan = plan_project(request.idea, template_id=request.template_id) + plan = plan_project( + request.idea, + template_id=request.template_id, + pack_id=request.pack_id, + ) validate_plan(plan) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc @@ -139,6 +184,8 @@ def plan_only(request: PlanRequest): "stack": plan.get("stack"), "files": plan["files"], } + if plan.get("pack_id"): + result["pack_id"] = plan["pack_id"] if plan.get("template_id"): result["template_id"] = plan["template_id"] if plan.get("description"): @@ -178,6 +225,8 @@ def generate_project(request: GenerateRequest): "quality": quality, "job_id": job_id, } + if plan.get("pack_id"): + result["pack_id"] = plan["pack_id"] if plan.get("template_id"): result["template_id"] = plan["template_id"] return result @@ -256,6 +305,8 @@ def event_stream() -> Iterator[str]: "stack": plan.get("stack") or "", "quality": quality, } + if plan.get("pack_id"): + done["pack_id"] = plan["pack_id"] if plan.get("template_id"): done["template_id"] = plan["template_id"] yield _sse(done) diff --git a/backend/packs/express-ts.yaml b/backend/packs/express-ts.yaml new file mode 100644 index 0000000..45b257d --- /dev/null +++ b/backend/packs/express-ts.yaml @@ -0,0 +1,13 @@ +id: express-ts +name: Express TypeScript +description: Express API in TypeScript with tsconfig and basic router. +stack: Node.js + Express + TypeScript +version: "1.0.0" +files: + - package.json + - tsconfig.json + - src/index.ts + - src/routes/health.ts + - README.md + - .gitignore + - .env.example diff --git a/backend/packs/fastapi-crud.json b/backend/packs/fastapi-crud.json new file mode 100644 index 0000000..443adcf --- /dev/null +++ b/backend/packs/fastapi-crud.json @@ -0,0 +1,17 @@ +{ + "id": "fastapi-crud", + "name": "FastAPI CRUD", + "description": "FastAPI starter with models, CRUD routes, and uvicorn entrypoint.", + "stack": "FastAPI + Uvicorn", + "version": "1.0.0", + "files": [ + "main.py", + "requirements.txt", + "README.md", + "app/__init__.py", + "app/models.py", + "app/routes.py", + ".env.example", + ".gitignore" + ] +} diff --git a/backend/packs/python-lib.json b/backend/packs/python-lib.json new file mode 100644 index 0000000..e9fcd8e --- /dev/null +++ b/backend/packs/python-lib.json @@ -0,0 +1,15 @@ +{ + "id": "python-lib", + "name": "Python Library", + "description": "Publishable Python library with pyproject, package layout, and tests.", + "stack": "Python library", + "version": "1.0.0", + "files": [ + "pyproject.toml", + "README.md", + "src/mylib/__init__.py", + "src/mylib/core.py", + "tests/test_core.py", + ".gitignore" + ] +} diff --git a/backend/requirements.txt b/backend/requirements.txt index f5c2db6..8726dde 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -4,3 +4,4 @@ openai>=1.55.0 python-dotenv>=1.0.0 pydantic>=2.9.0 httpx>=0.27.0 +PyYAML>=6.0.0 diff --git a/extension/package.json b/extension/package.json index d412ff4..0ffed8a 100644 --- a/extension/package.json +++ b/extension/package.json @@ -2,7 +2,7 @@ "name": "creer", "displayName": "Creer", "description": "AI-powered repo scaffolding inside your workspace", - "version": "0.4.0", + "version": "0.5.0", "publisher": "creer", "engines": { "vscode": "^1.85.0" @@ -96,6 +96,16 @@ "type": "boolean", "default": true, "description": "When true, QuickPick license and CI each run; when false, use creer.license and creer.ciPreset" + }, + "creer.contentPreview": { + "type": "boolean", + "default": true, + "description": "After generate, before write, show a content preview/diff and confirm" + }, + "creer.defaultWorkspaceFolder": { + "type": "string", + "default": "", + "description": "Optional workspace folder name or path hint for multi-root workspaces; empty prompts when multiple folders are open" } } } diff --git a/extension/src/api.ts b/extension/src/api.ts index 49f4182..71f8c93 100644 --- a/extension/src/api.ts +++ b/extension/src/api.ts @@ -9,11 +9,21 @@ export interface Template { files: string[]; } +export interface Pack { + id: string; + name: string; + description: string; + stack: string; + version?: string; + files: string[]; +} + export interface PlanResponse { project_name: string; stack?: string; files: string[]; template_id?: string; + pack_id?: string; description?: string; } @@ -48,6 +58,7 @@ export interface GenerateResponse { stack?: string; files: Record; template_id?: string; + pack_id?: string; quality?: QualityIssue[]; } @@ -57,8 +68,14 @@ export interface GitHubCreateRepoResponse { full_name: string; } +export interface PlanOptions { + templateId?: string; + packId?: string; +} + export interface GenerateOptions { templateId?: string; + packId?: string; plan?: PlanResponse; jobId?: string; bakeins?: BakeinOptions; @@ -94,6 +111,14 @@ export async function fetchTemplates(): Promise { return response.data.templates ?? []; } +export async function fetchPacks(): Promise { + const backendUrl = getBackendUrl(); + const response = await axios.get<{ packs: Pack[] }>(`${backendUrl}/packs`, { + timeout: 30_000, + }); + return response.data.packs ?? []; +} + export async function fetchBakeins(): Promise { const backendUrl = getBackendUrl(); const response = await axios.get(`${backendUrl}/bakeins`, { @@ -115,11 +140,13 @@ export async function postCancel(jobId: string): Promise<{ cancelled: true }> { return response.data; } -export async function postPlan(idea: string, templateId?: string): Promise { +export async function postPlan(idea: string, options?: PlanOptions): Promise { const backendUrl = getBackendUrl(); - const body: { idea: string; template_id?: string } = { idea }; - if (templateId) { - body.template_id = templateId; + const body: { idea: string; template_id?: string; pack_id?: string } = { idea }; + if (options?.packId) { + body.pack_id = options.packId; + } else if (options?.templateId) { + body.template_id = options.templateId; } const response = await axios.post(`${backendUrl}/plan`, body, { timeout: 300_000, @@ -135,11 +162,14 @@ export async function postGenerate( const body: { idea: string; template_id?: string; + pack_id?: string; plan?: PlanResponse; job_id?: string; bakeins?: BakeinOptions; } = { idea }; - if (options?.templateId) { + if (options?.packId) { + body.pack_id = options.packId; + } else if (options?.templateId) { body.template_id = options.templateId; } if (options?.plan) { diff --git a/extension/src/contentPreview.ts b/extension/src/contentPreview.ts new file mode 100644 index 0000000..011741b --- /dev/null +++ b/extension/src/contentPreview.ts @@ -0,0 +1,279 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import * as vscode from 'vscode'; + +const MAX_FILES_SHOWN = 30; +const MAX_PREVIEW_LINES = 40; +const MAX_DIFF_LINES = 60; + +function fenceLang(filePath: string): string { + const ext = path.extname(filePath).toLowerCase(); + const map: Record = { + '.ts': 'typescript', + '.tsx': 'tsx', + '.js': 'javascript', + '.jsx': 'jsx', + '.py': 'python', + '.json': 'json', + '.md': 'markdown', + '.yml': 'yaml', + '.yaml': 'yaml', + '.toml': 'toml', + '.sh': 'bash', + '.css': 'css', + '.html': 'html', + '.rs': 'rust', + '.go': 'go', + '.java': 'java', + '.rb': 'ruby', + }; + return map[ext] || ''; +} + +function byteSize(content: string): number { + return Buffer.byteLength(content, 'utf8'); +} + +function formatBytes(n: number): string { + if (n < 1024) { + return `${n} B`; + } + if (n < 1024 * 1024) { + return `${(n / 1024).toFixed(1)} KB`; + } + return `${(n / (1024 * 1024)).toFixed(1)} MB`; +} + +function truncateLines(text: string, maxLines: number): { text: string; truncated: boolean } { + const lines = text.split('\n'); + if (lines.length <= maxLines) { + return { text, truncated: false }; + } + return { + text: lines.slice(0, maxLines).join('\n'), + truncated: true, + }; +} + +/** + * Simple line-based unified diff (no external deps). + * Uses LCS on lines for moderate-sized inputs; truncates output. + */ +export function buildUnifiedDiff( + existing: string, + generated: string, + maxOutputLines = MAX_DIFF_LINES +): string { + const a = existing.split('\n'); + const b = generated.split('\n'); + + // Cap LCS input to keep it snappy on huge files + const CAP = 400; + const aCapped = a.length > CAP; + const bCapped = b.length > CAP; + const aLines = aCapped ? a.slice(0, CAP) : a; + const bLines = bCapped ? b.slice(0, CAP) : b; + + const n = aLines.length; + const m = bLines.length; + const dp: number[][] = Array.from({ length: n + 1 }, () => Array(m + 1).fill(0)); + for (let i = n - 1; i >= 0; i--) { + for (let j = m - 1; j >= 0; j--) { + if (aLines[i] === bLines[j]) { + dp[i][j] = dp[i + 1][j + 1] + 1; + } else { + dp[i][j] = Math.max(dp[i + 1][j], dp[i][j + 1]); + } + } + } + + const ops: Array<{ kind: 'eq' | 'del' | 'add'; line: string }> = []; + let i = 0; + let j = 0; + while (i < n && j < m) { + if (aLines[i] === bLines[j]) { + ops.push({ kind: 'eq', line: aLines[i] }); + i++; + j++; + } else if (dp[i + 1][j] >= dp[i][j + 1]) { + ops.push({ kind: 'del', line: aLines[i] }); + i++; + } else { + ops.push({ kind: 'add', line: bLines[j] }); + j++; + } + } + while (i < n) { + ops.push({ kind: 'del', line: aLines[i++] }); + } + while (j < m) { + ops.push({ kind: 'add', line: bLines[j++] }); + } + + const out: string[] = ['--- existing', '+++ generated']; + let shown = 0; + let skippedEq = 0; + + const flushSkipped = () => { + if (skippedEq > 0) { + out.push(` … ${skippedEq} unchanged line(s)`); + skippedEq = 0; + } + }; + + for (const op of ops) { + if (shown >= maxOutputLines) { + flushSkipped(); + out.push(` … diff truncated (${ops.length - shown} more op(s))`); + break; + } + if (op.kind === 'eq') { + skippedEq++; + continue; + } + flushSkipped(); + if (op.kind === 'del') { + out.push(`-${op.line}`); + } else { + out.push(`+${op.line}`); + } + shown++; + } + flushSkipped(); + + if (aCapped || bCapped) { + out.push( + ` … input capped for diff (existing ${a.length} lines, generated ${b.length} lines)` + ); + } + + return out.join('\n'); +} + +function buildNewFileSection(relPath: string, content: string): string { + const size = formatBytes(byteSize(content)); + const lang = fenceLang(relPath); + const { text, truncated } = truncateLines(content, MAX_PREVIEW_LINES); + const note = truncated + ? `\n_Preview truncated to ${MAX_PREVIEW_LINES} lines (${content.split('\n').length} total)._\n` + : ''; + + return [ + `### \`${relPath}\` — NEW · ${size}`, + '', + note, + '```' + lang, + text, + '```', + '', + ].join('\n'); +} + +function buildExistingFileSection( + relPath: string, + existing: string, + generated: string +): string { + const size = formatBytes(byteSize(generated)); + if (existing === generated) { + return [ + `### \`${relPath}\` — UNCHANGED · ${size}`, + '', + '_Generated content matches the existing file._', + '', + ].join('\n'); + } + + const diff = buildUnifiedDiff(existing, generated); + return [ + `### \`${relPath}\` — EXISTING (will overwrite) · ${size}`, + '', + '```diff', + diff, + '```', + '', + ].join('\n'); +} + +export function buildContentPreviewMarkdown( + projectName: string, + projectPath: string, + files: Record +): string { + const paths = Object.keys(files).sort((a, b) => a.localeCompare(b)); + const shown = paths.slice(0, MAX_FILES_SHOWN); + const omitted = paths.length - shown.length; + + const sections: string[] = [ + `# Creer content preview`, + '', + `**Project:** \`${projectName}\``, + '', + `**Path:** \`${projectPath}\``, + '', + `**Files:** ${paths.length}`, + '', + '---', + '', + ]; + + for (const relPath of shown) { + const generated = files[relPath] ?? ''; + const fullPath = path.join(projectPath, relPath); + let existing: string | undefined; + try { + if (fs.existsSync(fullPath) && fs.statSync(fullPath).isFile()) { + existing = fs.readFileSync(fullPath, 'utf8'); + } + } catch { + existing = undefined; + } + + if (existing === undefined) { + sections.push(buildNewFileSection(relPath, generated)); + } else { + sections.push(buildExistingFileSection(relPath, existing, generated)); + } + } + + if (omitted > 0) { + sections.push(`_…and ${omitted} more file(s) not shown in this preview._`, ''); + } + + sections.push('---', '', '_Confirm to write these files to disk._', ''); + return sections.join('\n'); +} + +/** + * Open an untitled markdown preview of generated file contents and ask the user + * to confirm writing. Returns true if the user confirms. + * When `creer.contentPreview` is false, skips the UI and returns true. + */ +export async function showContentPreviewAndConfirm( + projectName: string, + projectPath: string, + files: Record +): Promise { + const config = vscode.workspace.getConfiguration('creer'); + const enabled = config.get('contentPreview') ?? true; + if (!enabled) { + return true; + } + + const markdown = buildContentPreviewMarkdown(projectName, projectPath, files); + const doc = await vscode.workspace.openTextDocument({ + content: markdown, + language: 'markdown', + }); + await vscode.window.showTextDocument(doc, { preview: true, preserveFocus: false }); + + const fileCount = Object.keys(files).length; + const choice = await vscode.window.showInformationMessage( + `Creer content ready: ${projectName} (${fileCount} files). Write files to disk?`, + { modal: true }, + 'Write files', + 'Cancel' + ); + + return choice === 'Write files'; +} diff --git a/extension/src/preview.ts b/extension/src/preview.ts index 3449c80..e98c1ea 100644 --- a/extension/src/preview.ts +++ b/extension/src/preview.ts @@ -8,9 +8,12 @@ function buildFileTreeMarkdown(files: string[]): string { export function buildPlanPreviewMarkdown(plan: PlanResponse, idea: string): string { const stack = plan.stack?.trim() || '(unspecified)'; - const templateLine = plan.template_id - ? `\n**Template:** \`${plan.template_id}\`\n` - : '\n**Template:** AI plan (no template)\n'; + let sourceLine = '\n**Template:** AI plan (no template)\n'; + if (plan.pack_id) { + sourceLine = `\n**Pack:** \`${plan.pack_id}\`\n`; + } else if (plan.template_id) { + sourceLine = `\n**Template:** \`${plan.template_id}\`\n`; + } const description = plan.description?.trim() ? `\n**Description:** ${plan.description.trim()}\n` : ''; @@ -23,7 +26,7 @@ export function buildPlanPreviewMarkdown(plan: PlanResponse, idea: string): stri `**Project:** \`${plan.project_name}\``, '', `**Stack:** ${stack}`, - templateLine, + sourceLine, description, `**Files (${plan.files.length}):**`, '', diff --git a/extension/src/scaffold.ts b/extension/src/scaffold.ts index 5ed6fc4..f267d79 100644 --- a/extension/src/scaffold.ts +++ b/extension/src/scaffold.ts @@ -3,6 +3,7 @@ import * as vscode from 'vscode'; import { createGitHubRepo, fetchBakeins, + fetchPacks, fetchTemplates, formatAxiosError, postGenerate, @@ -11,10 +12,12 @@ import { type CiBakein, type GenerateResponse, type LicenseBakein, + type Pack, type PlanResponse, type QualityIssue, type Template, } from './api'; +import { showContentPreviewAndConfirm } from './contentPreview'; import { addRemoteAndPush, ensureGitRepo, initGit } from './git'; import { showPlanPreviewAndConfirm } from './preview'; import { resolveGitHubToken } from './secrets'; @@ -23,6 +26,7 @@ import { streamGenerate, type StreamProgressEvent, } from './streamGenerate'; +import { pickWorkspaceRoot } from './workspace'; import { assertSafeProjectName, findConflicts, @@ -39,6 +43,11 @@ export interface ScaffoldOptions { fromChat?: boolean; } +export type SourcePick = + | { kind: 'ai' } + | { kind: 'template'; id: string } + | { kind: 'pack'; id: string }; + const LICENSE_FALLBACK: Array<{ id: LicenseBakein; name: string }> = [ { id: 'mit', name: 'MIT' }, { id: 'apache-2.0', name: 'Apache-2.0' }, @@ -81,33 +90,78 @@ async function promptForIdea(fromChat: boolean, initial?: string): Promise { - // null = cancelled; undefined = AI plan (no template); string = template id - const items: Array = [ +/** + * QuickPick: AI plan | built-in templates | packs. + * Returns null if cancelled. + */ +async function pickSource( + templates: Template[], + packs: Pack[] +): Promise { + type Item = vscode.QuickPickItem & { + kindSelect?: SourcePick['kind']; + sourceId?: string; + }; + + const items: Item[] = [ { label: 'AI plan (no template)', description: 'Let Creer choose the stack and file layout', - templateId: undefined, + kindSelect: 'ai', }, - ...templates.map((t) => ({ - label: t.name, - description: t.stack, - detail: t.description, - templateId: t.id, - })), ]; + if (templates.length > 0) { + items.push({ + label: 'Built-in templates', + kind: vscode.QuickPickItemKind.Separator, + }); + for (const t of templates) { + items.push({ + label: t.name, + description: t.stack, + detail: t.description, + kindSelect: 'template', + sourceId: t.id, + }); + } + } + + if (packs.length > 0) { + items.push({ + label: 'Packs', + kind: vscode.QuickPickItemKind.Separator, + }); + for (const p of packs) { + const versionNote = p.version ? ` v${p.version}` : ''; + items.push({ + label: `[pack] ${p.name}`, + description: `${p.stack}${versionNote}`, + detail: p.description, + kindSelect: 'pack', + sourceId: p.id, + }); + } + } + const picked = await vscode.window.showQuickPick(items, { - placeHolder: 'Select a template or AI plan', + placeHolder: 'Select AI plan, template, or pack', ignoreFocusOut: true, matchOnDescription: true, matchOnDetail: true, }); - if (!picked) { + if (!picked || picked.kind === vscode.QuickPickItemKind.Separator) { return null; } - return picked.templateId; + + if (picked.kindSelect === 'template' && picked.sourceId) { + return { kind: 'template', id: picked.sourceId }; + } + if (picked.kindSelect === 'pack' && picked.sourceId) { + return { kind: 'pack', id: picked.sourceId }; + } + return { kind: 'ai' }; } function isLicenseBakein(v: string): v is LicenseBakein { @@ -314,15 +368,34 @@ function reportQualityIssues(quality: QualityIssue[] | undefined): void { ); } +function sourceToIds(source: SourcePick): { + templateId?: string; + packId?: string; +} { + if (source.kind === 'template') { + return { templateId: source.id }; + } + if (source.kind === 'pack') { + return { packId: source.id }; + } + return {}; +} + async function generateWithOptionalStream( idea: string, plan: PlanResponse, - templateId: string | undefined, + source: SourcePick, useStreaming: boolean, bakeins: BakeinOptions ): Promise { + const { templateId, packId } = sourceToIds(source); + const resolvedTemplateId = plan.template_id ?? templateId; + const resolvedPackId = plan.pack_id ?? packId; + + // pack_id and template_id are mutually exclusive const genOpts = { - templateId: plan.template_id ?? templateId, + templateId: resolvedPackId ? undefined : resolvedTemplateId, + packId: resolvedPackId, plan, bakeins, }; @@ -354,6 +427,7 @@ async function generateWithOptionalStream( return await streamGenerate({ idea, templateId: genOpts.templateId, + packId: genOpts.packId, plan: genOpts.plan, bakeins: genOpts.bakeins, signal: controller.signal, @@ -394,8 +468,8 @@ export async function runScaffoldFlow(options: ScaffoldOptions): Promise { return; } - const workspaceFolders = vscode.workspace.workspaceFolders; - if (!workspaceFolders) { + const rootPath = await pickWorkspaceRoot(); + if (!rootPath) { vscode.window.showErrorMessage('Open a workspace folder first.'); return; } @@ -404,11 +478,12 @@ export async function runScaffoldFlow(options: ScaffoldOptions): Promise { const shouldInitGit = config.get('initGit') ?? true; const previewBeforeWrite = config.get('previewBeforeWrite') ?? true; const useStreaming = config.get('useStreaming') ?? true; - const rootPath = workspaceFolders[0].uri.fsPath; try { - // 1) Templates + // 1) Templates + packs let templates: Template[] = []; + let packs: Pack[] = []; + try { templates = await vscode.window.withProgress( { @@ -425,11 +500,25 @@ export async function runScaffoldFlow(options: ScaffoldOptions): Promise { ); } - const templatePick = await pickTemplate(templates); - if (templatePick === null) { + try { + packs = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'Creer: loading packs…', + cancellable: false, + }, + () => fetchPacks() + ); + } catch { + // Backend /packs may not be ready yet — show templates only. + packs = []; + } + + const source = await pickSource(templates, packs); + if (!source) { return; } - const templateId = templatePick; // string | undefined + const { templateId, packId } = sourceToIds(source); // 2) Plan const plan: PlanResponse = await vscode.window.withProgress( @@ -438,7 +527,7 @@ export async function runScaffoldFlow(options: ScaffoldOptions): Promise { title: 'Creer: planning project…', cancellable: false, }, - () => postPlan(idea, templateId) + () => postPlan(idea, { templateId, packId }) ); if (!plan.project_name || !Array.isArray(plan.files)) { @@ -446,12 +535,14 @@ export async function runScaffoldFlow(options: ScaffoldOptions): Promise { return; } - // Ensure template_id is on the plan when selected if (templateId && !plan.template_id) { plan.template_id = templateId; } + if (packId && !plan.pack_id) { + plan.pack_id = packId; + } - // 3) Preview / confirm + // 3) Plan preview / confirm (tree) before generate if (previewBeforeWrite) { const confirmed = await showPlanPreviewAndConfirm(plan, idea); if (!confirmed) { @@ -471,7 +562,7 @@ export async function runScaffoldFlow(options: ScaffoldOptions): Promise { generated = await generateWithOptionalStream( idea, plan, - templateId, + source, useStreaming, bakeins ); @@ -515,21 +606,31 @@ export async function runScaffoldFlow(options: ScaffoldOptions): Promise { return; } - // 6) Conflict resolution + // 6) Content preview / diff before write + const contentConfirmed = await showContentPreviewAndConfirm( + projectName, + projectPath, + files + ); + if (!contentConfirmed) { + return; + } + + // 7) Conflict resolution const conflicts = findConflicts(projectPath, files); const resolution = await resolveConflicts(conflicts); if (resolution === 'cancel') { return; } - // 7) Write + // 8) Write const { written, skipped } = writeProjectFiles(projectPath, files, resolution); if (written === 0 && skipped === 0) { vscode.window.showWarningMessage('No files were written.'); return; } - // 8) Git init + // 9) Git init if (shouldInitGit) { try { await initGit(projectPath); @@ -539,7 +640,7 @@ export async function runScaffoldFlow(options: ScaffoldOptions): Promise { } } - // 9) GitHub remote (optional) + // 10) GitHub remote (optional) await maybeCreateGitHubRemote(context, projectPath, projectName, idea); const skipNote = skipped > 0 ? ` (${skipped} existing skipped)` : ''; diff --git a/extension/src/streamGenerate.ts b/extension/src/streamGenerate.ts index 01a260e..4517d8d 100644 --- a/extension/src/streamGenerate.ts +++ b/extension/src/streamGenerate.ts @@ -28,6 +28,7 @@ export type StreamDoneEvent = { stack: string; files: Record; template_id?: string; + pack_id?: string; quality?: QualityIssue[]; }; @@ -52,6 +53,7 @@ export type StreamProgressEvent = export interface StreamGenerateOptions { idea: string; templateId?: string; + packId?: string; plan?: PlanResponse; bakeins?: BakeinOptions; jobId?: string; @@ -137,11 +139,14 @@ export function streamGenerate(options: StreamGenerateOptions): Promise { + const folders = vscode.workspace.workspaceFolders; + if (!folders || folders.length === 0) { + return undefined; + } + + if (folders.length === 1) { + return folders[0].uri.fsPath; + } + + const config = vscode.workspace.getConfiguration('creer'); + const hint = (config.get('defaultWorkspaceFolder') || '').trim(); + if (hint) { + const matched = folders.find( + (f) => f.name === hint || f.uri.fsPath === hint || f.uri.fsPath.endsWith(hint) + ); + if (matched) { + return matched.uri.fsPath; + } + } + + const items: Array = folders.map((f) => ({ + label: f.name, + description: f.uri.fsPath, + fsPath: f.uri.fsPath, + })); + + const picked = await vscode.window.showQuickPick(items, { + placeHolder: 'Select a workspace folder for the new project', + ignoreFocusOut: true, + matchOnDescription: true, + }); + + return picked?.fsPath; +} From d92d451e573c51dab63ca59e114313a417d1bdad Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 08:48:29 +0000 Subject: [PATCH 06/13] Implement Creer v0.6: marketplace packs, conflict diffs, publish prep Add remote pack install/delete and marketplace catalog, side-by-side vscode.diff review for write conflicts, and vsce/ovsx packaging docs so the extension is ready to publish without shipping secrets. Co-authored-by: Sanath S Patil --- PLAN.md | 35 +-- README.md | 30 ++- backend/.env.example | 2 + backend/.gitignore | 2 + backend/app/packs.py | 390 +++++++++++++++++++++++++++--- backend/main.py | 51 +++- backend/packs/installed/.gitkeep | 0 backend/tests/test_marketplace.py | 203 ++++++++++++++++ extension/.vscodeignore | 16 ++ extension/LICENSE | 21 ++ extension/PUBLISH.md | 82 +++++++ extension/README.md | 18 ++ extension/package.json | 29 ++- extension/src/api.ts | 50 ++++ extension/src/conflictDiff.ts | 196 +++++++++++++++ extension/src/extension.ts | 26 +- extension/src/marketplace.ts | 175 ++++++++++++++ extension/src/scaffold.ts | 6 +- 18 files changed, 1255 insertions(+), 77 deletions(-) create mode 100644 backend/packs/installed/.gitkeep create mode 100644 backend/tests/test_marketplace.py create mode 100644 extension/.vscodeignore create mode 100644 extension/LICENSE create mode 100644 extension/PUBLISH.md create mode 100644 extension/README.md create mode 100644 extension/src/conflictDiff.ts create mode 100644 extension/src/marketplace.ts diff --git a/PLAN.md b/PLAN.md index 0b64265..6d7734c 100644 --- a/PLAN.md +++ b/PLAN.md @@ -1,33 +1,18 @@ # Creer — Final Plan -v0.1 delivered the foundation: FastAPI planner/generator + VS Code command that writes a generated repo into the workspace (optional git init). +## Done -## v0.2 — done +- **v0.1** — FastAPI planner/generator + VS Code write-to-workspace (+ optional git) +- **v0.2** — Preview, GitHub create/push, templates, overwrite protection, `/creer` chat +- **v0.3** — Streaming, offline/local models, bake-ins, SecretStorage + GIT_ASKPASS +- **v0.4** — Stream cancel, selectable license/CI bake-ins, quality gates +- **v0.5** — Content diff preview before write, multi-root workspace targeting, installable JSON/YAML template packs (`GET /packs`, `pack_id`) +- **v0.6** — Pack marketplace UI + install from URL, side-by-side conflict diffs (`creer-generated`), Open VSX / Marketplace publish packaging -1. **Preview before writing** — plan preview markdown + confirm before generate/write (`creer.previewBeforeWrite`). -2. **GitHub repo creation** — `POST /github/create-repo` + extension remote add/push. -3. **Curated templates** — `GET /templates` + template-anchored `/plan` & `/generate`. -4. **Overwrite protection** — per-file conflict detection with overwrite / skip / cancel. -5. **Chat command `/creer`** — `creer.createRepoFromChat` + `@creer` chat participant. +## Optional next -## v0.3 — done - -1. **Streaming generation** — `POST /generate/stream` (SSE) + extension progress UI. -2. **Local / offline backends** — `OPENAI_BASE_URL`, `CREER_OFFLINE`. -3. **Open-source bake-ins** — LICENSE / README / CI via `bakeins.py`. -4. **Hardening** — `GIT_ASKPASS` + SecretStorage for GitHub tokens. - -## v0.4 — done - -1. **Cancellation** — `job_id` on stream + `POST /generate/cancel`; extension AbortSignal + cancellable progress. -2. **Selectable bake-ins** — license (`mit` / `apache-2.0` / `none`) and CI presets (`auto` / `python` / `node` / `none`); `GET /bakeins`. -3. **Quality gates** — telemetry-free tree checks (`quality` on generate/done; `POST /quality`). - -## v0.5 (optional next) - -- Diff preview of generated file contents before write -- Multi-root workspace targeting -- Template packs as installable JSON/YAML +- Backend marketplace catalog + remote pack install/delete endpoints (extension already codes to contracts) +- Extension icon + signed publisher release ## Non-goals diff --git a/README.md b/README.md index 7d92e59..9a6485d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ AI-powered repo scaffolding inside your workspace. -**Current version: 0.5.0** +**Current version: 0.6.0** ## Architecture @@ -19,7 +19,7 @@ creer/ - OpenAI API key **or** `OPENAI_BASE_URL` **or** `CREER_OFFLINE=1` (with template/pack) - VS Code / Cursor -## Backend (v0.5) +## Backend ```bash cd backend @@ -43,10 +43,13 @@ uvicorn main:app --reload --port 8000 | Method | Path | Description | |---|---|---| -| `GET` | `/health` | Version `0.5.0`, packs_count | +| `GET` | `/health` | Health / version | | `GET` | `/templates` | Built-in templates | | `GET` | `/packs` | Installable JSON/YAML packs | | `GET` | `/packs/{id}` | Single pack | +| `POST` | `/packs/install` | Install pack from URL (`{ url, overwrite? }`) | +| `DELETE` | `/packs/{id}` | Delete installed pack | +| `GET` | `/marketplace` | Remote/bundled marketplace items | | `GET` | `/bakeins` | License/CI options | | `POST` | `/plan` | Plan (`template_id` **or** `pack_id`) | | `POST` | `/generate` | Generate + bake-ins + quality | @@ -55,6 +58,8 @@ uvicorn main:app --reload --port 8000 | `POST` | `/quality` | Dry-run gates | | `POST` | `/github/create-repo` | Create GitHub repo | +Marketplace and pack install/delete may be absent on older backends; the extension handles that gracefully. + ### Packs Drop `.json` / `.yaml` files into `backend/packs/` (or `CREER_PACKS_DIR`): @@ -72,13 +77,13 @@ Drop `.json` / `.yaml` files into `backend/packs/` (or `CREER_PACKS_DIR`): Shipped examples: `fastapi-crud`, `express-ts`, `python-lib`. -## Extension (v0.5) +## Extension (v0.6) ```bash cd extension && npm install && npm run compile ``` -Flow: idea → template/pack → bake-ins → plan preview → generate (cancellable) → **content diff preview** → write → optional git/GitHub. +Flow: idea → template/pack → bake-ins → plan preview → generate (cancellable) → content diff preview → **conflict review diffs** → write → optional git/GitHub. Multi-root: QuickPick workspace folder (or `creer.defaultWorkspaceFolder`). @@ -90,6 +95,8 @@ Multi-root: QuickPick workspace folder (or `creer.defaultWorkspaceFolder`). | `creer.createRepoFromChat` | Creer: Create from Chat Prompt | | `creer.setGitHubToken` | Creer: Set GitHub Token | | `creer.clearGitHubToken` | Creer: Clear GitHub Token | +| `creer.installPackFromUrl` | Creer: Install Pack from URL | +| `creer.browseMarketplace` | Creer: Browse Pack Marketplace | ### Settings @@ -97,6 +104,7 @@ Multi-root: QuickPick workspace folder (or `creer.defaultWorkspaceFolder`). |---|---|---| | `creer.backendUrl` | `http://localhost:8000` | Backend URL | | `creer.contentPreview` | `true` | Diff/content preview before write | +| `creer.showConflictDiffs` | `true` | Offer side-by-side Review diffs on conflicts | | `creer.defaultWorkspaceFolder` | `""` | Multi-root folder name/path hint | | `creer.previewBeforeWrite` | `true` | Plan tree confirm before generate | | `creer.useStreaming` | `true` | SSE progress | @@ -106,6 +114,18 @@ Multi-root: QuickPick workspace folder (or `creer.defaultWorkspaceFolder`). | `creer.createGitHubRepo` | `false` | Create remote | | `creer.githubPrivate` | `true` | Private repos | +## Publishing + +The extension is ready to package for the **VS Marketplace** and **Open VSX** (no credentials in-repo). + +```bash +cd extension +npm run compile +npm run package # → creer-0.6.0.vsix via npx @vscode/vsce +``` + +Full steps (tokens, `ovsx publish`, checklist): see [`extension/PUBLISH.md`](extension/PUBLISH.md). + ## License MIT diff --git a/backend/.env.example b/backend/.env.example index bb3c577..bb2253c 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -14,4 +14,6 @@ CREER_OFFLINE= # Optional extra packs directory (JSON/YAML). Merged with backend/packs; # user packs override built-in packs on the same id. +# When set, remote installs (POST /packs/install) write here. +# When unset, installs go to backend/packs/installed/. # CREER_PACKS_DIR=/path/to/my-packs diff --git a/backend/.gitignore b/backend/.gitignore index 7d49b78..491ffd0 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -4,3 +4,5 @@ __pycache__/ *.pyc .pytest_cache/ .mypy_cache/ +packs/installed/* +!packs/installed/.gitkeep diff --git a/backend/app/packs.py b/backend/app/packs.py index f3a06c8..0285492 100644 --- a/backend/app/packs.py +++ b/backend/app/packs.py @@ -1,4 +1,4 @@ -"""Installable template packs (JSON/YAML) for Creer v0.5.""" +"""Installable template packs (JSON/YAML) for Creer v0.6 — including remote install.""" from __future__ import annotations @@ -6,7 +6,9 @@ import os import re from pathlib import Path +from urllib.parse import urlparse +import httpx import yaml from app.templates import slugify @@ -15,10 +17,22 @@ # backend/packs — resolved relative to backend root (parent of app/) _BACKEND_ROOT = Path(__file__).resolve().parent.parent PACKS_DIR = _BACKEND_ROOT / "packs" +INSTALLED_DIR = PACKS_DIR / "installed" _PACK_FILE_SUFFIXES = (".json", ".yaml", ".yml") _SAFE_ID = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,63}$") +MAX_DOWNLOAD_BYTES = 1_000_000 +FETCH_TIMEOUT_SECONDS = 30.0 + + +class PackConflictError(Exception): + """Raised when installing a pack id that already exists and overwrite is false.""" + + +class PackNotInstalledError(Exception): + """Raised when uninstall targets a pack that is not in the writable install dir.""" + def _extra_packs_dir() -> Path | None: """Optional user packs directory via CREER_PACKS_DIR.""" @@ -28,78 +42,67 @@ def _extra_packs_dir() -> Path | None: return Path(raw).expanduser().resolve() -def load_pack_file(path: Path | str) -> dict: +def writable_packs_dir() -> Path: """ - Load and validate a pack from a JSON or YAML file. + Directory where remotely installed packs are written. - Required: id, name, non-empty files list with safe relative paths. - Optional: description, stack, version. + Prefer CREER_PACKS_DIR when set; otherwise backend/packs/installed/. """ - path = Path(path) - if not path.is_file(): - raise ValueError(f"Pack file not found: {path}") + extra = _extra_packs_dir() + if extra is not None: + return extra + return INSTALLED_DIR - suffix = path.suffix.lower() - try: - text = path.read_text(encoding="utf-8") - except OSError as exc: - raise ValueError(f"Cannot read pack file {path}: {exc}") from exc - if suffix == ".json": - try: - data = json.loads(text) - except json.JSONDecodeError as exc: - raise ValueError(f"Invalid JSON in pack {path}: {exc}") from exc - elif suffix in (".yaml", ".yml"): - try: - data = yaml.safe_load(text) - except yaml.YAMLError as exc: - raise ValueError(f"Invalid YAML in pack {path}: {exc}") from exc - else: - raise ValueError(f"Unsupported pack file type: {path.suffix!r}") +def validate_pack_dict(data: dict, *, source: str = "") -> dict: + """ + Validate a pack mapping and return a normalized dict. + Required: id, name, non-empty files list with safe relative paths. + Optional: description, stack, version. + """ if not isinstance(data, dict): - raise ValueError(f"Pack must be a mapping/object: {path}") + raise ValueError(f"Pack must be a mapping/object: {source}") pack_id = data.get("id") name = data.get("name") files = data.get("files") if not isinstance(pack_id, str) or not pack_id.strip(): - raise ValueError(f"Pack missing valid id: {path}") + raise ValueError(f"Pack missing valid id: {source}") pack_id = pack_id.strip() if not _SAFE_ID.match(pack_id): - raise ValueError(f"Invalid pack id {pack_id!r} in {path}") + raise ValueError(f"Invalid pack id {pack_id!r} in {source}") if not isinstance(name, str) or not name.strip(): - raise ValueError(f"Pack missing valid name: {path}") + raise ValueError(f"Pack missing valid name: {source}") if not isinstance(files, list) or not files: - raise ValueError(f"Pack must include a non-empty files list: {path}") + raise ValueError(f"Pack must include a non-empty files list: {source}") validated_files: list[str] = [] seen: set[str] = set() for entry in files: if not isinstance(entry, str) or not entry.strip(): - raise ValueError(f"Invalid file path in pack {path}: {entry!r}") + raise ValueError(f"Invalid file path in pack {source}: {entry!r}") fpath = entry.strip().replace("\\", "/") _reject_unsafe_path(fpath, label="pack file path") if fpath in seen: - raise ValueError(f"Duplicate file path in pack {path}: {fpath!r}") + raise ValueError(f"Duplicate file path in pack {source}: {fpath!r}") seen.add(fpath) validated_files.append(fpath) version = data.get("version", "1.0.0") if version is not None and not isinstance(version, str): - raise ValueError(f"Pack version must be a string: {path}") + raise ValueError(f"Pack version must be a string: {source}") stack = data.get("stack", "") if stack is not None and not isinstance(stack, str): - raise ValueError(f"Pack stack must be a string: {path}") + raise ValueError(f"Pack stack must be a string: {source}") description = data.get("description", "") if description is not None and not isinstance(description, str): - raise ValueError(f"Pack description must be a string: {path}") + raise ValueError(f"Pack description must be a string: {source}") return { "id": pack_id, @@ -111,6 +114,115 @@ def load_pack_file(path: Path | str) -> dict: } +def _detect_pack_format( + hint_filename: str | None = None, + content_type: str | None = None, + text: str | None = None, +) -> str: + """Return 'json' or 'yaml' based on filename, Content-Type, and/or body sniff.""" + if hint_filename: + lower = hint_filename.lower().split("?", 1)[0] + if lower.endswith(".json"): + return "json" + if lower.endswith(".yaml") or lower.endswith(".yml"): + return "yaml" + + if content_type: + ct = content_type.split(";", 1)[0].strip().lower() + if ct in ("application/json", "text/json"): + return "json" + if ct in ( + "application/yaml", + "application/x-yaml", + "text/yaml", + "text/x-yaml", + ): + return "yaml" + + if text is not None: + stripped = text.lstrip() + if stripped.startswith("{") or stripped.startswith("["): + return "json" + + # Default: try JSON first via caller; prefer yaml when ambiguous after sniff + return "yaml" if text is not None else "json" + + +def parse_pack_content( + text_or_bytes: str | bytes, + hint_filename: str | None = None, + *, + content_type: str | None = None, +) -> dict: + """Parse and validate pack content from text or bytes.""" + if isinstance(text_or_bytes, bytes): + try: + text = text_or_bytes.decode("utf-8") + except UnicodeDecodeError as exc: + raise ValueError(f"Pack content is not valid UTF-8: {exc}") from exc + else: + text = text_or_bytes + + source = hint_filename or "" + fmt = _detect_pack_format(hint_filename, content_type, text) + + data: object + if fmt == "json": + try: + data = json.loads(text) + except json.JSONDecodeError as exc: + # Fallback: maybe mis-detected YAML + if hint_filename and hint_filename.lower().endswith(".json"): + raise ValueError(f"Invalid JSON in pack {source}: {exc}") from exc + try: + data = yaml.safe_load(text) + except yaml.YAMLError as yexc: + raise ValueError(f"Invalid pack content {source}: {exc}") from yexc + else: + try: + data = yaml.safe_load(text) + except yaml.YAMLError as exc: + raise ValueError(f"Invalid YAML in pack {source}: {exc}") from exc + + if not isinstance(data, dict): + raise ValueError(f"Pack must be a mapping/object: {source}") + + return validate_pack_dict(data, source=source) + + +def parse_pack_bytes( + data: bytes, + hint_filename: str | None = None, + *, + content_type: str | None = None, +) -> dict: + """Parse pack bytes (alias of parse_pack_content for bytes).""" + return parse_pack_content(data, hint_filename, content_type=content_type) + + +def load_pack_file(path: Path | str) -> dict: + """ + Load and validate a pack from a JSON or YAML file. + + Required: id, name, non-empty files list with safe relative paths. + Optional: description, stack, version. + """ + path = Path(path) + if not path.is_file(): + raise ValueError(f"Pack file not found: {path}") + + suffix = path.suffix.lower() + if suffix not in _PACK_FILE_SUFFIXES: + raise ValueError(f"Unsupported pack file type: {path.suffix!r}") + + try: + text = path.read_text(encoding="utf-8") + except OSError as exc: + raise ValueError(f"Cannot read pack file {path}: {exc}") from exc + + return parse_pack_content(text, hint_filename=path.name) + + def _load_packs_from_dir(directory: Path) -> dict[str, dict]: """Load all valid pack files from a directory (non-recursive).""" result: dict[str, dict] = {} @@ -133,19 +245,21 @@ def _load_packs_from_dir(directory: Path) -> dict[str, dict]: def _all_packs() -> dict[str, dict]: """ - Merge built-in packs with optional CREER_PACKS_DIR. + Merge built-in packs, installed packs, and optional CREER_PACKS_DIR. - User packs override built-in packs on id collision. + Later sources override earlier ones on id collision: + built-in → installed/ → CREER_PACKS_DIR. """ packs = _load_packs_from_dir(PACKS_DIR) + packs.update(_load_packs_from_dir(INSTALLED_DIR)) extra = _extra_packs_dir() - if extra is not None: + if extra is not None and extra.resolve() != INSTALLED_DIR.resolve(): packs.update(_load_packs_from_dir(extra)) return packs def list_packs() -> list[dict]: - """Return all available packs (built-in + user), sorted by id.""" + """Return all available packs (built-in + installed + user), sorted by id.""" packs = _all_packs() return [dict(packs[k]) for k in sorted(packs.keys())] @@ -158,6 +272,204 @@ def get_pack(pack_id: str) -> dict | None: return dict(pack) if pack else None +def install_pack(pack: dict, *, overwrite: bool = False) -> dict: + """ + Write a validated pack dict as ``{id}.json`` into the writable packs dir. + + Raises PackConflictError if the id already exists and overwrite is False. + """ + validated = validate_pack_dict(pack, source="") + pack_id = validated["id"] + + if get_pack(pack_id) is not None and not overwrite: + raise PackConflictError(f"Pack already exists: {pack_id!r}") + + target_dir = writable_packs_dir() + target_dir.mkdir(parents=True, exist_ok=True) + target = target_dir / f"{pack_id}.json" + target.write_text( + json.dumps(validated, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + return dict(validated) + + +def install_pack_from_bytes( + data: bytes, + *, + hint_filename: str = "pack.json", + content_type: str | None = None, + overwrite: bool = False, +) -> dict: + """Parse pack bytes and install into the writable packs dir.""" + pack = parse_pack_bytes(data, hint_filename, content_type=content_type) + return install_pack(pack, overwrite=overwrite) + + +def uninstall_pack(pack_id: str) -> bool: + """ + Remove an installed pack from the writable dir only. + + Returns True if a file was deleted. + Raises PackNotInstalledError if the pack is not present under the writable dir + (built-in shipped packs cannot be deleted this way). + """ + if not pack_id or not _SAFE_ID.match(pack_id): + raise ValueError(f"Invalid pack id: {pack_id!r}") + + target_dir = writable_packs_dir() + if not target_dir.is_dir(): + raise PackNotInstalledError(f"Pack not installed (writable): {pack_id!r}") + + candidates = [ + target_dir / f"{pack_id}.json", + target_dir / f"{pack_id}.yaml", + target_dir / f"{pack_id}.yml", + ] + deleted = False + for path in candidates: + if path.is_file(): + path.unlink() + deleted = True + + if not deleted: + raise PackNotInstalledError(f"Pack not installed (writable): {pack_id!r}") + return True + + +def validate_remote_pack_url(url: str) -> str: + """Allow only http/https URLs with a host. Returns the stripped URL.""" + if not isinstance(url, str) or not url.strip(): + raise ValueError("url is required") + url = url.strip() + parsed = urlparse(url) + if parsed.scheme not in ("http", "https"): + raise ValueError( + f"Only http/https URLs are allowed, got scheme {parsed.scheme!r}" + ) + if not parsed.hostname: + raise ValueError("URL must include a host") + return url + + +def _hint_filename_from_url(url: str) -> str: + path = urlparse(url).path or "" + name = Path(path).name + if name and any(name.lower().endswith(s) for s in _PACK_FILE_SUFFIXES): + return name + return "pack.json" + + +def fetch_pack_bytes(url: str) -> tuple[bytes, str | None, str]: + """ + Download pack content from a remote URL. + + Returns (body, content_type, hint_filename). + Enforces http(s), ~30s timeout, and 1MB max size. + """ + url = validate_remote_pack_url(url) + hint = _hint_filename_from_url(url) + + try: + with httpx.Client( + timeout=FETCH_TIMEOUT_SECONDS, + follow_redirects=True, + ) as client: + with client.stream("GET", url) as resp: + if resp.status_code >= 400: + raise ValueError( + f"Failed to fetch pack ({resp.status_code}): {url}" + ) + content_type = resp.headers.get("content-type") + chunks: list[bytes] = [] + total = 0 + for chunk in resp.iter_bytes(): + if not chunk: + continue + total += len(chunk) + if total > MAX_DOWNLOAD_BYTES: + raise ValueError( + f"Pack download exceeds {MAX_DOWNLOAD_BYTES} byte limit" + ) + chunks.append(chunk) + body = b"".join(chunks) + except httpx.HTTPError as exc: + raise ValueError(f"Failed to fetch pack URL: {exc}") from exc + + if not body: + raise ValueError("Pack download was empty") + + return body, content_type, hint + + +def install_pack_from_url(url: str, *, overwrite: bool = False) -> dict: + """Fetch a remote pack URL, validate, and install.""" + body, content_type, hint = fetch_pack_bytes(url) + return install_pack_from_bytes( + body, + hint_filename=hint, + content_type=content_type, + overwrite=overwrite, + ) + + +def marketplace_catalog() -> list[dict]: + """ + Static curated marketplace catalog. + + Bundled entries mirror the three shipped packs (offline/demo). + Remote entries are illustrative placeholders — do not require network in tests. + """ + return [ + { + "id": "fastapi-crud", + "name": "FastAPI CRUD", + "description": "FastAPI starter with models, CRUD routes, and uvicorn entrypoint.", + "source": "bundled", + "url": None, + }, + { + "id": "python-lib", + "name": "Python Library", + "description": "Publishable Python library with pyproject, package layout, and tests.", + "source": "bundled", + "url": None, + }, + { + "id": "express-ts", + "name": "Express TypeScript", + "description": "Express API in TypeScript with tsconfig and basic router.", + "source": "bundled", + "url": None, + }, + { + "id": "demo-remote", + "name": "Demo Remote Pack", + "description": ( + "Example remote pack URL (placeholder). Install via POST /packs/install " + "with this url — for demos/docs only; tests should mock the fetch." + ), + "url": ( + "https://raw.githubusercontent.com/example/creer-packs/main/" + "demo-remote.json" + ), + "source": "remote", + }, + { + "id": "hello-cli", + "name": "Hello CLI Pack", + "description": ( + "Example remote CLI starter pack URL (placeholder for marketplace demos)." + ), + "url": ( + "https://raw.githubusercontent.com/example/creer-packs/main/" + "hello-cli.yaml" + ), + "source": "remote", + }, + ] + + def apply_pack(pack_id: str, idea: str) -> dict: """ Build a plan from an installable pack. diff --git a/backend/main.py b/backend/main.py index 8b98684..27b8ec3 100644 --- a/backend/main.py +++ b/backend/main.py @@ -16,12 +16,20 @@ from app.generator import generate_files, generate_files_iter from app.validator import validate_plan, validate_files from app.templates import list_templates, get_template -from app.packs import list_packs, get_pack +from app.packs import ( + PackConflictError, + PackNotInstalledError, + get_pack, + install_pack_from_url, + list_packs, + marketplace_catalog, + uninstall_pack, +) from app.github import create_github_repo from app.jobs import cancel_job, create_job, finish_job, is_cancelled from app.quality import has_errors, run_quality_gates -VERSION = "0.5.0" +VERSION = "0.6.0" app = FastAPI(title="Creer", version=VERSION) @@ -73,6 +81,11 @@ class GitHubCreateRepoRequest(BaseModel): token: str | None = None +class PackInstallRequest(BaseModel): + url: str = Field(..., min_length=1, max_length=2000) + overwrite: bool = False + + def _check_pack_template_exclusive( pack_id: str | None, template_id: str | None ) -> None: @@ -146,6 +159,28 @@ def packs(): return {"packs": list_packs()} +@app.get("/marketplace") +def marketplace(): + """Static curated catalog of bundled + example remote packs.""" + return {"items": marketplace_catalog()} + + +@app.post("/packs/install") +def packs_install(request: PackInstallRequest): + """Fetch a remote pack URL (http/https), validate, and install locally.""" + try: + pack = install_pack_from_url(request.url, overwrite=request.overwrite) + except PackConflictError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException( + status_code=500, detail=f"Pack install failed: {exc}" + ) from exc + return {"installed": True, "pack": pack} + + @app.get("/packs/{pack_id}") def pack_detail(pack_id: str): pack = get_pack(pack_id) @@ -154,6 +189,18 @@ def pack_detail(pack_id: str): return pack +@app.delete("/packs/{pack_id}") +def packs_delete(pack_id: str): + """Delete a pack from the writable installed dir only (not shipped examples).""" + try: + uninstall_pack(pack_id) + except PackNotInstalledError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return {"deleted": True, "pack_id": pack_id} + + @app.get("/bakeins") def bakeins(): return list_bakein_options() diff --git a/backend/packs/installed/.gitkeep b/backend/packs/installed/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/test_marketplace.py b/backend/tests/test_marketplace.py new file mode 100644 index 0000000..62fa9c7 --- /dev/null +++ b/backend/tests/test_marketplace.py @@ -0,0 +1,203 @@ +"""Creer v0.6 marketplace / remote pack install tests (offline).""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +# Ensure offline before importing the app +os.environ["CREER_OFFLINE"] = "1" +# Isolate installs into a temp dir via fixture; clear CREER_PACKS_DIR by default +os.environ.pop("CREER_PACKS_DIR", None) + +from app import packs as packs_mod +from main import VERSION, app + + +SAMPLE_PACK = { + "id": "marketplace-demo", + "name": "Marketplace Demo", + "description": "Installed via tests", + "stack": "demo", + "version": "1.0.0", + "files": ["README.md", "main.py"], +} + + +@pytest.fixture() +def install_dir(tmp_path, monkeypatch): + """Point writable installs at a temp directory (not repo packs/).""" + target = tmp_path / "installed" + target.mkdir() + monkeypatch.setenv("CREER_PACKS_DIR", str(target)) + # Also keep INSTALLED_DIR clean conceptually — writable_packs_dir uses env + yield target + + +@pytest.fixture() +def client(install_dir): + return TestClient(app) + + +def test_health_version_0_6(client): + resp = client.get("/health") + assert resp.status_code == 200 + data = resp.json() + assert data["version"] == "0.6.0" + assert VERSION == "0.6.0" + assert data["offline"] is True + assert data["packs_count"] >= 3 + + +def test_marketplace_catalog(client): + resp = client.get("/marketplace") + assert resp.status_code == 200 + items = resp.json()["items"] + assert len(items) >= 4 + bundled = [i for i in items if i.get("source") == "bundled"] + remote = [i for i in items if i.get("source") == "remote"] + assert len(bundled) == 3 + assert {b["id"] for b in bundled} == {"fastapi-crud", "python-lib", "express-ts"} + assert len(remote) >= 1 + assert all("url" in r for r in remote) + + +def test_install_pack_from_bytes(install_dir): + body = json.dumps(SAMPLE_PACK).encode("utf-8") + installed = packs_mod.install_pack_from_bytes(body, overwrite=False) + assert installed["id"] == "marketplace-demo" + assert (install_dir / "marketplace-demo.json").is_file() + ids = {p["id"] for p in packs_mod.list_packs()} + assert "marketplace-demo" in ids + + +def test_install_endpoint_mocked_httpx(client, install_dir, monkeypatch): + body = json.dumps(SAMPLE_PACK).encode("utf-8") + + class FakeResponse: + status_code = 200 + headers = {"content-type": "application/json"} + + def iter_bytes(self): + yield body + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + class FakeClient: + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def stream(self, method, url): + assert method == "GET" + assert url.startswith("https://") + return FakeResponse() + + monkeypatch.setattr(packs_mod.httpx, "Client", FakeClient) + + resp = client.post( + "/packs/install", + json={"url": "https://example.com/packs/marketplace-demo.json", "overwrite": False}, + ) + assert resp.status_code == 200 + data = resp.json() + assert data["installed"] is True + assert data["pack"]["id"] == "marketplace-demo" + + packs = client.get("/packs").json()["packs"] + assert any(p["id"] == "marketplace-demo" for p in packs) + + +def test_install_overwrite_false_conflict(client, install_dir, monkeypatch): + packs_mod.install_pack(SAMPLE_PACK, overwrite=False) + with pytest.raises(packs_mod.PackConflictError): + packs_mod.install_pack(SAMPLE_PACK, overwrite=False) + + body = json.dumps(SAMPLE_PACK).encode("utf-8") + + class FakeResponse: + status_code = 200 + headers = {"content-type": "application/json"} + + def iter_bytes(self): + yield body + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + class FakeClient: + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def stream(self, method, url): + return FakeResponse() + + monkeypatch.setattr(packs_mod.httpx, "Client", FakeClient) + resp = client.post( + "/packs/install", + json={ + "url": "https://example.com/packs/marketplace-demo.json", + "overwrite": False, + }, + ) + assert resp.status_code == 409 + + +def test_delete_installed_only(client, install_dir): + packs_mod.install_pack(SAMPLE_PACK, overwrite=True) + assert any(p["id"] == "marketplace-demo" for p in packs_mod.list_packs()) + + resp = client.delete("/packs/marketplace-demo") + assert resp.status_code == 200 + assert resp.json()["deleted"] is True + assert not (install_dir / "marketplace-demo.json").exists() + assert not any(p["id"] == "marketplace-demo" for p in packs_mod.list_packs()) + + # Built-in cannot be deleted via writable dir + resp = client.delete("/packs/fastapi-crud") + assert resp.status_code == 404 + # Still listed (bundled) + assert any(p["id"] == "fastapi-crud" for p in client.get("/packs").json()["packs"]) + + +def test_reject_non_http_scheme(): + with pytest.raises(ValueError, match="http/https"): + packs_mod.validate_remote_pack_url("file:///etc/passwd") + with pytest.raises(ValueError, match="http/https"): + packs_mod.validate_remote_pack_url("ftp://example.com/pack.json") + + +def test_parse_yaml_pack(): + text = """ +id: yaml-demo +name: YAML Demo +description: from yaml +stack: yaml +version: "1.0.0" +files: + - README.md +""" + pack = packs_mod.parse_pack_content(text, "pack.yaml") + assert pack["id"] == "yaml-demo" diff --git a/extension/.vscodeignore b/extension/.vscodeignore new file mode 100644 index 0000000..0637e65 --- /dev/null +++ b/extension/.vscodeignore @@ -0,0 +1,16 @@ +.vscode/** +.vscode-test/** +src/** +tsconfig.json +**/*.ts +!**/*.d.ts +**/*.map +.gitignore +.eslintignore +.eslintrc* +**/.DS_Store +**/*.vsix +node_modules/@types/** +node_modules/typescript/** +*.md +!README.md diff --git a/extension/LICENSE b/extension/LICENSE new file mode 100644 index 0000000..88cdccb --- /dev/null +++ b/extension/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Sanath S Patil + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/extension/PUBLISH.md b/extension/PUBLISH.md new file mode 100644 index 0000000..090781b --- /dev/null +++ b/extension/PUBLISH.md @@ -0,0 +1,82 @@ +# Publishing Creer (VS Code / Open VSX) + +The extension is packaged from compiled `out/` JavaScript. **Do not commit secrets** (personal access tokens). Tokens stay in your environment only. + +## Prerequisites + +- Node.js 18+ +- Compiled extension: `cd extension && npm install && npm run compile` +- Publisher identity: + - **VS Marketplace:** Azure DevOps PAT with Marketplace (Acquire / Publish) scopes, and a [publisher](https://marketplace.visualstudio.com/manage) matching `package.json` → `publisher` + - **Open VSX:** account + token from [open-vsx.org](https://open-vsx.org/) + +## Package a `.vsix` (no publish) + +```bash +cd extension +npm run compile +npm run package +# or: npx --yes @vscode/vsce package +``` + +This runs `vsce package`, respects `.vscodeignore`, and includes production dependencies (e.g. `axios`). Output: `creer-0.6.0.vsix` (version from `package.json`). + +Install locally for a smoke test: + +```bash +code --install-extension creer-0.6.0.vsix +# or Cursor: cursor --install-extension creer-0.6.0.vsix +``` + +## Publish to VS Marketplace + +1. Create a publisher if needed and ensure `package.json` `publisher` matches. +2. Export a PAT (do not paste into git or chat logs): + + ```bash + export VSCE_PAT=your_marketplace_pat + ``` + +3. Publish: + + ```bash + cd extension + npx --yes @vscode/vsce publish + # or: npm run publish:marketplace # prints the command reminder + ``` + +Optional: `npx @vscode/vsce publish -p "$VSCE_PAT"`. + +## Publish to Open VSX + +1. Create an Open VSX token. +2. Export it: + + ```bash + export OVSX_PAT=your_open_vsx_token + ``` + +3. Publish the same extension (package first if you want a local `.vsix`): + + ```bash + cd extension + npx --yes ovsx publish + # with an existing vsix: + # npx --yes ovsx publish creer-0.6.0.vsix + ``` + +`npm run publish:ovsx` only documents this flow (exits non-zero so CI does not publish by accident). + +## Checklist before publish + +- [ ] `version` bumped in `extension/package.json` +- [ ] `npm run compile` succeeds +- [ ] `license`, `repository`, `homepage`, `bugs` fields look correct +- [ ] `.vscodeignore` excludes `src/`, maps, and tooling (ships `out/`) +- [ ] Manual smoke: Create New Repo + Install Pack from URL against a running backend +- [ ] No tokens in repo files or commit history + +## Notes + +- Icon: add `icon.png` and `"icon": "icon.png"` in `package.json` when you have artwork. +- Prefer `npx @vscode/vsce` / `npx ovsx` so `@vscode/vsce` need not be a permanent `devDependency`. diff --git a/extension/README.md b/extension/README.md new file mode 100644 index 0000000..222decf --- /dev/null +++ b/extension/README.md @@ -0,0 +1,18 @@ +# Creer (VS Code extension) + +AI-powered repo scaffolding inside your workspace. + +See the [repository README](https://github.com/seven0070/Creer#readme) for backend setup, commands, and settings. + +## Develop + +```bash +npm install +npm run compile +``` + +Press **F5** in VS Code/Cursor to launch an Extension Development Host (see `.vscode/launch.json`). + +## Publishing + +Package and publish steps (Marketplace / Open VSX) are documented in [PUBLISH.md](./PUBLISH.md). No publish tokens belong in this repo. diff --git a/extension/package.json b/extension/package.json index 0ffed8a..6806bac 100644 --- a/extension/package.json +++ b/extension/package.json @@ -2,8 +2,17 @@ "name": "creer", "displayName": "Creer", "description": "AI-powered repo scaffolding inside your workspace", - "version": "0.5.0", + "version": "0.6.0", "publisher": "creer", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/seven0070/Creer.git" + }, + "homepage": "https://github.com/seven0070/Creer", + "bugs": { + "url": "https://github.com/seven0070/Creer/issues" + }, "engines": { "vscode": "^1.85.0" }, @@ -30,6 +39,14 @@ { "command": "creer.clearGitHubToken", "title": "Creer: Clear GitHub Token" + }, + { + "command": "creer.installPackFromUrl", + "title": "Creer: Install Pack from URL" + }, + { + "command": "creer.browseMarketplace", + "title": "Creer: Browse Pack Marketplace" } ], "chatParticipants": [ @@ -102,6 +119,11 @@ "default": true, "description": "After generate, before write, show a content preview/diff and confirm" }, + "creer.showConflictDiffs": { + "type": "boolean", + "default": true, + "description": "When conflicts exist, offer Review diffs (side-by-side vscode.diff). When false, only Overwrite all / Skip existing / Cancel" + }, "creer.defaultWorkspaceFolder": { "type": "string", "default": "", @@ -114,7 +136,10 @@ "vscode:prepublish": "npm run compile", "compile": "tsc -p ./", "watch": "tsc -watch -p ./", - "lint": "tsc --noEmit -p ./" + "lint": "tsc --noEmit -p ./", + "package": "npx --yes @vscode/vsce package", + "publish:ovsx": "echo \"Set OVSX_PAT then run: npx --yes ovsx publish\" && exit 1", + "publish:marketplace": "echo \"Set VSCE_PAT then run: npx --yes @vscode/vsce publish\" && exit 1" }, "dependencies": { "axios": "^1.7.9" diff --git a/extension/src/api.ts b/extension/src/api.ts index 71f8c93..36f0dd9 100644 --- a/extension/src/api.ts +++ b/extension/src/api.ts @@ -18,6 +18,15 @@ export interface Pack { files: string[]; } +export interface MarketplaceItem { + id: string; + name: string; + description: string; + /** e.g. bundled | remote | url host */ + source: string; + url?: string; +} + export interface PlanResponse { project_name: string; stack?: string; @@ -119,6 +128,47 @@ export async function fetchPacks(): Promise { return response.data.packs ?? []; } +/** + * GET /marketplace → { items: MarketplaceItem[] } + * Callers should treat missing/404 endpoints as “unavailable”. + */ +export async function fetchMarketplace(): Promise { + const backendUrl = getBackendUrl(); + const response = await axios.get<{ items: MarketplaceItem[] }>( + `${backendUrl}/marketplace`, + { timeout: 30_000 } + ); + return response.data.items ?? []; +} + +/** + * POST /packs/install → { url, overwrite? } → installed pack + */ +export async function installPack( + url: string, + overwrite?: boolean +): Promise { + const backendUrl = getBackendUrl(); + const body: { url: string; overwrite?: boolean } = { url }; + if (overwrite !== undefined) { + body.overwrite = overwrite; + } + const response = await axios.post(`${backendUrl}/packs/install`, body, { + timeout: 120_000, + }); + return response.data; +} + +/** + * DELETE /packs/{id} → delete installed pack + */ +export async function deletePack(id: string): Promise { + const backendUrl = getBackendUrl(); + await axios.delete(`${backendUrl}/packs/${encodeURIComponent(id)}`, { + timeout: 30_000, + }); +} + export async function fetchBakeins(): Promise { const backendUrl = getBackendUrl(); const response = await axios.get(`${backendUrl}/bakeins`, { diff --git a/extension/src/conflictDiff.ts b/extension/src/conflictDiff.ts new file mode 100644 index 0000000..1e1d42c --- /dev/null +++ b/extension/src/conflictDiff.ts @@ -0,0 +1,196 @@ +import * as path from 'path'; +import * as vscode from 'vscode'; +import { resolveSafeProjectPath, type ConflictResolution } from './writeFiles'; + +export const CREER_GENERATED_SCHEME = 'creer-generated'; + +const MAX_DIFF_FILES = 15; + +/** + * Serves generated file contents for side-by-side `vscode.diff` against on-disk files. + * Content is keyed by the `path` query parameter (relative project path). + */ +export class CreerGeneratedContentProvider implements vscode.TextDocumentContentProvider { + private readonly contents = new Map(); + private readonly _onDidChange = new vscode.EventEmitter(); + readonly onDidChange = this._onDidChange.event; + + setContent(relativePath: string, content: string): void { + const key = normalizeRelPath(relativePath); + this.contents.set(key, content); + this._onDidChange.fire(generatedUri(key)); + } + + setMany(files: Record, relativePaths: string[]): void { + for (const rel of relativePaths) { + this.setContent(rel, files[rel] ?? ''); + } + } + + clear(): void { + this.contents.clear(); + } + + provideTextDocumentContent(uri: vscode.Uri): string { + const key = pathFromUri(uri); + return this.contents.get(key) ?? ''; + } +} + +let sharedProvider: CreerGeneratedContentProvider | undefined; + +export function registerConflictDiffProvider( + context: vscode.ExtensionContext +): CreerGeneratedContentProvider { + const provider = new CreerGeneratedContentProvider(); + sharedProvider = provider; + context.subscriptions.push( + vscode.workspace.registerTextDocumentContentProvider(CREER_GENERATED_SCHEME, provider), + { dispose: () => { + if (sharedProvider === provider) { + sharedProvider = undefined; + } + provider.clear(); + } } + ); + return provider; +} + +function getProvider(): CreerGeneratedContentProvider { + if (!sharedProvider) { + // Lazy fallback if activate forgot to register (tests / unexpected hosts) + sharedProvider = new CreerGeneratedContentProvider(); + } + return sharedProvider; +} + +function normalizeRelPath(relativePath: string): string { + return relativePath.replace(/\\/g, '/'); +} + +function pathFromUri(uri: vscode.Uri): string { + const q = new URLSearchParams(uri.query); + const fromQuery = q.get('path'); + if (fromQuery) { + return normalizeRelPath(fromQuery); + } + return normalizeRelPath(uri.path.replace(/^\//, '')); +} + +export function generatedUri(relativePath: string): vscode.Uri { + const normalized = normalizeRelPath(relativePath); + return vscode.Uri.from({ + scheme: CREER_GENERATED_SCHEME, + path: '/' + normalized, + query: `path=${encodeURIComponent(normalized)}`, + }); +} + +/** + * Open side-by-side diffs: existing on-disk file (left) vs generated content (right). + * Caps at MAX_DIFF_FILES to avoid flooding the editor. + */ +export async function openConflictDiffs( + projectPath: string, + files: Record, + conflicts: string[], + provider?: CreerGeneratedContentProvider +): Promise { + const p = provider ?? getProvider(); + const toShow = conflicts.slice(0, MAX_DIFF_FILES); + p.setMany(files, toShow); + + for (const rel of toShow) { + let existingUri: vscode.Uri; + try { + const fullPath = resolveSafeProjectPath(projectPath, rel); + existingUri = vscode.Uri.file(fullPath); + } catch { + continue; + } + const generated = generatedUri(rel); + const title = `${path.basename(rel)} (existing ↔ generated)`; + await vscode.commands.executeCommand('vscode.diff', existingUri, generated, title); + } + + return toShow.length; +} + +function conflictMessage(conflicts: string[]): string { + const preview = conflicts.slice(0, 10); + const extra = conflicts.length > 10 ? `\n…and ${conflicts.length - 10} more` : ''; + const list = preview.map((p) => `• ${p}`).join('\n'); + return `${conflicts.length} file(s) already exist under the project folder:\n${list}${extra}`; +} + +async function promptOverwriteSkipCancel( + message: string, + detail?: string +): Promise { + const choice = await vscode.window.showWarningMessage( + message, + { modal: true, detail }, + 'Overwrite all', + 'Skip existing', + 'Cancel' + ); + if (choice === 'Overwrite all') { + return 'overwrite'; + } + if (choice === 'Skip existing') { + return 'skip'; + } + return 'cancel'; +} + +/** + * Conflict resolution with optional side-by-side review via `creer-generated` diffs. + * When `creer.showConflictDiffs` is false, only Overwrite / Skip / Cancel are offered. + */ +export async function resolveConflictsWithDiffs( + projectPath: string, + files: Record, + conflicts: string[] +): Promise { + if (conflicts.length === 0) { + return 'overwrite'; + } + + const config = vscode.workspace.getConfiguration('creer'); + const showDiffs = config.get('showConflictDiffs') ?? true; + const message = conflictMessage(conflicts); + + if (!showDiffs) { + return promptOverwriteSkipCancel(message); + } + + const choice = await vscode.window.showWarningMessage( + message, + { modal: true }, + 'Review diffs', + 'Overwrite all', + 'Skip existing', + 'Cancel' + ); + + if (choice === 'Overwrite all') { + return 'overwrite'; + } + if (choice === 'Skip existing') { + return 'skip'; + } + if (choice !== 'Review diffs') { + return 'cancel'; + } + + const opened = await openConflictDiffs(projectPath, files, conflicts); + const more = + conflicts.length > MAX_DIFF_FILES + ? ` (showing ${opened} of ${conflicts.length})` + : ''; + + return promptOverwriteSkipCancel( + `Reviewed ${opened} conflicted file(s)${more}. How should Creer proceed?`, + 'Overwrite all replaces existing files. Skip existing keeps them and writes only new paths.' + ); +} diff --git a/extension/src/extension.ts b/extension/src/extension.ts index abac31a..3b54884 100644 --- a/extension/src/extension.ts +++ b/extension/src/extension.ts @@ -1,8 +1,15 @@ import * as vscode from 'vscode'; +import { registerConflictDiffProvider } from './conflictDiff'; +import { + browseMarketplaceCommand, + installPackFromUrlCommand, +} from './marketplace'; import { runScaffoldFlow } from './scaffold'; import { clearGitHubToken, setGitHubToken } from './secrets'; export function activate(context: vscode.ExtensionContext) { + registerConflictDiffProvider(context); + const createRepo = vscode.commands.registerCommand('creer.createRepo', async () => { await runScaffoldFlow({ context, fromChat: false }); }); @@ -35,7 +42,24 @@ export function activate(context: vscode.ExtensionContext) { void vscode.window.showInformationMessage('Creer: GitHub token cleared from SecretStorage.'); }); - context.subscriptions.push(createRepo, createRepoFromChat, setToken, clearToken); + const installPackFromUrl = vscode.commands.registerCommand( + 'creer.installPackFromUrl', + () => installPackFromUrlCommand() + ); + + const browseMarketplace = vscode.commands.registerCommand( + 'creer.browseMarketplace', + () => browseMarketplaceCommand() + ); + + context.subscriptions.push( + createRepo, + createRepoFromChat, + setToken, + clearToken, + installPackFromUrl, + browseMarketplace + ); registerChatParticipant(context); } diff --git a/extension/src/marketplace.ts b/extension/src/marketplace.ts new file mode 100644 index 0000000..cb3db3d --- /dev/null +++ b/extension/src/marketplace.ts @@ -0,0 +1,175 @@ +import * as vscode from 'vscode'; +import { + deletePack, + fetchMarketplace, + formatAxiosError, + installPack, + type MarketplaceItem, +} from './api'; + +/** + * Creer: Install Pack from URL — prompt for URL, POST /packs/install. + */ +export async function installPackFromUrlCommand(): Promise { + const url = await vscode.window.showInputBox({ + prompt: 'Pack URL (JSON/YAML pack definition)', + placeHolder: 'https://example.com/packs/fastapi-crud.json', + ignoreFocusOut: true, + }); + const trimmed = url?.trim(); + if (!trimmed) { + return; + } + + try { + const pack = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'Creer: installing pack…', + cancellable: false, + }, + () => installPack(trimmed) + ); + const name = pack.name || pack.id || 'pack'; + void vscode.window.showInformationMessage( + `Creer: installed pack “${name}”${pack.id ? ` (${pack.id})` : ''}.` + ); + } catch (err) { + const message = formatAxiosError(err, 'Failed to install pack'); + void vscode.window.showErrorMessage(`Creer: could not install pack — ${message}`); + } +} + +function isBundledSource(source: string | undefined): boolean { + if (!source) { + return false; + } + const s = source.toLowerCase(); + return s === 'bundled' || s === 'builtin' || s === 'built-in' || s === 'local'; +} + +/** + * Creer: Browse Pack Marketplace — GET /marketplace, QuickPick, optional install. + */ +export async function browseMarketplaceCommand(): Promise { + let items: MarketplaceItem[]; + try { + items = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'Creer: loading marketplace…', + cancellable: false, + }, + () => fetchMarketplace() + ); + } catch (err) { + const message = formatAxiosError(err, 'Marketplace unavailable'); + void vscode.window.showWarningMessage( + `Creer: marketplace endpoint not available (${message}). ` + + 'You can still use “Creer: Install Pack from URL” if the backend supports /packs/install.' + ); + return; + } + + if (items.length === 0) { + void vscode.window.showInformationMessage('Creer: marketplace has no items.'); + return; + } + + type PickItem = vscode.QuickPickItem & { market?: MarketplaceItem }; + + const picks: PickItem[] = items.map((item) => { + const bundled = isBundledSource(item.source); + const hasUrl = Boolean(item.url?.trim()); + let description = item.source || ''; + if (bundled) { + description = description ? `${description} · bundled` : 'bundled'; + } else if (hasUrl) { + description = description ? `${description} · remote` : 'remote'; + } + return { + label: item.name || item.id, + description, + detail: item.description, + market: item, + }; + }); + + const picked = await vscode.window.showQuickPick(picks, { + placeHolder: 'Select a marketplace pack', + ignoreFocusOut: true, + matchOnDescription: true, + matchOnDetail: true, + }); + + if (!picked?.market) { + return; + } + + const item = picked.market; + if (isBundledSource(item.source)) { + void vscode.window.showInformationMessage( + `Creer: “${item.name || item.id}” is already available (bundled).` + ); + return; + } + + const url = item.url?.trim(); + if (!url) { + void vscode.window.showInformationMessage( + `Creer: “${item.name || item.id}” has no install URL.` + ); + return; + } + + const action = await vscode.window.showInformationMessage( + `Install pack “${item.name || item.id}” from marketplace?`, + 'Install', + 'Cancel' + ); + if (action !== 'Install') { + return; + } + + try { + const pack = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: `Creer: installing ${item.name || item.id}…`, + cancellable: false, + }, + () => installPack(url) + ); + void vscode.window.showInformationMessage( + `Creer: installed pack “${pack.name || pack.id || item.name}”.` + ); + } catch (err) { + const message = formatAxiosError(err, 'Failed to install pack'); + void vscode.window.showErrorMessage(`Creer: could not install pack — ${message}`); + } +} + +/** Optional helper for hosts that expose delete UI later. */ +export async function deletePackCommand(packId?: string): Promise { + let id = packId?.trim(); + if (!id) { + id = ( + await vscode.window.showInputBox({ + prompt: 'Pack id to delete', + placeHolder: 'fastapi-crud', + ignoreFocusOut: true, + }) + )?.trim(); + } + if (!id) { + return; + } + + try { + await deletePack(id); + void vscode.window.showInformationMessage(`Creer: deleted pack “${id}”.`); + } catch (err) { + const message = formatAxiosError(err, 'Failed to delete pack'); + void vscode.window.showErrorMessage(`Creer: could not delete pack — ${message}`); + } +} diff --git a/extension/src/scaffold.ts b/extension/src/scaffold.ts index f267d79..013a938 100644 --- a/extension/src/scaffold.ts +++ b/extension/src/scaffold.ts @@ -26,11 +26,11 @@ import { streamGenerate, type StreamProgressEvent, } from './streamGenerate'; +import { resolveConflictsWithDiffs } from './conflictDiff'; import { pickWorkspaceRoot } from './workspace'; import { assertSafeProjectName, findConflicts, - resolveConflicts, writeProjectFiles, } from './writeFiles'; @@ -616,9 +616,9 @@ export async function runScaffoldFlow(options: ScaffoldOptions): Promise { return; } - // 7) Conflict resolution + // 7) Conflict resolution (optional side-by-side diffs) const conflicts = findConflicts(projectPath, files); - const resolution = await resolveConflicts(conflicts); + const resolution = await resolveConflictsWithDiffs(projectPath, files, conflicts); if (resolution === 'cancel') { return; } From 05a954513a80f75f1c66e2f5f655b88586c9cb4a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 08:48:37 +0000 Subject: [PATCH 07/13] Add pytest to backend requirements for marketplace tests Co-authored-by: Sanath S Patil --- backend/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/requirements.txt b/backend/requirements.txt index 8726dde..4de1c63 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -5,3 +5,4 @@ python-dotenv>=1.0.0 pydantic>=2.9.0 httpx>=0.27.0 PyYAML>=6.0.0 +pytest>=8.0.0 From 6d514a0ea707e799943bb4274711668935e1363f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 08:48:42 +0000 Subject: [PATCH 08/13] Mark v0.6 complete in PLAN.md Co-authored-by: Sanath S Patil --- PLAN.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PLAN.md b/PLAN.md index 6d7734c..7ac15bc 100644 --- a/PLAN.md +++ b/PLAN.md @@ -7,12 +7,12 @@ - **v0.3** — Streaming, offline/local models, bake-ins, SecretStorage + GIT_ASKPASS - **v0.4** — Stream cancel, selectable license/CI bake-ins, quality gates - **v0.5** — Content diff preview before write, multi-root workspace targeting, installable JSON/YAML template packs (`GET /packs`, `pack_id`) -- **v0.6** — Pack marketplace UI + install from URL, side-by-side conflict diffs (`creer-generated`), Open VSX / Marketplace publish packaging +- **v0.6** — Pack marketplace + remote URL install/delete, side-by-side conflict diffs (`creer-generated`), Open VSX / Marketplace publish packaging ## Optional next -- Backend marketplace catalog + remote pack install/delete endpoints (extension already codes to contracts) - Extension icon + signed publisher release +- Hosted pack registry (beyond static `/marketplace` catalog) ## Non-goals From af463cd2699eafa2837273272c4528e13e4096fb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 08:52:04 +0000 Subject: [PATCH 09/13] Implement Creer v0.7: self-hosted registry, icon, release packaging Add searchable /registry with JSON pack downloads, extension marketplace icon, Browse Pack Registry command, changelog, and verified vsix packaging path for Marketplace/Open VSX (signing remains a human PAT step). Co-authored-by: Sanath S Patil --- PLAN.md | 9 +- README.md | 124 ++++++---------------- backend/.env.example | 4 + backend/app/registry.py | 166 ++++++++++++++++++++++++++++++ backend/config.py | 2 + backend/main.py | 53 ++++++++-- backend/tests/test_marketplace.py | 4 +- backend/tests/test_registry.py | 68 ++++++++++++ extension/.gitignore | 1 + extension/.vscodeignore | 2 + extension/CHANGELOG.md | 41 ++++++++ extension/PUBLISH.md | 12 ++- extension/media/icon.png | Bin 0 -> 13330 bytes extension/package.json | 7 +- extension/src/api.ts | 54 +++++++++- extension/src/extension.ts | 9 +- extension/src/marketplace.ts | 158 +++++++++++++++++++++++----- 17 files changed, 577 insertions(+), 137 deletions(-) create mode 100644 backend/app/registry.py create mode 100644 backend/tests/test_registry.py create mode 100644 extension/CHANGELOG.md create mode 100644 extension/media/icon.png diff --git a/PLAN.md b/PLAN.md index 7ac15bc..e258c53 100644 --- a/PLAN.md +++ b/PLAN.md @@ -6,13 +6,14 @@ - **v0.2** — Preview, GitHub create/push, templates, overwrite protection, `/creer` chat - **v0.3** — Streaming, offline/local models, bake-ins, SecretStorage + GIT_ASKPASS - **v0.4** — Stream cancel, selectable license/CI bake-ins, quality gates -- **v0.5** — Content diff preview before write, multi-root workspace targeting, installable JSON/YAML template packs (`GET /packs`, `pack_id`) -- **v0.6** — Pack marketplace + remote URL install/delete, side-by-side conflict diffs (`creer-generated`), Open VSX / Marketplace publish packaging +- **v0.5** — Content diff preview before write, multi-root workspace targeting, installable JSON/YAML template packs +- **v0.6** — Pack marketplace + remote URL install/delete, side-by-side conflict diffs, publish packaging +- **v0.7** — Self-hosted pack registry (`/registry` + download), extension icon, Browse Pack Registry, release changelog ## Optional next -- Extension icon + signed publisher release -- Hosted pack registry (beyond static `/marketplace` catalog) +- Signed publisher release with real Marketplace/Open VSX tokens (human step) +- Federated multi-host registry discovery ## Non-goals diff --git a/README.md b/README.md index 9a6485d..89dc7e4 100644 --- a/README.md +++ b/README.md @@ -2,129 +2,71 @@ AI-powered repo scaffolding inside your workspace. -**Current version: 0.6.0** +**Current version: 0.7.0** ## Architecture ``` creer/ -├── backend/ # Python FastAPI AI engine (+ packs/) -└── extension/ # VS Code extension +├── backend/ # Python FastAPI AI engine (+ packs/ + registry) +└── extension/ # VS Code extension (icon in media/) ``` -## Prerequisites - -- Python 3.10+ -- Node.js 18+ -- OpenAI API key **or** `OPENAI_BASE_URL` **or** `CREER_OFFLINE=1` (with template/pack) -- VS Code / Cursor - -## Backend +## Quick start ```bash -cd backend -python -m venv venv && source venv/bin/activate -pip install -r requirements.txt -cp .env.example .env +# Backend +cd backend && python -m venv venv && source venv/bin/activate +pip install -r requirements.txt && cp .env.example .env uvicorn main:app --reload --port 8000 -``` -### Environment +# Extension +cd extension && npm install && npm run compile +# F5 → Creer: Create New Repo +``` -| Variable | Description | -|---|---| -| `OPENAI_API_KEY` | OpenAI API key | -| `OPENAI_BASE_URL` | OpenAI-compatible base URL (Ollama, etc.) | -| `CREER_MODEL` | Model name (default `gpt-4o-mini`) | -| `CREER_OFFLINE` | Template/pack-only stubs | -| `CREER_PACKS_DIR` | Extra packs directory (overrides same ids) | +## Registry (v0.7) -### Endpoints +Self-hosted pack catalog: | Method | Path | Description | |---|---|---| -| `GET` | `/health` | Health / version | -| `GET` | `/templates` | Built-in templates | -| `GET` | `/packs` | Installable JSON/YAML packs | -| `GET` | `/packs/{id}` | Single pack | -| `POST` | `/packs/install` | Install pack from URL (`{ url, overwrite? }`) | -| `DELETE` | `/packs/{id}` | Delete installed pack | -| `GET` | `/marketplace` | Remote/bundled marketplace items | -| `GET` | `/bakeins` | License/CI options | -| `POST` | `/plan` | Plan (`template_id` **or** `pack_id`) | -| `POST` | `/generate` | Generate + bake-ins + quality | -| `POST` | `/generate/stream` | SSE progress (`job_id`) | -| `POST` | `/generate/cancel` | Cancel job | -| `POST` | `/quality` | Dry-run gates | -| `POST` | `/github/create-repo` | Create GitHub repo | - -Marketplace and pack install/delete may be absent on older backends; the extension handles that gracefully. - -### Packs - -Drop `.json` / `.yaml` files into `backend/packs/` (or `CREER_PACKS_DIR`): - -```json -{ - "id": "fastapi-crud", - "name": "FastAPI CRUD", - "description": "CRUD API starter", - "stack": "FastAPI + Uvicorn", - "version": "1.0.0", - "files": ["main.py", "requirements.txt", "README.md"] -} -``` +| `GET` | `/registry?q=&source=` | Searchable pack list | +| `GET` | `/registry/packs/{id}` | Pack metadata | +| `GET` | `/registry/packs/{id}/download` | Portable JSON pack (installable URL) | +| `GET` | `/marketplace` | Curated featured view | -Shipped examples: `fastapi-crud`, `express-ts`, `python-lib`. - -## Extension (v0.6) +Install from another Creer host: ```bash -cd extension && npm install && npm run compile +curl -X POST http://localhost:8000/packs/install \ + -H 'Content-Type: application/json' \ + -d '{"url":"http://other-host:8000/registry/packs/fastapi-crud/download"}' ``` -Flow: idea → template/pack → bake-ins → plan preview → generate (cancellable) → content diff preview → **conflict review diffs** → write → optional git/GitHub. - -Multi-root: QuickPick workspace folder (or `creer.defaultWorkspaceFolder`). +Set `CREER_PUBLIC_BASE_URL` for absolute download links in registry responses. -### Commands +## Extension commands | Command | Title | |---|---| -| `creer.createRepo` | Creer: Create New Repo | -| `creer.createRepoFromChat` | Creer: Create from Chat Prompt | -| `creer.setGitHubToken` | Creer: Set GitHub Token | -| `creer.clearGitHubToken` | Creer: Clear GitHub Token | -| `creer.installPackFromUrl` | Creer: Install Pack from URL | -| `creer.browseMarketplace` | Creer: Browse Pack Marketplace | - -### Settings - -| Setting | Default | Description | -|---|---|---| -| `creer.backendUrl` | `http://localhost:8000` | Backend URL | -| `creer.contentPreview` | `true` | Diff/content preview before write | -| `creer.showConflictDiffs` | `true` | Offer side-by-side Review diffs on conflicts | -| `creer.defaultWorkspaceFolder` | `""` | Multi-root folder name/path hint | -| `creer.previewBeforeWrite` | `true` | Plan tree confirm before generate | -| `creer.useStreaming` | `true` | SSE progress | -| `creer.promptBakeins` | `true` | QuickPick license/CI | -| `creer.license` / `creer.ciPreset` | `mit` / `auto` | Defaults when not prompting | -| `creer.initGit` | `true` | git init + commit | -| `creer.createGitHubRepo` | `false` | Create remote | -| `creer.githubPrivate` | `true` | Private repos | +| `creer.createRepo` | Create New Repo | +| `creer.createRepoFromChat` | Create from Chat Prompt | +| `creer.browseMarketplace` | Browse Pack Marketplace | +| `creer.browseRegistry` | Browse Pack Registry | +| `creer.installPackFromUrl` | Install Pack from URL | +| `creer.setGitHubToken` / `clearGitHubToken` | SecretStorage token | ## Publishing -The extension is ready to package for the **VS Marketplace** and **Open VSX** (no credentials in-repo). +See [`extension/PUBLISH.md`](extension/PUBLISH.md). Package: ```bash -cd extension -npm run compile -npm run package # → creer-0.6.0.vsix via npx @vscode/vsce +cd extension && npm run compile && npm run package +# → creer-0.7.0.vsix (includes media/icon.png) ``` -Full steps (tokens, `ovsx publish`, checklist): see [`extension/PUBLISH.md`](extension/PUBLISH.md). +Signed Marketplace / Open VSX publish requires your own `VSCE_PAT` / `OVSX_PAT` (never commit tokens). ## License diff --git a/backend/.env.example b/backend/.env.example index bb2253c..35100fa 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -17,3 +17,7 @@ CREER_OFFLINE= # When set, remote installs (POST /packs/install) write here. # When unset, installs go to backend/packs/installed/. # CREER_PACKS_DIR=/path/to/my-packs + +# Optional public base URL for absolute registry download links +# Example: http://localhost:8000 or https://creer.example.com +# CREER_PUBLIC_BASE_URL=http://localhost:8000 diff --git a/backend/app/registry.py b/backend/app/registry.py new file mode 100644 index 0000000..403c5ac --- /dev/null +++ b/backend/app/registry.py @@ -0,0 +1,166 @@ +"""Self-hosted pack registry — searchable catalog + downloadable pack JSON.""" + +from __future__ import annotations + +import json +from typing import Any + +from config import CREER_PUBLIC_BASE_URL +from app.packs import ( + INSTALLED_DIR, + PACKS_DIR, + _extra_packs_dir, + _load_packs_from_dir, + get_pack, + list_packs, + marketplace_catalog, +) + + +def _download_path(pack_id: str) -> str: + return f"/registry/packs/{pack_id}/download" + + +def _absolute_or_relative(path: str) -> str: + base = (CREER_PUBLIC_BASE_URL or "").rstrip("/") + if base: + return f"{base}{path}" + return path + + +def _packs_with_source() -> list[dict[str, Any]]: + """ + Build pack list with source tags. + + Precedence matches list_packs merge: built-in → installed → CREER_PACKS_DIR. + Final source reflects the winning file location. + """ + sources: dict[str, str] = {} + packs: dict[str, dict] = {} + + for pack_id, pack in _load_packs_from_dir(PACKS_DIR).items(): + packs[pack_id] = pack + sources[pack_id] = "bundled" + + for pack_id, pack in _load_packs_from_dir(INSTALLED_DIR).items(): + packs[pack_id] = pack + sources[pack_id] = "installed" + + extra = _extra_packs_dir() + if extra is not None and extra.resolve() != INSTALLED_DIR.resolve(): + for pack_id, pack in _load_packs_from_dir(extra).items(): + packs[pack_id] = pack + sources[pack_id] = "installed" + + items: list[dict[str, Any]] = [] + for pack_id in sorted(packs.keys()): + pack = dict(packs[pack_id]) + download = _absolute_or_relative(_download_path(pack_id)) + items.append( + { + "id": pack["id"], + "name": pack.get("name") or pack["id"], + "description": pack.get("description") or "", + "stack": pack.get("stack") or "", + "version": pack.get("version") or "1.0.0", + "source": sources.get(pack_id, "bundled"), + "files": list(pack.get("files") or []), + "download_url": download, + "install_url": download, + } + ) + return items + + +def list_registry( + *, + q: str | None = None, + source: str = "all", +) -> dict[str, Any]: + """Searchable registry listing.""" + source_filter = (source or "all").lower().strip() + if source_filter not in ("all", "bundled", "installed"): + source_filter = "all" + + items = _packs_with_source() + if source_filter != "all": + items = [i for i in items if i["source"] == source_filter] + + query = (q or "").strip().lower() + if query: + def matches(item: dict[str, Any]) -> bool: + blob = " ".join( + [ + str(item.get("id") or ""), + str(item.get("name") or ""), + str(item.get("description") or ""), + str(item.get("stack") or ""), + ] + ).lower() + return query in blob + + items = [i for i in items if matches(i)] + + return { + "version": "0.7.0", + "base_url": CREER_PUBLIC_BASE_URL or None, + "items": items, + } + + +def get_registry_pack(pack_id: str) -> dict[str, Any] | None: + for item in _packs_with_source(): + if item["id"] == pack_id: + # Include full pack body fields + pack = get_pack(pack_id) + if pack is None: + return None + out = dict(item) + out["files"] = list(pack.get("files") or []) + out["stack"] = pack.get("stack") or out.get("stack") or "" + out["description"] = pack.get("description") or out.get("description") or "" + return out + return None + + +def pack_download_bytes(pack_id: str) -> tuple[bytes, str] | None: + """Return (json_bytes, filename) for a pack, or None.""" + pack = get_pack(pack_id) + if pack is None: + return None + # Portable JSON export + payload = { + "id": pack["id"], + "name": pack.get("name") or pack["id"], + "description": pack.get("description") or "", + "stack": pack.get("stack") or "", + "version": pack.get("version") or "1.0.0", + "files": list(pack.get("files") or []), + } + data = (json.dumps(payload, indent=2, ensure_ascii=False) + "\n").encode("utf-8") + return data, f"{pack['id']}.json" + + +def featured_marketplace() -> list[dict[str, Any]]: + """ + Backward-compatible marketplace view. + + Bundled catalog entries plus locally available packs as installable via registry download. + Remote placeholder URLs from the static catalog are preserved. + """ + featured = marketplace_catalog() + # Enrich bundled items that exist locally with download/install URLs + local = {i["id"]: i for i in _packs_with_source()} + out: list[dict[str, Any]] = [] + for item in featured: + entry = dict(item) + local_item = local.get(item["id"]) + if local_item and not entry.get("url"): + entry["url"] = local_item["download_url"] + entry["download_url"] = local_item["download_url"] + out.append(entry) + return out + + +def registry_count() -> int: + return len(list_packs()) diff --git a/backend/config.py b/backend/config.py index 7845039..94f5fb8 100644 --- a/backend/config.py +++ b/backend/config.py @@ -9,3 +9,5 @@ CREER_OFFLINE = os.getenv("CREER_OFFLINE", "").lower() in ("1", "true", "yes") # Optional extra packs directory (merged with backend/packs; user overrides on id collision) CREER_PACKS_DIR = os.getenv("CREER_PACKS_DIR", "").strip() or None +# Optional public base URL for absolute registry download links +CREER_PUBLIC_BASE_URL = os.getenv("CREER_PUBLIC_BASE_URL", "").strip() or None diff --git a/backend/main.py b/backend/main.py index 27b8ec3..cc79725 100644 --- a/backend/main.py +++ b/backend/main.py @@ -6,11 +6,11 @@ from collections.abc import Iterator from typing import Any, Literal -from fastapi import FastAPI, Header, HTTPException -from fastapi.responses import StreamingResponse +from fastapi import FastAPI, Header, HTTPException, Query +from fastapi.responses import Response, StreamingResponse from pydantic import BaseModel, Field -from config import CREER_OFFLINE, MODEL, OPENAI_BASE_URL +from config import CREER_OFFLINE, CREER_PUBLIC_BASE_URL, MODEL, OPENAI_BASE_URL from app.bakeins import apply_bakeins, list_bakein_options from app.planner import plan_project from app.generator import generate_files, generate_files_iter @@ -22,14 +22,20 @@ get_pack, install_pack_from_url, list_packs, - marketplace_catalog, uninstall_pack, ) +from app.registry import ( + featured_marketplace, + get_registry_pack, + list_registry, + pack_download_bytes, + registry_count, +) from app.github import create_github_repo from app.jobs import cancel_job, create_job, finish_job, is_cancelled from app.quality import has_errors, run_quality_gates -VERSION = "0.6.0" +VERSION = "0.7.0" app = FastAPI(title="Creer", version=VERSION) @@ -146,6 +152,8 @@ def health(): "base_url_set": bool(OPENAI_BASE_URL), "model": MODEL, "packs_count": len(list_packs()), + "registry_count": registry_count(), + "public_base_url_set": bool(CREER_PUBLIC_BASE_URL), } @@ -159,10 +167,41 @@ def packs(): return {"packs": list_packs()} +@app.get("/registry") +def registry( + q: str | None = Query(default=None, description="Search name/description/id/stack"), + source: str = Query(default="all", description="bundled | installed | all"), +): + """Self-hosted searchable pack registry.""" + return list_registry(q=q, source=source) + + +@app.get("/registry/packs/{pack_id}") +def registry_pack_detail(pack_id: str): + item = get_registry_pack(pack_id) + if item is None: + raise HTTPException(status_code=404, detail=f"Unknown pack_id: {pack_id!r}") + return item + + +@app.get("/registry/packs/{pack_id}/download") +def registry_pack_download(pack_id: str): + """Download pack as portable JSON (usable as POST /packs/install url).""" + result = pack_download_bytes(pack_id) + if result is None: + raise HTTPException(status_code=404, detail=f"Unknown pack_id: {pack_id!r}") + data, filename = result + return Response( + content=data, + media_type="application/json", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + @app.get("/marketplace") def marketplace(): - """Static curated catalog of bundled + example remote packs.""" - return {"items": marketplace_catalog()} + """Curated marketplace view (featured + local download URLs when available).""" + return {"items": featured_marketplace()} @app.post("/packs/install") diff --git a/backend/tests/test_marketplace.py b/backend/tests/test_marketplace.py index 62fa9c7..97e21f9 100644 --- a/backend/tests/test_marketplace.py +++ b/backend/tests/test_marketplace.py @@ -47,8 +47,8 @@ def test_health_version_0_6(client): resp = client.get("/health") assert resp.status_code == 200 data = resp.json() - assert data["version"] == "0.6.0" - assert VERSION == "0.6.0" + assert data["version"] == "0.7.0" + assert VERSION == "0.7.0" assert data["offline"] is True assert data["packs_count"] >= 3 diff --git a/backend/tests/test_registry.py b/backend/tests/test_registry.py new file mode 100644 index 0000000..477f5bc --- /dev/null +++ b/backend/tests/test_registry.py @@ -0,0 +1,68 @@ +"""Tests for self-hosted pack registry (v0.7).""" + +from __future__ import annotations + +import json + +from fastapi.testclient import TestClient + +import main +from app.packs import install_pack, uninstall_pack + + +def test_health_registry_fields(): + c = TestClient(main.app) + h = c.get("/health").json() + assert h["version"] == "0.7.0" + assert h["registry_count"] >= 3 + + +def test_registry_lists_and_search(): + c = TestClient(main.app) + r = c.get("/registry") + assert r.status_code == 200 + items = r.json()["items"] + assert len(items) >= 3 + assert all("download_url" in i for i in items) + + r = c.get("/registry", params={"q": "fastapi"}) + assert r.status_code == 200 + ids = [i["id"] for i in r.json()["items"]] + assert "fastapi-crud" in ids + + r = c.get("/registry", params={"source": "bundled"}) + assert r.status_code == 200 + assert all(i["source"] == "bundled" for i in r.json()["items"]) + + +def test_registry_download_and_reinstall(): + c = TestClient(main.app) + r = c.get("/registry/packs/python-lib/download") + assert r.status_code == 200 + assert "attachment" in r.headers.get("content-disposition", "") + pack = r.json() + assert pack["id"] == "python-lib" + assert isinstance(pack["files"], list) and pack["files"] + + # Install under a new id via bytes path (avoid conflicting with bundled id) + data = dict(pack) + data["id"] = "python-lib-copy" + data["name"] = "Python Lib Copy" + install_pack(data, overwrite=True) + try: + ids = [p["id"] for p in c.get("/packs").json()["packs"]] + assert "python-lib-copy" in ids + detail = c.get("/registry/packs/python-lib-copy") + assert detail.status_code == 200 + assert detail.json()["source"] == "installed" + finally: + uninstall_pack("python-lib-copy") + + +def test_marketplace_enriched(): + c = TestClient(main.app) + items = c.get("/marketplace").json()["items"] + bundled = [i for i in items if i.get("source") == "bundled" and i["id"] == "fastapi-crud"] + assert bundled + # Local download URL attached when pack exists + assert bundled[0].get("url") or bundled[0].get("download_url") diff --git a/extension/.gitignore b/extension/.gitignore index e6b3087..9fdd493 100644 --- a/extension/.gitignore +++ b/extension/.gitignore @@ -2,3 +2,4 @@ node_modules/ out/ *.vsix .vscode-test/ +media/icon-1024.png diff --git a/extension/.vscodeignore b/extension/.vscodeignore index 0637e65..14b22ea 100644 --- a/extension/.vscodeignore +++ b/extension/.vscodeignore @@ -14,3 +14,5 @@ node_modules/@types/** node_modules/typescript/** *.md !README.md +!CHANGELOG.md +media/icon-1024.png diff --git a/extension/CHANGELOG.md b/extension/CHANGELOG.md new file mode 100644 index 0000000..f094ce6 --- /dev/null +++ b/extension/CHANGELOG.md @@ -0,0 +1,41 @@ +# Changelog + +## 0.7.0 + +- Self-hosted pack registry: `GET /registry`, pack detail + JSON download +- Extension icon (`media/icon.png`) for Marketplace / Open VSX packaging +- **Creer: Browse Pack Registry** command (search + install via download URL) +- `CREER_PUBLIC_BASE_URL` for absolute registry links +- Marketplace items enriched with local download URLs when available + +## 0.6.0 + +- Pack marketplace + remote URL install/delete +- Side-by-side conflict diffs (`creer-generated`) +- Publish packaging (`PUBLISH.md`, vsce/ovsx scripts) + +## 0.5.0 + +- Content diff preview before write +- Multi-root workspace targeting +- Installable JSON/YAML template packs + +## 0.4.0 + +- Stream cancellation +- Selectable license/CI bake-ins +- Quality gates + +## 0.3.0 + +- Streaming generation +- Offline / local model backends +- OSS bake-ins + token hardening + +## 0.2.0 + +- Plan preview, templates, overwrite protection, GitHub push, `/creer` chat + +## 0.1.0 + +- Initial FastAPI backend + VS Code scaffold command diff --git a/extension/PUBLISH.md b/extension/PUBLISH.md index 090781b..9b9fed4 100644 --- a/extension/PUBLISH.md +++ b/extension/PUBLISH.md @@ -19,7 +19,17 @@ npm run package # or: npx --yes @vscode/vsce package ``` -This runs `vsce package`, respects `.vscodeignore`, and includes production dependencies (e.g. `axios`). Output: `creer-0.6.0.vsix` (version from `package.json`). +This runs `vsce package`, respects `.vscodeignore`, and includes production dependencies (e.g. `axios`) plus `media/icon.png`. Output: `creer-0.7.0.vsix` (version from `package.json`). + +### Publisher signing (human step) + +vsce/ovsx publish with a PAT **signs the release to your publisher identity**. This repo is packaging-ready (icon, changelog, license, repository metadata). Actual signing/publish requires: + +1. Create publisher `creer` (or change `package.json` → `publisher`) +2. Export `VSCE_PAT` / `OVSX_PAT` in your shell only +3. Run `npx @vscode/vsce publish` or `npx ovsx publish` + +No tokens are stored in this repository. Install locally for a smoke test: diff --git a/extension/media/icon.png b/extension/media/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..d4fb02a8528682bca9864798d3dfa4f5a8163481 GIT binary patch literal 13330 zcmV+tH0{fYP)004R> z004l5008;`004mK004C`008P>0026e000+ooVrmw00006VoOIv0RI600RN!9r;`8x z00(qQO+^Rl2nYx=2^4Tl2LJ#wh)G02RCwBzeR;TLMRo65RePU1_U+*|G~IN!v;#C? z>tkpc6!}yX#4(QGl*DKBJ@cZHN210lpLxcS2%jVfBryq5)WjE=3_~PL0#90O!YoKL zb{M*w9`5aX?>W0_y+5jI)mpXpY0UX9%^CKtVU54FYE^9n`7#!kuE42j8W{s)0Vw~3 z;U5I~Es^*Y0SGW&6MrTEMC4m2wgVWx<_m$sZwLT{nEC?(M4;3sabIF!+#wN&t`Lzx zc1-cx@LBeb{7>#y%q@>HOeIW$5HL(Iw#==e93G~|$vc=^@H=+EQ%{3)y7TawS`G&s z1oa3|8ZL>Z<%k90iZ;|_c(BEbS;B)8O(D}gw@&h^j&YP_PE;COPq-h=L5M zCYa@jNRlr|P%r>lMoITW06=H}5$5OU;YZ=wo#A&IuUozv)*J$8z%`opl$eP81!M{B zntzb)Ab==`hJXNwnEpj7I7@tQYL}@m2#ER*0I-k*>=FvpLXKImTGw`Q7hCNJj;3n@ z{gbXCW;I6&7kLMDjr%W$rh%E+#DGQ1V9g=W^uc@J`NtV*>O(PH%s&Z=WUiy2z8I#Y zkzg!OL9ir=fE18y#7v|;-7z-;7ih?gmFX)N{K!5~56yM~rYc@pL=Hat4W%_DKMZ&B!`R1sFn2anb zk*7>9%`8klL-1Wlr;=j!GHhQkD^r{lCgmeaJYs{xgNQ?CwrdH&=#PSI@ukk;^VWd1 z7M87qCHsSKGW7wI#Zb&u?+ZF=X{2lsq;g7mm3PLWVeT$K+x(}^QN3XcO25s2QMWOA7$Oh78U{@TQ`1U26k(B&gjDEBiwR@KD@et~ z#VAWRs{H_?DawGWuuAc+2#&Oc8TLXgBSGpZDN4h0p*42cQ3sw_Vh(fo98Z?G+ zOfNAoIu7IoZIpN>T}3rXZ3nPT)nRl{a*}uYh$>u*uuB58D)tb| zw6ynBfQ}5%2s(H#6hhHc!tX9MRnRQ59mmHiFg9ENf*Ng!Pymf2#;4*(mJ8JMW$GlW zm_dc^7BgD~9%bD?5>W>gq!&4c_F{FqS*%cOd5k+V!d>#=2L=6yO zMjCAv%_OOsc7QRS7q&3RP=fZ+&gK+Bo0lZ$gU}MkY#|jK0%|cfaVRRA4t*skkf2t% zTpj{-bXeLiJpoj?5Yc5~v3B)=zLj8Ml!+mDj%2K%qa_g&%3-s2%tk4ypAZHBAOU$4 zz7^4cOx`FHOu+4FPlYXF6z{ddP}Yoq{WMp#S3xOr+(rps;^>LG{6Xrj5|~fuyXt%> zxK`#48h0?M^Rf0psbb>BL#B4hi;h_+Ho*X-v(##dNYCKioG7_z4S4FyU>=2iX zGPLA##7;m+EwvAVghYf$9N1yN5}wGQeMcugg{YUu*&1Z7Q1BMk~Zvhsl2Ld9Bf) zkP<45Y4k#JByDd?`CqB(2zlj8NL}m;r7dVrM6o94{t|V%=${D{MHN-Psk}JJeK$@O z<{{D~Cr7m2QqpnK{y#TW6`HJvqTo#zIjBS*+H8>03`%Suo%?L5k_PZY+E!N&8(+AF z18P)hHOLCpiibgJinJ=oUYJdpXtJa)+uoM_j;pW{1$EOnMQ=o%3KeCy@Bli5$}$tZ zDshBMZ8$WH)xW{3J%1?%D1Hu8LK5qVBpbjXs+TXyA(3c3;2(|?gkuIO9y+S&#_yumiR^wyEUlsk z*%2yuSlD&Jeq^0c5QC)x^G}#ys8pgzYvO1Kq+BNhQj)g=m@%MjDG*}$t@a%PCJT(w zVk+{aF~xEcYDr^5LPqlXAcy~Xv=Al^1^+tMSh89xILmU8>mkHXMWjoK0m(!QwZ*)u zUJyI2+bXT%$pj3O$VRDFqze^yzexR06A|)EP2jFczr}O~K}xWy!Dq%M+bfeUkxHS` zB=sgDJCJmiBt@y3l4tT)P1y2PaT7}Qh9MBH=SeqZP0X>PoPG{DfEPwNKu;+i)mfBg z%o#~QG-!+oRUytf@*bQ6?}2lg2rCK&*tx5`wHOG-ZB+_ESU9u0%#;oRv1KM(0T9s z1K;n1Yrq&78Cf*7XxZe%#N@>I=vc3+h-ksPy?bYO&(7}Hy=Q*U9zPfW08|x>j#PDp z0M7g5eV`~UiPX=NQqWFhjs@oFSVU1)lvu4~jtdJf3II^00C0%lXUg$t6lK%mS~8=e ziwt<=(52j98G3Rr))?=}&CR)m1sLtET($DZ^@kj}e*KGI^!yjBK5)hItg7m!TAnbm^p#tq^B@$AmdS~ZaR;Ljn z8f%;W0A^;OS0B3mkT;)r!YMC5_Q*rmPfboFS_6f00gCnoqG?1l7_yVBc27L@^i99q zeA$gRUUlm&+qZ6oy0Q}!2B7gCL?ua%lDMz{<@nmvE5Zp<`ofgMoO|cHv}pmv4aaw; zl_UeuWSIzzjp1tcLPdbVk*^35QyN19s~oFYYh1riGc%LRr+@3Uule1#yzv!BytJ6QY@I&+feR`s*+J@sDravWp3OonahCni&j&UqoOH99FcXbpPj{qEgM zR<8KSJKy%dPJ8F-16Bk&HqIfSF&ztOLjy=T8Y1sKAXe5Qg7zASt;kTFCQ=eTBwCbsoV2bZt5p#}v9tgIA#Qj%i>$)9+y=`kf#Q*_aw{g@O{(Oe zAUOp@G*#8?-d)$#hu`z=&wluW2P~fsbO!=!&2v$>gXEnlR3dWTTVn#6y7uR{ee#=s zv1#*WGc{?9b>5}QI3|^_D)M8ZmFh<pArLJix8#PIB9(Kmrt?7WyY)5g636Cy$;5QcbU(;F-pd7bjS4J*` zd74HH_@SdXvV2PfRaLpYduxM#{>d{h{QRd^?zem}7z9sHmco&sNf~i=y@qj%01$#T zYn%hXV-7$3U9WxJU5`EX>wE96#>Pl5J}dl^OLPjwR7F`)?RV`x$i#Zq{ch%y|q?3j3G=cVQc5I z0nbK9Ui`g9g8&29OfOyX{!>qF&|H7Z&BhwER{O2git<4%(9QvY4vIV6d+`mTWI(zE zvn5o%GiXGsVu_Np2vmts%5aK@0c~ZQ?b}~<;t5w@_*Vx%Z*6}tu-2OF=ey8IF;7kk z6j#!7F;BNa|S2WJ=1`C2S)iSrAg6O_sG)vwho#-v6HOe(g($=$unH2isjf$Ll$+ z?Zox%s}QQn_WOeqk38zC^Uhy3v8b7!Gsb8hkWz0DvaBpJh0KO@9!w=d)((n)L?)yU zsl_*m_roh@wGz@Vv`d&l(H`cjs;X+XZ~NeTPrLBXKHoGA5*X8kGTSoyTvFUY>*r)i z3)Tl79J67=<>&s@qS3LYzfcZOdEG>r>=ijRo#<5FD6*K54+KCKSF9GsJ({Bu?1R=0 zUTRSLe2BFzjWeeLtI7^`?0CzmCx7p(uXyJ|+Cbc-NnE&1O$8z#^4@z-B>!ab<9~q@ z;#y^br!FTV8K-|AJB_oQHcTPnK%TZn0iz=jNiF_l#>V@$wN-jfTdp0;Bj z=4Lbx)~{PPy?DtL*IZeRj}v*w1eDIF%g3h9G*B!m&mbB!LkD~ra*22EpGwM9!*v?1PmPefEzHUzPA^YdGtc;dmwAAju0 zC%5mJSr`ne$}SlnU%g_*^H;5U;hHsz7cC+(h&X5(1Te-7rFfBgh^EpG27}*!_i4Yp z_nyD`=ZkyO%bLb%B*M979A*>SCL}I6sGJZ0LRBhAzJg@7OC>Y{QXW#52B4OU5fK*V z|MII}T)*z1{$Nm9+rDL(VGsGv%34I+xo6K6H{NvF_1FLW-uoVV^2uiJEDRbz0As*= zKmcp8?j3Nz0Y|*(g{K^M+*@D$>NP7?gasS`%sN*9SyGA|Ud9+A_=_|D_||*xy8EFA ztI-kfRRWIhV~xs5HwHpQGAVh2Kp@O{$_>W{`itm-OG-sNpom3MaJhA20|;>>m9@=| z9UnOD9e@AjFAfHcwWb@YcYW!+H;C5SEl)gg&iB55@zvKny>%-%2VtQ6o_^B_C!G1d_aAxaq27DvyltxjX@1Dwov-Wa-#6X%>fievJ3dOn z0kCB&_J|1rmET7SYlt8AU0PVNMnRdxVzKcuBBxlb+T|1o%K`uZFo>?-U$t`oAD;b< zMdRa8`Z4ipQ{VZzu6EDNeD*KT`{18`@ur(^ogECS@o_sg7U=E)qWcB%$vpsov9=l; zvty$R3k!GNb?4t)cIg9;J+|?%!x+<1ptsDxl<(uDWMVAHr4tXcE)_U%94{0nvl zImRt9uCS-AATj&Z?3?2%sf875_%`0%jVh>1`OCCxbo_Mb9GfsOaP+y-jml(N9QC{EiI8pYpby_ zGI+-?fBBuCT(M$$`lv$>3v9}A&>18PNjC zAhI?P0AvQ}9Yty%Pby;3dnFb1%OtQd*3Hgt{BIl1{oJR)`w~-2tEQFJvCXDa> z^k@Fz;ve+(Tkaq%WY}TJ|(mU!hR=#z?45g2rE1z`z^obx@+I~i9gsoKOa}~ zJ0m_}h!CMa7%Z8Z`p8?}0efbRmH7#AW#&C4jHz=N5&~()nG{Tyjw6{W)YF6HJw(bo zQ(t%?@*h3@-2f1YsCm?WPvot&|MN><`TmdorMLfnP1BGn72((#LY<|OJGl~qqG`E& zf-o2~Bh$-oxb zZQE-n9-yRXv}HvwscdQR)#&^EgVwG-=Tm<)(yJl4k5a_Mn}B)FzUaGO`qsC4%a=Dz z!zped4zeZUajAH*=p|1o7H@g)dJ~iP-gU+0us-Tle4&oPUqnsl0!9C5JzMc5~VOC(;pg()m%69P0%Q!iWg z^}jy(0MAzNrAgCg^CM&9{idG^w~7)!>2Qms z03JD|(-m?YovNdw#p=o~VcT7aWl6mS)fjYp=U#H?!ABi_nD^eG5q)WbAON6quCA-| z|LLD^*>qdIY+2JZ(xwG0A&Nyfx>u4=zR~?5klJ~N;E8N+;&+P2i|NbHv8}rU5zH(wYr#GmWNNx>_S(lDJml|dlMnqUuy`{_22L#?Tk!GCw zO|4NXYh+2ymu6mr+EuCClEC))b`=MZx`Ft+q#$y~cLuD|?GO=I`>JC$hGBMNp7bVw z@$X-G^{*d%upS!=aTC{cNc0>V&uPd)Kw6H4Cy$u7d88wTAUT-EIp-ai#9>NHsvK~n z!~g(*F~%5mv`FYi21B919SKvFQONoS@RM3639GB35TlfP5Td87SuxolXy}?|VtUyT zhaCzW@poq5003g$)!$umDbzJlLaQk!9u1vfN9zn2E|bI-YxJ=VT6J>+27SNpo_!W< zI-^sUw#AT}QtVyTW_*n64Ji*JvwZ00!ZMW|f2!nRXC1f6tdE*~OAd+97#Q@|y=dL) z75kGH%grsv3jp3zRoOf4yZ_d^@4~TBPd-N*j5Qfcv#3ETK~u&Tpg6xO838uq>8f(#`20J>Efnw1v!>U z7)Pc>E9JW=nWs{6^G@8(!oNfcgu&pDwQG$rYT>2Jxq0#c_Ntq2@q72yOP4f_6Ch0V zUAR_EJkZjLQa`0r&E}ZVVX7fSgoXK+yzoU|KjTd2Tv+WHHnpZ{s;c_V<(FK2?bUXC ztclJ*F+6K*fg1QtGA6ME8wf~l)Xq0_?b1bU9H9Lk5COb9_<3uAlSS>eIRHTffSWdN z2DKLo1*uTP{kj~iu$6MRm&}-J7n-7$us@Q32#jeA0>HpE#_y{R)j3~PRevzZPBRZU zriznJWa4m%qL@92!2qP=YT9<1@@!e0Q1ergGcuk4aovHdI!G^~>ENrYs+yZ$xciYu zU}S`xhiFzMs1=*#6f_Xi&U4Babgf}CR-&vg2tYoZB3v2UU8131*4jjE4H@G;lf1t` z5xMh~?l$5BCt8xvDH&5lqOCVXP}vpBr%Q^ppnB+V>(ftfedZYmm1qTqR%ODP@u)h^ z(j|YT6dEcU9nYXrU~hOx+wl{}ohi&sqfpQ}{vX8wk_k_7kIX-#Mi0W_VQ#w|Z0??; z5(EHG1h!hfc!?ZHHv}RAfTwosnBO~VWg>wJaD{&x2r?f$oa&wUWS2*E) z*esAa09~Ray0wVCy6(C+T_MzPc6Jtsz#3>=Huac$pt5;s1H*POkkTak%CurY(gntb z&b7od0FcgJORiPKzydsXTdv}3wo_o@1iB*#$vVDMcpx#=76#C4>KvFcygf1cE`ja| zrN|}lkjX>&!me>^8@_LcxKBRFHrZEkwt+Ygz{^Ggswpc@tv?_FQGJK|(!SDkvKtWo z1QDJ0{jf(j+#{VxtMQ46aX=GOeUicazC+KF0}v8Ga#XTNP_m)DhT28q%->FSRL7fd3h?#f*4q355zn`!^@{*9prDlP(^ghRxEXX}(M<+?fPiSAasT<$(~!-j zzK~l)1b~(M?YCm-QpgLf3d6)NvIHpTj-7Hsc`(wnGVsYbwFRr%MDJinvNnZ-AtZ4` zNk`nHN6Di{KrmvdPdpK&J)U+{Q1%xz%|G(gQ$xE4mO*G5H!(JL=s^cT(-_$v3Lh(q z^HKwPc$f#BjvO9NL}es$!2|#}@Av%|GUqNA7(~e<@kx~Gh{?1)s~($)a#I8sg#z(; zfa>Px57Z&^#y;@y!{O_;F3OJ~`~S+Lj!GIpib>+QO5TL}OvafV32DA_=YTcrCE zt}rd867;GEo_O4QUsMjP*BkqhHoO!--qMk5`}l)rn`GdEZ6OjdLwM}Enosk{$vax;gJlMaA@Wi0_3Hk@Tt0J)0}_FzMYxbhRhZ2r`x5vQAcaGjxR{hunWtZq%MM!5JL>x;>u#9wwMgJhM;VSKzORtsBnIMF!=D>-)@Fgfv0;A zv1#0*iHZ67g==rPz8)X zx?ZztwRcWbFDr1LaRHGrX1*Cb^5oW3YDVqniiGv3Y_t-TQ(J&#tp^1kQ1~7xFZp(P z^mfUlf_s&`S*W6>hQxl7AAEsh%jiH$?@@ToN?wG|KOv_MS*^=;yOJRZ9Tf? z!bBw?s;a8LbKBQG`{y5h$LaI)3&t8bBDP@sP2-xe(b2Om{`)h(`Db?V;#fS{@rVen zs8KUTkt`D~CB95)n|K(QDtSe5dP>-dd|n(&9*6fH#>Otb>dL)yb9Gg#vO37IGY`R9 zGiaJOoN&S){muuQZQH7<3dfdB^sq&^>)Gxv|%>KBls{W2`@A<8Fee@ltd+$d^ zdI{-=m#s4PkKc@ri~zw8uK5X6RZbnsXOeXKk#=$+&WFT9Ezso1UQAB~n3;b)2#*Fb+6gfn`O2YJzmtg@#gW#G!KmWX_1cjJ(33b>R zx_EN(%5%?q&)ZMyZ{JSd*~)SdK!u4fHWp&qM1TqJ0|1aU76}%%J^k7fPrB~hzgxTF zfI-t#a<~5QR{#;;`0n4%&+Mw}Dhn~+smx9aVo-bpVX-y!!><-str3}lOnhLmPMWDR z6vP-H1i~rD9!Wq%wl}h6%R>i0{{=5S_+TY*onZmQ9=#`P%{yOn(ux%;uiJFn{H|y1 z=%_WuYljylM1kuJRs*ZT`4NMeJqYwCAO6H&fAS0CBcskawPAMnV7&KrRo!;)T^~Q^ z%Q!khUBi{U>e`BeI>Uj&Z3a{w0kk~U2`CICb-X7l~`zyH)X zOpcEin?th@=ur1d@!9$LQ(yVY#<{2jZ4=be z1B2K!ZhG;O(_eSWt2Z9=%ora@SpHBnPOu5Ym!0vY?Qus?wv5`=OiW5(U_4%Q-`cM=%k~MefMilK6?FO#u(?E zC$hhR%N6}nRoO?L+`94i-u2APuBxg%d7-_n=(EL9`5?pu=dz7}L_z|J3_##xwp31- z<9=brio}yBly3_)5gK$03#*ncyX7ChxAuSo8g>SpPQ7o@AsnBYYtoY!gZ?-3~YAJdGD>U9nB86Jb^CO+Si}? zkt=_8ZEq@szC~0aSW(2i`ggrBn@v&3nh_D=QOCwRMJ;1GGb`?tlOQl^UVHTHGepWV zxIvNYm|yN3vJf>u7UOnBWTveAhJY*{xLO-8rVf=IaiAd3UO5i+KIXJN z`D%P@%Y(mq$HwnYtuJAZ0$urg9Z(Dv?&~0XLpTj zdUfv~ue$7vZ~j@mc(L=IkzMOHJyTJX3yfAUGYBYw`~rr*Wi+puNP~6`i&R z83sk(*Ao-J{Kcl7^9yfy#R(-3Xm|@PISnq_bicU#0fx3b90D|r>-Bn9-*)q9U-(1l z_2S+!Kw;4nN<`sH6ZA+4nV7uHP%`1pIwfy?Q zPhxXEvfcndh$<(>Z@%^BIpZ>>1^43rP{>)&Z>h;nQ z7YS46=t%{&xiVEC78#%@x&km3;xSmWYIQ;VbhF65v;aPcv5BdQ!SS7X4*?NX6XUnu zdh?xIw!G<;CytJclxdl^jn<@O4tJX6q1ruLDTe4wNc{?6q${?s>v-sEIB@=_uhIaA376attB1rc8Op@|(2OhFKT|kA(9Owl|Am=MnXY8($_ksCcfPKeG>U z<3aF2408EC!j?DzPAiF~Rr=2uGgw%Fy|Z>~{K&&MyyfL5o_y@_8(whmNUz6F8Vdh< zPau}1d0|%`qrPqDv$x#6`O=?V|F5_G?1@LWfHl?lWJr;U`nyCqvBr^WmQ5BEzF1ma zLEo(W&{Cj9(C~gqYgd%#p_=9`~z3-zjWEu_(WAz`MG(BKs2|ouybbj)@@J!>aj;Q zKXBis`|tjbhaTL%?J4jMM#t)r5#&c1O2#X2B2FpHmb_KLkHWqnPY%m!t(NSwwFtu| z0yHmsS^Oz`kP^v8o(JoK++K7Qx%w&-+h5qZch9rCckQ0tLvwqIE0on-n7C{| zDIK68L24eZ;gut3QmAauU~QnTA;ChHQc8_G^5A&JNsKsf$&V`TpnhvsGDlOoNl2bD z$x-p_%o4^+=Z;7OVW+&x3sXs`c)75GX-Op+BB>b6ePLmBIR#%JogYcCpvVwQRE15; z@-qY(a9zJ1lB6mRQNfq;3V^^~uTcXe7Lr0lIT8{|5Sk$6Omdv94{Q?ai&>PJ=EXEb z>?W_3UZ^F}YLtjYrIGqQnzDpu+ZXWxdnj84zBl!*ga|oPFQ}coZrY5)c0G?jJrEqS zMncNokPh{MB|n#Aq2h`38|eueq`%hfwIxV9QW8AM1v-4a60hiQQ_O-LtD6kW<(=^wB8!uaBOjhds~fIdikuMi%~rUJ@x^jF>G%13L=qN8T$+U=eG+b$?UICtRml~O@+DRg zdY8iw_Kgw33)nB$u|YmwJ)uvTu?W|wps_?yk#N~BNPaMZv%+LFC+u*N_$a}cil3Af zBH{lq52TfODmN1{^>dlBR&c?S3||PK5M23sNwwm?PL$MK%o+*z9V|WYkhGqko~~ zl0?}vw#|2dnFBKFj>ChOK!^$E zzIf0wG|_IZEhafPn14^*VS8waTpe}7ZNC>$v)o9%4~!(Qkb9IdlOno;jMu?IWqtpxzM*eh9F{7)OSaQr?qoK;2=XkTSsD7cvqw zLP=b%+7_Q=^TjviNi$;21l0(5&o?mjNmE}#ghUI4NL%(hH7+K>hxWB~M4zvz3yYZD zkrbUuo?Ut>tIiS`>6MO_)HtM|_;Aipk}*>7Mq6nthNbc)lrbKP->6JHv|y0tUvWAT zl&io(*;Q-~d}^YcY|6^oe2c;z zNqsTIeA-2Vq+-Cy#9=Wdfo%&RC}F7pVI+o}G$~h#Dj7(pj4*AL4K1p@c+pd68|FR~ z3NJ^er=Vgw5M2ta!j~(ffgL@B4Q%2uT#8bZLO10~O+IW>}tZp*4UK>Bs!30Q=$796W~Jt@z9sBqgb#+15MrbP@#kDXo?Ybct|SN!;=lw zsejhp3?)?PS1_KFm&(zW<3J97TSk-sD)0`h!3^BQ^!cnEX;jeNxOuB^hwy%ZEQtS zFKkV9nogAYaAg`O;-)XTgfjgkBL)H@ut+lt2DWVv3;)W3W}t)|{fW9!AzbPB9epKs z$i93hffmm*{+ks+${k7jC0#SKERszBoDOu%y-Q~!1T?}a!^6m%U}Y1;YX;mxVfaZ9 z5H#5pABYV4Cw3Zo>KWKO2gU&TD7;dnr1@)k8O0Gy&KU?*)GTf<#8aJVD&r%;{!ILQ zd_^LRNHwUrQcR|_jG-*F)HFd&FM<^LqprOa)5bzX*s@4!=@(`4R+je6!=pP4I7eF^ z0n?$~d2R`nre;ay5)kD83De>CJp3qD#AaWNn_8VR;k57o^S9#w62Xado04?QJQCDz zcc8vUj#MH&%X(~58|TGyk}?w*muBN0Mrx4m+6n__0RZObVKBhu(*OYJk+~V1wUM6y zX$b=;91K}}+ee+nQ$&hLCu@YBp_q>j;#z~z;+X-xJm5h2V|j?645@Y~(@E8Wx2!~&3e@nZeOJbKA}Yqbme93mn4=*fhJZ!IOX$T~ zs4UGc_}d>1)Njd;wXkBjSw4*uqhw8*L$XdNiOdUQJs c^6@eM9|>UCqGb0c(f|Me07*qoM6N<$g4CE%F8}}l literal 0 HcmV?d00001 diff --git a/extension/package.json b/extension/package.json index 6806bac..31d0fcb 100644 --- a/extension/package.json +++ b/extension/package.json @@ -2,9 +2,10 @@ "name": "creer", "displayName": "Creer", "description": "AI-powered repo scaffolding inside your workspace", - "version": "0.6.0", + "version": "0.7.0", "publisher": "creer", "license": "MIT", + "icon": "media/icon.png", "repository": { "type": "git", "url": "https://github.com/seven0070/Creer.git" @@ -47,6 +48,10 @@ { "command": "creer.browseMarketplace", "title": "Creer: Browse Pack Marketplace" + }, + { + "command": "creer.browseRegistry", + "title": "Creer: Browse Pack Registry" } ], "chatParticipants": [ diff --git a/extension/src/api.ts b/extension/src/api.ts index 36f0dd9..bdf3227 100644 --- a/extension/src/api.ts +++ b/extension/src/api.ts @@ -25,6 +25,7 @@ export interface MarketplaceItem { /** e.g. bundled | remote | url host */ source: string; url?: string; + download_url?: string; } export interface PlanResponse { @@ -143,6 +144,7 @@ export async function fetchMarketplace(): Promise { /** * POST /packs/install → { url, overwrite? } → installed pack + * Backend returns `{ installed: true, pack: {...} }` (also tolerate a bare pack body). */ export async function installPack( url: string, @@ -153,10 +155,56 @@ export async function installPack( if (overwrite !== undefined) { body.overwrite = overwrite; } - const response = await axios.post(`${backendUrl}/packs/install`, body, { - timeout: 120_000, + const response = await axios.post<{ installed?: boolean; pack?: Pack } & Pack>( + `${backendUrl}/packs/install`, + body, + { timeout: 120_000 } + ); + const data = response.data; + if (data?.pack && typeof data.pack === 'object') { + return data.pack; + } + return data as Pack; +} + +export interface RegistryItem { + id: string; + name: string; + description?: string; + stack?: string; + version?: string; + source?: string; + files?: string[]; + download_url?: string; + install_url?: string; +} + +export interface RegistryResponse { + version: string; + base_url?: string | null; + items: RegistryItem[]; +} + +/** + * GET /registry?q=&source= — searchable self-hosted pack registry. + */ +export async function fetchRegistry(options?: { + q?: string; + source?: 'all' | 'bundled' | 'installed'; +}): Promise { + const backendUrl = getBackendUrl(); + const response = await axios.get(`${backendUrl}/registry`, { + timeout: 30_000, + params: { + q: options?.q || undefined, + source: options?.source || undefined, + }, }); - return response.data; + return { + version: response.data.version, + base_url: response.data.base_url, + items: response.data.items ?? [], + }; } /** diff --git a/extension/src/extension.ts b/extension/src/extension.ts index 3b54884..9047d1b 100644 --- a/extension/src/extension.ts +++ b/extension/src/extension.ts @@ -2,6 +2,7 @@ import * as vscode from 'vscode'; import { registerConflictDiffProvider } from './conflictDiff'; import { browseMarketplaceCommand, + browseRegistryCommand, installPackFromUrlCommand, } from './marketplace'; import { runScaffoldFlow } from './scaffold'; @@ -52,13 +53,19 @@ export function activate(context: vscode.ExtensionContext) { () => browseMarketplaceCommand() ); + const browseRegistry = vscode.commands.registerCommand( + 'creer.browseRegistry', + () => browseRegistryCommand() + ); + context.subscriptions.push( createRepo, createRepoFromChat, setToken, clearToken, installPackFromUrl, - browseMarketplace + browseMarketplace, + browseRegistry ); registerChatParticipant(context); } diff --git a/extension/src/marketplace.ts b/extension/src/marketplace.ts index cb3db3d..f17222d 100644 --- a/extension/src/marketplace.ts +++ b/extension/src/marketplace.ts @@ -2,18 +2,40 @@ import * as vscode from 'vscode'; import { deletePack, fetchMarketplace, + fetchRegistry, formatAxiosError, installPack, type MarketplaceItem, + type RegistryItem, } from './api'; +function resolveInstallUrl(url: string | undefined): string | undefined { + const trimmed = url?.trim(); + if (!trimmed) { + return undefined; + } + if (trimmed.startsWith('http://') || trimmed.startsWith('https://')) { + return trimmed; + } + // Relative registry paths → absolute against backend + const config = vscode.workspace.getConfiguration('creer'); + const backendUrl = (config.get('backendUrl') || 'http://localhost:8000').replace( + /\/$/, + '' + ); + if (trimmed.startsWith('/')) { + return `${backendUrl}${trimmed}`; + } + return `${backendUrl}/${trimmed}`; +} + /** * Creer: Install Pack from URL — prompt for URL, POST /packs/install. */ export async function installPackFromUrlCommand(): Promise { const url = await vscode.window.showInputBox({ - prompt: 'Pack URL (JSON/YAML pack definition)', - placeHolder: 'https://example.com/packs/fastapi-crud.json', + prompt: 'Pack URL (JSON/YAML pack definition or registry download URL)', + placeHolder: 'http://localhost:8000/registry/packs/fastapi-crud/download', ignoreFocusOut: true, }); const trimmed = url?.trim(); @@ -48,6 +70,43 @@ function isBundledSource(source: string | undefined): boolean { return s === 'bundled' || s === 'builtin' || s === 'built-in' || s === 'local'; } +async function installFromResolvedUrl( + label: string, + url: string | undefined +): Promise { + const resolved = resolveInstallUrl(url); + if (!resolved) { + void vscode.window.showInformationMessage(`Creer: “${label}” has no install URL.`); + return; + } + + const action = await vscode.window.showInformationMessage( + `Install pack “${label}”?\n${resolved}`, + 'Install', + 'Cancel' + ); + if (action !== 'Install') { + return; + } + + try { + const pack = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: `Creer: installing ${label}…`, + cancellable: false, + }, + () => installPack(resolved) + ); + void vscode.window.showInformationMessage( + `Creer: installed pack “${pack.name || pack.id || label}”.` + ); + } catch (err) { + const message = formatAxiosError(err, 'Failed to install pack'); + void vscode.window.showErrorMessage(`Creer: could not install pack — ${message}`); + } +} + /** * Creer: Browse Pack Marketplace — GET /marketplace, QuickPick, optional install. */ @@ -66,7 +125,7 @@ export async function browseMarketplaceCommand(): Promise { const message = formatAxiosError(err, 'Marketplace unavailable'); void vscode.window.showWarningMessage( `Creer: marketplace endpoint not available (${message}). ` + - 'You can still use “Creer: Install Pack from URL” if the backend supports /packs/install.' + 'Try “Creer: Browse Pack Registry” or “Install Pack from URL”.' ); return; } @@ -80,7 +139,7 @@ export async function browseMarketplaceCommand(): Promise { const picks: PickItem[] = items.map((item) => { const bundled = isBundledSource(item.source); - const hasUrl = Boolean(item.url?.trim()); + const hasUrl = Boolean(item.url?.trim() || item.download_url?.trim()); let description = item.source || ''; if (bundled) { description = description ? `${description} · bundled` : 'bundled'; @@ -107,46 +166,91 @@ export async function browseMarketplaceCommand(): Promise { } const item = picked.market; - if (isBundledSource(item.source)) { + if (isBundledSource(item.source) && !item.url && !item.download_url) { void vscode.window.showInformationMessage( `Creer: “${item.name || item.id}” is already available (bundled).` ); return; } - const url = item.url?.trim(); - if (!url) { - void vscode.window.showInformationMessage( - `Creer: “${item.name || item.id}” has no install URL.` - ); - return; - } - - const action = await vscode.window.showInformationMessage( - `Install pack “${item.name || item.id}” from marketplace?`, - 'Install', - 'Cancel' + await installFromResolvedUrl( + item.name || item.id, + item.url || item.download_url ); - if (action !== 'Install') { +} + +/** + * Creer: Browse Pack Registry — searchable self-hosted /registry catalog. + */ +export async function browseRegistryCommand(): Promise { + const q = await vscode.window.showInputBox({ + prompt: 'Search registry (leave empty for all packs)', + placeHolder: 'fastapi, express, cli…', + ignoreFocusOut: true, + }); + if (q === undefined) { return; } + let items: RegistryItem[]; try { - const pack = await vscode.window.withProgress( + const registry = await vscode.window.withProgress( { location: vscode.ProgressLocation.Notification, - title: `Creer: installing ${item.name || item.id}…`, + title: 'Creer: loading registry…', cancellable: false, }, - () => installPack(url) - ); - void vscode.window.showInformationMessage( - `Creer: installed pack “${pack.name || pack.id || item.name}”.` + () => fetchRegistry({ q: q.trim() || undefined }) ); + items = registry.items; } catch (err) { - const message = formatAxiosError(err, 'Failed to install pack'); - void vscode.window.showErrorMessage(`Creer: could not install pack — ${message}`); + const message = formatAxiosError(err, 'Registry unavailable'); + void vscode.window.showWarningMessage( + `Creer: registry endpoint not available (${message}).` + ); + return; } + + if (items.length === 0) { + void vscode.window.showInformationMessage('Creer: no registry packs matched.'); + return; + } + + type PickItem = vscode.QuickPickItem & { reg?: RegistryItem }; + const picks: PickItem[] = items.map((item) => ({ + label: item.name || item.id, + description: [item.source, item.stack, item.version].filter(Boolean).join(' · '), + detail: item.description, + reg: item, + })); + + const picked = await vscode.window.showQuickPick(picks, { + placeHolder: 'Select a registry pack to install (or reinstall)', + ignoreFocusOut: true, + matchOnDescription: true, + matchOnDetail: true, + }); + + if (!picked?.reg) { + return; + } + + const item = picked.reg; + if (isBundledSource(item.source)) { + const choice = await vscode.window.showInformationMessage( + `“${item.name || item.id}” is bundled. Install a local copy via registry download anyway?`, + 'Install copy', + 'Cancel' + ); + if (choice !== 'Install copy') { + return; + } + } + + await installFromResolvedUrl( + item.name || item.id, + item.install_url || item.download_url + ); } /** Optional helper for hosts that expose delete UI later. */ @@ -155,7 +259,7 @@ export async function deletePackCommand(packId?: string): Promise { if (!id) { id = ( await vscode.window.showInputBox({ - prompt: 'Pack id to delete', + prompt: 'Pack id to delete (installed packs only)', placeHolder: 'fastapi-crud', ignoreFocusOut: true, }) From 3e5a6103fb6dafd194a1f98b14729e2acc0673ab Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 08:55:54 +0000 Subject: [PATCH 10/13] Implement Creer v0.8: federated registry and release CI Add multi-host registry federation via CREER_REGISTRY_PEERS and /registry/federated, Browse Federated Registry in the extension, plus GitHub Actions CI and artifact-first release that publishes only when VSCE_PAT/OVSX_PAT secrets are configured. Co-authored-by: Sanath S Patil --- .github/workflows/ci.yml | 52 ++++++++ .github/workflows/release.yml | 78 ++++++++++++ PLAN.md | 5 +- README.md | 23 ++-- backend/.env.example | 4 + backend/app/federation.py | 190 ++++++++++++++++++++++++++++++ backend/app/registry.py | 2 +- backend/config.py | 2 + backend/main.py | 15 ++- backend/tests/test_federation.py | 189 +++++++++++++++++++++++++++++ backend/tests/test_marketplace.py | 8 +- backend/tests/test_registry.py | 3 +- extension/CHANGELOG.md | 7 ++ extension/PUBLISH.md | 25 +++- extension/package.json | 6 +- extension/src/api.ts | 53 +++++++++ extension/src/extension.ts | 9 +- extension/src/marketplace.ts | 6 +- extension/src/registry.ts | 118 +++++++++++++++++++ 19 files changed, 770 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 backend/app/federation.py create mode 100644 backend/tests/test_federation.py create mode 100644 extension/src/registry.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3547046 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,52 @@ +name: CI + +on: + push: + pull_request: + +permissions: + contents: read + +jobs: + backend: + runs-on: ubuntu-latest + defaults: + run: + working-directory: backend + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Pytest + env: + PYTHONPATH: . + run: pytest -q + + extension: + runs-on: ubuntu-latest + defaults: + run: + working-directory: extension + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: npm + cache-dependency-path: extension/package-lock.json + + - name: Install and compile + run: | + npm ci + npm run compile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..4614c75 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,78 @@ +name: Release + +on: + workflow_dispatch: + push: + tags: + - 'v*' + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: npm + cache-dependency-path: extension/package-lock.json + + - name: Install, compile, package + working-directory: extension + run: | + npm ci + npm run compile + npx --yes @vscode/vsce package + + - name: Upload VSIX artifact + uses: actions/upload-artifact@v4 + with: + name: creer-vsix + path: extension/*.vsix + if-no-files-found: error + + publish: + needs: [build] + runs-on: ubuntu-latest + steps: + - name: Download VSIX artifact + uses: actions/download-artifact@v4 + with: + name: creer-vsix + path: dist + + - name: Publish (Marketplace / Open VSX when tokens present) + env: + VSCE_PAT: ${{ secrets.VSCE_PAT }} + OVSX_PAT: ${{ secrets.OVSX_PAT }} + run: | + set -euo pipefail + cd dist + shopt -s nullglob + vsix=(*.vsix) + if [ ${#vsix[@]} -eq 0 ]; then + echo "No .vsix artifact found" + exit 1 + fi + echo "Packaged: ${vsix[*]}" + + published=false + if [ -n "${VSCE_PAT:-}" ]; then + echo "Publishing to VS Marketplace…" + npx --yes @vscode/vsce publish --packagePath "${vsix[0]}" -p "$VSCE_PAT" + published=true + fi + if [ -n "${OVSX_PAT:-}" ]; then + echo "Publishing to Open VSX…" + npx --yes ovsx publish "${vsix[0]}" -p "$OVSX_PAT" + published=true + fi + if [ "$published" = false ]; then + echo "No publish tokens configured — artifact only" + fi diff --git a/PLAN.md b/PLAN.md index e258c53..227b41a 100644 --- a/PLAN.md +++ b/PLAN.md @@ -9,11 +9,12 @@ - **v0.5** — Content diff preview before write, multi-root workspace targeting, installable JSON/YAML template packs - **v0.6** — Pack marketplace + remote URL install/delete, side-by-side conflict diffs, publish packaging - **v0.7** — Self-hosted pack registry (`/registry` + download), extension icon, Browse Pack Registry, release changelog +- **v0.8** — Federated registry (`/registry/federated` + peers), Browse Federated Registry UI, GitHub Actions release + CI (artifact-first; signed publish when secrets exist) ## Optional next -- Signed publisher release with real Marketplace/Open VSX tokens (human step) -- Federated multi-host registry discovery +- Human: configure `VSCE_PAT` / `OVSX_PAT` repository secrets for signed Marketplace / Open VSX publish +- More registry peers / richer federation discovery UX ## Non-goals diff --git a/README.md b/README.md index 89dc7e4..4843fc2 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,15 @@ AI-powered repo scaffolding inside your workspace. -**Current version: 0.7.0** +**Current version: 0.8.0** ## Architecture ``` creer/ ├── backend/ # Python FastAPI AI engine (+ packs/ + registry) -└── extension/ # VS Code extension (icon in media/) +├── extension/ # VS Code extension (icon in media/) +└── .github/ # CI + release workflows ``` ## Quick start @@ -25,17 +26,20 @@ cd extension && npm install && npm run compile # F5 → Creer: Create New Repo ``` -## Registry (v0.7) +## Registry & federation (v0.7–v0.8) -Self-hosted pack catalog: +Self-hosted pack catalog plus optional multi-host federation: | Method | Path | Description | |---|---|---| | `GET` | `/registry?q=&source=` | Searchable pack list | +| `GET` | `/registry/federated?q=&source=` | Local + peer merge | | `GET` | `/registry/packs/{id}` | Pack metadata | | `GET` | `/registry/packs/{id}/download` | Portable JSON pack (installable URL) | | `GET` | `/marketplace` | Curated featured view | +Extension: **Creer: Browse Federated Registry** searches the federated catalog and installs via absolute `download_url` / `install_url`. + Install from another Creer host: ```bash @@ -45,6 +49,7 @@ curl -X POST http://localhost:8000/packs/install \ ``` Set `CREER_PUBLIC_BASE_URL` for absolute download links in registry responses. +Set `CREER_REGISTRY_PEERS` (comma-separated base URLs) for federated discovery. ## Extension commands @@ -54,16 +59,20 @@ Set `CREER_PUBLIC_BASE_URL` for absolute download links in registry responses. | `creer.createRepoFromChat` | Create from Chat Prompt | | `creer.browseMarketplace` | Browse Pack Marketplace | | `creer.browseRegistry` | Browse Pack Registry | +| `creer.browseFederatedRegistry` | Browse Federated Registry | | `creer.installPackFromUrl` | Install Pack from URL | | `creer.setGitHubToken` / `clearGitHubToken` | SecretStorage token | -## Publishing +## CI & publishing + +- **CI** (`.github/workflows/ci.yml`): pytest + extension compile on push/PR +- **Release** (`.github/workflows/release.yml`): tag `v*` or `workflow_dispatch` → package `.vsix` artifact; publish only when `VSCE_PAT` / `OVSX_PAT` secrets are set -See [`extension/PUBLISH.md`](extension/PUBLISH.md). Package: +See [`extension/PUBLISH.md`](extension/PUBLISH.md). Package locally: ```bash cd extension && npm run compile && npm run package -# → creer-0.7.0.vsix (includes media/icon.png) +# → creer-0.8.0.vsix (includes media/icon.png) ``` Signed Marketplace / Open VSX publish requires your own `VSCE_PAT` / `OVSX_PAT` (never commit tokens). diff --git a/backend/.env.example b/backend/.env.example index 35100fa..999649a 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -21,3 +21,7 @@ CREER_OFFLINE= # Optional public base URL for absolute registry download links # Example: http://localhost:8000 or https://creer.example.com # CREER_PUBLIC_BASE_URL=http://localhost:8000 + +# Comma-separated peer Creer registry base URLs (federated discovery, v0.8) +# Example: http://127.0.0.1:8001,https://creer-packs.example.com +# CREER_REGISTRY_PEERS= diff --git a/backend/app/federation.py b/backend/app/federation.py new file mode 100644 index 0000000..120492b --- /dev/null +++ b/backend/app/federation.py @@ -0,0 +1,190 @@ +"""Federated multi-host registry discovery — query peer Creer registries and merge.""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any +from urllib.parse import urljoin, urlparse + +import httpx + +from config import CREER_REGISTRY_PEERS +from app.registry import list_registry + +FEDERATION_VERSION = "0.8.0" +_MAX_PEERS = 8 + + +def parse_peers(raw: str | None = None) -> list[str]: + """ + Normalize peer base URLs from CREER_REGISTRY_PEERS (or raw override). + + Strips whitespace/trailing slash, dedupes (order preserved), keeps http/https only. + """ + text = CREER_REGISTRY_PEERS if raw is None else raw + if not text: + return [] + + seen: set[str] = set() + out: list[str] = [] + for part in text.split(","): + url = part.strip().rstrip("/") + if not url: + continue + parsed = urlparse(url) + if parsed.scheme not in ("http", "https"): + continue + if not parsed.netloc: + continue + if url in seen: + continue + seen.add(url) + out.append(url) + return out + + +def _absolutize_url(base_url: str, value: Any) -> Any: + if not isinstance(value, str) or not value: + return value + if value.startswith("http://") or value.startswith("https://"): + return value + # Relative path → absolute against peer base + return urljoin(base_url.rstrip("/") + "/", value.lstrip("/")) + + +def _tag_peer_items(base_url: str, items: list[dict[str, Any]]) -> list[dict[str, Any]]: + tagged: list[dict[str, Any]] = [] + for raw in items: + if not isinstance(raw, dict): + continue + item = dict(raw) + item["peer"] = base_url + if "download_url" in item: + item["download_url"] = _absolutize_url(base_url, item.get("download_url")) + if "install_url" in item: + item["install_url"] = _absolutize_url(base_url, item.get("install_url")) + if "url" in item and isinstance(item.get("url"), str): + item["url"] = _absolutize_url(base_url, item["url"]) + tagged.append(item) + return tagged + + +def fetch_peer_registry( + base_url: str, + *, + q: str | None = None, + source: str | None = None, + timeout: float = 8.0, +) -> list[dict[str, Any]]: + """ + GET {base}/registry and return tagged items. + + On any failure returns [] (never raises for federation callers). + """ + items, _err = _fetch_peer_registry(base_url, q=q, source=source, timeout=timeout) + return items + + +def _fetch_peer_registry( + base_url: str, + *, + q: str | None = None, + source: str | None = None, + timeout: float = 8.0, +) -> tuple[list[dict[str, Any]], str | None]: + base = (base_url or "").strip().rstrip("/") + if not base: + return [], "empty base_url" + + params: dict[str, str] = {} + if q: + params["q"] = q + if source: + params["source"] = source + + url = f"{base}/registry" + try: + with httpx.Client(timeout=timeout, follow_redirects=True) as client: + resp = client.get(url, params=params or None) + resp.raise_for_status() + data = resp.json() + except Exception as exc: # noqa: BLE001 — federation must never crash + return [], str(exc) + + if isinstance(data, dict): + raw_items = data.get("items") or [] + elif isinstance(data, list): + raw_items = data + else: + return [], "unexpected registry response shape" + + if not isinstance(raw_items, list): + return [], "unexpected registry items shape" + + return _tag_peer_items(base, raw_items), None + + +def list_federated( + *, + q: str | None = None, + source: str | None = None, + include_local: bool = True, +) -> dict[str, Any]: + """Merge local registry with peer registries (local ids win on collision).""" + local = list_registry(q=q, source=source or "all") if include_local else { + "version": FEDERATION_VERSION, + "base_url": None, + "items": [], + } + + peers = parse_peers()[:_MAX_PEERS] + peer_meta: list[dict[str, Any]] = [] + peer_items_by_url: dict[str, list[dict[str, Any]]] = {} + + def _one(peer: str) -> tuple[str, list[dict[str, Any]], str | None]: + items, err = _fetch_peer_registry(peer, q=q, source=source) + return peer, items, err + + if peers: + workers = min(8, len(peers)) + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = {pool.submit(_one, p): p for p in peers} + for fut in as_completed(futures): + peer, items, err = fut.result() + peer_items_by_url[peer] = items + peer_meta.append( + { + "base_url": peer, + "ok": err is None, + "count": len(items) if err is None else 0, + "error": err, + } + ) + # Stable peer order matching parse_peers() + order = {p: i for i, p in enumerate(peers)} + peer_meta.sort(key=lambda m: order.get(m["base_url"], 0)) + + merged: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + + for item in local.get("items") or []: + pid = item.get("id") + if isinstance(pid, str) and pid: + seen_ids.add(pid) + merged.append(item) + + for peer in peers: + for item in peer_items_by_url.get(peer, []): + pid = item.get("id") + if isinstance(pid, str) and pid in seen_ids: + continue + if isinstance(pid, str) and pid: + seen_ids.add(pid) + merged.append(item) + + return { + "version": FEDERATION_VERSION, + "local": local, + "peers": peer_meta, + "items": merged, + } diff --git a/backend/app/registry.py b/backend/app/registry.py index 403c5ac..ccc6eef 100644 --- a/backend/app/registry.py +++ b/backend/app/registry.py @@ -102,7 +102,7 @@ def matches(item: dict[str, Any]) -> bool: items = [i for i in items if matches(i)] return { - "version": "0.7.0", + "version": "0.8.0", "base_url": CREER_PUBLIC_BASE_URL or None, "items": items, } diff --git a/backend/config.py b/backend/config.py index 94f5fb8..4b5f303 100644 --- a/backend/config.py +++ b/backend/config.py @@ -11,3 +11,5 @@ CREER_PACKS_DIR = os.getenv("CREER_PACKS_DIR", "").strip() or None # Optional public base URL for absolute registry download links CREER_PUBLIC_BASE_URL = os.getenv("CREER_PUBLIC_BASE_URL", "").strip() or None +# Comma-separated peer Creer registry base URLs for federation (v0.8) +CREER_REGISTRY_PEERS = os.getenv("CREER_REGISTRY_PEERS", "").strip() or None diff --git a/backend/main.py b/backend/main.py index cc79725..8bf89fc 100644 --- a/backend/main.py +++ b/backend/main.py @@ -31,11 +31,12 @@ pack_download_bytes, registry_count, ) +from app.federation import list_federated, parse_peers from app.github import create_github_repo from app.jobs import cancel_job, create_job, finish_job, is_cancelled from app.quality import has_errors, run_quality_gates -VERSION = "0.7.0" +VERSION = "0.8.0" app = FastAPI(title="Creer", version=VERSION) @@ -154,6 +155,7 @@ def health(): "packs_count": len(list_packs()), "registry_count": registry_count(), "public_base_url_set": bool(CREER_PUBLIC_BASE_URL), + "peers_configured": len(parse_peers()), } @@ -172,10 +174,19 @@ def registry( q: str | None = Query(default=None, description="Search name/description/id/stack"), source: str = Query(default="all", description="bundled | installed | all"), ): - """Self-hosted searchable pack registry.""" + """Self-hosted searchable pack registry (local only).""" return list_registry(q=q, source=source) +@app.get("/registry/federated") +def registry_federated( + q: str | None = Query(default=None, description="Search name/description/id/stack"), + source: str = Query(default="all", description="bundled | installed | all"), +): + """Federated registry: local packs plus peer Creer registries.""" + return list_federated(q=q, source=source, include_local=True) + + @app.get("/registry/packs/{pack_id}") def registry_pack_detail(pack_id: str): item = get_registry_pack(pack_id) diff --git a/backend/tests/test_federation.py b/backend/tests/test_federation.py new file mode 100644 index 0000000..5905a95 --- /dev/null +++ b/backend/tests/test_federation.py @@ -0,0 +1,189 @@ +"""Tests for federated multi-host registry discovery (v0.8).""" + +from __future__ import annotations + +from fastapi.testclient import TestClient + +import main +from app import federation as fed +from app.federation import list_federated, parse_peers +from main import VERSION + + +def test_parse_peers_normalize_dedupe(): + raw = ( + " http://127.0.0.1:8001/,https://creer-packs.example.com," + "http://127.0.0.1:8001,ftp://bad.example,not-a-url,,https://creer-packs.example.com/ " + ) + peers = parse_peers(raw) + assert peers == [ + "http://127.0.0.1:8001", + "https://creer-packs.example.com", + ] + + +def test_parse_peers_from_config(monkeypatch): + monkeypatch.setattr( + fed, + "CREER_REGISTRY_PEERS", + "https://a.example/,https://b.example", + ) + assert parse_peers() == ["https://a.example", "https://b.example"] + + +def test_list_federated_mocked_peer(monkeypatch): + monkeypatch.setattr( + fed, + "CREER_REGISTRY_PEERS", + "http://peer.test:9000", + ) + + peer_item = { + "id": "peer-only-pack", + "name": "Peer Pack", + "description": "from peer", + "stack": "python", + "version": "1.0.0", + "source": "bundled", + "files": ["README.md"], + "download_url": "/registry/packs/peer-only-pack/download", + "install_url": "/registry/packs/peer-only-pack/download", + } + + def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): + assert base_url == "http://peer.test:9000" + return [ + { + **peer_item, + "peer": base_url, + "download_url": f"{base_url}/registry/packs/peer-only-pack/download", + "install_url": f"{base_url}/registry/packs/peer-only-pack/download", + } + ], None + + monkeypatch.setattr(fed, "_fetch_peer_registry", fake_fetch) + + result = list_federated() + assert result["version"] == "0.8.0" + assert "items" in result["local"] + assert len(result["peers"]) == 1 + assert result["peers"][0]["ok"] is True + assert result["peers"][0]["count"] == 1 + assert result["peers"][0]["error"] is None + + ids = [i["id"] for i in result["items"]] + assert "peer-only-pack" in ids + # Local packs come first + local_ids = {i["id"] for i in result["local"]["items"]} + assert ids[: len(local_ids)] == [i["id"] for i in result["local"]["items"]] + + peer_entry = next(i for i in result["items"] if i["id"] == "peer-only-pack") + assert peer_entry["peer"] == "http://peer.test:9000" + assert peer_entry["download_url"].startswith("http://peer.test:9000/") + + +def test_peer_failure_keeps_local(monkeypatch): + monkeypatch.setattr( + fed, + "CREER_REGISTRY_PEERS", + "http://dead.peer:9999", + ) + + def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): + return [], "connection refused" + + monkeypatch.setattr(fed, "_fetch_peer_registry", fake_fetch) + + result = list_federated() + assert result["version"] == "0.8.0" + local_count = len(result["local"]["items"]) + assert local_count >= 3 + assert len(result["items"]) == local_count + assert result["peers"][0]["ok"] is False + assert result["peers"][0]["count"] == 0 + assert "connection refused" in (result["peers"][0]["error"] or "") + + +def test_dedupe_prefers_local(monkeypatch): + monkeypatch.setattr( + fed, + "CREER_REGISTRY_PEERS", + "http://peer.test:9000", + ) + + def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): + # Overlap with bundled fastapi-crud id + return [ + { + "id": "fastapi-crud", + "name": "Peer FastAPI", + "peer": base_url, + "download_url": f"{base_url}/registry/packs/fastapi-crud/download", + } + ], None + + monkeypatch.setattr(fed, "_fetch_peer_registry", fake_fetch) + result = list_federated() + matches = [i for i in result["items"] if i["id"] == "fastapi-crud"] + assert len(matches) == 1 + assert matches[0].get("peer") is None # local wins + + +def test_health_0_8_and_federated_route(monkeypatch): + monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "https://a.example,https://b.example") + # Also patch config import used if parse_peers reads module-level — already patched fed + c = TestClient(main.app) + h = c.get("/health").json() + assert h["version"] == "0.8.0" + assert VERSION == "0.8.0" + assert h["peers_configured"] == 2 + + # No live peers — empty mock via monkeypatch on fetch + monkeypatch.setattr( + fed, + "_fetch_peer_registry", + lambda *a, **k: ([], "offline"), + ) + r = c.get("/registry/federated") + assert r.status_code == 200 + body = r.json() + assert body["version"] == "0.8.0" + assert "local" in body + assert len(body["items"]) == len(body["local"]["items"]) + + +def test_fetch_peer_registry_absolutizes(monkeypatch): + class FakeResp: + def raise_for_status(self): + return None + + def json(self): + return { + "items": [ + { + "id": "remote-pack", + "download_url": "/registry/packs/remote-pack/download", + "install_url": "/registry/packs/remote-pack/download", + } + ] + } + + class FakeClient: + def __init__(self, *a, **k): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get(self, url, params=None): + assert url == "http://peer.example/registry" + return FakeResp() + + monkeypatch.setattr(fed.httpx, "Client", FakeClient) + items = fed.fetch_peer_registry("http://peer.example") + assert len(items) == 1 + assert items[0]["peer"] == "http://peer.example" + assert items[0]["download_url"] == "http://peer.example/registry/packs/remote-pack/download" diff --git a/backend/tests/test_marketplace.py b/backend/tests/test_marketplace.py index 97e21f9..2a2b039 100644 --- a/backend/tests/test_marketplace.py +++ b/backend/tests/test_marketplace.py @@ -43,14 +43,16 @@ def client(install_dir): return TestClient(app) -def test_health_version_0_6(client): +def test_health_version_0_8(client, monkeypatch): + monkeypatch.setattr("main.CREER_OFFLINE", True) resp = client.get("/health") assert resp.status_code == 200 data = resp.json() - assert data["version"] == "0.7.0" - assert VERSION == "0.7.0" + assert data["version"] == "0.8.0" + assert VERSION == "0.8.0" assert data["offline"] is True assert data["packs_count"] >= 3 + assert "peers_configured" in data def test_marketplace_catalog(client): diff --git a/backend/tests/test_registry.py b/backend/tests/test_registry.py index 477f5bc..99c0cc3 100644 --- a/backend/tests/test_registry.py +++ b/backend/tests/test_registry.py @@ -13,8 +13,9 @@ def test_health_registry_fields(): c = TestClient(main.app) h = c.get("/health").json() - assert h["version"] == "0.7.0" + assert h["version"] == "0.8.0" assert h["registry_count"] >= 3 + assert "peers_configured" in h def test_registry_lists_and_search(): diff --git a/extension/CHANGELOG.md b/extension/CHANGELOG.md index f094ce6..b38eaea 100644 --- a/extension/CHANGELOG.md +++ b/extension/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 0.8.0 + +- Federated multi-host registry: `GET /registry/federated`, `CREER_REGISTRY_PEERS` +- Extension: **Creer: Browse Federated Registry** (search + install via absolute download/install URLs) +- GitHub Actions release workflow (`.vsix` artifact; optional Marketplace / Open VSX publish when secrets exist) +- Lightweight CI (backend pytest + extension compile) + ## 0.7.0 - Self-hosted pack registry: `GET /registry`, pack detail + JSON download diff --git a/extension/PUBLISH.md b/extension/PUBLISH.md index 9b9fed4..3bb869b 100644 --- a/extension/PUBLISH.md +++ b/extension/PUBLISH.md @@ -19,7 +19,7 @@ npm run package # or: npx --yes @vscode/vsce package ``` -This runs `vsce package`, respects `.vscodeignore`, and includes production dependencies (e.g. `axios`) plus `media/icon.png`. Output: `creer-0.7.0.vsix` (version from `package.json`). +This runs `vsce package`, respects `.vscodeignore`, and includes production dependencies (e.g. `axios`) plus `media/icon.png`. Output: `creer-0.8.0.vsix` (version from `package.json`). ### Publisher signing (human step) @@ -31,11 +31,28 @@ vsce/ovsx publish with a PAT **signs the release to your publisher identity**. T No tokens are stored in this repository. +## GitHub Actions release + +Workflow: [`.github/workflows/release.yml`](../.github/workflows/release.yml) + +| Trigger | Behavior | +|---|---| +| Tag `v*` (e.g. `v0.8.0`) | Build + package `.vsix`, upload artifact | +| `workflow_dispatch` | Same | + +**Publish job** downloads the artifact and: + +1. If repository secret `VSCE_PAT` is set → `npx @vscode/vsce publish --packagePath *.vsix -p "$VSCE_PAT"` +2. If repository secret `OVSX_PAT` is set → `npx ovsx publish *.vsix -p "$OVSX_PAT"` +3. If neither → logs `No publish tokens configured — artifact only` (signed publish is skipped) + +Configure secrets under **Settings → Secrets and variables → Actions** (never commit PATs). CI (`.github/workflows/ci.yml`) runs backend pytest + extension compile on push/PR and does not publish. + Install locally for a smoke test: ```bash -code --install-extension creer-0.6.0.vsix -# or Cursor: cursor --install-extension creer-0.6.0.vsix +code --install-extension creer-0.8.0.vsix +# or Cursor: cursor --install-extension creer-0.8.0.vsix ``` ## Publish to VS Marketplace @@ -72,7 +89,7 @@ Optional: `npx @vscode/vsce publish -p "$VSCE_PAT"`. cd extension npx --yes ovsx publish # with an existing vsix: - # npx --yes ovsx publish creer-0.6.0.vsix + # npx --yes ovsx publish creer-0.8.0.vsix ``` `npm run publish:ovsx` only documents this flow (exits non-zero so CI does not publish by accident). diff --git a/extension/package.json b/extension/package.json index 31d0fcb..24a956d 100644 --- a/extension/package.json +++ b/extension/package.json @@ -2,7 +2,7 @@ "name": "creer", "displayName": "Creer", "description": "AI-powered repo scaffolding inside your workspace", - "version": "0.7.0", + "version": "0.8.0", "publisher": "creer", "license": "MIT", "icon": "media/icon.png", @@ -52,6 +52,10 @@ { "command": "creer.browseRegistry", "title": "Creer: Browse Pack Registry" + }, + { + "command": "creer.browseFederatedRegistry", + "title": "Creer: Browse Federated Registry" } ], "chatParticipants": [ diff --git a/extension/src/api.ts b/extension/src/api.ts index bdf3227..12b4965 100644 --- a/extension/src/api.ts +++ b/extension/src/api.ts @@ -207,6 +207,59 @@ export async function fetchRegistry(options?: { }; } +/** Item from a federated registry merge; `peer` is set when sourced from a remote host. */ +export interface FederatedRegistryItem extends RegistryItem { + /** Peer base URL when the item came from a remote Creer registry. */ + peer?: string; +} + +export interface FederatedRegistryPeer { + base_url: string; + ok?: boolean; + count?: number; + error?: string | null; +} + +export interface FederatedRegistryResponse { + version?: string; + /** Local registry snapshot. */ + local: RegistryResponse | { + version?: string; + base_url?: string | null; + items?: RegistryItem[]; + }; + /** Configured peer hosts (reachable or not). */ + peers: FederatedRegistryPeer[]; + /** Merged catalog; peer-sourced rows may include `peer`. */ + items: FederatedRegistryItem[]; +} + +/** + * GET /registry/federated?q=&source= — local + peer registry merge. + */ +export async function fetchFederatedRegistry(options?: { + q?: string; + source?: string; +}): Promise { + const backendUrl = getBackendUrl(); + const response = await axios.get( + `${backendUrl}/registry/federated`, + { + timeout: 60_000, + params: { + q: options?.q || undefined, + source: options?.source || undefined, + }, + } + ); + return { + version: response.data.version, + local: response.data.local ?? { items: [] }, + peers: response.data.peers ?? [], + items: response.data.items ?? [], + }; +} + /** * DELETE /packs/{id} → delete installed pack */ diff --git a/extension/src/extension.ts b/extension/src/extension.ts index 9047d1b..243b154 100644 --- a/extension/src/extension.ts +++ b/extension/src/extension.ts @@ -5,6 +5,7 @@ import { browseRegistryCommand, installPackFromUrlCommand, } from './marketplace'; +import { browseFederatedRegistryCommand } from './registry'; import { runScaffoldFlow } from './scaffold'; import { clearGitHubToken, setGitHubToken } from './secrets'; @@ -58,6 +59,11 @@ export function activate(context: vscode.ExtensionContext) { () => browseRegistryCommand() ); + const browseFederatedRegistry = vscode.commands.registerCommand( + 'creer.browseFederatedRegistry', + () => browseFederatedRegistryCommand() + ); + context.subscriptions.push( createRepo, createRepoFromChat, @@ -65,7 +71,8 @@ export function activate(context: vscode.ExtensionContext) { clearToken, installPackFromUrl, browseMarketplace, - browseRegistry + browseRegistry, + browseFederatedRegistry ); registerChatParticipant(context); } diff --git a/extension/src/marketplace.ts b/extension/src/marketplace.ts index f17222d..a9a2167 100644 --- a/extension/src/marketplace.ts +++ b/extension/src/marketplace.ts @@ -9,7 +9,7 @@ import { type RegistryItem, } from './api'; -function resolveInstallUrl(url: string | undefined): string | undefined { +export function resolveInstallUrl(url: string | undefined): string | undefined { const trimmed = url?.trim(); if (!trimmed) { return undefined; @@ -62,7 +62,7 @@ export async function installPackFromUrlCommand(): Promise { } } -function isBundledSource(source: string | undefined): boolean { +export function isBundledSource(source: string | undefined): boolean { if (!source) { return false; } @@ -70,7 +70,7 @@ function isBundledSource(source: string | undefined): boolean { return s === 'bundled' || s === 'builtin' || s === 'built-in' || s === 'local'; } -async function installFromResolvedUrl( +export async function installFromResolvedUrl( label: string, url: string | undefined ): Promise { diff --git a/extension/src/registry.ts b/extension/src/registry.ts new file mode 100644 index 0000000..1598c54 --- /dev/null +++ b/extension/src/registry.ts @@ -0,0 +1,118 @@ +import * as vscode from 'vscode'; +import { + fetchFederatedRegistry, + formatAxiosError, + type FederatedRegistryItem, +} from './api'; +import { + installFromResolvedUrl, + isBundledSource, +} from './marketplace'; + +function peerHostLabel(peer: string | undefined): string | undefined { + if (!peer?.trim()) { + return undefined; + } + try { + return new URL(peer).host || peer; + } catch { + return peer; + } +} + +/** + * Creer: Browse Federated Registry — GET /registry/federated, QuickPick, install. + */ +export async function browseFederatedRegistryCommand(): Promise { + const q = await vscode.window.showInputBox({ + prompt: 'Search federated registry (leave empty for all packs)', + placeHolder: 'fastapi, express, peer host…', + ignoreFocusOut: true, + }); + if (q === undefined) { + return; + } + + let items: FederatedRegistryItem[]; + let peerCount = 0; + try { + const federated = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'Creer: loading federated registry…', + cancellable: false, + }, + () => fetchFederatedRegistry({ q: q.trim() || undefined }) + ); + items = federated.items; + peerCount = federated.peers?.length ?? 0; + } catch (err) { + const message = formatAxiosError(err, 'Federated registry unavailable'); + void vscode.window.showWarningMessage( + `Creer: federated registry endpoint not available (${message}). ` + + 'Try “Creer: Browse Pack Registry” or configure CREER_REGISTRY_PEERS on the backend.' + ); + return; + } + + if (items.length === 0) { + void vscode.window.showInformationMessage( + peerCount > 0 + ? 'Creer: no federated packs matched (peers configured).' + : 'Creer: no federated packs matched.' + ); + return; + } + + type PickItem = vscode.QuickPickItem & { fed?: FederatedRegistryItem }; + const picks: PickItem[] = items.map((item) => { + const parts: string[] = []; + if (item.source) { + parts.push(item.source); + } + const host = peerHostLabel(item.peer); + if (host) { + parts.push(host); + } + if (item.stack) { + parts.push(item.stack); + } + if (item.version) { + parts.push(item.version); + } + return { + label: item.name || item.id, + description: parts.join(' · '), + detail: item.description, + fed: item, + }; + }); + + const picked = await vscode.window.showQuickPick(picks, { + placeHolder: 'Select a federated pack to install', + ignoreFocusOut: true, + matchOnDescription: true, + matchOnDetail: true, + }); + + if (!picked?.fed) { + return; + } + + const item = picked.fed; + if (isBundledSource(item.source) && !item.peer) { + const choice = await vscode.window.showInformationMessage( + `“${item.name || item.id}” is bundled locally. Install a copy via download URL anyway?`, + 'Install copy', + 'Cancel' + ); + if (choice !== 'Install copy') { + return; + } + } + + await installFromResolvedUrl( + item.name || item.id, + item.install_url || item.download_url + ); +} From 33ffaa7987159bfe9d99802b0a43175f6e3717d7 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 08:59:30 +0000 Subject: [PATCH 11/13] Implement Creer v0.9: richer federation UX and release polish Add peer status/probe APIs and ad-hoc peers on federated queries, Manage Registry Peers + peer-aware browse UI, GitHub Release assets on tags, and RELEASE.md for the human VSCE/OVSX token publish path. Co-authored-by: Sanath S Patil --- .github/workflows/release.yml | 22 +- PLAN.md | 5 +- README.md | 22 +- RELEASE.md | 56 +++++ backend/app/federation.py | 141 ++++++++++- backend/app/registry.py | 2 +- backend/config.py | 2 +- backend/main.py | 37 ++- backend/tests/test_federation.py | 12 +- backend/tests/test_federation_ux.py | 292 +++++++++++++++++++++++ backend/tests/test_marketplace.py | 6 +- backend/tests/test_registry.py | 2 +- extension/CHANGELOG.md | 8 + extension/PUBLISH.md | 12 +- extension/package-lock.json | 4 +- extension/package.json | 16 +- extension/src/api.ts | 70 +++++- extension/src/extension.ts | 10 +- extension/src/registry.ts | 347 +++++++++++++++++++++++++++- 19 files changed, 1016 insertions(+), 50 deletions(-) create mode 100644 RELEASE.md create mode 100644 backend/tests/test_federation_ux.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4614c75..af42c6e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: - 'v*' permissions: - contents: read + contents: write jobs: build: @@ -37,6 +37,26 @@ jobs: path: extension/*.vsix if-no-files-found: error + github-release: + needs: [build] + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + steps: + - name: Download VSIX artifact + uses: actions/download-artifact@v4 + with: + name: creer-vsix + path: dist + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + files: dist/*.vsix + generate_release_notes: true + fail_on_unmatched_files: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + publish: needs: [build] runs-on: ubuntu-latest diff --git a/PLAN.md b/PLAN.md index 227b41a..db6c97a 100644 --- a/PLAN.md +++ b/PLAN.md @@ -10,11 +10,12 @@ - **v0.6** — Pack marketplace + remote URL install/delete, side-by-side conflict diffs, publish packaging - **v0.7** — Self-hosted pack registry (`/registry` + download), extension icon, Browse Pack Registry, release changelog - **v0.8** — Federated registry (`/registry/federated` + peers), Browse Federated Registry UI, GitHub Actions release + CI (artifact-first; signed publish when secrets exist) +- **v0.9** — Peer status/probe UX (`creer.registryPeers`, Manage Registry Peers), federated browse enrichment, GitHub Release on tag + `RELEASE.md` ## Optional next -- Human: configure `VSCE_PAT` / `OVSX_PAT` repository secrets for signed Marketplace / Open VSX publish -- More registry peers / richer federation discovery UX +- Human: configure `VSCE_PAT` / `OVSX_PAT` repository secrets; tag `v0.9.0` (see [`RELEASE.md`](RELEASE.md)) +- More peer discovery / registry auth ## Non-goals diff --git a/README.md b/README.md index 4843fc2..0237c04 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ AI-powered repo scaffolding inside your workspace. -**Current version: 0.8.0** +**Current version: 0.9.0** ## Architecture @@ -26,19 +26,23 @@ cd extension && npm install && npm run compile # F5 → Creer: Create New Repo ``` -## Registry & federation (v0.7–v0.8) +## Registry & federation (v0.7–v0.9) Self-hosted pack catalog plus optional multi-host federation: | Method | Path | Description | |---|---|---| | `GET` | `/registry?q=&source=` | Searchable pack list | -| `GET` | `/registry/federated?q=&source=` | Local + peer merge | +| `GET` | `/registry/federated?q=&source=&peers=` | Local + peer merge (extra peers CSV) | +| `GET` | `/registry/peers` | Peer health + configured URLs | +| `POST` | `/registry/peers/probe` | Probe one peer `{ url }` | | `GET` | `/registry/packs/{id}` | Pack metadata | | `GET` | `/registry/packs/{id}/download` | Portable JSON pack (installable URL) | | `GET` | `/marketplace` | Curated featured view | -Extension: **Creer: Browse Federated Registry** searches the federated catalog and installs via absolute `download_url` / `install_url`. +Extension settings: `creer.registryPeers` (comma-separated peer base URLs), `creer.showPeerStatus` (peer health in federated browse). + +Commands: **Creer: Browse Federated Registry**, **Creer: Manage Registry Peers**. Install from another Creer host: @@ -49,7 +53,8 @@ curl -X POST http://localhost:8000/packs/install \ ``` Set `CREER_PUBLIC_BASE_URL` for absolute download links in registry responses. -Set `CREER_REGISTRY_PEERS` (comma-separated base URLs) for federated discovery. +Set `CREER_REGISTRY_PEERS` (comma-separated base URLs) for backend-configured federated discovery. +Use `creer.registryPeers` in the extension for client-side extra peers when browsing. ## Extension commands @@ -60,19 +65,20 @@ Set `CREER_REGISTRY_PEERS` (comma-separated base URLs) for federated discovery. | `creer.browseMarketplace` | Browse Pack Marketplace | | `creer.browseRegistry` | Browse Pack Registry | | `creer.browseFederatedRegistry` | Browse Federated Registry | +| `creer.manageRegistryPeers` | Manage Registry Peers | | `creer.installPackFromUrl` | Install Pack from URL | | `creer.setGitHubToken` / `clearGitHubToken` | SecretStorage token | ## CI & publishing - **CI** (`.github/workflows/ci.yml`): pytest + extension compile on push/PR -- **Release** (`.github/workflows/release.yml`): tag `v*` or `workflow_dispatch` → package `.vsix` artifact; publish only when `VSCE_PAT` / `OVSX_PAT` secrets are set +- **Release** (`.github/workflows/release.yml`): tag `v*` → package `.vsix`, create GitHub Release with attachment; publish to Marketplace / Open VSX only when `VSCE_PAT` / `OVSX_PAT` secrets are set -See [`extension/PUBLISH.md`](extension/PUBLISH.md). Package locally: +See [`RELEASE.md`](RELEASE.md) and [`extension/PUBLISH.md`](extension/PUBLISH.md). Package locally: ```bash cd extension && npm run compile && npm run package -# → creer-0.8.0.vsix (includes media/icon.png) +# → creer-0.9.0.vsix (includes media/icon.png) ``` Signed Marketplace / Open VSX publish requires your own `VSCE_PAT` / `OVSX_PAT` (never commit tokens). diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..fce98f3 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,56 @@ +# Releasing Creer v0.9.0 + +Exact steps for a human maintainer to cut a tagged release with GitHub Release + optional Marketplace / Open VSX publish. + +## 1. Set repository secrets (once) + +In GitHub → **Settings → Secrets and variables → Actions**, add: + +| Secret | Purpose | +|---|---| +| `VSCE_PAT` | Azure DevOps PAT with Marketplace **Acquire** + **Publish** (publisher must match `extension/package.json` → `publisher`) | +| `OVSX_PAT` | Open VSX access token from [open-vsx.org](https://open-vsx.org/) | + +Both are optional. If neither is set, the release workflow still builds the `.vsix`, uploads it as an artifact, and (on tag pushes) creates a **GitHub Release** with the `.vsix` attached. Marketplace / Open VSX publish is skipped with `No publish tokens configured — artifact only`. + +Never commit PATs. Prefer repo secrets over exporting tokens in shared shells. + +## 2. Bump & verify locally + +```bash +# Confirm extension version is 0.9.0 +grep '"version"' extension/package.json + +cd extension +npm ci +npm run compile +npm run package +# → creer-0.9.0.vsix +``` + +Smoke-test: `code --install-extension creer-0.9.0.vsix` (or Cursor equivalent) against a running backend. + +## 3. Tag v0.9.0 and push + +From a clean `main` (or the release commit): + +```bash +git tag -a v0.9.0 -m "Creer v0.9.0" +git push origin v0.9.0 +``` + +Tag pattern `v*` triggers [`.github/workflows/release.yml`](.github/workflows/release.yml). + +## 4. What the workflow does + +1. **build** — `npm ci` → `compile` → `vsce package` → upload `creer-vsix` artifact +2. **github-release** (tag pushes only) — create a GitHub Release and attach the `.vsix` (`contents: write`) +3. **publish** — if `VSCE_PAT` / `OVSX_PAT` secrets exist, publish to Marketplace / Open VSX; otherwise artifact-only + +You can also run the workflow via **Actions → Release → Run workflow** (`workflow_dispatch`) for a package/artifact without a tag (no GitHub Release job in that case). + +## 5. After release + +- Confirm the GitHub Release page lists `creer-0.9.0.vsix` +- If secrets were set, confirm Marketplace / Open VSX listing updated to 0.9.0 +- See [`extension/PUBLISH.md`](extension/PUBLISH.md) for manual `vsce` / `ovsx` publish from a laptop diff --git a/backend/app/federation.py b/backend/app/federation.py index 120492b..2ae88ef 100644 --- a/backend/app/federation.py +++ b/backend/app/federation.py @@ -2,6 +2,7 @@ from __future__ import annotations +import time from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Any from urllib.parse import urljoin, urlparse @@ -11,7 +12,7 @@ from config import CREER_REGISTRY_PEERS from app.registry import list_registry -FEDERATION_VERSION = "0.8.0" +FEDERATION_VERSION = "0.9.0" _MAX_PEERS = 8 @@ -43,6 +44,33 @@ def parse_peers(raw: str | None = None) -> list[str]: return out +def resolve_peers(extra_peers: list[str] | None = None) -> list[str]: + """ + Merge configured peers with optional ad-hoc extras. + + Dedupes (configured first), http/https only, capped at _MAX_PEERS. + """ + configured = parse_peers() + extras: list[str] = [] + if extra_peers: + # Normalize each entry (allow raw URLs or comma-joined strings) + for raw in extra_peers: + if not raw: + continue + extras.extend(parse_peers(raw)) + + seen: set[str] = set() + out: list[str] = [] + for url in configured + extras: + if url in seen: + continue + seen.add(url) + out.append(url) + if len(out) >= _MAX_PEERS: + break + return out + + def _absolutize_url(base_url: str, value: Any) -> Any: if not isinstance(value, str) or not value: return value @@ -124,11 +152,118 @@ def _fetch_peer_registry( return _tag_peer_items(base, raw_items), None +def probe_peer(base_url: str, timeout: float = 5.0) -> dict[str, Any]: + """ + Probe a peer host via /health (preferred) and/or /registry for pack count. + + Never raises — returns a status dict with ok/latency/count/version/error. + """ + base = (base_url or "").strip().rstrip("/") + result: dict[str, Any] = { + "base_url": base, + "ok": False, + "latency_ms": None, + "count": None, + "version": None, + "error": None, + } + if not base: + result["error"] = "empty base_url" + return result + + parsed = urlparse(base) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + result["error"] = "url must use http or https with a host" + return result + + started = time.perf_counter() + version: str | None = None + count: int | None = None + health_ok = False + last_error: str | None = None + + try: + with httpx.Client(timeout=timeout, follow_redirects=True) as client: + # Prefer /health for liveness + version (+ registry_count when present) + try: + hresp = client.get(f"{base}/health") + hresp.raise_for_status() + hdata = hresp.json() + health_ok = True + if isinstance(hdata, dict): + ver = hdata.get("version") + if isinstance(ver, str): + version = ver + if "registry_count" in hdata and hdata["registry_count"] is not None: + try: + count = int(hdata["registry_count"]) + except (TypeError, ValueError): + pass + except Exception as exc: # noqa: BLE001 + last_error = str(exc) + + # Use /registry for count (and version fallback) when needed + if count is None or version is None: + try: + rresp = client.get(f"{base}/registry") + rresp.raise_for_status() + rdata = rresp.json() + if isinstance(rdata, dict): + if version is None: + ver = rdata.get("version") + if isinstance(ver, str): + version = ver + if count is None: + raw_items = rdata.get("items") or [] + count = len(raw_items) if isinstance(raw_items, list) else 0 + elif isinstance(rdata, list) and count is None: + count = len(rdata) + except Exception as exc: # noqa: BLE001 + if not health_ok: + last_error = str(exc) + elif last_error is None: + last_error = str(exc) + + latency_ms = round((time.perf_counter() - started) * 1000, 1) + result["latency_ms"] = latency_ms + + if health_ok or count is not None: + result["ok"] = True + result["count"] = count if count is not None else 0 + result["version"] = version + result["error"] = None + return result + + result["error"] = last_error or "unreachable" + return result + except Exception as exc: # noqa: BLE001 + result["latency_ms"] = round((time.perf_counter() - started) * 1000, 1) + result["error"] = str(exc) + return result + + +def list_peer_status() -> list[dict[str, Any]]: + """Probe all configured peers concurrently; preserve configured order.""" + peers = parse_peers()[:_MAX_PEERS] + if not peers: + return [] + + by_url: dict[str, dict[str, Any]] = {} + workers = min(8, len(peers)) + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = {pool.submit(probe_peer, p): p for p in peers} + for fut in as_completed(futures): + peer = futures[fut] + by_url[peer] = fut.result() + return [by_url[p] for p in peers] + + def list_federated( *, q: str | None = None, source: str | None = None, include_local: bool = True, + extra_peers: list[str] | None = None, ) -> dict[str, Any]: """Merge local registry with peer registries (local ids win on collision).""" local = list_registry(q=q, source=source or "all") if include_local else { @@ -137,7 +272,7 @@ def list_federated( "items": [], } - peers = parse_peers()[:_MAX_PEERS] + peers = resolve_peers(extra_peers) peer_meta: list[dict[str, Any]] = [] peer_items_by_url: dict[str, list[dict[str, Any]]] = {} @@ -160,7 +295,7 @@ def _one(peer: str) -> tuple[str, list[dict[str, Any]], str | None]: "error": err, } ) - # Stable peer order matching parse_peers() + # Stable peer order matching resolve_peers() order = {p: i for i, p in enumerate(peers)} peer_meta.sort(key=lambda m: order.get(m["base_url"], 0)) diff --git a/backend/app/registry.py b/backend/app/registry.py index ccc6eef..a09901b 100644 --- a/backend/app/registry.py +++ b/backend/app/registry.py @@ -102,7 +102,7 @@ def matches(item: dict[str, Any]) -> bool: items = [i for i in items if matches(i)] return { - "version": "0.8.0", + "version": "0.9.0", "base_url": CREER_PUBLIC_BASE_URL or None, "items": items, } diff --git a/backend/config.py b/backend/config.py index 4b5f303..0e5fc9f 100644 --- a/backend/config.py +++ b/backend/config.py @@ -11,5 +11,5 @@ CREER_PACKS_DIR = os.getenv("CREER_PACKS_DIR", "").strip() or None # Optional public base URL for absolute registry download links CREER_PUBLIC_BASE_URL = os.getenv("CREER_PUBLIC_BASE_URL", "").strip() or None -# Comma-separated peer Creer registry base URLs for federation (v0.8) +# Comma-separated peer Creer registry base URLs for federation (v0.8+) CREER_REGISTRY_PEERS = os.getenv("CREER_REGISTRY_PEERS", "").strip() or None diff --git a/backend/main.py b/backend/main.py index 8bf89fc..54609c7 100644 --- a/backend/main.py +++ b/backend/main.py @@ -5,6 +5,7 @@ import json from collections.abc import Iterator from typing import Any, Literal +from urllib.parse import urlparse from fastapi import FastAPI, Header, HTTPException, Query from fastapi.responses import Response, StreamingResponse @@ -31,12 +32,12 @@ pack_download_bytes, registry_count, ) -from app.federation import list_federated, parse_peers +from app.federation import list_federated, list_peer_status, parse_peers, probe_peer from app.github import create_github_repo from app.jobs import cancel_job, create_job, finish_job, is_cancelled from app.quality import has_errors, run_quality_gates -VERSION = "0.8.0" +VERSION = "0.9.0" app = FastAPI(title="Creer", version=VERSION) @@ -93,6 +94,10 @@ class PackInstallRequest(BaseModel): overwrite: bool = False +class PeerProbeRequest(BaseModel): + url: str = Field(..., min_length=1, max_length=2000) + + def _check_pack_template_exclusive( pack_id: str | None, template_id: str | None ) -> None: @@ -182,9 +187,35 @@ def registry( def registry_federated( q: str | None = Query(default=None, description="Search name/description/id/stack"), source: str = Query(default="all", description="bundled | installed | all"), + peers: str | None = Query( + default=None, + description="Comma-separated extra peer base URLs for this request only", + ), ): """Federated registry: local packs plus peer Creer registries.""" - return list_federated(q=q, source=source, include_local=True) + extra = parse_peers(peers) if peers else None + return list_federated( + q=q, source=source, include_local=True, extra_peers=extra + ) + + +@app.get("/registry/peers") +def registry_peers(): + """Configured peer list plus live probe status for each peer.""" + return {"peers": list_peer_status(), "configured": parse_peers()} + + +@app.post("/registry/peers/probe") +def registry_peers_probe(request: PeerProbeRequest): + """Ad-hoc probe of a single peer base URL.""" + url = (request.url or "").strip().rstrip("/") + parsed = urlparse(url) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + raise HTTPException( + status_code=400, + detail="url must use http or https with a host", + ) + return probe_peer(url) @app.get("/registry/packs/{pack_id}") diff --git a/backend/tests/test_federation.py b/backend/tests/test_federation.py index 5905a95..171c111 100644 --- a/backend/tests/test_federation.py +++ b/backend/tests/test_federation.py @@ -64,7 +64,7 @@ def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): monkeypatch.setattr(fed, "_fetch_peer_registry", fake_fetch) result = list_federated() - assert result["version"] == "0.8.0" + assert result["version"] == "0.9.0" assert "items" in result["local"] assert len(result["peers"]) == 1 assert result["peers"][0]["ok"] is True @@ -95,7 +95,7 @@ def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): monkeypatch.setattr(fed, "_fetch_peer_registry", fake_fetch) result = list_federated() - assert result["version"] == "0.8.0" + assert result["version"] == "0.9.0" local_count = len(result["local"]["items"]) assert local_count >= 3 assert len(result["items"]) == local_count @@ -129,13 +129,13 @@ def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): assert matches[0].get("peer") is None # local wins -def test_health_0_8_and_federated_route(monkeypatch): +def test_health_0_9_and_federated_route(monkeypatch): monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "https://a.example,https://b.example") # Also patch config import used if parse_peers reads module-level — already patched fed c = TestClient(main.app) h = c.get("/health").json() - assert h["version"] == "0.8.0" - assert VERSION == "0.8.0" + assert h["version"] == "0.9.0" + assert VERSION == "0.9.0" assert h["peers_configured"] == 2 # No live peers — empty mock via monkeypatch on fetch @@ -147,7 +147,7 @@ def test_health_0_8_and_federated_route(monkeypatch): r = c.get("/registry/federated") assert r.status_code == 200 body = r.json() - assert body["version"] == "0.8.0" + assert body["version"] == "0.9.0" assert "local" in body assert len(body["items"]) == len(body["local"]["items"]) diff --git a/backend/tests/test_federation_ux.py b/backend/tests/test_federation_ux.py new file mode 100644 index 0000000..d7e66f6 --- /dev/null +++ b/backend/tests/test_federation_ux.py @@ -0,0 +1,292 @@ +"""Tests for federation UX — peer status, probe, ad-hoc peers (v0.9).""" + +from __future__ import annotations + +from fastapi.testclient import TestClient + +import main +from app import federation as fed +from app.federation import list_federated, list_peer_status, probe_peer, resolve_peers +from main import VERSION + + +def test_probe_peer_ok(monkeypatch): + calls: list[str] = [] + + class FakeResp: + def __init__(self, data): + self._data = data + + def raise_for_status(self): + return None + + def json(self): + return self._data + + class FakeClient: + def __init__(self, *a, **k): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get(self, url, params=None): + calls.append(url) + if url.endswith("/health"): + return FakeResp( + {"status": "ok", "version": "0.8.0", "registry_count": 3} + ) + raise AssertionError(f"unexpected url: {url}") + + monkeypatch.setattr(fed.httpx, "Client", FakeClient) + result = probe_peer("http://peer.example:8001") + assert result["ok"] is True + assert result["base_url"] == "http://peer.example:8001" + assert result["version"] == "0.8.0" + assert result["count"] == 3 + assert result["error"] is None + assert isinstance(result["latency_ms"], (int, float)) + assert result["latency_ms"] >= 0 + assert any(u.endswith("/health") for u in calls) + + +def test_probe_peer_fail(monkeypatch): + class FakeClient: + def __init__(self, *a, **k): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get(self, url, params=None): + raise ConnectionError("connection refused") + + monkeypatch.setattr(fed.httpx, "Client", FakeClient) + result = probe_peer("http://dead.peer:9999") + assert result["ok"] is False + assert result["base_url"] == "http://dead.peer:9999" + assert result["error"] + assert "connection refused" in result["error"] + assert result["count"] is None + assert isinstance(result["latency_ms"], (int, float)) + + +def test_probe_peer_registry_fallback(monkeypatch): + """When /health fails, /registry still yields ok + count.""" + + class FakeResp: + def __init__(self, data): + self._data = data + + def raise_for_status(self): + return None + + def json(self): + return self._data + + class FakeClient: + def __init__(self, *a, **k): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def get(self, url, params=None): + if url.endswith("/health"): + raise ConnectionError("no health") + if url.endswith("/registry"): + return FakeResp( + { + "version": "0.8.0", + "items": [{"id": "a"}, {"id": "b"}], + } + ) + raise AssertionError(url) + + monkeypatch.setattr(fed.httpx, "Client", FakeClient) + result = probe_peer("http://peer.example") + assert result["ok"] is True + assert result["count"] == 2 + assert result["version"] == "0.8.0" + assert result["error"] is None + + +def test_resolve_peers_merges_extra_dedupe_max(): + # configured via monkeypatch below in other tests; unit-test via parse_peers path + peers = resolve_peers(None) + assert isinstance(peers, list) + + # With extras only when no config + from app import federation as f + + # Temporarily empty config + original = f.CREER_REGISTRY_PEERS + try: + f.CREER_REGISTRY_PEERS = "http://a.example,http://b.example" + merged = resolve_peers( + [ + "http://b.example/", + "http://c.example", + "ftp://bad", + "http://d.example", + ] + ) + assert merged == [ + "http://a.example", + "http://b.example", + "http://c.example", + "http://d.example", + ] + + f.CREER_REGISTRY_PEERS = ",".join(f"http://p{i}.example" for i in range(10)) + capped = resolve_peers(["http://extra.example"]) + assert len(capped) == 8 + assert "http://extra.example" not in capped # configured fills the cap + finally: + f.CREER_REGISTRY_PEERS = original + + +def test_list_peer_status_concurrent(monkeypatch): + monkeypatch.setattr( + fed, + "CREER_REGISTRY_PEERS", + "http://a.example,http://b.example", + ) + + def fake_probe(base_url, timeout=5.0): + return { + "base_url": base_url, + "ok": base_url.endswith("a.example"), + "latency_ms": 1.0, + "count": 1 if base_url.endswith("a.example") else None, + "version": "0.9.0" if base_url.endswith("a.example") else None, + "error": None if base_url.endswith("a.example") else "down", + } + + monkeypatch.setattr(fed, "probe_peer", fake_probe) + statuses = list_peer_status() + assert len(statuses) == 2 + assert statuses[0]["base_url"] == "http://a.example" + assert statuses[0]["ok"] is True + assert statuses[1]["base_url"] == "http://b.example" + assert statuses[1]["ok"] is False + + +def test_registry_peers_route(monkeypatch): + monkeypatch.setattr( + fed, + "CREER_REGISTRY_PEERS", + "https://peer.example", + ) + monkeypatch.setattr( + fed, + "probe_peer", + lambda base_url, timeout=5.0: { + "base_url": base_url, + "ok": True, + "latency_ms": 2.5, + "count": 4, + "version": "0.9.0", + "error": None, + }, + ) + c = TestClient(main.app) + r = c.get("/registry/peers") + assert r.status_code == 200 + body = r.json() + assert body["configured"] == ["https://peer.example"] + assert len(body["peers"]) == 1 + assert body["peers"][0]["ok"] is True + assert body["peers"][0]["count"] == 4 + + +def test_registry_peers_probe_validation(): + c = TestClient(main.app) + bad = c.post("/registry/peers/probe", json={"url": "ftp://evil.example"}) + assert bad.status_code == 400 + + bad2 = c.post("/registry/peers/probe", json={"url": "not-a-url"}) + assert bad2.status_code == 400 + + bad3 = c.post("/registry/peers/probe", json={"url": "http://"}) + assert bad3.status_code == 400 + + +def test_registry_peers_probe_ok(monkeypatch): + monkeypatch.setattr( + main, + "probe_peer", + lambda base_url, timeout=5.0: { + "base_url": base_url.rstrip("/"), + "ok": True, + "latency_ms": 10.0, + "count": 2, + "version": "0.9.0", + "error": None, + }, + ) + c = TestClient(main.app) + r = c.post("/registry/peers/probe", json={"url": "http://peer.test:9000/"}) + assert r.status_code == 200 + body = r.json() + assert body["ok"] is True + assert body["base_url"] == "http://peer.test:9000" + assert body["count"] == 2 + + +def test_federated_with_extra_peers_query(monkeypatch): + monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "") + + def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): + assert base_url == "http://adhoc.peer:8002" + return [ + { + "id": "adhoc-pack", + "name": "Adhoc", + "peer": base_url, + "download_url": f"{base_url}/registry/packs/adhoc-pack/download", + } + ], None + + monkeypatch.setattr(fed, "_fetch_peer_registry", fake_fetch) + c = TestClient(main.app) + r = c.get("/registry/federated", params={"peers": "http://adhoc.peer:8002/"}) + assert r.status_code == 200 + body = r.json() + assert body["version"] == "0.9.0" + assert any(p["base_url"] == "http://adhoc.peer:8002" for p in body["peers"]) + assert any(i["id"] == "adhoc-pack" for i in body["items"]) + + +def test_list_federated_extra_peers_arg(monkeypatch): + monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "http://cfg.peer") + + seen: list[str] = [] + + def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): + seen.append(base_url) + return [], None + + monkeypatch.setattr(fed, "_fetch_peer_registry", fake_fetch) + result = list_federated(extra_peers=["http://extra.peer/", "http://cfg.peer"]) + assert result["version"] == "0.9.0" + assert seen == ["http://cfg.peer", "http://extra.peer"] + + +def test_health_0_9(monkeypatch): + monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "https://a.example,https://b.example") + c = TestClient(main.app) + h = c.get("/health").json() + assert h["version"] == "0.9.0" + assert VERSION == "0.9.0" + assert h["peers_configured"] == 2 + assert h["status"] == "ok" diff --git a/backend/tests/test_marketplace.py b/backend/tests/test_marketplace.py index 2a2b039..5a70869 100644 --- a/backend/tests/test_marketplace.py +++ b/backend/tests/test_marketplace.py @@ -43,13 +43,13 @@ def client(install_dir): return TestClient(app) -def test_health_version_0_8(client, monkeypatch): +def test_health_version_0_9(client, monkeypatch): monkeypatch.setattr("main.CREER_OFFLINE", True) resp = client.get("/health") assert resp.status_code == 200 data = resp.json() - assert data["version"] == "0.8.0" - assert VERSION == "0.8.0" + assert data["version"] == "0.9.0" + assert VERSION == "0.9.0" assert data["offline"] is True assert data["packs_count"] >= 3 assert "peers_configured" in data diff --git a/backend/tests/test_registry.py b/backend/tests/test_registry.py index 99c0cc3..9b3cf64 100644 --- a/backend/tests/test_registry.py +++ b/backend/tests/test_registry.py @@ -13,7 +13,7 @@ def test_health_registry_fields(): c = TestClient(main.app) h = c.get("/health").json() - assert h["version"] == "0.8.0" + assert h["version"] == "0.9.0" assert h["registry_count"] >= 3 assert "peers_configured" in h diff --git a/extension/CHANGELOG.md b/extension/CHANGELOG.md index b38eaea..6bd19ed 100644 --- a/extension/CHANGELOG.md +++ b/extension/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.9.0 + +- Richer federation UX: `creer.registryPeers` + `creer.showPeerStatus` settings +- API: `fetchPeerStatus`, `probePeer`; federated fetch passes `peers` query +- **Creer: Browse Federated Registry** — peer health summary, local/`$(cloud)` labels +- **Creer: Manage Registry Peers** — add/remove setting peers, probe all (soft-fail) +- Release polish: GitHub Release on tag with `.vsix` attached; [`RELEASE.md`](../RELEASE.md) + ## 0.8.0 - Federated multi-host registry: `GET /registry/federated`, `CREER_REGISTRY_PEERS` diff --git a/extension/PUBLISH.md b/extension/PUBLISH.md index 3bb869b..53a5072 100644 --- a/extension/PUBLISH.md +++ b/extension/PUBLISH.md @@ -19,7 +19,7 @@ npm run package # or: npx --yes @vscode/vsce package ``` -This runs `vsce package`, respects `.vscodeignore`, and includes production dependencies (e.g. `axios`) plus `media/icon.png`. Output: `creer-0.8.0.vsix` (version from `package.json`). +This runs `vsce package`, respects `.vscodeignore`, and includes production dependencies (e.g. `axios`) plus `media/icon.png`. Output: `creer-0.9.0.vsix` (version from `package.json`). ### Publisher signing (human step) @@ -37,8 +37,8 @@ Workflow: [`.github/workflows/release.yml`](../.github/workflows/release.yml) | Trigger | Behavior | |---|---| -| Tag `v*` (e.g. `v0.8.0`) | Build + package `.vsix`, upload artifact | -| `workflow_dispatch` | Same | +| Tag `v*` (e.g. `v0.9.0`) | Build + package `.vsix`, upload artifact, create GitHub Release with `.vsix` attached | +| `workflow_dispatch` | Build + package + artifact (no GitHub Release); publish only if secrets set | **Publish job** downloads the artifact and: @@ -51,8 +51,8 @@ Configure secrets under **Settings → Secrets and variables → Actions** (neve Install locally for a smoke test: ```bash -code --install-extension creer-0.8.0.vsix -# or Cursor: cursor --install-extension creer-0.8.0.vsix +code --install-extension creer-0.9.0.vsix +# or Cursor: cursor --install-extension creer-0.9.0.vsix ``` ## Publish to VS Marketplace @@ -89,7 +89,7 @@ Optional: `npx @vscode/vsce publish -p "$VSCE_PAT"`. cd extension npx --yes ovsx publish # with an existing vsix: - # npx --yes ovsx publish creer-0.8.0.vsix + # npx --yes ovsx publish creer-0.9.0.vsix ``` `npm run publish:ovsx` only documents this flow (exits non-zero so CI does not publish by accident). diff --git a/extension/package-lock.json b/extension/package-lock.json index 54caa3e..af6747f 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -1,12 +1,12 @@ { "name": "creer", - "version": "0.1.0", + "version": "0.9.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "creer", - "version": "0.1.0", + "version": "0.9.0", "dependencies": { "axios": "^1.7.9" }, diff --git a/extension/package.json b/extension/package.json index 24a956d..eebcb33 100644 --- a/extension/package.json +++ b/extension/package.json @@ -2,7 +2,7 @@ "name": "creer", "displayName": "Creer", "description": "AI-powered repo scaffolding inside your workspace", - "version": "0.8.0", + "version": "0.9.0", "publisher": "creer", "license": "MIT", "icon": "media/icon.png", @@ -56,6 +56,10 @@ { "command": "creer.browseFederatedRegistry", "title": "Creer: Browse Federated Registry" + }, + { + "command": "creer.manageRegistryPeers", + "title": "Creer: Manage Registry Peers" } ], "chatParticipants": [ @@ -137,6 +141,16 @@ "type": "string", "default": "", "description": "Optional workspace folder name or path hint for multi-root workspaces; empty prompts when multiple folders are open" + }, + "creer.registryPeers": { + "type": "string", + "default": "", + "description": "Comma-separated peer base URLs used when browsing the federated registry (passed as peers= query)" + }, + "creer.showPeerStatus": { + "type": "boolean", + "default": true, + "description": "When browsing the federated registry, show peer health summary (ok/fail/latency)" } } } diff --git a/extension/src/api.ts b/extension/src/api.ts index 12b4965..7e62e8b 100644 --- a/extension/src/api.ts +++ b/extension/src/api.ts @@ -235,13 +235,23 @@ export interface FederatedRegistryResponse { } /** - * GET /registry/federated?q=&source= — local + peer registry merge. + * GET /registry/federated?q=&source=&peers= — local + peer registry merge. + * `peers` is a comma-separated list of extra peer base URLs (from settings or callers). */ export async function fetchFederatedRegistry(options?: { q?: string; source?: string; + /** Extra peer base URLs (comma-separated string or array). */ + peers?: string | string[]; }): Promise { const backendUrl = getBackendUrl(); + let peersParam: string | undefined; + if (Array.isArray(options?.peers)) { + const joined = options.peers.map((p) => p.trim()).filter(Boolean).join(','); + peersParam = joined || undefined; + } else if (typeof options?.peers === 'string' && options.peers.trim()) { + peersParam = options.peers.trim(); + } const response = await axios.get( `${backendUrl}/registry/federated`, { @@ -249,6 +259,7 @@ export async function fetchFederatedRegistry(options?: { params: { q: options?.q || undefined, source: options?.source || undefined, + peers: peersParam, }, } ); @@ -260,6 +271,63 @@ export async function fetchFederatedRegistry(options?: { }; } +/** Live peer probe / status from GET /registry/peers or POST /registry/peers/probe. */ +export interface PeerStatus { + /** Peer base URL (preferred). */ + url?: string; + /** Alternate field some backends may return. */ + base_url?: string; + ok: boolean; + latency_ms?: number | null; + error?: string | null; + count?: number | null; +} + +export interface PeerStatusListResponse { + peers: PeerStatus[]; + /** Backend-configured peer URLs (CREER_REGISTRY_PEERS). */ + configured: string[]; +} + +function normalizePeerStatus(raw: PeerStatus): PeerStatus { + return { + url: raw.url || raw.base_url || '', + base_url: raw.base_url || raw.url || '', + ok: Boolean(raw.ok), + latency_ms: raw.latency_ms ?? null, + error: raw.error ?? null, + count: raw.count ?? null, + }; +} + +/** + * GET /registry/peers → `{ peers: PeerStatus[], configured: string[] }` + */ +export async function fetchPeerStatus(): Promise { + const backendUrl = getBackendUrl(); + const response = await axios.get( + `${backendUrl}/registry/peers`, + { timeout: 30_000 } + ); + return { + peers: (response.data.peers ?? []).map(normalizePeerStatus), + configured: response.data.configured ?? [], + }; +} + +/** + * POST /registry/peers/probe `{ url }` → PeerStatus + */ +export async function probePeer(url: string): Promise { + const backendUrl = getBackendUrl(); + const response = await axios.post( + `${backendUrl}/registry/peers/probe`, + { url }, + { timeout: 30_000 } + ); + return normalizePeerStatus(response.data); +} + /** * DELETE /packs/{id} → delete installed pack */ diff --git a/extension/src/extension.ts b/extension/src/extension.ts index 243b154..406d206 100644 --- a/extension/src/extension.ts +++ b/extension/src/extension.ts @@ -5,7 +5,7 @@ import { browseRegistryCommand, installPackFromUrlCommand, } from './marketplace'; -import { browseFederatedRegistryCommand } from './registry'; +import { browseFederatedRegistryCommand, manageRegistryPeersCommand } from './registry'; import { runScaffoldFlow } from './scaffold'; import { clearGitHubToken, setGitHubToken } from './secrets'; @@ -64,6 +64,11 @@ export function activate(context: vscode.ExtensionContext) { () => browseFederatedRegistryCommand() ); + const manageRegistryPeers = vscode.commands.registerCommand( + 'creer.manageRegistryPeers', + () => manageRegistryPeersCommand() + ); + context.subscriptions.push( createRepo, createRepoFromChat, @@ -72,7 +77,8 @@ export function activate(context: vscode.ExtensionContext) { installPackFromUrl, browseMarketplace, browseRegistry, - browseFederatedRegistry + browseFederatedRegistry, + manageRegistryPeers ); registerChatParticipant(context); } diff --git a/extension/src/registry.ts b/extension/src/registry.ts index 1598c54..dbd2916 100644 --- a/extension/src/registry.ts +++ b/extension/src/registry.ts @@ -1,8 +1,11 @@ import * as vscode from 'vscode'; import { fetchFederatedRegistry, + fetchPeerStatus, formatAxiosError, + probePeer, type FederatedRegistryItem, + type PeerStatus, } from './api'; import { installFromResolvedUrl, @@ -20,10 +23,44 @@ function peerHostLabel(peer: string | undefined): string | undefined { } } +function peerUrlOf(status: PeerStatus): string { + return (status.url || status.base_url || '').trim(); +} + +function formatPeerHealth(status: PeerStatus): string { + const host = peerHostLabel(peerUrlOf(status)) || peerUrlOf(status) || 'peer'; + if (status.ok) { + const latency = + typeof status.latency_ms === 'number' ? ` ${Math.round(status.latency_ms)}ms` : ''; + return `$(check) ${host}${latency}`; + } + const err = status.error ? `: ${status.error}` : ''; + return `$(error) ${host}${err}`; +} + +function parseRegistryPeersSetting(): string[] { + const raw = vscode.workspace.getConfiguration('creer').get('registryPeers') || ''; + return raw + .split(',') + .map((p) => p.trim().replace(/\/$/, '')) + .filter(Boolean); +} + +async function saveRegistryPeersSetting(peers: string[]): Promise { + const value = peers.join(', '); + await vscode.workspace + .getConfiguration('creer') + .update('registryPeers', value, vscode.ConfigurationTarget.Global); +} + /** * Creer: Browse Federated Registry — GET /registry/federated, QuickPick, install. */ export async function browseFederatedRegistryCommand(): Promise { + const config = vscode.workspace.getConfiguration('creer'); + const showPeerStatus = config.get('showPeerStatus') !== false; + const registryPeers = config.get('registryPeers') || ''; + const q = await vscode.window.showInputBox({ prompt: 'Search federated registry (leave empty for all packs)', placeHolder: 'fastapi, express, peer host…', @@ -33,6 +70,31 @@ export async function browseFederatedRegistryCommand(): Promise { return; } + let peerStatuses: PeerStatus[] | undefined; + if (showPeerStatus) { + try { + const statusResp = await fetchPeerStatus(); + peerStatuses = statusResp.peers; + const ok = peerStatuses.filter((p) => p.ok).length; + const fail = peerStatuses.length - ok; + if (peerStatuses.length > 0) { + void vscode.window.showInformationMessage( + `Creer peers: ${ok} ok, ${fail} fail` + + (peerStatuses.some((p) => typeof p.latency_ms === 'number') + ? ` · ${peerStatuses + .filter((p) => p.ok && typeof p.latency_ms === 'number') + .map((p) => `${peerHostLabel(peerUrlOf(p))}:${Math.round(p.latency_ms!)}ms`) + .slice(0, 3) + .join(', ')}` + : '') + ); + } + } catch { + // Soft-fail: peer status endpoints may be missing on older backends. + peerStatuses = undefined; + } + } + let items: FederatedRegistryItem[]; let peerCount = 0; try { @@ -42,7 +104,11 @@ export async function browseFederatedRegistryCommand(): Promise { title: 'Creer: loading federated registry…', cancellable: false, }, - () => fetchFederatedRegistry({ q: q.trim() || undefined }) + () => + fetchFederatedRegistry({ + q: q.trim() || undefined, + peers: registryPeers.trim() || undefined, + }) ); items = federated.items; peerCount = federated.peers?.length ?? 0; @@ -50,7 +116,7 @@ export async function browseFederatedRegistryCommand(): Promise { const message = formatAxiosError(err, 'Federated registry unavailable'); void vscode.window.showWarningMessage( `Creer: federated registry endpoint not available (${message}). ` + - 'Try “Creer: Browse Pack Registry” or configure CREER_REGISTRY_PEERS on the backend.' + 'Try “Creer: Browse Pack Registry” or configure creer.registryPeers / CREER_REGISTRY_PEERS.' ); return; } @@ -65,14 +131,43 @@ export async function browseFederatedRegistryCommand(): Promise { } type PickItem = vscode.QuickPickItem & { fed?: FederatedRegistryItem }; - const picks: PickItem[] = items.map((item) => { - const parts: string[] = []; - if (item.source) { - parts.push(item.source); + const picks: PickItem[] = []; + + if (showPeerStatus && peerStatuses && peerStatuses.length > 0) { + picks.push({ + label: 'Peer health', + kind: vscode.QuickPickItemKind.Separator, + }); + for (const status of peerStatuses) { + picks.push({ + label: formatPeerHealth(status), + description: peerUrlOf(status), + detail: status.ok + ? typeof status.count === 'number' + ? `${status.count} packs` + : 'reachable' + : status.error || 'unreachable', + }); } + picks.push({ + label: 'Packs', + kind: vscode.QuickPickItemKind.Separator, + }); + } + + const localItems = items.filter((i) => !i.peer); + const remoteItems = items.filter((i) => Boolean(i.peer)); + + const toPick = (item: FederatedRegistryItem): PickItem => { const host = peerHostLabel(item.peer); + const parts: string[] = []; if (host) { parts.push(host); + } else { + parts.push('local'); + } + if (item.source) { + parts.push(item.source); } if (item.stack) { parts.push(item.stack); @@ -81,15 +176,24 @@ export async function browseFederatedRegistryCommand(): Promise { parts.push(item.version); } return { - label: item.name || item.id, + label: host + ? `$(cloud) ${item.name || item.id}` + : `$(home) ${item.name || item.id}`, description: parts.join(' · '), detail: item.description, fed: item, }; - }); + }; + + for (const item of localItems) { + picks.push(toPick(item)); + } + for (const item of remoteItems) { + picks.push(toPick(item)); + } const picked = await vscode.window.showQuickPick(picks, { - placeHolder: 'Select a federated pack to install', + placeHolder: 'Select a federated pack to install (peer host shown in description)', ignoreFocusOut: true, matchOnDescription: true, matchOnDetail: true, @@ -116,3 +220,228 @@ export async function browseFederatedRegistryCommand(): Promise { item.install_url || item.download_url ); } + +/** + * Creer: Manage Registry Peers — view/add/remove setting peers; probe via backend. + */ +export async function manageRegistryPeersCommand(): Promise { + const settingPeers = parseRegistryPeersSetting(); + + let configured: string[] = []; + let livePeers: PeerStatus[] = []; + let peersEndpointOk = true; + try { + const status = await fetchPeerStatus(); + configured = status.configured ?? []; + livePeers = status.peers ?? []; + } catch { + peersEndpointOk = false; + } + + const lines: string[] = []; + lines.push( + settingPeers.length + ? `Setting peers (${settingPeers.length}): ${settingPeers.join(', ')}` + : 'Setting peers: (none)' + ); + if (peersEndpointOk) { + lines.push( + configured.length + ? `Backend configured (${configured.length}): ${configured.join(', ')}` + : 'Backend configured: (none)' + ); + if (livePeers.length > 0) { + lines.push( + 'Status: ' + + livePeers + .map((p) => { + const host = peerHostLabel(peerUrlOf(p)) || peerUrlOf(p); + return p.ok + ? `${host} ok${typeof p.latency_ms === 'number' ? ` ${Math.round(p.latency_ms)}ms` : ''}` + : `${host} fail`; + }) + .join('; ') + ); + } + } else { + lines.push('Backend peer endpoints unavailable (soft-fail). You can still edit creer.registryPeers.'); + } + + type Action = 'add' | 'remove' | 'probe' | 'done'; + type ActionPick = vscode.QuickPickItem & { action?: Action }; + + const actions: ActionPick[] = [ + { + label: '$(info) Current peers', + description: 'Summary', + detail: lines.join('\n'), + }, + { + label: '$(add) Add peer', + description: 'Probe URL then append to creer.registryPeers', + action: 'add', + }, + { + label: '$(remove) Remove peer', + description: 'Remove from creer.registryPeers', + action: 'remove', + }, + { + label: '$(debug-alt) Probe all', + description: peersEndpointOk + ? 'POST /registry/peers/probe for each setting peer' + : 'Requires backend probe endpoint', + action: 'probe', + }, + ]; + + const picked = await vscode.window.showQuickPick(actions, { + placeHolder: 'Creer: Manage Registry Peers', + ignoreFocusOut: true, + matchOnDescription: true, + matchOnDetail: true, + }); + + if (!picked?.action) { + if (picked && !picked.action) { + void vscode.window.showInformationMessage(`Creer:\n${lines.join('\n')}`); + } + return; + } + + if (picked.action === 'add') { + const url = await vscode.window.showInputBox({ + prompt: 'Peer base URL (e.g. https://creer.example.com)', + placeHolder: 'http://localhost:8001', + ignoreFocusOut: true, + }); + const trimmed = url?.trim().replace(/\/$/, ''); + if (!trimmed) { + return; + } + + let ok = true; + let probeError: string | undefined; + try { + const result = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: `Creer: probing ${trimmed}…`, + cancellable: false, + }, + () => probePeer(trimmed) + ); + ok = result.ok; + probeError = result.error || undefined; + if (ok) { + const latency = + typeof result.latency_ms === 'number' + ? ` (${Math.round(result.latency_ms)}ms)` + : ''; + void vscode.window.showInformationMessage( + `Creer: peer reachable${latency}.` + ); + } + } catch (err) { + // Soft-fail: if probe endpoint missing, still allow adding after confirm. + const message = formatAxiosError(err, 'Probe failed'); + const choice = await vscode.window.showWarningMessage( + `Creer: could not probe peer (${message}). Add anyway?`, + 'Add', + 'Cancel' + ); + if (choice !== 'Add') { + return; + } + ok = true; + } + + if (!ok) { + void vscode.window.showErrorMessage( + `Creer: peer probe failed${probeError ? ` — ${probeError}` : ''}. Not added.` + ); + return; + } + + const next = parseRegistryPeersSetting(); + if (!next.includes(trimmed)) { + next.push(trimmed); + await saveRegistryPeersSetting(next); + } + void vscode.window.showInformationMessage( + `Creer: added peer ${trimmed} to creer.registryPeers.` + ); + return; + } + + if (picked.action === 'remove') { + const current = parseRegistryPeersSetting(); + if (current.length === 0) { + void vscode.window.showInformationMessage('Creer: no setting peers to remove.'); + return; + } + const toRemove = await vscode.window.showQuickPick( + current.map((url) => ({ label: url, description: peerHostLabel(url) })), + { + placeHolder: 'Select peer to remove from creer.registryPeers', + ignoreFocusOut: true, + canPickMany: true, + } + ); + if (!toRemove || toRemove.length === 0) { + return; + } + const removeSet = new Set(toRemove.map((p) => p.label)); + const remaining = current.filter((u) => !removeSet.has(u)); + await saveRegistryPeersSetting(remaining); + void vscode.window.showInformationMessage( + `Creer: removed ${toRemove.length} peer(s) from creer.registryPeers.` + ); + return; + } + + if (picked.action === 'probe') { + const current = parseRegistryPeersSetting(); + const targets = + current.length > 0 + ? current + : configured.length > 0 + ? configured + : []; + if (targets.length === 0) { + void vscode.window.showInformationMessage( + 'Creer: no peers to probe. Add peers via this command or CREER_REGISTRY_PEERS.' + ); + return; + } + + const results: string[] = []; + await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'Creer: probing peers…', + cancellable: false, + }, + async (progress) => { + for (let i = 0; i < targets.length; i++) { + const url = targets[i]; + progress.report({ + message: `${i + 1}/${targets.length} ${peerHostLabel(url) || url}`, + }); + try { + const status = await probePeer(url); + results.push(formatPeerHealth(status).replace(/\$\([^)]+\)\s*/g, '')); + } catch (err) { + results.push( + `${peerHostLabel(url) || url}: ${formatAxiosError(err, 'probe failed')}` + ); + } + } + } + ); + + void vscode.window.showInformationMessage( + `Creer probe results:\n${results.join('\n')}` + ); + } +} From 2c6c6d4198f0c1022d79d000f2c2d9f12caaff95 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 09:04:02 +0000 Subject: [PATCH 12/13] Release Creer v1.0.0: registry auth and one-hop peer discovery Add optional CREER_REGISTRY_TOKEN for pack/peer write endpoints, gossip-lite /registry/discover with federated discover=true expansion, and extension SecretStorage registry token plus Discover peers UX. Core generate stays public. Co-authored-by: Sanath S Patil --- PLAN.md | 5 +- README.md | 29 ++- RELEASE.md | 20 +- backend/.env.example | 6 + backend/app/auth.py | 53 ++++ backend/app/federation.py | 103 +++++++- backend/app/registry.py | 2 +- backend/config.py | 3 + backend/main.py | 47 +++- backend/tests/test_federation.py | 10 +- backend/tests/test_federation_ux.py | 14 +- backend/tests/test_marketplace.py | 4 +- backend/tests/test_registry.py | 2 +- backend/tests/test_v1_auth_discovery.py | 328 ++++++++++++++++++++++++ extension/CHANGELOG.md | 11 + extension/PUBLISH.md | 10 +- extension/package-lock.json | 4 +- extension/package.json | 21 +- extension/src/api.ts | 106 +++++++- extension/src/extension.ts | 50 +++- extension/src/marketplace.ts | 36 ++- extension/src/registry.ts | 119 ++++++++- extension/src/secrets.ts | 42 +++ 23 files changed, 936 insertions(+), 89 deletions(-) create mode 100644 backend/app/auth.py create mode 100644 backend/tests/test_v1_auth_discovery.py diff --git a/PLAN.md b/PLAN.md index db6c97a..7c47375 100644 --- a/PLAN.md +++ b/PLAN.md @@ -11,11 +11,12 @@ - **v0.7** — Self-hosted pack registry (`/registry` + download), extension icon, Browse Pack Registry, release changelog - **v0.8** — Federated registry (`/registry/federated` + peers), Browse Federated Registry UI, GitHub Actions release + CI (artifact-first; signed publish when secrets exist) - **v0.9** — Peer status/probe UX (`creer.registryPeers`, Manage Registry Peers), federated browse enrichment, GitHub Release on tag + `RELEASE.md` +- **v1.0** — Stable foundation: optional registry write auth (Bearer / `X-Creer-Token`), `GET /registry/discover`, federated `discover=true`, SecretStorage registry token + Discover peers UX ## Optional next -- Human: configure `VSCE_PAT` / `OVSX_PAT` repository secrets; tag `v0.9.0` (see [`RELEASE.md`](RELEASE.md)) -- More peer discovery / registry auth +- Human: configure `VSCE_PAT` / `OVSX_PAT` repository secrets; tag `v1.0.0` (see [`RELEASE.md`](RELEASE.md)) — agents cannot set GitHub Actions secrets +- Hardened multi-hop discovery policies / signed peer trust ## Non-goals diff --git a/README.md b/README.md index 0237c04..cc0a5d4 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ AI-powered repo scaffolding inside your workspace. -**Current version: 0.9.0** +**Current version: 1.0.0** (stable foundation) ## Architecture @@ -26,23 +26,28 @@ cd extension && npm install && npm run compile # F5 → Creer: Create New Repo ``` -## Registry & federation (v0.7–v0.9) +## Registry & federation (v0.7–v1.0) -Self-hosted pack catalog plus optional multi-host federation: +Self-hosted pack catalog plus optional multi-host federation and write auth: | Method | Path | Description | |---|---|---| | `GET` | `/registry?q=&source=` | Searchable pack list | -| `GET` | `/registry/federated?q=&source=&peers=` | Local + peer merge (extra peers CSV) | +| `GET` | `/registry/federated?q=&source=&peers=&discover=` | Local + peer merge; `discover=true` expands one hop | +| `GET` | `/registry/discover` | One-hop peer discovery | | `GET` | `/registry/peers` | Peer health + configured URLs | -| `POST` | `/registry/peers/probe` | Probe one peer `{ url }` | +| `POST` | `/registry/peers/probe` | Probe one peer `{ url }` (auth when configured) | | `GET` | `/registry/packs/{id}` | Pack metadata | | `GET` | `/registry/packs/{id}/download` | Portable JSON pack (installable URL) | | `GET` | `/marketplace` | Curated featured view | +| `POST` | `/packs/install` | Install pack from URL (auth when configured) | +| `DELETE` | `/packs/{id}` | Delete installed pack (auth when configured) | -Extension settings: `creer.registryPeers` (comma-separated peer base URLs), `creer.showPeerStatus` (peer health in federated browse). +When the backend sets `CREER_REGISTRY_TOKEN`, mutating routes expect `Authorization: Bearer ` and/or `X-Creer-Token`. -Commands: **Creer: Browse Federated Registry**, **Creer: Manage Registry Peers**. +Extension settings: `creer.registryPeers`, `creer.showPeerStatus`, `creer.federatedDiscover` (pass `discover=true`), `creer.registryToken` (deprecated plaintext — prefer SecretStorage). + +Commands: **Browse Federated Registry**, **Manage Registry Peers** (includes Discover peers), **Set / Clear Registry Token**. Install from another Creer host: @@ -53,7 +58,8 @@ curl -X POST http://localhost:8000/packs/install \ ``` Set `CREER_PUBLIC_BASE_URL` for absolute download links in registry responses. -Set `CREER_REGISTRY_PEERS` (comma-separated base URLs) for backend-configured federated discovery. +Set `CREER_REGISTRY_PEERS` for backend-configured federated discovery. +Set `CREER_REGISTRY_TOKEN` to require write auth on install/delete/probe. Use `creer.registryPeers` in the extension for client-side extra peers when browsing. ## Extension commands @@ -67,7 +73,8 @@ Use `creer.registryPeers` in the extension for client-side extra peers when brow | `creer.browseFederatedRegistry` | Browse Federated Registry | | `creer.manageRegistryPeers` | Manage Registry Peers | | `creer.installPackFromUrl` | Install Pack from URL | -| `creer.setGitHubToken` / `clearGitHubToken` | SecretStorage token | +| `creer.setGitHubToken` / `clearGitHubToken` | GitHub SecretStorage token | +| `creer.setRegistryToken` / `clearRegistryToken` | Registry write SecretStorage token | ## CI & publishing @@ -78,10 +85,10 @@ See [`RELEASE.md`](RELEASE.md) and [`extension/PUBLISH.md`](extension/PUBLISH.md ```bash cd extension && npm run compile && npm run package -# → creer-0.9.0.vsix (includes media/icon.png) +# → creer-1.0.0.vsix (includes media/icon.png) ``` -Signed Marketplace / Open VSX publish requires your own `VSCE_PAT` / `OVSX_PAT` (never commit tokens). +Signed Marketplace / Open VSX publish requires your own `VSCE_PAT` / `OVSX_PAT` (never commit tokens). Agents cannot set GitHub Actions secrets — that remains a human step. ## License diff --git a/RELEASE.md b/RELEASE.md index fce98f3..147f050 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,4 +1,4 @@ -# Releasing Creer v0.9.0 +# Releasing Creer v1.0.0 Exact steps for a human maintainer to cut a tagged release with GitHub Release + optional Marketplace / Open VSX publish. @@ -15,28 +15,30 @@ Both are optional. If neither is set, the release workflow still builds the `.vs Never commit PATs. Prefer repo secrets over exporting tokens in shared shells. +**Note:** Cloud agents cannot configure GitHub Actions secrets — a human must set `VSCE_PAT` / `OVSX_PAT` before signed Marketplace / Open VSX publish. + ## 2. Bump & verify locally ```bash -# Confirm extension version is 0.9.0 +# Confirm extension version is 1.0.0 grep '"version"' extension/package.json cd extension npm ci npm run compile npm run package -# → creer-0.9.0.vsix +# → creer-1.0.0.vsix ``` -Smoke-test: `code --install-extension creer-0.9.0.vsix` (or Cursor equivalent) against a running backend. +Smoke-test: `code --install-extension creer-1.0.0.vsix` (or Cursor equivalent) against a running backend. -## 3. Tag v0.9.0 and push +## 3. Tag v1.0.0 and push From a clean `main` (or the release commit): ```bash -git tag -a v0.9.0 -m "Creer v0.9.0" -git push origin v0.9.0 +git tag -a v1.0.0 -m "Creer v1.0.0" +git push origin v1.0.0 ``` Tag pattern `v*` triggers [`.github/workflows/release.yml`](.github/workflows/release.yml). @@ -51,6 +53,6 @@ You can also run the workflow via **Actions → Release → Run workflow** (`wor ## 5. After release -- Confirm the GitHub Release page lists `creer-0.9.0.vsix` -- If secrets were set, confirm Marketplace / Open VSX listing updated to 0.9.0 +- Confirm the GitHub Release page lists `creer-1.0.0.vsix` +- If secrets were set, confirm Marketplace / Open VSX listing updated to 1.0.0 - See [`extension/PUBLISH.md`](extension/PUBLISH.md) for manual `vsce` / `ovsx` publish from a laptop diff --git a/backend/.env.example b/backend/.env.example index 999649a..9608235 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -25,3 +25,9 @@ CREER_OFFLINE= # Comma-separated peer Creer registry base URLs (federated discovery, v0.8) # Example: http://127.0.0.1:8001,https://creer-packs.example.com # CREER_REGISTRY_PEERS= + +# Optional registry write token (v1.0). When set (non-empty), mutating endpoints +# require Authorization: Bearer or X-Creer-Token: . +# Protects: POST /packs/install, DELETE /packs/{id}, POST /registry/peers/probe. +# Reads (GET /registry, /health, /plan, /generate) stay public. +# CREER_REGISTRY_TOKEN= diff --git a/backend/app/auth.py b/backend/app/auth.py new file mode 100644 index 0000000..138809f --- /dev/null +++ b/backend/app/auth.py @@ -0,0 +1,53 @@ +"""Optional registry write auth — Bearer or X-Creer-Token when CREER_REGISTRY_TOKEN is set.""" + +from __future__ import annotations + +from fastapi import Header, HTTPException + +from config import CREER_REGISTRY_TOKEN as _TOKEN_FROM_CONFIG + +# Module-level binding so tests can monkeypatch app.auth.CREER_REGISTRY_TOKEN +CREER_REGISTRY_TOKEN = _TOKEN_FROM_CONFIG + + +def registry_auth_required() -> bool: + """True when mutating registry/pack endpoints require a token.""" + return bool(CREER_REGISTRY_TOKEN) + + +def _extract_token( + authorization: str | None, + x_creer_token: str | None, +) -> str | None: + if x_creer_token and x_creer_token.strip(): + return x_creer_token.strip() + if authorization: + parts = authorization.split(" ", 1) + if len(parts) == 2 and parts[0].lower() == "bearer": + token = parts[1].strip() + return token or None + stripped = authorization.strip() + return stripped or None + return None + + +def require_registry_write( + authorization: str | None = Header(default=None), + x_creer_token: str | None = Header(default=None, alias="X-Creer-Token"), +) -> None: + """ + Gate mutating registry/pack endpoints. + + If CREER_REGISTRY_TOKEN is unset/empty → allow. + Otherwise require Authorization: Bearer or X-Creer-Token. + """ + expected = CREER_REGISTRY_TOKEN + if not expected: + return + + provided = _extract_token(authorization, x_creer_token) + if provided != expected: + raise HTTPException( + status_code=401, + detail="Registry write authentication required", + ) diff --git a/backend/app/federation.py b/backend/app/federation.py index 2ae88ef..c9a1c34 100644 --- a/backend/app/federation.py +++ b/backend/app/federation.py @@ -9,11 +9,13 @@ import httpx -from config import CREER_REGISTRY_PEERS -from app.registry import list_registry +from config import CREER_PUBLIC_BASE_URL, CREER_REGISTRY_PEERS +from app.auth import registry_auth_required +from app.registry import list_registry, registry_count -FEDERATION_VERSION = "0.9.0" +FEDERATION_VERSION = "1.0.0" _MAX_PEERS = 8 +_DISCOVER_TIMEOUT = 3.0 def parse_peers(raw: str | None = None) -> list[str]: @@ -71,6 +73,17 @@ def resolve_peers(extra_peers: list[str] | None = None) -> list[str]: return out +def discover_self() -> dict[str, Any]: + """Local gossip-lite discovery payload (configured peers only).""" + return { + "version": FEDERATION_VERSION, + "base_url": CREER_PUBLIC_BASE_URL or None, + "packs_count": registry_count(), + "peers": parse_peers(), + "auth_required": registry_auth_required(), + } + + def _absolutize_url(base_url: str, value: Any) -> Any: if not isinstance(value, str) or not value: return value @@ -152,6 +165,82 @@ def _fetch_peer_registry( return _tag_peer_items(base, raw_items), None +def fetch_peer_discover( + base_url: str, + *, + timeout: float = _DISCOVER_TIMEOUT, +) -> tuple[dict[str, Any] | None, str | None]: + """ + GET {base}/registry/discover. + + Returns (payload, None) on success or (None, error) on failure. Never raises. + """ + base = (base_url or "").strip().rstrip("/") + if not base: + return None, "empty base_url" + + url = f"{base}/registry/discover" + try: + with httpx.Client(timeout=timeout, follow_redirects=True) as client: + resp = client.get(url) + resp.raise_for_status() + data = resp.json() + except Exception as exc: # noqa: BLE001 + return None, str(exc) + + if not isinstance(data, dict): + return None, "unexpected discover response shape" + return data, None + + +def expand_peers_one_hop( + peers: list[str], + *, + timeout: float = _DISCOVER_TIMEOUT, +) -> tuple[list[str], list[str]]: + """ + One-hop gossip-lite: for each ok peer, fetch /registry/discover and collect + advertised peers not already in the list. Cap total at _MAX_PEERS. + + Returns (expanded_peers, discovered_peers_added_this_hop). + """ + if not peers: + return [], [] + + known: set[str] = set(peers) + discovered: list[str] = [] + by_peer: dict[str, dict[str, Any] | None] = {} + + workers = min(8, len(peers)) + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = {pool.submit(fetch_peer_discover, p, timeout=timeout): p for p in peers} + for fut in as_completed(futures): + peer = futures[fut] + data, err = fut.result() + by_peer[peer] = data if err is None else None + + # Preserve seed peer order when reading advertised lists + for peer in peers: + data = by_peer.get(peer) + if not data: + continue + advertised = data.get("peers") or [] + if not isinstance(advertised, list): + continue + for raw in advertised: + if not isinstance(raw, str): + continue + for cand in parse_peers(raw): + if cand in known: + continue + if len(peers) + len(discovered) >= _MAX_PEERS: + return peers + discovered, discovered + known.add(cand) + discovered.append(cand) + + return peers + discovered, discovered + + def probe_peer(base_url: str, timeout: float = 5.0) -> dict[str, Any]: """ Probe a peer host via /health (preferred) and/or /registry for pack count. @@ -264,6 +353,7 @@ def list_federated( source: str | None = None, include_local: bool = True, extra_peers: list[str] | None = None, + discover: bool = False, ) -> dict[str, Any]: """Merge local registry with peer registries (local ids win on collision).""" local = list_registry(q=q, source=source or "all") if include_local else { @@ -273,6 +363,10 @@ def list_federated( } peers = resolve_peers(extra_peers) + discovered_peers: list[str] = [] + if discover and peers: + peers, discovered_peers = expand_peers_one_hop(peers) + peer_meta: list[dict[str, Any]] = [] peer_items_by_url: dict[str, list[dict[str, Any]]] = {} @@ -295,7 +389,7 @@ def _one(peer: str) -> tuple[str, list[dict[str, Any]], str | None]: "error": err, } ) - # Stable peer order matching resolve_peers() + # Stable peer order matching resolve_peers() / expand order order = {p: i for i, p in enumerate(peers)} peer_meta.sort(key=lambda m: order.get(m["base_url"], 0)) @@ -322,4 +416,5 @@ def _one(peer: str) -> tuple[str, list[dict[str, Any]], str | None]: "local": local, "peers": peer_meta, "items": merged, + "discovered_peers": discovered_peers, } diff --git a/backend/app/registry.py b/backend/app/registry.py index a09901b..1ad793a 100644 --- a/backend/app/registry.py +++ b/backend/app/registry.py @@ -102,7 +102,7 @@ def matches(item: dict[str, Any]) -> bool: items = [i for i in items if matches(i)] return { - "version": "0.9.0", + "version": "1.0.0", "base_url": CREER_PUBLIC_BASE_URL or None, "items": items, } diff --git a/backend/config.py b/backend/config.py index 0e5fc9f..d92feac 100644 --- a/backend/config.py +++ b/backend/config.py @@ -13,3 +13,6 @@ CREER_PUBLIC_BASE_URL = os.getenv("CREER_PUBLIC_BASE_URL", "").strip() or None # Comma-separated peer Creer registry base URLs for federation (v0.8+) CREER_REGISTRY_PEERS = os.getenv("CREER_REGISTRY_PEERS", "").strip() or None +# Optional token for mutating registry/pack write endpoints (v1.0+) +# When set, POST /packs/install, DELETE /packs/{id}, POST /registry/peers/probe require auth +CREER_REGISTRY_TOKEN = os.getenv("CREER_REGISTRY_TOKEN", "").strip() or None diff --git a/backend/main.py b/backend/main.py index 54609c7..d682871 100644 --- a/backend/main.py +++ b/backend/main.py @@ -7,11 +7,12 @@ from typing import Any, Literal from urllib.parse import urlparse -from fastapi import FastAPI, Header, HTTPException, Query +from fastapi import Depends, FastAPI, Header, HTTPException, Query from fastapi.responses import Response, StreamingResponse from pydantic import BaseModel, Field from config import CREER_OFFLINE, CREER_PUBLIC_BASE_URL, MODEL, OPENAI_BASE_URL +from app.auth import registry_auth_required, require_registry_write from app.bakeins import apply_bakeins, list_bakein_options from app.planner import plan_project from app.generator import generate_files, generate_files_iter @@ -32,12 +33,18 @@ pack_download_bytes, registry_count, ) -from app.federation import list_federated, list_peer_status, parse_peers, probe_peer +from app.federation import ( + discover_self, + list_federated, + list_peer_status, + parse_peers, + probe_peer, +) from app.github import create_github_repo from app.jobs import cancel_job, create_job, finish_job, is_cancelled from app.quality import has_errors, run_quality_gates -VERSION = "0.9.0" +VERSION = "1.0.0" app = FastAPI(title="Creer", version=VERSION) @@ -161,6 +168,7 @@ def health(): "registry_count": registry_count(), "public_base_url_set": bool(CREER_PUBLIC_BASE_URL), "peers_configured": len(parse_peers()), + "auth_required": registry_auth_required(), } @@ -191,14 +199,28 @@ def registry_federated( default=None, description="Comma-separated extra peer base URLs for this request only", ), + discover: bool = Query( + default=False, + description="One-hop peer discovery via GET {peer}/registry/discover", + ), ): """Federated registry: local packs plus peer Creer registries.""" extra = parse_peers(peers) if peers else None return list_federated( - q=q, source=source, include_local=True, extra_peers=extra + q=q, + source=source, + include_local=True, + extra_peers=extra, + discover=discover, ) +@app.get("/registry/discover") +def registry_discover(): + """Gossip-lite self advertisement: version, packs, configured peers, auth flag.""" + return discover_self() + + @app.get("/registry/peers") def registry_peers(): """Configured peer list plus live probe status for each peer.""" @@ -206,8 +228,11 @@ def registry_peers(): @app.post("/registry/peers/probe") -def registry_peers_probe(request: PeerProbeRequest): - """Ad-hoc probe of a single peer base URL.""" +def registry_peers_probe( + request: PeerProbeRequest, + _: None = Depends(require_registry_write), +): + """Ad-hoc probe of a single peer base URL (auth required when token configured).""" url = (request.url or "").strip().rstrip("/") parsed = urlparse(url) if parsed.scheme not in ("http", "https") or not parsed.netloc: @@ -247,7 +272,10 @@ def marketplace(): @app.post("/packs/install") -def packs_install(request: PackInstallRequest): +def packs_install( + request: PackInstallRequest, + _: None = Depends(require_registry_write), +): """Fetch a remote pack URL (http/https), validate, and install locally.""" try: pack = install_pack_from_url(request.url, overwrite=request.overwrite) @@ -271,7 +299,10 @@ def pack_detail(pack_id: str): @app.delete("/packs/{pack_id}") -def packs_delete(pack_id: str): +def packs_delete( + pack_id: str, + _: None = Depends(require_registry_write), +): """Delete a pack from the writable installed dir only (not shipped examples).""" try: uninstall_pack(pack_id) diff --git a/backend/tests/test_federation.py b/backend/tests/test_federation.py index 171c111..35f5e8c 100644 --- a/backend/tests/test_federation.py +++ b/backend/tests/test_federation.py @@ -64,7 +64,7 @@ def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): monkeypatch.setattr(fed, "_fetch_peer_registry", fake_fetch) result = list_federated() - assert result["version"] == "0.9.0" + assert result["version"] == "1.0.0" assert "items" in result["local"] assert len(result["peers"]) == 1 assert result["peers"][0]["ok"] is True @@ -95,7 +95,7 @@ def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): monkeypatch.setattr(fed, "_fetch_peer_registry", fake_fetch) result = list_federated() - assert result["version"] == "0.9.0" + assert result["version"] == "1.0.0" local_count = len(result["local"]["items"]) assert local_count >= 3 assert len(result["items"]) == local_count @@ -134,8 +134,8 @@ def test_health_0_9_and_federated_route(monkeypatch): # Also patch config import used if parse_peers reads module-level — already patched fed c = TestClient(main.app) h = c.get("/health").json() - assert h["version"] == "0.9.0" - assert VERSION == "0.9.0" + assert h["version"] == "1.0.0" + assert VERSION == "1.0.0" assert h["peers_configured"] == 2 # No live peers — empty mock via monkeypatch on fetch @@ -147,7 +147,7 @@ def test_health_0_9_and_federated_route(monkeypatch): r = c.get("/registry/federated") assert r.status_code == 200 body = r.json() - assert body["version"] == "0.9.0" + assert body["version"] == "1.0.0" assert "local" in body assert len(body["items"]) == len(body["local"]["items"]) diff --git a/backend/tests/test_federation_ux.py b/backend/tests/test_federation_ux.py index d7e66f6..18eb61e 100644 --- a/backend/tests/test_federation_ux.py +++ b/backend/tests/test_federation_ux.py @@ -168,7 +168,7 @@ def fake_probe(base_url, timeout=5.0): "ok": base_url.endswith("a.example"), "latency_ms": 1.0, "count": 1 if base_url.endswith("a.example") else None, - "version": "0.9.0" if base_url.endswith("a.example") else None, + "version": "1.0.0" if base_url.endswith("a.example") else None, "error": None if base_url.endswith("a.example") else "down", } @@ -195,7 +195,7 @@ def test_registry_peers_route(monkeypatch): "ok": True, "latency_ms": 2.5, "count": 4, - "version": "0.9.0", + "version": "1.0.0", "error": None, }, ) @@ -230,7 +230,7 @@ def test_registry_peers_probe_ok(monkeypatch): "ok": True, "latency_ms": 10.0, "count": 2, - "version": "0.9.0", + "version": "1.0.0", "error": None, }, ) @@ -262,7 +262,7 @@ def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): r = c.get("/registry/federated", params={"peers": "http://adhoc.peer:8002/"}) assert r.status_code == 200 body = r.json() - assert body["version"] == "0.9.0" + assert body["version"] == "1.0.0" assert any(p["base_url"] == "http://adhoc.peer:8002" for p in body["peers"]) assert any(i["id"] == "adhoc-pack" for i in body["items"]) @@ -278,7 +278,7 @@ def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): monkeypatch.setattr(fed, "_fetch_peer_registry", fake_fetch) result = list_federated(extra_peers=["http://extra.peer/", "http://cfg.peer"]) - assert result["version"] == "0.9.0" + assert result["version"] == "1.0.0" assert seen == ["http://cfg.peer", "http://extra.peer"] @@ -286,7 +286,7 @@ def test_health_0_9(monkeypatch): monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "https://a.example,https://b.example") c = TestClient(main.app) h = c.get("/health").json() - assert h["version"] == "0.9.0" - assert VERSION == "0.9.0" + assert h["version"] == "1.0.0" + assert VERSION == "1.0.0" assert h["peers_configured"] == 2 assert h["status"] == "ok" diff --git a/backend/tests/test_marketplace.py b/backend/tests/test_marketplace.py index 5a70869..e9ed0a7 100644 --- a/backend/tests/test_marketplace.py +++ b/backend/tests/test_marketplace.py @@ -48,8 +48,8 @@ def test_health_version_0_9(client, monkeypatch): resp = client.get("/health") assert resp.status_code == 200 data = resp.json() - assert data["version"] == "0.9.0" - assert VERSION == "0.9.0" + assert data["version"] == "1.0.0" + assert VERSION == "1.0.0" assert data["offline"] is True assert data["packs_count"] >= 3 assert "peers_configured" in data diff --git a/backend/tests/test_registry.py b/backend/tests/test_registry.py index 9b3cf64..a140578 100644 --- a/backend/tests/test_registry.py +++ b/backend/tests/test_registry.py @@ -13,7 +13,7 @@ def test_health_registry_fields(): c = TestClient(main.app) h = c.get("/health").json() - assert h["version"] == "0.9.0" + assert h["version"] == "1.0.0" assert h["registry_count"] >= 3 assert "peers_configured" in h diff --git a/backend/tests/test_v1_auth_discovery.py b/backend/tests/test_v1_auth_discovery.py new file mode 100644 index 0000000..edb68eb --- /dev/null +++ b/backend/tests/test_v1_auth_discovery.py @@ -0,0 +1,328 @@ +"""Tests for optional registry write auth + one-hop discovery (v1.0).""" + +from __future__ import annotations + +import json + +from fastapi.testclient import TestClient + +import main +from app import auth as auth_mod +from app import federation as fed +from app import packs as packs_mod +from app.federation import expand_peers_one_hop, list_federated +from main import VERSION + + +SAMPLE_PACK = { + "id": "auth-demo-pack", + "name": "Auth Demo", + "description": "Installed via auth tests", + "stack": "demo", + "version": "1.0.0", + "files": ["README.md"], +} + + +def test_install_without_token_still_works(tmp_path, monkeypatch): + """When CREER_REGISTRY_TOKEN is unset, install works as before.""" + target = tmp_path / "installed" + target.mkdir() + monkeypatch.setenv("CREER_PACKS_DIR", str(target)) + monkeypatch.setattr(auth_mod, "CREER_REGISTRY_TOKEN", None) + + body = json.dumps(SAMPLE_PACK).encode("utf-8") + + class FakeResponse: + status_code = 200 + headers = {"content-type": "application/json"} + + def iter_bytes(self): + yield body + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + class FakeClient: + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def stream(self, method, url): + return FakeResponse() + + monkeypatch.setattr(packs_mod.httpx, "Client", FakeClient) + c = TestClient(main.app) + resp = c.post( + "/packs/install", + json={"url": "https://example.com/packs/auth-demo-pack.json", "overwrite": True}, + ) + assert resp.status_code == 200 + assert resp.json()["installed"] is True + assert resp.json()["pack"]["id"] == "auth-demo-pack" + + +def test_install_requires_auth_when_token_set(tmp_path, monkeypatch): + target = tmp_path / "installed" + target.mkdir() + monkeypatch.setenv("CREER_PACKS_DIR", str(target)) + monkeypatch.setattr(auth_mod, "CREER_REGISTRY_TOKEN", "secret-token") + + body = json.dumps(SAMPLE_PACK).encode("utf-8") + + class FakeResponse: + status_code = 200 + headers = {"content-type": "application/json"} + + def iter_bytes(self): + yield body + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + class FakeClient: + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def stream(self, method, url): + return FakeResponse() + + monkeypatch.setattr(packs_mod.httpx, "Client", FakeClient) + c = TestClient(main.app) + + denied = c.post( + "/packs/install", + json={"url": "https://example.com/packs/auth-demo-pack.json", "overwrite": True}, + ) + assert denied.status_code == 401 + + wrong = c.post( + "/packs/install", + json={"url": "https://example.com/packs/auth-demo-pack.json", "overwrite": True}, + headers={"Authorization": "Bearer wrong"}, + ) + assert wrong.status_code == 401 + + ok = c.post( + "/packs/install", + json={"url": "https://example.com/packs/auth-demo-pack.json", "overwrite": True}, + headers={"Authorization": "Bearer secret-token"}, + ) + assert ok.status_code == 200 + assert ok.json()["installed"] is True + + +def test_install_accepts_x_creer_token(tmp_path, monkeypatch): + target = tmp_path / "installed" + target.mkdir() + monkeypatch.setenv("CREER_PACKS_DIR", str(target)) + monkeypatch.setattr(auth_mod, "CREER_REGISTRY_TOKEN", "header-secret") + + body = json.dumps({**SAMPLE_PACK, "id": "x-token-pack"}).encode("utf-8") + + class FakeResponse: + status_code = 200 + headers = {"content-type": "application/json"} + + def iter_bytes(self): + yield body + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + class FakeClient: + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def stream(self, method, url): + return FakeResponse() + + monkeypatch.setattr(packs_mod.httpx, "Client", FakeClient) + c = TestClient(main.app) + ok = c.post( + "/packs/install", + json={"url": "https://example.com/packs/x-token-pack.json", "overwrite": True}, + headers={"X-Creer-Token": "header-secret"}, + ) + assert ok.status_code == 200 + + +def test_delete_and_probe_require_auth(monkeypatch): + monkeypatch.setattr(auth_mod, "CREER_REGISTRY_TOKEN", "probe-secret") + c = TestClient(main.app) + + assert c.delete("/packs/nope").status_code == 401 + assert c.post("/registry/peers/probe", json={"url": "http://peer.example"}).status_code == 401 + + # Reads stay public + assert c.get("/registry").status_code == 200 + assert c.get("/health").status_code == 200 + assert c.get("/health").json()["auth_required"] is True + + +def test_discover_endpoint_shape(monkeypatch): + monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "http://a.example,http://b.example") + monkeypatch.setattr(auth_mod, "CREER_REGISTRY_TOKEN", "tok") + monkeypatch.setattr(fed, "CREER_PUBLIC_BASE_URL", "http://me.example:8000") + c = TestClient(main.app) + r = c.get("/registry/discover") + assert r.status_code == 200 + body = r.json() + assert body["version"] == "1.0.0" + assert body["base_url"] == "http://me.example:8000" + assert isinstance(body["packs_count"], int) + assert body["packs_count"] >= 0 + assert body["peers"] == ["http://a.example", "http://b.example"] + assert body["auth_required"] is True + + +def test_discover_auth_required_false(monkeypatch): + monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "") + monkeypatch.setattr(auth_mod, "CREER_REGISTRY_TOKEN", None) + monkeypatch.setattr(fed, "CREER_PUBLIC_BASE_URL", None) + c = TestClient(main.app) + body = c.get("/registry/discover").json() + assert body["auth_required"] is False + assert body["base_url"] is None + assert body["peers"] == [] + + +def test_federated_discover_expands_peers(monkeypatch): + monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "http://seed.example") + + def fake_discover(base_url, *, timeout=3.0): + if base_url == "http://seed.example": + return { + "version": "1.0.0", + "peers": ["http://hop.example", "http://seed.example"], + "packs_count": 1, + "auth_required": False, + }, None + return None, "unreachable" + + def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): + if base_url == "http://seed.example": + return ( + [ + { + "id": "seed-pack", + "name": "Seed", + "peer": base_url, + "download_url": f"{base_url}/registry/packs/seed-pack/download", + } + ], + None, + ) + if base_url == "http://hop.example": + return ( + [ + { + "id": "hop-pack", + "name": "Hop", + "peer": base_url, + "download_url": f"{base_url}/registry/packs/hop-pack/download", + } + ], + None, + ) + return [], "unknown" + + monkeypatch.setattr(fed, "fetch_peer_discover", fake_discover) + monkeypatch.setattr(fed, "_fetch_peer_registry", fake_fetch) + + result = list_federated(discover=True) + assert result["version"] == "1.0.0" + assert result["discovered_peers"] == ["http://hop.example"] + peer_urls = [p["base_url"] for p in result["peers"]] + assert peer_urls == ["http://seed.example", "http://hop.example"] + ids = {i["id"] for i in result["items"]} + assert "seed-pack" in ids + assert "hop-pack" in ids + + +def test_federated_discover_false_no_expansion(monkeypatch): + monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "http://seed.example") + + def boom(*a, **k): + raise AssertionError("discover should not be called") + + monkeypatch.setattr(fed, "fetch_peer_discover", boom) + monkeypatch.setattr( + fed, + "_fetch_peer_registry", + lambda *a, **k: ([], None), + ) + result = list_federated(discover=False) + assert result["discovered_peers"] == [] + assert [p["base_url"] for p in result["peers"]] == ["http://seed.example"] + + +def test_federated_discover_query_param(monkeypatch): + monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "http://seed.example") + + def fake_discover(base_url, *, timeout=3.0): + return {"peers": ["http://extra.example"]}, None + + monkeypatch.setattr(fed, "fetch_peer_discover", fake_discover) + monkeypatch.setattr(fed, "_fetch_peer_registry", lambda *a, **k: ([], None)) + + c = TestClient(main.app) + r = c.get("/registry/federated", params={"discover": "true"}) + assert r.status_code == 200 + body = r.json() + assert body["discovered_peers"] == ["http://extra.example"] + assert any(p["base_url"] == "http://extra.example" for p in body["peers"]) + + +def test_expand_peers_respects_cap(monkeypatch): + seeds = [f"http://s{i}.example" for i in range(6)] + + def fake_discover(base_url, *, timeout=3.0): + # Each seed advertises many peers + return { + "peers": [f"http://n{i}.example" for i in range(10)], + }, None + + monkeypatch.setattr(fed, "fetch_peer_discover", fake_discover) + expanded, discovered = expand_peers_one_hop(seeds) + assert len(expanded) == 8 + assert len(discovered) == 2 + assert all(d.startswith("http://n") for d in discovered) + + +def test_health_1_0(monkeypatch): + monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "https://a.example") + monkeypatch.setattr(auth_mod, "CREER_REGISTRY_TOKEN", None) + c = TestClient(main.app) + h = c.get("/health").json() + assert h["version"] == "1.0.0" + assert VERSION == "1.0.0" + assert h["peers_configured"] == 1 + assert h["auth_required"] is False + assert h["status"] == "ok" diff --git a/extension/CHANGELOG.md b/extension/CHANGELOG.md index 6bd19ed..a97c727 100644 --- a/extension/CHANGELOG.md +++ b/extension/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 1.0.0 + +Stable foundation release: optional registry write auth, one-hop peer discovery, and federated discover browse. + +- Optional registry write auth: `creer.registryToken` (deprecated plaintext) + SecretStorage via **Creer: Set / Clear Registry Token** +- Mutating API calls (`installPack`, `deletePack`, `probePeer`) send `Authorization: Bearer` and `X-Creer-Token` when a token is available +- `GET /registry/discover` client + `fetchFederatedRegistry({ discover })` for one-hop peer expansion +- Setting `creer.federatedDiscover` (default false) passes `discover=true` on federated browse +- **Creer: Manage Registry Peers** — Discover peers (one hop); probe/install/delete honor registry token +- Human publish step unchanged: set `VSCE_PAT` / `OVSX_PAT` yourself (agents cannot configure GitHub Actions secrets) + ## 0.9.0 - Richer federation UX: `creer.registryPeers` + `creer.showPeerStatus` settings diff --git a/extension/PUBLISH.md b/extension/PUBLISH.md index 53a5072..cd648c4 100644 --- a/extension/PUBLISH.md +++ b/extension/PUBLISH.md @@ -19,7 +19,7 @@ npm run package # or: npx --yes @vscode/vsce package ``` -This runs `vsce package`, respects `.vscodeignore`, and includes production dependencies (e.g. `axios`) plus `media/icon.png`. Output: `creer-0.9.0.vsix` (version from `package.json`). +This runs `vsce package`, respects `.vscodeignore`, and includes production dependencies (e.g. `axios`) plus `media/icon.png`. Output: `creer-1.0.0.vsix` (version from `package.json`). ### Publisher signing (human step) @@ -37,7 +37,7 @@ Workflow: [`.github/workflows/release.yml`](../.github/workflows/release.yml) | Trigger | Behavior | |---|---| -| Tag `v*` (e.g. `v0.9.0`) | Build + package `.vsix`, upload artifact, create GitHub Release with `.vsix` attached | +| Tag `v*` (e.g. `v1.0.0`) | Build + package `.vsix`, upload artifact, create GitHub Release with `.vsix` attached | | `workflow_dispatch` | Build + package + artifact (no GitHub Release); publish only if secrets set | **Publish job** downloads the artifact and: @@ -51,8 +51,8 @@ Configure secrets under **Settings → Secrets and variables → Actions** (neve Install locally for a smoke test: ```bash -code --install-extension creer-0.9.0.vsix -# or Cursor: cursor --install-extension creer-0.9.0.vsix +code --install-extension creer-1.0.0.vsix +# or Cursor: cursor --install-extension creer-1.0.0.vsix ``` ## Publish to VS Marketplace @@ -89,7 +89,7 @@ Optional: `npx @vscode/vsce publish -p "$VSCE_PAT"`. cd extension npx --yes ovsx publish # with an existing vsix: - # npx --yes ovsx publish creer-0.9.0.vsix + # npx --yes ovsx publish creer-1.0.0.vsix ``` `npm run publish:ovsx` only documents this flow (exits non-zero so CI does not publish by accident). diff --git a/extension/package-lock.json b/extension/package-lock.json index af6747f..4d74cb5 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -1,12 +1,12 @@ { "name": "creer", - "version": "0.9.0", + "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "creer", - "version": "0.9.0", + "version": "1.0.0", "dependencies": { "axios": "^1.7.9" }, diff --git a/extension/package.json b/extension/package.json index eebcb33..97cd971 100644 --- a/extension/package.json +++ b/extension/package.json @@ -2,7 +2,7 @@ "name": "creer", "displayName": "Creer", "description": "AI-powered repo scaffolding inside your workspace", - "version": "0.9.0", + "version": "1.0.0", "publisher": "creer", "license": "MIT", "icon": "media/icon.png", @@ -41,6 +41,14 @@ "command": "creer.clearGitHubToken", "title": "Creer: Clear GitHub Token" }, + { + "command": "creer.setRegistryToken", + "title": "Creer: Set Registry Token" + }, + { + "command": "creer.clearRegistryToken", + "title": "Creer: Clear Registry Token" + }, { "command": "creer.installPackFromUrl", "title": "Creer: Install Pack from URL" @@ -151,6 +159,17 @@ "type": "boolean", "default": true, "description": "When browsing the federated registry, show peer health summary (ok/fail/latency)" + }, + "creer.federatedDiscover": { + "type": "boolean", + "default": false, + "description": "When browsing the federated registry, pass discover=true for one-hop peer expansion" + }, + "creer.registryToken": { + "type": "string", + "default": "", + "description": "DEPRECATED: Prefer SecretStorage via 'Creer: Set Registry Token'. Used only as a fallback when SecretStorage is empty. Sent as Bearer / X-Creer-Token on mutating registry requests when the backend has CREER_REGISTRY_TOKEN.", + "deprecationMessage": "Use 'Creer: Set Registry Token' (SecretStorage) instead of storing the token in settings." } } } diff --git a/extension/src/api.ts b/extension/src/api.ts index 7e62e8b..be6e2bb 100644 --- a/extension/src/api.ts +++ b/extension/src/api.ts @@ -142,23 +142,47 @@ export async function fetchMarketplace(): Promise { return response.data.items ?? []; } +/** + * Auth headers for mutating registry endpoints when a token is available. + * Backend accepts Authorization: Bearer and/or X-Creer-Token when CREER_REGISTRY_TOKEN is set. + */ +export function registryAuthHeaders(token?: string): Record { + const trimmed = token?.trim(); + if (!trimmed) { + return {}; + } + return { + Authorization: `Bearer ${trimmed}`, + 'X-Creer-Token': trimmed, + }; +} + +export interface PackInstallOptions { + overwrite?: boolean; + /** Optional registry write token (from SecretStorage / setting). */ + token?: string; +} + /** * POST /packs/install → { url, overwrite? } → installed pack * Backend returns `{ installed: true, pack: {...} }` (also tolerate a bare pack body). */ export async function installPack( url: string, - overwrite?: boolean + options?: PackInstallOptions ): Promise { const backendUrl = getBackendUrl(); const body: { url: string; overwrite?: boolean } = { url }; - if (overwrite !== undefined) { - body.overwrite = overwrite; + if (options?.overwrite !== undefined) { + body.overwrite = options.overwrite; } const response = await axios.post<{ installed?: boolean; pack?: Pack } & Pack>( `${backendUrl}/packs/install`, body, - { timeout: 120_000 } + { + timeout: 120_000, + headers: registryAuthHeaders(options?.token), + } ); const data = response.data; if (data?.pack && typeof data.pack === 'object') { @@ -235,14 +259,17 @@ export interface FederatedRegistryResponse { } /** - * GET /registry/federated?q=&source=&peers= — local + peer registry merge. + * GET /registry/federated?q=&source=&peers=&discover= — local + peer registry merge. * `peers` is a comma-separated list of extra peer base URLs (from settings or callers). + * When `discover` is true, the backend expands one hop of peer-of-peer URLs. */ export async function fetchFederatedRegistry(options?: { q?: string; source?: string; /** Extra peer base URLs (comma-separated string or array). */ peers?: string | string[]; + /** One-hop peer expansion (discover=true). */ + discover?: boolean; }): Promise { const backendUrl = getBackendUrl(); let peersParam: string | undefined; @@ -260,6 +287,7 @@ export async function fetchFederatedRegistry(options?: { q: options?.q || undefined, source: options?.source || undefined, peers: peersParam, + discover: options?.discover === true ? true : undefined, }, } ); @@ -271,6 +299,53 @@ export async function fetchFederatedRegistry(options?: { }; } +export interface RegistryDiscoverResponse { + /** Discovered peer base URLs (one hop). */ + peers: string[]; + /** Optional notes from the backend. */ + discovered?: string[]; +} + +function normalizePeerUrlList(raw: unknown): string[] { + if (!Array.isArray(raw)) { + return []; + } + const out: string[] = []; + const seen = new Set(); + for (const entry of raw) { + let url = ''; + if (typeof entry === 'string') { + url = entry.trim().replace(/\/$/, ''); + } else if (entry && typeof entry === 'object') { + const obj = entry as { url?: string; base_url?: string }; + url = (obj.url || obj.base_url || '').trim().replace(/\/$/, ''); + } + if (url && !seen.has(url)) { + seen.add(url); + out.push(url); + } + } + return out; +} + +/** + * GET /registry/discover — one-hop peer discovery from the local backend. + */ +export async function fetchRegistryDiscover(): Promise { + const backendUrl = getBackendUrl(); + const response = await axios.get>( + `${backendUrl}/registry/discover`, + { timeout: 60_000 } + ); + const data = response.data ?? {}; + const peers = normalizePeerUrlList(data.peers ?? data.discovered ?? data.urls); + const discovered = normalizePeerUrlList(data.discovered); + return { + peers: peers.length > 0 ? peers : discovered, + discovered: discovered.length > 0 ? discovered : undefined, + }; +} + /** Live peer probe / status from GET /registry/peers or POST /registry/peers/probe. */ export interface PeerStatus { /** Peer base URL (preferred). */ @@ -315,15 +390,26 @@ export async function fetchPeerStatus(): Promise { }; } +export interface RegistryMutateOptions { + /** Optional registry write token (from SecretStorage / setting). */ + token?: string; +} + /** * POST /registry/peers/probe `{ url }` → PeerStatus */ -export async function probePeer(url: string): Promise { +export async function probePeer( + url: string, + options?: RegistryMutateOptions +): Promise { const backendUrl = getBackendUrl(); const response = await axios.post( `${backendUrl}/registry/peers/probe`, { url }, - { timeout: 30_000 } + { + timeout: 30_000, + headers: registryAuthHeaders(options?.token), + } ); return normalizePeerStatus(response.data); } @@ -331,10 +417,14 @@ export async function probePeer(url: string): Promise { /** * DELETE /packs/{id} → delete installed pack */ -export async function deletePack(id: string): Promise { +export async function deletePack( + id: string, + options?: RegistryMutateOptions +): Promise { const backendUrl = getBackendUrl(); await axios.delete(`${backendUrl}/packs/${encodeURIComponent(id)}`, { timeout: 30_000, + headers: registryAuthHeaders(options?.token), }); } diff --git a/extension/src/extension.ts b/extension/src/extension.ts index 406d206..a3d58ca 100644 --- a/extension/src/extension.ts +++ b/extension/src/extension.ts @@ -7,7 +7,12 @@ import { } from './marketplace'; import { browseFederatedRegistryCommand, manageRegistryPeersCommand } from './registry'; import { runScaffoldFlow } from './scaffold'; -import { clearGitHubToken, setGitHubToken } from './secrets'; +import { + clearGitHubToken, + clearRegistryToken, + setGitHubToken, + setRegistryToken, +} from './secrets'; export function activate(context: vscode.ExtensionContext) { registerConflictDiffProvider(context); @@ -44,29 +49,60 @@ export function activate(context: vscode.ExtensionContext) { void vscode.window.showInformationMessage('Creer: GitHub token cleared from SecretStorage.'); }); + const setRegistryTok = vscode.commands.registerCommand( + 'creer.setRegistryToken', + async () => { + const token = await vscode.window.showInputBox({ + prompt: + 'Creer registry write token (matches CREER_REGISTRY_TOKEN) — stored in SecretStorage', + placeHolder: 'registry token', + password: true, + ignoreFocusOut: true, + }); + const trimmed = token?.trim(); + if (!trimmed) { + return; + } + await setRegistryToken(context, trimmed); + void vscode.window.showInformationMessage( + 'Creer: registry token saved to SecretStorage.' + ); + } + ); + + const clearRegistryTok = vscode.commands.registerCommand( + 'creer.clearRegistryToken', + async () => { + await clearRegistryToken(context); + void vscode.window.showInformationMessage( + 'Creer: registry token cleared from SecretStorage.' + ); + } + ); + const installPackFromUrl = vscode.commands.registerCommand( 'creer.installPackFromUrl', - () => installPackFromUrlCommand() + () => installPackFromUrlCommand(context) ); const browseMarketplace = vscode.commands.registerCommand( 'creer.browseMarketplace', - () => browseMarketplaceCommand() + () => browseMarketplaceCommand(context) ); const browseRegistry = vscode.commands.registerCommand( 'creer.browseRegistry', - () => browseRegistryCommand() + () => browseRegistryCommand(context) ); const browseFederatedRegistry = vscode.commands.registerCommand( 'creer.browseFederatedRegistry', - () => browseFederatedRegistryCommand() + () => browseFederatedRegistryCommand(context) ); const manageRegistryPeers = vscode.commands.registerCommand( 'creer.manageRegistryPeers', - () => manageRegistryPeersCommand() + () => manageRegistryPeersCommand(context) ); context.subscriptions.push( @@ -74,6 +110,8 @@ export function activate(context: vscode.ExtensionContext) { createRepoFromChat, setToken, clearToken, + setRegistryTok, + clearRegistryTok, installPackFromUrl, browseMarketplace, browseRegistry, diff --git a/extension/src/marketplace.ts b/extension/src/marketplace.ts index a9a2167..fe9419a 100644 --- a/extension/src/marketplace.ts +++ b/extension/src/marketplace.ts @@ -8,6 +8,7 @@ import { type MarketplaceItem, type RegistryItem, } from './api'; +import { resolveRegistryToken } from './secrets'; export function resolveInstallUrl(url: string | undefined): string | undefined { const trimmed = url?.trim(); @@ -32,7 +33,9 @@ export function resolveInstallUrl(url: string | undefined): string | undefined { /** * Creer: Install Pack from URL — prompt for URL, POST /packs/install. */ -export async function installPackFromUrlCommand(): Promise { +export async function installPackFromUrlCommand( + context: vscode.ExtensionContext +): Promise { const url = await vscode.window.showInputBox({ prompt: 'Pack URL (JSON/YAML pack definition or registry download URL)', placeHolder: 'http://localhost:8000/registry/packs/fastapi-crud/download', @@ -43,6 +46,7 @@ export async function installPackFromUrlCommand(): Promise { return; } + const token = await resolveRegistryToken(context); try { const pack = await vscode.window.withProgress( { @@ -50,7 +54,7 @@ export async function installPackFromUrlCommand(): Promise { title: 'Creer: installing pack…', cancellable: false, }, - () => installPack(trimmed) + () => installPack(trimmed, { token }) ); const name = pack.name || pack.id || 'pack'; void vscode.window.showInformationMessage( @@ -72,7 +76,8 @@ export function isBundledSource(source: string | undefined): boolean { export async function installFromResolvedUrl( label: string, - url: string | undefined + url: string | undefined, + context?: vscode.ExtensionContext ): Promise { const resolved = resolveInstallUrl(url); if (!resolved) { @@ -89,6 +94,7 @@ export async function installFromResolvedUrl( return; } + const token = context ? await resolveRegistryToken(context) : undefined; try { const pack = await vscode.window.withProgress( { @@ -96,7 +102,7 @@ export async function installFromResolvedUrl( title: `Creer: installing ${label}…`, cancellable: false, }, - () => installPack(resolved) + () => installPack(resolved, { token }) ); void vscode.window.showInformationMessage( `Creer: installed pack “${pack.name || pack.id || label}”.` @@ -110,7 +116,9 @@ export async function installFromResolvedUrl( /** * Creer: Browse Pack Marketplace — GET /marketplace, QuickPick, optional install. */ -export async function browseMarketplaceCommand(): Promise { +export async function browseMarketplaceCommand( + context: vscode.ExtensionContext +): Promise { let items: MarketplaceItem[]; try { items = await vscode.window.withProgress( @@ -175,14 +183,17 @@ export async function browseMarketplaceCommand(): Promise { await installFromResolvedUrl( item.name || item.id, - item.url || item.download_url + item.url || item.download_url, + context ); } /** * Creer: Browse Pack Registry — searchable self-hosted /registry catalog. */ -export async function browseRegistryCommand(): Promise { +export async function browseRegistryCommand( + context: vscode.ExtensionContext +): Promise { const q = await vscode.window.showInputBox({ prompt: 'Search registry (leave empty for all packs)', placeHolder: 'fastapi, express, cli…', @@ -249,12 +260,16 @@ export async function browseRegistryCommand(): Promise { await installFromResolvedUrl( item.name || item.id, - item.install_url || item.download_url + item.install_url || item.download_url, + context ); } /** Optional helper for hosts that expose delete UI later. */ -export async function deletePackCommand(packId?: string): Promise { +export async function deletePackCommand( + packId: string | undefined, + context: vscode.ExtensionContext +): Promise { let id = packId?.trim(); if (!id) { id = ( @@ -269,8 +284,9 @@ export async function deletePackCommand(packId?: string): Promise { return; } + const token = await resolveRegistryToken(context); try { - await deletePack(id); + await deletePack(id, { token }); void vscode.window.showInformationMessage(`Creer: deleted pack “${id}”.`); } catch (err) { const message = formatAxiosError(err, 'Failed to delete pack'); diff --git a/extension/src/registry.ts b/extension/src/registry.ts index dbd2916..4a91824 100644 --- a/extension/src/registry.ts +++ b/extension/src/registry.ts @@ -2,6 +2,7 @@ import * as vscode from 'vscode'; import { fetchFederatedRegistry, fetchPeerStatus, + fetchRegistryDiscover, formatAxiosError, probePeer, type FederatedRegistryItem, @@ -11,6 +12,7 @@ import { installFromResolvedUrl, isBundledSource, } from './marketplace'; +import { resolveRegistryToken } from './secrets'; function peerHostLabel(peer: string | undefined): string | undefined { if (!peer?.trim()) { @@ -56,10 +58,13 @@ async function saveRegistryPeersSetting(peers: string[]): Promise { /** * Creer: Browse Federated Registry — GET /registry/federated, QuickPick, install. */ -export async function browseFederatedRegistryCommand(): Promise { +export async function browseFederatedRegistryCommand( + context: vscode.ExtensionContext +): Promise { const config = vscode.workspace.getConfiguration('creer'); const showPeerStatus = config.get('showPeerStatus') !== false; const registryPeers = config.get('registryPeers') || ''; + const federatedDiscover = config.get('federatedDiscover') === true; const q = await vscode.window.showInputBox({ prompt: 'Search federated registry (leave empty for all packs)', @@ -101,13 +106,16 @@ export async function browseFederatedRegistryCommand(): Promise { const federated = await vscode.window.withProgress( { location: vscode.ProgressLocation.Notification, - title: 'Creer: loading federated registry…', + title: federatedDiscover + ? 'Creer: loading federated registry (discover)…' + : 'Creer: loading federated registry…', cancellable: false, }, () => fetchFederatedRegistry({ q: q.trim() || undefined, peers: registryPeers.trim() || undefined, + discover: federatedDiscover || undefined, }) ); items = federated.items; @@ -217,15 +225,45 @@ export async function browseFederatedRegistryCommand(): Promise { await installFromResolvedUrl( item.name || item.id, - item.install_url || item.download_url + item.install_url || item.download_url, + context ); } +/** + * Discover one-hop peers via GET /registry/discover, falling back to federated?discover=true. + */ +async function discoverPeerUrls(): Promise { + try { + const discovered = await fetchRegistryDiscover(); + if (discovered.peers.length > 0) { + return discovered.peers; + } + } catch { + // Soft-fail: try federated discover instead. + } + + const registryPeers = vscode.workspace + .getConfiguration('creer') + .get('registryPeers') || ''; + const federated = await fetchFederatedRegistry({ + peers: registryPeers.trim() || undefined, + discover: true, + }); + const fromPeers = (federated.peers ?? []) + .map((p) => (p.base_url || '').trim().replace(/\/$/, '')) + .filter(Boolean); + return [...new Set(fromPeers)]; +} + /** * Creer: Manage Registry Peers — view/add/remove setting peers; probe via backend. */ -export async function manageRegistryPeersCommand(): Promise { +export async function manageRegistryPeersCommand( + context: vscode.ExtensionContext +): Promise { const settingPeers = parseRegistryPeersSetting(); + const token = await resolveRegistryToken(context); let configured: string[] = []; let livePeers: PeerStatus[] = []; @@ -267,7 +305,7 @@ export async function manageRegistryPeersCommand(): Promise { lines.push('Backend peer endpoints unavailable (soft-fail). You can still edit creer.registryPeers.'); } - type Action = 'add' | 'remove' | 'probe' | 'done'; + type Action = 'add' | 'remove' | 'probe' | 'discover' | 'done'; type ActionPick = vscode.QuickPickItem & { action?: Action }; const actions: ActionPick[] = [ @@ -293,6 +331,11 @@ export async function manageRegistryPeersCommand(): Promise { : 'Requires backend probe endpoint', action: 'probe', }, + { + label: '$(search) Discover peers (one hop)', + description: 'GET /registry/discover or federated?discover=true — offer to add', + action: 'discover', + }, ]; const picked = await vscode.window.showQuickPick(actions, { @@ -329,7 +372,7 @@ export async function manageRegistryPeersCommand(): Promise { title: `Creer: probing ${trimmed}…`, cancellable: false, }, - () => probePeer(trimmed) + () => probePeer(trimmed, { token }) ); ok = result.ok; probeError = result.error || undefined; @@ -429,7 +472,7 @@ export async function manageRegistryPeersCommand(): Promise { message: `${i + 1}/${targets.length} ${peerHostLabel(url) || url}`, }); try { - const status = await probePeer(url); + const status = await probePeer(url, { token }); results.push(formatPeerHealth(status).replace(/\$\([^)]+\)\s*/g, '')); } catch (err) { results.push( @@ -443,5 +486,67 @@ export async function manageRegistryPeersCommand(): Promise { void vscode.window.showInformationMessage( `Creer probe results:\n${results.join('\n')}` ); + return; + } + + if (picked.action === 'discover') { + let discovered: string[] = []; + try { + discovered = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'Creer: discovering peers (one hop)…', + cancellable: false, + }, + () => discoverPeerUrls() + ); + } catch (err) { + const message = formatAxiosError(err, 'Discovery unavailable'); + void vscode.window.showWarningMessage( + `Creer: peer discovery not available (${message}).` + ); + return; + } + + if (discovered.length === 0) { + void vscode.window.showInformationMessage('Creer: no peers discovered.'); + return; + } + + const existing = new Set(parseRegistryPeersSetting()); + const picks = discovered.map((url) => ({ + label: url, + description: existing.has(url) + ? 'already in creer.registryPeers' + : peerHostLabel(url), + picked: !existing.has(url), + })); + + const selected = await vscode.window.showQuickPick(picks, { + placeHolder: 'Select discovered peers to add to creer.registryPeers', + ignoreFocusOut: true, + canPickMany: true, + }); + if (!selected || selected.length === 0) { + return; + } + + const next = parseRegistryPeersSetting(); + let added = 0; + for (const item of selected) { + const url = item.label.trim().replace(/\/$/, ''); + if (url && !next.includes(url)) { + next.push(url); + added += 1; + } + } + if (added > 0) { + await saveRegistryPeersSetting(next); + } + void vscode.window.showInformationMessage( + added > 0 + ? `Creer: added ${added} discovered peer(s) to creer.registryPeers.` + : 'Creer: selected peers were already in creer.registryPeers.' + ); } } diff --git a/extension/src/secrets.ts b/extension/src/secrets.ts index a1341fd..b8e112d 100644 --- a/extension/src/secrets.ts +++ b/extension/src/secrets.ts @@ -3,6 +3,9 @@ import * as vscode from 'vscode'; /** SecretStorage key for the GitHub personal access token. */ export const GITHUB_TOKEN_SECRET_KEY = 'creer.githubToken'; +/** SecretStorage key for the optional Creer registry write token. */ +export const REGISTRY_TOKEN_SECRET_KEY = 'creer.registryToken'; + /** * Read GitHub token: SecretStorage first, then deprecated config fallback. */ @@ -89,3 +92,42 @@ export async function resolveGitHubToken( } return promptAndStoreGitHubToken(context); } + +/** + * Read registry token: SecretStorage first, then deprecated config fallback. + * Auth is optional — only needed when the backend sets CREER_REGISTRY_TOKEN. + */ +export async function getRegistryToken( + context: vscode.ExtensionContext +): Promise { + const secret = (await context.secrets.get(REGISTRY_TOKEN_SECRET_KEY))?.trim(); + if (secret) { + return secret; + } + + const fromConfig = ( + vscode.workspace.getConfiguration('creer').get('registryToken') || '' + ).trim(); + return fromConfig || undefined; +} + +export async function setRegistryToken( + context: vscode.ExtensionContext, + token: string +): Promise { + await context.secrets.store(REGISTRY_TOKEN_SECRET_KEY, token.trim()); +} + +export async function clearRegistryToken(context: vscode.ExtensionContext): Promise { + await context.secrets.delete(REGISTRY_TOKEN_SECRET_KEY); +} + +/** + * Resolve registry token for mutating API calls (SecretStorage → deprecated setting). + * Does not prompt — registry auth is optional when the server has no CREER_REGISTRY_TOKEN. + */ +export async function resolveRegistryToken( + context: vscode.ExtensionContext +): Promise { + return getRegistryToken(context); +} From 6f393933747d952bc9cd5de690798f65a74edd6b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 09:10:12 +0000 Subject: [PATCH 13/13] Implement Creer v1.1: hardened multi-hop discovery policies Add SSRF/private-IP peer guards, allow/deny lists, hop budgets with cycle detection, max_hops on federated queries, and extension trust warnings for private peers plus policy-aware federation status. Co-authored-by: Sanath S Patil --- PLAN.md | 5 +- README.md | 29 ++- backend/.env.example | 16 ++ backend/app/federation.py | 152 ++++++++---- backend/app/peer_policy.py | 194 +++++++++++++++ backend/config.py | 27 ++ backend/main.py | 27 +- backend/tests/test_federation.py | 19 +- backend/tests/test_federation_ux.py | 22 +- backend/tests/test_marketplace.py | 4 +- backend/tests/test_peer_policy.py | 219 +++++++++++++++++ backend/tests/test_registry.py | 2 +- backend/tests/test_v1_auth_discovery.py | 27 +- extension/CHANGELOG.md | 10 + extension/package-lock.json | 4 +- extension/package.json | 14 +- extension/src/api.ts | 193 +++++++++++++-- extension/src/registry.ts | 313 +++++++++++++++++++++--- 18 files changed, 1138 insertions(+), 139 deletions(-) create mode 100644 backend/app/peer_policy.py create mode 100644 backend/tests/test_peer_policy.py diff --git a/PLAN.md b/PLAN.md index 7c47375..1288c62 100644 --- a/PLAN.md +++ b/PLAN.md @@ -12,11 +12,12 @@ - **v0.8** — Federated registry (`/registry/federated` + peers), Browse Federated Registry UI, GitHub Actions release + CI (artifact-first; signed publish when secrets exist) - **v0.9** — Peer status/probe UX (`creer.registryPeers`, Manage Registry Peers), federated browse enrichment, GitHub Release on tag + `RELEASE.md` - **v1.0** — Stable foundation: optional registry write auth (Bearer / `X-Creer-Token`), `GET /registry/discover`, federated `discover=true`, SecretStorage registry token + Discover peers UX +- **v1.1** — Discovery hardening: peer policy (SSRF / private IP blocks, allow/deny, max hops); extension surfaces policy errors, `creer.federationMaxHops` / `creer.warnPrivatePeers`, blocked-discover UX ## Optional next -- Human: configure `VSCE_PAT` / `OVSX_PAT` repository secrets; tag `v1.0.0` (see [`RELEASE.md`](RELEASE.md)) — agents cannot set GitHub Actions secrets -- Hardened multi-hop discovery policies / signed peer trust +- Human: configure `VSCE_PAT` / `OVSX_PAT` repository secrets; tag `v1.1.0` (see [`RELEASE.md`](RELEASE.md)) — agents cannot set GitHub Actions secrets +- Signed peer trust / mutual TLS between registries ## Non-goals diff --git a/README.md b/README.md index cc0a5d4..392a92a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ AI-powered repo scaffolding inside your workspace. -**Current version: 1.0.0** (stable foundation) +**Current version: 1.1.0** (discovery hardening) ## Architecture @@ -26,17 +26,17 @@ cd extension && npm install && npm run compile # F5 → Creer: Create New Repo ``` -## Registry & federation (v0.7–v1.0) +## Registry & federation (v0.7–v1.1) -Self-hosted pack catalog plus optional multi-host federation and write auth: +Self-hosted pack catalog plus optional multi-host federation, write auth, and peer policy: | Method | Path | Description | |---|---|---| | `GET` | `/registry?q=&source=` | Searchable pack list | -| `GET` | `/registry/federated?q=&source=&peers=&discover=` | Local + peer merge; `discover=true` expands one hop | -| `GET` | `/registry/discover` | One-hop peer discovery | +| `GET` | `/registry/federated?q=&source=&peers=&discover=&max_hops=` | Local + peer merge; `discover=true` expands peers; `max_hops` caps hop depth (0–2) | +| `GET` | `/registry/discover` | Peer discovery (policy-filtered in v1.1+) | | `GET` | `/registry/peers` | Peer health + configured URLs | -| `POST` | `/registry/peers/probe` | Probe one peer `{ url }` (auth when configured) | +| `POST` | `/registry/peers/probe` | Probe one peer `{ url }` (auth when configured; may 400 on policy) | | `GET` | `/registry/packs/{id}` | Pack metadata | | `GET` | `/registry/packs/{id}/download` | Portable JSON pack (installable URL) | | `GET` | `/marketplace` | Curated featured view | @@ -45,9 +45,20 @@ Self-hosted pack catalog plus optional multi-host federation and write auth: When the backend sets `CREER_REGISTRY_TOKEN`, mutating routes expect `Authorization: Bearer ` and/or `X-Creer-Token`. -Extension settings: `creer.registryPeers`, `creer.showPeerStatus`, `creer.federatedDiscover` (pass `discover=true`), `creer.registryToken` (deprecated plaintext — prefer SecretStorage). +### Peer policy env vars (v1.1) -Commands: **Browse Federated Registry**, **Manage Registry Peers** (includes Discover peers), **Set / Clear Registry Token**. +Backend peer/federation policy (SSRF and private-IP hardening). The extension surfaces 400 `detail` strings and per-peer `error` / optional `policy` fields. + +| Env | Purpose | +|---|---| +| `CREER_FEDERATION_MAX_HOPS` | Default max discovery hops (0–2; default 1). Extension may also send `max_hops` on federated browse when the backend accepts it. | +| `CREER_PEER_ALLOWLIST` | Comma-separated hostnames/URLs; if non-empty, only these peers may be contacted | +| `CREER_PEER_DENYLIST` | Comma-separated hostnames/URLs always blocked | +| `CREER_ALLOW_PRIVATE_PEERS` | When true (`1`/`true`/`yes`), allow loopback/private/link-local peers (default off) | + +Extension settings: `creer.registryPeers`, `creer.showPeerStatus`, `creer.federatedDiscover` (`discover=true`), `creer.federationMaxHops` (`max_hops`), `creer.warnPrivatePeers`, `creer.registryToken` (deprecated plaintext — prefer SecretStorage). + +Commands: **Browse Federated Registry**, **Manage Registry Peers** (Discover peers; private-host warning; policy-blocked suggestions), **Set / Clear Registry Token**. Install from another Creer host: @@ -85,7 +96,7 @@ See [`RELEASE.md`](RELEASE.md) and [`extension/PUBLISH.md`](extension/PUBLISH.md ```bash cd extension && npm run compile && npm run package -# → creer-1.0.0.vsix (includes media/icon.png) +# → creer-1.1.0.vsix (includes media/icon.png) ``` Signed Marketplace / Open VSX publish requires your own `VSCE_PAT` / `OVSX_PAT` (never commit tokens). Agents cannot set GitHub Actions secrets — that remains a human step. diff --git a/backend/.env.example b/backend/.env.example index 9608235..a5a4082 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -31,3 +31,19 @@ CREER_OFFLINE= # Protects: POST /packs/install, DELETE /packs/{id}, POST /registry/peers/probe. # Reads (GET /registry, /health, /plan, /generate) stay public. # CREER_REGISTRY_TOKEN= + +# Federation discovery hop budget (v1.1). Integer 0–2 (default 1). +# 0 = no discover expansion (configured/seed peers still queried as hop0). +# CREER_FEDERATION_MAX_HOPS=1 + +# Comma-separated peer hostnames or base URLs. If non-empty, ONLY these peers +# may be contacted for outbound probe/fetch/discover. +# CREER_PEER_ALLOWLIST= + +# Comma-separated peer hostnames or base URLs always blocked. +# CREER_PEER_DENYLIST= + +# Allow outbound peers on private/link-local/loopback/metadata addresses (default off). +# Set to 1/true/yes for local multi-instance testing. Loopback is also allowed when +# that host appears explicitly on CREER_PEER_ALLOWLIST. +# CREER_ALLOW_PRIVATE_PEERS= diff --git a/backend/app/federation.py b/backend/app/federation.py index c9a1c34..cd0c03d 100644 --- a/backend/app/federation.py +++ b/backend/app/federation.py @@ -11,9 +11,15 @@ from config import CREER_PUBLIC_BASE_URL, CREER_REGISTRY_PEERS from app.auth import registry_auth_required +from app.peer_policy import ( + assert_peer_allowed, + clamped_max_hops, + normalize_peer_url, + policy_summary, +) from app.registry import list_registry, registry_count -FEDERATION_VERSION = "1.0.0" +FEDERATION_VERSION = "1.1.0" _MAX_PEERS = 8 _DISCOVER_TIMEOUT = 3.0 @@ -31,14 +37,9 @@ def parse_peers(raw: str | None = None) -> list[str]: seen: set[str] = set() out: list[str] = [] for part in text.split(","): - url = part.strip().rstrip("/") + url = normalize_peer_url(part) if not url: continue - parsed = urlparse(url) - if parsed.scheme not in ("http", "https"): - continue - if not parsed.netloc: - continue if url in seen: continue seen.add(url) @@ -81,6 +82,7 @@ def discover_self() -> dict[str, Any]: "packs_count": registry_count(), "peers": parse_peers(), "auth_required": registry_auth_required(), + "policy": policy_summary(), } @@ -137,6 +139,11 @@ def _fetch_peer_registry( if not base: return [], "empty base_url" + try: + assert_peer_allowed(base) + except ValueError as exc: + return [], str(exc) + params: dict[str, str] = {} if q: params["q"] = q @@ -179,6 +186,11 @@ def fetch_peer_discover( if not base: return None, "empty base_url" + try: + assert_peer_allowed(base) + except ValueError as exc: + return None, str(exc) + url = f"{base}/registry/discover" try: with httpx.Client(timeout=timeout, follow_redirects=True) as client: @@ -193,52 +205,97 @@ def fetch_peer_discover( return data, None -def expand_peers_one_hop( - peers: list[str], +def expand_peers( + seeds: list[str], + max_hops: int | None = None, *, timeout: float = _DISCOVER_TIMEOUT, ) -> tuple[list[str], list[str]]: """ - One-hop gossip-lite: for each ok peer, fetch /registry/discover and collect - advertised peers not already in the list. Cap total at _MAX_PEERS. + Multi-hop gossip-lite discovery with cycle detection. - Returns (expanded_peers, discovered_peers_added_this_hop). + max_hops 0 = no expansion (return seeds only; configured peers remain hop0). + Cap total peers at _MAX_PEERS. Skips candidates blocked by peer policy. + Returns (expanded_peers, discovered_peers_added). """ - if not peers: + hops = clamped_max_hops(max_hops) + if not seeds: return [], [] - known: set[str] = set(peers) - discovered: list[str] = [] - by_peer: dict[str, dict[str, Any] | None] = {} + # Preserve seed order; cap immediately + known: set[str] = set() + all_peers: list[str] = [] + for s in seeds: + url = normalize_peer_url(s) or (s.strip().rstrip("/") if s else "") + if not url or url in known: + continue + known.add(url) + all_peers.append(url) + if len(all_peers) >= _MAX_PEERS: + break - workers = min(8, len(peers)) - with ThreadPoolExecutor(max_workers=workers) as pool: - futures = {pool.submit(fetch_peer_discover, p, timeout=timeout): p for p in peers} - for fut in as_completed(futures): - peer = futures[fut] - data, err = fut.result() - by_peer[peer] = data if err is None else None + if hops <= 0 or not all_peers: + return all_peers, [] - # Preserve seed peer order when reading advertised lists - for peer in peers: - data = by_peer.get(peer) - if not data: - continue - advertised = data.get("peers") or [] - if not isinstance(advertised, list): - continue - for raw in advertised: - if not isinstance(raw, str): + discovered_all: list[str] = [] + frontier = list(all_peers) + fetched: set[str] = set() + + for _ in range(hops): + if len(all_peers) >= _MAX_PEERS: + break + to_query = [p for p in frontier if p not in fetched] + if not to_query: + break + + by_peer: dict[str, dict[str, Any] | None] = {} + workers = min(8, len(to_query)) + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = { + pool.submit(fetch_peer_discover, p, timeout=timeout): p for p in to_query + } + for fut in as_completed(futures): + peer = futures[fut] + data, err = fut.result() + by_peer[peer] = data if err is None else None + fetched.add(peer) + + new_frontier: list[str] = [] + for peer in to_query: + data = by_peer.get(peer) + if not data: + continue + advertised = data.get("peers") or [] + if not isinstance(advertised, list): continue - for cand in parse_peers(raw): - if cand in known: + for raw in advertised: + if not isinstance(raw, str): continue - if len(peers) + len(discovered) >= _MAX_PEERS: - return peers + discovered, discovered - known.add(cand) - discovered.append(cand) + for cand in parse_peers(raw): + if cand in known: + continue # cycle / already have + try: + assert_peer_allowed(cand) + except ValueError: + continue + if len(all_peers) >= _MAX_PEERS: + return all_peers, discovered_all + known.add(cand) + all_peers.append(cand) + discovered_all.append(cand) + new_frontier.append(cand) + frontier = new_frontier + + return all_peers, discovered_all - return peers + discovered, discovered + +def expand_peers_one_hop( + peers: list[str], + *, + timeout: float = _DISCOVER_TIMEOUT, +) -> tuple[list[str], list[str]]: + """Backward-compatible one-hop expand (ignores CREER_FEDERATION_MAX_HOPS).""" + return expand_peers(peers, max_hops=1, timeout=timeout) def probe_peer(base_url: str, timeout: float = 5.0) -> dict[str, Any]: @@ -265,6 +322,12 @@ def probe_peer(base_url: str, timeout: float = 5.0) -> dict[str, Any]: result["error"] = "url must use http or https with a host" return result + try: + assert_peer_allowed(base) + except ValueError as exc: + result["error"] = str(exc) + return result + started = time.perf_counter() version: str | None = None count: int | None = None @@ -354,6 +417,7 @@ def list_federated( include_local: bool = True, extra_peers: list[str] | None = None, discover: bool = False, + max_hops: int | None = None, ) -> dict[str, Any]: """Merge local registry with peer registries (local ids win on collision).""" local = list_registry(q=q, source=source or "all") if include_local else { @@ -365,7 +429,7 @@ def list_federated( peers = resolve_peers(extra_peers) discovered_peers: list[str] = [] if discover and peers: - peers, discovered_peers = expand_peers_one_hop(peers) + peers, discovered_peers = expand_peers(peers, max_hops=max_hops) peer_meta: list[dict[str, Any]] = [] peer_items_by_url: dict[str, list[dict[str, Any]]] = {} @@ -411,10 +475,16 @@ def _one(peer: str) -> tuple[str, list[dict[str, Any]], str | None]: seen_ids.add(pid) merged.append(item) + summary = policy_summary() + if max_hops is not None: + summary = dict(summary) + summary["request_max_hops"] = clamped_max_hops(max_hops) + return { "version": FEDERATION_VERSION, "local": local, "peers": peer_meta, "items": merged, "discovered_peers": discovered_peers, + "policy": summary, } diff --git a/backend/app/peer_policy.py b/backend/app/peer_policy.py new file mode 100644 index 0000000..abe9fa3 --- /dev/null +++ b/backend/app/peer_policy.py @@ -0,0 +1,194 @@ +"""Outbound peer SSRF / allow-deny policy for federation (v1.1).""" + +from __future__ import annotations + +import ipaddress +import socket +from typing import Any +from urllib.parse import urlparse + +from config import ( + CREER_ALLOW_PRIVATE_PEERS as _ALLOW_PRIVATE_FROM_CONFIG, + CREER_FEDERATION_MAX_HOPS as _MAX_HOPS_FROM_CONFIG, + CREER_PEER_ALLOWLIST as _ALLOWLIST_FROM_CONFIG, + CREER_PEER_DENYLIST as _DENYLIST_FROM_CONFIG, +) + +# Module-level bindings so tests can monkeypatch +CREER_ALLOW_PRIVATE_PEERS = _ALLOW_PRIVATE_FROM_CONFIG +CREER_FEDERATION_MAX_HOPS = _MAX_HOPS_FROM_CONFIG +CREER_PEER_ALLOWLIST = _ALLOWLIST_FROM_CONFIG +CREER_PEER_DENYLIST = _DENYLIST_FROM_CONFIG + + +def normalize_peer_url(url: str | None) -> str | None: + """Strip, drop trailing slash; return http(s) base URL or None if invalid.""" + if not url or not isinstance(url, str): + return None + text = url.strip().rstrip("/") + if not text: + return None + parsed = urlparse(text) + if parsed.scheme not in ("http", "https"): + return None + if not parsed.netloc: + return None + return text + + +def host_of(url: str) -> str: + """Extract lowercase hostname from a URL or bare host string.""" + if not url or not isinstance(url, str): + return "" + text = url.strip() + if "://" not in text: + text = f"http://{text}" + parsed = urlparse(text) + host = parsed.hostname or "" + return host.lower().rstrip(".") + + +def _hosts_from_csv(raw: str | None) -> set[str]: + """Parse comma-separated hostnames/URLs into a set of lowercase hosts.""" + if not raw or not str(raw).strip(): + return set() + out: set[str] = set() + for part in str(raw).split(","): + part = part.strip() + if not part: + continue + h = host_of(part) + if h: + out.add(h) + return out + + +def allowlist_hosts() -> set[str]: + return _hosts_from_csv(CREER_PEER_ALLOWLIST) + + +def denylist_hosts() -> set[str]: + return _hosts_from_csv(CREER_PEER_DENYLIST) + + +def allowlist_active() -> bool: + return bool(allowlist_hosts()) + + +def is_blocked_host(host: str) -> bool: + """ + True if host is denylisted, or allowlist is active and host is not listed. + """ + h = (host or "").lower().rstrip(".") + if not h: + return True + if h in denylist_hosts(): + return True + allow = allowlist_hosts() + if allow and h not in allow: + return True + return False + + +def _ip_is_private_or_unsafe(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: + """Block private, loopback, link-local, unspecified, multicast, reserved.""" + if ( + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_unspecified + or ip.is_multicast + or ip.is_reserved + ): + return True + # Explicit metadata IP (also link-local, but keep visible intent) + if ip.version == 4 and str(ip) == "169.254.169.254": + return True + return False + + +def is_private_or_unsafe_host(host: str) -> bool: + """ + True if host is a private/unsafe literal IP, or any resolved address is. + + On DNS failure, treat as unsafe (blocked for outbound). + """ + h = (host or "").lower().rstrip(".") + if not h: + return True + + # Literal IP (IPv4 / IPv6) + try: + ip = ipaddress.ip_address(h) + return _ip_is_private_or_unsafe(ip) + except ValueError: + pass + + # Hostname → resolve all addresses + try: + infos = socket.getaddrinfo(h, None) + except OSError: + return True + + if not infos: + return True + + for info in infos: + sockaddr = info[4] + addr = sockaddr[0] + try: + ip = ipaddress.ip_address(addr) + except ValueError: + continue + if _ip_is_private_or_unsafe(ip): + return True + return False + + +def assert_peer_allowed(url: str) -> None: + """ + Raise ValueError with a clear message when the peer URL is not allowed. + + Loopback/private hosts are allowed when CREER_ALLOW_PRIVATE_PEERS is true, + or when the host is explicitly present on CREER_PEER_ALLOWLIST (testing). + """ + normalized = normalize_peer_url(url) + if not normalized: + raise ValueError("peer url must use http or https with a host") + + host = host_of(normalized) + if not host: + raise ValueError("peer url missing host") + + if is_blocked_host(host): + if host in denylist_hosts(): + raise ValueError(f"peer host denied by denylist: {host}") + raise ValueError(f"peer host not on allowlist: {host}") + + if not CREER_ALLOW_PRIVATE_PEERS: + # Explicit allowlist entry may opt into private/loopback for local testing + if host not in allowlist_hosts(): + if is_private_or_unsafe_host(host): + raise ValueError( + f"peer host is private or unsafe (set CREER_ALLOW_PRIVATE_PEERS=1 " + f"or allowlist the host to permit): {host}" + ) + + +def policy_summary() -> dict[str, Any]: + """Compact policy snapshot for discover / federated responses.""" + return { + "max_hops": int(CREER_FEDERATION_MAX_HOPS), + "allow_private": bool(CREER_ALLOW_PRIVATE_PEERS), + "allowlist_active": allowlist_active(), + } + + +def clamped_max_hops(override: int | None = None) -> int: + """Return max hops clamped to 0–2.""" + raw = CREER_FEDERATION_MAX_HOPS if override is None else override + try: + value = int(raw) + except (TypeError, ValueError): + value = 1 + return max(0, min(2, value)) diff --git a/backend/config.py b/backend/config.py index d92feac..960296b 100644 --- a/backend/config.py +++ b/backend/config.py @@ -16,3 +16,30 @@ # Optional token for mutating registry/pack write endpoints (v1.0+) # When set, POST /packs/install, DELETE /packs/{id}, POST /registry/peers/probe require auth CREER_REGISTRY_TOKEN = os.getenv("CREER_REGISTRY_TOKEN", "").strip() or None + + +def _clamp_federation_max_hops(raw: str | None) -> int: + """Parse CREER_FEDERATION_MAX_HOPS; default 1, clamp to 0–2.""" + if raw is None or str(raw).strip() == "": + return 1 + try: + value = int(str(raw).strip()) + except (TypeError, ValueError): + return 1 + return max(0, min(2, value)) + + +# Max discovery hops for federated expand (v1.1+). 0 = no expand (configured peers only). +CREER_FEDERATION_MAX_HOPS = _clamp_federation_max_hops( + os.getenv("CREER_FEDERATION_MAX_HOPS", "1") +) +# Comma-separated hostnames/URLs; if non-empty, ONLY these peers may be contacted +CREER_PEER_ALLOWLIST = os.getenv("CREER_PEER_ALLOWLIST", "").strip() +# Comma-separated hostnames/URLs always blocked +CREER_PEER_DENYLIST = os.getenv("CREER_PEER_DENYLIST", "").strip() +# When false (default), block private/link-local/loopback/metadata for outbound peers +CREER_ALLOW_PRIVATE_PEERS = os.getenv("CREER_ALLOW_PRIVATE_PEERS", "").lower() in ( + "1", + "true", + "yes", +) diff --git a/backend/main.py b/backend/main.py index d682871..9da67dd 100644 --- a/backend/main.py +++ b/backend/main.py @@ -11,7 +11,14 @@ from fastapi.responses import Response, StreamingResponse from pydantic import BaseModel, Field -from config import CREER_OFFLINE, CREER_PUBLIC_BASE_URL, MODEL, OPENAI_BASE_URL +from config import ( + CREER_ALLOW_PRIVATE_PEERS, + CREER_FEDERATION_MAX_HOPS, + CREER_OFFLINE, + CREER_PUBLIC_BASE_URL, + MODEL, + OPENAI_BASE_URL, +) from app.auth import registry_auth_required, require_registry_write from app.bakeins import apply_bakeins, list_bakein_options from app.planner import plan_project @@ -40,11 +47,12 @@ parse_peers, probe_peer, ) +from app.peer_policy import assert_peer_allowed from app.github import create_github_repo from app.jobs import cancel_job, create_job, finish_job, is_cancelled from app.quality import has_errors, run_quality_gates -VERSION = "1.0.0" +VERSION = "1.1.0" app = FastAPI(title="Creer", version=VERSION) @@ -169,6 +177,8 @@ def health(): "public_base_url_set": bool(CREER_PUBLIC_BASE_URL), "peers_configured": len(parse_peers()), "auth_required": registry_auth_required(), + "federation_max_hops": CREER_FEDERATION_MAX_HOPS, + "allow_private_peers": CREER_ALLOW_PRIVATE_PEERS, } @@ -201,7 +211,13 @@ def registry_federated( ), discover: bool = Query( default=False, - description="One-hop peer discovery via GET {peer}/registry/discover", + description="Multi-hop peer discovery via GET {peer}/registry/discover", + ), + max_hops: int | None = Query( + default=None, + ge=0, + le=2, + description="Hop budget for discover (overrides CREER_FEDERATION_MAX_HOPS for this request)", ), ): """Federated registry: local packs plus peer Creer registries.""" @@ -212,6 +228,7 @@ def registry_federated( include_local=True, extra_peers=extra, discover=discover, + max_hops=max_hops, ) @@ -240,6 +257,10 @@ def registry_peers_probe( status_code=400, detail="url must use http or https with a host", ) + try: + assert_peer_allowed(url) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc return probe_peer(url) diff --git a/backend/tests/test_federation.py b/backend/tests/test_federation.py index 35f5e8c..5933955 100644 --- a/backend/tests/test_federation.py +++ b/backend/tests/test_federation.py @@ -6,10 +6,17 @@ import main from app import federation as fed +from app import peer_policy as policy from app.federation import list_federated, parse_peers from main import VERSION +def _allow_fake_hosts(monkeypatch): + monkeypatch.setattr(policy, "CREER_ALLOW_PRIVATE_PEERS", True) + monkeypatch.setattr(policy, "CREER_PEER_ALLOWLIST", "") + monkeypatch.setattr(policy, "CREER_PEER_DENYLIST", "") + monkeypatch.setattr(policy, "is_private_or_unsafe_host", lambda host: False) + def test_parse_peers_normalize_dedupe(): raw = ( " http://127.0.0.1:8001/,https://creer-packs.example.com," @@ -64,7 +71,7 @@ def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): monkeypatch.setattr(fed, "_fetch_peer_registry", fake_fetch) result = list_federated() - assert result["version"] == "1.0.0" + assert result["version"] == "1.1.0" assert "items" in result["local"] assert len(result["peers"]) == 1 assert result["peers"][0]["ok"] is True @@ -95,7 +102,7 @@ def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): monkeypatch.setattr(fed, "_fetch_peer_registry", fake_fetch) result = list_federated() - assert result["version"] == "1.0.0" + assert result["version"] == "1.1.0" local_count = len(result["local"]["items"]) assert local_count >= 3 assert len(result["items"]) == local_count @@ -134,8 +141,8 @@ def test_health_0_9_and_federated_route(monkeypatch): # Also patch config import used if parse_peers reads module-level — already patched fed c = TestClient(main.app) h = c.get("/health").json() - assert h["version"] == "1.0.0" - assert VERSION == "1.0.0" + assert h["version"] == "1.1.0" + assert VERSION == "1.1.0" assert h["peers_configured"] == 2 # No live peers — empty mock via monkeypatch on fetch @@ -147,12 +154,14 @@ def test_health_0_9_and_federated_route(monkeypatch): r = c.get("/registry/federated") assert r.status_code == 200 body = r.json() - assert body["version"] == "1.0.0" + assert body["version"] == "1.1.0" assert "local" in body assert len(body["items"]) == len(body["local"]["items"]) def test_fetch_peer_registry_absolutizes(monkeypatch): + _allow_fake_hosts(monkeypatch) + class FakeResp: def raise_for_status(self): return None diff --git a/backend/tests/test_federation_ux.py b/backend/tests/test_federation_ux.py index 18eb61e..ba837c2 100644 --- a/backend/tests/test_federation_ux.py +++ b/backend/tests/test_federation_ux.py @@ -6,11 +6,21 @@ import main from app import federation as fed +from app import peer_policy as policy from app.federation import list_federated, list_peer_status, probe_peer, resolve_peers from main import VERSION +def _allow_fake_hosts(monkeypatch): + """Skip SSRF DNS checks for .example test hosts.""" + monkeypatch.setattr(policy, "CREER_ALLOW_PRIVATE_PEERS", True) + monkeypatch.setattr(policy, "CREER_PEER_ALLOWLIST", "") + monkeypatch.setattr(policy, "CREER_PEER_DENYLIST", "") + monkeypatch.setattr(policy, "is_private_or_unsafe_host", lambda host: False) + + def test_probe_peer_ok(monkeypatch): + _allow_fake_hosts(monkeypatch) calls: list[str] = [] class FakeResp: @@ -54,6 +64,8 @@ def get(self, url, params=None): def test_probe_peer_fail(monkeypatch): + _allow_fake_hosts(monkeypatch) + class FakeClient: def __init__(self, *a, **k): pass @@ -79,6 +91,7 @@ def get(self, url, params=None): def test_probe_peer_registry_fallback(monkeypatch): """When /health fails, /registry still yields ok + count.""" + _allow_fake_hosts(monkeypatch) class FakeResp: def __init__(self, data): @@ -222,6 +235,7 @@ def test_registry_peers_probe_validation(): def test_registry_peers_probe_ok(monkeypatch): + _allow_fake_hosts(monkeypatch) monkeypatch.setattr( main, "probe_peer", @@ -262,7 +276,7 @@ def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): r = c.get("/registry/federated", params={"peers": "http://adhoc.peer:8002/"}) assert r.status_code == 200 body = r.json() - assert body["version"] == "1.0.0" + assert body["version"] == "1.1.0" assert any(p["base_url"] == "http://adhoc.peer:8002" for p in body["peers"]) assert any(i["id"] == "adhoc-pack" for i in body["items"]) @@ -278,7 +292,7 @@ def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): monkeypatch.setattr(fed, "_fetch_peer_registry", fake_fetch) result = list_federated(extra_peers=["http://extra.peer/", "http://cfg.peer"]) - assert result["version"] == "1.0.0" + assert result["version"] == "1.1.0" assert seen == ["http://cfg.peer", "http://extra.peer"] @@ -286,7 +300,7 @@ def test_health_0_9(monkeypatch): monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "https://a.example,https://b.example") c = TestClient(main.app) h = c.get("/health").json() - assert h["version"] == "1.0.0" - assert VERSION == "1.0.0" + assert h["version"] == "1.1.0" + assert VERSION == "1.1.0" assert h["peers_configured"] == 2 assert h["status"] == "ok" diff --git a/backend/tests/test_marketplace.py b/backend/tests/test_marketplace.py index e9ed0a7..dad5458 100644 --- a/backend/tests/test_marketplace.py +++ b/backend/tests/test_marketplace.py @@ -48,8 +48,8 @@ def test_health_version_0_9(client, monkeypatch): resp = client.get("/health") assert resp.status_code == 200 data = resp.json() - assert data["version"] == "1.0.0" - assert VERSION == "1.0.0" + assert data["version"] == "1.1.0" + assert VERSION == "1.1.0" assert data["offline"] is True assert data["packs_count"] >= 3 assert "peers_configured" in data diff --git a/backend/tests/test_peer_policy.py b/backend/tests/test_peer_policy.py new file mode 100644 index 0000000..0a8f98b --- /dev/null +++ b/backend/tests/test_peer_policy.py @@ -0,0 +1,219 @@ +"""Tests for peer SSRF / allow-deny policy + multi-hop discovery (v1.1).""" + +from __future__ import annotations + +import socket + +from fastapi.testclient import TestClient + +import main +from app import federation as fed +from app import peer_policy as policy +from app.federation import expand_peers, list_federated +from main import VERSION + + +def _public_addrinfo(host, *args, **kwargs): + """Fake DNS: all hostnames resolve to a public IP.""" + return [ + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.8.8", 0)), + ] + + +def test_block_loopback_when_private_not_allowed(monkeypatch): + monkeypatch.setattr(policy, "CREER_ALLOW_PRIVATE_PEERS", False) + monkeypatch.setattr(policy, "CREER_PEER_ALLOWLIST", "") + monkeypatch.setattr(policy, "CREER_PEER_DENYLIST", "") + + try: + policy.assert_peer_allowed("http://127.0.0.1:8001") + raised = False + except ValueError as exc: + raised = True + assert "private or unsafe" in str(exc).lower() or "127.0.0.1" in str(exc) + assert raised is True + + assert policy.is_private_or_unsafe_host("127.0.0.1") is True + assert policy.is_private_or_unsafe_host("10.0.0.5") is True + assert policy.is_private_or_unsafe_host("192.168.1.1") is True + assert policy.is_private_or_unsafe_host("169.254.169.254") is True + assert policy.is_private_or_unsafe_host("::1") is True + + +def test_allow_private_when_flag_set(monkeypatch): + monkeypatch.setattr(policy, "CREER_ALLOW_PRIVATE_PEERS", True) + monkeypatch.setattr(policy, "CREER_PEER_ALLOWLIST", "") + monkeypatch.setattr(policy, "CREER_PEER_DENYLIST", "") + + policy.assert_peer_allowed("http://127.0.0.1:8001") + policy.assert_peer_allowed("http://10.1.2.3:9000") + + +def test_allowlist_permits_loopback_when_private_off(monkeypatch): + monkeypatch.setattr(policy, "CREER_ALLOW_PRIVATE_PEERS", False) + monkeypatch.setattr(policy, "CREER_PEER_ALLOWLIST", "127.0.0.1,localhost") + monkeypatch.setattr(policy, "CREER_PEER_DENYLIST", "") + + policy.assert_peer_allowed("http://127.0.0.1:8001") + + +def test_denylist_blocks(monkeypatch): + monkeypatch.setattr(policy, "CREER_ALLOW_PRIVATE_PEERS", True) + monkeypatch.setattr(policy, "CREER_PEER_ALLOWLIST", "") + monkeypatch.setattr(policy, "CREER_PEER_DENYLIST", "evil.example,http://bad.peer:8000") + + try: + policy.assert_peer_allowed("http://evil.example") + raised = False + except ValueError as exc: + raised = True + assert "denylist" in str(exc).lower() or "denied" in str(exc).lower() + assert raised is True + + try: + policy.assert_peer_allowed("http://bad.peer:8000") + raised2 = False + except ValueError: + raised2 = True + assert raised2 is True + + +def test_allowlist_restricts(monkeypatch): + monkeypatch.setattr(policy, "CREER_ALLOW_PRIVATE_PEERS", True) + monkeypatch.setattr(policy, "CREER_PEER_ALLOWLIST", "ok.example") + monkeypatch.setattr(policy, "CREER_PEER_DENYLIST", "") + monkeypatch.setattr(policy.socket, "getaddrinfo", _public_addrinfo) + + policy.assert_peer_allowed("https://ok.example") + try: + policy.assert_peer_allowed("https://other.example") + raised = False + except ValueError as exc: + raised = True + assert "allowlist" in str(exc).lower() + assert raised is True + assert policy.allowlist_active() is True + assert policy.is_blocked_host("other.example") is True + assert policy.is_blocked_host("ok.example") is False + + +def test_dns_failure_treated_as_unsafe(monkeypatch): + monkeypatch.setattr(policy, "CREER_ALLOW_PRIVATE_PEERS", False) + monkeypatch.setattr(policy, "CREER_PEER_ALLOWLIST", "") + monkeypatch.setattr(policy, "CREER_PEER_DENYLIST", "") + + def boom(*a, **k): + raise socket.gaierror(-2, "Name or service not known") + + monkeypatch.setattr(policy.socket, "getaddrinfo", boom) + assert policy.is_private_or_unsafe_host("no-such-host.invalid") is True + + +def test_max_hops_cycle_no_infinite_loop(monkeypatch): + """A discovers B, B discovers A → cycle detected, no infinite loop.""" + monkeypatch.setattr(policy, "CREER_ALLOW_PRIVATE_PEERS", True) + monkeypatch.setattr(policy, "CREER_PEER_ALLOWLIST", "") + monkeypatch.setattr(policy, "CREER_PEER_DENYLIST", "") + monkeypatch.setattr(policy, "CREER_FEDERATION_MAX_HOPS", 2) + monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "http://a.example") + + def fake_discover(base_url, *, timeout=3.0): + if base_url == "http://a.example": + return {"peers": ["http://b.example"]}, None + if base_url == "http://b.example": + return {"peers": ["http://a.example"]}, None + return None, "unknown" + + monkeypatch.setattr(fed, "fetch_peer_discover", fake_discover) + + expanded, discovered = expand_peers(["http://a.example"], max_hops=2) + assert expanded == ["http://a.example", "http://b.example"] + assert discovered == ["http://b.example"] + # Second hop sees A again but does not re-add or loop + assert expanded.count("http://a.example") == 1 + assert expanded.count("http://b.example") == 1 + + +def test_max_hops_zero_no_expansion(monkeypatch): + monkeypatch.setattr(policy, "CREER_FEDERATION_MAX_HOPS", 0) + + def boom(*a, **k): + raise AssertionError("discover must not be called when max_hops=0") + + monkeypatch.setattr(fed, "fetch_peer_discover", boom) + expanded, discovered = expand_peers(["http://seed.example"], max_hops=0) + assert expanded == ["http://seed.example"] + assert discovered == [] + + +def test_list_federated_includes_policy(monkeypatch): + monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "") + monkeypatch.setattr(policy, "CREER_FEDERATION_MAX_HOPS", 1) + monkeypatch.setattr(policy, "CREER_ALLOW_PRIVATE_PEERS", False) + monkeypatch.setattr(policy, "CREER_PEER_ALLOWLIST", "") + + result = list_federated(discover=False) + assert result["version"] == "1.1.0" + assert result["policy"] == { + "max_hops": 1, + "allow_private": False, + "allowlist_active": False, + } + + +def test_discover_includes_policy(monkeypatch): + monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "") + monkeypatch.setattr(policy, "CREER_FEDERATION_MAX_HOPS", 2) + monkeypatch.setattr(policy, "CREER_ALLOW_PRIVATE_PEERS", True) + monkeypatch.setattr(policy, "CREER_PEER_ALLOWLIST", "x.example") + + c = TestClient(main.app) + body = c.get("/registry/discover").json() + assert body["version"] == "1.1.0" + assert body["policy"]["max_hops"] == 2 + assert body["policy"]["allow_private"] is True + assert body["policy"]["allowlist_active"] is True + + +def test_probe_endpoint_400_when_policy_blocks(monkeypatch): + monkeypatch.setattr(policy, "CREER_ALLOW_PRIVATE_PEERS", False) + monkeypatch.setattr(policy, "CREER_PEER_ALLOWLIST", "") + monkeypatch.setattr(policy, "CREER_PEER_DENYLIST", "") + + c = TestClient(main.app) + r = c.post("/registry/peers/probe", json={"url": "http://127.0.0.1:8001"}) + assert r.status_code == 400 + detail = r.json()["detail"] + assert "private" in detail.lower() or "unsafe" in detail.lower() + + +def test_probe_peer_returns_error_when_blocked(monkeypatch): + monkeypatch.setattr(policy, "CREER_ALLOW_PRIVATE_PEERS", False) + monkeypatch.setattr(policy, "CREER_PEER_ALLOWLIST", "") + monkeypatch.setattr(policy, "CREER_PEER_DENYLIST", "") + + result = fed.probe_peer("http://127.0.0.1:9") + assert result["ok"] is False + assert result["error"] + assert "private" in result["error"].lower() or "unsafe" in result["error"].lower() + + +def test_health_1_1(monkeypatch): + monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "https://a.example") + c = TestClient(main.app) + h = c.get("/health").json() + assert h["version"] == "1.1.0" + assert VERSION == "1.1.0" + assert "federation_max_hops" in h + assert h["federation_max_hops"] in (0, 1, 2) + assert "allow_private_peers" in h + assert isinstance(h["allow_private_peers"], bool) + assert h["status"] == "ok" + + +def test_normalize_and_host_of(): + assert policy.normalize_peer_url("https://Peer.Example:8443/") == "https://Peer.Example:8443" + assert policy.normalize_peer_url("ftp://x") is None + assert policy.normalize_peer_url("") is None + assert policy.host_of("https://Peer.Example:8443/path") == "peer.example" + assert policy.host_of("127.0.0.1") == "127.0.0.1" diff --git a/backend/tests/test_registry.py b/backend/tests/test_registry.py index a140578..fbff412 100644 --- a/backend/tests/test_registry.py +++ b/backend/tests/test_registry.py @@ -13,7 +13,7 @@ def test_health_registry_fields(): c = TestClient(main.app) h = c.get("/health").json() - assert h["version"] == "1.0.0" + assert h["version"] == "1.1.0" assert h["registry_count"] >= 3 assert "peers_configured" in h diff --git a/backend/tests/test_v1_auth_discovery.py b/backend/tests/test_v1_auth_discovery.py index edb68eb..227e94c 100644 --- a/backend/tests/test_v1_auth_discovery.py +++ b/backend/tests/test_v1_auth_discovery.py @@ -10,6 +10,7 @@ from app import auth as auth_mod from app import federation as fed from app import packs as packs_mod +from app import peer_policy as policy from app.federation import expand_peers_one_hop, list_federated from main import VERSION @@ -24,6 +25,13 @@ } +def _allow_fake_hosts(monkeypatch): + monkeypatch.setattr(policy, "CREER_ALLOW_PRIVATE_PEERS", True) + monkeypatch.setattr(policy, "CREER_PEER_ALLOWLIST", "") + monkeypatch.setattr(policy, "CREER_PEER_DENYLIST", "") + monkeypatch.setattr(policy, "is_private_or_unsafe_host", lambda host: False) + + def test_install_without_token_still_works(tmp_path, monkeypatch): """When CREER_REGISTRY_TOKEN is unset, install works as before.""" target = tmp_path / "installed" @@ -194,12 +202,16 @@ def test_discover_endpoint_shape(monkeypatch): r = c.get("/registry/discover") assert r.status_code == 200 body = r.json() - assert body["version"] == "1.0.0" + assert body["version"] == "1.1.0" assert body["base_url"] == "http://me.example:8000" assert isinstance(body["packs_count"], int) assert body["packs_count"] >= 0 assert body["peers"] == ["http://a.example", "http://b.example"] assert body["auth_required"] is True + assert "policy" in body + assert body["policy"]["max_hops"] in (0, 1, 2) + assert "allow_private" in body["policy"] + assert "allowlist_active" in body["policy"] def test_discover_auth_required_false(monkeypatch): @@ -214,7 +226,9 @@ def test_discover_auth_required_false(monkeypatch): def test_federated_discover_expands_peers(monkeypatch): + _allow_fake_hosts(monkeypatch) monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "http://seed.example") + monkeypatch.setattr(policy, "CREER_FEDERATION_MAX_HOPS", 1) def fake_discover(base_url, *, timeout=3.0): if base_url == "http://seed.example": @@ -257,7 +271,7 @@ def fake_fetch(base_url, *, q=None, source=None, timeout=8.0): monkeypatch.setattr(fed, "_fetch_peer_registry", fake_fetch) result = list_federated(discover=True) - assert result["version"] == "1.0.0" + assert result["version"] == "1.1.0" assert result["discovered_peers"] == ["http://hop.example"] peer_urls = [p["base_url"] for p in result["peers"]] assert peer_urls == ["http://seed.example", "http://hop.example"] @@ -284,7 +298,9 @@ def boom(*a, **k): def test_federated_discover_query_param(monkeypatch): + _allow_fake_hosts(monkeypatch) monkeypatch.setattr(fed, "CREER_REGISTRY_PEERS", "http://seed.example") + monkeypatch.setattr(policy, "CREER_FEDERATION_MAX_HOPS", 1) def fake_discover(base_url, *, timeout=3.0): return {"peers": ["http://extra.example"]}, None @@ -301,6 +317,7 @@ def fake_discover(base_url, *, timeout=3.0): def test_expand_peers_respects_cap(monkeypatch): + _allow_fake_hosts(monkeypatch) seeds = [f"http://s{i}.example" for i in range(6)] def fake_discover(base_url, *, timeout=3.0): @@ -321,8 +338,10 @@ def test_health_1_0(monkeypatch): monkeypatch.setattr(auth_mod, "CREER_REGISTRY_TOKEN", None) c = TestClient(main.app) h = c.get("/health").json() - assert h["version"] == "1.0.0" - assert VERSION == "1.0.0" + assert h["version"] == "1.1.0" + assert VERSION == "1.1.0" assert h["peers_configured"] == 1 assert h["auth_required"] is False assert h["status"] == "ok" + assert "federation_max_hops" in h + assert "allow_private_peers" in h diff --git a/extension/CHANGELOG.md b/extension/CHANGELOG.md index a97c727..405f4a4 100644 --- a/extension/CHANGELOG.md +++ b/extension/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 1.1.0 + +Discovery hardening: surface backend peer policy (SSRF / private IP / allow-deny / max hops) cleanly in the extension. + +- Settings: `creer.federationMaxHops` (0–2, default 1) passed as `max_hops` on federated browse; `creer.warnPrivatePeers` (default true) +- Federated fetch types include optional `policy` / peer `blocked` fields; `formatAxiosError` surfaces FastAPI 400 policy `detail` +- **Manage Registry Peers** — modal warning before adding/probing localhost or private-looking hosts; discover shows policy-blocked suggestions as blocked (skipped on add) +- **Browse Federated Registry** — brief status when peers are ok / blocked by policy / failed (e.g. “2 peers ok, 1 blocked by policy”) +- Keeps `creer.federatedDiscover` (`discover=true`) alongside `max_hops` + ## 1.0.0 Stable foundation release: optional registry write auth, one-hop peer discovery, and federated discover browse. diff --git a/extension/package-lock.json b/extension/package-lock.json index 4d74cb5..14ba27b 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -1,12 +1,12 @@ { "name": "creer", - "version": "1.0.0", + "version": "1.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "creer", - "version": "1.0.0", + "version": "1.1.0", "dependencies": { "axios": "^1.7.9" }, diff --git a/extension/package.json b/extension/package.json index 97cd971..aab5264 100644 --- a/extension/package.json +++ b/extension/package.json @@ -2,7 +2,7 @@ "name": "creer", "displayName": "Creer", "description": "AI-powered repo scaffolding inside your workspace", - "version": "1.0.0", + "version": "1.1.0", "publisher": "creer", "license": "MIT", "icon": "media/icon.png", @@ -165,6 +165,18 @@ "default": false, "description": "When browsing the federated registry, pass discover=true for one-hop peer expansion" }, + "creer.federationMaxHops": { + "type": "number", + "default": 1, + "minimum": 0, + "maximum": 2, + "description": "Max peer-discovery hops for federated browse (sent as max_hops when the backend supports it). 0 = configured peers only; 1 = one hop; 2 = two hops." + }, + "creer.warnPrivatePeers": { + "type": "boolean", + "default": true, + "description": "Warn before adding or probing peers whose host looks like localhost or a private IP" + }, "creer.registryToken": { "type": "string", "default": "", diff --git a/extension/src/api.ts b/extension/src/api.ts index be6e2bb..8cb9d75 100644 --- a/extension/src/api.ts +++ b/extension/src/api.ts @@ -96,12 +96,46 @@ function getBackendUrl(): string { return (config.get('backendUrl') || 'http://localhost:8000').replace(/\/$/, ''); } +function formatDetailValue(detail: unknown): string | undefined { + if (typeof detail === 'string' && detail.trim()) { + return detail.trim(); + } + if (Array.isArray(detail)) { + const parts = detail + .map((entry) => { + if (typeof entry === 'string') { + return entry.trim(); + } + if (entry && typeof entry === 'object') { + const obj = entry as { msg?: string; message?: string; detail?: string }; + return (obj.msg || obj.message || obj.detail || '').trim(); + } + return ''; + }) + .filter(Boolean); + if (parts.length > 0) { + return parts.join('; '); + } + } + if (detail && typeof detail === 'object') { + const obj = detail as { message?: string; error?: string; reason?: string }; + const nested = obj.message || obj.error || obj.reason; + if (typeof nested === 'string' && nested.trim()) { + return nested.trim(); + } + } + return undefined; +} + +/** + * Prefer backend `detail` (including FastAPI 400 policy / validation messages). + */ export function formatAxiosError(err: unknown, fallback: string): string { if (axios.isAxiosError(err)) { - const ax = err as AxiosError<{ detail?: string }>; - const detail = ax.response?.data?.detail; - if (typeof detail === 'string' && detail.trim()) { - return detail; + const ax = err as AxiosError<{ detail?: unknown }>; + const fromDetail = formatDetailValue(ax.response?.data?.detail); + if (fromDetail) { + return fromDetail; } if (ax.message) { return ax.message; @@ -113,6 +147,16 @@ export function formatAxiosError(err: unknown, fallback: string): string { return fallback; } +/** True when an error string indicates peer policy / private / SSRF block. */ +export function isPeerPolicyBlockMessage(error: string | null | undefined): boolean { + if (!error?.trim()) { + return false; + } + return /\b(blocked|private|unsafe|ssrf|deny|denied|denylist|allowlist|not allowed|disallowed|policy)\b/i.test( + error + ); +} + export async function fetchTemplates(): Promise { const backendUrl = getBackendUrl(); const response = await axios.get<{ templates: Template[] }>(`${backendUrl}/templates`, { @@ -242,6 +286,23 @@ export interface FederatedRegistryPeer { ok?: boolean; count?: number; error?: string | null; + /** Explicit policy block flag when the backend sets it. */ + blocked?: boolean; + /** Optional policy reason / code from the backend. */ + policy?: string | null; +} + +/** Optional peer-policy snapshot returned by federated/discover responses (v1.1+). */ +export interface FederationPolicyInfo { + max_hops?: number | null; + allow_private?: boolean | null; + allowlist_active?: boolean | null; + block_private?: boolean | null; + allow?: string[] | null; + deny?: string[] | null; + /** Free-form notes or summary from the backend. */ + message?: string | null; + [key: string]: unknown; } export interface FederatedRegistryResponse { @@ -256,20 +317,26 @@ export interface FederatedRegistryResponse { peers: FederatedRegistryPeer[]; /** Merged catalog; peer-sourced rows may include `peer`. */ items: FederatedRegistryItem[]; + /** Peers added via discover expansion (when present). */ + discovered_peers?: string[]; + /** Backend peer policy summary when present (v1.1+). */ + policy?: FederationPolicyInfo | null; } /** - * GET /registry/federated?q=&source=&peers=&discover= — local + peer registry merge. + * GET /registry/federated?q=&source=&peers=&discover=&max_hops= — local + peer merge. * `peers` is a comma-separated list of extra peer base URLs (from settings or callers). - * When `discover` is true, the backend expands one hop of peer-of-peer URLs. + * When `discover` is true, the backend expands peer-of-peer URLs (hop depth via `max_hops`). */ export async function fetchFederatedRegistry(options?: { q?: string; source?: string; /** Extra peer base URLs (comma-separated string or array). */ peers?: string | string[]; - /** One-hop peer expansion (discover=true). */ + /** Peer expansion (discover=true). */ discover?: boolean; + /** Max discovery hops (0–2); sent as `max_hops` when set. */ + maxHops?: number; }): Promise { const backendUrl = getBackendUrl(); let peersParam: string | undefined; @@ -279,6 +346,10 @@ export async function fetchFederatedRegistry(options?: { } else if (typeof options?.peers === 'string' && options.peers.trim()) { peersParam = options.peers.trim(); } + let maxHops: number | undefined; + if (typeof options?.maxHops === 'number' && Number.isFinite(options.maxHops)) { + maxHops = Math.max(0, Math.min(2, Math.trunc(options.maxHops))); + } const response = await axios.get( `${backendUrl}/registry/federated`, { @@ -288,6 +359,7 @@ export async function fetchFederatedRegistry(options?: { source: options?.source || undefined, peers: peersParam, discover: options?.discover === true ? true : undefined, + max_hops: maxHops, }, } ); @@ -296,40 +368,79 @@ export async function fetchFederatedRegistry(options?: { local: response.data.local ?? { items: [] }, peers: response.data.peers ?? [], items: response.data.items ?? [], + discovered_peers: response.data.discovered_peers, + policy: response.data.policy ?? null, }; } +/** Discovered peer suggestion (string URL or structured entry with policy error). */ +export interface RegistryDiscoverPeer { + url: string; + error?: string | null; + blocked?: boolean; + policy?: string | null; +} + export interface RegistryDiscoverResponse { - /** Discovered peer base URLs (one hop). */ + /** Discovered peer base URLs (one hop / policy-filtered). */ peers: string[]; + /** Structured peer entries when the backend returns objects (v1.1+). */ + peerDetails: RegistryDiscoverPeer[]; /** Optional notes from the backend. */ discovered?: string[]; + /** Backend peer policy summary when present (v1.1+). */ + policy?: FederationPolicyInfo | null; } -function normalizePeerUrlList(raw: unknown): string[] { +function normalizeDiscoverPeerEntry(entry: unknown): RegistryDiscoverPeer | undefined { + if (typeof entry === 'string') { + const url = entry.trim().replace(/\/$/, ''); + return url ? { url } : undefined; + } + if (entry && typeof entry === 'object') { + const obj = entry as { + url?: string; + base_url?: string; + error?: string | null; + blocked?: boolean; + policy?: string | null; + }; + const url = (obj.url || obj.base_url || '').trim().replace(/\/$/, ''); + if (!url) { + return undefined; + } + const error = obj.error ?? null; + const blocked = + obj.blocked === true || isPeerPolicyBlockMessage(error) || isPeerPolicyBlockMessage(obj.policy); + return { + url, + error, + blocked, + policy: obj.policy ?? null, + }; + } + return undefined; +} + +function normalizeDiscoverPeerList(raw: unknown): RegistryDiscoverPeer[] { if (!Array.isArray(raw)) { return []; } - const out: string[] = []; + const out: RegistryDiscoverPeer[] = []; const seen = new Set(); for (const entry of raw) { - let url = ''; - if (typeof entry === 'string') { - url = entry.trim().replace(/\/$/, ''); - } else if (entry && typeof entry === 'object') { - const obj = entry as { url?: string; base_url?: string }; - url = (obj.url || obj.base_url || '').trim().replace(/\/$/, ''); - } - if (url && !seen.has(url)) { - seen.add(url); - out.push(url); + const peer = normalizeDiscoverPeerEntry(entry); + if (!peer || seen.has(peer.url)) { + continue; } + seen.add(peer.url); + out.push(peer); } return out; } /** - * GET /registry/discover — one-hop peer discovery from the local backend. + * GET /registry/discover — peer discovery from the local backend (policy-aware in v1.1+). */ export async function fetchRegistryDiscover(): Promise { const backendUrl = getBackendUrl(); @@ -338,11 +449,26 @@ export async function fetchRegistryDiscover(): Promise { timeout: 60_000 } ); const data = response.data ?? {}; - const peers = normalizePeerUrlList(data.peers ?? data.discovered ?? data.urls); - const discovered = normalizePeerUrlList(data.discovered); + const peerDetails = normalizeDiscoverPeerList( + data.peers ?? data.discovered ?? data.urls + ); + const discoveredDetails = normalizeDiscoverPeerList(data.discovered); + const merged = + peerDetails.length > 0 + ? peerDetails + : discoveredDetails; + const policy = + data.policy && typeof data.policy === 'object' + ? (data.policy as FederationPolicyInfo) + : null; return { - peers: peers.length > 0 ? peers : discovered, - discovered: discovered.length > 0 ? discovered : undefined, + peers: merged.map((p) => p.url), + peerDetails: merged, + discovered: + discoveredDetails.length > 0 + ? discoveredDetails.map((p) => p.url) + : undefined, + policy, }; } @@ -356,22 +482,34 @@ export interface PeerStatus { latency_ms?: number | null; error?: string | null; count?: number | null; + /** Explicit policy block flag when the backend sets it. */ + blocked?: boolean; + policy?: string | null; } export interface PeerStatusListResponse { peers: PeerStatus[]; /** Backend-configured peer URLs (CREER_REGISTRY_PEERS). */ configured: string[]; + /** Backend peer policy summary when present (v1.1+). */ + policy?: FederationPolicyInfo | null; } function normalizePeerStatus(raw: PeerStatus): PeerStatus { + const error = raw.error ?? null; + const blocked = + raw.blocked === true || + isPeerPolicyBlockMessage(error) || + isPeerPolicyBlockMessage(raw.policy); return { url: raw.url || raw.base_url || '', base_url: raw.base_url || raw.url || '', - ok: Boolean(raw.ok), + ok: Boolean(raw.ok) && !blocked, latency_ms: raw.latency_ms ?? null, - error: raw.error ?? null, + error, count: raw.count ?? null, + blocked, + policy: raw.policy ?? null, }; } @@ -387,6 +525,7 @@ export async function fetchPeerStatus(): Promise { return { peers: (response.data.peers ?? []).map(normalizePeerStatus), configured: response.data.configured ?? [], + policy: response.data.policy ?? null, }; } diff --git a/extension/src/registry.ts b/extension/src/registry.ts index 4a91824..1ae6ec9 100644 --- a/extension/src/registry.ts +++ b/extension/src/registry.ts @@ -4,9 +4,12 @@ import { fetchPeerStatus, fetchRegistryDiscover, formatAxiosError, + isPeerPolicyBlockMessage, probePeer, type FederatedRegistryItem, + type FederatedRegistryPeer, type PeerStatus, + type RegistryDiscoverPeer, } from './api'; import { installFromResolvedUrl, @@ -31,6 +34,10 @@ function peerUrlOf(status: PeerStatus): string { function formatPeerHealth(status: PeerStatus): string { const host = peerHostLabel(peerUrlOf(status)) || peerUrlOf(status) || 'peer'; + if (status.blocked || isPeerPolicyBlockMessage(status.error) || isPeerPolicyBlockMessage(status.policy)) { + const reason = status.error || status.policy || 'policy'; + return `$(circle-slash) ${host}: blocked (${reason})`; + } if (status.ok) { const latency = typeof status.latency_ms === 'number' ? ` ${Math.round(status.latency_ms)}ms` : ''; @@ -40,6 +47,115 @@ function formatPeerHealth(status: PeerStatus): string { return `$(error) ${host}${err}`; } +/** + * Heuristic: localhost / loopback / RFC1918 / link-local / .local hosts. + * Used only for client-side warnPrivatePeers UX (backend enforces real policy). + */ +export function looksLikePrivateOrLocalhostHost(urlOrHost: string): boolean { + let host = (urlOrHost || '').trim().toLowerCase(); + if (!host) { + return false; + } + try { + if (host.includes('://') || host.includes('/')) { + host = new URL(host.includes('://') ? host : `http://${host}`).hostname.toLowerCase(); + } + } catch { + // fall through with raw host + } + // Strip IPv6 brackets + if (host.startsWith('[') && host.endsWith(']')) { + host = host.slice(1, -1); + } + + if ( + host === 'localhost' || + host === '127.0.0.1' || + host === '0.0.0.0' || + host === '::1' || + host === '::' || + host.endsWith('.localhost') || + host.endsWith('.local') + ) { + return true; + } + + // IPv4 private / loopback / link-local + if (/^10\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host)) { + return true; + } + if (/^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host)) { + return true; + } + if (/^192\.168\.\d{1,3}\.\d{1,3}$/.test(host)) { + return true; + } + if (/^172\.(1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3}$/.test(host)) { + return true; + } + if (/^169\.254\.\d{1,3}\.\d{1,3}$/.test(host)) { + return true; + } + + // IPv6 ULA (fc00::/7) / link-local (fe80::/10) — simple prefix check + const bare = host.replace(/%.*$/, ''); + if (/^(fc|fd)[0-9a-f]*:/i.test(bare) || /^fe[89ab][0-9a-f]*:/i.test(bare)) { + return true; + } + + return false; +} + +async function confirmPrivatePeerIfNeeded(url: string): Promise { + const warn = vscode.workspace.getConfiguration('creer').get('warnPrivatePeers') !== false; + if (!warn || !looksLikePrivateOrLocalhostHost(url)) { + return true; + } + const host = peerHostLabel(url) || url; + const choice = await vscode.window.showWarningMessage( + `Creer: “${host}” looks like localhost or a private IP. ` + + 'Backend peer policy may block it (SSRF / private IP). Add or probe anyway?', + { modal: true }, + 'Continue', + 'Cancel' + ); + return choice === 'Continue'; +} + +function readFederationMaxHops(): number { + const raw = vscode.workspace.getConfiguration('creer').get('federationMaxHops'); + if (typeof raw !== 'number' || !Number.isFinite(raw)) { + return 1; + } + return Math.max(0, Math.min(2, Math.trunc(raw))); +} + +function isFederatedPeerBlocked(peer: FederatedRegistryPeer): boolean { + return ( + peer.blocked === true || + isPeerPolicyBlockMessage(peer.error) || + isPeerPolicyBlockMessage(peer.policy) + ); +} + +function summarizeFederatedPeerPolicy(peers: FederatedRegistryPeer[]): string | undefined { + if (!peers.length) { + return undefined; + } + const blocked = peers.filter(isFederatedPeerBlocked); + const ok = peers.filter((p) => p.ok && !isFederatedPeerBlocked(p)); + const fail = peers.length - ok.length - blocked.length; + const parts: string[] = []; + parts.push(`${ok.length} peer${ok.length === 1 ? '' : 's'} ok`); + if (blocked.length > 0) { + parts.push(`${blocked.length} blocked by policy`); + } + if (fail > 0) { + parts.push(`${fail} fail`); + } + return parts.join(', '); +} + function parseRegistryPeersSetting(): string[] { const raw = vscode.workspace.getConfiguration('creer').get('registryPeers') || ''; return raw @@ -65,6 +181,7 @@ export async function browseFederatedRegistryCommand( const showPeerStatus = config.get('showPeerStatus') !== false; const registryPeers = config.get('registryPeers') || ''; const federatedDiscover = config.get('federatedDiscover') === true; + const maxHops = readFederationMaxHops(); const q = await vscode.window.showInputBox({ prompt: 'Search federated registry (leave empty for all packs)', @@ -80,12 +197,20 @@ export async function browseFederatedRegistryCommand( try { const statusResp = await fetchPeerStatus(); peerStatuses = statusResp.peers; - const ok = peerStatuses.filter((p) => p.ok).length; - const fail = peerStatuses.length - ok; + const ok = peerStatuses.filter((p) => p.ok && !p.blocked).length; + const blocked = peerStatuses.filter((p) => p.blocked).length; + const fail = peerStatuses.length - ok - blocked; if (peerStatuses.length > 0) { + const parts = [`${ok} ok`]; + if (blocked > 0) { + parts.push(`${blocked} blocked by policy`); + } + if (fail > 0) { + parts.push(`${fail} fail`); + } void vscode.window.showInformationMessage( - `Creer peers: ${ok} ok, ${fail} fail` + - (peerStatuses.some((p) => typeof p.latency_ms === 'number') + `Creer peers: ${parts.join(', ')}` + + (peerStatuses.some((p) => p.ok && typeof p.latency_ms === 'number') ? ` · ${peerStatuses .filter((p) => p.ok && typeof p.latency_ms === 'number') .map((p) => `${peerHostLabel(peerUrlOf(p))}:${Math.round(p.latency_ms!)}ms`) @@ -102,12 +227,13 @@ export async function browseFederatedRegistryCommand( let items: FederatedRegistryItem[]; let peerCount = 0; + let federatedPeers: FederatedRegistryPeer[] = []; try { const federated = await vscode.window.withProgress( { location: vscode.ProgressLocation.Notification, title: federatedDiscover - ? 'Creer: loading federated registry (discover)…' + ? `Creer: loading federated registry (discover, max_hops=${maxHops})…` : 'Creer: loading federated registry…', cancellable: false, }, @@ -116,10 +242,19 @@ export async function browseFederatedRegistryCommand( q: q.trim() || undefined, peers: registryPeers.trim() || undefined, discover: federatedDiscover || undefined, + maxHops, }) ); items = federated.items; - peerCount = federated.peers?.length ?? 0; + federatedPeers = federated.peers ?? []; + peerCount = federatedPeers.length; + const policySummary = summarizeFederatedPeerPolicy(federatedPeers); + const hasPeerErrors = federatedPeers.some( + (p) => !p.ok || Boolean(p.error) || isFederatedPeerBlocked(p) + ); + if (policySummary && (federated.policy || hasPeerErrors)) { + void vscode.window.showInformationMessage(`Creer federation: ${policySummary}`); + } } catch (err) { const message = formatAxiosError(err, 'Federated registry unavailable'); void vscode.window.showWarningMessage( @@ -150,11 +285,13 @@ export async function browseFederatedRegistryCommand( picks.push({ label: formatPeerHealth(status), description: peerUrlOf(status), - detail: status.ok - ? typeof status.count === 'number' - ? `${status.count} packs` - : 'reachable' - : status.error || 'unreachable', + detail: status.blocked + ? status.error || status.policy || 'blocked by policy' + : status.ok + ? typeof status.count === 'number' + ? `${status.count} packs` + : 'reachable' + : status.error || 'unreachable', }); } picks.push({ @@ -231,13 +368,14 @@ export async function browseFederatedRegistryCommand( } /** - * Discover one-hop peers via GET /registry/discover, falling back to federated?discover=true. + * Discover peers via GET /registry/discover, falling back to federated?discover=true. + * Returns structured entries so policy-blocked suggestions can be shown grayed. */ -async function discoverPeerUrls(): Promise { +async function discoverPeerDetails(): Promise { try { const discovered = await fetchRegistryDiscover(); - if (discovered.peers.length > 0) { - return discovered.peers; + if (discovered.peerDetails.length > 0) { + return discovered.peerDetails; } } catch { // Soft-fail: try federated discover instead. @@ -246,14 +384,38 @@ async function discoverPeerUrls(): Promise { const registryPeers = vscode.workspace .getConfiguration('creer') .get('registryPeers') || ''; + const maxHops = readFederationMaxHops(); const federated = await fetchFederatedRegistry({ peers: registryPeers.trim() || undefined, discover: true, + maxHops, }); - const fromPeers = (federated.peers ?? []) - .map((p) => (p.base_url || '').trim().replace(/\/$/, '')) - .filter(Boolean); - return [...new Set(fromPeers)]; + const fromPeers = (federated.peers ?? []).map((p) => { + const url = (p.base_url || '').trim().replace(/\/$/, ''); + const blocked = isFederatedPeerBlocked(p); + return { + url, + error: p.error ?? null, + blocked, + policy: p.policy ?? null, + } satisfies RegistryDiscoverPeer; + }).filter((p) => Boolean(p.url)); + + const discoveredExtra = (federated.discovered_peers ?? []) + .map((u) => (typeof u === 'string' ? u.trim().replace(/\/$/, '') : '')) + .filter(Boolean) + .map((url) => ({ url } satisfies RegistryDiscoverPeer)); + + const seen = new Set(); + const out: RegistryDiscoverPeer[] = []; + for (const peer of [...fromPeers, ...discoveredExtra]) { + if (seen.has(peer.url)) { + continue; + } + seen.add(peer.url); + out.push(peer); + } + return out; } /** @@ -294,6 +456,9 @@ export async function manageRegistryPeersCommand( livePeers .map((p) => { const host = peerHostLabel(peerUrlOf(p)) || peerUrlOf(p); + if (p.blocked) { + return `${host} blocked`; + } return p.ok ? `${host} ok${typeof p.latency_ms === 'number' ? ` ${Math.round(p.latency_ms)}ms` : ''}` : `${host} fail`; @@ -332,8 +497,8 @@ export async function manageRegistryPeersCommand( action: 'probe', }, { - label: '$(search) Discover peers (one hop)', - description: 'GET /registry/discover or federated?discover=true — offer to add', + label: '$(search) Discover peers', + description: 'GET /registry/discover or federated?discover=true — offer to add (policy-aware)', action: 'discover', }, ]; @@ -363,6 +528,10 @@ export async function manageRegistryPeersCommand( return; } + if (!(await confirmPrivatePeerIfNeeded(trimmed))) { + return; + } + let ok = true; let probeError: string | undefined; try { @@ -374,8 +543,14 @@ export async function manageRegistryPeersCommand( }, () => probePeer(trimmed, { token }) ); - ok = result.ok; - probeError = result.error || undefined; + ok = result.ok && !result.blocked; + probeError = result.error || result.policy || undefined; + if (result.blocked || isPeerPolicyBlockMessage(probeError)) { + void vscode.window.showErrorMessage( + `Creer: peer blocked by policy${probeError ? ` — ${probeError}` : ''}. Not added.` + ); + return; + } if (ok) { const latency = typeof result.latency_ms === 'number' @@ -388,6 +563,12 @@ export async function manageRegistryPeersCommand( } catch (err) { // Soft-fail: if probe endpoint missing, still allow adding after confirm. const message = formatAxiosError(err, 'Probe failed'); + if (isPeerPolicyBlockMessage(message)) { + void vscode.window.showErrorMessage( + `Creer: peer blocked by policy — ${message}. Not added.` + ); + return; + } const choice = await vscode.window.showWarningMessage( `Creer: could not probe peer (${message}). Add anyway?`, 'Add', @@ -458,6 +639,24 @@ export async function manageRegistryPeersCommand( return; } + // Warn once if any target looks private + const privateTargets = targets.filter(looksLikePrivateOrLocalhostHost); + if (privateTargets.length > 0) { + const warn = + vscode.workspace.getConfiguration('creer').get('warnPrivatePeers') !== false; + if (warn) { + const choice = await vscode.window.showWarningMessage( + `Creer: ${privateTargets.length} peer(s) look like localhost/private IPs. Probe anyway?`, + { modal: true }, + 'Continue', + 'Cancel' + ); + if (choice !== 'Continue') { + return; + } + } + } + const results: string[] = []; await vscode.window.withProgress( { @@ -490,15 +689,15 @@ export async function manageRegistryPeersCommand( } if (picked.action === 'discover') { - let discovered: string[] = []; + let discovered: RegistryDiscoverPeer[] = []; try { discovered = await vscode.window.withProgress( { location: vscode.ProgressLocation.Notification, - title: 'Creer: discovering peers (one hop)…', + title: 'Creer: discovering peers…', cancellable: false, }, - () => discoverPeerUrls() + () => discoverPeerDetails() ); } catch (err) { const message = formatAxiosError(err, 'Discovery unavailable'); @@ -514,16 +713,39 @@ export async function manageRegistryPeersCommand( } const existing = new Set(parseRegistryPeersSetting()); - const picks = discovered.map((url) => ({ - label: url, - description: existing.has(url) - ? 'already in creer.registryPeers' - : peerHostLabel(url), - picked: !existing.has(url), - })); + type DiscoverPick = vscode.QuickPickItem & { + peerUrl?: string; + blocked?: boolean; + }; + + const picks: DiscoverPick[] = discovered.map((peer) => { + const blocked = + peer.blocked === true || + isPeerPolicyBlockMessage(peer.error) || + isPeerPolicyBlockMessage(peer.policy); + if (blocked) { + return { + label: `$(circle-slash) ${peer.url}`, + description: 'blocked', + detail: peer.error || peer.policy || 'blocked by policy', + peerUrl: peer.url, + blocked: true, + }; + } + return { + label: peer.url, + description: existing.has(peer.url) + ? 'already in creer.registryPeers' + : peerHostLabel(peer.url), + peerUrl: peer.url, + blocked: false, + picked: !existing.has(peer.url), + }; + }); const selected = await vscode.window.showQuickPick(picks, { - placeHolder: 'Select discovered peers to add to creer.registryPeers', + placeHolder: + 'Select discovered peers to add (blocked suggestions are shown grayed and skipped)', ignoreFocusOut: true, canPickMany: true, }); @@ -533,9 +755,20 @@ export async function manageRegistryPeersCommand( const next = parseRegistryPeersSetting(); let added = 0; + let skippedBlocked = 0; for (const item of selected) { - const url = item.label.trim().replace(/\/$/, ''); - if (url && !next.includes(url)) { + if (item.blocked) { + skippedBlocked += 1; + continue; + } + const url = (item.peerUrl || item.label).trim().replace(/\/$/, ''); + if (!url) { + continue; + } + if (!(await confirmPrivatePeerIfNeeded(url))) { + continue; + } + if (!next.includes(url)) { next.push(url); added += 1; } @@ -543,10 +776,14 @@ export async function manageRegistryPeersCommand( if (added > 0) { await saveRegistryPeersSetting(next); } + const suffix = + skippedBlocked > 0 + ? ` Skipped ${skippedBlocked} blocked by policy.` + : ''; void vscode.window.showInformationMessage( added > 0 - ? `Creer: added ${added} discovered peer(s) to creer.registryPeers.` - : 'Creer: selected peers were already in creer.registryPeers.' + ? `Creer: added ${added} discovered peer(s) to creer.registryPeers.${suffix}` + : `Creer: selected peers were already in creer.registryPeers or blocked.${suffix}` ); } }