Skip to content

Commit b798515

Browse files
committed
feat: Pluggable cache with redis as fallback if ENV var is set
1 parent 8254311 commit b798515

9 files changed

Lines changed: 243 additions & 66 deletions

File tree

README.md

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ After the test completes, you can run `npx playwright show-report` to see a deta
103103

104104
- **Core Execution**`runSteps()` and `runUserFlow()` for flexible test orchestration in natural language, with smart caching and auto-healing
105105
- **Multi-Model Assertion Engine** — Consensus-based validation using Claude and Gemini, with an arbiter model to resolve disagreements
106-
- **Redis-Based Step Caching** — Cache-first execution with AI fallback and automatic self-healing when cached steps fail
106+
- **Pluggable Step Caching** — Cache-first execution with AI fallback and automatic self-healing. Supports Redis, file-based, or custom cache backends.
107107
- **Configurable AI Models** — 8 dedicated model slots for step execution, assertions, extraction, and more
108108
- **AI Gateway Support** — Route requests through Vercel AI Gateway, OpenRouter, Cloudflare AI Gateway, or connect directly to provider SDKs
109109
- **Dynamic Placeholders** — Inject values at runtime with `{{run.*}}`, `{{global.*}}`, `{{data.*}}`, and `{{email.*}}` expressions for repeatable and data-driven tests
@@ -184,6 +184,8 @@ configure({
184184

185185
| Variable | Required | Default | Description |
186186
|----------|----------|---------|-------------|
187+
| `CACHE_PROVIDER` | No | - | Cache backend: `redis`, `file`, or `none`. Falls back to Redis if `REDIS_URL` is set. |
188+
| `CACHE_DIR` | No | `.passmark-cache` | Directory for file-based cache (when `CACHE_PROVIDER=file`) |
187189
| `REDIS_URL` | No | - | Redis connection URL for step caching and global state |
188190
| `ANTHROPIC_API_KEY` | Yes | - | Anthropic API key for Claude models |
189191
| `GOOGLE_GENERATIVE_AI_API_KEY` | Yes | - | Google API key for Gemini models |
@@ -212,7 +214,35 @@ All models are configurable via `configure({ ai: { models: { ... } } })`:
212214

213215
## Caching
214216

215-
Passmark caches successful step actions in Redis. On subsequent runs, cached steps execute directly without AI calls, dramatically reducing latency and cost.
217+
Passmark caches successful step actions so that subsequent runs execute directly without AI calls, dramatically reducing latency and cost. The cache backend is pluggable — choose between Redis, file-based, or no caching at all.
218+
219+
### Cache Providers
220+
221+
Set the `CACHE_PROVIDER` environment variable to select a backend:
222+
223+
| Provider | `CACHE_PROVIDER` | Additional Config | Description |
224+
|----------|-------------------|-------------------|-------------|
225+
| **Redis** | `redis` | `REDIS_URL` | Uses Redis via ioredis. Default when `REDIS_URL` is set. |
226+
| **File** | `file` | `CACHE_DIR` (optional, defaults to `.passmark-cache`) | JSON files on disk. No external dependencies — great for local development and CI. |
227+
| **None** | `none` || Disables caching entirely. Every step uses AI execution. |
228+
229+
For backwards compatibility, if `CACHE_PROVIDER` is not set, Passmark will use Redis when `REDIS_URL` is present, otherwise caching is disabled.
230+
231+
### Custom Cache Store
232+
233+
You can implement a custom cache backend by conforming to the `CacheStore` interface:
234+
235+
```typescript
236+
import { CacheStore } from "passmark";
237+
238+
interface CacheStore {
239+
hgetall(key: string): Promise<Record<string, string>>;
240+
hset(key: string, values: Record<string, string>): Promise<void>;
241+
expire(key: string, seconds: number): Promise<void>;
242+
}
243+
```
244+
245+
### Caching Behavior
216246

217247
- Steps are cached by `userFlow` + `step.description`
218248
- Set `bypassCache: true` on individual steps or the entire run to force AI execution

src/__tests__/data-cache.test.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { describe, it, expect, vi, beforeEach } from "vitest";
22

3-
vi.mock("../redis", () => ({
4-
redis: { hgetall: vi.fn(), hset: vi.fn(), expire: vi.fn() },
3+
vi.mock("../cache", () => ({
4+
cache: { hgetall: vi.fn(), hset: vi.fn(), expire: vi.fn() },
55
}));
66

77
vi.mock("../email", () => ({

src/__tests__/integration/run-steps.test.ts

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@ import { describe, it, expect, vi, beforeEach } from "vitest";
33
// Mock instrumentation (imported as side effect)
44
vi.mock("../../instrumentation", () => ({ axiomEnabled: false }));
55

6-
// Mock Redis
7-
vi.mock("../../redis", () => ({
8-
redis: {
6+
// Mock Cache
7+
vi.mock("../../cache", () => ({
8+
cache: {
99
hgetall: vi.fn().mockResolvedValue({}),
10-
hset: vi.fn().mockResolvedValue("OK"),
11-
expire: vi.fn().mockResolvedValue(1),
10+
hset: vi.fn().mockResolvedValue(undefined),
11+
expire: vi.fn().mockResolvedValue(undefined),
1212
},
1313
}));
1414

@@ -89,7 +89,7 @@ vi.mock("../../utils/secure-script-runner", () => ({
8989

9090
import { runSteps } from "../../index";
9191
import { resetConfig } from "../../config";
92-
import { redis } from "../../redis";
92+
import { cache } from "../../cache";
9393
import { generateText } from "ai";
9494
import type { Page } from "@playwright/test";
9595
import type { Step } from "../../types";
@@ -121,8 +121,8 @@ describe("runSteps", () => {
121121
beforeEach(() => {
122122
vi.clearAllMocks();
123123
resetConfig();
124-
// Reset redis mock to default empty
125-
vi.mocked(redis!.hgetall).mockResolvedValue({});
124+
// Reset cache mock to default empty
125+
vi.mocked(cache!.hgetall).mockResolvedValue({});
126126
});
127127

128128
it("executes a simple step", async () => {
@@ -198,8 +198,8 @@ describe("runSteps", () => {
198198
const page = createMockPage();
199199
const steps: Step[] = [{ description: "Click submit" }];
200200

201-
// Mock redis to return cached step data
202-
vi.mocked(redis!.hgetall).mockResolvedValue({
201+
// Mock cache to return cached step data
202+
vi.mocked(cache!.hgetall).mockResolvedValue({
203203
locator: 'getByRole("button", { name: "Submit" })',
204204
action: "click",
205205
description: "Submit button",
@@ -220,8 +220,8 @@ describe("runSteps", () => {
220220
const page = createMockPage();
221221
const steps: Step[] = [{ description: "Click submit" }];
222222

223-
// Mock redis to return cached step data
224-
vi.mocked(redis!.hgetall).mockResolvedValue({
223+
// Mock cache to return cached step data
224+
vi.mocked(cache!.hgetall).mockResolvedValue({
225225
locator: 'getByRole("button", { name: "Submit" })',
226226
action: "click",
227227
description: "Submit button",
@@ -291,8 +291,8 @@ describe("runSteps", () => {
291291
it("bypasses cache for individual step when step.bypassCache is true", async () => {
292292
const page = createMockPage();
293293

294-
// Mock redis to return cached data
295-
vi.mocked(redis!.hgetall).mockResolvedValue({
294+
// Mock cache to return cached data
295+
vi.mocked(cache!.hgetall).mockResolvedValue({
296296
locator: 'getByRole("button", { name: "Go" })',
297297
action: "click",
298298
description: "Go button",

src/cache.ts

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
import { logger } from "./logger";
2+
3+
// =============================================================================
4+
// Cache Store Interface
5+
// =============================================================================
6+
7+
/**
8+
* Interface for a hash-based cache store.
9+
* Implementations must support hash get/set and key expiration.
10+
*/
11+
export interface CacheStore {
12+
hgetall(key: string): Promise<Record<string, string>>;
13+
hset(key: string, values: Record<string, string>): Promise<void>;
14+
expire(key: string, seconds: number): Promise<void>;
15+
}
16+
17+
// =============================================================================
18+
// Redis Store
19+
// =============================================================================
20+
21+
class RedisStore implements CacheStore {
22+
private client: import("ioredis").default;
23+
24+
constructor(url: string) {
25+
// eslint-disable-next-line @typescript-eslint/no-require-imports
26+
const Redis = require("ioredis") as typeof import("ioredis").default;
27+
this.client = new Redis(url);
28+
}
29+
30+
async hgetall(key: string): Promise<Record<string, string>> {
31+
return this.client.hgetall(key);
32+
}
33+
34+
async hset(key: string, values: Record<string, string>): Promise<void> {
35+
await this.client.hset(key, values);
36+
}
37+
38+
async expire(key: string, seconds: number): Promise<void> {
39+
await this.client.expire(key, seconds);
40+
}
41+
}
42+
43+
// =============================================================================
44+
// File Store
45+
// =============================================================================
46+
47+
import * as fs from "fs";
48+
import * as path from "path";
49+
50+
class FileStore implements CacheStore {
51+
private dir: string;
52+
53+
constructor(dir: string) {
54+
this.dir = dir;
55+
if (!fs.existsSync(dir)) {
56+
fs.mkdirSync(dir, { recursive: true });
57+
}
58+
}
59+
60+
private filePath(key: string): string {
61+
// Encode key to a safe filename
62+
const safeKey = encodeURIComponent(key);
63+
return path.join(this.dir, `${safeKey}.json`);
64+
}
65+
66+
private read(key: string): { data: Record<string, string>; expiresAt?: number } | null {
67+
const fp = this.filePath(key);
68+
if (!fs.existsSync(fp)) return null;
69+
70+
try {
71+
const raw = JSON.parse(fs.readFileSync(fp, "utf-8"));
72+
73+
// Check expiration
74+
if (raw.expiresAt && Date.now() > raw.expiresAt) {
75+
fs.unlinkSync(fp);
76+
return null;
77+
}
78+
79+
return raw;
80+
} catch {
81+
return null;
82+
}
83+
}
84+
85+
private write(key: string, entry: { data: Record<string, string>; expiresAt?: number }): void {
86+
const fp = this.filePath(key);
87+
fs.writeFileSync(fp, JSON.stringify(entry), "utf-8");
88+
}
89+
90+
async hgetall(key: string): Promise<Record<string, string>> {
91+
const entry = this.read(key);
92+
return entry?.data ?? {};
93+
}
94+
95+
async hset(key: string, values: Record<string, string>): Promise<void> {
96+
const existing = this.read(key);
97+
const merged = { ...(existing?.data ?? {}), ...values };
98+
this.write(key, { data: merged, expiresAt: existing?.expiresAt });
99+
}
100+
101+
async expire(key: string, seconds: number): Promise<void> {
102+
const existing = this.read(key);
103+
if (!existing) return;
104+
this.write(key, { ...existing, expiresAt: Date.now() + seconds * 1000 });
105+
}
106+
}
107+
108+
// =============================================================================
109+
// Factory
110+
// =============================================================================
111+
112+
/**
113+
* Creates the cache store based on environment variables.
114+
*
115+
* CACHE_PROVIDER selects the backend:
116+
* - "redis" (default when REDIS_URL is set): uses Redis via ioredis
117+
* - "file": uses JSON files on disk at CACHE_DIR (defaults to .passmark-cache)
118+
* - "none": disables caching entirely
119+
*
120+
* For backwards compatibility, if CACHE_PROVIDER is not set:
121+
* - If REDIS_URL is set → uses Redis
122+
* - Otherwise → caching is disabled (null)
123+
*/
124+
function createCacheStore(): CacheStore | null {
125+
const provider = process.env.CACHE_PROVIDER?.toLowerCase();
126+
127+
if (provider === "none") {
128+
logger.warn("Cache provider set to 'none'. Caching is disabled.");
129+
return null;
130+
}
131+
132+
if (provider === "file") {
133+
const dir = process.env.CACHE_DIR || ".passmark-cache";
134+
logger.info(`Using file-based cache at: ${dir}`);
135+
return new FileStore(dir);
136+
}
137+
138+
if (provider === "redis" || (!provider && process.env.REDIS_URL)) {
139+
if (!process.env.REDIS_URL) {
140+
logger.warn("CACHE_PROVIDER is 'redis' but REDIS_URL is not set. Caching is disabled.");
141+
return null;
142+
}
143+
logger.info("Using Redis cache.");
144+
return new RedisStore(process.env.REDIS_URL);
145+
}
146+
147+
if (provider) {
148+
logger.warn(`Unknown CACHE_PROVIDER '${provider}'. Caching is disabled.`);
149+
return null;
150+
}
151+
152+
// No CACHE_PROVIDER and no REDIS_URL
153+
logger.warn(
154+
"No cache provider configured. Set CACHE_PROVIDER=redis|file|none or REDIS_URL. " +
155+
"Step caching, global placeholders, and project data are disabled.",
156+
);
157+
return null;
158+
}
159+
160+
export const cache: CacheStore | null = createCacheStore();

src/constants.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,5 +22,5 @@ export const MAX_RETRIES = 3;
2222
// Thinking budgets (tokens)
2323
export const THINKING_BUDGET_DEFAULT = 1024;
2424

25-
// Redis
25+
// Cache
2626
export const GLOBAL_VALUES_TTL_SECONDS = 86400;

0 commit comments

Comments
 (0)