Skip to content

Commit 2687f16

Browse files
deacon-mpdeacon-mp
authored
Add exhaustive Playwright E2E tests for stockpile UI (#602)
* Add exhaustive Playwright E2E tests for stockpile plugin UI Covers plugin page load, ability/adversary browsing, planner display, obfuscator configuration, and error states. Tests run against a full Caldera instance via CALDERA_URL env var. * Address Copilot review feedback on e2e tests - Replace waitUntil: 'networkidle' with 'domcontentloaded' plus explicit heading locator guard in navigateToStockpile helper - Fix obfuscator test to assert specific stockpile obfuscator names (base64, caesar, steganography) instead of just non-empty length - Add dedicated adversary test asserting both name AND description fields - Replace brittle positional selectors (first()/nth(1)) with card-label-anchored locators (.card with hasText: 'abilities'/'adversaries')
1 parent 1b4f7c5 commit 2687f16

File tree

3 files changed

+327
-0
lines changed

3 files changed

+327
-0
lines changed

‎package.json‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"name": "stockpile-e2e-tests",
3+
"version": "1.0.0",
4+
"private": true,
5+
"scripts": {
6+
"test:e2e": "npx playwright test",
7+
"test:e2e:headed": "npx playwright test --headed",
8+
"test:e2e:debug": "npx playwright test --debug"
9+
},
10+
"devDependencies": {
11+
"@playwright/test": "^1.49.0"
12+
}
13+
}

‎playwright.config.js‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// @ts-check
2+
const { defineConfig, devices } = require('@playwright/test');
3+
4+
const CALDERA_URL = process.env.CALDERA_URL || 'http://localhost:8888';
5+
6+
module.exports = defineConfig({
7+
testDir: './tests/e2e',
8+
fullyParallel: false,
9+
forbidOnly: !!process.env.CI,
10+
retries: process.env.CI ? 2 : 0,
11+
workers: 1,
12+
reporter: [['html', { open: 'never' }], ['list']],
13+
timeout: 60_000,
14+
use: {
15+
baseURL: CALDERA_URL,
16+
trace: 'on-first-retry',
17+
screenshot: 'only-on-failure',
18+
headless: true,
19+
httpCredentials: {
20+
username: process.env.CALDERA_USER || 'admin',
21+
password: process.env.CALDERA_PASS || 'admin',
22+
},
23+
},
24+
projects: [
25+
{
26+
name: 'chromium',
27+
use: { ...devices['Desktop Chrome'] },
28+
},
29+
],
30+
});

‎tests/e2e/stockpile.spec.js‎

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,284 @@
1+
// @ts-check
2+
const { test, expect } = require('@playwright/test');
3+
4+
const CALDERA_URL = process.env.CALDERA_URL || 'http://localhost:8888';
5+
const PLUGIN_ROUTE = '/#/plugins/stockpile';
6+
7+
// ---------------------------------------------------------------------------
8+
// Helper: navigate to the stockpile plugin page inside magma.
9+
// Uses 'domcontentloaded' (faster than 'networkidle') and waits for the
10+
// Stockpile heading to confirm the Vue component has rendered.
11+
// ---------------------------------------------------------------------------
12+
async function navigateToStockpile(page) {
13+
await page.goto(`${CALDERA_URL}${PLUGIN_ROUTE}`, { waitUntil: 'domcontentloaded' });
14+
// Explicit UI guard: wait until the plugin heading is present before proceeding.
15+
await page.locator('h2', { hasText: 'Stockpile' }).waitFor({ state: 'visible', timeout: 15_000 });
16+
}
17+
18+
// ===========================================================================
19+
// 1. Plugin page loads
20+
// ===========================================================================
21+
test.describe('Stockpile plugin page load', () => {
22+
test('should display the Stockpile heading', async ({ page }) => {
23+
await navigateToStockpile(page);
24+
const heading = page.locator('h2', { hasText: 'Stockpile' });
25+
await expect(heading).toBeVisible({ timeout: 15_000 });
26+
});
27+
28+
test('should display introductory description text', async ({ page }) => {
29+
await navigateToStockpile(page);
30+
await expect(
31+
page.locator('text=The stockpile plugin contains a collection of TTPs')
32+
).toBeVisible({ timeout: 15_000 });
33+
});
34+
35+
test('should render ability count card', async ({ page }) => {
36+
await navigateToStockpile(page);
37+
const abilityCard = page.locator('.card', { hasText: 'abilities' });
38+
await expect(abilityCard).toBeVisible({ timeout: 15_000 });
39+
});
40+
41+
test('should render adversary count card', async ({ page }) => {
42+
await navigateToStockpile(page);
43+
const adversaryCard = page.locator('.card', { hasText: 'adversaries' });
44+
await expect(adversaryCard).toBeVisible({ timeout: 15_000 });
45+
});
46+
47+
test('should show numeric ability count or placeholder', async ({ page }) => {
48+
await navigateToStockpile(page);
49+
// Anchor to the abilities card by label rather than positional index.
50+
const abilityCard = page.locator('.card', { hasText: 'abilities' });
51+
const abilityCount = abilityCard.locator('h1.is-size-1');
52+
await expect(abilityCount).toBeVisible({ timeout: 15_000 });
53+
const text = await abilityCount.textContent();
54+
expect(text?.trim()).toMatch(/^(\d+|---)$/);
55+
});
56+
57+
test('should show numeric adversary count or placeholder', async ({ page }) => {
58+
await navigateToStockpile(page);
59+
// Anchor to the adversaries card by label rather than positional index.
60+
const adversaryCard = page.locator('.card', { hasText: 'adversaries' });
61+
const adversaryCount = adversaryCard.locator('h1.is-size-1');
62+
await expect(adversaryCount).toBeVisible({ timeout: 15_000 });
63+
const text = await adversaryCount.textContent();
64+
expect(text?.trim()).toMatch(/^(\d+|---)$/);
65+
});
66+
67+
test('should have a horizontal rule separator', async ({ page }) => {
68+
await navigateToStockpile(page);
69+
await expect(page.locator('hr')).toBeVisible({ timeout: 15_000 });
70+
});
71+
});
72+
73+
// ===========================================================================
74+
// 2. Ability browsing within stockpile context
75+
// ===========================================================================
76+
test.describe('Stockpile ability browsing', () => {
77+
test('should have a "View Abilities" link pointing to abilities page with stockpile filter', async ({ page }) => {
78+
await navigateToStockpile(page);
79+
const viewAbilitiesBtn = page.locator('a', { hasText: 'View Abilities' });
80+
await expect(viewAbilitiesBtn).toBeVisible({ timeout: 15_000 });
81+
const href = await viewAbilitiesBtn.getAttribute('href');
82+
expect(href).toContain('/abilities');
83+
expect(href).toContain('plugin=stockpile');
84+
});
85+
86+
test('should have a "View Adversaries" link pointing to adversaries page', async ({ page }) => {
87+
await navigateToStockpile(page);
88+
const viewAdversariesBtn = page.locator('a', { hasText: 'View Adversaries' });
89+
await expect(viewAdversariesBtn).toBeVisible({ timeout: 15_000 });
90+
const href = await viewAdversariesBtn.getAttribute('href');
91+
expect(href).toContain('/adversaries');
92+
});
93+
94+
test('clicking "View Abilities" navigates to abilities page', async ({ page }) => {
95+
await navigateToStockpile(page);
96+
const viewAbilitiesBtn = page.locator('a', { hasText: 'View Abilities' });
97+
await viewAbilitiesBtn.click();
98+
await page.waitForURL(/abilities/, { timeout: 15_000 });
99+
expect(page.url()).toContain('abilities');
100+
});
101+
102+
test('clicking "View Adversaries" navigates to adversaries page', async ({ page }) => {
103+
await navigateToStockpile(page);
104+
const viewAdversariesBtn = page.locator('a', { hasText: 'View Adversaries' });
105+
await viewAdversariesBtn.click();
106+
await page.waitForURL(/adversaries/, { timeout: 15_000 });
107+
expect(page.url()).toContain('adversaries');
108+
});
109+
110+
test('ability card should contain a right-arrow icon', async ({ page }) => {
111+
await navigateToStockpile(page);
112+
const abilityCard = page.locator('.card', { hasText: 'abilities' });
113+
const icon = abilityCard.locator('.icon');
114+
await expect(icon).toBeVisible({ timeout: 15_000 });
115+
});
116+
117+
test('adversary card should contain a right-arrow icon', async ({ page }) => {
118+
await navigateToStockpile(page);
119+
const adversaryCard = page.locator('.card', { hasText: 'adversaries' });
120+
const icon = adversaryCard.locator('.icon');
121+
await expect(icon).toBeVisible({ timeout: 15_000 });
122+
});
123+
});
124+
125+
// ===========================================================================
126+
// 3. Planner display (stockpile ships planners visible through the main app)
127+
// ===========================================================================
128+
test.describe('Stockpile planner display', () => {
129+
test('should load planners API without error', async ({ page }) => {
130+
const response = await page.request.get(`${CALDERA_URL}/api/v2/planners`);
131+
expect(response.ok()).toBeTruthy();
132+
const planners = await response.json();
133+
expect(Array.isArray(planners)).toBeTruthy();
134+
});
135+
136+
test('should include stockpile-provided planners in API response', async ({ page }) => {
137+
const response = await page.request.get(`${CALDERA_URL}/api/v2/planners`);
138+
const planners = await response.json();
139+
// Stockpile ships planners like atomic, batch, buckets
140+
const plannerNames = planners.map((p) => p.name?.toLowerCase() || '');
141+
const hasStockpilePlanner = plannerNames.some(
142+
(n) => n.includes('atomic') || n.includes('batch') || n.includes('buckets')
143+
);
144+
expect(hasStockpilePlanner).toBeTruthy();
145+
});
146+
147+
test('planners should have required fields', async ({ page }) => {
148+
const response = await page.request.get(`${CALDERA_URL}/api/v2/planners`);
149+
const planners = await response.json();
150+
for (const planner of planners) {
151+
expect(planner).toHaveProperty('name');
152+
expect(planner).toHaveProperty('id');
153+
}
154+
});
155+
});
156+
157+
// ===========================================================================
158+
// 4. Obfuscator selection / configuration
159+
// ===========================================================================
160+
test.describe('Stockpile obfuscator configuration', () => {
161+
test('should load obfuscators API without error', async ({ page }) => {
162+
const response = await page.request.get(`${CALDERA_URL}/api/v2/obfuscators`);
163+
expect(response.ok()).toBeTruthy();
164+
const obfuscators = await response.json();
165+
expect(Array.isArray(obfuscators)).toBeTruthy();
166+
});
167+
168+
test('should include plain-text obfuscator', async ({ page }) => {
169+
const response = await page.request.get(`${CALDERA_URL}/api/v2/obfuscators`);
170+
const obfuscators = await response.json();
171+
const names = obfuscators.map((o) => o.name?.toLowerCase() || '');
172+
expect(names).toContain('plain-text');
173+
});
174+
175+
test('obfuscators should have name and description', async ({ page }) => {
176+
const response = await page.request.get(`${CALDERA_URL}/api/v2/obfuscators`);
177+
const obfuscators = await response.json();
178+
for (const obf of obfuscators) {
179+
expect(obf).toHaveProperty('name');
180+
expect(typeof obf.name).toBe('string');
181+
}
182+
});
183+
184+
test('should include base64 or caesar obfuscator from stockpile', async ({ page }) => {
185+
const response = await page.request.get(`${CALDERA_URL}/api/v2/obfuscators`);
186+
const obfuscators = await response.json();
187+
// Stockpile ships base64_jumble, base64_no_padding, caesar_cipher, and steganography.
188+
// Verify at least one of the known stockpile obfuscator names is present.
189+
const names = obfuscators.map((o) => o.name?.toLowerCase() || '');
190+
const hasStockpileObfuscator = names.some(
191+
(n) => n.includes('base64') || n.includes('caesar') || n.includes('steganography')
192+
);
193+
expect(hasStockpileObfuscator).toBeTruthy();
194+
});
195+
});
196+
197+
// ===========================================================================
198+
// 5. Adversary listing
199+
// ===========================================================================
200+
test.describe('Stockpile adversary listing', () => {
201+
test('adversaries API should return objects with name and description', async ({ page }) => {
202+
const response = await page.request.get(`${CALDERA_URL}/api/v2/adversaries`);
203+
expect(response.ok()).toBeTruthy();
204+
const adversaries = await response.json();
205+
expect(Array.isArray(adversaries)).toBeTruthy();
206+
// Every adversary must expose both name and description fields.
207+
for (const adv of adversaries) {
208+
expect(adv).toHaveProperty('name');
209+
expect(adv).toHaveProperty('description');
210+
}
211+
});
212+
});
213+
214+
// ===========================================================================
215+
// 6. Error states
216+
// ===========================================================================
217+
test.describe('Stockpile error states', () => {
218+
test('should handle invalid plugin route gracefully', async ({ page }) => {
219+
const resp = await page.goto(`${CALDERA_URL}/#/plugins/nonexistent-plugin`, {
220+
waitUntil: 'domcontentloaded',
221+
});
222+
// The app should still load (Vue router fallback) even if plugin doesn't exist
223+
expect(resp?.status()).toBeLessThan(500);
224+
});
225+
226+
test('should handle abilities API failure gracefully', async ({ page }) => {
227+
// Intercept abilities API and force a failure
228+
await page.route('**/api/v2/abilities', (route) =>
229+
route.fulfill({ status: 500, body: 'Internal Server Error' })
230+
);
231+
await navigateToStockpile(page);
232+
// Page should still render without crashing
233+
const heading = page.locator('h2', { hasText: 'Stockpile' });
234+
await expect(heading).toBeVisible({ timeout: 15_000 });
235+
});
236+
237+
test('should show placeholder when abilities API returns empty', async ({ page }) => {
238+
await page.route('**/api/v2/abilities', (route) =>
239+
route.fulfill({
240+
status: 200,
241+
contentType: 'application/json',
242+
body: JSON.stringify([]),
243+
})
244+
);
245+
await navigateToStockpile(page);
246+
// Count should show "---" placeholder when 0 stockpile abilities; anchor by label.
247+
const abilityCard = page.locator('.card', { hasText: 'abilities' });
248+
const abilityCount = abilityCard.locator('h1.is-size-1');
249+
await expect(abilityCount).toBeVisible({ timeout: 15_000 });
250+
const text = await abilityCount.textContent();
251+
expect(text?.trim()).toMatch(/^(0|---)$/);
252+
});
253+
254+
test('should show placeholder when adversaries API returns empty', async ({ page }) => {
255+
await page.route('**/api/v2/adversaries', (route) =>
256+
route.fulfill({
257+
status: 200,
258+
contentType: 'application/json',
259+
body: JSON.stringify([]),
260+
})
261+
);
262+
await navigateToStockpile(page);
263+
// Anchor to the adversaries card by label rather than positional index.
264+
const adversaryCard = page.locator('.card', { hasText: 'adversaries' });
265+
const adversaryCount = adversaryCard.locator('h1.is-size-1');
266+
await expect(adversaryCount).toBeVisible({ timeout: 15_000 });
267+
const text = await adversaryCount.textContent();
268+
expect(text?.trim()).toMatch(/^(0|---)$/);
269+
});
270+
271+
test('should handle network timeout on abilities API', async ({ page }) => {
272+
await page.route('**/api/v2/abilities', (route) => route.abort('timedout'));
273+
await navigateToStockpile(page);
274+
const heading = page.locator('h2', { hasText: 'Stockpile' });
275+
await expect(heading).toBeVisible({ timeout: 15_000 });
276+
});
277+
278+
test('should handle network timeout on adversaries API', async ({ page }) => {
279+
await page.route('**/api/v2/adversaries', (route) => route.abort('timedout'));
280+
await navigateToStockpile(page);
281+
const heading = page.locator('h2', { hasText: 'Stockpile' });
282+
await expect(heading).toBeVisible({ timeout: 15_000 });
283+
});
284+
});

0 commit comments

Comments
 (0)