Skip to content

Commit 3f0ee2b

Browse files
authored
Create release testing package (elastic#239506)
1 parent 717d4a6 commit 3f0ee2b

16 files changed

Lines changed: 460 additions & 1 deletion

File tree

.buildkite/scout_ci_config.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,6 @@ plugins:
1717
packages:
1818
enabled:
1919
- kbn-streamlang-tests
20+
- kbn-scout-release-testing
2021
disabled:
2122
- kbn-scout # Internal scout tests are run in advance to validate Scout integrity (see .buildkite/scripts/steps/test/scout_test_run_builder.sh)

.github/CODEOWNERS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -567,6 +567,7 @@ src/platform/packages/shared/kbn-rule-data-utils @elastic/security-detections-re
567567
src/platform/packages/shared/kbn-safer-lodash-set @elastic/kibana-security
568568
src/platform/packages/shared/kbn-saved-search-component @elastic/obs-ux-logs-team
569569
src/platform/packages/shared/kbn-scout @elastic/appex-qa
570+
src/platform/packages/shared/kbn-scout-release-testing @elastic/appex-qa
570571
src/platform/packages/shared/kbn-search-api-panels @elastic/search-kibana
571572
src/platform/packages/shared/kbn-search-connectors @elastic/search-kibana
572573
src/platform/packages/shared/kbn-search-errors @elastic/kibana-data-discovery

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1640,6 +1640,7 @@
16401640
"@kbn/scout": "link:src/platform/packages/shared/kbn-scout",
16411641
"@kbn/scout-info": "link:src/platform/packages/private/kbn-scout-info",
16421642
"@kbn/scout-oblt": "link:x-pack/solutions/observability/packages/kbn-scout-oblt",
1643+
"@kbn/scout-release-testing": "link:src/platform/packages/shared/kbn-scout-release-testing",
16431644
"@kbn/scout-reporting": "link:src/platform/packages/private/kbn-scout-reporting",
16441645
"@kbn/scout-security": "link:x-pack/solutions/security/packages/kbn-scout-security",
16451646
"@kbn/security-api-integration-helpers": "link:x-pack/platform/test/security_api_integration/packages/helpers",
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# @kbn/scout-upgrade-testing
2+
3+
Release upgrade testing package for Kibana stateful deployments.
4+
5+
## Overview
6+
7+
This package contains end-to-end tests that verify Kibana functionality across version upgrades in stateful Elastic Cloud deployments. Tests are executed through the [AppEx QA Stateful Cloud Upgrade Tests pipeline](https://buildkite.com/elastic/appex-qa-stateful-cloud-upgrade-tests) and are built using the [@kbn/scout](../kbn-scout) testing framework.
8+
9+
## Purpose
10+
11+
The primary goal of this package is to ensure:
12+
13+
- ✅ Core Kibana features work correctly after version upgrades
14+
- ✅ Data and saved objects persist through upgrade cycles
15+
- ✅ User workflows remain functional across versions
16+
17+
## Testing Framework
18+
19+
Tests are written using **Scout**, Kibana's Playwright-based testing framework, which provides:
20+
21+
- Page object abstractions for Kibana applications
22+
- Elastic Cloud deployment management
23+
- Authentication helpers
24+
- Advanced debugging capabilities
25+
26+
## Directory Structure
27+
28+
```
29+
kbn-scout-release-testing/
30+
├── test/
31+
│ └── scout/
32+
│ └── ui/
33+
│ ├── playwright.config.ts # Playwright configuration
34+
│ └── discovery_tests/ # Test suites organized by feature
35+
│ └── discovery.spec.ts # Discover app tests
36+
├── kibana.jsonc # Package manifest
37+
├── package.json
38+
├── tsconfig.json
39+
└── README.md
40+
```
41+
42+
## CI/CD Pipeline
43+
44+
Tests run in the [AppEx QA Stateful Cloud Upgrade Tests](https://buildkite.com/elastic/appex-qa-stateful-cloud-upgrade-tests) Buildkite pipeline:
45+
46+
- **Trigger**: Manual triggers, VERSION and UPGRADE_VERSION should be defined as ENV variables
47+
- **Environment**: Stateful Elastic Cloud deployments
48+
- **Upgrade Scenarios**: Tests execute after version upgrades
49+
50+
### Code Owners
51+
52+
This package is maintained by **@elastic/appex-qa**
53+
54+
For questions or support, reach out in:
55+
- Slack: `#appex-qa`
56+
57+
## Related Documentation
58+
59+
- [Scout Framework Documentation](src/platform/packages/shared/kbn-scout/README.md)
60+
- [AppEx QA Buildkite Pipeline](https://buildkite.com/elastic/appex-qa-stateful-cloud-upgrade-tests)
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"type": "functional-tests",
3+
"id": "@kbn/scout-release-testing",
4+
"owner": [
5+
"@elastic/appex-qa"
6+
],
7+
"group": "platform",
8+
"visibility": "private",
9+
"devOnly": true
10+
}
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"name": "@kbn/scout-release-testing",
3+
"private": true,
4+
"version": "1.0.0",
5+
"license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0"
6+
}
Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
import { expect, test } from '@kbn/scout';
11+
12+
const defaultSettings = {
13+
defaultIndex: 'logstash-*',
14+
'dateFormat:tz': 'UTC',
15+
};
16+
const defaultStartTime = 'Sep 19, 2015 @ 06:31:44.000';
17+
const defaultEndTime = 'Sep 23, 2015 @ 18:31:44.000';
18+
const endTimeNoResults = 'Sep 19, 2015 @ 18:45:00.000';
19+
const queryName1 = 'Query # 1';
20+
const queryName2 = 'Query # 2';
21+
22+
test.describe('Discover app', { tag: ['@ess'] }, () => {
23+
test.beforeAll(async ({ kbnClient, esArchiver }) => {
24+
await kbnClient.importExport.load(
25+
'src/platform/test/functional/fixtures/kbn_archiver/discover'
26+
);
27+
await esArchiver.loadIfNeeded(
28+
'src/platform/test/functional/fixtures/es_archiver/logstash_functional'
29+
);
30+
await kbnClient.uiSettings.update(defaultSettings);
31+
});
32+
33+
test.beforeEach(async ({ browserAuth, pageObjects, uiSettings }) => {
34+
await browserAuth.loginAsAdmin();
35+
await uiSettings.setDefaultTime({
36+
from: defaultStartTime,
37+
to: defaultEndTime,
38+
});
39+
await pageObjects.discover.goto();
40+
});
41+
42+
test.afterAll(async ({ kbnClient }) => {
43+
await kbnClient.savedObjects.clean({ types: ['search', 'index-pattern'] });
44+
});
45+
46+
test('should display selected time range in date picker and matching docs in table', async ({
47+
pageObjects,
48+
}) => {
49+
await pageObjects.datePicker.setAbsoluteRange({
50+
from: defaultStartTime,
51+
to: defaultEndTime,
52+
});
53+
const time = await pageObjects.datePicker.getTimeConfig();
54+
expect(time.start).toBe(defaultStartTime);
55+
expect(time.end).toBe(defaultEndTime);
56+
57+
const rowData = await pageObjects.discover.getDocTableIndex(1);
58+
expect(rowData).toContain('Sep 22, 2015 @ 23:50:13.253');
59+
});
60+
61+
test('save query should show toast message and display query name', async ({ pageObjects }) => {
62+
await pageObjects.discover.saveSearch(queryName1);
63+
const actualQueryNameString = await pageObjects.discover.getCurrentQueryName();
64+
expect(actualQueryNameString).toBe(queryName1);
65+
});
66+
67+
test('should refetch when autofresh is enabled', async ({ pageObjects }) => {
68+
const interval = 5;
69+
await pageObjects.datePicker.startAutoRefresh(interval);
70+
const getRequestTimestamp = async () => {
71+
await pageObjects.inspector.open();
72+
const requestTimestamp = await pageObjects.inspector.getRequestTimestamp();
73+
await pageObjects.inspector.close();
74+
return requestTimestamp;
75+
};
76+
77+
const requestTimestampBefore = await getRequestTimestamp();
78+
79+
await expect
80+
.poll(
81+
async () => {
82+
const requestTimestampAfter = await getRequestTimestamp();
83+
return Boolean(requestTimestampAfter) && requestTimestampBefore !== requestTimestampAfter;
84+
},
85+
{ timeout: 7000 }
86+
)
87+
.toBe(true);
88+
});
89+
90+
test('load query should show query name', async ({ pageObjects }) => {
91+
await pageObjects.discover.saveSearch(queryName2);
92+
await pageObjects.discover.loadSavedSearch(queryName2);
93+
await expect
94+
.poll(async () => await pageObjects.discover.getCurrentQueryName())
95+
.toBe(queryName2);
96+
});
97+
98+
test('should show the correct hit count', async ({ pageObjects }) => {
99+
const expectedHitCount = 14004;
100+
expect(await pageObjects.discover.getHitCountInt()).toBe(expectedHitCount);
101+
});
102+
103+
test('should show correct time range string in chart', async ({ pageObjects }) => {
104+
const actualTimeString = await pageObjects.discover.getChartTimespan();
105+
const expectedTimeString = `${defaultStartTime} - ${defaultEndTime} (interval: Auto - 3 hours)`;
106+
expect(actualTimeString).toBe(expectedTimeString);
107+
});
108+
109+
test('should modify the time range when a bar is clicked', async ({ pageObjects }) => {
110+
await pageObjects.discover.clickHistogramBar();
111+
await pageObjects.discover.waitUntilSearchingHasFinished();
112+
113+
const time = await pageObjects.datePicker.getTimeConfig();
114+
expect(time.start).toBe('Sep 21, 2015 @ 09:00:00.000');
115+
expect(time.end).toBe('Sep 21, 2015 @ 12:00:00.000');
116+
117+
await expect
118+
.poll(
119+
async () => {
120+
const rowData = await pageObjects.discover.getDocTableField(1);
121+
return rowData.includes('Sep 21, 2015 @ 11:59:22.316');
122+
},
123+
{ timeout: 3000 }
124+
)
125+
.toBe(true);
126+
});
127+
128+
test('should show correct initial chart interval of Auto', async ({ page, pageObjects }) => {
129+
await page.testSubj.click('discoverQueryHits'); // cancel out tooltips
130+
const actualInterval = await pageObjects.discover.getChartInterval();
131+
const expectedInterval = 'auto';
132+
expect(actualInterval).toBe(expectedInterval);
133+
});
134+
135+
test('should show "no results"', async ({ page, pageObjects }) => {
136+
await pageObjects.datePicker.setAbsoluteRange({
137+
from: defaultStartTime,
138+
to: endTimeNoResults,
139+
});
140+
await expect(page.testSubj.locator('discoverNoResults')).toBeVisible();
141+
});
142+
143+
test('should suggest a new time range is picked', async ({ page, pageObjects, uiSettings }) => {
144+
await uiSettings.setDefaultTime({
145+
from: defaultStartTime,
146+
to: endTimeNoResults,
147+
});
148+
await pageObjects.discover.goto();
149+
await expect(page.testSubj.locator('discoverNoResultsTimefilter')).toBeVisible();
150+
});
151+
152+
test('should show matches when time range is expanded', async ({
153+
page,
154+
pageObjects,
155+
uiSettings,
156+
}) => {
157+
await uiSettings.setDefaultTime({
158+
from: defaultStartTime,
159+
to: endTimeNoResults,
160+
});
161+
await pageObjects.discover.goto();
162+
await pageObjects.discover.expandTimeRangeAsSuggestedInNoResultsMessage();
163+
164+
await expect(page.testSubj.locator('discoverNoResultsTimefilter')).toBeHidden();
165+
await expect.poll(async () => await pageObjects.discover.getHitCountInt()).toBeGreaterThan(0);
166+
});
167+
});
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
import { createPlaywrightConfig } from '@kbn/scout';
11+
12+
export default createPlaywrightConfig({
13+
testDir: './discovery_tests',
14+
});
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"extends": "../../../../../tsconfig.base.json",
3+
"compilerOptions": {
4+
"outDir": "target/types",
5+
"types": [
6+
"jest",
7+
"node"
8+
]
9+
},
10+
"include": [
11+
"**/*.ts",
12+
],
13+
"exclude": [
14+
"target/**/*"
15+
],
16+
"kbn_references": [
17+
"@kbn/scout"
18+
]
19+
}

src/platform/packages/shared/kbn-scout/src/playwright/page_objects/date_picker.ts

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,12 @@
1010
import type { ScoutPage } from '..';
1111
import { expect } from '..';
1212

13+
export enum DateUnitSelector {
14+
Seconds = 's',
15+
Minutes = 'm',
16+
Hours = 'h',
17+
}
18+
1319
export class DatePicker {
1420
constructor(private readonly page: ScoutPage) {}
1521

@@ -70,4 +76,30 @@ export class DatePicker {
7076
).toHaveText(to);
7177
await this.page.testSubj.click('querySubmitButton');
7278
}
79+
80+
async getTimeConfig(): Promise<{ start: string; end: string }> {
81+
await this.showStartEndTimes();
82+
const start = await this.page.testSubj.innerText('superDatePickerstartDatePopoverButton');
83+
const end = await this.page.testSubj.innerText('superDatePickerendDatePopoverButton');
84+
return { start, end };
85+
}
86+
87+
async startAutoRefresh(interval: number, dateUnit: DateUnitSelector = DateUnitSelector.Seconds) {
88+
await this.page.testSubj.click('superDatePickerToggleQuickMenuButton');
89+
// Check if refresh is already running
90+
const toggleButton = this.page.testSubj.locator('superDatePickerToggleRefreshButton');
91+
const isPaused = (await toggleButton.getAttribute('aria-checked')) === 'false';
92+
if (isPaused) {
93+
await toggleButton.click();
94+
}
95+
// Set interval
96+
const intervalInput = this.page.testSubj.locator('superDatePickerRefreshIntervalInput');
97+
await intervalInput.clear();
98+
await intervalInput.fill(interval.toString());
99+
const timeUnit = this.page.testSubj.locator('superDatePickerRefreshIntervalUnitsSelect');
100+
await timeUnit.selectOption({ value: dateUnit });
101+
await intervalInput.press('Enter');
102+
103+
await this.page.testSubj.click('superDatePickerToggleQuickMenuButton');
104+
}
73105
}

0 commit comments

Comments
 (0)