Skip to content

Commit f83fe2d

Browse files
RHTAP-5994: Add retry to component creation testcase (#308)
1 parent 2ab81c1 commit f83fe2d

8 files changed

Lines changed: 257 additions & 13 deletions

File tree

package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
"@playwright/test": "^1.54.1",
5555
"@trivago/prettier-plugin-sort-imports": "^6.0.0",
5656
"@types/node": "^24.0.14",
57+
"@types/proper-lockfile": "^4.1.4",
5758
"@types/sodium-native": "^2.3.9",
5859
"@types/uuid": "^11.0.0",
5960
"@types/winston": "^2.4.4",
@@ -71,6 +72,7 @@
7172
"pino": "^10.0.0",
7273
"pino-pretty": "^13.0.0",
7374
"prettier": "^3.6.2",
75+
"proper-lockfile": "^4.1.2",
7476
"rimraf": "^6.0.1",
7577
"ts-node": "^10.9.2",
7678
"typescript": "^5.8.3"

scripts/generateProjectConfig.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
55
import path from 'path';
66
import { LoggerFactory } from '../src/logger/logger';
77
import type { Logger } from '../src/logger/logger';
8+
import { PROJECT_CONFIGS_FILE } from '../src/constants';
89

910
interface ProjectConfig {
1011
name: string;
@@ -68,7 +69,7 @@ function generateProjectConfig(): void {
6869
}));
6970

7071
// Ensure output directory exists
71-
const outputPath = './tmp/project-configs.json';
72+
const outputPath = PROJECT_CONFIGS_FILE;
7273
const outputDir = path.dirname(outputPath);
7374
if (!existsSync(outputDir)) {
7475
mkdirSync(outputDir, { recursive: true });

src/api/rhdh/developerhub.ts

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -132,16 +132,21 @@ export class DeveloperHub {
132132
} else if (status === 'failed' || status === 'cancelled') {
133133
this.logger.error(`Component creation ${status}.`, { taskId });
134134

135-
// Get logs to understand what went wrong
135+
// Get logs to understand what went wrong and include in error
136+
let taskLogs = '';
136137
try {
137-
const logs = await this.getComponentLogs(taskId);
138-
this.logger.error(`Task logs: ${logs}`);
138+
taskLogs = await this.getComponentLogs(taskId);
139+
this.logger.error(`Task logs:\n${taskLogs}`);
139140
} catch (logError) {
140141
this.logger.error(`Failed to retrieve task logs: ${logError instanceof Error ? logError.message : String(logError)}`);
141142
}
142143

144+
const errorMessage = taskLogs
145+
? `Component creation ${status} for task ${taskId}\n---TASK_LOGS_START---\n${taskLogs}\n---TASK_LOGS_END---`
146+
: `Component creation ${status} for task ${taskId}`;
147+
143148
// Use bail to immediately exit the retry loop for terminal failure states
144-
bail(new Error(`Component creation ${status} for task ${taskId}`));
149+
bail(new Error(errorMessage));
145150
return; // This line won't be reached after bail, but added for clarity
146151
}
147152

src/constants.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,6 @@ export const DEFAULT_APP_NAMESPACE = 'tssc-app';
33
export const TSSC_APP_DEPLOYMENT_NAMESPACE = process.env.TSSC_APP_DEPLOYMENT_NAMESPACE || DEFAULT_APP_NAMESPACE;
44

55
export const TSSC_CI_NAMESPACE = `${TSSC_APP_DEPLOYMENT_NAMESPACE}-ci`;
6+
7+
// Path to the project configs file (shared between config generator, E2E and UI tests)
8+
export const PROJECT_CONFIGS_FILE = './tmp/project-configs.json';

src/playwright/testItem.ts

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,14 @@ import { CIType } from '../../src/rhtap/core/integration/ci';
22
import { GitType } from '../../src/rhtap/core/integration/git';
33
import { TemplateType } from '../../src/rhtap/core/integration/git';
44
import { ImageRegistryType } from '../../src/rhtap/core/integration/registry';
5+
import { randomString } from '../utils/util';
6+
import { existsSync, readFileSync, writeFileSync } from 'fs';
7+
import { lock } from 'proper-lockfile';
8+
import { PROJECT_CONFIGS_FILE } from '../constants';
9+
import { LoggerFactory, Logger } from '../logger/logger';
510

611
export class TestItem {
12+
private readonly logger: Logger;
713
private name: string;
814
private template: TemplateType;
915
private registryType: ImageRegistryType;
@@ -24,6 +30,7 @@ export class TestItem {
2430
acs: string = '',
2531
planName: string = 'legacy'
2632
) {
33+
this.logger = LoggerFactory.getLogger('playwright.testItem');
2734
this.name = name;
2835
this.template = template;
2936
this.registryType = registryType;
@@ -100,6 +107,109 @@ export class TestItem {
100107
this.planName = planName;
101108
}
102109

110+
/**
111+
* Regenerates the component name with a new random suffix.
112+
* This is useful when component creation fails due to name conflicts
113+
* and a fresh name is needed for retry.
114+
*
115+
* Example: "backend-tests-python-abc123" -> "backend-tests-python-xyz789"
116+
*
117+
* @returns The new generated name
118+
*/
119+
public regenerateName(): string {
120+
const currentName = this.name;
121+
const lastHyphenIndex = currentName.lastIndexOf('-');
122+
123+
const baseName = lastHyphenIndex > 0
124+
? currentName.substring(0, lastHyphenIndex)
125+
: currentName;
126+
127+
const newRandomSuffix = randomString(8);
128+
this.name = `${baseName}-${newRandomSuffix}`;
129+
130+
this.logger.info(`Regenerated component name: '${currentName}' -> '${this.name}'`);
131+
return this.name;
132+
}
133+
134+
/**
135+
* Gets the base name (without the random suffix) for identification.
136+
*/
137+
public getBaseName(): string {
138+
const lastHyphenIndex = this.name.lastIndexOf('-');
139+
return lastHyphenIndex > 0 ? this.name.substring(0, lastHyphenIndex) : this.name;
140+
}
141+
142+
/**
143+
* Saves the current component name to existing project-configs.json.
144+
*
145+
* IMPORTANT: Matches by exact config (template + gitType + ciType + registryType)
146+
* to avoid conflicts when multiple workers run in parallel with same base name.
147+
*
148+
* @throws Error if the file is missing, no matching config entry, write/lock fails
149+
*/
150+
public async saveComponentName(): Promise<void> {
151+
const filePath = PROJECT_CONFIGS_FILE;
152+
const lockfilePath = `${filePath}.lock`;
153+
let release: (() => Promise<void>) | undefined;
154+
155+
try {
156+
if (!existsSync(filePath)) {
157+
const msg = `Project configs file not found: ${filePath}`;
158+
this.logger.error(msg);
159+
throw new Error(msg);
160+
}
161+
162+
// Acquire a lock to prevent race conditions
163+
release = await lock(filePath, { lockfilePath, retries: 5 });
164+
this.logger.info(`Acquired lock for ${filePath}`);
165+
166+
// Re-read file to get latest state
167+
const configs = JSON.parse(readFileSync(filePath, 'utf-8'));
168+
169+
// Find and update the matching testItem by EXACT config match only
170+
// (template + gitType + ciType + registryType) to avoid cross-worker conflicts
171+
let updated = false;
172+
for (const config of configs) {
173+
if (config.testItem) {
174+
if (config.testItem.template === this.template &&
175+
config.testItem.gitType === this.gitType &&
176+
config.testItem.ciType === this.ciType &&
177+
config.testItem.registryType === this.registryType) {
178+
const oldName = config.testItem.name;
179+
config.testItem.name = this.name;
180+
updated = true;
181+
this.logger.info(`Updated component name in project-configs.json: '${oldName}' -> '${this.name}' (config: ${this.template}/${this.gitType}/${this.ciType}/${this.registryType})`);
182+
break;
183+
}
184+
}
185+
}
186+
187+
if (updated) {
188+
writeFileSync(filePath, JSON.stringify(configs, null, 2));
189+
this.logger.info(`Updated project configs file: ${filePath}`);
190+
return;
191+
}
192+
const msg = `Could not find matching config to update for: ${this.name} (config: ${this.template}/${this.gitType}/${this.ciType}/${this.registryType})`;
193+
this.logger.error(msg);
194+
throw new Error(msg);
195+
} catch (error) {
196+
if (error instanceof Error && error.message.startsWith('Project configs file not found')) {
197+
throw error;
198+
}
199+
if (error instanceof Error && error.message.startsWith('Could not find matching config')) {
200+
throw error;
201+
}
202+
this.logger.error(`Failed to save component name with lock: ${error}`);
203+
throw error instanceof Error ? error : new Error(String(error));
204+
} finally {
205+
// Only release if acquired the lock
206+
if (release) {
207+
await release();
208+
this.logger.info(`Released lock for ${filePath}`);
209+
}
210+
}
211+
}
212+
103213
/**
104214
* Convert the TestItem to a JSON-serializable object
105215
*/

src/utils/projectConfigLoader.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { TestItem } from '../playwright/testItem';
22
import { readFileSync, existsSync } from 'fs';
33
import { LoggerFactory, Logger } from '../logger/logger';
4+
import { PROJECT_CONFIGS_FILE } from '../constants';
45

56
const logger: Logger = LoggerFactory.getLogger('utils.project-config-loader');
67

@@ -19,7 +20,7 @@ interface SerializedProjectConfig {
1920
* Configurations are generated by scripts/generateProjectConfig.ts before Playwright runs
2021
*/
2122
export function loadProjectConfigurations(): ProjectConfig[] {
22-
const configFilePath = './tmp/project-configs.json';
23+
const configFilePath = PROJECT_CONFIGS_FILE;
2324

2425
if (!existsSync(configFilePath)) {
2526
logger.error(`Project configuration file not found: ${configFilePath}`);

src/utils/test/common.ts

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { EventType } from '../../rhtap/core/integration/ci';
66
import { Git, PullRequest } from '../../rhtap/core/integration/git';
77
import { TPA } from '../../rhtap/core/integration/tpa';
88
import { SBOMResult } from '../../api/tpa/tpaClient';
9+
import { Component } from '../../rhtap/core/component';
910
import { sleep } from '../util';
1011
import { expectPipelineSuccess } from './assertionHelpers';
1112
import { expect } from '@playwright/test';
@@ -14,6 +15,124 @@ import { LoggerFactory, Logger } from '../../logger/logger';
1415

1516
const logger: Logger = LoggerFactory.getLogger('utils.test.common');
1617

18+
/**
19+
* Determines if an error is non-retryable (should not trigger retry attempts).
20+
* For example "Name already taken" errors are retryable because we regenerate the name.
21+
*/
22+
function isNonRetryableError(errorMessage: string): boolean {
23+
const nonRetryablePatterns = [
24+
/unauthorized/i,
25+
/status code 401/i,
26+
/authentication failed/i,
27+
/invalid.*token/i,
28+
/template.*not found/i,
29+
/invalid.*template/i,
30+
/permission denied/i,
31+
/environment variable .* is not defined or is empty/i,
32+
/failed to retrieve secret/i,
33+
];
34+
35+
return nonRetryablePatterns.some((pattern) => pattern.test(errorMessage));
36+
}
37+
38+
/**
39+
* Creates a component and waits for completion with retry support.
40+
*
41+
* This function handles transient failures and name conflicts by:
42+
* 1. Creating the component with the current testItem name
43+
* 2. Waiting for component creation to complete (prints detailed task logs on failure)
44+
* 3. If either step fails, regenerating a new component name and retrying
45+
* 4. Continuing until success or max retries reached
46+
*
47+
* @param testItem The test item containing configuration details (name will be modified on retry)
48+
* @param options Retry configuration options
49+
* @returns Promise<Component> The successfully created and completed component
50+
* @throws Error when component creation fails after all retry attempts
51+
*/
52+
export async function createComponentAndWaitForCompletion(
53+
testItem: TestItem,
54+
options: { maxRetries?: number; retryDelayMs?: number; regenerateNameOnRetry?: boolean } = {}
55+
): Promise<Component> {
56+
const DEFAULT_MAX_RETRIES = 3;
57+
const DEFAULT_RETRY_DELAY_MS = 10000;
58+
59+
const {
60+
maxRetries = DEFAULT_MAX_RETRIES,
61+
retryDelayMs = DEFAULT_RETRY_DELAY_MS,
62+
regenerateNameOnRetry = true,
63+
} = options;
64+
65+
let lastError: Error | null = null;
66+
let attemptNumber = 0;
67+
const originalName = testItem.getName();
68+
const totalAttempts = maxRetries + 1;
69+
70+
while (attemptNumber <= maxRetries) {
71+
attemptNumber++;
72+
const componentName = testItem.getName();
73+
const imageName = componentName;
74+
75+
logger.info(`[Attempt ${attemptNumber}/${totalAttempts}] Creating component '${componentName}'...`);
76+
77+
try {
78+
// Step 1: Create the component
79+
const component = await Component.new(componentName, testItem, imageName, true);
80+
81+
// Step 2: Wait for completion
82+
await component.waitUntilComponentIsCompleted();
83+
84+
// Success!
85+
logger.info(`✅ Component '${componentName}' created successfully on attempt ${attemptNumber}/${totalAttempts}`);
86+
87+
return component;
88+
} catch (error) {
89+
lastError = error instanceof Error ? error : new Error(String(error));
90+
const errorMessage = lastError.message;
91+
92+
// Extract error summary (exclude task logs from summary display)
93+
const errorSummary = errorMessage.includes('---TASK_LOGS_START---')
94+
? errorMessage.split('---TASK_LOGS_START---')[0].trim()
95+
: errorMessage;
96+
97+
logger.error(`❌ COMPONENT CREATION FAILED - Attempt ${attemptNumber}/${totalAttempts} | Component: ${componentName} | Error: ${errorSummary}`);
98+
99+
// Check if we've exhausted all retries
100+
if (attemptNumber > maxRetries) {
101+
logger.error(`❌ All ${totalAttempts} attempts exhausted. Original: '${originalName}', Last tried: '${componentName}'`);
102+
break;
103+
}
104+
105+
// Check if this is a non-retryable error
106+
if (isNonRetryableError(errorMessage)) {
107+
logger.error(`Non-retryable error detected. Stopping retry attempts.`);
108+
break;
109+
}
110+
111+
logger.info(`🔄 Will retry. Attempts remaining: ${totalAttempts - attemptNumber}`);
112+
113+
// Regenerate name for next attempt if enabled
114+
if (regenerateNameOnRetry) {
115+
const newName = testItem.regenerateName();
116+
logger.info(`New component name: '${newName}'`);
117+
118+
// Save the new name immediately to project-configs.json
119+
await testItem.saveComponentName();
120+
}
121+
122+
// Wait before next retry
123+
logger.info(`Waiting ${retryDelayMs}ms before next attempt...`);
124+
await sleep(retryDelayMs);
125+
}
126+
}
127+
128+
// If we reach here, all retries failed
129+
throw new Error(
130+
`Component creation failed after ${attemptNumber} attempts. ` +
131+
`Original name: '${originalName}'. ` +
132+
`Last error: ${lastError?.message?.split('---TASK_LOGS_START---')[0].trim() || 'Unknown error'}`
133+
);
134+
}
135+
17136
/**
18137
* Promotes an application to a specific environment using a pull request workflow
19138
*

tests/tssc/full_workflow.test.ts

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { Git } from '../../src/rhtap/core/integration/git';
55
import { TPA } from '../../src/rhtap/core/integration/tpa';
66
import { ComponentPostCreateAction } from '../../src/rhtap/postcreation/componentPostCreateAction';
77
import {
8+
createComponentAndWaitForCompletion,
89
runAndWaitforAppSync,
910
handleSourceRepoCodeChanges,
1011
handlePromotionToEnvironmentandGetPipeline,
@@ -42,16 +43,18 @@ test.describe('TSSC Complete Workflow', () => {
4243

4344
test.describe('Component Creation', () => {
4445
test('should create a component successfully', async ({ testItem, logger }) => {
45-
// Generate component name directly in the test
46+
// Generate initial component name directly in the test
4647
const componentName = testItem.getName();
47-
const imageName = `${componentName}`;
48-
logger.info(`Creating component: ${componentName}`);
48+
logger.info(`Creating component with retry support. Initial name: ${componentName}`);
4949

50-
// Create the component directly in the test
51-
component = await Component.new(componentName, testItem, imageName);
50+
// Create the component and wait for completion with automatic retry
51+
component = await createComponentAndWaitForCompletion(testItem, {
52+
maxRetries: 2,
53+
retryDelayMs: 10000,
54+
regenerateNameOnRetry: true
55+
});
5256

53-
// Wait for the component to be created
54-
await component.waitUntilComponentIsCompleted();
57+
logger.info(`Component created successfully with name: ${component.getName()}`);
5558

5659
// Initialize shared resources
5760
cd = component.getCD();

0 commit comments

Comments
 (0)