Skip to content

feat: enhance function removal with retry mechanism for ProvisionConf… #71

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rsonghuster merged 1 commit into from
Aug 11, 2025
Merged

feat: enhance function removal with retry mechanism for ProvisionConf… #71

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/resources/fc/error-code.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ export enum FC_API_ERROR_CODE {
TriggerAlreadyExists = 'TriggerAlreadyExists', // 函数已存在
AliasNotFound = 'AliasNotFound', // 别名不存在
AliasAlreadyExists = 'AliasAlreadyExists', // 别名已存在
ProvisionConfigExist = 'ProvisionConfigExist', // 预配置存在
}

export const isSlsNotExistException = (project: string, logstore: string, ex) => {
Expand Down
2 changes: 1 addition & 1 deletion src/resources/fc/impl/client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -543,7 +543,7 @@ export default class FC_Client {
}

async getLayerVersionByArn(arn: string) {
const result = await this.fc20230330Client.getLayerVersionByArn(arn);
const result = await this.fc20230330InvokeClient.getLayerVersionByArn(arn);
const { body } = result.toMap();
logger.debug(`getLayerVersionByArn response body: ${JSON.stringify(body)}`);
return body;
Expand Down
2 changes: 1 addition & 1 deletion src/subCommands/deploy/impl/concurrency_config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ export default class ConcurrencyConfig extends Base {
`Online concurrencyConfig exists, specified not to deploy, skipping deployment ${id}`,
);
}
} else {
} else if (this.needDeploy) {
try {
if (!_.isEmpty(this.remote)) {
logger.info(`Remove remote concurrencyConfig of ${this.functionName}`);
Expand Down
232 changes: 144 additions & 88 deletions src/subCommands/deploy/impl/provision_config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,33 +43,71 @@ export default class ProvisionConfig extends Base {
const id = `${this.functionName}/provisionConfig`;
const provisionConfig = _.get(this.inputs, 'props.provisionConfig', {});
const qualifier = _.get(provisionConfig, 'qualifier', 'LATEST');

if (!_.isEmpty(localConfig)) {
if (this.needDeploy) {
await this.fcSdk.putFunctionProvisionConfig(this.functionName, qualifier, localConfig);

if (this.ProvisionMode === 'sync') {
await this.waitForProvisionReady(qualifier, localConfig);
} else {
logger.info(
`Waiting for provisionConfig of ${this.functionName}/${qualifier} to instance up ...`,
`Skip wait provisionConfig of ${this.functionName}/${qualifier} to instance up`,
);
const { defaultTarget, target } = localConfig;
let realTarget = defaultTarget;
if (!defaultTarget) {
realTarget = target;
}
let getCurrentErrorCount = 0;
if (realTarget && realTarget > 0) {
const maxRetries = 180;
for (let index = 0; index < maxRetries; index++) {
// eslint-disable-next-line no-await-in-loop
const result = await this.fcSdk.getFunctionProvisionConfig(
this.functionName,
qualifier,
);
const { current, currentError } = result || {};
if (current && current === realTarget) {
break;
}
if (currentError && currentError.length > 0) {
/*
}
} else if (_.isEmpty(remoteConfig)) {
// 如果不需要部署，但是远端资源不存在，则尝试创建一下
logger.debug(
`Online provisionConfig does not exist, specified not to deploy, attempting to create ${id}`,
);
await this.fcSdk.putFunctionProvisionConfig(this.functionName, qualifier, localConfig);
} else {
logger.debug(
`Online provisionConfig exists, specified not to deploy, skipping deployment ${id}`,
);
}
} else if (this.needDeploy) {
await this.removeProvisionConfig(qualifier);
}

return this.needDeploy;
}

/**
* 等待预配置实例就绪
*/
private async waitForProvisionReady(qualifier: string, config: any) {
logger.info(
`Waiting for provisionConfig of ${this.functionName}/${qualifier} to instance up ...`,
);

const { defaultTarget, target } = config;
const realTarget = defaultTarget || target;

// 如果没有目标值或目标值为0，则无需等待
if (!realTarget || realTarget <= 0) {
return;
}

let getCurrentErrorCount = 0;
const maxRetries = 180;

for (let index = 0; index < maxRetries; index++) {
// eslint-disable-next-line no-await-in-loop
const result = await this.fcSdk.getFunctionProvisionConfig(this.functionName, qualifier);
const { current, currentError } = result || {};

// 检查是否已达到目标值
if (current && current === realTarget) {
logger.info(
`ProvisionConfig of ${this.functionName}/${qualifier} is ready. Current: ${current}, Target: ${realTarget}`,
);
return;
}

// 处理错误情况
if (currentError && currentError.length > 0) {
/*
logger.warn('=========== LAST 10 MINUTES LOGS START ==========');
const iInput = _.cloneDeep(this.inputs);
iInput.args = [
Expand All @@ -82,89 +120,107 @@ export default class ProvisionConfig extends Base {
// eslint-disable-next-line no-await-in-loop
await logs.run();
logger.warn('=========== LAST 10 MINUTES END =========== ');
*/
// 如果是系统内部错误，则继续尝试
if (!currentError.includes('an internal error has occurred')) {
// 不是系统内部错误，满足一定的重试次数则退出
getCurrentErrorCount++;
if (getCurrentErrorCount > 3 || (index > 6 && getCurrentErrorCount > 0)) {
logger.error(
`get ${this.functionName}/${qualifier} provision config getCurrentErrorCount=${getCurrentErrorCount}`,
);
throw new Error(
`get ${this.functionName}/${qualifier} provision config error: ${currentError}`,
);
}
}
}
logger.info(
`waiting ${this.functionName}/${qualifier} provision OK: current: ${current}, target: ${realTarget}`,
);
// eslint-disable-next-line no-await-in-loop
await sleep(5);
}
*/
// 如果是系统内部错误，则继续尝试
if (!currentError.includes('an internal error has occurred')) {
// 不是系统内部错误，满足一定的重试次数则退出
getCurrentErrorCount++;
if (getCurrentErrorCount > 3 || (index > 6 && getCurrentErrorCount > 0)) {
logger.error(
`get ${this.functionName}/${qualifier} provision config getCurrentErrorCount=${getCurrentErrorCount}`,
);
throw new Error(
`get ${this.functionName}/${qualifier} provision config error: ${currentError}`,
);
}
} else {
logger.info(
`Skip wait provisionConfig of ${this.functionName}/${qualifier} to instance up`,
);
}
} else if (_.isEmpty(remoteConfig)) {
// 如果不需要部署，但是远端资源不存在，则尝试创建一下
logger.debug(
`Online provisionConfig does not exist, specified not to deploy, attempting to create ${id}`,
);
await this.fcSdk.putFunctionProvisionConfig(this.functionName, qualifier, localConfig);
} else {
logger.debug(
`Online provisionConfig exists, specified not to deploy, skipping deployment ${id}`,
);
}
} else {
try {
if (!_.isEmpty(this.remote)) {
logger.info(`Remove remote provisionConfig of ${this.functionName}/${qualifier}`);
await this.fcSdk.removeFunctionProvisionConfig(this.functionName, qualifier);
for (let index = 0; index < 12; index++) {
// eslint-disable-next-line no-await-in-loop
const result = await this.fcSdk.getFunctionProvisionConfig(
this.functionName,
qualifier,

logger.info(
`waiting ${this.functionName}/${qualifier} provision OK: current: ${current}, target: ${realTarget}`,
);

// eslint-disable-next-line no-await-in-loop
await sleep(5);
}

logger.warn(
`Timeout waiting for provisionConfig of ${this.functionName}/${qualifier} to be ready`,
);
}

/**
* 删除预配置
*/
private async removeProvisionConfig(qualifier: string) {
try {
if (!_.isEmpty(this.remote)) {
logger.info(`Remove remote provisionConfig of ${this.functionName}/${qualifier}`);
await this.fcSdk.removeFunctionProvisionConfig(this.functionName, qualifier);

// 等待预配置实例数降至0
const maxRetries = 12;
for (let index = 0; index < maxRetries; index++) {
// eslint-disable-next-line no-await-in-loop
const result = await this.fcSdk.getFunctionProvisionConfig(this.functionName, qualifier);
const { current } = result || {};

if (current === 0 || !current) {
logger.info(
`ProvisionConfig of ${this.functionName}/${qualifier} removed successfully`,
);
const { current } = result || {};
if (current === 0 || !current) {
break;
}
logger.info(`waiting ${this.functionName}/${qualifier} provision current to 0 ...`);
// eslint-disable-next-line no-await-in-loop
await sleep(5);
return;
}

logger.info(`waiting ${this.functionName}/${qualifier} provision current to 0 ...`);
// eslint-disable-next-line no-await-in-loop
await sleep(5);
}
} catch (ex) {
logger.error(
`Remove remote provisionConfig of ${this.functionName}/${qualifier} error: ${ex.message}`,

logger.warn(
`Timeout waiting for provisionConfig of ${this.functionName}/${qualifier} to be removed`,
);
}
} catch (ex) {
logger.error(
`Remove remote provisionConfig of ${this.functionName}/${qualifier} error: ${ex.message}`,
);
throw ex;
}
return this.needDeploy;
}

/**
* 清理预配置配置对象
*/
private sanitizeProvisionConfig(config: any): any {
if (!config) return {};

let sanitized = _.omit(config, ['current', 'functionArn', 'currentError']);

// 如果同时存在target和defaultTarget，移除target
if ('target' in sanitized && 'defaultTarget' in sanitized) {
sanitized = _.omit(sanitized, ['target']);
}

// 移除空的策略配置
if (_.isEmpty(sanitized?.targetTrackingPolicies)) {
sanitized = _.omit(sanitized, ['targetTrackingPolicies']);
}

if (_.isEmpty(sanitized?.scheduledActions)) {
sanitized = _.omit(sanitized, ['scheduledActions']);
}

return sanitized;
}

private async getRemote() {
const provisionConfig = _.get(this.inputs, 'props.provisionConfig', {});
const qualifier = _.get(provisionConfig, 'qualifier', 'LATEST');

try {
const result = await this.fcSdk.getFunctionProvisionConfig(this.functionName, qualifier);
let r = _.omit(result, ['current', 'functionArn', 'currentError']);
if ('target' in r && 'defaultTarget' in r) {
r = _.omit(r, ['target']);
}
if (_.isEmpty(r?.targetTrackingPolicies)) {
r = _.omit(r, ['targetTrackingPolicies']);
}
if (_.isEmpty(r?.scheduledActions)) {
r = _.omit(r, ['scheduledActions']);
}
this.remote = r;
this.remote = this.sanitizeProvisionConfig(result);
} catch (ex) {
logger.debug(`Get remote provisionConfig of ${this.functionName} error: ${ex.message}`);
this.remote = {};
Expand Down
45 changes: 44 additions & 1 deletion src/subCommands/remove/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,7 +442,50 @@ export default class Remove {
}

logger.spin('removing', 'function', `${this.region}/${this.functionName}`);
await this.fcSdk.fc20230330Client.deleteFunction(this.functionName);
try {
await this.fcSdk.fc20230330Client.deleteFunction(this.functionName);
} catch (ex) {
// 如果是 ProvisionConfigExist 错误，尝试重试
if (ex.code === FC_API_ERROR_CODE.ProvisionConfigExist) {
logger.warn(
`Remove function ${this.functionName} failed with ProvisionConfigExist error, retrying...`,
);

// 重试 20 次，每次间隔 2 秒
for (let i = 1; i <= 20; i++) {
logger.info(`Retry attempt ${i}/20...`);
await sleep(2);

try {
await this.fcSdk.fc20230330Client.deleteFunction(this.functionName);
logger.info(`Function ${this.functionName} removed successfully on retry attempt ${i}`);
break;
} catch (retryEx) {
if (i === 20) {
// 最后一次重试仍然失败
logger.error(
`Remove function ${this.functionName} error after 20 retries: ${retryEx.message}`,
);
throw retryEx;
}

// 如果不是 ProvisionConfigExist 错误，直接抛出
if (retryEx.code !== FC_API_ERROR_CODE.ProvisionConfigExist) {
logger.error(`Remove function ${this.functionName} error: ${retryEx.message}`);
throw retryEx;
}

// 如果是 ProvisionConfigExist 错误，继续重试
logger.warn(
`Remove function ${this.functionName} still failed with ProvisionConfigExist error, continuing retries...`,
);
}
}
} else {
logger.error(`Remove function ${this.functionName} error: ${ex.message}`);
throw ex;
}
}
logger.spin('removed', 'function', `${this.region}/${this.functionName}`);
}

Expand Down
Loading