diff --git a/.github/workflows/enum-static-mapping-updater.yml b/.github/workflows/enum-static-mapping-updater.yml index 8eddf4a47cb27..de9bd0abc4e48 100644 --- a/.github/workflows/enum-static-mapping-updater.yml +++ b/.github/workflows/enum-static-mapping-updater.yml @@ -52,10 +52,10 @@ jobs: git checkout -b "$branchName" git add . # Add all files changed - git commit -m "chore: update enum static mapping" + git commit -m "chore(enum-updater): update enum static mapping" git push -f origin "$branchName" - gh pr create --title "chore: update enum static mapping" \ + gh pr create --title "chore(enum-updater): update enum static mapping" \ --body "This PR updates the CDK enum mapping file." \ --base main \ --head "$branchName" \ diff --git a/tools/@aws-cdk/enum-updater/README.md b/tools/@aws-cdk/enum-updater/README.md index d39e6da9085fe..abd41b69167e6 100644 --- a/tools/@aws-cdk/enum-updater/README.md +++ b/tools/@aws-cdk/enum-updater/README.md @@ -1,6 +1,8 @@ # CDK Enum Updater -This tool updates CDK enums with missing enum values on a weekly basis. +This tool updates CDK enums with missing enum values on a weekly basis. It is triggered by the following two Github workflows: + * [Update Enum Static Mapping](../../../.github/workflows/enum-static-mapping-updater.yml) - Runs every week on Monday 12am + * [Update Missing Enum Values](../../../.github/workflows/enum-auto-updater.yml) - Runs every week on Monday 1pm To run the tool locally, run the following commands to install dependencies and build: @@ -101,4 +103,50 @@ All exclusions for a module should be listed together. For example if you want t } } } +``` + +### Manual Static Mapping Overrides +Sometimes the script will calculate the wrong static enum mapping. You can override this value by adding an entry to the `lib/manual-enum-mapping.json` file. + +Manual mappings should be added in the form: +``` +{ + "serviceName": { + "CdkEnumName": { + "cdk_path": "path/to/cdkEnum", + "sdk_service": "serviceName", + "sdk_enum_name": "AwsEnumThatWeWantThisToMapTo", + "match_percentage": 1.0, + } + } +} +``` + +For example: +``` +{ + "ec2": { + "VpcEndpointIpAddressType": { + "cdk_path": "aws-cdk/packages/aws-cdk-lib/aws-ec2/lib/vpc-endpoint.ts", + "sdk_service": "ec2", + "sdk_enum_name": "IpAddressType", + "match_percentage": 1.0, + } + } +} +``` + +The entry that is overwritten in the `lib/static-enum-mapping.json` file will contain a field `manual: true`: +``` +{ + "ec2": { + "VpcEndpointIpAddressType": { + "cdk_path": "aws-cdk/packages/aws-cdk-lib/aws-ec2/lib/vpc-endpoint.ts", + "sdk_service": "ec2", + "sdk_enum_name": "IpAddressType", + "match_percentage": 1.0, + "manual": true, + } + } +} ``` \ No newline at end of file diff --git a/tools/@aws-cdk/enum-updater/lib/manual-enum-mapping.json b/tools/@aws-cdk/enum-updater/lib/manual-enum-mapping.json new file mode 100644 index 0000000000000..a82e5d4d0c641 --- /dev/null +++ b/tools/@aws-cdk/enum-updater/lib/manual-enum-mapping.json @@ -0,0 +1,10 @@ +{ + "ec2": { + "VpcEndpointIpAddressType": { + "cdk_path": "aws-cdk/packages/aws-cdk-lib/aws-ec2/lib/vpc-endpoint.ts", + "sdk_service": "ec2", + "sdk_enum_name": "IpAddressType", + "match_percentage": 1.0 + } + } +} \ No newline at end of file diff --git a/tools/@aws-cdk/enum-updater/lib/static-enum-mapping-updater.ts b/tools/@aws-cdk/enum-updater/lib/static-enum-mapping-updater.ts index fe361f1c0ff98..4b740018e6ae8 100644 --- a/tools/@aws-cdk/enum-updater/lib/static-enum-mapping-updater.ts +++ b/tools/@aws-cdk/enum-updater/lib/static-enum-mapping-updater.ts @@ -7,11 +7,13 @@ import * as extract from 'extract-zip'; const ENUMS_URL = "https://raw.githubusercontent.com/aws/aws-cdk/main/packages/aws-cdk-lib/core/lib/analytics-data-source/enums/module-enums.json"; const ENUM_LIKE_CLASSES_URL = "https://raw.githubusercontent.com/aws/aws-cdk/main/packages/aws-cdk-lib/core/lib/analytics-data-source/enums/module-enumlikes.json"; const CFN_LINT_URL = "https://github.com/aws-cloudformation/cfn-lint/archive/refs/heads/main.zip" -const MODULE_MAPPING = path.join(__dirname, "module-mapping.json"); -const STATIC_MAPPING_FILE_NAME = "static-enum-mapping.json"; +export const STATIC_MAPPING_FILE_NAME = "static-enum-mapping.json"; const PARSED_CDK_ENUMS_FILE_NAME = "cdk-enums.json"; const EXCLUDE_FILE = "exclude-values.json"; -export const PARSED_SDK_ENUMS_FILE_NAME = "sdk-enums.json"; +const MODULE_MAPPING = path.join(__dirname, "module-mapping.json"); +const PARSED_SDK_ENUMS_FILE_NAME = "sdk-enums.json"; +export const MANUAL_MAPPING_FILE_NAME = "manual-enum-mapping.json"; +export const MANUAL_MAPPING = path.join(__dirname, MANUAL_MAPPING_FILE_NAME); export const STATIC_MAPPING = path.join(__dirname, STATIC_MAPPING_FILE_NAME); export const CDK_ENUMS = path.join(__dirname, PARSED_CDK_ENUMS_FILE_NAME); export const SDK_ENUMS = path.join(__dirname, PARSED_SDK_ENUMS_FILE_NAME); @@ -31,7 +33,6 @@ export interface SdkEnums { }; } - function extractEnums(schema: Record, enums: { [enumName: string]: (string | number)[] }) { // Helper function to process a property and its potential enum values function processProperty(propertyName: string, property: any) { @@ -85,6 +86,7 @@ export interface StaticMappingEntry { sdk_service: string; sdk_enum_name: string; match_percentage: number; + manual?: boolean; } export interface StaticMapping { @@ -453,19 +455,22 @@ function isValidMatch(cdkValues: Set, sdkValues: Set): boolean { * * @param {CdkEnums} cdkEnums - The extracted CDK enums. * @param {SdkEnums} sdkEnums - The extracted AWS SDK enums. - * @param {Record} manualMappings - The manually defined service mappings. + * @param {Record} moduleMappings - The manually defined service mappings. + * @param {StaticMapping} manualEnumMappings - Manually defined overrides for the static enum mappings. * @returns {Promise} */ export async function generateAndSaveStaticMapping( cdkEnums: CdkEnums, sdkEnums: SdkEnums, - manualMappings: Record, + moduleMappings: Record, + manualEnumMappings: StaticMapping ): Promise { const staticMapping: StaticMapping = {}; const unmatchedEnums: UnmatchedEnums = {}; + // Process CDK enums as before for (const [module, enums] of Object.entries(cdkEnums)) { - if (!manualMappings[module]) { + if (!moduleMappings[module]) { // Add to unmatched if no SDK mapping exists unmatchedEnums[module] = Object.fromEntries( Object.entries(enums).map(([enumName, enumData]) => [ @@ -476,9 +481,21 @@ export async function generateAndSaveStaticMapping( continue; } - const sdkServices = manualMappings[module]; + const sdkServices = moduleMappings[module]; for (const [enumName, enumData] of Object.entries(enums)) { + // Check if we have a manual mapping for this enum + if (manualEnumMappings && manualEnumMappings[module]?.[enumName]) { + // Use the manual mapping + if (!staticMapping[module]) { + staticMapping[module] = {}; + } + staticMapping[module][enumName] = manualEnumMappings[module][enumName]; + staticMapping[module][enumName]["manual"] = true; + continue; + } + + // Otherwise, proceed with automatic mapping const match = findMatchingEnum( enumName, enumData.values, @@ -519,7 +536,7 @@ export async function generateAndSaveStaticMapping( fs.writeFileSync(`lib/${STATIC_MAPPING_FILE_NAME}`, JSON.stringify(staticMapping, null, 2)); fs.writeFileSync('lib/unmatched-enums.json', JSON.stringify(unmatchedEnums, null, 2)); - console.log(`Total matched enums: ${Object.values(staticMapping).reduce((sum, moduleEnums) => + console.log(`Total matched enums: ${Object.values(staticMapping).reduce((sum, moduleEnums) => sum + Object.keys(moduleEnums).length, 0)}`); } @@ -547,9 +564,10 @@ export async function entryMethod(): Promise { const cdkEnums: CdkEnums = JSON.parse(fs.readFileSync(CDK_ENUMS, 'utf8')); const sdkEnums: SdkEnums = JSON.parse(fs.readFileSync(SDK_ENUMS, 'utf8')); const manualMappings: Record = JSON.parse(fs.readFileSync(MODULE_MAPPING, 'utf8')); + const manualEnumMappings = JSON.parse(fs.readFileSync(MANUAL_MAPPING, 'utf8')); // Generate and save static mapping - await generateAndSaveStaticMapping(cdkEnums, sdkEnums, manualMappings); + await generateAndSaveStaticMapping(cdkEnums, sdkEnums, manualMappings, manualEnumMappings); console.log("Static mapping and missing values analysis completed."); diff --git a/tools/@aws-cdk/enum-updater/test/static-enum-mapping-updater.test.ts b/tools/@aws-cdk/enum-updater/test/static-enum-mapping-updater.test.ts index b41a329d3cd3e..625915b9b0f85 100644 --- a/tools/@aws-cdk/enum-updater/test/static-enum-mapping-updater.test.ts +++ b/tools/@aws-cdk/enum-updater/test/static-enum-mapping-updater.test.ts @@ -1,13 +1,13 @@ import axios from 'axios'; import * as fs from 'fs'; import * as tmp from 'tmp'; -import { +import { normalizeValue, normalizeEnumValues, extractModuleName, entryMethod, generateAndSaveStaticMapping, - findMatchingEnum + findMatchingEnum, } from '../lib/static-enum-mapping-updater'; jest.mock('axios'); @@ -146,6 +146,7 @@ describe('Static Mapping Generation', () => { mockCdkEnums, mockSdkEnums, mockManualMappings, + {} ); // Verify the file write operation @@ -162,6 +163,7 @@ describe('Static Mapping Generation', () => { mockCdkEnums, mockSdkEnums, {}, + {} ); // Verify the file write operation @@ -171,6 +173,77 @@ describe('Static Mapping Generation', () => { expect(writtenContent.service1).toBeUndefined(); }); + + + it('should prioritize manual mappings over automatic ones', async () => { + // Setup test data + const mockCdkEnums = { + amplify: { + ManualEnum: { + path: "path/to/enum", + enumLike: false, + values: ['VALUE1', 'VALUE2'] + }, + AutoEnum: { + path: "path/to/auto/enum", + enumLike: false, + values: ['AUTO1', 'AUTO2'] + } + } + }; + + const mockSdkEnums = { + amplify: { + SomeEnum: ['VALUE1', 'VALUE2', 'VALUE3'], + AutoEnum: ['AUTO1', 'AUTO2', 'AUTO3'] + } + }; + + const mockModuleMappings = { + amplify: ['amplify'] + }; + + const testManualEnumMappings = { + amplify: { + ManualEnum: { + cdk_path: "path/to/enum", + sdk_service: "amplify", + sdk_enum_name: "OverrideManualEnum", + match_percentage: 1.0, + } + } + }; + + // Execute the functionI + await generateAndSaveStaticMapping( + mockCdkEnums, + mockSdkEnums, + mockModuleMappings, + testManualEnumMappings + ); + + // Find the call that writes to static-enum-mapping.json + const writeCall = (fs.writeFileSync as jest.Mock).mock.calls.find( + call => String(call[0]).includes('static-enum-mapping.json') + ); + expect(writeCall).toBeDefined(); + + const writtenContent = JSON.parse(writeCall[1]); + + // Check that the manual mapping was used for ManualEnum + expect(writtenContent.amplify.ManualEnum).toBeDefined(); + expect(writtenContent.amplify.ManualEnum.sdk_service).toBe('amplify'); + expect(writtenContent.amplify.ManualEnum.sdk_enum_name).toBe('OverrideManualEnum'); + expect(writtenContent.amplify.ManualEnum.cdk_path).toBe('path/to/enum'); + expect(writtenContent.amplify.ManualEnum.manual).toBe(true); + + // Check that automatic mapping was used for AutoEnum + expect(writtenContent.amplify.AutoEnum).toBeDefined(); + expect(writtenContent.amplify.AutoEnum.sdk_service).toBe('amplify'); + expect(writtenContent.amplify.AutoEnum.sdk_enum_name).toBe('AutoEnum'); + expect(writtenContent.amplify.AutoEnum.cdk_path).toBe('path/to/auto/enum'); + expect(writtenContent.amplify.AutoEnum.manual).toBeUndefined(); + }); });