From 48e9c6dbe9b23dcc6dec498f025556eea42e30af Mon Sep 17 00:00:00 2001 From: go-to-k <24818752+go-to-k@users.noreply.github.com> Date: Sun, 9 Aug 2026 12:42:09 +0900 Subject: [PATCH 1/3] fix(emr): rename ConfigurationProperties / StepProperties to the SDK Properties member CFn spells the EMR application-configuration bag `Configuration.ConfigurationProperties` and the step bag `HadoopJarStepConfig.StepProperties`, while both SDK members are named `Properties`. The EMR providers cast the CFn blobs straight to the SDK types, and the AWS SDK v3 serializer drops unknown members - so every EMR application configuration (spark-defaults / hive-site / yarn-site ...) silently vanished and the cluster came up unconfigured while cdkd reported the deploy as successful. Step properties were dropped the same way. The value shapes already match (Record for the former, KeyValue[] for the latter - verified against the live CFn registry schema), so both directions are pure key renames. - new shared helper src/provisioning/emr-configuration.ts: toSdkConfigurations (recursive - Configurations nests into itself), toSdkStepConfigs, toSdkInstanceTypeConfigs - wired at every forwarding site: EMRClusterProvider top-level Configurations / Steps, per-instance-group Configurations, per-fleet InstanceTypeConfigs; EMRInstanceGroupConfigProvider create; EMRInstanceFleetConfigProvider create + ModifyInstanceFleet update - no inverse needed: Configurations / Steps / InstanceTypeConfigs are all in EMRClusterProvider.getDriftUnknownPaths and neither instance provider implements readCurrentState - unit tests for the helper plus per-provider wiring tests, the four provider-level ones verified to fail without the fix - emr-cluster fixture gains top-level + NESTED Configurations, a per-master-group Configurations block and a step with StepProperties; emr-instance-configs gains Configurations on the standalone TASK group. Both verify.sh files read the values back from AWS through a new SDK-based list_instance_groups_json helper (aws emr list-instance-groups is CLI-customized and unusable non-interactively) Closes #1383 --- .claude/rules/code-layout.md | 1 + src/provisioning/emr-configuration.ts | 91 ++++++++++++ .../providers/emr-cluster-provider.ts | 19 ++- .../emr-instance-fleet-config-provider.ts | 9 +- .../emr-instance-group-config-provider.ts | 5 +- .../emr-cluster/lib/emr-cluster-stack.ts | 48 +++++++ tests/integration/emr-cluster/verify.sh | 63 ++++++++ .../lib/emr-instance-configs-stack.ts | 12 ++ .../emr-instance-configs/verify.sh | 34 +++++ .../provisioning/emr-configuration.test.ts | 136 ++++++++++++++++++ .../providers/emr-cluster-provider.test.ts | 92 ++++++++++++ ...emr-instance-fleet-config-provider.test.ts | 48 +++++++ ...emr-instance-group-config-provider.test.ts | 21 +++ 13 files changed, 560 insertions(+), 19 deletions(-) create mode 100644 src/provisioning/emr-configuration.ts create mode 100644 tests/unit/provisioning/emr-configuration.test.ts diff --git a/.claude/rules/code-layout.md b/.claude/rules/code-layout.md index 61a4dd609..fb69b230d 100644 --- a/.claude/rules/code-layout.md +++ b/.claude/rules/code-layout.md @@ -77,6 +77,7 @@ paths: - **src/provisioning/register-providers.ts** - Shared provider registration (called from deploy.ts and destroy.ts) - **src/provisioning/data-delete-intent.ts** - Shared destroy data-guard intent helpers (issue #1340): `hasCdkAutoDeleteTag(properties, tagKey)` / `isTruthyCfnBoolean(value)` plus the CDK tag-key constants `S3_AUTO_DELETE_OBJECTS_TAG` (`aws-cdk:auto-delete-objects`, stamped by `autoDeleteObjects: true`) and `ECR_AUTO_DELETE_IMAGES_TAG` (`aws-cdk:auto-delete-images`). Consumed by `S3BucketProvider.delete` (auto-empty of a non-empty bucket only with the tag / `DeleteContext.forceDataDelete`), `S3DirectoryBucketProvider.delete` (issue #1344 — same gate; no CDK opt-in sugar exists for directory buckets, so plain destroy of a non-empty one fails with a manual-empty remediation), and `ECRProvider.delete` (`force: true` only with `EmptyOnDelete: true`, the tag, or `forceDataDelete`) — without an opt-in the AWS not-empty error surfaces like CloudFormation DELETE_FAILED. `DeleteContext.forceDataDelete` (src/provisioning/region-check.ts) is set ONLY by the deploy engine's replacement/recreate delete sites under `--force-stateful-recreation`. See the "Destroy data guards" section in docs/cli-reference.md and the DeleteContext contract note in .claude/rules/providers.md. - **src/provisioning/final-snapshot.ts** - `DeletionPolicy` / `UpdateReplacePolicy: Snapshot` support (issues #1352 / #1353 / #1354): `ATOMIC_FINAL_SNAPSHOT_TYPES` (RDS DBInstance / DBCluster, Neptune / DocDB clusters, ElastiCache CacheCluster — the delete call sites generate `buildFinalSnapshotIdentifier(physicalId, resourceType)` and thread it via `DeleteContext.finalSnapshotIdentifier`; each provider flips its delete from `SkipFinalSnapshot: true` to the API's atomic final-snapshot form; ONLY on the SDK route — a cc-api-routed atomic type is refused and `CloudControlProvider.delete` fail-closes on the field), `PRE_DELETE_SNAPSHOT_TYPES` + `createPreDeleteFinalSnapshot` dispatcher (all CC-routed: `AWS::EC2::Volume` via EC2 `CreateSnapshot` tagged `cdkd:final-snapshot-of`; `AWS::Redshift::Cluster` via `CreateClusterSnapshot`; `AWS::ElastiCache::ReplicationGroup` via ElastiCache `CreateSnapshot` — each waited to ready, idempotent reuse via the tag / the `finalSnapshotNamePrefix` name prefix across delete re-runs), `unsupportedFinalSnapshotError` / `ccRoutedFinalSnapshotError` refusals, and (issue #1366) `finalSnapshotMechanism(type, route)` / `refusesFinalSnapshot(type, route)` — the mechanism matrix as a PURE function, so the executor that ACTS on it and the `cdkd rollback` plan preview that DESCRIBES it read one source (issue #1368 extends that to the preview's STATE effect: a refused Snapshot delete no longer unwinds the record, since the next-older segment is classified against it). The two type sets are DISJOINT by construction — `finalSnapshotMechanism` tests the atomic set first, so a type in both would silently take the atomic arm and never reach the pre-delete snapshot; pinned in `final-snapshot.test.ts` alongside the union-equals-the-CFn-documented-list fence (re-homed there from the deleted `supportsFinalSnapshot` predicate, #1368). Consumed by the deploy engine (`prepareFinalSnapshotForDelete` — the shared gate for the DELETE branch AND the four replacement / recreate delete sites), `destroy-runner.ts`, and `rollback-executor.ts` — the latter twice: `rollbackFinalSnapshotId` for the delete-of-the-NEW-resource under `UpdateReplacePolicy` (honors only the atomic SDK-routed shape, plain-deletes otherwise — scope decision on #1354), and `prepareCreateRollbackFinalSnapshot` for a rolled-back CREATE under `DeletionPolicy` (the FULL matrix, refusing what it cannot snapshot — issue #1358). The engine's clients come from `DeployEngineOptions.finalSnapshotClients` (stack-region-pinned `AwsClients`, structurally a `PreDeleteSnapshotClients`), threaded on to `RollbackExecutorContext.finalSnapshotClients`; `--skip-final-snapshot` (deploy / destroy / state destroy / rollback, `skipFinalSnapshotOption` in `src/cli/options.ts` — deliberately NOT in the shared `destroyOptions` array `cdkd orphan` consumes) is the explicit data-loss opt-out. +- **src/provisioning/emr-configuration.ts** - Shared CFn -> SDK shape converters for the `AWS::EMR::*` nested config blobs whose CFn key spelling diverges from `@aws-sdk/client-emr` (issue #1383): `toSdkConfigurations` (renames `Configuration.ConfigurationProperties` -> the SDK's `Properties` at EVERY `Configurations` nesting level), `toSdkStepConfigs` (`HadoopJarStepConfig.StepProperties` -> `Properties`), and `toSdkInstanceTypeConfigs` (per-instance-type nested `Configurations`). Both are pure key renames — the VALUE shapes already match (`Record` / `KeyValue[]`, verified against the live CFn registry schema) — but the AWS SDK v3 serializer drops unknown members, so before the conversion every EMR application configuration (spark-defaults / hive-site / yarn-site ...) silently vanished while cdkd reported success. Consumed by `EMRClusterProvider` (top-level `Configurations` / `Steps`, per-group `Configurations`, per-fleet `InstanceTypeConfigs`), `EMRInstanceGroupConfigProvider` (create), and `EMRInstanceFleetConfigProvider` (create + the `ModifyInstanceFleet` update). No inverse is needed: `Configurations` / `Steps` / `InstanceTypeConfigs` are all declared in `EMRClusterProvider.getDriftUnknownPaths` and neither instance provider implements `readCurrentState`. Non-object / non-array inputs (an unresolved intrinsic) pass through untouched so AWS surfaces the real validation error. The `AWS::EMR::*` types are NOT yet in `NESTED_KEY_TARGETS` (`scripts/gen-nested-key-coverage.ts`) — critic target expansion is tracked in issue #1393. - **src/provisioning/ec2-termination-protection.ts** - Shared `--remove-protection` helper for `AWS::EC2::Instance`: `disableInstanceApiTermination()` (flip `DisableApiTermination` off, idempotent, errors swallowed at debug), `isTerminationProtectionPropagationError()` (matches the "may not be terminated. Modify its disableApiTermination" 400 from both `TerminateInstances` and the Cloud Control `DeleteResource` wrapper), and `TERMINATION_PROTECTION_MAX_ATTEMPTS`. Used by `EC2Provider.deleteInstance` (SDK path) and `CloudControlProvider.delete` (CC-API path — an instance routes through Cloud Control whenever its template trips the #614 silent-drop routing) so `--remove-protection` works regardless of which delete path the instance takes; the modify WRITE lags the delete READ, so both callers flip-off + retry the delete to close the propagation window. ALSO used by `ASGProvider.delete` (issue #796): an `AWS::AutoScaling::AutoScalingGroup` whose launch template sets `DisableApiTermination: true` launches instances that survive the group's `DeleteAutoScalingGroup(ForceDelete: true)` (ASG-level DeletionProtection + ForceDelete governs only the group + scale-in protection, not EC2-level termination protection), so under `--remove-protection` the provider enumerates the group's current instances and flips each one's `DisableApiTermination` off before the force delete — the ASG's own async terminate loop then absorbs the modify-WRITE propagation lag, so no per-instance delete retry is needed there. An ASG can ALSO route via Cloud Control when its template sets a silent-drop property such as `AvailabilityZoneIds` (#614 routing) — Cloud Control's `DeleteResource` cannot `ForceDelete` a protected ASG or clear its protection, so `CloudControlProvider.delete` detects `removeProtection === true && resourceType === 'AWS::AutoScaling::AutoScalingGroup'` and delegates to `new ASGProvider().delete(...)` (the single source of truth for protected-ASG deletion), keeping the SDK and CC routing paths behaviourally identical (issue #798; CDK's L2 emits `availabilityZones` names not `AvailabilityZoneIds`, so this CC path only fires for hand-written L1 / imported templates). - **src/provisioning/unsupported-types.ts** + **unsupported-types.generated.ts** - Pre-flight unsupported-type rejection. The `.generated.ts` ships the provider-coverage Tier 3 set (`ProvisioningType: NON_PROVISIONABLE`) into the runtime, codegen'd from `docs/_generated/provider-coverage.json` by `scripts/gen-unsupported-types.ts` (`vp run gen:unsupported-types`; CI fails on drift). The hand-written `.ts` adds `isNonProvisionable()` + `unsupportedTypeIssueUrl()`; both are consulted by `CloudControlProvider.isSupportedResourceType` (rejects Tier 3) and `ProviderRegistry.validateResourceTypes` (per-type error + issue link). The `--allow-unsupported-types` escape hatch routes named types through Cloud Control via `ProviderRegistry.allowUnsupportedTypes()`. - **src/provisioning/property-coverage.ts** + **property-coverage.generated.ts** - Pre-flight property-level rejection (parallel to unsupported-types but at top-level CFn property granularity). The `.generated.ts` ships per-Tier-1-type `{ handled, silentDrop }` records, codegen'd from `tests/fixtures/cfn-schemas/*.json` + each SDK provider's `handledProperties` / `unhandledByDesign` declarations by `scripts/gen-property-coverage.ts` (`vp run gen:property-coverage`; CI fails on drift; the codegen parses provider sources via the TypeScript Compiler API so no `dist/` bootstrap is needed). The hand-written `.ts` adds `getPropertyCoverage()` + `findSilentDropProperties()` + `unsupportedPropertyIssueUrl()`; all are consulted by `ProviderRegistry.validateResourceProperties` (per-resource per-property error + 1-click GitHub issue link + dedup'd re-run command). The `--allow-unsupported-properties` escape hatch (deploy only) routes named `:` entries past the reject via `ProviderRegistry.allowUnsupportedProperties()`. Tier 2 (Cloud Control) types are intentionally NOT in the generated map — CC forwards the full property map to AWS, so no write-side silent drop is possible. diff --git a/src/provisioning/emr-configuration.ts b/src/provisioning/emr-configuration.ts new file mode 100644 index 000000000..86bcc52af --- /dev/null +++ b/src/provisioning/emr-configuration.ts @@ -0,0 +1,91 @@ +import type { Configuration, InstanceTypeConfig, StepConfig } from '@aws-sdk/client-emr'; + +/** + * CFn -> SDK shape converters for the two `AWS::EMR::*` nested blobs whose CFn + * key spelling diverges from the `@aws-sdk/client-emr` model (issue #1383). + * + * | CFn key | SDK member | + * | ------------------------------------------------ | ------------------------------ | + * | `Configuration.ConfigurationProperties` | `Configuration.Properties` | + * | `HadoopJarStepConfig.StepProperties` | `HadoopJarStepConfig.Properties` | + * + * Both are plain renames — the VALUE shapes already match (`Record` + * for the former, `KeyValue[]` for the latter, verified against the live CFn + * registry schema on 2026-08-09). The AWS SDK v3 serializer drops unknown + * members, so before this conversion every EMR application configuration + * (spark-defaults / hive-site / yarn-site ...) silently vanished while cdkd + * reported the deploy as successful. + * + * `Configurations` nests into itself, so the rename is applied at EVERY level. + * Non-object / non-array inputs (an unresolved intrinsic, a malformed template) + * pass through untouched so this layer never turns a bad template into a + * confusing crash — AWS surfaces the real validation error instead. + */ + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function toSdkConfiguration(raw: unknown): Configuration { + if (!isRecord(raw)) return raw as Configuration; + + const { ConfigurationProperties, Configurations, ...rest } = raw; + return { + ...rest, + ...(ConfigurationProperties !== undefined ? { Properties: ConfigurationProperties } : {}), + ...(Configurations !== undefined + ? { Configurations: toSdkConfigurations(Configurations) } + : {}), + } as Configuration; +} + +/** + * CFn `Configurations` list -> SDK `Configuration[]`, renaming + * `ConfigurationProperties` -> `Properties` at every nesting level. + */ +export function toSdkConfigurations(value: unknown): Configuration[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value)) return value as Configuration[]; + return value.map(toSdkConfiguration); +} + +/** + * CFn `Steps` list -> SDK `StepConfig[]`, renaming + * `HadoopJarStep.StepProperties` -> `HadoopJarStep.Properties`. + */ +export function toSdkStepConfigs(value: unknown): StepConfig[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value)) return value as StepConfig[]; + + return value.map((step) => { + if (!isRecord(step)) return step as StepConfig; + const hadoopJarStep = step['HadoopJarStep']; + if (!isRecord(hadoopJarStep) || hadoopJarStep['StepProperties'] === undefined) { + return step as unknown as StepConfig; + } + const { StepProperties, ...restHadoopJarStep } = hadoopJarStep; + return { + ...step, + HadoopJarStep: { ...restHadoopJarStep, Properties: StepProperties }, + } as unknown as StepConfig; + }); +} + +/** + * CFn `InstanceTypeConfigs` list -> SDK `InstanceTypeConfig[]`, converting each + * element's nested per-instance-type `Configurations`. + */ +export function toSdkInstanceTypeConfigs(value: unknown): InstanceTypeConfig[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value)) return value as InstanceTypeConfig[]; + + return value.map((instanceTypeConfig) => { + if (!isRecord(instanceTypeConfig) || instanceTypeConfig['Configurations'] === undefined) { + return instanceTypeConfig as InstanceTypeConfig; + } + return { + ...instanceTypeConfig, + Configurations: toSdkConfigurations(instanceTypeConfig['Configurations']), + } as InstanceTypeConfig; + }); +} diff --git a/src/provisioning/providers/emr-cluster-provider.ts b/src/provisioning/providers/emr-cluster-provider.ts index 256595dbb..1c198ef43 100644 --- a/src/provisioning/providers/emr-cluster-provider.ts +++ b/src/provisioning/providers/emr-cluster-provider.ts @@ -32,6 +32,11 @@ import { getLogger } from '../../utils/logger.js'; import { ProvisioningError, ResourceUpdateNotSupportedError } from '../../utils/error-handler.js'; import { assertRegionMatch, type DeleteContext } from '../region-check.js'; import { normalizeAwsTagsToCfn, resolveExplicitPhysicalId } from '../import-helpers.js'; +import { + toSdkConfigurations, + toSdkInstanceTypeConfigs, + toSdkStepConfigs, +} from '../emr-configuration.js'; import type { ResourceProvider, ResourceCreateResult, @@ -258,13 +263,11 @@ export class EMRClusterProvider implements ResourceProvider { Applications: properties['Applications'] as | import('@aws-sdk/client-emr').Application[] | undefined, - Configurations: properties['Configurations'] as - | import('@aws-sdk/client-emr').Configuration[] - | undefined, + Configurations: toSdkConfigurations(properties['Configurations']), BootstrapActions: properties['BootstrapActions'] as | import('@aws-sdk/client-emr').BootstrapActionConfig[] | undefined, - Steps: properties['Steps'] as import('@aws-sdk/client-emr').StepConfig[] | undefined, + Steps: toSdkStepConfigs(properties['Steps']), KerberosAttributes: properties['KerberosAttributes'] as | import('@aws-sdk/client-emr').KerberosAttributes | undefined, @@ -413,9 +416,7 @@ export class EMRClusterProvider implements ResourceProvider { Name: raw['Name'] as string | undefined, Market: raw['Market'] as import('@aws-sdk/client-emr').MarketType | undefined, BidPrice: raw['BidPrice'] as string | undefined, - Configurations: raw['Configurations'] as - | import('@aws-sdk/client-emr').Configuration[] - | undefined, + Configurations: toSdkConfigurations(raw['Configurations']), EbsConfiguration: raw['EbsConfiguration'] as | import('@aws-sdk/client-emr').EbsConfiguration | undefined, @@ -435,9 +436,7 @@ export class EMRClusterProvider implements ResourceProvider { Name: raw['Name'] as string | undefined, TargetOnDemandCapacity: toNumber(raw['TargetOnDemandCapacity']), TargetSpotCapacity: toNumber(raw['TargetSpotCapacity']), - InstanceTypeConfigs: raw['InstanceTypeConfigs'] as - | import('@aws-sdk/client-emr').InstanceTypeConfig[] - | undefined, + InstanceTypeConfigs: toSdkInstanceTypeConfigs(raw['InstanceTypeConfigs']), LaunchSpecifications: raw['LaunchSpecifications'] as | import('@aws-sdk/client-emr').InstanceFleetProvisioningSpecifications | undefined, diff --git a/src/provisioning/providers/emr-instance-fleet-config-provider.ts b/src/provisioning/providers/emr-instance-fleet-config-provider.ts index 53469a288..3832b1232 100644 --- a/src/provisioning/providers/emr-instance-fleet-config-provider.ts +++ b/src/provisioning/providers/emr-instance-fleet-config-provider.ts @@ -13,6 +13,7 @@ import { import { getLogger } from '../../utils/logger.js'; import { ProvisioningError, ResourceUpdateNotSupportedError } from '../../utils/error-handler.js'; import { assertRegionMatch, type DeleteContext } from '../region-check.js'; +import { toSdkInstanceTypeConfigs } from '../emr-configuration.js'; import type { ResourceProvider, ResourceCreateResult, @@ -218,9 +219,7 @@ export class EMRInstanceFleetConfigProvider implements ResourceProvider { Name: properties['Name'] as string | undefined, TargetOnDemandCapacity: toNumber(properties['TargetOnDemandCapacity']), TargetSpotCapacity: toNumber(properties['TargetSpotCapacity']), - InstanceTypeConfigs: properties['InstanceTypeConfigs'] as - | import('@aws-sdk/client-emr').InstanceTypeConfig[] - | undefined, + InstanceTypeConfigs: toSdkInstanceTypeConfigs(properties['InstanceTypeConfigs']), LaunchSpecifications: properties['LaunchSpecifications'] as | import('@aws-sdk/client-emr').InstanceFleetProvisioningSpecifications | undefined, @@ -288,9 +287,7 @@ export class EMRInstanceFleetConfigProvider implements ResourceProvider { ResizeSpecifications: properties['ResizeSpecifications'] as | import('@aws-sdk/client-emr').InstanceFleetResizingSpecifications | undefined, - InstanceTypeConfigs: properties['InstanceTypeConfigs'] as - | import('@aws-sdk/client-emr').InstanceTypeConfig[] - | undefined, + InstanceTypeConfigs: toSdkInstanceTypeConfigs(properties['InstanceTypeConfigs']), }; await this.getClient().send( new ModifyInstanceFleetCommand({ ClusterId: clusterId, InstanceFleet: modify }) diff --git a/src/provisioning/providers/emr-instance-group-config-provider.ts b/src/provisioning/providers/emr-instance-group-config-provider.ts index 29646c0df..2beb8a894 100644 --- a/src/provisioning/providers/emr-instance-group-config-provider.ts +++ b/src/provisioning/providers/emr-instance-group-config-provider.ts @@ -14,6 +14,7 @@ import { import { getLogger } from '../../utils/logger.js'; import { ProvisioningError, ResourceUpdateNotSupportedError } from '../../utils/error-handler.js'; import { assertRegionMatch, type DeleteContext } from '../region-check.js'; +import { toSdkConfigurations } from '../emr-configuration.js'; import type { ResourceProvider, ResourceCreateResult, @@ -220,9 +221,7 @@ export class EMRInstanceGroupConfigProvider implements ResourceProvider { Name: properties['Name'] as string | undefined, Market: properties['Market'] as import('@aws-sdk/client-emr').MarketType | undefined, BidPrice: properties['BidPrice'] as string | undefined, - Configurations: properties['Configurations'] as - | import('@aws-sdk/client-emr').Configuration[] - | undefined, + Configurations: toSdkConfigurations(properties['Configurations']), EbsConfiguration: properties['EbsConfiguration'] as | import('@aws-sdk/client-emr').EbsConfiguration | undefined, diff --git a/tests/integration/emr-cluster/lib/emr-cluster-stack.ts b/tests/integration/emr-cluster/lib/emr-cluster-stack.ts index 421937c84..be21b2571 100644 --- a/tests/integration/emr-cluster/lib/emr-cluster-stack.ts +++ b/tests/integration/emr-cluster/lib/emr-cluster-stack.ts @@ -96,6 +96,45 @@ export class EmrClusterStack extends cdk.Stack { // the normal deploy+verify+destroy window, so it never races the test. // The UPDATE phase bumps it to exercise PutAutoTerminationPolicy. autoTerminationPolicy: { idleTimeout: isUpdate ? 7200 : 3600 }, + // Issue #1383: CFn spells the property bag `ConfigurationProperties` + // while the SDK member is `Properties`, so the AWS SDK v3 serializer + // silently dropped EVERY application configuration and the cluster came + // up unconfigured while cdkd reported success. Two levels are used on + // purpose: the rename must apply at every `Configurations` depth. + // `core-site` / `hadoop-env`+`export` are the canonical benign + // classifications — an unknown key is just an extra Hadoop config entry, + // so the cluster still bootstraps. + configurations: [ + { + classification: 'core-site', + configurationProperties: { 'cdkd.integ.marker': 'top-level' }, + }, + { + classification: 'hadoop-env', + configurations: [ + { + classification: 'export', + configurationProperties: { CDKD_INTEG_NESTED: 'yes' }, + }, + ], + }, + ], + // Issue #1383, second key: CFn `HadoopJarStep.StepProperties` -> SDK + // `Properties`. `command-runner.jar` with a trivial echo keeps the step + // fast and CONTINUE-on-failure keeps a step error from killing the + // cluster. Steps are create-only, so this block is identical in the + // UPDATE phase. + steps: [ + { + name: 'cdkd-integ-step', + actionOnFailure: 'CONTINUE', + hadoopJarStep: { + jar: 'command-runner.jar', + args: ['echo', 'cdkd-integ'], + stepProperties: [{ key: 'cdkd.integ.step', value: 'yes' }], + }, + }, + ], instances: { ec2SubnetId: vpc.publicSubnets[0].subnetId, // Provide the managed SG for both roles (see EmrSg comment above). @@ -109,6 +148,15 @@ export class EmrClusterStack extends cdk.Stack { instanceType: 'm5.xlarge', market: 'ON_DEMAND', name: 'Master', + // Issue #1383: the per-instance-group `Configurations` blob goes + // through the SAME rename inside `Cluster.Instances` — a separate + // conversion site from the top-level one above. + configurations: [ + { + classification: 'core-site', + configurationProperties: { 'cdkd.integ.marker': 'master-group' }, + }, + ], }, }, tags: [ diff --git a/tests/integration/emr-cluster/verify.sh b/tests/integration/emr-cluster/verify.sh index ab86fa976..e0d40bbe0 100755 --- a/tests/integration/emr-cluster/verify.sh +++ b/tests/integration/emr-cluster/verify.sh @@ -93,6 +93,23 @@ CLEANUP_TAG_VALUE="emr-cluster" LOCAL_DIST="${PWD}/../../../dist/cli.js" +# `aws emr list-instance-groups` is an AWS-CLI-CUSTOMIZED command: in a +# non-interactive shell it prints `Warning: Input is not a terminal (fd=0).` +# and then dies with `aws: [ERROR]: [Errno 22] Invalid argument` (verified +# 2026-08-09; `--no-paginate --no-cli-pager JSON array of InstanceGroups + ( cd "${REPO_ROOT}" && REGION="${REGION}" node --input-type=module -e " +import { EMRClient, ListInstanceGroupsCommand } from '@aws-sdk/client-emr'; +const client = new EMRClient({ region: process.env.REGION }); +const res = await client.send(new ListInstanceGroupsCommand({ ClusterId: process.argv[1] })); +process.stdout.write(JSON.stringify(res.InstanceGroups ?? [])); +" "$1" ) || return 1 +} + + # Ids of ACTIVE (not terminated) clusters named like the fixture and carrying # the fixture's constant tag. # Ids of ACTIVE clusters named like the fixture and carrying its constant tag. @@ -430,6 +447,52 @@ if [ "${PROVISIONED_BY}" != "sdk" ]; then fi echo " cluster routed via SDK provider (provisionedBy=sdk)" +# --- Assertions: issue #1383 nested-key renames reached AWS ------------ +# CFn spells the property bag `ConfigurationProperties` and the step bag +# `StepProperties`, while the SDK members are both `Properties`. The AWS SDK v3 +# serializer drops unknown members, so before the fix EVERY application +# configuration silently vanished and the cluster came up unconfigured while +# cdkd reported success. Read the values back from AWS, not from cdkd state. +TOP_CFG="$(aws emr describe-cluster --cluster-id "${CID_P1}" --region "${REGION}" \ + --query "Cluster.Configurations[?Classification=='core-site'].Properties.\"cdkd.integ.marker\" | [0]" \ + --output text)" +if [ "${TOP_CFG}" != "top-level" ]; then + echo "FAIL: top-level Configurations core-site 'cdkd.integ.marker' is '${TOP_CFG}', expected 'top-level' (issue #1383 NOT closed)" >&2 + aws emr describe-cluster --cluster-id "${CID_P1}" --region "${REGION}" --query 'Cluster.Configurations' >&2 + exit 1 +fi +NESTED_CFG="$(aws emr describe-cluster --cluster-id "${CID_P1}" --region "${REGION}" \ + --query "Cluster.Configurations[?Classification=='hadoop-env'].Configurations[0].Properties.CDKD_INTEG_NESTED | [0]" \ + --output text)" +if [ "${NESTED_CFG}" != "yes" ]; then + echo "FAIL: NESTED Configurations hadoop-env/export CDKD_INTEG_NESTED is '${NESTED_CFG}', expected 'yes' (recursive rename NOT applied)" >&2 + aws emr describe-cluster --cluster-id "${CID_P1}" --region "${REGION}" --query 'Cluster.Configurations' >&2 + exit 1 +fi +echo " top-level + NESTED ConfigurationProperties reached AWS (issue #1383 closed)" + +GROUPS_JSON="$(list_instance_groups_json "${CID_P1}")" +GROUP_CFG="$(printf '%s' "${GROUPS_JSON}" | jq -r ' + [ .[] | select(.InstanceGroupType == "MASTER") + | .Configurations[]? | select(.Classification == "core-site") + | .Properties["cdkd.integ.marker"] ] | first // empty')" +if [ "${GROUP_CFG}" != "master-group" ]; then + echo "FAIL: master instance-group Configurations 'cdkd.integ.marker' is '${GROUP_CFG}', expected 'master-group' (per-group rename NOT applied)" >&2 + echo " raw groups: ${GROUPS_JSON}" >&2 + exit 1 +fi +echo " per-instance-group ConfigurationProperties reached AWS" + +STEP_PROP="$(aws emr list-steps --cluster-id "${CID_P1}" --region "${REGION}" \ + --query "Steps[?Name=='cdkd-integ-step'].Config.Properties.\"cdkd.integ.step\" | [0]" \ + --output text)" +if [ "${STEP_PROP}" != "yes" ]; then + echo "FAIL: step Properties 'cdkd.integ.step' is '${STEP_PROP}', expected 'yes' (StepProperties rename NOT applied)" >&2 + aws emr list-steps --cluster-id "${CID_P1}" --region "${REGION}" --query 'Steps[].Config' >&2 + exit 1 +fi +echo " HadoopJarStep StepProperties reached AWS" + # --- Phase 2: in-place update ------------------------------------------ echo "==> Phase 2: re-deploy with CDKD_TEST_UPDATE=true (step concurrency, auto-termination, tags)" CDKD_TEST_UPDATE=true node "${LOCAL_DIST}" deploy "${STACK}" \ diff --git a/tests/integration/emr-instance-configs/lib/emr-instance-configs-stack.ts b/tests/integration/emr-instance-configs/lib/emr-instance-configs-stack.ts index e9459b585..433748a7c 100644 --- a/tests/integration/emr-instance-configs/lib/emr-instance-configs-stack.ts +++ b/tests/integration/emr-instance-configs/lib/emr-instance-configs-stack.ts @@ -143,6 +143,18 @@ export class EmrInstanceConfigsStack extends cdk.Stack { instanceCount: isUpdate ? 2 : 1, market: 'ON_DEMAND', name: 'cdkd-integ-task-group', + // Issue #1383: the standalone InstanceGroupConfig provider forwards this + // blob to AddInstanceGroups, where CFn `ConfigurationProperties` must be + // renamed to the SDK's `Properties` — a conversion site distinct from the + // inline `Cluster.Instances` one the emr-cluster fixture covers. + // `Configurations` is create-only, so it is identical in the UPDATE phase + // (which resizes InstanceCount only). + configurations: [ + { + classification: 'core-site', + configurationProperties: { 'cdkd.integ.marker': 'task-group' }, + }, + ], }); // AddInstanceGroups requires the cluster to be WAITING/RUNNING first; the // JobFlowId Ref already induces this ordering, but make it explicit. diff --git a/tests/integration/emr-instance-configs/verify.sh b/tests/integration/emr-instance-configs/verify.sh index ab59b7f4e..05c941a13 100755 --- a/tests/integration/emr-instance-configs/verify.sh +++ b/tests/integration/emr-instance-configs/verify.sh @@ -86,6 +86,23 @@ CLEANUP_TAG_VALUE="emr-instance-configs" LOCAL_DIST="${PWD}/../../../dist/cli.js" +# `aws emr list-instance-groups` is an AWS-CLI-CUSTOMIZED command: in a +# non-interactive shell it prints `Warning: Input is not a terminal (fd=0).` +# and then dies with `aws: [ERROR]: [Errno 22] Invalid argument` (verified +# 2026-08-09; `--no-paginate --no-cli-pager JSON array of InstanceGroups + ( cd "${REPO_ROOT}" && REGION="${REGION}" node --input-type=module -e " +import { EMRClient, ListInstanceGroupsCommand } from '@aws-sdk/client-emr'; +const client = new EMRClient({ region: process.env.REGION }); +const res = await client.send(new ListInstanceGroupsCommand({ ClusterId: process.argv[1] })); +process.stdout.write(JSON.stringify(res.InstanceGroups ?? [])); +" "$1" ) || return 1 +} + + # Ids of ACTIVE (not terminated) clusters named like the fixture and carrying # the fixture's constant tag. active_tagged_cluster_ids() { @@ -280,6 +297,23 @@ if [ "${PROVISIONED_BY}" != "sdk" ]; then fi echo " group routed via SDK provider (provisionedBy=sdk)" +# --- Assertion: issue #1383 per-group ConfigurationProperties ---------- +# CFn spells the property bag `ConfigurationProperties`; the SDK member is +# `Properties`, and the AWS SDK v3 serializer drops unknown members — so before +# the fix AddInstanceGroups silently created the group WITHOUT its application +# configuration while cdkd reported success. Read it back from AWS. +GROUPS_JSON="$(list_instance_groups_json "${CID_P1}")" +GROUP_CFG="$(printf '%s' "${GROUPS_JSON}" | jq -r --arg gid "${GID_P1}" ' + [ .[] | select(.Id == $gid) + | .Configurations[]? | select(.Classification == "core-site") + | .Properties["cdkd.integ.marker"] ] | first // empty')" +if [ "${GROUP_CFG}" != "task-group" ]; then + echo "FAIL: TASK group Configurations 'cdkd.integ.marker' is '${GROUP_CFG}', expected 'task-group' (issue #1383 NOT closed)" >&2 + echo " raw groups: ${GROUPS_JSON}" >&2 + exit 1 +fi +echo " standalone-group ConfigurationProperties reached AWS (issue #1383 closed)" + # --- Phase 2: in-place resize ------------------------------------------ echo "==> Phase 2: re-deploy with CDKD_TEST_UPDATE=true (resize TASK group 1 -> 2)" CDKD_TEST_UPDATE=true node "${LOCAL_DIST}" deploy "${STACK}" \ diff --git a/tests/unit/provisioning/emr-configuration.test.ts b/tests/unit/provisioning/emr-configuration.test.ts new file mode 100644 index 000000000..00f29745d --- /dev/null +++ b/tests/unit/provisioning/emr-configuration.test.ts @@ -0,0 +1,136 @@ +import { describe, it, expect } from 'vite-plus/test'; +import { + toSdkConfigurations, + toSdkInstanceTypeConfigs, + toSdkStepConfigs, +} from '../../../src/provisioning/emr-configuration.js'; + +describe('toSdkConfigurations (issue #1383)', () => { + it('renames ConfigurationProperties to the SDK Properties member', () => { + expect( + toSdkConfigurations([ + { + Classification: 'spark-defaults', + ConfigurationProperties: { 'spark.executor.memory': '4g' }, + }, + ]) + ).toEqual([ + { Classification: 'spark-defaults', Properties: { 'spark.executor.memory': '4g' } }, + ]); + }); + + it('renames at EVERY nesting level of Configurations', () => { + expect( + toSdkConfigurations([ + { + Classification: 'hadoop-env', + ConfigurationProperties: { TOP: '1' }, + Configurations: [ + { + Classification: 'export', + ConfigurationProperties: { JAVA_HOME: '/usr/lib/jvm' }, + Configurations: [ + { Classification: 'deep', ConfigurationProperties: { DEEP: 'yes' } }, + ], + }, + ], + }, + ]) + ).toEqual([ + { + Classification: 'hadoop-env', + Properties: { TOP: '1' }, + Configurations: [ + { + Classification: 'export', + Properties: { JAVA_HOME: '/usr/lib/jvm' }, + Configurations: [{ Classification: 'deep', Properties: { DEEP: 'yes' } }], + }, + ], + }, + ]); + }); + + it('leaves a configuration without ConfigurationProperties untouched', () => { + expect(toSdkConfigurations([{ Classification: 'core-site' }])).toEqual([ + { Classification: 'core-site' }, + ]); + }); + + it('passes undefined and non-array inputs through unchanged', () => { + expect(toSdkConfigurations(undefined)).toBeUndefined(); + // An unresolved intrinsic must not become a confusing local crash — AWS + // surfaces the real validation error instead. + const intrinsic = { Ref: 'SomeParam' }; + expect(toSdkConfigurations(intrinsic)).toBe(intrinsic); + }); +}); + +describe('toSdkStepConfigs (issue #1383)', () => { + it('renames HadoopJarStep.StepProperties to the SDK Properties member', () => { + expect( + toSdkStepConfigs([ + { + Name: 'my-step', + ActionOnFailure: 'CONTINUE', + HadoopJarStep: { + Jar: 's3://bucket/job.jar', + MainClass: 'com.example.Main', + Args: ['--input', 's3://in'], + StepProperties: [{ Key: 'k', Value: 'v' }], + }, + }, + ]) + ).toEqual([ + { + Name: 'my-step', + ActionOnFailure: 'CONTINUE', + HadoopJarStep: { + Jar: 's3://bucket/job.jar', + MainClass: 'com.example.Main', + Args: ['--input', 's3://in'], + Properties: [{ Key: 'k', Value: 'v' }], + }, + }, + ]); + }); + + it('leaves a step without StepProperties untouched', () => { + const steps = [{ Name: 's', HadoopJarStep: { Jar: 's3://bucket/job.jar' } }]; + expect(toSdkStepConfigs(steps)).toEqual(steps); + }); + + it('passes undefined and non-array inputs through unchanged', () => { + expect(toSdkStepConfigs(undefined)).toBeUndefined(); + const intrinsic = { Ref: 'SomeParam' }; + expect(toSdkStepConfigs(intrinsic)).toBe(intrinsic); + }); +}); + +describe('toSdkInstanceTypeConfigs (issue #1383)', () => { + it('converts each element nested Configurations', () => { + expect( + toSdkInstanceTypeConfigs([ + { + InstanceType: 'm5.xlarge', + Configurations: [ + { Classification: 'yarn-site', ConfigurationProperties: { 'yarn.a': 'b' } }, + ], + }, + { InstanceType: 'm5.2xlarge' }, + ]) + ).toEqual([ + { + InstanceType: 'm5.xlarge', + Configurations: [{ Classification: 'yarn-site', Properties: { 'yarn.a': 'b' } }], + }, + { InstanceType: 'm5.2xlarge' }, + ]); + }); + + it('passes undefined and non-array inputs through unchanged', () => { + expect(toSdkInstanceTypeConfigs(undefined)).toBeUndefined(); + const intrinsic = { Ref: 'SomeParam' }; + expect(toSdkInstanceTypeConfigs(intrinsic)).toBe(intrinsic); + }); +}); diff --git a/tests/unit/provisioning/providers/emr-cluster-provider.test.ts b/tests/unit/provisioning/providers/emr-cluster-provider.test.ts index a670a8d74..1d82c818e 100644 --- a/tests/unit/provisioning/providers/emr-cluster-provider.test.ts +++ b/tests/unit/provisioning/providers/emr-cluster-provider.test.ts @@ -212,6 +212,98 @@ describe('EMRClusterProvider create', () => { expect(fleets[1]!['TargetSpotCapacity']).toBe(2); }); + it('renames ConfigurationProperties / StepProperties to the SDK Properties member on RunJobFlow (issue #1383)', async () => { + routeSend({ + RunJobFlowCommand: { JobFlowId: CLUSTER_ID }, + DescribeClusterCommand: [clusterOf('WAITING')], + }); + + await newProvider().create('MyCluster', RESOURCE_TYPE, { + ...BASE_PROPS, + Configurations: [ + { + Classification: 'spark-defaults', + ConfigurationProperties: { 'spark.executor.memory': '4g' }, + }, + ], + Steps: [ + { + Name: 'my-step', + ActionOnFailure: 'CONTINUE', + HadoopJarStep: { + Jar: 's3://bucket/job.jar', + StepProperties: [{ Key: 'k', Value: 'v' }], + }, + }, + ], + Instances: { + Ec2SubnetId: 'subnet-abc', + MasterInstanceGroup: { + InstanceCount: 1, + InstanceType: 'm5.xlarge', + Configurations: [ + { Classification: 'yarn-site', ConfigurationProperties: { 'yarn.a': 'b' } }, + ], + }, + }, + }); + + const input = callsOf(RunJobFlowCommand)[0]!.input; + expect(input['Configurations']).toEqual([ + { Classification: 'spark-defaults', Properties: { 'spark.executor.memory': '4g' } }, + ]); + expect(input['Steps']).toEqual([ + { + Name: 'my-step', + ActionOnFailure: 'CONTINUE', + HadoopJarStep: { + Jar: 's3://bucket/job.jar', + Properties: [{ Key: 'k', Value: 'v' }], + }, + }, + ]); + const groups = (input['Instances'] as { InstanceGroups: Array> }) + .InstanceGroups; + expect(groups[0]!['Configurations']).toEqual([ + { Classification: 'yarn-site', Properties: { 'yarn.a': 'b' } }, + ]); + }); + + it('renames per-instance-type-config Configurations inside an instance fleet (issue #1383)', async () => { + routeSend({ + RunJobFlowCommand: { JobFlowId: CLUSTER_ID }, + DescribeClusterCommand: [clusterOf('WAITING')], + }); + + await newProvider().create('MyCluster', RESOURCE_TYPE, { + ...BASE_PROPS, + Instances: { + Ec2SubnetIds: ['subnet-abc'], + MasterInstanceFleet: { + TargetOnDemandCapacity: 1, + InstanceTypeConfigs: [ + { + InstanceType: 'm5.xlarge', + Configurations: [ + { Classification: 'hive-site', ConfigurationProperties: { 'hive.a': 'b' } }, + ], + }, + ], + }, + }, + }); + + const input = callsOf(RunJobFlowCommand)[0]!.input; + const fleets = (input['Instances'] as { InstanceFleets: Array> }) + .InstanceFleets; + expect(fleets[0]!['InstanceTypeConfigs']).toEqual([ + { + InstanceType: 'm5.xlarge', + Configurations: [{ Classification: 'hive-site', Properties: { 'hive.a': 'b' } }], + }, + ]); + }); + it('errors and best-effort terminates when the cluster reaches a terminal state during create', async () => { routeSend({ RunJobFlowCommand: { JobFlowId: CLUSTER_ID }, diff --git a/tests/unit/provisioning/providers/emr-instance-fleet-config-provider.test.ts b/tests/unit/provisioning/providers/emr-instance-fleet-config-provider.test.ts index 555a9ff8b..3a8864ff5 100644 --- a/tests/unit/provisioning/providers/emr-instance-fleet-config-provider.test.ts +++ b/tests/unit/provisioning/providers/emr-instance-fleet-config-provider.test.ts @@ -140,6 +140,54 @@ describe('EMRInstanceFleetConfigProvider create', () => { }); }); + it('renames InstanceTypeConfigs[].Configurations[].ConfigurationProperties on create AND update (issue #1383)', async () => { + const cfnInstanceTypeConfigs = [ + { + InstanceType: 'm5.xlarge', + WeightedCapacity: 1, + Configurations: [ + { Classification: 'spark-defaults', ConfigurationProperties: { 'spark.a': 'b' } }, + ], + }, + ]; + const sdkInstanceTypeConfigs = [ + { + InstanceType: 'm5.xlarge', + WeightedCapacity: 1, + Configurations: [{ Classification: 'spark-defaults', Properties: { 'spark.a': 'b' } }], + }, + ]; + + routeSend({ + AddInstanceFleetCommand: { ClusterId: CLUSTER_ID, InstanceFleetId: FLEET_ID }, + ListInstanceFleetsCommand: fleetOf('RUNNING'), + ModifyInstanceFleetCommand: {}, + }); + + await newProvider().create('Fleet', RESOURCE_TYPE, { + ...BASE_PROPS, + InstanceTypeConfigs: cfnInstanceTypeConfigs, + }); + expect( + (callsOf(AddInstanceFleetCommand)[0]!.input.InstanceFleet as Record)[ + 'InstanceTypeConfigs' + ] + ).toEqual(sdkInstanceTypeConfigs); + + await newProvider().update( + 'Fleet', + FLEET_ID, + RESOURCE_TYPE, + { ...BASE_PROPS, InstanceTypeConfigs: cfnInstanceTypeConfigs }, + { ...BASE_PROPS } + ); + expect( + (callsOf(ModifyInstanceFleetCommand)[0]!.input.InstanceFleet as Record)[ + 'InstanceTypeConfigs' + ] + ).toEqual(sdkInstanceTypeConfigs); + }); + it('rejects when ClusterId is absent', async () => { routeSend({}); const { ClusterId: _drop, ...noParent } = BASE_PROPS; diff --git a/tests/unit/provisioning/providers/emr-instance-group-config-provider.test.ts b/tests/unit/provisioning/providers/emr-instance-group-config-provider.test.ts index ee9fa2866..6f8332d87 100644 --- a/tests/unit/provisioning/providers/emr-instance-group-config-provider.test.ts +++ b/tests/unit/provisioning/providers/emr-instance-group-config-provider.test.ts @@ -143,6 +143,27 @@ describe('EMRInstanceGroupConfigProvider create', () => { }); }); + it('renames Configurations[].ConfigurationProperties to the SDK Properties member (issue #1383)', async () => { + routeSend({ + AddInstanceGroupsCommand: { JobFlowId: CLUSTER_ID, InstanceGroupIds: [GROUP_ID] }, + ListInstanceGroupsCommand: groupOf('RUNNING'), + }); + + await newProvider().create('Grp', RESOURCE_TYPE, { + ...BASE_PROPS, + Configurations: [ + { Classification: 'yarn-site', ConfigurationProperties: { 'yarn.nodemanager.a': 'b' } }, + ], + }); + + const groups = callsOf(AddInstanceGroupsCommand)[0]!.input.InstanceGroups as Array< + Record + >; + expect(groups[0]!['Configurations']).toEqual([ + { Classification: 'yarn-site', Properties: { 'yarn.nodemanager.a': 'b' } }, + ]); + }); + it('rejects when JobFlowId is absent', async () => { routeSend({}); const { JobFlowId: _drop, ...noParent } = BASE_PROPS; From eca3870111c21b7e047803001daeefe66c87fd81 Mon Sep 17 00:00:00 2001 From: go-to-k <24818752+go-to-k@users.noreply.github.com> Date: Sun, 9 Aug 2026 13:20:29 +0900 Subject: [PATCH 2/3] test(integ): record the emr-cluster and emr-instance-configs runs --- docs/_generated/integ-last-run.tsv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_generated/integ-last-run.tsv b/docs/_generated/integ-last-run.tsv index f9b5d0d0a..dbdfd2f0f 100644 --- a/docs/_generated/integ-last-run.tsv +++ b/docs/_generated/integ-last-run.tsv @@ -97,8 +97,8 @@ efs-immutable-replacement 2026-08-01T09:51:12Z PASS 63 verify.sh 0801 regression efs-lambda 2026-07-26T18:36:31Z PASS 499 standard 0727b sweep-b4 staleness re-run (rc=0); account clean efs-standalone 2026-07-26T18:36:31Z PASS 148 verify.sh 0727b sweep-b4 staleness re-run (rc=0); account clean elasticache-replicationgroup-getatt 2026-07-20T14:51:32Z PASS 596 verify.sh CC GetAtt PrimaryEndPoint real hostname, 0 orphans -emr-cluster 2026-08-02T14:15:43Z PASS 605 verify.sh #1339 fix verified: cleanup warning gone on clean run (gone_probe-gated); 12 del/0 err, 0 orphans -emr-instance-configs 2026-08-02T06:50:46Z PASS 997 verify.sh 0802 heavy TTL sweep; deploy+resize+destroy ok, 0 orphans +emr-cluster 2026-08-09T04:01:57Z PASS 700 verify.sh #1383 top-level+nested+group Configurations and StepProperties asserted; 12 deleted, 0 errors, 0 orphans +emr-instance-configs 2026-08-09T04:19:21Z PASS 1250 verify.sh #1383 standalone-group ConfigurationProperties asserted; destroy clean, 0 orphans event-driven 2026-07-26T18:36:31Z PASS 42 standard 0727b sweep-b4 staleness re-run (rc=0); account clean eventbridge 2026-07-27T11:04:52Z PASS 87 verify.sh issue #1267 post-review re-run; 10 del 0 err, 0 orphans eventbridge-api-destination 2026-07-21T14:48:06Z PASS 70 verify.sh rc ok, orph clean From f5369b9225b583be3f6ace7da6500fe6b7e6e42f Mon Sep 17 00:00:00 2001 From: go-to-k <24818752+go-to-k@users.noreply.github.com> Date: Sun, 9 Aug 2026 13:29:20 +0900 Subject: [PATCH 3/3] test(integ): regenerate cli-flag-coverage and paginate the instance-group helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cli-flag-coverage was stale: the new node/SDK helper introduces --input-type and the CLI-customization comment names --no-paginate / --no-cli-pager, and the matrix counts DECLARED flags found in verify.sh. CI caught it; regenerating here (integ- and scenario-coverage were already regenerated in the first commit). - the helper now follows Marker, for parity with the provider's own paginated listInstanceGroups — a partial first page would silently satisfy the Configurations assertions. --- docs/_generated/cli-flag-coverage.json | 3 +++ docs/cli-flag-coverage.md | 10 +++++----- tests/integration/emr-cluster/verify.sh | 15 ++++++++++++--- tests/integration/emr-instance-configs/verify.sh | 15 ++++++++++++--- 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/docs/_generated/cli-flag-coverage.json b/docs/_generated/cli-flag-coverage.json index 5a8f33702..51bd7fb72 100644 --- a/docs/_generated/cli-flag-coverage.json +++ b/docs/_generated/cli-flag-coverage.json @@ -1317,6 +1317,7 @@ "--include-non-importable", "--include-planned-deletion", "--input", + "--input-type", "--instance-group-id", "--instance-id", "--instance-ids", @@ -1360,10 +1361,12 @@ "--no-audit", "--no-buffer", "--no-build", + "--no-cli-pager", "--no-deletion-protection", "--no-deletion-protection-enabled", "--no-disable-api-termination", "--no-fund", + "--no-paginate", "--no-path-metadata", "--no-prefix-user-supplied-names", "--no-pull", diff --git a/docs/cli-flag-coverage.md b/docs/cli-flag-coverage.md index 21bcd5d56..6ed778559 100644 --- a/docs/cli-flag-coverage.md +++ b/docs/cli-flag-coverage.md @@ -62,7 +62,7 @@ Reviewer judgment required per flag — many of these are pure-logic flags adequ | `--verbose` | [`cache-streaming`](../tests/integration/cache-streaming/)
[`conditions`](../tests/integration/conditions/)
[`cross-region-state-bucket`](../tests/integration/cross-region-state-bucket/)
[`deletion-ordering-complex`](../tests/integration/deletion-ordering-complex/)
[`destroy-interrupt`](../tests/integration/destroy-interrupt/)
[`docdb-neptune`](../tests/integration/docdb-neptune/)
[`drift-revert`](../tests/integration/drift-revert/)
[`drift-revert-arrays`](../tests/integration/drift-revert-arrays/)
[`drift-revert-vpc`](../tests/integration/drift-revert-vpc/)
[`dynamodb-globaltable`](../tests/integration/dynamodb-globaltable/)
[`ec2-instance-fanout`](../tests/integration/ec2-instance-fanout/)
[`elasticache-replicationgroup-getatt`](../tests/integration/elasticache-replicationgroup-getatt/)
[`export`](../tests/integration/export/)
[`export-nested-stack`](../tests/integration/export-nested-stack/)
[`iam-managed-policy`](../tests/integration/iam-managed-policy/)
[`import-attributes`](../tests/integration/import-attributes/)
[`import-auto-mode`](../tests/integration/import-auto-mode/)
[`import-nested-stack`](../tests/integration/import-nested-stack/)
[`local-invoke-agentcore-from-state`](../tests/integration/local-invoke-agentcore-from-state/)
[`local-start-alb-from-state`](../tests/integration/local-start-alb-from-state/)
[`macro-expansion`](../tests/integration/macro-expansion/)
[`migrate-from-bare-cfn`](../tests/integration/migrate-from-bare-cfn/)
[`opensearch-domain-getatt`](../tests/integration/opensearch-domain-getatt/)
[`redshift-cluster-getatt`](../tests/integration/redshift-cluster-getatt/)
[`remove-protection`](../tests/integration/remove-protection/)
[`rollback-sqs-cooldown`](../tests/integration/rollback-sqs-cooldown/)
[`stepfunctions`](../tests/integration/stepfunctions/)
[`tags-propagation`](../tests/integration/tags-propagation/)
[`throttle-wide-dag`](../tests/integration/throttle-wide-dag/) | | `--yes` | [`agentcore-tools`](../tests/integration/agentcore-tools/)
[`apigateway`](../tests/integration/apigateway/)
[`apigatewayv2-update-removal`](../tests/integration/apigatewayv2-update-removal/)
[`apigw-gateway-response`](../tests/integration/apigw-gateway-response/)
[`apigw-stage-throttling`](../tests/integration/apigw-stage-throttling/)
[`apigw-usage-plan-key`](../tests/integration/apigw-usage-plan-key/)
[`appconfig`](../tests/integration/appconfig/)
[`asset-auto-create`](../tests/integration/asset-auto-create/)
[`asset-bootstrap`](../tests/integration/asset-bootstrap/)
[`asset-migration`](../tests/integration/asset-migration/)
[`aws-custom-resource`](../tests/integration/aws-custom-resource/)
[`backup`](../tests/integration/backup/)
[`bootstrap-free-region`](../tests/integration/bootstrap-free-region/)
[`bucket-deployment`](../tests/integration/bucket-deployment/)
[`budgets`](../tests/integration/budgets/)
[`cache-streaming`](../tests/integration/cache-streaming/)
[`cc-api-fallback`](../tests/integration/cc-api-fallback/)
[`cc-api-fallback-transitions`](../tests/integration/cc-api-fallback-transitions/)
[`cc-getatt-readback`](../tests/integration/cc-getatt-readback/)
[`cc-protection-flip`](../tests/integration/cc-protection-flip/)
[`cc-protection-flip-eks`](../tests/integration/cc-protection-flip-eks/)
[`ci-cd`](../tests/integration/ci-cd/)
[`cloudfront-function-url`](../tests/integration/cloudfront-function-url/)
[`cloudwatch-anomaly-detector`](../tests/integration/cloudwatch-anomaly-detector/)
[`codecommit`](../tests/integration/codecommit/)
[`codedeploy-lambda-deployment-group`](../tests/integration/codedeploy-lambda-deployment-group/)
[`cognito`](../tests/integration/cognito/)
[`cognito-custom-attribute-add`](../tests/integration/cognito-custom-attribute-add/)
[`cognito-identity-pool`](../tests/integration/cognito-identity-pool/)
[`cognito-lambda-triggers`](../tests/integration/cognito-lambda-triggers/)
[`cognito-resource-server`](../tests/integration/cognito-resource-server/)
[`cognito-userpool-user-ref`](../tests/integration/cognito-userpool-user-ref/)
[`conditions`](../tests/integration/conditions/)
[`conditions-and-if`](../tests/integration/conditions-and-if/)
[`conditions-update-2`](../tests/integration/conditions-update-2/)
[`custom-resource-getatt-data`](../tests/integration/custom-resource-getatt-data/)
[`data-analytics`](../tests/integration/data-analytics/)
[`deep-getatt-chains`](../tests/integration/deep-getatt-chains/)
[`deletion-ordering-complex`](../tests/integration/deletion-ordering-complex/)
[`deletion-policy-snapshot`](../tests/integration/deletion-policy-snapshot/)
[`deletion-policy-snapshot-heavy`](../tests/integration/deletion-policy-snapshot-heavy/)
[`destroy-data-guard`](../tests/integration/destroy-data-guard/)
[`destroy-interrupt`](../tests/integration/destroy-interrupt/)
[`diff-intrinsic-target-change`](../tests/integration/diff-intrinsic-target-change/)
[`dlm-lifecycle-policy`](../tests/integration/dlm-lifecycle-policy/)
[`docker-image-asset`](../tests/integration/docker-image-asset/)
[`dsql`](../tests/integration/dsql/)
[`dynamodb-autoscaling`](../tests/integration/dynamodb-autoscaling/)
[`dynamodb-gsi-update`](../tests/integration/dynamodb-gsi-update/)
[`dynamodb-ondemand`](../tests/integration/dynamodb-ondemand/)
[`dynamodb-sse`](../tests/integration/dynamodb-sse/)
[`dynamodb-stream-filter`](../tests/integration/dynamodb-stream-filter/)
[`dynamodb-streams`](../tests/integration/dynamodb-streams/)
[`dynamodb-tableclass-switch`](../tests/integration/dynamodb-tableclass-switch/)
[`dynamodb-ttl-attr-change`](../tests/integration/dynamodb-ttl-attr-change/)
[`ec2-instance`](../tests/integration/ec2-instance/)
[`ec2-instance-fanout`](../tests/integration/ec2-instance-fanout/)
[`ecr-scanning`](../tests/integration/ecr-scanning/)
[`ecs-fargate`](../tests/integration/ecs-fargate/)
[`ecs-schedule-targets`](../tests/integration/ecs-schedule-targets/)
[`ecs-service-update-props`](../tests/integration/ecs-service-update-props/)
[`efs-immutable-replacement`](../tests/integration/efs-immutable-replacement/)
[`efs-standalone`](../tests/integration/efs-standalone/)
[`elasticache-replicationgroup-getatt`](../tests/integration/elasticache-replicationgroup-getatt/)
[`emr-cluster`](../tests/integration/emr-cluster/)
[`emr-instance-configs`](../tests/integration/emr-instance-configs/)
[`eventbridge`](../tests/integration/eventbridge/)
[`eventbridge-api-destination`](../tests/integration/eventbridge-api-destination/)
[`eventbridge-archive`](../tests/integration/eventbridge-archive/)
[`eventbridge-input-transformer`](../tests/integration/eventbridge-input-transformer/)
[`eventbridge-pipes`](../tests/integration/eventbridge-pipes/)
[`eventbridge-scheduler`](../tests/integration/eventbridge-scheduler/)
[`eventsourcemapping-race`](../tests/integration/eventsourcemapping-race/)
[`export`](../tests/integration/export/)
[`export-nested-stack`](../tests/integration/export-nested-stack/)
[`fifo-sqs-event-source`](../tests/integration/fifo-sqs-event-source/)
[`fsx-lustre`](../tests/integration/fsx-lustre/)
[`fsx-ontap`](../tests/integration/fsx-ontap/)
[`fsx-openzfs`](../tests/integration/fsx-openzfs/)
[`fsx-windows`](../tests/integration/fsx-windows/)
[`gc-custom-asset-names`](../tests/integration/gc-custom-asset-names/)
[`getatt-fallback-guard`](../tests/integration/getatt-fallback-guard/)
[`getstackoutput-crossregion`](../tests/integration/getstackoutput-crossregion/)
[`glue-securityconfig-replace`](../tests/integration/glue-securityconfig-replace/)
[`glue-update-hardening`](../tests/integration/glue-update-hardening/)
[`iam-access-key`](../tests/integration/iam-access-key/)
[`iam-managed-policy`](../tests/integration/iam-managed-policy/)
[`iam-oidc-provider`](../tests/integration/iam-oidc-provider/)
[`iam-propagation-stress`](../tests/integration/iam-propagation-stress/)
[`iam-role-policies-drift-clean`](../tests/integration/iam-role-policies-drift-clean/)
[`iam-role-prefixed-name-update`](../tests/integration/iam-role-prefixed-name-update/)
[`import-attributes`](../tests/integration/import-attributes/)
[`import-auto-mode`](../tests/integration/import-auto-mode/)
[`import-nested-stack`](../tests/integration/import-nested-stack/)
[`inplace-attr-propagation`](../tests/integration/inplace-attr-propagation/)
[`kinesis-esm-filter`](../tests/integration/kinesis-esm-filter/)
[`kinesis-stream-mode-switch`](../tests/integration/kinesis-stream-mode-switch/)
[`lambda`](../tests/integration/lambda/)
[`lambda-alias-provisioned-concurrency`](../tests/integration/lambda-alias-provisioned-concurrency/)
[`lambda-arch-switch`](../tests/integration/lambda-arch-switch/)
[`lambda-config-field-removal`](../tests/integration/lambda-config-field-removal/)
[`lambda-destinations`](../tests/integration/lambda-destinations/)
[`lambda-env-removal`](../tests/integration/lambda-env-removal/)
[`lambda-event-invoke-config-update`](../tests/integration/lambda-event-invoke-config-update/)
[`lambda-layer-version-update`](../tests/integration/lambda-layer-version-update/)
[`lambda-log-retention`](../tests/integration/lambda-log-retention/)
[`lambda-microvm-image`](../tests/integration/lambda-microvm-image/)
[`lambda-reserved-concurrency`](../tests/integration/lambda-reserved-concurrency/)
[`lambda-snapstart`](../tests/integration/lambda-snapstart/)
[`launchtemplate-asg-inplace`](../tests/integration/launchtemplate-asg-inplace/)
[`loggroup-class-guard`](../tests/integration/loggroup-class-guard/)
[`loggroup-kms-associate`](../tests/integration/loggroup-kms-associate/)
[`migrate-from-bare-cfn`](../tests/integration/migrate-from-bare-cfn/)
[`migrate-from-cfn`](../tests/integration/migrate-from-cfn/)
[`multi-asset`](../tests/integration/multi-asset/)
[`nested-stack-3level`](../tests/integration/nested-stack-3level/)
[`nested-stack-deep`](../tests/integration/nested-stack-deep/)
[`nodejs-function`](../tests/integration/nodejs-function/)
[`opensearch-domain-getatt`](../tests/integration/opensearch-domain-getatt/)
[`propagation-races-2`](../tests/integration/propagation-races-2/)
[`raw-cfn-conditions-params`](../tests/integration/raw-cfn-conditions-params/)
[`rds-aurora`](../tests/integration/rds-aurora/)
[`rds-dbinstance-backfill`](../tests/integration/rds-dbinstance-backfill/)
[`rds-full-stack`](../tests/integration/rds-full-stack/)
[`recreate-mixed-direction`](../tests/integration/recreate-mixed-direction/)
[`recreate-via-cc-api`](../tests/integration/recreate-via-cc-api/)
[`recreate-via-sdk-provider`](../tests/integration/recreate-via-sdk-provider/)
[`redshift-cluster-getatt`](../tests/integration/redshift-cluster-getatt/)
[`rename-refactor`](../tests/integration/rename-refactor/)
[`replacement-fanout`](../tests/integration/replacement-fanout/)
[`replacement-immutable-name`](../tests/integration/replacement-immutable-name/)
[`rollback-deletion-policy-snapshot`](../tests/integration/rollback-deletion-policy-snapshot/)
[`route53`](../tests/integration/route53/)
[`s3-asset-deploy`](../tests/integration/s3-asset-deploy/)
[`s3-cloudfront`](../tests/integration/s3-cloudfront/)
[`s3-directory-bucket`](../tests/integration/s3-directory-bucket/)
[`s3-event-notification`](../tests/integration/s3-event-notification/)
[`s3-lifecycle`](../tests/integration/s3-lifecycle/)
[`s3-object-lock`](../tests/integration/s3-object-lock/)
[`s3-replication-and-filter`](../tests/integration/s3-replication-and-filter/)
[`s3-tables`](../tests/integration/s3-tables/)
[`s3-vectors`](../tests/integration/s3-vectors/)
[`scheduler-custom-group`](../tests/integration/scheduler-custom-group/)
[`schema-v5-to-v6-migration`](../tests/integration/schema-v5-to-v6-migration/)
[`schema-v6-to-v7-migration`](../tests/integration/schema-v6-to-v7-migration/)
[`schema-v7-to-v8-migration`](../tests/integration/schema-v7-to-v8-migration/)
[`sdk-ccapi-crossref`](../tests/integration/sdk-ccapi-crossref/)
[`secrets-dynamic-ref`](../tests/integration/secrets-dynamic-ref/)
[`secrets-rotation-schedule`](../tests/integration/secrets-rotation-schedule/)
[`serverless-api`](../tests/integration/serverless-api/)
[`servicediscovery`](../tests/integration/servicediscovery/)
[`servicediscovery-namespaces`](../tests/integration/servicediscovery-namespaces/)
[`ses-identity`](../tests/integration/ses-identity/)
[`sg-circular-dependency`](../tests/integration/sg-circular-dependency/)
[`sns-event-source`](../tests/integration/sns-event-source/)
[`sns-inline-subscription`](../tests/integration/sns-inline-subscription/)
[`sns-pending-subscription`](../tests/integration/sns-pending-subscription/)
[`sns-sqs-event`](../tests/integration/sns-sqs-event/)
[`sns-subscription-filter`](../tests/integration/sns-subscription-filter/)
[`sqs-esm-max-concurrency`](../tests/integration/sqs-esm-max-concurrency/)
[`stepfunctions`](../tests/integration/stepfunctions/)
[`stepfunctions-logging`](../tests/integration/stepfunctions-logging/)
[`stepfunctions-s3-definition`](../tests/integration/stepfunctions-s3-definition/)
[`synthetics-canary`](../tests/integration/synthetics-canary/)
[`throttle-wide-dag`](../tests/integration/throttle-wide-dag/)
[`update-replace`](../tests/integration/update-replace/)
[`wait-condition-handle`](../tests/integration/wait-condition-handle/) | -## Long-form flags referenced in integs but NOT declared in src/cli/options.ts (276) +## Long-form flags referenced in integs but NOT declared in src/cli/options.ts (279) These are mostly third-party CLI flags (`--query` for `aws` / `--region` for `aws s3 ls` / `--no-paginate` / etc.) OR typos of cdkd flag names. Listed here for visibility — review only if a row matches a cdkd flag with a misspelling. @@ -174,6 +174,7 @@ These are mostly third-party CLI flags (`--query` for `aws` / `--region` for `aw - `--include-non-importable` - `--include-planned-deletion` - `--input` +- `--input-type` - `--instance-group-id` - `--instance-id` - `--instance-ids` @@ -217,10 +218,12 @@ These are mostly third-party CLI flags (`--query` for `aws` / `--region` for `aw - `--no-audit` - `--no-buffer` - `--no-build` +- `--no-cli-pager` - `--no-deletion-protection` - `--no-deletion-protection-enabled` - `--no-disable-api-termination` - `--no-fund` +- `--no-paginate` - `--no-path-metadata` - `--no-prefix-user-supplied-names` - `--no-pull` @@ -263,8 +266,5 @@ These are mostly third-party CLI flags (`--query` for `aws` / `--region` for `aw - `--recursive` - `--replication-group-id` - `--repository-description` -- `--repository-name` -- `--repository-names` -- `--require-approval` -_(76 more entries truncated — see `docs/_generated/cli-flag-coverage.json` for the full list.)_ +_(79 more entries truncated — see `docs/_generated/cli-flag-coverage.json` for the full list.)_ diff --git a/tests/integration/emr-cluster/verify.sh b/tests/integration/emr-cluster/verify.sh index e0d40bbe0..8c064bd1f 100755 --- a/tests/integration/emr-cluster/verify.sh +++ b/tests/integration/emr-cluster/verify.sh @@ -104,12 +104,21 @@ list_instance_groups_json() { # $1 = cluster id -> JSON array of InstanceGroups ( cd "${REPO_ROOT}" && REGION="${REGION}" node --input-type=module -e " import { EMRClient, ListInstanceGroupsCommand } from '@aws-sdk/client-emr'; const client = new EMRClient({ region: process.env.REGION }); -const res = await client.send(new ListInstanceGroupsCommand({ ClusterId: process.argv[1] })); -process.stdout.write(JSON.stringify(res.InstanceGroups ?? [])); +const groups = []; +let marker; +// Follow Marker for parity with the provider's own paginated listInstanceGroups +// — a partial first page would silently satisfy the assertions below. +do { + const res = await client.send( + new ListInstanceGroupsCommand({ ClusterId: process.argv[1], Marker: marker }) + ); + groups.push(...(res.InstanceGroups ?? [])); + marker = res.Marker; +} while (marker); +process.stdout.write(JSON.stringify(groups)); " "$1" ) || return 1 } - # Ids of ACTIVE (not terminated) clusters named like the fixture and carrying # the fixture's constant tag. # Ids of ACTIVE clusters named like the fixture and carrying its constant tag. diff --git a/tests/integration/emr-instance-configs/verify.sh b/tests/integration/emr-instance-configs/verify.sh index 05c941a13..20d41e14a 100755 --- a/tests/integration/emr-instance-configs/verify.sh +++ b/tests/integration/emr-instance-configs/verify.sh @@ -97,12 +97,21 @@ list_instance_groups_json() { # $1 = cluster id -> JSON array of InstanceGroups ( cd "${REPO_ROOT}" && REGION="${REGION}" node --input-type=module -e " import { EMRClient, ListInstanceGroupsCommand } from '@aws-sdk/client-emr'; const client = new EMRClient({ region: process.env.REGION }); -const res = await client.send(new ListInstanceGroupsCommand({ ClusterId: process.argv[1] })); -process.stdout.write(JSON.stringify(res.InstanceGroups ?? [])); +const groups = []; +let marker; +// Follow Marker for parity with the provider's own paginated listInstanceGroups +// — a partial first page would silently satisfy the assertions below. +do { + const res = await client.send( + new ListInstanceGroupsCommand({ ClusterId: process.argv[1], Marker: marker }) + ); + groups.push(...(res.InstanceGroups ?? [])); + marker = res.Marker; +} while (marker); +process.stdout.write(JSON.stringify(groups)); " "$1" ) || return 1 } - # Ids of ACTIVE (not terminated) clusters named like the fixture and carrying # the fixture's constant tag. active_tagged_cluster_ids() {