aws_route53: delete_existing is creating and deleting 2x

[BwL1289]

Describe the bug

It's possible there's some user error in here.

When delete_existing=True, and something goes awry from another resource related to the A record (see below for reproduction steps), I am seeing the custom resource run twice so the a record gets created, then deleted, then created, and then finally deleted so by the time the stack finishes updating, there's no A record at all.

Here are the logs:

INIT_START Runtime Version: nodejs:20.v57	Runtime Version ARN: arn:aws:lambda:<redacted>>::runtime:9d084cce5cc7578c503eb8fe4bf7891c94c8f5f0ccb036f3f8c3a01cf5212db6
START RequestId: 2c14be6e-b073-4b8c-ac2b-aae1e0e2cd41 Version: $LATEST
2025-04-23T13:17:12.072Z	2c14be6e-b073-4b8c-ac2b-aae1e0e2cd41	INFO	
{
    "RequestType": "Create",
    "ServiceToken": "arn:aws:lambda:<redacted>>:<redacted>:function:CoreStackDev-CustomDeleteExistingRecordSetCustomRe-8ZwMf05clmhl",
    "ResponseURL": "...",
    "StackId": "arn:aws:cloudformation:<redacted>>:<redacted>:stack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb",
    "RequestId": "49412ef1-b119-4959-a442-db1326a97e04",
    "LogicalResourceId": "WebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordTestDeleteExistingRecordSetCustomResourceA9FD1425",
    "ResourceType": "Custom::DeleteExistingRecordSet",
    "ResourceProperties": {
        "ServiceToken": "arn:aws:lambda:<redacted>>:<redacted>:function:CoreStackDev-CustomDeleteExistingRecordSetCustomRe-8ZwMf05clmhl",
        "RecordName": "api.dev.<redacted>.com.",
        "RecordType": "A",
        "HostedZoneId": "<redacted>"
    }
}

2025-04-23T13:17:16.254Z	2c14be6e-b073-4b8c-ac2b-aae1e0e2cd41	INFO	submit response to cloudformation https://cloudformation-custom-resource-response-<redacted>.s3.<redacted>>.amazonaws.com//arn%3Aaws%3Acloudformation%3A<redacted>>%3A<redacted>%3Astack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb%7CWebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordTestDeleteExistingRecordSetCustomResourceA9FD1425%7C49412ef1-b119-4959-a442-db1326a97e04?*** {
  Status: 'SUCCESS',
  Reason: 'SUCCESS',
  StackId: 'arn:aws:cloudformation:<redacted>>:<redacted>:stack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb',
  RequestId: '49412ef1-b119-4959-a442-db1326a97e04',
  PhysicalResourceId: '49412ef1-b119-4959-a442-db1326a97e04',
  LogicalResourceId: 'WebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordTestDeleteExistingRecordSetCustomResourceA9FD1425',
  NoEcho: undefined,
  Data: undefined
}
END RequestId: 2c14be6e-b073-4b8c-ac2b-aae1e0e2cd41
REPORT RequestId: 2c14be6e-b073-4b8c-ac2b-aae1e0e2cd41	Duration: 4390.45 ms	Billed Duration: 4391 ms	Memory Size: 128 MB	Max Memory Used: 88 MB	Init Duration: 160.14 ms	
START RequestId: 00d5ccf0-8492-45c6-aef7-8841946360dd Version: $LATEST
2025-04-23T13:17:23.497Z	00d5ccf0-8492-45c6-aef7-8841946360dd	INFO	
{
    "RequestType": "Delete",
    "ServiceToken": "arn:aws:lambda:<redacted>>:<redacted>:function:CoreStackDev-CustomDeleteExistingRecordSetCustomRe-8ZwMf05clmhl",
    "ResponseURL": "...",
    "StackId": "arn:aws:cloudformation:<redacted>>:<redacted>:stack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb",
    "RequestId": "c8e33677-92bd-4fe3-bff6-379b2d73532e",
    "LogicalResourceId": "WebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordDeleteExistingRecordSetCustomResourceF51AF170",
    "PhysicalResourceId": "7d099ac6-c8b9-475b-bc4a-84680622fac6",
    "ResourceType": "Custom::DeleteExistingRecordSet",
    "ResourceProperties": {
        "ServiceToken": "arn:aws:lambda:<redacted>>:<redacted>:function:CoreStackDev-CustomDeleteExistingRecordSetCustomRe-8ZwMf05clmhl",
        "RecordName": "api.dev.<redacted>.com.",
        "RecordType": "A",
        "HostedZoneId": "<redacted>"
    }
}

2025-04-23T13:17:23.498Z	00d5ccf0-8492-45c6-aef7-8841946360dd	INFO	submit response to cloudformation https://cloudformation-custom-resource-response-<redacted>.s3.<redacted>>.amazonaws.com//arn%3Aaws%3Acloudformation%3A<redacted>>%3A<redacted>%3Astack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb%7CWebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordDeleteExistingRecordSetCustomResourceF51AF170%7Cc8e33677-92bd-4fe3-bff6-379b2d73532e?*** {
  Status: 'SUCCESS',
  Reason: 'SUCCESS',
  StackId: 'arn:aws:cloudformation:<redacted>>:<redacted>:stack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb',
  RequestId: 'c8e33677-92bd-4fe3-bff6-379b2d73532e',
  PhysicalResourceId: '7d099ac6-c8b9-475b-bc4a-84680622fac6',
  LogicalResourceId: 'WebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordDeleteExistingRecordSetCustomResourceF51AF170',
  NoEcho: undefined,
  Data: undefined
}
END RequestId: 00d5ccf0-8492-45c6-aef7-8841946360dd
REPORT RequestId: 00d5ccf0-8492-45c6-aef7-8841946360dd	Duration: 120.10 ms	Billed Duration: 121 ms	Memory Size: 128 MB	Max Memory Used: 88 MB	
START RequestId: 882d23c0-226a-4461-881f-5306de3602f8 Version: $LATEST
2025-04-23T13:22:42.452Z	882d23c0-226a-4461-881f-5306de3602f8	INFO	
{
    "RequestType": "Create",
    "ServiceToken": "arn:aws:lambda:<redacted>>:<redacted>:function:CoreStackDev-CustomDeleteExistingRecordSetCustomRe-8ZwMf05clmhl",
    "ResponseURL": "...",
    "StackId": "arn:aws:cloudformation:<redacted>>:<redacted>:stack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb",
    "RequestId": "5ee90893-a5c8-40b0-a989-90c0bb925043",
    "LogicalResourceId": "WebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordDeleteExistingRecordSetCustomResourceF51AF170",
    "ResourceType": "Custom::DeleteExistingRecordSet",
    "ResourceProperties": {
        "ServiceToken": "arn:aws:lambda:<redacted>>:<redacted>:function:CoreStackDev-CustomDeleteExistingRecordSetCustomRe-8ZwMf05clmhl",
        "RecordName": "api.dev.<redacted>.com.",
        "RecordType": "A",
        "HostedZoneId": "<redacted>"
    }
}

2025-04-23T13:22:42.772Z	882d23c0-226a-4461-881f-5306de3602f8	INFO	submit response to cloudformation https://cloudformation-custom-resource-response-<redacted>.s3.<redacted>>.amazonaws.com//arn%3Aaws%3Acloudformation%3A<redacted>>%3A<redacted>%3Astack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb%7CWebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordDeleteExistingRecordSetCustomResourceF51AF170%7C5ee90893-a5c8-40b0-a989-90c0bb925043?*** {
  Status: 'SUCCESS',
  Reason: 'SUCCESS',
  StackId: 'arn:aws:cloudformation:<redacted>>:<redacted>:stack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb',
  RequestId: '5ee90893-a5c8-40b0-a989-90c0bb925043',
  PhysicalResourceId: '5ee90893-a5c8-40b0-a989-90c0bb925043',
  LogicalResourceId: 'WebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordDeleteExistingRecordSetCustomResourceF51AF170',
  NoEcho: undefined,
  Data: undefined
}
END RequestId: 882d23c0-226a-4461-881f-5306de3602f8
REPORT RequestId: 882d23c0-226a-4461-881f-5306de3602f8	Duration: 620.24 ms	Billed Duration: 621 ms	Memory Size: 128 MB	Max Memory Used: 88 MB	
START RequestId: 43437399-5250-4621-adf4-6845830879a4 Version: $LATEST
2025-04-23T13:23:53.667Z	43437399-5250-4621-adf4-6845830879a4	INFO	
{
    "RequestType": "Delete",
    "ServiceToken": "arn:aws:lambda:<redacted>>:<redacted>:function:CoreStackDev-CustomDeleteExistingRecordSetCustomRe-8ZwMf05clmhl",
    "ResponseURL": "...",
    "StackId": "arn:aws:cloudformation:<redacted>>:<redacted>:stack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb",
    "RequestId": "1a791d51-f495-445e-a66b-4b1a934e05dd",
    "LogicalResourceId": "WebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordTestDeleteExistingRecordSetCustomResourceA9FD1425",
    "PhysicalResourceId": "49412ef1-b119-4959-a442-db1326a97e04",
    "ResourceType": "Custom::DeleteExistingRecordSet",
    "ResourceProperties": {
        "ServiceToken": "arn:aws:lambda:<redacted>>:<redacted>:function:CoreStackDev-CustomDeleteExistingRecordSetCustomRe-8ZwMf05clmhl",
        "RecordName": "api.dev.<redacted>.com.",
        "RecordType": "A",
        "HostedZoneId": "<redacted>"
    }
}

2025-04-23T13:23:53.731Z	43437399-5250-4621-adf4-6845830879a4	INFO	submit response to cloudformation https://cloudformation-custom-resource-response-<redacted>.s3.<redacted>>.amazonaws.com//arn%3Aaws%3Acloudformation%3A<redacted>>%3A<redacted>%3Astack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb%7CWebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordTestDeleteExistingRecordSetCustomResourceA9FD1425%7C1a791d51-f495-445e-a66b-4b1a934e05dd?*** {
  Status: 'SUCCESS',
  Reason: 'SUCCESS',
  StackId: 'arn:aws:cloudformation:<redacted>>:<redacted>:stack/CoreStackDev/6fdcbfc0-fde9-11ef-a37b-06a0be8e5ebb',
  RequestId: '1a791d51-f495-445e-a66b-4b1a934e05dd',
  PhysicalResourceId: '49412ef1-b119-4959-a442-db1326a97e04',
  LogicalResourceId: 'WebAppSvcFlaskAppServiceDataPlaneRoute53SvcAlbEndpointRecordTestDeleteExistingRecordSetCustomResourceA9FD1425',
  NoEcho: undefined,
  Data: undefined
}

Regression Issue

• Select this option if this issue appears to be a regression.

Last Known Working CDK Version

No response

Expected Behavior

The existing a record gets deleted and then created with the new value.

Current Behavior

By the time the stack finishes updating, there's no A record at all.

Reproduction Steps

1. Deploy with:
   • An A record with delete_existing=True
   • An ALB w/ protocol=lb.ApplicationProtocol.HTTP
2. Attempt to move ALB to protocol=lb.ApplicationProtocol.HTTPS which may require domain_name and domain_zone to be set. In my case there was a mistake in here somewhere, and the domain_name got set to some other value (i.e. original value was api.dev.example.com an the new domain_name gets set to just dev.example.com), the A record was deleted and recreated with the new record name.
3. Try to force Cfn to recreate the record set by changing the logicalid of the A record
4. Experience error

Possible Solution

Detect if the a record has already been deleted and (re)created by the custom resource and if so, don't run the CR again.

Additional Information/Context

No response

CDK CLI Version

2.1000.3 (build 321a46a)

Framework Version

No response

Node.js Version

v22.12.0

OS

Mac

Language

Python

Language Version

No response

Other information

No response

[ykethan]

appears to be related to #26754

[ykethan]

Hey @BwL1289, thank you for reporting this issue and providing us the logs. On diving into this issue, I was able to observe the following when reproducing this issue.

CDK versions

 "aws-cdk-lib": "2.191.0",
"aws-cdk": "^2.1012.0"

Reproduction Steps:

1. Created a CDK stack with a Route53 PublicHostedZone and an ARecord:

export class Route53IssueStack extends cdk.Stack {
  constructor(scope: Construct, id: string, props?: cdk.StackProps) {
    super(scope, id, props);

    // Step 1: Create a VPC
    const vpc = new ec2.Vpc(this, 'VPC', {
      maxAzs: 2,
      natGateways: 0,
    });

    // Step 1: Create an ALB
    const alb = new elbv2.ApplicationLoadBalancer(this, 'ALB', {
      vpc,
      internetFacing: true,
    });

    // Create a hosted zone
    const hostedZone = new route53.PublicHostedZone(this, 'HostedZone', {
      zoneName: 'mydemo-domain.net',
    });

    // Create an A record
    new route53.ARecord(this, 'ARecord', {
      zone: hostedZone,
      recordName: 'api.dev.mydemo-domain.net',
      target: route53.RecordTarget.fromAlias(
        new targets.LoadBalancerTarget(alb)
      ),
      deleteExisting: true,
    });

    // Add HTTP listener
    alb.addListener('HttpListener', {
      port: 80,
      protocol: elbv2.ApplicationProtocol.HTTP,
      defaultAction: elbv2.ListenerAction.fixedResponse(200, {
        contentType: 'text/plain',
        messageBody: 'OK',
      }),
    });
  }
}

2. ran cdk synth && cdk deploy, the deploy was succesfull
3. Updated zoneName and recordName:

    // Create a hosted zone
    const hostedZone = new route53.PublicHostedZone(this, 'HostedZone', {
      zoneName: 'mydemo-domain-abc.net',
    });

    // Create an A record
    new route53.ARecord(this, 'ARecord', {
      zone: hostedZone,
      recordName: 'api.dev.mydemo-domain-abc.net',
      target: route53.RecordTarget.fromAlias(
        new targets.LoadBalancerTarget(alb)
      ),
      deleteExisting: true,
    });

4. ran cdk synth

HostedZoneDB99F866:
  Type: AWS::Route53::HostedZone
  Properties:
    Name: mydemo-domain.net.  # Should be mydemo-domain-abc.net.

5. Tried clearing the CDK context with cdk context --clear and re-synthesizing, the issue persists.

Marking this as P1

[BwL1289]

@ykethan thank you