diff --git a/.editorconfig b/.editorconfig index 4b89edcf9b..b5f1f0448f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -20,3 +20,7 @@ insert_final_newline = true [{Makefile,go.mod,go.sum,*.go}] indent_style = tab indent_size = 4 + +[*.py] +indent_style = space +indent_size = 4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 953261ab60..d27eddee6d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,5 +23,6 @@ repos: rev: v0.44.0 hooks: - id: markdownlint + args: ["--fix"] minimum_pre_commit_version: !!str 3.2 diff --git a/docs/flags.md b/docs/flags.md index bbe78c71d2..0639f6b3f9 100644 --- a/docs/flags.md +++ b/docs/flags.md @@ -161,7 +161,7 @@ | `--txt-wildcard-replacement=""` | When using the TXT registry, a custom string that's used instead of an asterisk for TXT records corresponding to wildcard DNS records (optional) | | `--[no-]txt-encrypt-enabled` | When using the TXT registry, set if TXT records should be encrypted before stored (default: disabled) | | `--txt-encrypt-aes-key=""` | When using the TXT registry, set TXT record decryption and encryption 32 byte aes key (required when --txt-encrypt=true) | -| `--[no-]txt-new-format-only` | When using the TXT registry, only use new format records which include record type information (e.g., prefix: 'a-'). Reduces number of TXT records (default: disabled) | +| `--[no-]txt-new-format-only` | Was used during TXT format migration. Enabling or disabling it will not take any effect. (deprecated) | | `--dynamodb-region=""` | When using the DynamoDB registry, the AWS region of the DynamoDB table (optional) | | `--dynamodb-table="external-dns"` | When using the DynamoDB registry, the name of the DynamoDB table (default: "external-dns") | | `--txt-cache-interval=0s` | The interval between cache synchronizations in duration format (default: disabled) | diff --git a/docs/registry/txt.md b/docs/registry/txt.md index 815c7c3b39..c3426395c4 100644 --- a/docs/registry/txt.md +++ b/docs/registry/txt.md @@ -3,8 +3,51 @@ The TXT registry is the default registry. It stores DNS record metadata in TXT records, using the same provider. +If you plan to manage apex domains with external-dns whilst using a txt registry, you should ensure when using --txt-prefix that you specify the record type substitution and that it ends in a period (**.**). The record should be created under the same domain as the apex record being managed, i.e. --txt-prefix=someprefix-%{record_type}. + +> Note: `--txt-prefix` and `--txt-suffix` contribute to the 63-byte maximum record length. To avoid errors, use them only if absolutely required and keep them as short as possible. + ## Record Format Options +### For version `v0.16.2+` + +The TXT registry supports single format for storing DNS record metadata: + +- Creates a TXT record with record type information (e.g., 'a-' prefix for A records) + +The TXT registry would try to guarantee a consistency in between providers and sources, if provider supports the behaviour + +If you are dealing with APEX domains, example `example.com` and TXT records are failing to be created for managed record types specified by `--managed-record-types`, consider following options: + +1. TXT record with prefix based on requirements. Example `--txt-prefix="%{record_type}-abc-"` or `--txt-prefix="%{record_type}.abc-"` +2. TXT record with suffix based on requirements. Example `--txt-suffix="-abc-%{record_type}"` or `--txt-suffix="-abc.%{record_type}."` + +Example when configured `--txt-prefix="%{record_type}-abc-"` for apex domain `example.com` the expected result is + +| Name | TYPE | +|:-----------------------------:|:--------:| +| `a-abc-nginx-v2.ex.com.` | `TXT` | +| `nginx-v2.ex.com.` | `CNAME` | + +And when configured `--txt-suffix="-abc.%{record_type}"` for apex domain `example.com` the expected result is + +| Name | TYPE | +|:-----------------------------:|:-------:| +| `nginx-v2-abc.a.ex.com.` | `TXT` | +| `nginx-v3.ex.com..` | `CNAME` | + +### Manually Cleanup Legacy TXT Records + +> While deleting registry TXT records won't cause downtime, a well-thought-out migration and cleanup plan is crucial. + +Occasionally, it may be necessary to remove outdated TXT records from your registry. + +An example script for AWS can be found in [scripts/aws-cleanup-legacy-txt-records.py](../../scripts/aws-cleanup-legacy-txt-records.py) with instructions on how to run it. +The script performs targeted deletion of TXT records that include `ResourceRecords` matching the `heritage=external-dns,external-dns/owner=default` or similar pattern. +In the event of unintended deletion of all TXT records managed by `external-dns`, `external-dns` will initiate a full DNS record regeneration, along with`TXT` and `non-TXT` records. Just be aware, this operation's duration is directly proportional to the DNS estate size." + +### For version `v0.16.0 & v0.16.1` + The TXT registry supports two formats for storing DNS record metadata: - Legacy format: Creates a TXT record without record type information @@ -31,14 +74,14 @@ The `--txt-new-format-only` flag should be used in addition to your existing ext ### Migration to New Format Only +> Note: `external-dns` will not automatically remove legacy format records when switching to new-format-only mode. You'll need to clean up the old records manually if desired. + When transitioning from dual-format to new-format-only records: - Ensure all your `external-dns` instances support the new format - Enable the `--txt-new-format-only` flag on your external-dns instances Manually clean up any existing legacy format TXT records from your DNS provider -Note: `external-dns` will not automatically remove legacy format records when switching to new-format-only mode. You'll need to clean up the old records manually if desired. - ## Prefixes and Suffixes In order to avoid having the registry TXT records collide with diff --git a/pkg/apis/externaldns/types.go b/pkg/apis/externaldns/types.go index f0697d4982..340975a04e 100644 --- a/pkg/apis/externaldns/types.go +++ b/pkg/apis/externaldns/types.go @@ -615,7 +615,7 @@ func App(cfg *Config) *kingpin.Application { app.Flag("txt-wildcard-replacement", "When using the TXT registry, a custom string that's used instead of an asterisk for TXT records corresponding to wildcard DNS records (optional)").Default(defaultConfig.TXTWildcardReplacement).StringVar(&cfg.TXTWildcardReplacement) app.Flag("txt-encrypt-enabled", "When using the TXT registry, set if TXT records should be encrypted before stored (default: disabled)").BoolVar(&cfg.TXTEncryptEnabled) app.Flag("txt-encrypt-aes-key", "When using the TXT registry, set TXT record decryption and encryption 32 byte aes key (required when --txt-encrypt=true)").Default(defaultConfig.TXTEncryptAESKey).StringVar(&cfg.TXTEncryptAESKey) - app.Flag("txt-new-format-only", "When using the TXT registry, only use new format records which include record type information (e.g., prefix: 'a-'). Reduces number of TXT records (default: disabled)").BoolVar(&cfg.TXTNewFormatOnly) + app.Flag("txt-new-format-only", "Was used during TXT format migration. Enabling or disabling it will not take any effect. (deprecated)").BoolVar(&cfg.TXTNewFormatOnly) app.Flag("dynamodb-region", "When using the DynamoDB registry, the AWS region of the DynamoDB table (optional)").Default(cfg.AWSDynamoDBRegion).StringVar(&cfg.AWSDynamoDBRegion) app.Flag("dynamodb-table", "When using the DynamoDB registry, the name of the DynamoDB table (default: \"external-dns\")").Default(defaultConfig.AWSDynamoDBTable).StringVar(&cfg.AWSDynamoDBTable) diff --git a/registry/txt.go b/registry/txt.go index 06a8314a78..ffd2a6e6a6 100644 --- a/registry/txt.go +++ b/registry/txt.go @@ -58,8 +58,6 @@ type TXTRegistry struct { // encrypt text records txtEncryptEnabled bool txtEncryptAESKey []byte - - newFormatOnly bool } // NewTXTRegistry returns a new TXTRegistry object. When newFormatOnly is true, it will only @@ -74,6 +72,10 @@ func NewTXTRegistry(provider provider.Provider, txtPrefix, txtSuffix, ownerID st return nil, errors.New("owner id cannot be empty") } + if newFormatOnly { + log.Warn("--txt-new-format-only is left for backward compatibility, it will be removed in future releases") + } + if len(txtEncryptAESKey) == 0 { txtEncryptAESKey = nil } else if len(txtEncryptAESKey) != 32 { @@ -103,7 +105,6 @@ func NewTXTRegistry(provider provider.Provider, txtPrefix, txtSuffix, ownerID st excludeRecordTypes: excludeRecordTypes, txtEncryptEnabled: txtEncryptEnabled, txtEncryptAESKey: txtEncryptAESKey, - newFormatOnly: newFormatOnly, }, nil } @@ -236,25 +237,13 @@ func (im *TXTRegistry) Records(ctx context.Context) ([]*endpoint.Endpoint, error func (im *TXTRegistry) generateTXTRecord(r *endpoint.Endpoint) []*endpoint.Endpoint { endpoints := make([]*endpoint.Endpoint, 0) - // Create legacy format record by default unless newFormatOnly is true - if !im.newFormatOnly && !im.txtEncryptEnabled && !im.mapper.recordTypeInAffix() && r.RecordType != endpoint.RecordTypeAAAA { - // old TXT record format - txt := endpoint.NewEndpoint(im.mapper.toTXTName(r.DNSName), endpoint.RecordTypeTXT, r.Labels.Serialize(true, im.txtEncryptEnabled, im.txtEncryptAESKey)) - if txt != nil { - txt.WithSetIdentifier(r.SetIdentifier) - txt.Labels[endpoint.OwnedRecordLabelKey] = r.DNSName - txt.ProviderSpecific = r.ProviderSpecific - endpoints = append(endpoints, txt) - } - } - // Always create new format record recordType := r.RecordType // AWS Alias records are encoded as type "cname" if isAlias, found := r.GetProviderSpecificProperty("alias"); found && isAlias == "true" && recordType == endpoint.RecordTypeA { recordType = endpoint.RecordTypeCNAME } - txtNew := endpoint.NewEndpoint(im.mapper.toNewTXTName(r.DNSName, recordType), endpoint.RecordTypeTXT, r.Labels.Serialize(true, im.txtEncryptEnabled, im.txtEncryptAESKey)) + txtNew := endpoint.NewEndpoint(im.mapper.toTXTName(r.DNSName, recordType), endpoint.RecordTypeTXT, r.Labels.Serialize(true, im.txtEncryptEnabled, im.txtEncryptAESKey)) if txtNew != nil { txtNew.WithSetIdentifier(r.SetIdentifier) txtNew.Labels[endpoint.OwnedRecordLabelKey] = r.DNSName @@ -336,8 +325,7 @@ func (im *TXTRegistry) AdjustEndpoints(endpoints []*endpoint.Endpoint) ([]*endpo type nameMapper interface { toEndpointName(string) (endpointName string, recordType string) - toTXTName(string) string - toNewTXTName(string, string) string + toTXTName(string, string) string recordTypeInAffix() bool } @@ -437,22 +425,6 @@ func (pr affixNameMapper) toEndpointName(txtDNSName string) (endpointName string return "", "" } -func (pr affixNameMapper) toTXTName(endpointDNSName string) string { - DNSName := strings.SplitN(endpointDNSName, ".", 2) - - prefix := pr.dropAffixTemplate(pr.prefix) - suffix := pr.dropAffixTemplate(pr.suffix) - // If specified, replace a leading asterisk in the generated txt record name with some other string - if pr.wildcardReplacement != "" && DNSName[0] == "*" { - DNSName[0] = pr.wildcardReplacement - } - - if len(DNSName) < 2 { - return prefix + DNSName[0] + suffix - } - return prefix + DNSName[0] + suffix + "." + DNSName[1] -} - func (pr affixNameMapper) recordTypeInAffix() bool { if strings.Contains(pr.prefix, recordTemplate) { return true @@ -470,7 +442,7 @@ func (pr affixNameMapper) normalizeAffixTemplate(afix, recordType string) string return afix } -func (pr affixNameMapper) toNewTXTName(endpointDNSName, recordType string) string { +func (pr affixNameMapper) toTXTName(endpointDNSName, recordType string) string { DNSName := strings.SplitN(endpointDNSName, ".", 2) recordType = strings.ToLower(recordType) recordT := recordType + "-" diff --git a/registry/txt_test.go b/registry/txt_test.go index 6cb598b25a..4ff36c949b 100644 --- a/registry/txt_test.go +++ b/registry/txt_test.go @@ -18,6 +18,7 @@ package registry import ( "context" + "fmt" "reflect" "strings" "testing" @@ -590,37 +591,29 @@ func testTXTRegistryApplyChangesWithPrefix(t *testing.T) { expected := &plan.Changes{ Create: []*endpoint.Endpoint{ newEndpointWithOwnerResource("new-record-1.test-zone.example.org", "new-loadbalancer-1.lb.com", endpoint.RecordTypeCNAME, "owner", "ingress/default/my-ingress"), - newEndpointWithOwnerAndOwnedRecord("txt.new-record-1.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress\"", endpoint.RecordTypeTXT, "", "new-record-1.test-zone.example.org"), + newEndpointWithOwnerAndOwnedRecord("txt.cname-new-record-1.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress\"", endpoint.RecordTypeTXT, "", "new-record-1.test-zone.example.org"), newEndpointWithOwnerResource("multiple.test-zone.example.org", "lb3.loadbalancer.com", endpoint.RecordTypeCNAME, "owner", "ingress/default/my-ingress").WithSetIdentifier("test-set-3"), - newEndpointWithOwnerAndOwnedRecord("txt.multiple.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress\"", endpoint.RecordTypeTXT, "", "multiple.test-zone.example.org").WithSetIdentifier("test-set-3"), newEndpointWithOwnerAndOwnedRecord("txt.cname-multiple.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress\"", endpoint.RecordTypeTXT, "", "multiple.test-zone.example.org").WithSetIdentifier("test-set-3"), newEndpointWithOwnerResource("example", "new-loadbalancer-1.lb.com", endpoint.RecordTypeCNAME, "owner", "ingress/default/my-ingress"), - newEndpointWithOwnerAndOwnedRecord("txt.example", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress\"", endpoint.RecordTypeTXT, "", "example"), newEndpointWithOwnerAndOwnedRecord("txt.cname-example", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress\"", endpoint.RecordTypeTXT, "", "example"), }, Delete: []*endpoint.Endpoint{ newEndpointWithOwner("foobar.test-zone.example.org", "foobar.loadbalancer.com", endpoint.RecordTypeCNAME, "owner"), - newEndpointWithOwnerAndOwnedRecord("txt.foobar.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "foobar.test-zone.example.org"), newEndpointWithOwnerAndOwnedRecord("txt.cname-foobar.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "foobar.test-zone.example.org"), newEndpointWithOwner("multiple.test-zone.example.org", "lb1.loadbalancer.com", endpoint.RecordTypeCNAME, "owner").WithSetIdentifier("test-set-1"), - newEndpointWithOwnerAndOwnedRecord("txt.multiple.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "multiple.test-zone.example.org").WithSetIdentifier("test-set-1"), newEndpointWithOwnerAndOwnedRecord("txt.cname-multiple.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "multiple.test-zone.example.org").WithSetIdentifier("test-set-1"), }, UpdateNew: []*endpoint.Endpoint{ newEndpointWithOwnerResource("tar.test-zone.example.org", "new-tar.loadbalancer.com", endpoint.RecordTypeCNAME, "owner", "ingress/default/my-ingress-2"), - newEndpointWithOwnerAndOwnedRecord("txt.tar.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress-2\"", endpoint.RecordTypeTXT, "", "tar.test-zone.example.org"), newEndpointWithOwnerAndOwnedRecord("txt.cname-tar.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress-2\"", endpoint.RecordTypeTXT, "", "tar.test-zone.example.org"), newEndpointWithOwnerResource("multiple.test-zone.example.org", "new.loadbalancer.com", endpoint.RecordTypeCNAME, "owner", "ingress/default/my-ingress-2").WithSetIdentifier("test-set-2"), - newEndpointWithOwnerAndOwnedRecord("txt.multiple.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress-2\"", endpoint.RecordTypeTXT, "", "multiple.test-zone.example.org").WithSetIdentifier("test-set-2"), newEndpointWithOwnerAndOwnedRecord("txt.cname-multiple.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress-2\"", endpoint.RecordTypeTXT, "", "multiple.test-zone.example.org").WithSetIdentifier("test-set-2"), }, UpdateOld: []*endpoint.Endpoint{ newEndpointWithOwner("tar.test-zone.example.org", "tar.loadbalancer.com", endpoint.RecordTypeCNAME, "owner"), - newEndpointWithOwnerAndOwnedRecord("txt.tar.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "tar.test-zone.example.org"), newEndpointWithOwnerAndOwnedRecord("txt.cname-tar.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "tar.test-zone.example.org"), newEndpointWithOwner("multiple.test-zone.example.org", "lb2.loadbalancer.com", endpoint.RecordTypeCNAME, "owner").WithSetIdentifier("test-set-2"), - newEndpointWithOwnerAndOwnedRecord("txt.multiple.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "multiple.test-zone.example.org").WithSetIdentifier("test-set-2"), newEndpointWithOwnerAndOwnedRecord("txt.cname-multiple.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "multiple.test-zone.example.org").WithSetIdentifier("test-set-2"), }, } @@ -788,40 +781,30 @@ func testTXTRegistryApplyChangesWithSuffix(t *testing.T) { expected := &plan.Changes{ Create: []*endpoint.Endpoint{ newEndpointWithOwnerResource("new-record-1.test-zone.example.org", "new-loadbalancer-1.lb.com", endpoint.RecordTypeCNAME, "owner", "ingress/default/my-ingress"), - newEndpointWithOwnerAndOwnedRecord("new-record-1-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress\"", endpoint.RecordTypeTXT, "", "new-record-1.test-zone.example.org"), newEndpointWithOwnerAndOwnedRecord("cname-new-record-1-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress\"", endpoint.RecordTypeTXT, "", "new-record-1.test-zone.example.org"), newEndpointWithOwnerResource("multiple.test-zone.example.org", "lb3.loadbalancer.com", endpoint.RecordTypeCNAME, "owner", "ingress/default/my-ingress").WithSetIdentifier("test-set-3"), - newEndpointWithOwnerAndOwnedRecord("multiple-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress\"", endpoint.RecordTypeTXT, "", "multiple.test-zone.example.org").WithSetIdentifier("test-set-3"), newEndpointWithOwnerAndOwnedRecord("cname-multiple-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress\"", endpoint.RecordTypeTXT, "", "multiple.test-zone.example.org").WithSetIdentifier("test-set-3"), newEndpointWithOwnerResource("example", "new-loadbalancer-1.lb.com", endpoint.RecordTypeCNAME, "owner", "ingress/default/my-ingress"), - newEndpointWithOwnerAndOwnedRecord("example-txt", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress\"", endpoint.RecordTypeTXT, "", "example"), newEndpointWithOwnerAndOwnedRecord("cname-example-txt", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress\"", endpoint.RecordTypeTXT, "", "example"), newEndpointWithOwnerResource("*.wildcard.test-zone.example.org", "new-loadbalancer-1.lb.com", endpoint.RecordTypeCNAME, "owner", "ingress/default/my-ingress"), - newEndpointWithOwnerAndOwnedRecord("wildcard-txt.wildcard.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress\"", endpoint.RecordTypeTXT, "", "*.wildcard.test-zone.example.org"), newEndpointWithOwnerAndOwnedRecord("cname-wildcard-txt.wildcard.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress\"", endpoint.RecordTypeTXT, "", "*.wildcard.test-zone.example.org"), }, Delete: []*endpoint.Endpoint{ newEndpointWithOwner("foobar.test-zone.example.org", "foobar.loadbalancer.com", endpoint.RecordTypeCNAME, "owner"), - newEndpointWithOwnerAndOwnedRecord("foobar-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "foobar.test-zone.example.org"), newEndpointWithOwnerAndOwnedRecord("cname-foobar-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "foobar.test-zone.example.org"), newEndpointWithOwner("multiple.test-zone.example.org", "lb1.loadbalancer.com", endpoint.RecordTypeCNAME, "owner").WithSetIdentifier("test-set-1"), - newEndpointWithOwnerAndOwnedRecord("multiple-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "multiple.test-zone.example.org").WithSetIdentifier("test-set-1"), newEndpointWithOwnerAndOwnedRecord("cname-multiple-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "multiple.test-zone.example.org").WithSetIdentifier("test-set-1"), }, UpdateNew: []*endpoint.Endpoint{ newEndpointWithOwnerResource("tar.test-zone.example.org", "new-tar.loadbalancer.com", endpoint.RecordTypeCNAME, "owner", "ingress/default/my-ingress-2"), - newEndpointWithOwnerAndOwnedRecord("tar-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress-2\"", endpoint.RecordTypeTXT, "", "tar.test-zone.example.org"), newEndpointWithOwnerAndOwnedRecord("cname-tar-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress-2\"", endpoint.RecordTypeTXT, "", "tar.test-zone.example.org"), newEndpointWithOwnerResource("multiple.test-zone.example.org", "new.loadbalancer.com", endpoint.RecordTypeCNAME, "owner", "ingress/default/my-ingress-2").WithSetIdentifier("test-set-2"), - newEndpointWithOwnerAndOwnedRecord("multiple-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress-2\"", endpoint.RecordTypeTXT, "", "multiple.test-zone.example.org").WithSetIdentifier("test-set-2"), newEndpointWithOwnerAndOwnedRecord("cname-multiple-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner,external-dns/resource=ingress/default/my-ingress-2\"", endpoint.RecordTypeTXT, "", "multiple.test-zone.example.org").WithSetIdentifier("test-set-2"), }, UpdateOld: []*endpoint.Endpoint{ newEndpointWithOwner("tar.test-zone.example.org", "tar.loadbalancer.com", endpoint.RecordTypeCNAME, "owner"), - newEndpointWithOwnerAndOwnedRecord("tar-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "tar.test-zone.example.org"), newEndpointWithOwnerAndOwnedRecord("cname-tar-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "tar.test-zone.example.org"), newEndpointWithOwner("multiple.test-zone.example.org", "lb2.loadbalancer.com", endpoint.RecordTypeCNAME, "owner").WithSetIdentifier("test-set-2"), - newEndpointWithOwnerAndOwnedRecord("multiple-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "multiple.test-zone.example.org").WithSetIdentifier("test-set-2"), newEndpointWithOwnerAndOwnedRecord("cname-multiple-txt.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "multiple.test-zone.example.org").WithSetIdentifier("test-set-2"), }, } @@ -888,19 +871,14 @@ func testTXTRegistryApplyChangesNoPrefix(t *testing.T) { expected := &plan.Changes{ Create: []*endpoint.Endpoint{ newEndpointWithOwner("new-record-1.test-zone.example.org", "new-loadbalancer-1.lb.com", endpoint.RecordTypeCNAME, "owner"), - newEndpointWithOwnerAndOwnedRecord("new-record-1.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "new-record-1.test-zone.example.org"), newEndpointWithOwnerAndOwnedRecord("cname-new-record-1.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "new-record-1.test-zone.example.org"), newEndpointWithOwner("example", "new-loadbalancer-1.lb.com", endpoint.RecordTypeCNAME, "owner"), - newEndpointWithOwnerAndOwnedRecord("example", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "example"), newEndpointWithOwnerAndOwnedRecord("cname-example", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "example"), newEndpointWithOwner("new-alias.test-zone.example.org", "my-domain.com", endpoint.RecordTypeA, "owner").WithProviderSpecific("alias", "true"), - // TODO: It's not clear why the TXT registry copies ProviderSpecificProperties to ownership records; that doesn't seem correct. - newEndpointWithOwnerAndOwnedRecord("new-alias.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "new-alias.test-zone.example.org").WithProviderSpecific("alias", "true"), newEndpointWithOwnerAndOwnedRecord("cname-new-alias.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "new-alias.test-zone.example.org").WithProviderSpecific("alias", "true"), }, Delete: []*endpoint.Endpoint{ newEndpointWithOwner("foobar.test-zone.example.org", "foobar.loadbalancer.com", endpoint.RecordTypeCNAME, "owner"), - newEndpointWithOwnerAndOwnedRecord("foobar.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "foobar.test-zone.example.org"), newEndpointWithOwnerAndOwnedRecord("cname-foobar.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "foobar.test-zone.example.org"), }, UpdateNew: []*endpoint.Endpoint{}, @@ -1404,7 +1382,7 @@ func TestToEndpointNameNewTXT(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - txtDomain := tc.mapper.toNewTXTName(tc.domain, tc.recordType) + txtDomain := tc.mapper.toTXTName(tc.domain, tc.recordType) assert.Equal(t, tc.txtDomain, txtDomain) domain, _ := tc.mapper.toEndpointName(txtDomain) @@ -1455,15 +1433,12 @@ func TestNewTXTScheme(t *testing.T) { expected := &plan.Changes{ Create: []*endpoint.Endpoint{ newEndpointWithOwner("new-record-1.test-zone.example.org", "new-loadbalancer-1.lb.com", endpoint.RecordTypeCNAME, "owner"), - newEndpointWithOwnerAndOwnedRecord("new-record-1.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "new-record-1.test-zone.example.org"), newEndpointWithOwnerAndOwnedRecord("cname-new-record-1.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "new-record-1.test-zone.example.org"), newEndpointWithOwner("example", "new-loadbalancer-1.lb.com", endpoint.RecordTypeCNAME, "owner"), - newEndpointWithOwnerAndOwnedRecord("example", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "example"), newEndpointWithOwnerAndOwnedRecord("cname-example", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "example"), }, Delete: []*endpoint.Endpoint{ newEndpointWithOwner("foobar.test-zone.example.org", "foobar.loadbalancer.com", endpoint.RecordTypeCNAME, "owner"), - newEndpointWithOwnerAndOwnedRecord("foobar.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "foobar.test-zone.example.org"), newEndpointWithOwnerAndOwnedRecord("cname-foobar.test-zone.example.org", "\"heritage=external-dns,external-dns/owner=owner\"", endpoint.RecordTypeTXT, "", "foobar.test-zone.example.org"), }, UpdateNew: []*endpoint.Endpoint{}, @@ -1486,20 +1461,13 @@ func TestNewTXTScheme(t *testing.T) { assert.Equal(t, nil, ctx.Value(provider.RecordsContextKey)) } err := r.ApplyChanges(ctx, changes) + fmt.Println(err) require.NoError(t, err) } func TestGenerateTXT(t *testing.T) { record := newEndpointWithOwner("foo.test-zone.example.org", "new-foo.loadbalancer.com", endpoint.RecordTypeCNAME, "owner") expectedTXT := []*endpoint.Endpoint{ - { - DNSName: "foo.test-zone.example.org", - Targets: endpoint.Targets{"\"heritage=external-dns,external-dns/owner=owner\""}, - RecordType: endpoint.RecordTypeTXT, - Labels: map[string]string{ - endpoint.OwnedRecordLabelKey: "foo.test-zone.example.org", - }, - }, { DNSName: "cname-foo.test-zone.example.org", Targets: endpoint.Targets{"\"heritage=external-dns,external-dns/owner=owner\""}, @@ -1655,46 +1623,17 @@ func TestMultiClusterDifferentRecordTypeOwnership(t *testing.T) { } } -/** - -helper methods - -*/ - -func newEndpointWithOwner(dnsName, target, recordType, ownerID string) *endpoint.Endpoint { - return newEndpointWithOwnerAndLabels(dnsName, target, recordType, ownerID, nil) -} - -func newEndpointWithOwnerAndOwnedRecord(dnsName, target, recordType, ownerID, ownedRecord string) *endpoint.Endpoint { - return newEndpointWithOwnerAndLabels(dnsName, target, recordType, ownerID, endpoint.Labels{endpoint.OwnedRecordLabelKey: ownedRecord}) -} - -func newEndpointWithOwnerAndLabels(dnsName, target, recordType, ownerID string, labels endpoint.Labels) *endpoint.Endpoint { - e := endpoint.NewEndpoint(dnsName, recordType, target) - e.Labels[endpoint.OwnerLabelKey] = ownerID - for k, v := range labels { - e.Labels[k] = v - } - return e -} - -func newEndpointWithOwnerResource(dnsName, target, recordType, ownerID, resource string) *endpoint.Endpoint { - e := endpoint.NewEndpoint(dnsName, recordType, target) - e.Labels[endpoint.OwnerLabelKey] = ownerID - e.Labels[endpoint.ResourceLabelKey] = resource - return e -} - func TestNewTXTRegistryWithNewFormatOnly(t *testing.T) { p := inmemory.NewInMemoryProvider() - r, err := NewTXTRegistry(p, "txt", "", "owner", time.Hour, "", []string{}, []string{}, false, nil, false) + buf := testutils.LogsToBuffer(log.DebugLevel, t) + _, err := NewTXTRegistry(p, "txt", "", "owner", time.Hour, "", []string{}, []string{}, false, nil, false) require.NoError(t, err) - assert.False(t, r.newFormatOnly) + assert.NotContains(t, buf.String(), "--txt-new-format-only is left for backward compatibility") - r, err = NewTXTRegistry(p, "txt", "", "owner", time.Hour, "", []string{}, []string{}, false, nil, true) + _, err = NewTXTRegistry(p, "txt", "", "owner", time.Hour, "", []string{}, []string{}, false, nil, true) require.NoError(t, err) - assert.True(t, r.newFormatOnly) + assert.Contains(t, buf.String(), "--txt-new-format-only is left for backward compatibility") } func TestGenerateTXTRecordWithNewFormatOnly(t *testing.T) { @@ -1711,9 +1650,9 @@ func TestGenerateTXTRecordWithNewFormatOnly(t *testing.T) { name: "legacy format enabled - standard record", newFormatOnly: false, endpoint: newEndpointWithOwner("foo.test-zone.example.org", "1.2.3.4", endpoint.RecordTypeA, "owner"), - expectedRecords: 2, + expectedRecords: 1, expectedPrefix: "a-", - description: "Should generate both old and new format TXT records", + description: "Should generate only new format TXT records", }, { name: "new format only - standard record", diff --git a/registry/txt_utils_test.go b/registry/txt_utils_test.go new file mode 100644 index 0000000000..96097209cb --- /dev/null +++ b/registry/txt_utils_test.go @@ -0,0 +1,51 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "sigs.k8s.io/external-dns/endpoint" +) + +/** + +helper methods + +*/ + +func newEndpointWithOwner(dnsName, target, recordType, ownerID string) *endpoint.Endpoint { + return newEndpointWithOwnerAndLabels(dnsName, target, recordType, ownerID, nil) +} + +func newEndpointWithOwnerAndOwnedRecord(dnsName, target, recordType, ownerID, ownedRecord string) *endpoint.Endpoint { + return newEndpointWithOwnerAndLabels(dnsName, target, recordType, ownerID, endpoint.Labels{endpoint.OwnedRecordLabelKey: ownedRecord}) +} + +func newEndpointWithOwnerAndLabels(dnsName, target, recordType, ownerID string, labels endpoint.Labels) *endpoint.Endpoint { + e := endpoint.NewEndpoint(dnsName, recordType, target) + e.Labels[endpoint.OwnerLabelKey] = ownerID + for k, v := range labels { + e.Labels[k] = v + } + return e +} + +func newEndpointWithOwnerResource(dnsName, target, recordType, ownerID, resource string) *endpoint.Endpoint { + e := endpoint.NewEndpoint(dnsName, recordType, target) + e.Labels[endpoint.OwnerLabelKey] = ownerID + e.Labels[endpoint.ResourceLabelKey] = resource + return e +} diff --git a/scripts/aws-cleanup-legacy-txt-records.py b/scripts/aws-cleanup-legacy-txt-records.py new file mode 100755 index 0000000000..c8827d1421 --- /dev/null +++ b/scripts/aws-cleanup-legacy-txt-records.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python + +# Copyright 2025 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Warning: The script deletes all records that match certain values. It could delete both legacy and new records if there is no way to differentiate them. + +# This Python script is designed to help migrate DNS management to `external-dns` by cleaning up legacy TXT records in AWS Route 53. +# It identifies and deletes TXT records that match a specified pattern, ensuring that `external-dns` can take over managing these resources. +# The script performs the following steps: +# +# 1. **Setup and Configuration**: +# - Imports necessary libraries (`boto3`, `argparse`, etc.). +# - Defines constants and utility functions. +# - Parses command-line arguments for configuration. +# +# 2. **Record Class**: +# - Represents a DNS record with methods to check if it should be deleted. +# +# 3. **Main Functionality**: +# - Connects to AWS Route 53 using `boto3`. +# - Support single zone cleanup at a time. +# - Lists and filters TXT records based on the specified pattern. +# - Deletes the filtered records in batches, with an option for a dry run or actual deletion. +# +# 4. **Execution**: +# - The script is executed with command-line arguments specifying the hosted zone ID, record pattern, total items to delete, batch size, and whether to perform a dry run or actual deletion. +# - Check 'To Run script' section for more details + +# WARNING: run this script at your own RISK. This will delete all the TXT records that do contain certain string. +# To Run script +# 1. Python, pip and pipenv installed https://pipenv.pypa.io/en/latest/ +# 2. AWS Access https://docs.aws.amazon.com/signin/latest/userguide/command-line-sign-in.html +# 3. pipenv shell +# 4. pip install boto3 +# 5. python scripts/aws-cleanup-legacy-txt-records.py --help +# 6. DRY RUN python scripts/aws-cleanup-legacy-txt-records.py --zone-id ASDFQEQREWRQADF --record-match text +# 6.1 Before execution consider to stop `external-dns` +# 7. Execute Deletion. First few times with reduced number of items +# - python scripts/aws-cleanup-legacy-txt-records.py --zone-id ASDFQEQREWRQADF --total-items 3 --batch-delete-count 1 --record-match 'external-dns' +# - python scripts/aws-cleanup-legacy-txt-records.py --zone-id ASDFQEQREWRQADF --total-items 10000 --batch-delete-count 50 --run --record-match "external-dns/owner=default" + +# python scripts/aws-cleanup-legacy-txt-records.py --help +# python scripts/aws-cleanup-legacy-txt-records.py --zone-id Z06155043AVN8RVC88TYY --total-items 300 --batch-delete-count 20 --record-match "external-dns/owner=default" --run + +import boto3 +from botocore.config import Config as AwsConfig +import json, argparse, os, uuid, time + +MAX_ITEMS=300 # max is 300 https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/route53/client/list_resource_record_sets.html +SLEEP=1 # in seconds, required to make sure Route53 API is not throttled +SESSION_ID=uuid.uuid4() + +def json_prettify(data): + return json.dumps(data, indent=4, default=str) + +class Record: + + def __init__(self, record): + # static + self.type = 'TXT' + self.record = record + self.name = record['Name'] + self.resource_records = record['ResourceRecords'] + resource_record = '' + for r in self.resource_records: + resource_record += r['Value'] + self.resource_record = resource_record + + def is_for_deletion(self, contains): + + if contains in self.resource_record: + return True + return False + + def __str__(self): + return f'record: name: {self.name}, type: {self.type}, records: {self.resource_record}' + +class Config: + + def __init__(self, zone_id, contain, total_items, batch, run): + self.zone_id = zone_id + self.record_contain = contain + self.total_items = total_items + self.batch_size = batch + self.run = run + self.contain = contain + +def records(config: Config) -> None: + print(f"calculate TXT records to cleanup for 'zone:{config.zone_id}' and 'max records:{config.total_items}'") + # https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html + cfg = AwsConfig( + user_agent=f"ExternalDNS/boto3-{SESSION_ID}", + + ) + r53client = boto3.client('route53', config=cfg) + dns_records_to_cleanup = [] + items = 0 + try: + params = { + 'HostedZoneId': config.zone_id, + 'MaxItems': str(MAX_ITEMS), + } + dns_in_iteration = r53client.list_resource_record_sets(**params) + elements = dns_in_iteration['ResourceRecordSets'] + for el in elements: + if el['Type'] == 'TXT': + record = Record(el) + if record.is_for_deletion(config.contain): + dns_records_to_cleanup.append(record) + print("to cleanup >>", record) + items += 1 + if items >= config.total_items: + break + + while len(elements) > 0 and 'NextRecordName' in dns_in_iteration.keys() and items < config.total_items: + dns_in_iteration = r53client.list_resource_record_sets( + HostedZoneId= config.zone_id, + StartRecordName= dns_in_iteration['NextRecordName'], + MaxItems= str(MAX_ITEMS), + ) + elements = dns_in_iteration['ResourceRecordSets'] + for el in elements: + if el['Type'] == 'TXT': + record = Record(el) + if record.is_for_deletion(config.contain): + dns_records_to_cleanup.append(record) + print("to cleanup >>", record) + items += 1 + if items >= config.total_items: + break + + if len(dns_records_to_cleanup) > 0: + delete_records(r53client, config, dns_records_to_cleanup) + else: + print("No 'TXT' records found to cleanup....") + except Exception as e: + print(f"An error occurred: {e}") + os._exit(os.EX_OSERR) + +def delete_records(client: boto3.client, config: Config, records: list[Record]) -> None: + total=len(records) + print(f"will cleanup '{total}' records with batch '{config.batch_size}' at a time") + count = 0 + + if config.run: + print("deletion of records!!") + else: + print("dry run execution") + + for i in range(0, total, config.batch_size): + if config.batch_size <= 0: + break + batch = records[i:min(i + config.batch_size, total)] + count += config.batch_size + if count >= total: + count = total + + changes = [] + + for el in batch: + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/route53/client/change_resource_record_sets.html + changes.append({ + 'Action': 'DELETE', + 'ResourceRecordSet': el.record + }) + + print(f"BATCH deletion(start). {len(changes)} records > {changes}") + + if config.run: + client.change_resource_record_sets( + HostedZoneId=config.zone_id, + ChangeBatch={ + "Comment": "external-dns legacy record cleanup. batch of ", + "Changes": changes, + } + ) + time.sleep(SLEEP) + + print(f"BATCH deletion(success). {count}/{total}(deleted/total)") + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Cleanup legacy TXT records") + parser.add_argument("--zone-id", type=str, required=True, help="Hosted Zone ID for which to run a cleanup.") + parser.add_argument("--record-match", type=str, required=True, help="Record to match specific value. Example 'external-dns/owner=default'") + parser.add_argument("--total-items", type=int, required=False, default=100, help="Number of items to delete. Default to 10") + parser.add_argument("--batch-delete-count", type=int, required=False, default=2, help="Number of items to delete in single DELETE batch. Default to 2") + parser.add_argument("--run", action="store_true", help="Execute the cleanup. The tool will do a dry-run if --run is not specified.") + + print("Run this script at your own RISKS!!!!") + print(f"Session ID '{SESSION_ID}'") + + args = parser.parse_args() + print("arguments:",args) + cfg = Config( + zone_id=args.zone_id, + contain=args.record_match, + total_items=args.total_items, + batch=args.batch_delete_count, + run=args.run, + ) + records(cfg)