From a0811eb3644788c38915a50574e0ce2a3b722f97 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 20 Oct 2025 17:45:22 -0600 Subject: [PATCH 01/21] add oauth2 support --- .../filebeat/filebeat-input-azure-eventhub.md | 49 ++++++- x-pack/filebeat/input/azureeventhub/config.go | 67 ++++++++-- .../input/azureeventhub/config_test.go | 122 ++++++++++++++++++ .../filebeat/input/azureeventhub/v2_input.go | 88 +++++++++++-- 4 files changed, 303 insertions(+), 23 deletions(-) diff --git a/docs/reference/filebeat/filebeat-input-azure-eventhub.md b/docs/reference/filebeat/filebeat-input-azure-eventhub.md index 493080f9b920..e1bfab41c0c5 100644 --- a/docs/reference/filebeat/filebeat-input-azure-eventhub.md +++ b/docs/reference/filebeat/filebeat-input-azure-eventhub.md @@ -13,7 +13,7 @@ Users can make use of the `azure-eventhub` input in order to read messages from Users can enable internal logs tracing for this input by setting the environment variable `BEATS_AZURE_EVENTHUB_INPUT_TRACING_ENABLED: true`. When enabled, this input will log additional information to the logs. Additional information includes partition ownership, blob lease information, and other internal state. -Example configuration: +Example configuration using Shared Access Key authentication: ```yaml filebeat.inputs: @@ -27,6 +27,24 @@ filebeat.inputs: resource_manager_endpoint: "" ``` +Example configuration using OAuth2 authentication: + +```yaml +filebeat.inputs: +- type: azure-eventhub + eventhub: "insights-operational-logs" + consumer_group: "test" + # No connection_string provided - automatically uses OAuth2 + eventhub_namespace: "your-eventhub-namespace.servicebus.windows.net" + tenant_id: "your-tenant-id" + client_id: "your-client-id" + client_secret: "your-client-secret" + authority_host: "https://login.microsoftonline.com" + storage_account: "azureeph" + storage_account_connection_string: "DefaultEndpointsProtocol=https;AccountName=..." + storage_account_container: "" +``` + ## Configuration options [_configuration_options] The `azure-eventhub` input supports the following configuration: @@ -44,10 +62,37 @@ Optional, we recommend using a dedicated consumer group for the azure input. Reu ## `connection_string` [_connection_string] -The connection string required to communicate with Event Hubs, steps here [https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string). +The connection string required to communicate with Event Hubs when using Shared Access Key authentication, steps here [https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string). + +**Note**: If `connection_string` is not provided, the input will automatically use OAuth2 authentication and require the OAuth2 configuration parameters below. A Blob Storage account is required in order to store/retrieve/update the offset or state of the eventhub messages. This means that after stopping filebeat it can start back up at the spot that it stopped processing messages. +## `eventhub_namespace` [_eventhub_namespace] + +The fully qualified namespace for the Event Hub. Required when `connection_string` is not provided (OAuth2 authentication). Format: `your-eventhub-namespace.servicebus.windows.net` + +## `tenant_id` [_tenant_id] + +The Azure Active Directory tenant ID. Required when `connection_string` is not provided (OAuth2 authentication). + +## `client_id` [_client_id] + +The Azure Active Directory application (client) ID. Required when `connection_string` is not provided (OAuth2 authentication). + +## `client_secret` [_client_secret] + +The Azure Active Directory application client secret. Required when `connection_string` is not provided (OAuth2 authentication). + +## `authority_host` [_authority_host] + +The Azure Active Directory authority host. Optional when using OAuth2 authentication. Defaults to Azure Public Cloud (`https://login.microsoftonline.com`). + +Supported values: +- `https://login.microsoftonline.com` (Azure Public Cloud - default) +- `https://login.microsoftonline.us` (Azure Government) +- `https://login.chinacloudapi.cn` (Azure China) + ## `storage_account` [_storage_account] diff --git a/x-pack/filebeat/input/azureeventhub/config.go b/x-pack/filebeat/input/azureeventhub/config.go index eb7157f9b0c6..30e4b689b614 100644 --- a/x-pack/filebeat/input/azureeventhub/config.go +++ b/x-pack/filebeat/input/azureeventhub/config.go @@ -23,7 +23,8 @@ type azureInputConfig struct { // EventHubName is the name of the event hub to connect to. EventHubName string `config:"eventhub" validate:"required"` // ConnectionString is the connection string to connect to the event hub. - ConnectionString string `config:"connection_string" validate:"required"` + // This is required when using Shared Access Key authentication. + ConnectionString string `config:"connection_string"` // ConsumerGroup is the name of the consumer group to use. ConsumerGroup string `config:"consumer_group"` // Azure Storage container to store leases and checkpoints @@ -38,6 +39,26 @@ type azureInputConfig struct { SAContainer string `config:"storage_account_container"` // by default the azure public environment is used, to override, users can provide a specific resource manager endpoint OverrideEnvironment string `config:"resource_manager_endpoint"` + + // --------------------------------------- + // OAuth2 authentication configuration + // --------------------------------------- + + // EventHubNamespace is the fully qualified namespace for the Event Hub. + // Required when using OAuth2 authentication (when connection_string is not provided). + EventHubNamespace string `config:"eventhub_namespace"` + // TenantID is the Azure Active Directory tenant ID. + // Required when using OAuth2 authentication (when connection_string is not provided). + TenantID string `config:"tenant_id"` + // ClientID is the Azure Active Directory application (client) ID. + // Required when using OAuth2 authentication (when connection_string is not provided). + ClientID string `config:"client_id"` + // ClientSecret is the Azure Active Directory application client secret. + // Required when using OAuth2 authentication with client credentials flow. + ClientSecret string `config:"client_secret"` + // AuthorityHost is the Azure Active Directory authority host. + // Optional, defaults to Azure Public Cloud. + AuthorityHost string `config:"authority_host"` // LegacySanitizeOptions is a list of sanitization options to apply to messages. // // The supported options are: @@ -125,19 +146,39 @@ func defaultConfig() azureInputConfig { func (conf *azureInputConfig) Validate() error { logger := logp.NewLogger("azureeventhub.config") - connectionStringProperties, err := parseConnectionString(conf.ConnectionString) - if err != nil { - return fmt.Errorf("invalid connection string: %w", err) - } + // Determine authentication method based on whether connection_string is provided + useOAuth2 := conf.ConnectionString == "" - // If the connection string contains an entity path, we need to double - // check that it matches the event hub name. - if connectionStringProperties.EntityPath != nil && *connectionStringProperties.EntityPath != conf.EventHubName { - return fmt.Errorf( - "invalid config: the entity path (%s) in the connection string does not match event hub name (%s)", - *connectionStringProperties.EntityPath, - conf.EventHubName, - ) + if useOAuth2 { + // Validate OAuth2 configuration + if conf.EventHubNamespace == "" { + return errors.New("eventhub_namespace is required when connection_string is not provided (OAuth2 authentication)") + } + if conf.TenantID == "" { + return errors.New("tenant_id is required when connection_string is not provided (OAuth2 authentication)") + } + if conf.ClientID == "" { + return errors.New("client_id is required when connection_string is not provided (OAuth2 authentication)") + } + if conf.ClientSecret == "" { + return errors.New("client_secret is required when connection_string is not provided (OAuth2 authentication)") + } + } else { + // Validate connection string configuration + connectionStringProperties, err := parseConnectionString(conf.ConnectionString) + if err != nil { + return fmt.Errorf("invalid connection string: %w", err) + } + + // If the connection string contains an entity path, we need to double + // check that it matches the event hub name. + if connectionStringProperties.EntityPath != nil && *connectionStringProperties.EntityPath != conf.EventHubName { + return fmt.Errorf( + "invalid config: the entity path (%s) in the connection string does not match event hub name (%s)", + *connectionStringProperties.EntityPath, + conf.EventHubName, + ) + } } if conf.EventHubName == "" { diff --git a/x-pack/filebeat/input/azureeventhub/config_test.go b/x-pack/filebeat/input/azureeventhub/config_test.go index e2b21dcabafd..24435a1c9ca1 100644 --- a/x-pack/filebeat/input/azureeventhub/config_test.go +++ b/x-pack/filebeat/input/azureeventhub/config_test.go @@ -154,3 +154,125 @@ func TestValidateConnectionStringV2(t *testing.T) { assert.ErrorContains(t, err, "invalid config: the entity path (my-event-hub) in the connection string does not match event hub name (not-my-event-hub)") }) } + +func TestOAuth2ConfigValidation(t *testing.T) { + tests := []struct { + name string + config azureInputConfig + expectError bool + errorMsg string + }{ + { + name: "valid oauth2 config (no connection_string)", + config: azureInputConfig{ + EventHubName: "test-hub", + EventHubNamespace: "test-namespace.servicebus.windows.net", + TenantID: "test-tenant-id", + ClientID: "test-client-id", + ClientSecret: "test-client-secret", + SAName: "test-storage", + SAConnectionString: "test-connection-string", + }, + expectError: false, + }, + { + name: "oauth2 config missing namespace", + config: azureInputConfig{ + EventHubName: "test-hub", + TenantID: "test-tenant-id", + ClientID: "test-client-id", + ClientSecret: "test-client-secret", + SAName: "test-storage", + SAConnectionString: "test-connection-string", + }, + expectError: true, + errorMsg: "eventhub_namespace is required when connection_string is not provided (OAuth2 authentication)", + }, + { + name: "oauth2 config missing tenant_id", + config: azureInputConfig{ + EventHubName: "test-hub", + EventHubNamespace: "test-namespace.servicebus.windows.net", + ClientID: "test-client-id", + ClientSecret: "test-client-secret", + SAName: "test-storage", + SAConnectionString: "test-connection-string", + }, + expectError: true, + errorMsg: "tenant_id is required when connection_string is not provided (OAuth2 authentication)", + }, + { + name: "oauth2 config missing client_id", + config: azureInputConfig{ + EventHubName: "test-hub", + EventHubNamespace: "test-namespace.servicebus.windows.net", + TenantID: "test-tenant-id", + ClientSecret: "test-client-secret", + SAName: "test-storage", + SAConnectionString: "test-connection-string", + }, + expectError: true, + errorMsg: "client_id is required when connection_string is not provided (OAuth2 authentication)", + }, + { + name: "oauth2 config missing client_secret", + config: azureInputConfig{ + EventHubName: "test-hub", + EventHubNamespace: "test-namespace.servicebus.windows.net", + TenantID: "test-tenant-id", + ClientID: "test-client-id", + SAName: "test-storage", + SAConnectionString: "test-connection-string", + }, + expectError: true, + errorMsg: "client_secret is required when connection_string is not provided (OAuth2 authentication)", + }, + { + name: "valid connection_string config", + config: azureInputConfig{ + EventHubName: "test-hub", + ConnectionString: "Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=test;SharedAccessKey=test", + SAName: "test-storage", + SAConnectionString: "test-connection-string", + }, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.config.Validate() + if tt.expectError { + if err == nil { + t.Errorf("expected error but got none") + return + } + if tt.errorMsg != "" && err.Error() != tt.errorMsg { + t.Errorf("expected error message %q, got %q", tt.errorMsg, err.Error()) + } + } else { + if err != nil { + t.Errorf("unexpected error: %v", err) + } + } + }) + } +} + +func TestGetAzureCloud(t *testing.T) { + // Test that the function doesn't panic for different authority hosts + testCases := []string{ + "https://login.microsoftonline.com", + "https://login.microsoftonline.us", + "https://login.chinacloudapi.cn", + "", + } + + for _, authorityHost := range testCases { + t.Run("authority_host_"+authorityHost, func(t *testing.T) { + cloud := getAzureCloud(authorityHost) + // Just verify we got a result - we can't easily compare cloud configurations + _ = cloud + }) + } +} diff --git a/x-pack/filebeat/input/azureeventhub/v2_input.go b/x-pack/filebeat/input/azureeventhub/v2_input.go index ab249b110cde..19bdfc6ed870 100644 --- a/x-pack/filebeat/input/azureeventhub/v2_input.go +++ b/x-pack/filebeat/input/azureeventhub/v2_input.go @@ -17,6 +17,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs/checkpoints" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" @@ -215,14 +216,26 @@ func (in *eventHubInputV2) setup(ctx context.Context) error { } // Create the event hub consumerClient to receive events. - consumerClient, err := azeventhubs.NewConsumerClientFromConnectionString( - in.config.ConnectionString, - eventHubName, - in.config.ConsumerGroup, - nil, - ) - if err != nil { - return fmt.Errorf("failed to create consumer client: %w", err) + var consumerClient *azeventhubs.ConsumerClient + + // Determine authentication method based on whether connection_string is provided + if in.config.ConnectionString == "" { + // Use OAuth2 authentication + consumerClient, err = createConsumerClientWithOAuth2(in.config, in.log) + if err != nil { + return fmt.Errorf("failed to create consumer client with OAuth2: %w", err) + } + } else { + // Use connection string authentication (default) + consumerClient, err = azeventhubs.NewConsumerClientFromConnectionString( + in.config.ConnectionString, + eventHubName, + in.config.ConsumerGroup, + nil, + ) + if err != nil { + return fmt.Errorf("failed to create consumer client: %w", err) + } } in.consumerClient = consumerClient @@ -659,3 +672,62 @@ func shutdownPartitionResources(ctx context.Context, partitionClient *azeventhub // processing events for this partition. defer pipelineClient.Close() } + +// createConsumerClientWithOAuth2 creates a new Event Hub consumer client using OAuth2 authentication. +func createConsumerClientWithOAuth2( + config azureInputConfig, + log *logp.Logger, +) (*azeventhubs.ConsumerClient, error) { + log = log.Named("oauth2") + + // Create credential options + credentialOptions := &azidentity.ClientSecretCredentialOptions{ + ClientOptions: azcore.ClientOptions{ + Cloud: getAzureCloud(config.AuthorityHost), + }, + } + + // Create the credential + credential, err := azidentity.NewClientSecretCredential( + config.TenantID, + config.ClientID, + config.ClientSecret, + credentialOptions, + ) + if err != nil { + return nil, fmt.Errorf("failed to create client secret credential: %w", err) + } + + // Create the consumer client with OAuth2 authentication + consumerClient, err := azeventhubs.NewConsumerClient( + config.EventHubNamespace, + config.EventHubName, + config.ConsumerGroup, + credential, + nil, + ) + if err != nil { + return nil, fmt.Errorf("failed to create consumer client with OAuth2: %w", err) + } + + log.Infow("successfully created consumer client with OAuth2 authentication", + "namespace", config.EventHubNamespace, + "eventhub", config.EventHubName, + "tenant_id", config.TenantID, + "client_id", config.ClientID, + ) + + return consumerClient, nil +} + +// getAzureCloud returns the appropriate Azure cloud configuration based on the authority host. +func getAzureCloud(authorityHost string) cloud.Configuration { + switch authorityHost { + case "https://login.microsoftonline.us": + return cloud.AzureGovernment + case "https://login.chinacloudapi.cn": + return cloud.AzureChina + default: + return cloud.AzurePublic + } +} From c6ff76b151aefec6185b9380a485506af0dfccf4 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 21 Oct 2025 21:11:10 -0600 Subject: [PATCH 02/21] fix parsing connection string when using oauth2 --- .../filebeat/input/azureeventhub/v2_input.go | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/x-pack/filebeat/input/azureeventhub/v2_input.go b/x-pack/filebeat/input/azureeventhub/v2_input.go index 19bdfc6ed870..4769d9ad4fc0 100644 --- a/x-pack/filebeat/input/azureeventhub/v2_input.go +++ b/x-pack/filebeat/input/azureeventhub/v2_input.go @@ -189,32 +189,6 @@ func (in *eventHubInputV2) setup(ctx context.Context) error { } in.checkpointStore = checkpointStore - // There is a mismatch between how the azure-eventhub input and the new - // Event Hub SDK expect the event hub name in the connection string. - // - // The azure-eventhub input was designed to work with the old Event Hub SDK, - // which worked using the event hub name in the connection string. - // - // The new Event Hub SDK expects clients to pass the event hub name as a - // parameter, or in the connection string as the entity path. - // - // We need to handle both cases. - eventHubName := in.config.EventHubName - - connectionStringProperties, err := parseConnectionString(in.config.ConnectionString) - if err != nil { - return fmt.Errorf("failed to parse connection string: %w", err) - } - if connectionStringProperties.EntityPath != nil { - // If the connection string contains an entity path, we need to - // set the event hub name to an empty string. - // - // This is a requirement of the new Event Hub SDK. - // - // See: https://github.com/Azure/azure-sdk-for-go/blob/4ece3e50652223bba502f2b73e7f297de34a799c/sdk/messaging/azeventhubs/producer_client.go#L304-L306 - eventHubName = "" - } - // Create the event hub consumerClient to receive events. var consumerClient *azeventhubs.ConsumerClient @@ -227,6 +201,32 @@ func (in *eventHubInputV2) setup(ctx context.Context) error { } } else { // Use connection string authentication (default) + // There is a mismatch between how the azure-eventhub input and the new + // Event Hub SDK expect the event hub name in the connection string. + // + // The azure-eventhub input was designed to work with the old Event Hub SDK, + // which worked using the event hub name in the connection string. + // + // The new Event Hub SDK expects clients to pass the event hub name as a + // parameter, or in the connection string as the entity path. + // + // We need to handle both cases. + eventHubName := in.config.EventHubName + + connectionStringProperties, err := parseConnectionString(in.config.ConnectionString) + if err != nil { + return fmt.Errorf("failed to parse connection string: %w", err) + } + if connectionStringProperties.EntityPath != nil { + // If the connection string contains an entity path, we need to + // set the event hub name to an empty string. + // + // This is a requirement of the new Event Hub SDK. + // + // See: https://github.com/Azure/azure-sdk-for-go/blob/4ece3e50652223bba502f2b73e7f297de34a799c/sdk/messaging/azeventhubs/producer_client.go#L304-L306 + eventHubName = "" + } + consumerClient, err = azeventhubs.NewConsumerClientFromConnectionString( in.config.ConnectionString, eventHubName, From 25a4dd1f940db7d419f75dae0e9aec8a56f63fa9 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 28 Oct 2025 13:44:23 -0600 Subject: [PATCH 03/21] support oauth2 with storage account --- ...37-feat-add-oauth2-for-azure-eventhub.yaml | 45 +++++++++++ x-pack/filebeat/input/azureeventhub/config.go | 19 ++++- .../filebeat/input/azureeventhub/v2_input.go | 80 ++++++++++++++++--- 3 files changed, 131 insertions(+), 13 deletions(-) create mode 100644 changelog/fragments/1761678237-feat-add-oauth2-for-azure-eventhub.yaml diff --git a/changelog/fragments/1761678237-feat-add-oauth2-for-azure-eventhub.yaml b/changelog/fragments/1761678237-feat-add-oauth2-for-azure-eventhub.yaml new file mode 100644 index 000000000000..0412ace9da15 --- /dev/null +++ b/changelog/fragments/1761678237-feat-add-oauth2-for-azure-eventhub.yaml @@ -0,0 +1,45 @@ +# REQUIRED +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: feature + +# REQUIRED for all kinds +# Change summary; a 80ish characters long description of the change. +summary: add oauth2 authentication for azure eventhub and storage in Filebeat + +# REQUIRED for breaking-change, deprecation, known-issue +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# description: + +# REQUIRED for breaking-change, deprecation, known-issue +# impact: + +# REQUIRED for breaking-change, deprecation, known-issue +# action: + +# REQUIRED for all kinds +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: "filebeat" + +# AUTOMATED +# OPTIONAL to manually add other PR URLs +# PR URL: A link the PR that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +# pr: https://github.com/owner/repo/1234 + +# AUTOMATED +# OPTIONAL to manually add other issue URLs +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +# issue: https://github.com/owner/repo/1234 diff --git a/x-pack/filebeat/input/azureeventhub/config.go b/x-pack/filebeat/input/azureeventhub/config.go index 30e4b689b614..ef2523a50b33 100644 --- a/x-pack/filebeat/input/azureeventhub/config.go +++ b/x-pack/filebeat/input/azureeventhub/config.go @@ -236,8 +236,23 @@ func (conf *azureInputConfig) Validate() error { return errors.New("no storage account key configured (config: storage_account_key)") } case processorV2: - if conf.SAConnectionString == "" { - return errors.New("no storage account connection string configured (config: storage_account_connection_string)") + // For processor v2, either connection string or OAuth2 must be configured + useOAuth2ForStorage := conf.SAConnectionString == "" + if useOAuth2ForStorage { + // Validate OAuth2 configuration for storage + if conf.TenantID == "" { + return errors.New("tenant_id is required when storage_account_connection_string is not provided (OAuth2 authentication)") + } + if conf.ClientID == "" { + return errors.New("client_id is required when storage_account_connection_string is not provided (OAuth2 authentication)") + } + if conf.ClientSecret == "" { + return errors.New("client_secret is required when storage_account_connection_string is not provided (OAuth2 authentication)") + } + } else { + if conf.SAConnectionString == "" { + return errors.New("no storage account connection string configured (config: storage_account_connection_string)") + } } default: return fmt.Errorf( diff --git a/x-pack/filebeat/input/azureeventhub/v2_input.go b/x-pack/filebeat/input/azureeventhub/v2_input.go index 4769d9ad4fc0..2a4ff8195049 100644 --- a/x-pack/filebeat/input/azureeventhub/v2_input.go +++ b/x-pack/filebeat/input/azureeventhub/v2_input.go @@ -147,18 +147,32 @@ func (in *eventHubInputV2) setup(ctx context.Context) error { sanitizers: sanitizers, } - containerClient, err := container.NewClientFromConnectionString( - in.config.SAConnectionString, - in.config.SAContainer, - &container.ClientOptions{ - ClientOptions: azcore.ClientOptions{ - Cloud: cloud.AzurePublic, + // Create the container client + var containerClient *container.Client + + // Determine the authentication method for the storage account based on whether SAConnectionString is provided + if in.config.SAConnectionString == "" { + // Use OAuth2 authentication for storage + containerClient, err = createContainerClientWithOAuth2(in.config, in.log) + if err != nil { + in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating blob container client with OAuth2: %s", err.Error())) + return fmt.Errorf("failed to create blob container client with OAuth2: %w", err) + } + } else { + // Use connection string authentication (default) + containerClient, err = container.NewClientFromConnectionString( + in.config.SAConnectionString, + in.config.SAContainer, + &container.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Cloud: cloud.AzurePublic, + }, }, - }, - ) - if err != nil { - in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating blob container client: %s", err.Error())) - return fmt.Errorf("failed to create blob container client: %w", err) + ) + if err != nil { + in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating blob container client: %s", err.Error())) + return fmt.Errorf("failed to create blob container client: %w", err) + } } // The modern event hub SDK does not create the container @@ -720,6 +734,50 @@ func createConsumerClientWithOAuth2( return consumerClient, nil } +// createContainerClientWithOAuth2 creates a new Blob Storage container client using OAuth2 authentication. +func createContainerClientWithOAuth2( + config azureInputConfig, + log *logp.Logger, +) (*container.Client, error) { + log = log.Named("oauth2") + + // Build the storage account URL + storageAccountURL := fmt.Sprintf("https://%s.blob.core.windows.net", config.SAName) + + // Create credential options + credentialOptions := &azidentity.ClientSecretCredentialOptions{ + ClientOptions: azcore.ClientOptions{ + Cloud: getAzureCloud(config.AuthorityHost), + }, + } + + // Create the credential + credential, err := azidentity.NewClientSecretCredential( + config.TenantID, + config.ClientID, + config.ClientSecret, + credentialOptions, + ) + if err != nil { + return nil, fmt.Errorf("failed to create client secret credential: %w", err) + } + + // Create the container client with OAuth2 authentication + containerClient, err := container.NewClient(storageAccountURL+"/"+config.SAContainer, credential, nil) + if err != nil { + return nil, fmt.Errorf("failed to create container client with OAuth2: %w", err) + } + + log.Infow("successfully created container client with OAuth2 authentication", + "storage_account", config.SAName, + "container", config.SAContainer, + "tenant_id", config.TenantID, + "client_id", config.ClientID, + ) + + return containerClient, nil +} + // getAzureCloud returns the appropriate Azure cloud configuration based on the authority host. func getAzureCloud(authorityHost string) cloud.Configuration { switch authorityHost { From ba53a00f35f128b1777651f9a4f47733e13bd49a Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 28 Oct 2025 17:56:19 -0600 Subject: [PATCH 04/21] add tests --- .../input/azureeventhub/config_test.go | 144 ++++++++++++------ 1 file changed, 96 insertions(+), 48 deletions(-) diff --git a/x-pack/filebeat/input/azureeventhub/config_test.go b/x-pack/filebeat/input/azureeventhub/config_test.go index 24435a1c9ca1..caa21d57fdd5 100644 --- a/x-pack/filebeat/input/azureeventhub/config_test.go +++ b/x-pack/filebeat/input/azureeventhub/config_test.go @@ -163,78 +163,126 @@ func TestOAuth2ConfigValidation(t *testing.T) { errorMsg string }{ { - name: "valid oauth2 config (no connection_string)", - config: azureInputConfig{ - EventHubName: "test-hub", - EventHubNamespace: "test-namespace.servicebus.windows.net", - TenantID: "test-tenant-id", - ClientID: "test-client-id", - ClientSecret: "test-client-secret", - SAName: "test-storage", - SAConnectionString: "test-connection-string", - }, + name: "valid oauth2 config for eventhub (no connection_string)", + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.EventHubNamespace = "test-namespace.servicebus.windows.net" + c.TenantID = "test-tenant-id" + c.ClientID = "test-client-id" + c.ClientSecret = "test-client-secret" + c.SAName = "test-storage" + c.SAConnectionString = "test-connection-string" + c.ProcessorVersion = "v2" + return c + }(), + expectError: false, + }, + { + name: "valid oauth2 config for only storage account (no storage_account_connection_string)", + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.ConnectionString = "Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=test;SharedAccessKey=test" + c.TenantID = "test-tenant-id" + c.ClientID = "test-client-id" + c.ClientSecret = "test-client-secret" + c.SAName = "test-storage" + c.ProcessorVersion = "v2" + return c + }(), + expectError: false, + }, + { + name: "valid oauth2 config for both eventhub and storage account (no connection strings)", + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.EventHubNamespace = "test-namespace.servicebus.windows.net" + c.TenantID = "test-tenant-id" + c.ClientID = "test-client-id" + c.ClientSecret = "test-client-secret" + c.SAName = "test-storage" + c.ProcessorVersion = "v2" + return c + }(), expectError: false, }, { name: "oauth2 config missing namespace", - config: azureInputConfig{ - EventHubName: "test-hub", - TenantID: "test-tenant-id", - ClientID: "test-client-id", - ClientSecret: "test-client-secret", - SAName: "test-storage", - SAConnectionString: "test-connection-string", - }, + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.TenantID = "test-tenant-id" + c.ClientID = "test-client-id" + c.ClientSecret = "test-client-secret" + c.SAName = "test-storage" + c.SAConnectionString = "test-connection-string" + c.ProcessorVersion = "v2" + return c + }(), expectError: true, errorMsg: "eventhub_namespace is required when connection_string is not provided (OAuth2 authentication)", }, { name: "oauth2 config missing tenant_id", - config: azureInputConfig{ - EventHubName: "test-hub", - EventHubNamespace: "test-namespace.servicebus.windows.net", - ClientID: "test-client-id", - ClientSecret: "test-client-secret", - SAName: "test-storage", - SAConnectionString: "test-connection-string", - }, + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.EventHubNamespace = "test-namespace.servicebus.windows.net" + c.ClientID = "test-client-id" + c.ClientSecret = "test-client-secret" + c.SAName = "test-storage" + c.SAConnectionString = "test-connection-string" + c.ProcessorVersion = "v2" + return c + }(), expectError: true, errorMsg: "tenant_id is required when connection_string is not provided (OAuth2 authentication)", }, { name: "oauth2 config missing client_id", - config: azureInputConfig{ - EventHubName: "test-hub", - EventHubNamespace: "test-namespace.servicebus.windows.net", - TenantID: "test-tenant-id", - ClientSecret: "test-client-secret", - SAName: "test-storage", - SAConnectionString: "test-connection-string", - }, + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.EventHubNamespace = "test-namespace.servicebus.windows.net" + c.TenantID = "test-tenant-id" + c.ClientSecret = "test-client-secret" + c.SAName = "test-storage" + c.SAConnectionString = "test-connection-string" + c.ProcessorVersion = "v2" + return c + }(), expectError: true, errorMsg: "client_id is required when connection_string is not provided (OAuth2 authentication)", }, { name: "oauth2 config missing client_secret", - config: azureInputConfig{ - EventHubName: "test-hub", - EventHubNamespace: "test-namespace.servicebus.windows.net", - TenantID: "test-tenant-id", - ClientID: "test-client-id", - SAName: "test-storage", - SAConnectionString: "test-connection-string", - }, + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.EventHubNamespace = "test-namespace.servicebus.windows.net" + c.TenantID = "test-tenant-id" + c.ClientID = "test-client-id" + c.SAName = "test-storage" + c.SAConnectionString = "test-connection-string" + c.ProcessorVersion = "v2" + return c + }(), expectError: true, errorMsg: "client_secret is required when connection_string is not provided (OAuth2 authentication)", }, { name: "valid connection_string config", - config: azureInputConfig{ - EventHubName: "test-hub", - ConnectionString: "Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=test;SharedAccessKey=test", - SAName: "test-storage", - SAConnectionString: "test-connection-string", - }, + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.ConnectionString = "Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=test;SharedAccessKey=test" + c.SAName = "test-storage" + c.SAConnectionString = "test-connection-string" + c.ProcessorVersion = "v2" + return c + }(), expectError: false, }, } From 8f9f12b0547f18060a7f83089c75f8c5bbea490b Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 29 Oct 2025 17:21:46 -0600 Subject: [PATCH 05/21] update doc --- docs/reference/filebeat/filebeat-input-azure-eventhub.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/filebeat/filebeat-input-azure-eventhub.md b/docs/reference/filebeat/filebeat-input-azure-eventhub.md index e1bfab41c0c5..b9ba2aa29c31 100644 --- a/docs/reference/filebeat/filebeat-input-azure-eventhub.md +++ b/docs/reference/filebeat/filebeat-input-azure-eventhub.md @@ -34,15 +34,15 @@ filebeat.inputs: - type: azure-eventhub eventhub: "insights-operational-logs" consumer_group: "test" - # No connection_string provided - automatically uses OAuth2 + # No connection_string provided - automatically uses OAuth2 for both eventhub and storage account eventhub_namespace: "your-eventhub-namespace.servicebus.windows.net" tenant_id: "your-tenant-id" client_id: "your-client-id" client_secret: "your-client-secret" authority_host: "https://login.microsoftonline.com" storage_account: "azureeph" - storage_account_connection_string: "DefaultEndpointsProtocol=https;AccountName=..." storage_account_container: "" + processor_version: "v2" ``` ## Configuration options [_configuration_options] From 147f4f15f82ec7503a31bd46d39b48bc3a3b0838 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 31 Oct 2025 16:06:05 -0600 Subject: [PATCH 06/21] update doc --- ...37-feat-add-oauth2-for-azure-eventhub.yaml | 2 +- .../filebeat/filebeat-input-azure-eventhub.md | 22 ++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/changelog/fragments/1761678237-feat-add-oauth2-for-azure-eventhub.yaml b/changelog/fragments/1761678237-feat-add-oauth2-for-azure-eventhub.yaml index 0412ace9da15..1ae4fda1da97 100644 --- a/changelog/fragments/1761678237-feat-add-oauth2-for-azure-eventhub.yaml +++ b/changelog/fragments/1761678237-feat-add-oauth2-for-azure-eventhub.yaml @@ -13,7 +13,7 @@ kind: feature # REQUIRED for all kinds # Change summary; a 80ish characters long description of the change. -summary: add oauth2 authentication for azure eventhub and storage in Filebeat +summary: Add OAuth2 authentication for Azure Event Hub and storage in Filebeat. # REQUIRED for breaking-change, deprecation, known-issue # Long description; in case the summary is not enough to describe the change diff --git a/docs/reference/filebeat/filebeat-input-azure-eventhub.md b/docs/reference/filebeat/filebeat-input-azure-eventhub.md index b9ba2aa29c31..358298063965 100644 --- a/docs/reference/filebeat/filebeat-input-azure-eventhub.md +++ b/docs/reference/filebeat/filebeat-input-azure-eventhub.md @@ -27,7 +27,7 @@ filebeat.inputs: resource_manager_endpoint: "" ``` -Example configuration using OAuth2 authentication: +{applies_to}`stack: ga 9.3.0` Example configuration using OAuth2 authentication: ```yaml filebeat.inputs: @@ -70,22 +70,42 @@ A Blob Storage account is required in order to store/retrieve/update the offset ## `eventhub_namespace` [_eventhub_namespace] +```{applies_to} +stack: ga 9.3.0 +``` + The fully qualified namespace for the Event Hub. Required when `connection_string` is not provided (OAuth2 authentication). Format: `your-eventhub-namespace.servicebus.windows.net` ## `tenant_id` [_tenant_id] +```{applies_to} +stack: ga 9.3.0 +``` + The Azure Active Directory tenant ID. Required when `connection_string` is not provided (OAuth2 authentication). ## `client_id` [_client_id] +```{applies_to} +stack: ga 9.3.0 +``` + The Azure Active Directory application (client) ID. Required when `connection_string` is not provided (OAuth2 authentication). ## `client_secret` [_client_secret] +```{applies_to} +stack: ga 9.3.0 +``` + The Azure Active Directory application client secret. Required when `connection_string` is not provided (OAuth2 authentication). ## `authority_host` [_authority_host] +```{applies_to} +stack: ga 9.3.0 +``` + The Azure Active Directory authority host. Optional when using OAuth2 authentication. Defaults to Azure Public Cloud (`https://login.microsoftonline.com`). Supported values: From 1c807949443bb637ceff1c0c947141d4092ef42b Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 3 Nov 2025 22:48:00 -0700 Subject: [PATCH 07/21] add auth_type --- .../filebeat/filebeat-input-azure-eventhub.md | 68 ++++- x-pack/filebeat/input/azureeventhub/auth.go | 55 ++++ .../input/azureeventhub/client_secret.go | 68 +++++ .../input/azureeventhub/client_secret_test.go | 29 ++ .../filebeat/input/azureeventhub/clients.go | 122 ++++++++ x-pack/filebeat/input/azureeventhub/config.go | 88 +++--- .../input/azureeventhub/config_test.go | 18 -- x-pack/filebeat/input/azureeventhub/input.go | 10 +- .../filebeat/input/azureeventhub/v2_input.go | 270 +++++++----------- 9 files changed, 496 insertions(+), 232 deletions(-) create mode 100644 x-pack/filebeat/input/azureeventhub/auth.go create mode 100644 x-pack/filebeat/input/azureeventhub/client_secret.go create mode 100644 x-pack/filebeat/input/azureeventhub/client_secret_test.go create mode 100644 x-pack/filebeat/input/azureeventhub/clients.go diff --git a/docs/reference/filebeat/filebeat-input-azure-eventhub.md b/docs/reference/filebeat/filebeat-input-azure-eventhub.md index 358298063965..b45981c30979 100644 --- a/docs/reference/filebeat/filebeat-input-azure-eventhub.md +++ b/docs/reference/filebeat/filebeat-input-azure-eventhub.md @@ -27,14 +27,14 @@ filebeat.inputs: resource_manager_endpoint: "" ``` -{applies_to}`stack: ga 9.3.0` Example configuration using OAuth2 authentication: +{applies_to}`stack: ga 9.3.0` Example configuration using client secret authentication: ```yaml filebeat.inputs: - type: azure-eventhub eventhub: "insights-operational-logs" consumer_group: "test" - # No connection_string provided - automatically uses OAuth2 for both eventhub and storage account + auth_type: "client_secret" eventhub_namespace: "your-eventhub-namespace.servicebus.windows.net" tenant_id: "your-tenant-id" client_id: "your-client-id" @@ -45,6 +45,7 @@ filebeat.inputs: processor_version: "v2" ``` + ## Configuration options [_configuration_options] The `azure-eventhub` input supports the following configuration: @@ -60,21 +61,70 @@ The name of the eventhub users would like to read from, field required. Optional, we recommend using a dedicated consumer group for the azure input. Reusing consumer groups among non-related consumers can cause unexpected behavior and possibly lost events. +## Authentication [_authentication] + +The azure-eventhub input supports multiple authentication methods. The `auth_type` configuration option controls the authentication method used for both Event Hub and Storage Account. + +### Authentication Types + +The following authentication types are supported: + +- **`connection_string`** (default if `auth_type` is not specified): Uses Azure Event Hubs and Storage Account connection strings +- **`client_secret`**: Uses Azure Active Directory service principal with client secret credentials + +### Required Permissions + +When using `client_secret` authentication, the service principal needs the following Azure RBAC permissions: + +**For Azure Event Hubs:** +- `Azure Event Hubs Data Receiver` role on the Event Hubs namespace or Event Hub +- Alternatively, a custom role with the following permissions: + - `Microsoft.EventHub/namespaces/eventhubs/read` + - `Microsoft.EventHub/namespaces/eventhubs/consumergroups/read` + +**For Azure Storage Account:** +- `Storage Blob Data Contributor` role on the Storage Account or container +- Alternatively, a custom role with the following permissions: + - `Microsoft.Storage/storageAccounts/blobServices/containers/read` + - `Microsoft.Storage/storageAccounts/blobServices/containers/write` + - `Microsoft.Storage/storageAccounts/blobServices/containers/delete` + - `Microsoft.Storage/storageAccounts/blobServices/generateUserDelegationKey/action` + +For detailed instructions on how to set up an Azure AD service principal and configure permissions, refer to the official Microsoft documentation: + +- [Create an Azure service principal with Azure CLI](https://learn.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) +- [Create an Azure AD app registration using the Azure portal](https://learn.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal) +- [Assign Azure roles using Azure CLI](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments-cli) +- [Azure Event Hubs authentication and authorization](https://learn.microsoft.com/en-us/azure/event-hubs/authorize-access-azure-active-directory) +- [Authorize access to blobs using Azure Active Directory](https://learn.microsoft.com/en-us/azure/storage/blobs/authorize-access-azure-active-directory) + ## `connection_string` [_connection_string] -The connection string required to communicate with Event Hubs when using Shared Access Key authentication, steps here [https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string). +The connection string required to communicate with Event Hubs when using `connection_string` authentication, steps here [https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string). -**Note**: If `connection_string` is not provided, the input will automatically use OAuth2 authentication and require the OAuth2 configuration parameters below. +Required when `auth_type` is set to `connection_string` or when `auth_type` is not specified (defaults to `connection_string` for backwards compatibility). A Blob Storage account is required in order to store/retrieve/update the offset or state of the eventhub messages. This means that after stopping filebeat it can start back up at the spot that it stopped processing messages. +## `auth_type` [_auth_type] + +```{applies_to} +stack: ga 9.3.0 +``` + +Specifies the authentication method to use for both Event Hub and Storage Account. If not specified, defaults to `connection_string` for backwards compatibility. + +Valid values: +- `connection_string`: Uses connection string authentication (default) +- `client_secret`: Uses Azure Active Directory service principal with client secret credentials + ## `eventhub_namespace` [_eventhub_namespace] ```{applies_to} stack: ga 9.3.0 ``` -The fully qualified namespace for the Event Hub. Required when `connection_string` is not provided (OAuth2 authentication). Format: `your-eventhub-namespace.servicebus.windows.net` +The fully qualified namespace for the Event Hub. Required when using `client_secret` authentication (`auth_type` is set to `client_secret`). Format: `your-eventhub-namespace.servicebus.windows.net` ## `tenant_id` [_tenant_id] @@ -82,7 +132,7 @@ The fully qualified namespace for the Event Hub. Required when `connection_strin stack: ga 9.3.0 ``` -The Azure Active Directory tenant ID. Required when `connection_string` is not provided (OAuth2 authentication). +The Azure Active Directory tenant ID. Required when using `client_secret` authentication for Event Hub or Storage Account. ## `client_id` [_client_id] @@ -90,7 +140,7 @@ The Azure Active Directory tenant ID. Required when `connection_string` is not p stack: ga 9.3.0 ``` -The Azure Active Directory application (client) ID. Required when `connection_string` is not provided (OAuth2 authentication). +The Azure Active Directory application (client) ID. Required when using `client_secret` authentication for Event Hub or Storage Account. ## `client_secret` [_client_secret] @@ -98,7 +148,7 @@ The Azure Active Directory application (client) ID. Required when `connection_st stack: ga 9.3.0 ``` -The Azure Active Directory application client secret. Required when `connection_string` is not provided (OAuth2 authentication). +The Azure Active Directory application client secret. Required when using `client_secret` authentication for Event Hub or Storage Account. ## `authority_host` [_authority_host] @@ -106,7 +156,7 @@ The Azure Active Directory application client secret. Required when `connection_ stack: ga 9.3.0 ``` -The Azure Active Directory authority host. Optional when using OAuth2 authentication. Defaults to Azure Public Cloud (`https://login.microsoftonline.com`). +The Azure Active Directory authority host. Optional when using `client_secret` authentication. Defaults to Azure Public Cloud (`https://login.microsoftonline.com`). Supported values: - `https://login.microsoftonline.com` (Azure Public Cloud - default) diff --git a/x-pack/filebeat/input/azureeventhub/auth.go b/x-pack/filebeat/input/azureeventhub/auth.go new file mode 100644 index 000000000000..b94961810dd2 --- /dev/null +++ b/x-pack/filebeat/input/azureeventhub/auth.go @@ -0,0 +1,55 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build !aix + +package azureeventhub + +import ( + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/elastic/elastic-agent-libs/logp" +) + +const ( + // AuthTypeConnectionString uses connection string authentication (default). + AuthTypeConnectionString string = "connection_string" + // AuthTypeClientSecret uses client secret credentials (OAuth2). + AuthTypeClientSecret string = "client_secret" +) + +// authConfig represents the authentication configuration. +type authConfig struct { + // AuthType specifies the authentication method to use. + // If not specified, will be inferred from other fields: + // - If connection_string is provided, defaults to connection_string + // - Otherwise, defaults to client_secret + AuthType string + + // Connection string authentication (legacy) + ConnectionString string + + // Client secret authentication + TenantID string + ClientID string + ClientSecret string + AuthorityHost string +} + +// newCredential creates a new TokenCredential based on the configured auth type. +// This function is not required right now for only supporting client_secret. +// But we will need it once we start supporting more auth types. +func newCredential(config authConfig, authType string, log *logp.Logger) (azcore.TokenCredential, error) { + switch authType { + case AuthTypeConnectionString: + // Connection string authentication doesn't use TokenCredential + // This is handled separately in the client creation + return nil, fmt.Errorf("connection_string authentication does not use TokenCredential") + case AuthTypeClientSecret: + return newClientSecretCredential(config, log) + default: + return nil, fmt.Errorf("unknown auth_type: %s (valid values: connection_string, client_secret)", authType) + } +} diff --git a/x-pack/filebeat/input/azureeventhub/client_secret.go b/x-pack/filebeat/input/azureeventhub/client_secret.go new file mode 100644 index 000000000000..131516420c8f --- /dev/null +++ b/x-pack/filebeat/input/azureeventhub/client_secret.go @@ -0,0 +1,68 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build !aix + +package azureeventhub + +import ( + "fmt" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/elastic/elastic-agent-libs/logp" +) + +// newClientSecretCredential creates a new client secret credential(Oauth2). +func newClientSecretCredential(config authConfig, log *logp.Logger) (azcore.TokenCredential, error) { + log = log.Named("client_secret") + + if config.TenantID == "" { + return nil, fmt.Errorf("tenant_id is required for client_secret authentication") + } + if config.ClientID == "" { + return nil, fmt.Errorf("client_id is required for client_secret authentication") + } + if config.ClientSecret == "" { + return nil, fmt.Errorf("client_secret is required for client_secret authentication") + } + + // Create credential options + credentialOptions := &azidentity.ClientSecretCredentialOptions{ + ClientOptions: azcore.ClientOptions{ + Cloud: getAzureCloud(config.AuthorityHost), + }, + } + + // Create the credential + credential, err := azidentity.NewClientSecretCredential( + config.TenantID, + config.ClientID, + config.ClientSecret, + credentialOptions, + ) + if err != nil { + return nil, fmt.Errorf("failed to create client secret credential: %w", err) + } + + log.Infow("successfully created client secret credential", + "tenant_id", config.TenantID, + "client_id", config.ClientID, + ) + + return credential, nil +} + +// getAzureCloud returns the appropriate Azure cloud configuration based on the authority host. +func getAzureCloud(authorityHost string) cloud.Configuration { + switch authorityHost { + case "https://login.microsoftonline.us": + return cloud.AzureGovernment + case "https://login.chinacloudapi.cn": + return cloud.AzureChina + default: + return cloud.AzurePublic + } +} diff --git a/x-pack/filebeat/input/azureeventhub/client_secret_test.go b/x-pack/filebeat/input/azureeventhub/client_secret_test.go new file mode 100644 index 000000000000..11b6548ff57f --- /dev/null +++ b/x-pack/filebeat/input/azureeventhub/client_secret_test.go @@ -0,0 +1,29 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build !aix + +package azureeventhub + +import ( + "testing" +) + +func TestGetAzureCloud(t *testing.T) { + // Test that the function doesn't panic for different authority hosts + testCases := []string{ + "https://login.microsoftonline.com", + "https://login.microsoftonline.us", + "https://login.chinacloudapi.cn", + "", + } + + for _, authorityHost := range testCases { + t.Run("authority_host_"+authorityHost, func(t *testing.T) { + cloud := getAzureCloud(authorityHost) + // Just verify we got a result - we can't easily compare cloud configurations + _ = cloud + }) + } +} diff --git a/x-pack/filebeat/input/azureeventhub/clients.go b/x-pack/filebeat/input/azureeventhub/clients.go new file mode 100644 index 000000000000..3bc867f9ab7a --- /dev/null +++ b/x-pack/filebeat/input/azureeventhub/clients.go @@ -0,0 +1,122 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build !aix + +package azureeventhub + +import ( + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/elastic/elastic-agent-libs/logp" +) + +// eventHubClientConfig holds configuration for creating an Event Hub consumer client. +type eventHubClientConfig struct { + Namespace string + EventHubName string + ConsumerGroup string + Credential azcore.TokenCredential + ConnectionString string +} + +// newEventHubConsumerClient creates a new Event Hub consumer client using the provided credential or connection string. +func newEventHubConsumerClient(config eventHubClientConfig, log *logp.Logger) (*azeventhubs.ConsumerClient, error) { + if config.ConnectionString != "" { + // Use connection string authentication + consumerClient, err := azeventhubs.NewConsumerClientFromConnectionString( + config.ConnectionString, + config.EventHubName, + config.ConsumerGroup, + nil, + ) + if err != nil { + return nil, fmt.Errorf("failed to create consumer client from connection string: %w", err) + } + return consumerClient, nil + } + + if config.Credential == nil { + return nil, fmt.Errorf("credential is required when connection_string is not provided") + } + + // Use credential authentication + consumerClient, err := azeventhubs.NewConsumerClient( + config.Namespace, + config.EventHubName, + config.ConsumerGroup, + config.Credential, + nil, + ) + if err != nil { + return nil, fmt.Errorf("failed to create consumer client with credential: %w", err) + } + + log.Infow("successfully created consumer client with credential authentication", + "namespace", config.Namespace, + "eventhub", config.EventHubName, + ) + + return consumerClient, nil +} + +// storageContainerClientConfig holds configuration for creating a Storage container client. +type storageContainerClientConfig struct { + StorageAccount string + Container string + Credential azcore.TokenCredential + ConnectionString string + Cloud cloud.Configuration +} + +// newStorageContainerClient creates a new Storage container client using the provided credential or connection string. +func newStorageContainerClient(config storageContainerClientConfig, authType string, log *logp.Logger) (*container.Client, error) { + if authType == AuthTypeConnectionString { + if config.ConnectionString != "" { + // Use connection string authentication (legacy) + if config.Cloud.ActiveDirectoryAuthorityHost == "" { + config.Cloud = cloud.AzurePublic + } + containerClient, err := container.NewClientFromConnectionString( + config.ConnectionString, + config.Container, + &container.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Cloud: config.Cloud, + }, + }, + ) + if err != nil { + return nil, fmt.Errorf("failed to create container client from connection string: %w", err) + } + return containerClient, nil + } else { + + } + } + + if config.Credential == nil { + return nil, fmt.Errorf("credential is required when connection_string is not provided") + } + + // Build the storage account URL + storageAccountURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s", config.StorageAccount, config.Container) + + // Use credential authentication + containerClient, err := container.NewClient(storageAccountURL, config.Credential, nil) + if err != nil { + return nil, fmt.Errorf("failed to create container client with credential: %w", err) + } + + log.Infow("successfully created container client with credential authentication", + "storage_account", config.StorageAccount, + "container", config.Container, + ) + + return containerClient, nil +} diff --git a/x-pack/filebeat/input/azureeventhub/config.go b/x-pack/filebeat/input/azureeventhub/config.go index ef2523a50b33..ea12159cbe22 100644 --- a/x-pack/filebeat/input/azureeventhub/config.go +++ b/x-pack/filebeat/input/azureeventhub/config.go @@ -41,23 +41,28 @@ type azureInputConfig struct { OverrideEnvironment string `config:"resource_manager_endpoint"` // --------------------------------------- - // OAuth2 authentication configuration + // Authentication configuration // --------------------------------------- + // AuthType specifies the authentication method to use for both Event Hub and Storage Account. + // If not specified, defaults to connection_string for backwards compatibility. + // Valid values: connection_string, client_secret + AuthType string `config:"auth_type"` + // EventHubNamespace is the fully qualified namespace for the Event Hub. - // Required when using OAuth2 authentication (when connection_string is not provided). + // Required when using client_secret authentication. EventHubNamespace string `config:"eventhub_namespace"` // TenantID is the Azure Active Directory tenant ID. - // Required when using OAuth2 authentication (when connection_string is not provided). + // Required when using client_secret authentication. TenantID string `config:"tenant_id"` // ClientID is the Azure Active Directory application (client) ID. - // Required when using OAuth2 authentication (when connection_string is not provided). + // Required when using client_secret authentication. ClientID string `config:"client_id"` // ClientSecret is the Azure Active Directory application client secret. - // Required when using OAuth2 authentication with client credentials flow. + // Required when using client_secret authentication. ClientSecret string `config:"client_secret"` // AuthorityHost is the Azure Active Directory authority host. - // Optional, defaults to Azure Public Cloud. + // Optional, defaults to Azure Public Cloud (https://login.microsoftonline.com). AuthorityHost string `config:"authority_host"` // LegacySanitizeOptions is a list of sanitization options to apply to messages. // @@ -145,33 +150,27 @@ func defaultConfig() azureInputConfig { // Validate validates the config. func (conf *azureInputConfig) Validate() error { logger := logp.NewLogger("azureeventhub.config") + logger.Infof("DEBUG: Validate() method called") - // Determine authentication method based on whether connection_string is provided - useOAuth2 := conf.ConnectionString == "" + // Determine authentication method + authType := conf.AuthType + if authType == "" { + // Default to connection_string for backwards compatibility + authType = AuthTypeConnectionString + } - if useOAuth2 { - // Validate OAuth2 configuration - if conf.EventHubNamespace == "" { - return errors.New("eventhub_namespace is required when connection_string is not provided (OAuth2 authentication)") - } - if conf.TenantID == "" { - return errors.New("tenant_id is required when connection_string is not provided (OAuth2 authentication)") - } - if conf.ClientID == "" { - return errors.New("client_id is required when connection_string is not provided (OAuth2 authentication)") - } - if conf.ClientSecret == "" { - return errors.New("client_secret is required when connection_string is not provided (OAuth2 authentication)") - } - } else { + switch authType { + case AuthTypeConnectionString: // Validate connection string configuration + if conf.ConnectionString == "" { + return errors.New("connection_string is required when auth_type is empty or set to connection_string") + } connectionStringProperties, err := parseConnectionString(conf.ConnectionString) if err != nil { return fmt.Errorf("invalid connection string: %w", err) } - // If the connection string contains an entity path, we need to double - // check that it matches the event hub name. + // If the connection string contains an entity path, we need to double-check that it matches the event hub name. if connectionStringProperties.EntityPath != nil && *connectionStringProperties.EntityPath != conf.EventHubName { return fmt.Errorf( "invalid config: the entity path (%s) in the connection string does not match event hub name (%s)", @@ -179,6 +178,22 @@ func (conf *azureInputConfig) Validate() error { conf.EventHubName, ) } + case AuthTypeClientSecret: + // Validate client secret configuration + if conf.EventHubNamespace == "" { + return errors.New("eventhub_namespace is required when using client_secret authentication") + } + if conf.TenantID == "" { + return errors.New("tenant_id is required when using client_secret authentication") + } + if conf.ClientID == "" { + return errors.New("client_id is required when using client_secret authentication") + } + if conf.ClientSecret == "" { + return errors.New("client_secret is required when using client_secret authentication") + } + default: + return fmt.Errorf("unknown auth_type: %s (valid values: connection_string, client_secret)", authType) } if conf.EventHubName == "" { @@ -236,22 +251,23 @@ func (conf *azureInputConfig) Validate() error { return errors.New("no storage account key configured (config: storage_account_key)") } case processorV2: - // For processor v2, either connection string or OAuth2 must be configured - useOAuth2ForStorage := conf.SAConnectionString == "" - if useOAuth2ForStorage { - // Validate OAuth2 configuration for storage + // For processor v2, either connection string or credential-based authentication must be configured + useCredentialForStorage := conf.SAConnectionString == "" + if useCredentialForStorage { + // Storage account uses the same auth_type as Event Hub + if authType != AuthTypeClientSecret { + return errors.New("storage account requires client_secret authentication when storage_account_connection_string is not provided") + } + + // Validate client secret configuration for storage if conf.TenantID == "" { - return errors.New("tenant_id is required when storage_account_connection_string is not provided (OAuth2 authentication)") + return errors.New("tenant_id is required when using client_secret authentication") } if conf.ClientID == "" { - return errors.New("client_id is required when storage_account_connection_string is not provided (OAuth2 authentication)") + return errors.New("client_id is required when using client_secret authentication") } if conf.ClientSecret == "" { - return errors.New("client_secret is required when storage_account_connection_string is not provided (OAuth2 authentication)") - } - } else { - if conf.SAConnectionString == "" { - return errors.New("no storage account connection string configured (config: storage_account_connection_string)") + return errors.New("client_secret is required when using client_secret authentication") } } default: diff --git a/x-pack/filebeat/input/azureeventhub/config_test.go b/x-pack/filebeat/input/azureeventhub/config_test.go index caa21d57fdd5..d85b201d1375 100644 --- a/x-pack/filebeat/input/azureeventhub/config_test.go +++ b/x-pack/filebeat/input/azureeventhub/config_test.go @@ -306,21 +306,3 @@ func TestOAuth2ConfigValidation(t *testing.T) { }) } } - -func TestGetAzureCloud(t *testing.T) { - // Test that the function doesn't panic for different authority hosts - testCases := []string{ - "https://login.microsoftonline.com", - "https://login.microsoftonline.us", - "https://login.chinacloudapi.cn", - "", - } - - for _, authorityHost := range testCases { - t.Run("authority_host_"+authorityHost, func(t *testing.T) { - cloud := getAzureCloud(authorityHost) - // Just verify we got a result - we can't easily compare cloud configurations - _ = cloud - }) - } -} diff --git a/x-pack/filebeat/input/azureeventhub/input.go b/x-pack/filebeat/input/azureeventhub/input.go index 664123c564ea..88486c0155c7 100644 --- a/x-pack/filebeat/input/azureeventhub/input.go +++ b/x-pack/filebeat/input/azureeventhub/input.go @@ -8,16 +8,14 @@ package azureeventhub import ( "fmt" - "os" - "github.com/Azure/go-autorest/autorest/azure" "github.com/devigned/tab" - v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/feature" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/go-concert/unison" + "os" ) const ( @@ -71,7 +69,6 @@ func (m *eventHubInputManager) Init(unison.Group) error { // Create creates a new azure-eventhub input based on the configuration. func (m *eventHubInputManager) Create(cfg *conf.C) (v2.Input, error) { - // Register the logs tracer only if the environment variable is // set to avoid the overhead of the tracer in environments where // it's not needed. @@ -86,6 +83,11 @@ func (m *eventHubInputManager) Create(cfg *conf.C) (v2.Input, error) { config.checkUnsupportedParams(m.log) + // Validate config + if err := config.Validate(); err != nil { + return nil, fmt.Errorf("config validation failed: %w", err) + } + switch config.ProcessorVersion { case processorV1: return newEventHubInputV1(config, m.log) diff --git a/x-pack/filebeat/input/azureeventhub/v2_input.go b/x-pack/filebeat/input/azureeventhub/v2_input.go index 2a4ff8195049..6d8d5a784f58 100644 --- a/x-pack/filebeat/input/azureeventhub/v2_input.go +++ b/x-pack/filebeat/input/azureeventhub/v2_input.go @@ -17,7 +17,6 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs/checkpoints" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" @@ -89,6 +88,9 @@ func (in *eventHubInputV2) Run( ) error { var err error + // DEBUG: Run method called + in.log.Infof("DEBUG: Run method called for azure-eventhub input") + // Setting up the status reporter helper in.status = statusreporterhelper.New(inputContext.StatusReporter, in.log, "Azure Event Hub") @@ -133,6 +135,9 @@ func (in *eventHubInputV2) Run( // setup initializes the components needed to process events. func (in *eventHubInputV2) setup(ctx context.Context) error { + // DEBUG: setup method called + in.log.Infof("------- DEBUG: setup method called") + sanitizers, err := newSanitizers(in.config.Sanitizers, in.config.LegacySanitizeOptions) if err != nil { return fmt.Errorf("failed to create sanitizers: %w", err) @@ -147,74 +152,68 @@ func (in *eventHubInputV2) setup(ctx context.Context) error { sanitizers: sanitizers, } + // Determine authentication method (shared by both Event Hub and Storage Account) + authType := in.config.AuthType + if authType == "" { + authType = AuthTypeConnectionString + } + + // DEBUG: authType determined + in.log.Infof("-----DEBUG: authType = %s", authType) + // Create the credential if needed (shared by both Event Hub and Storage Account) + // Both services use the same credential since they share the same auth_type + var credential azcore.TokenCredential // Create the container client var containerClient *container.Client + // Create the event hub consumerClient to receive events. + var consumerClient *azeventhubs.ConsumerClient + + useCredentials := authType == AuthTypeClientSecret + if useCredentials { + credConfig := authConfig{ + AuthType: authType, + TenantID: in.config.TenantID, + ClientID: in.config.ClientID, + ClientSecret: in.config.ClientSecret, + AuthorityHost: in.config.AuthorityHost, + } - // Determine the authentication method for the storage account based on whether SAConnectionString is provided - if in.config.SAConnectionString == "" { - // Use OAuth2 authentication for storage - containerClient, err = createContainerClientWithOAuth2(in.config, in.log) + credential, err = newCredential(credConfig, authType, in.log) if err != nil { - in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating blob container client with OAuth2: %s", err.Error())) - return fmt.Errorf("failed to create blob container client with OAuth2: %w", err) + in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating credential: %s", err.Error())) + return fmt.Errorf("failed to create credential: %w", err) } - } else { - // Use connection string authentication (default) - containerClient, err = container.NewClientFromConnectionString( - in.config.SAConnectionString, - in.config.SAContainer, - &container.ClientOptions{ - ClientOptions: azcore.ClientOptions{ - Cloud: cloud.AzurePublic, - }, + + consumerClient, err = newEventHubConsumerClient( + eventHubClientConfig{ + Namespace: in.config.EventHubNamespace, + EventHubName: in.config.EventHubName, + ConsumerGroup: in.config.ConsumerGroup, + Credential: credential, }, + in.log, ) if err != nil { - in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating blob container client: %s", err.Error())) - return fmt.Errorf("failed to create blob container client: %w", err) - } - } - - // The modern event hub SDK does not create the container - // automatically like the old SDK. - // - // The new `BlobStore` explicitly says: - // "the container must exist before the checkpoint store can be used." - // - // We need to ensure it exists before we can use it. - err = in.ensureContainerExists(ctx, containerClient) - if err != nil { - var respError *azcore.ResponseError - if errors.As(err, &respError) && respError != nil && respError.StatusCode == 403 { - in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed due to authentication error: %s", err.Error())) - return fmt.Errorf("authentication error: %w", err) + return fmt.Errorf("failed to create consumer client with credential: %w", err) } - return fmt.Errorf("failed to ensure blob container exists: %w", err) - } - - // Create the checkpoint store. - // - // The processor uses the checkpoint store to persist - // checkpoint information in the container using blobs (one blob - // for each event hub partition). - checkpointStore, err := checkpoints.NewBlobStore(containerClient, nil) - if err != nil { - return fmt.Errorf("failed to create checkpoint store: %w", err) - } - in.checkpointStore = checkpointStore - - // Create the event hub consumerClient to receive events. - var consumerClient *azeventhubs.ConsumerClient - // Determine authentication method based on whether connection_string is provided - if in.config.ConnectionString == "" { - // Use OAuth2 authentication - consumerClient, err = createConsumerClientWithOAuth2(in.config, in.log) + // Use credential-based authentication for storage account + containerClient, err = newStorageContainerClient( + storageContainerClientConfig{ + StorageAccount: in.config.SAName, + Container: in.config.SAContainer, + Credential: credential, + Cloud: getAzureCloud(in.config.AuthorityHost), + }, + authType, + in.log, + ) if err != nil { - return fmt.Errorf("failed to create consumer client with OAuth2: %w", err) + in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating blob container client with credential: %s", err.Error())) + return fmt.Errorf("failed to create blob container client with credential: %w", err) } } else { - // Use connection string authentication (default) + // Use connection string authentication for Event Hub // There is a mismatch between how the azure-eventhub input and the new // Event Hub SDK expect the event hub name in the connection string. // @@ -241,16 +240,60 @@ func (in *eventHubInputV2) setup(ctx context.Context) error { eventHubName = "" } - consumerClient, err = azeventhubs.NewConsumerClientFromConnectionString( - in.config.ConnectionString, - eventHubName, - in.config.ConsumerGroup, - nil, + consumerClient, err = newEventHubConsumerClient( + eventHubClientConfig{ + ConnectionString: in.config.ConnectionString, + EventHubName: eventHubName, + ConsumerGroup: in.config.ConsumerGroup, + }, + in.log, ) if err != nil { return fmt.Errorf("failed to create consumer client: %w", err) } + + containerClient, err = newStorageContainerClient( + storageContainerClientConfig{ + ConnectionString: in.config.SAConnectionString, + Container: in.config.SAContainer, + Cloud: cloud.AzurePublic, + }, + authType, + in.log, + ) + if err != nil { + in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating blob container client: %s", err.Error())) + return fmt.Errorf("failed to create blob container client: %w", err) + } + } + + // The modern event hub SDK does not create the container + // automatically like the old SDK. + // + // The new `BlobStore` explicitly says: + // "the container must exist before the checkpoint store can be used." + // + // We need to ensure it exists before we can use it. + err = in.ensureContainerExists(ctx, containerClient) + if err != nil { + var respError *azcore.ResponseError + if errors.As(err, &respError) && respError != nil && respError.StatusCode == 403 { + in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed due to authentication error: %s", err.Error())) + return fmt.Errorf("authentication error: %w", err) + } + return fmt.Errorf("failed to ensure blob container exists: %w", err) + } + + // Create the checkpoint store. + // + // The processor uses the checkpoint store to persist + // checkpoint information in the container using blobs (one blob + // for each event hub partition). + checkpointStore, err := checkpoints.NewBlobStore(containerClient, nil) + if err != nil { + return fmt.Errorf("failed to create checkpoint store: %w", err) } + in.checkpointStore = checkpointStore in.consumerClient = consumerClient // Manage the migration of the checkpoint information @@ -616,7 +659,7 @@ func (in *eventHubInputV2) processReceivedEvents(receivedEvents []*azeventhubs.R // Update input metrics. in.metrics.sentEvents.Inc() } - + records = nil // Update input metrics. in.metrics.processedMessages.Inc() in.metrics.processingTime.Update(time.Since(processingStartTime).Nanoseconds()) @@ -686,106 +729,3 @@ func shutdownPartitionResources(ctx context.Context, partitionClient *azeventhub // processing events for this partition. defer pipelineClient.Close() } - -// createConsumerClientWithOAuth2 creates a new Event Hub consumer client using OAuth2 authentication. -func createConsumerClientWithOAuth2( - config azureInputConfig, - log *logp.Logger, -) (*azeventhubs.ConsumerClient, error) { - log = log.Named("oauth2") - - // Create credential options - credentialOptions := &azidentity.ClientSecretCredentialOptions{ - ClientOptions: azcore.ClientOptions{ - Cloud: getAzureCloud(config.AuthorityHost), - }, - } - - // Create the credential - credential, err := azidentity.NewClientSecretCredential( - config.TenantID, - config.ClientID, - config.ClientSecret, - credentialOptions, - ) - if err != nil { - return nil, fmt.Errorf("failed to create client secret credential: %w", err) - } - - // Create the consumer client with OAuth2 authentication - consumerClient, err := azeventhubs.NewConsumerClient( - config.EventHubNamespace, - config.EventHubName, - config.ConsumerGroup, - credential, - nil, - ) - if err != nil { - return nil, fmt.Errorf("failed to create consumer client with OAuth2: %w", err) - } - - log.Infow("successfully created consumer client with OAuth2 authentication", - "namespace", config.EventHubNamespace, - "eventhub", config.EventHubName, - "tenant_id", config.TenantID, - "client_id", config.ClientID, - ) - - return consumerClient, nil -} - -// createContainerClientWithOAuth2 creates a new Blob Storage container client using OAuth2 authentication. -func createContainerClientWithOAuth2( - config azureInputConfig, - log *logp.Logger, -) (*container.Client, error) { - log = log.Named("oauth2") - - // Build the storage account URL - storageAccountURL := fmt.Sprintf("https://%s.blob.core.windows.net", config.SAName) - - // Create credential options - credentialOptions := &azidentity.ClientSecretCredentialOptions{ - ClientOptions: azcore.ClientOptions{ - Cloud: getAzureCloud(config.AuthorityHost), - }, - } - - // Create the credential - credential, err := azidentity.NewClientSecretCredential( - config.TenantID, - config.ClientID, - config.ClientSecret, - credentialOptions, - ) - if err != nil { - return nil, fmt.Errorf("failed to create client secret credential: %w", err) - } - - // Create the container client with OAuth2 authentication - containerClient, err := container.NewClient(storageAccountURL+"/"+config.SAContainer, credential, nil) - if err != nil { - return nil, fmt.Errorf("failed to create container client with OAuth2: %w", err) - } - - log.Infow("successfully created container client with OAuth2 authentication", - "storage_account", config.SAName, - "container", config.SAContainer, - "tenant_id", config.TenantID, - "client_id", config.ClientID, - ) - - return containerClient, nil -} - -// getAzureCloud returns the appropriate Azure cloud configuration based on the authority host. -func getAzureCloud(authorityHost string) cloud.Configuration { - switch authorityHost { - case "https://login.microsoftonline.us": - return cloud.AzureGovernment - case "https://login.chinacloudapi.cn": - return cloud.AzureChina - default: - return cloud.AzurePublic - } -} From 17d85a366bb46b7bb8afa4adb8794bc972500189 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 4 Nov 2025 16:48:26 -0700 Subject: [PATCH 08/21] fix unit tests --- x-pack/filebeat/input/azureeventhub/auth.go | 7 ++- .../input/azureeventhub/client_secret.go | 3 +- .../filebeat/input/azureeventhub/clients.go | 1 + .../input/azureeventhub/config_test.go | 58 ++++++++----------- x-pack/filebeat/input/azureeventhub/input.go | 4 +- .../filebeat/input/azureeventhub/v2_input.go | 11 +--- 6 files changed, 36 insertions(+), 48 deletions(-) diff --git a/x-pack/filebeat/input/azureeventhub/auth.go b/x-pack/filebeat/input/azureeventhub/auth.go index b94961810dd2..fe6119c92c37 100644 --- a/x-pack/filebeat/input/azureeventhub/auth.go +++ b/x-pack/filebeat/input/azureeventhub/auth.go @@ -10,6 +10,7 @@ import ( "fmt" "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/elastic/elastic-agent-libs/logp" ) @@ -28,7 +29,7 @@ type authConfig struct { // - Otherwise, defaults to client_secret AuthType string - // Connection string authentication (legacy) + // Connection string authentication ConnectionString string // Client secret authentication @@ -39,8 +40,6 @@ type authConfig struct { } // newCredential creates a new TokenCredential based on the configured auth type. -// This function is not required right now for only supporting client_secret. -// But we will need it once we start supporting more auth types. func newCredential(config authConfig, authType string, log *logp.Logger) (azcore.TokenCredential, error) { switch authType { case AuthTypeConnectionString: @@ -48,6 +47,8 @@ func newCredential(config authConfig, authType string, log *logp.Logger) (azcore // This is handled separately in the client creation return nil, fmt.Errorf("connection_string authentication does not use TokenCredential") case AuthTypeClientSecret: + // This function is not required right now for only supporting client_secret. + // But we will need it once we start supporting more auth types. return newClientSecretCredential(config, log) default: return nil, fmt.Errorf("unknown auth_type: %s (valid values: connection_string, client_secret)", authType) diff --git a/x-pack/filebeat/input/azureeventhub/client_secret.go b/x-pack/filebeat/input/azureeventhub/client_secret.go index 131516420c8f..86827e797860 100644 --- a/x-pack/filebeat/input/azureeventhub/client_secret.go +++ b/x-pack/filebeat/input/azureeventhub/client_secret.go @@ -8,10 +8,11 @@ package azureeventhub import ( "fmt" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/x-pack/filebeat/input/azureeventhub/clients.go b/x-pack/filebeat/input/azureeventhub/clients.go index 3bc867f9ab7a..55c7e9b142d2 100644 --- a/x-pack/filebeat/input/azureeventhub/clients.go +++ b/x-pack/filebeat/input/azureeventhub/clients.go @@ -13,6 +13,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/x-pack/filebeat/input/azureeventhub/config_test.go b/x-pack/filebeat/input/azureeventhub/config_test.go index d85b201d1375..a30b717135a6 100644 --- a/x-pack/filebeat/input/azureeventhub/config_test.go +++ b/x-pack/filebeat/input/azureeventhub/config_test.go @@ -163,7 +163,7 @@ func TestOAuth2ConfigValidation(t *testing.T) { errorMsg string }{ { - name: "valid oauth2 config for eventhub (no connection_string)", + name: "valid client_secret config for both eventhub and storage account", config: func() azureInputConfig { c := defaultConfig() c.EventHubName = "test-hub" @@ -172,108 +172,96 @@ func TestOAuth2ConfigValidation(t *testing.T) { c.ClientID = "test-client-id" c.ClientSecret = "test-client-secret" c.SAName = "test-storage" - c.SAConnectionString = "test-connection-string" c.ProcessorVersion = "v2" + c.AuthType = "client_secret" return c }(), expectError: false, }, { - name: "valid oauth2 config for only storage account (no storage_account_connection_string)", + name: "client_secret config missing namespace", config: func() azureInputConfig { c := defaultConfig() c.EventHubName = "test-hub" - c.ConnectionString = "Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=test;SharedAccessKey=test" c.TenantID = "test-tenant-id" c.ClientID = "test-client-id" c.ClientSecret = "test-client-secret" c.SAName = "test-storage" + c.SAConnectionString = "test-connection-string" c.ProcessorVersion = "v2" + c.AuthType = "client_secret" return c }(), - expectError: false, + expectError: true, + errorMsg: "eventhub_namespace is required when using client_secret authentication", }, { - name: "valid oauth2 config for both eventhub and storage account (no connection strings)", + name: "client_secret config missing tenant_id", config: func() azureInputConfig { c := defaultConfig() c.EventHubName = "test-hub" c.EventHubNamespace = "test-namespace.servicebus.windows.net" - c.TenantID = "test-tenant-id" - c.ClientID = "test-client-id" - c.ClientSecret = "test-client-secret" - c.SAName = "test-storage" - c.ProcessorVersion = "v2" - return c - }(), - expectError: false, - }, - { - name: "oauth2 config missing namespace", - config: func() azureInputConfig { - c := defaultConfig() - c.EventHubName = "test-hub" - c.TenantID = "test-tenant-id" c.ClientID = "test-client-id" c.ClientSecret = "test-client-secret" c.SAName = "test-storage" c.SAConnectionString = "test-connection-string" c.ProcessorVersion = "v2" + c.AuthType = "client_secret" return c }(), expectError: true, - errorMsg: "eventhub_namespace is required when connection_string is not provided (OAuth2 authentication)", + errorMsg: "tenant_id is required when using client_secret authentication", }, { - name: "oauth2 config missing tenant_id", + name: "client_secret config missing client_id", config: func() azureInputConfig { c := defaultConfig() c.EventHubName = "test-hub" c.EventHubNamespace = "test-namespace.servicebus.windows.net" - c.ClientID = "test-client-id" + c.TenantID = "test-tenant-id" c.ClientSecret = "test-client-secret" c.SAName = "test-storage" c.SAConnectionString = "test-connection-string" c.ProcessorVersion = "v2" + c.AuthType = "client_secret" return c }(), expectError: true, - errorMsg: "tenant_id is required when connection_string is not provided (OAuth2 authentication)", + errorMsg: "client_id is required when using client_secret authentication", }, { - name: "oauth2 config missing client_id", + name: "auth type client_secret config missing client_secret", config: func() azureInputConfig { c := defaultConfig() c.EventHubName = "test-hub" c.EventHubNamespace = "test-namespace.servicebus.windows.net" c.TenantID = "test-tenant-id" - c.ClientSecret = "test-client-secret" + c.ClientID = "test-client-id" c.SAName = "test-storage" c.SAConnectionString = "test-connection-string" c.ProcessorVersion = "v2" + c.AuthType = "client_secret" return c }(), expectError: true, - errorMsg: "client_id is required when connection_string is not provided (OAuth2 authentication)", + errorMsg: "client_secret is required when using client_secret authentication", }, { - name: "oauth2 config missing client_secret", + name: "valid connection_string config", config: func() azureInputConfig { c := defaultConfig() c.EventHubName = "test-hub" - c.EventHubNamespace = "test-namespace.servicebus.windows.net" - c.TenantID = "test-tenant-id" - c.ClientID = "test-client-id" + c.ConnectionString = "Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=test;SharedAccessKey=test" c.SAName = "test-storage" c.SAConnectionString = "test-connection-string" c.ProcessorVersion = "v2" + c.AuthType = "connection_string" return c }(), - expectError: true, - errorMsg: "client_secret is required when connection_string is not provided (OAuth2 authentication)", + expectError: false, }, { - name: "valid connection_string config", + name: "valid connection_string config without auth_type", config: func() azureInputConfig { c := defaultConfig() c.EventHubName = "test-hub" diff --git a/x-pack/filebeat/input/azureeventhub/input.go b/x-pack/filebeat/input/azureeventhub/input.go index 88486c0155c7..01b010175c47 100644 --- a/x-pack/filebeat/input/azureeventhub/input.go +++ b/x-pack/filebeat/input/azureeventhub/input.go @@ -8,14 +8,16 @@ package azureeventhub import ( "fmt" + "os" + "github.com/Azure/go-autorest/autorest/azure" "github.com/devigned/tab" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/feature" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/go-concert/unison" - "os" ) const ( diff --git a/x-pack/filebeat/input/azureeventhub/v2_input.go b/x-pack/filebeat/input/azureeventhub/v2_input.go index 6d8d5a784f58..3f34c34121a5 100644 --- a/x-pack/filebeat/input/azureeventhub/v2_input.go +++ b/x-pack/filebeat/input/azureeventhub/v2_input.go @@ -135,9 +135,6 @@ func (in *eventHubInputV2) Run( // setup initializes the components needed to process events. func (in *eventHubInputV2) setup(ctx context.Context) error { - // DEBUG: setup method called - in.log.Infof("------- DEBUG: setup method called") - sanitizers, err := newSanitizers(in.config.Sanitizers, in.config.LegacySanitizeOptions) if err != nil { return fmt.Errorf("failed to create sanitizers: %w", err) @@ -158,15 +155,13 @@ func (in *eventHubInputV2) setup(ctx context.Context) error { authType = AuthTypeConnectionString } - // DEBUG: authType determined - in.log.Infof("-----DEBUG: authType = %s", authType) // Create the credential if needed (shared by both Event Hub and Storage Account) // Both services use the same credential since they share the same auth_type var credential azcore.TokenCredential - // Create the container client - var containerClient *container.Client // Create the event hub consumerClient to receive events. var consumerClient *azeventhubs.ConsumerClient + // Create the container client + var containerClient *container.Client useCredentials := authType == AuthTypeClientSecret if useCredentials { @@ -197,7 +192,7 @@ func (in *eventHubInputV2) setup(ctx context.Context) error { return fmt.Errorf("failed to create consumer client with credential: %w", err) } - // Use credential-based authentication for storage account + // Use credential-based authentication for the storage account containerClient, err = newStorageContainerClient( storageContainerClientConfig{ StorageAccount: in.config.SAName, From 2d1f3d967eee6d9f73b40169988b754d6d46582a Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 4 Nov 2025 23:08:25 -0700 Subject: [PATCH 09/21] add more tests for the config options --- x-pack/filebeat/input/azureeventhub/config.go | 80 ++++---- .../input/azureeventhub/config_test.go | 178 +++++++++++++++++- 2 files changed, 213 insertions(+), 45 deletions(-) diff --git a/x-pack/filebeat/input/azureeventhub/config.go b/x-pack/filebeat/input/azureeventhub/config.go index ea12159cbe22..b057c92b9017 100644 --- a/x-pack/filebeat/input/azureeventhub/config.go +++ b/x-pack/filebeat/input/azureeventhub/config.go @@ -159,9 +159,20 @@ func (conf *azureInputConfig) Validate() error { authType = AuthTypeConnectionString } + // Validate the processor version first to ensure it's valid + if conf.ProcessorVersion != processorV1 && conf.ProcessorVersion != processorV2 { + return fmt.Errorf( + "invalid processor_version: %s (available versions: %s, %s)", + conf.ProcessorVersion, + processorV1, + processorV2, + ) + } + + // Validate authentication for both Event Hub and Storage Account together switch authType { case AuthTypeConnectionString: - // Validate connection string configuration + // Validate Event Hub connection string configuration if conf.ConnectionString == "" { return errors.New("connection_string is required when auth_type is empty or set to connection_string") } @@ -178,8 +189,23 @@ func (conf *azureInputConfig) Validate() error { conf.EventHubName, ) } + + // Validate Storage Account authentication for connection_string auth type + switch conf.ProcessorVersion { + case processorV1: + // Processor v1 requires storage account key + if conf.SAKey == "" { + return errors.New("storage_account_key is required when using connection_string authentication with processor v1") + } + case processorV2: + // Processor v2 requires storage account connection string + if conf.SAConnectionString == "" { + return errors.New("storage_account_connection_string is required when using connection_string authentication with processor v2") + } + } + case AuthTypeClientSecret: - // Validate client secret configuration + // Validate Event Hub client secret configuration if conf.EventHubNamespace == "" { return errors.New("eventhub_namespace is required when using client_secret authentication") } @@ -192,10 +218,25 @@ func (conf *azureInputConfig) Validate() error { if conf.ClientSecret == "" { return errors.New("client_secret is required when using client_secret authentication") } + + // Validate Storage Account authentication for client_secret auth type + switch conf.ProcessorVersion { + case processorV1: + // Processor v1 requires storage account key + if conf.SAKey == "" { + return errors.New("storage_account_key is required when using client_secret authentication with processor v1") + } + case processorV2: + // Processor v2 with client_secret auth type: Storage Account uses the same client_secret credentials as Event Hub + // The client_secret credentials are already validated above for Event Hub + // The storage account will use the same TenantID, ClientID, and ClientSecret as Event Hub + } + default: return fmt.Errorf("unknown auth_type: %s (valid values: connection_string, client_secret)", authType) } + // Validate required fields if conf.EventHubName == "" { return errors.New("no event hub name configured") } @@ -227,6 +268,7 @@ func (conf *azureInputConfig) Validate() error { return err } + // Validate processor-specific settings if conf.ProcessorUpdateInterval < 1*time.Second { return errors.New("processor_update_interval must be at least 1 second") } @@ -245,40 +287,6 @@ func (conf *azureInputConfig) Validate() error { ) } - switch conf.ProcessorVersion { - case processorV1: - if conf.SAKey == "" { - return errors.New("no storage account key configured (config: storage_account_key)") - } - case processorV2: - // For processor v2, either connection string or credential-based authentication must be configured - useCredentialForStorage := conf.SAConnectionString == "" - if useCredentialForStorage { - // Storage account uses the same auth_type as Event Hub - if authType != AuthTypeClientSecret { - return errors.New("storage account requires client_secret authentication when storage_account_connection_string is not provided") - } - - // Validate client secret configuration for storage - if conf.TenantID == "" { - return errors.New("tenant_id is required when using client_secret authentication") - } - if conf.ClientID == "" { - return errors.New("client_id is required when using client_secret authentication") - } - if conf.ClientSecret == "" { - return errors.New("client_secret is required when using client_secret authentication") - } - } - default: - return fmt.Errorf( - "invalid processor_version: %s (available versions: %s, %s)", - conf.ProcessorVersion, - processorV1, - processorV2, - ) - } - return nil } diff --git a/x-pack/filebeat/input/azureeventhub/config_test.go b/x-pack/filebeat/input/azureeventhub/config_test.go index a30b717135a6..335d7c308cc4 100644 --- a/x-pack/filebeat/input/azureeventhub/config_test.go +++ b/x-pack/filebeat/input/azureeventhub/config_test.go @@ -155,7 +155,7 @@ func TestValidateConnectionStringV2(t *testing.T) { }) } -func TestOAuth2ConfigValidation(t *testing.T) { +func TestClientSecretConfigValidation(t *testing.T) { tests := []struct { name string config azureInputConfig @@ -163,7 +163,7 @@ func TestOAuth2ConfigValidation(t *testing.T) { errorMsg string }{ { - name: "valid client_secret config for both eventhub and storage account", + name: "valid client_secret config for both eventhub and storage account with processor v2", config: func() azureInputConfig { c := defaultConfig() c.EventHubName = "test-hub" @@ -187,7 +187,6 @@ func TestOAuth2ConfigValidation(t *testing.T) { c.ClientID = "test-client-id" c.ClientSecret = "test-client-secret" c.SAName = "test-storage" - c.SAConnectionString = "test-connection-string" c.ProcessorVersion = "v2" c.AuthType = "client_secret" return c @@ -204,7 +203,6 @@ func TestOAuth2ConfigValidation(t *testing.T) { c.ClientID = "test-client-id" c.ClientSecret = "test-client-secret" c.SAName = "test-storage" - c.SAConnectionString = "test-connection-string" c.ProcessorVersion = "v2" c.AuthType = "client_secret" return c @@ -221,7 +219,6 @@ func TestOAuth2ConfigValidation(t *testing.T) { c.TenantID = "test-tenant-id" c.ClientSecret = "test-client-secret" c.SAName = "test-storage" - c.SAConnectionString = "test-connection-string" c.ProcessorVersion = "v2" c.AuthType = "client_secret" return c @@ -230,7 +227,7 @@ func TestOAuth2ConfigValidation(t *testing.T) { errorMsg: "client_id is required when using client_secret authentication", }, { - name: "auth type client_secret config missing client_secret", + name: "client_secret config missing client_secret", config: func() azureInputConfig { c := defaultConfig() c.EventHubName = "test-hub" @@ -238,7 +235,6 @@ func TestOAuth2ConfigValidation(t *testing.T) { c.TenantID = "test-tenant-id" c.ClientID = "test-client-id" c.SAName = "test-storage" - c.SAConnectionString = "test-connection-string" c.ProcessorVersion = "v2" c.AuthType = "client_secret" return c @@ -247,7 +243,87 @@ func TestOAuth2ConfigValidation(t *testing.T) { errorMsg: "client_secret is required when using client_secret authentication", }, { - name: "valid connection_string config", + name: "valid client_secret config with processor v1", + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.EventHubNamespace = "test-namespace.servicebus.windows.net" + c.TenantID = "test-tenant-id" + c.ClientID = "test-client-id" + c.ClientSecret = "test-client-secret" + c.SAName = "test-storage" + c.SAKey = "test-storage-key" + c.ProcessorVersion = "v1" + c.AuthType = "client_secret" + return c + }(), + expectError: false, + }, + { + name: "client_secret config with processor v1 missing storage account key", + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.EventHubNamespace = "test-namespace.servicebus.windows.net" + c.TenantID = "test-tenant-id" + c.ClientID = "test-client-id" + c.ClientSecret = "test-client-secret" + c.SAName = "test-storage" + c.ProcessorVersion = "v1" + c.AuthType = "client_secret" + return c + }(), + expectError: true, + errorMsg: "storage_account_key is required when using client_secret authentication with processor v1", + }, + { + name: "client_secret config with processor v2 uses same credentials for storage account", + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.EventHubNamespace = "test-namespace.servicebus.windows.net" + c.TenantID = "test-tenant-id" + c.ClientID = "test-client-id" + c.ClientSecret = "test-client-secret" + c.SAName = "test-storage" + // No SAConnectionString - should use client_secret credentials + c.ProcessorVersion = "v2" + c.AuthType = "client_secret" + return c + }(), + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.config.Validate() + if tt.expectError { + if err == nil { + t.Errorf("expected error but got none") + return + } + if tt.errorMsg != "" && err.Error() != tt.errorMsg { + t.Errorf("expected error message %q, got %q", tt.errorMsg, err.Error()) + } + } else { + if err != nil { + t.Errorf("unexpected error: %v", err) + } + } + }) + } +} + +func TestConnectionStringConfigValidation(t *testing.T) { + tests := []struct { + name string + config azureInputConfig + expectError bool + errorMsg string + }{ + { + name: "valid connection_string config with processor v2", config: func() azureInputConfig { c := defaultConfig() c.EventHubName = "test-hub" @@ -261,7 +337,7 @@ func TestOAuth2ConfigValidation(t *testing.T) { expectError: false, }, { - name: "valid connection_string config without auth_type", + name: "valid connection_string config without auth_type (defaults to connection_string)", config: func() azureInputConfig { c := defaultConfig() c.EventHubName = "test-hub" @@ -273,6 +349,90 @@ func TestOAuth2ConfigValidation(t *testing.T) { }(), expectError: false, }, + { + name: "valid connection_string config with processor v1", + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.ConnectionString = "Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=test;SharedAccessKey=test" + c.SAName = "test-storage" + c.SAKey = "test-storage-key" + c.ProcessorVersion = "v1" + c.AuthType = "connection_string" + return c + }(), + expectError: false, + }, + { + name: "connection_string config missing connection_string", + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.SAName = "test-storage" + c.SAConnectionString = "test-connection-string" + c.ProcessorVersion = "v2" + c.AuthType = "connection_string" + return c + }(), + expectError: true, + errorMsg: "connection_string is required when auth_type is empty or set to connection_string", + }, + { + name: "connection_string config with processor v1 missing storage account key", + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.ConnectionString = "Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=test;SharedAccessKey=test" + c.SAName = "test-storage" + c.ProcessorVersion = "v1" + c.AuthType = "connection_string" + return c + }(), + expectError: true, + errorMsg: "storage_account_key is required when using connection_string authentication with processor v1", + }, + { + name: "connection_string config with processor v2 missing storage account connection string", + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.ConnectionString = "Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=test;SharedAccessKey=test" + c.SAName = "test-storage" + c.ProcessorVersion = "v2" + c.AuthType = "connection_string" + return c + }(), + expectError: true, + errorMsg: "storage_account_connection_string is required when using connection_string authentication with processor v2", + }, + { + name: "invalid auth_type", + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.SAName = "test-storage" + c.ProcessorVersion = "v2" + c.AuthType = "invalid_auth_type" + return c + }(), + expectError: true, + errorMsg: "unknown auth_type: invalid_auth_type (valid values: connection_string, client_secret)", + }, + { + name: "invalid processor_version", + config: func() azureInputConfig { + c := defaultConfig() + c.EventHubName = "test-hub" + c.ConnectionString = "Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=test;SharedAccessKey=test" + c.SAName = "test-storage" + c.SAKey = "test-storage-key" + c.ProcessorVersion = "v3" + c.AuthType = "connection_string" + return c + }(), + expectError: true, + errorMsg: "invalid processor_version: v3 (available versions: v1, v2)", + }, } for _, tt := range tests { From 732f98437fff7f0f419436e92092a85536f99e8c Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 4 Nov 2025 23:10:24 -0700 Subject: [PATCH 10/21] update changelog --- ...1678237-feat-add-client-secret-auth-for-azure-eventhub.yaml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename changelog/fragments/{1761678237-feat-add-oauth2-for-azure-eventhub.yaml => 1761678237-feat-add-client-secret-auth-for-azure-eventhub.yaml} (95%) diff --git a/changelog/fragments/1761678237-feat-add-oauth2-for-azure-eventhub.yaml b/changelog/fragments/1761678237-feat-add-client-secret-auth-for-azure-eventhub.yaml similarity index 95% rename from changelog/fragments/1761678237-feat-add-oauth2-for-azure-eventhub.yaml rename to changelog/fragments/1761678237-feat-add-client-secret-auth-for-azure-eventhub.yaml index 1ae4fda1da97..6e58a3e66dee 100644 --- a/changelog/fragments/1761678237-feat-add-oauth2-for-azure-eventhub.yaml +++ b/changelog/fragments/1761678237-feat-add-client-secret-auth-for-azure-eventhub.yaml @@ -13,7 +13,7 @@ kind: feature # REQUIRED for all kinds # Change summary; a 80ish characters long description of the change. -summary: Add OAuth2 authentication for Azure Event Hub and storage in Filebeat. +summary: Add client secret authentication method for Azure Event Hub and storage in Filebeat. # REQUIRED for breaking-change, deprecation, known-issue # Long description; in case the summary is not enough to describe the change From bdba1592cc25fc96573d5e580f035a12d96ea4d9 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 5 Nov 2025 13:42:32 -0700 Subject: [PATCH 11/21] fix lint --- .../filebeat/input/azureeventhub/clients.go | 34 ++++++++----------- x-pack/filebeat/input/azureeventhub/config.go | 1 - .../filebeat/input/azureeventhub/v2_input.go | 5 +-- 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/x-pack/filebeat/input/azureeventhub/clients.go b/x-pack/filebeat/input/azureeventhub/clients.go index 55c7e9b142d2..162e0b0a0736 100644 --- a/x-pack/filebeat/input/azureeventhub/clients.go +++ b/x-pack/filebeat/input/azureeventhub/clients.go @@ -78,27 +78,23 @@ type storageContainerClientConfig struct { // newStorageContainerClient creates a new Storage container client using the provided credential or connection string. func newStorageContainerClient(config storageContainerClientConfig, authType string, log *logp.Logger) (*container.Client, error) { if authType == AuthTypeConnectionString { - if config.ConnectionString != "" { - // Use connection string authentication (legacy) - if config.Cloud.ActiveDirectoryAuthorityHost == "" { - config.Cloud = cloud.AzurePublic - } - containerClient, err := container.NewClientFromConnectionString( - config.ConnectionString, - config.Container, - &container.ClientOptions{ - ClientOptions: azcore.ClientOptions{ - Cloud: config.Cloud, - }, + // Use connection string authentication + if config.Cloud.ActiveDirectoryAuthorityHost == "" { + config.Cloud = cloud.AzurePublic + } + containerClient, err := container.NewClientFromConnectionString( + config.ConnectionString, + config.Container, + &container.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Cloud: config.Cloud, }, - ) - if err != nil { - return nil, fmt.Errorf("failed to create container client from connection string: %w", err) - } - return containerClient, nil - } else { - + }, + ) + if err != nil { + return nil, fmt.Errorf("failed to create container client from connection string: %w", err) } + return containerClient, nil } if config.Credential == nil { diff --git a/x-pack/filebeat/input/azureeventhub/config.go b/x-pack/filebeat/input/azureeventhub/config.go index b057c92b9017..49f0ede9ffda 100644 --- a/x-pack/filebeat/input/azureeventhub/config.go +++ b/x-pack/filebeat/input/azureeventhub/config.go @@ -150,7 +150,6 @@ func defaultConfig() azureInputConfig { // Validate validates the config. func (conf *azureInputConfig) Validate() error { logger := logp.NewLogger("azureeventhub.config") - logger.Infof("DEBUG: Validate() method called") // Determine authentication method authType := conf.AuthType diff --git a/x-pack/filebeat/input/azureeventhub/v2_input.go b/x-pack/filebeat/input/azureeventhub/v2_input.go index 3f34c34121a5..478cebaf96b5 100644 --- a/x-pack/filebeat/input/azureeventhub/v2_input.go +++ b/x-pack/filebeat/input/azureeventhub/v2_input.go @@ -88,9 +88,6 @@ func (in *eventHubInputV2) Run( ) error { var err error - // DEBUG: Run method called - in.log.Infof("DEBUG: Run method called for azure-eventhub input") - // Setting up the status reporter helper in.status = statusreporterhelper.New(inputContext.StatusReporter, in.log, "Azure Event Hub") @@ -654,7 +651,7 @@ func (in *eventHubInputV2) processReceivedEvents(receivedEvents []*azeventhubs.R // Update input metrics. in.metrics.sentEvents.Inc() } - records = nil + // Update input metrics. in.metrics.processedMessages.Inc() in.metrics.processingTime.Update(time.Since(processingStartTime).Nanoseconds()) From 44c0d1740b4d3d409ac2ee294d7249cb50238849 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 5 Nov 2025 15:04:06 -0700 Subject: [PATCH 12/21] move checking authType into the client funcs --- .../filebeat/input/azureeventhub/clients.go | 38 ++++- .../filebeat/input/azureeventhub/v2_input.go | 144 +++++------------- 2 files changed, 72 insertions(+), 110 deletions(-) diff --git a/x-pack/filebeat/input/azureeventhub/clients.go b/x-pack/filebeat/input/azureeventhub/clients.go index 162e0b0a0736..e6db1132e3d4 100644 --- a/x-pack/filebeat/input/azureeventhub/clients.go +++ b/x-pack/filebeat/input/azureeventhub/clients.go @@ -27,8 +27,33 @@ type eventHubClientConfig struct { } // newEventHubConsumerClient creates a new Event Hub consumer client using the provided credential or connection string. -func newEventHubConsumerClient(config eventHubClientConfig, log *logp.Logger) (*azeventhubs.ConsumerClient, error) { - if config.ConnectionString != "" { +func newEventHubConsumerClient(config eventHubClientConfig, authType string, log *logp.Logger) (*azeventhubs.ConsumerClient, error) { + if authType == AuthTypeConnectionString { + // Use connection string authentication for Event Hub + // There is a mismatch between how the azure-eventhub input and the new + // Event Hub SDK expect the event hub name in the connection string. + // + // The azure-eventhub input was designed to work with the old Event Hub SDK, + // which worked using the event hub name in the connection string. + // + // The new Event Hub SDK expects clients to pass the event hub name as a + // parameter, or in the connection string as the entity path. + // + // We need to handle both cases. + connectionStringProperties, err := parseConnectionString(config.ConnectionString) + if err != nil { + return nil, fmt.Errorf("failed to parse connection string: %w", err) + } + if connectionStringProperties.EntityPath != nil { + // If the connection string contains an entity path, we need to + // set the event hub name to an empty string. + // + // This is a requirement of the new Event Hub SDK. + // + // See: https://github.com/Azure/azure-sdk-for-go/blob/4ece3e50652223bba502f2b73e7f297de34a799c/sdk/messaging/azeventhubs/producer_client.go#L304-L306 + config.EventHubName = "" + } + // Use connection string authentication consumerClient, err := azeventhubs.NewConsumerClientFromConnectionString( config.ConnectionString, @@ -42,11 +67,11 @@ func newEventHubConsumerClient(config eventHubClientConfig, log *logp.Logger) (* return consumerClient, nil } + // Use credential authentication if config.Credential == nil { - return nil, fmt.Errorf("credential is required when connection_string is not provided") + return nil, fmt.Errorf("credential cannot be empty when auth_type is not connection_string") } - // Use credential authentication consumerClient, err := azeventhubs.NewConsumerClient( config.Namespace, config.EventHubName, @@ -97,14 +122,13 @@ func newStorageContainerClient(config storageContainerClientConfig, authType str return containerClient, nil } + // Use credential authentication if config.Credential == nil { - return nil, fmt.Errorf("credential is required when connection_string is not provided") + return nil, fmt.Errorf("credential cannot be empty when auth_type is not connection_string") } // Build the storage account URL storageAccountURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s", config.StorageAccount, config.Container) - - // Use credential authentication containerClient, err := container.NewClient(storageAccountURL, config.Credential, nil) if err != nil { return nil, fmt.Errorf("failed to create container client with credential: %w", err) diff --git a/x-pack/filebeat/input/azureeventhub/v2_input.go b/x-pack/filebeat/input/azureeventhub/v2_input.go index 478cebaf96b5..e2ad4b5b9993 100644 --- a/x-pack/filebeat/input/azureeventhub/v2_input.go +++ b/x-pack/filebeat/input/azureeventhub/v2_input.go @@ -13,8 +13,6 @@ import ( "strings" "time" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" - "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" @@ -154,109 +152,49 @@ func (in *eventHubInputV2) setup(ctx context.Context) error { // Create the credential if needed (shared by both Event Hub and Storage Account) // Both services use the same credential since they share the same auth_type - var credential azcore.TokenCredential - // Create the event hub consumerClient to receive events. - var consumerClient *azeventhubs.ConsumerClient - // Create the container client - var containerClient *container.Client - - useCredentials := authType == AuthTypeClientSecret - if useCredentials { - credConfig := authConfig{ - AuthType: authType, - TenantID: in.config.TenantID, - ClientID: in.config.ClientID, - ClientSecret: in.config.ClientSecret, - AuthorityHost: in.config.AuthorityHost, - } - - credential, err = newCredential(credConfig, authType, in.log) - if err != nil { - in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating credential: %s", err.Error())) - return fmt.Errorf("failed to create credential: %w", err) - } - - consumerClient, err = newEventHubConsumerClient( - eventHubClientConfig{ - Namespace: in.config.EventHubNamespace, - EventHubName: in.config.EventHubName, - ConsumerGroup: in.config.ConsumerGroup, - Credential: credential, - }, - in.log, - ) - if err != nil { - return fmt.Errorf("failed to create consumer client with credential: %w", err) - } - - // Use credential-based authentication for the storage account - containerClient, err = newStorageContainerClient( - storageContainerClientConfig{ - StorageAccount: in.config.SAName, - Container: in.config.SAContainer, - Credential: credential, - Cloud: getAzureCloud(in.config.AuthorityHost), - }, - authType, - in.log, - ) - if err != nil { - in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating blob container client with credential: %s", err.Error())) - return fmt.Errorf("failed to create blob container client with credential: %w", err) - } - } else { - // Use connection string authentication for Event Hub - // There is a mismatch between how the azure-eventhub input and the new - // Event Hub SDK expect the event hub name in the connection string. - // - // The azure-eventhub input was designed to work with the old Event Hub SDK, - // which worked using the event hub name in the connection string. - // - // The new Event Hub SDK expects clients to pass the event hub name as a - // parameter, or in the connection string as the entity path. - // - // We need to handle both cases. - eventHubName := in.config.EventHubName - - connectionStringProperties, err := parseConnectionString(in.config.ConnectionString) - if err != nil { - return fmt.Errorf("failed to parse connection string: %w", err) - } - if connectionStringProperties.EntityPath != nil { - // If the connection string contains an entity path, we need to - // set the event hub name to an empty string. - // - // This is a requirement of the new Event Hub SDK. - // - // See: https://github.com/Azure/azure-sdk-for-go/blob/4ece3e50652223bba502f2b73e7f297de34a799c/sdk/messaging/azeventhubs/producer_client.go#L304-L306 - eventHubName = "" - } + credConfig := authConfig{ + AuthType: authType, + TenantID: in.config.TenantID, + ClientID: in.config.ClientID, + ClientSecret: in.config.ClientSecret, + AuthorityHost: in.config.AuthorityHost, + } + credential, err := newCredential(credConfig, authType, in.log) + if err != nil { + in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating credential: %s", err.Error())) + return fmt.Errorf("failed to create credential: %w", err) + } - consumerClient, err = newEventHubConsumerClient( - eventHubClientConfig{ - ConnectionString: in.config.ConnectionString, - EventHubName: eventHubName, - ConsumerGroup: in.config.ConsumerGroup, - }, - in.log, - ) - if err != nil { - return fmt.Errorf("failed to create consumer client: %w", err) - } + // Create the event hub consumerClient to receive events. + consumerClient, err := newEventHubConsumerClient( + eventHubClientConfig{ + Namespace: in.config.EventHubNamespace, + EventHubName: in.config.EventHubName, + ConsumerGroup: in.config.ConsumerGroup, + Credential: credential, + }, + authType, + in.log, + ) + if err != nil { + return fmt.Errorf("failed to create consumer client with credential: %w", err) + } - containerClient, err = newStorageContainerClient( - storageContainerClientConfig{ - ConnectionString: in.config.SAConnectionString, - Container: in.config.SAContainer, - Cloud: cloud.AzurePublic, - }, - authType, - in.log, - ) - if err != nil { - in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating blob container client: %s", err.Error())) - return fmt.Errorf("failed to create blob container client: %w", err) - } + // Create the container client + containerClient, err := newStorageContainerClient( + storageContainerClientConfig{ + ConnectionString: in.config.SAConnectionString, + StorageAccount: in.config.SAName, + Container: in.config.SAContainer, + Credential: credential, + Cloud: getAzureCloud(in.config.AuthorityHost), + }, + authType, + in.log, + ) + if err != nil { + in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating blob container client with credential: %s", err.Error())) + return fmt.Errorf("failed to create blob container client with credential: %w", err) } // The modern event hub SDK does not create the container From e90998224169dc0aabeee55db94b81b19e319af5 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 6 Nov 2025 17:10:21 -0700 Subject: [PATCH 13/21] fix v2_migration to check for both connection string and eventhub namespace --- x-pack/filebeat/input/azureeventhub/config.go | 12 +++--- .../filebeat/input/azureeventhub/v2_input.go | 43 ++++++++++--------- .../input/azureeventhub/v2_migration.go | 25 ++++++++--- 3 files changed, 47 insertions(+), 33 deletions(-) diff --git a/x-pack/filebeat/input/azureeventhub/config.go b/x-pack/filebeat/input/azureeventhub/config.go index 49f0ede9ffda..33285e1ce156 100644 --- a/x-pack/filebeat/input/azureeventhub/config.go +++ b/x-pack/filebeat/input/azureeventhub/config.go @@ -151,11 +151,9 @@ func defaultConfig() azureInputConfig { func (conf *azureInputConfig) Validate() error { logger := logp.NewLogger("azureeventhub.config") - // Determine authentication method - authType := conf.AuthType - if authType == "" { - // Default to connection_string for backwards compatibility - authType = AuthTypeConnectionString + // Normalize authentication method (default to connection_string if empty) + if conf.AuthType == "" { + conf.AuthType = AuthTypeConnectionString } // Validate the processor version first to ensure it's valid @@ -169,7 +167,7 @@ func (conf *azureInputConfig) Validate() error { } // Validate authentication for both Event Hub and Storage Account together - switch authType { + switch conf.AuthType { case AuthTypeConnectionString: // Validate Event Hub connection string configuration if conf.ConnectionString == "" { @@ -232,7 +230,7 @@ func (conf *azureInputConfig) Validate() error { } default: - return fmt.Errorf("unknown auth_type: %s (valid values: connection_string, client_secret)", authType) + return fmt.Errorf("unknown auth_type: %s (valid values: connection_string, client_secret)", conf.AuthType) } // Validate required fields diff --git a/x-pack/filebeat/input/azureeventhub/v2_input.go b/x-pack/filebeat/input/azureeventhub/v2_input.go index e2ad4b5b9993..24e50309fa50 100644 --- a/x-pack/filebeat/input/azureeventhub/v2_input.go +++ b/x-pack/filebeat/input/azureeventhub/v2_input.go @@ -146,38 +146,40 @@ func (in *eventHubInputV2) setup(ctx context.Context) error { // Determine authentication method (shared by both Event Hub and Storage Account) authType := in.config.AuthType - if authType == "" { - authType = AuthTypeConnectionString - } - // Create the credential if needed (shared by both Event Hub and Storage Account) + // Create the credential if needed (only for client_secret auth) // Both services use the same credential since they share the same auth_type - credConfig := authConfig{ - AuthType: authType, - TenantID: in.config.TenantID, - ClientID: in.config.ClientID, - ClientSecret: in.config.ClientSecret, - AuthorityHost: in.config.AuthorityHost, - } - credential, err := newCredential(credConfig, authType, in.log) - if err != nil { - in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating credential: %s", err.Error())) - return fmt.Errorf("failed to create credential: %w", err) + var credential azcore.TokenCredential + if authType == AuthTypeClientSecret { + credConfig := authConfig{ + AuthType: authType, + TenantID: in.config.TenantID, + ClientID: in.config.ClientID, + ClientSecret: in.config.ClientSecret, + AuthorityHost: in.config.AuthorityHost, + } + var err error + credential, err = newCredential(credConfig, authType, in.log) + if err != nil { + in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating credential: %s", err.Error())) + return fmt.Errorf("failed to create credential: %w", err) + } } // Create the event hub consumerClient to receive events. consumerClient, err := newEventHubConsumerClient( eventHubClientConfig{ - Namespace: in.config.EventHubNamespace, - EventHubName: in.config.EventHubName, - ConsumerGroup: in.config.ConsumerGroup, - Credential: credential, + Namespace: in.config.EventHubNamespace, + EventHubName: in.config.EventHubName, + ConsumerGroup: in.config.ConsumerGroup, + Credential: credential, + ConnectionString: in.config.ConnectionString, }, authType, in.log, ) if err != nil { - return fmt.Errorf("failed to create consumer client with credential: %w", err) + return fmt.Errorf("failed to create consumer client: %w", err) } // Create the container client @@ -261,7 +263,6 @@ func (in *eventHubInputV2) run(ctx context.Context) error { // Check if we need to migrate the checkpoint store. err := in.migrationAssistant.checkAndMigrate( ctx, - in.config.ConnectionString, in.config.ConsumerGroup, ) if err != nil { diff --git a/x-pack/filebeat/input/azureeventhub/v2_migration.go b/x-pack/filebeat/input/azureeventhub/v2_migration.go index 60590cc5e038..305ebf02ae0b 100644 --- a/x-pack/filebeat/input/azureeventhub/v2_migration.go +++ b/x-pack/filebeat/input/azureeventhub/v2_migration.go @@ -57,7 +57,7 @@ func newMigrationAssistant(config azureInputConfig, log *logp.Logger, consumerCl // checkAndMigrate checks if the v1 checkpoint information for the partitions // exists and migrates it to v2 if it does. -func (m *migrationAssistant) checkAndMigrate(ctx context.Context, eventHubConnectionString, consumerGroup string) error { +func (m *migrationAssistant) checkAndMigrate(ctx context.Context, consumerGroup string) error { // Fetching event hub information eventHubProperties, err := m.consumerClient.GetEventHubProperties(ctx, nil) if err != nil { @@ -78,9 +78,24 @@ func (m *migrationAssistant) checkAndMigrate(ctx context.Context, eventHubConnec return err } - connectionStringProperties, err := parseConnectionString(m.config.ConnectionString) - if err != nil { - return fmt.Errorf("migration assistant: failed to parse connection string: %w", err) + // Determine the fully qualified namespace based on the auth type + var fullyQualifiedNamespace string + switch m.config.AuthType { + case AuthTypeConnectionString: + // When using connection_string auth, parse it to get the namespace + connectionStringProperties, err := parseConnectionString(m.config.ConnectionString) + if err != nil { + return fmt.Errorf("migration assistant: failed to parse connection string: %w", err) + } + fullyQualifiedNamespace = connectionStringProperties.FullyQualifiedNamespace + case AuthTypeClientSecret: + // When using client_secret auth, use EventHubNamespace directly + if m.config.EventHubNamespace == "" { + return fmt.Errorf("migration assistant: eventhub_namespace is required when using client_secret authentication") + } + fullyQualifiedNamespace = m.config.EventHubNamespace + default: + return fmt.Errorf("migration assistant: unknown auth_type: %s", m.config.AuthType) } for _, partitionID := range eventHubProperties.PartitionIDs { @@ -88,7 +103,7 @@ func (m *migrationAssistant) checkAndMigrate(ctx context.Context, eventHubConnec ctx, blobs, partitionID, - connectionStringProperties.FullyQualifiedNamespace, + fullyQualifiedNamespace, eventHubProperties.Name, consumerGroup, ) From 0af0f551ceb3e3aac8847d29674e6d4b9d93b0ff Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 6 Nov 2025 17:19:53 -0700 Subject: [PATCH 14/21] change migrate_checkpoint to true --- x-pack/filebeat/input/azureeventhub/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/filebeat/input/azureeventhub/README.md b/x-pack/filebeat/input/azureeventhub/README.md index 146f0cbf50ae..878e899b3d35 100644 --- a/x-pack/filebeat/input/azureeventhub/README.md +++ b/x-pack/filebeat/input/azureeventhub/README.md @@ -282,7 +282,7 @@ Using the following configuration: storage_account_key: "" storage_account_connection_string: "" processor_version: "v1" - migrate_checkpoint: yes + migrate_checkpoint: true start_position: "earliest" ``` @@ -377,7 +377,7 @@ Stop Filebeat and update the config with the following changes: storage_account_key: "" storage_account_connection_string: "" # NOTE: make sure this is set processor_version: "v2" # CHANGE: v1 > v2 - migrate_checkpoint: yes + migrate_checkpoint: true start_position: "earliest" ``` @@ -438,7 +438,7 @@ Using the following configuration: storage_account_key: "" storage_account_connection_string: "" processor_version: "v2" - migrate_checkpoint: yes + migrate_checkpoint: true start_position: "earliest" ``` @@ -564,7 +564,7 @@ Using the following configuration for all inputs: storage_account_key: "" storage_account_connection_string: "" processor_version: "v2" - migrate_checkpoint: yes + migrate_checkpoint: true start_position: "earliest" ``` @@ -783,7 +783,7 @@ Using the following configuration: storage_account_key: "" storage_account_connection_string: "" processor_version: "v2" - migrate_checkpoint: yes + migrate_checkpoint: true start_position: "earliest" ``` From fdc43ad6dc0f74c73326b6323e4e42d56af498c9 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 8 Dec 2025 16:02:17 -0700 Subject: [PATCH 15/21] move to GetFullyQualifiedEventHubNamespace function --- x-pack/filebeat/input/azureeventhub/config.go | 22 ++++ .../input/azureeventhub/config_test.go | 113 ++++++++++++++++++ .../input/azureeventhub/v2_migration.go | 20 +--- 3 files changed, 138 insertions(+), 17 deletions(-) diff --git a/x-pack/filebeat/input/azureeventhub/config.go b/x-pack/filebeat/input/azureeventhub/config.go index 33285e1ce156..d4609376fc67 100644 --- a/x-pack/filebeat/input/azureeventhub/config.go +++ b/x-pack/filebeat/input/azureeventhub/config.go @@ -287,6 +287,28 @@ func (conf *azureInputConfig) Validate() error { return nil } +// GetFullyQualifiedEventHubNamespace returns the fully qualified namespace for the Event Hub +// based on the configured authentication type. +func (conf *azureInputConfig) GetFullyQualifiedEventHubNamespace() (string, error) { + switch conf.AuthType { + case AuthTypeConnectionString: + // When using connection_string auth, parse it to get the namespace + connectionStringProperties, err := parseConnectionString(conf.ConnectionString) + if err != nil { + return "", fmt.Errorf("failed to parse connection string: %w", err) + } + return connectionStringProperties.FullyQualifiedNamespace, nil + case AuthTypeClientSecret: + // When using client_secret auth, use EventHubNamespace directly + if conf.EventHubNamespace == "" { + return "", fmt.Errorf("eventhub_namespace is required when using client_secret authentication") + } + return conf.EventHubNamespace, nil + default: + return "", fmt.Errorf("unknown auth_type: %s", conf.AuthType) + } +} + // checkUnsupportedParams checks if unsupported/deprecated/discouraged parameters are set and logs a warning func (conf *azureInputConfig) checkUnsupportedParams(logger *logp.Logger) { logger = logger.Named("azureeventhub.config") diff --git a/x-pack/filebeat/input/azureeventhub/config_test.go b/x-pack/filebeat/input/azureeventhub/config_test.go index 335d7c308cc4..b73b8d0961f9 100644 --- a/x-pack/filebeat/input/azureeventhub/config_test.go +++ b/x-pack/filebeat/input/azureeventhub/config_test.go @@ -454,3 +454,116 @@ func TestConnectionStringConfigValidation(t *testing.T) { }) } } + +func TestGetFullyQualifiedEventHubNamespace(t *testing.T) { + tests := []struct { + name string + config azureInputConfig + expectedResult string + expectError bool + errorMsg string + }{ + { + name: "connection_string auth with valid connection string", + config: func() azureInputConfig { + c := defaultConfig() + c.AuthType = AuthTypeConnectionString + c.ConnectionString = "Endpoint=sb://my-namespace.servicebus.windows.net/;SharedAccessKeyName=my-key;SharedAccessKey=my-secret" + return c + }(), + expectedResult: "my-namespace.servicebus.windows.net", + expectError: false, + }, + { + name: "connection_string auth with connection string containing entity path", + config: func() azureInputConfig { + c := defaultConfig() + c.AuthType = AuthTypeConnectionString + c.ConnectionString = "Endpoint=sb://test-ns.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=SECRET;EntityPath=my-event-hub" + return c + }(), + expectedResult: "test-ns.servicebus.windows.net", + expectError: false, + }, + { + name: "connection_string auth with invalid connection string", + config: func() azureInputConfig { + c := defaultConfig() + c.AuthType = AuthTypeConnectionString + c.ConnectionString = "InvalidConnectionString" + return c + }(), + expectError: true, + errorMsg: "failed to parse connection string", + }, + { + name: "connection_string auth with empty connection string", + config: func() azureInputConfig { + c := defaultConfig() + c.AuthType = AuthTypeConnectionString + c.ConnectionString = "" + return c + }(), + expectError: true, + errorMsg: "failed to parse connection string", + }, + { + name: "client_secret auth with valid namespace", + config: func() azureInputConfig { + c := defaultConfig() + c.AuthType = AuthTypeClientSecret + c.EventHubNamespace = "test-namespace.servicebus.windows.net" + return c + }(), + expectedResult: "test-namespace.servicebus.windows.net", + expectError: false, + }, + { + name: "client_secret auth with empty namespace", + config: func() azureInputConfig { + c := defaultConfig() + c.AuthType = AuthTypeClientSecret + c.EventHubNamespace = "" + return c + }(), + expectError: true, + errorMsg: "eventhub_namespace is required when using client_secret authentication", + }, + { + name: "unknown auth type", + config: func() azureInputConfig { + c := defaultConfig() + c.AuthType = "unknown_auth_type" + return c + }(), + expectError: true, + errorMsg: "unknown auth_type: unknown_auth_type", + }, + { + name: "empty auth type (should default but method doesn't handle default)", + config: func() azureInputConfig { + c := defaultConfig() + c.AuthType = "" + return c + }(), + expectError: true, + errorMsg: "unknown auth_type:", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := tt.config.GetFullyQualifiedEventHubNamespace() + if tt.expectError { + require.Error(t, err) + if tt.errorMsg != "" { + assert.Contains(t, err.Error(), tt.errorMsg) + } + assert.Empty(t, result) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expectedResult, result) + } + }) + } +} diff --git a/x-pack/filebeat/input/azureeventhub/v2_migration.go b/x-pack/filebeat/input/azureeventhub/v2_migration.go index 305ebf02ae0b..fc20b2812dc1 100644 --- a/x-pack/filebeat/input/azureeventhub/v2_migration.go +++ b/x-pack/filebeat/input/azureeventhub/v2_migration.go @@ -79,23 +79,9 @@ func (m *migrationAssistant) checkAndMigrate(ctx context.Context, consumerGroup } // Determine the fully qualified namespace based on the auth type - var fullyQualifiedNamespace string - switch m.config.AuthType { - case AuthTypeConnectionString: - // When using connection_string auth, parse it to get the namespace - connectionStringProperties, err := parseConnectionString(m.config.ConnectionString) - if err != nil { - return fmt.Errorf("migration assistant: failed to parse connection string: %w", err) - } - fullyQualifiedNamespace = connectionStringProperties.FullyQualifiedNamespace - case AuthTypeClientSecret: - // When using client_secret auth, use EventHubNamespace directly - if m.config.EventHubNamespace == "" { - return fmt.Errorf("migration assistant: eventhub_namespace is required when using client_secret authentication") - } - fullyQualifiedNamespace = m.config.EventHubNamespace - default: - return fmt.Errorf("migration assistant: unknown auth_type: %s", m.config.AuthType) + fullyQualifiedNamespace, err := m.config.GetFullyQualifiedEventHubNamespace() + if err != nil { + return fmt.Errorf("migration assistant: %w", err) } for _, partitionID := range eventHubProperties.PartitionIDs { From 820861a9aea94a853ac0594abc97d06a9ecddf2b Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 8 Dec 2025 17:24:23 -0700 Subject: [PATCH 16/21] refactor authenticator --- .../input/azureeventhub/authenticator.go | 173 ++++++++++++++++++ x-pack/filebeat/input/azureeventhub/config.go | 63 ++++--- .../filebeat/input/azureeventhub/v2_input.go | 53 +----- 3 files changed, 213 insertions(+), 76 deletions(-) create mode 100644 x-pack/filebeat/input/azureeventhub/authenticator.go diff --git a/x-pack/filebeat/input/azureeventhub/authenticator.go b/x-pack/filebeat/input/azureeventhub/authenticator.go new file mode 100644 index 000000000000..33346a6e0e27 --- /dev/null +++ b/x-pack/filebeat/input/azureeventhub/authenticator.go @@ -0,0 +1,173 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build !aix + +package azureeventhub + +import ( + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + + "github.com/elastic/elastic-agent-libs/logp" +) + +// createCredential creates a TokenCredential if needed based on the authentication type. +// Returns nil for connection_string authentication (which doesn't use credentials). +func createCredential(cfg *azureInputConfig, log *logp.Logger) (azcore.TokenCredential, error) { + switch cfg.AuthType { + case AuthTypeConnectionString: + // No credential needed for connection string authentication + return nil, nil + case AuthTypeClientSecret: + credConfig := authConfig{ + AuthType: cfg.AuthType, + TenantID: cfg.TenantID, + ClientID: cfg.ClientID, + ClientSecret: cfg.ClientSecret, + AuthorityHost: cfg.AuthorityHost, + } + credential, err := newClientSecretCredential(credConfig, log) + if err != nil { + return nil, fmt.Errorf("failed to create client secret credential: %w", err) + } + return credential, nil + default: + return nil, fmt.Errorf("invalid auth_type: %s", cfg.AuthType) + } +} + +// CreateEventHubConsumerClient creates an Event Hub consumer client +// using the configured authentication method from the provided config. +func CreateEventHubConsumerClient(cfg *azureInputConfig, log *logp.Logger) (*azeventhubs.ConsumerClient, error) { + switch cfg.AuthType { + case AuthTypeConnectionString: + // Use connection string authentication for Event Hub + // There is a mismatch between how the azure-eventhub input and the new + // Event Hub SDK expect the event hub name in the connection string. + // + // The azure-eventhub input was designed to work with the old Event Hub SDK, + // which worked using the event hub name in the connection string. + // + // The new Event Hub SDK expects clients to pass the event hub name as a + // parameter, or in the connection string as the entity path. + // + // We need to handle both cases. + connectionStringProperties, err := parseConnectionString(cfg.ConnectionString) + if err != nil { + return nil, fmt.Errorf("failed to parse connection string: %w", err) + } + + // Determine the event hub name to use + // If the connection string contains an entity path, we need to + // set the event hub name to an empty string. + // + // This is a requirement of the new Event Hub SDK. + // + // See: https://github.com/Azure/azure-sdk-for-go/blob/4ece3e50652223bba502f2b73e7f297de34a799c/sdk/messaging/azeventhubs/producer_client.go#L304-L306 + eventHubName := cfg.EventHubName + if connectionStringProperties.EntityPath != nil { + eventHubName = "" + } + + // Use connection string authentication + consumerClient, err := azeventhubs.NewConsumerClientFromConnectionString( + cfg.ConnectionString, + eventHubName, + cfg.ConsumerGroup, + nil, + ) + if err != nil { + return nil, fmt.Errorf("failed to create consumer client from connection string: %w", err) + } + return consumerClient, nil + + case AuthTypeClientSecret: + credential, err := createCredential(cfg, log) + if err != nil { + return nil, err + } + if credential == nil { + return nil, fmt.Errorf("credential cannot be empty when auth_type is client_secret") + } + + consumerClient, err := azeventhubs.NewConsumerClient( + cfg.EventHubNamespace, + cfg.EventHubName, + cfg.ConsumerGroup, + credential, + nil, + ) + if err != nil { + return nil, fmt.Errorf("failed to create consumer client with credential: %w", err) + } + + log.Infow("successfully created consumer client with credential authentication", + "namespace", cfg.EventHubNamespace, + "eventhub", cfg.EventHubName, + ) + + return consumerClient, nil + + default: + return nil, fmt.Errorf("invalid auth_type: %s", cfg.AuthType) + } +} + +// CreateStorageAccountContainerClient creates a Storage Account container client +// using the configured authentication method from the provided config. +func CreateStorageAccountContainerClient(cfg *azureInputConfig, log *logp.Logger) (*container.Client, error) { + switch cfg.AuthType { + case AuthTypeConnectionString: + // Use connection string authentication + cloudConfig := getAzureCloud(cfg.AuthorityHost) + if cloudConfig.ActiveDirectoryAuthorityHost == "" { + cloudConfig = cloud.AzurePublic + } + + containerClient, err := container.NewClientFromConnectionString( + cfg.SAConnectionString, + cfg.SAContainer, + &container.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Cloud: cloudConfig, + }, + }, + ) + if err != nil { + return nil, fmt.Errorf("failed to create container client from connection string: %w", err) + } + return containerClient, nil + + case AuthTypeClientSecret: + credential, err := createCredential(cfg, log) + if err != nil { + return nil, err + } + if credential == nil { + return nil, fmt.Errorf("credential cannot be empty when auth_type is client_secret") + } + + // Build the storage account URL + storageAccountURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s", cfg.SAName, cfg.SAContainer) + containerClient, err := container.NewClient(storageAccountURL, credential, nil) + if err != nil { + return nil, fmt.Errorf("failed to create container client with credential: %w", err) + } + + log.Infow("successfully created container client with credential authentication", + "storage_account", cfg.SAName, + "container", cfg.SAContainer, + ) + + return containerClient, nil + + default: + return nil, fmt.Errorf("invalid auth_type: %s", cfg.AuthType) + } +} diff --git a/x-pack/filebeat/input/azureeventhub/config.go b/x-pack/filebeat/input/azureeventhub/config.go index 7f15460063ad..d8c308d924e6 100644 --- a/x-pack/filebeat/input/azureeventhub/config.go +++ b/x-pack/filebeat/input/azureeventhub/config.go @@ -198,10 +198,8 @@ func (conf *azureInputConfig) Validate() error { return errors.New("storage_account_key is required when using connection_string authentication with processor v1") } case processorV2: - // Processor v2 requires storage account connection string - if conf.SAConnectionString == "" { - return errors.New("storage_account_connection_string is required when using connection_string authentication with processor v2") - } + // Processor v2 requires storage account connection string, but it can be auto-constructed + // from SAName and SAKey later in validation. We don't validate it here. } case AuthTypeClientSecret: @@ -293,34 +291,41 @@ func (conf *azureInputConfig) Validate() error { return errors.New("no storage account key configured (config: storage_account_key)") } case processorV2: - if conf.SAConnectionString == "" { - if conf.SAName != "" && conf.SAKey != "" { - // To avoid breaking changes, and ease the migration from v1 to v2, - // we can build the connection string using the following settings: - // - // - DefaultEndpointsProtocol=https; - // - AccountName=; - // - AccountKey=; - // - EndpointSuffix=core.windows.net - env, err := getAzureEnvironment(conf.OverrideEnvironment) - if err != nil { - return fmt.Errorf("failed to get azure environment: %w", err) + // For processor v2, storage account authentication depends on auth_type: + // - connection_string: needs SAConnectionString (can be auto-constructed from SAName+SAKey) + // - client_secret: uses the same credentials as Event Hub, no connection string needed + if conf.AuthType == AuthTypeConnectionString { + if conf.SAConnectionString == "" { + if conf.SAName != "" && conf.SAKey != "" { + // To avoid breaking changes, and ease the migration from v1 to v2, + // we can build the connection string using the following settings: + // + // - DefaultEndpointsProtocol=https; + // - AccountName=; + // - AccountKey=; + // - EndpointSuffix=core.windows.net + env, err := getAzureEnvironment(conf.OverrideEnvironment) + if err != nil { + return fmt.Errorf("failed to get azure environment: %w", err) + } + conf.SAConnectionString = fmt.Sprintf( + "DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s;EndpointSuffix=%s", + conf.SAName, + conf.SAKey, + env.StorageEndpointSuffix, + ) + logger.Warn("storage_account_connection_string is not configured, but storage_account and storage_account_key are configured. " + + "The connection string has been constructed from the storage account and key. " + + "Please configure storage_account_connection_string directly as storage_account_key is deprecated in processor v2.") + conf.SAKey = "" + } else { + // No connection string and no key, so we can't proceed. + return errors.New("no storage account connection string configured (config: storage_account_connection_string)") } - conf.SAConnectionString = fmt.Sprintf( - "DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s;EndpointSuffix=%s", - conf.SAName, - conf.SAKey, - env.StorageEndpointSuffix, - ) - logger.Warn("storage_account_connection_string is not configured, but storage_account and storage_account_key are configured. " + - "The connection string has been constructed from the storage account and key. " + - "Please configure storage_account_connection_string directly as storage_account_key is deprecated in processor v2.") - conf.SAKey = "" - } else { - // No connection string and no key, so we can't proceed. - return errors.New("no storage account connection string configured (config: storage_account_connection_string)") } } + // For client_secret auth with processor v2, storage account uses the same credentials + // No connection string validation needed default: return fmt.Errorf( "invalid processor_version: %s (available versions: %s, %s)", diff --git a/x-pack/filebeat/input/azureeventhub/v2_input.go b/x-pack/filebeat/input/azureeventhub/v2_input.go index 24e50309fa50..cb9e1b09668a 100644 --- a/x-pack/filebeat/input/azureeventhub/v2_input.go +++ b/x-pack/filebeat/input/azureeventhub/v2_input.go @@ -144,59 +144,18 @@ func (in *eventHubInputV2) setup(ctx context.Context) error { sanitizers: sanitizers, } - // Determine authentication method (shared by both Event Hub and Storage Account) - authType := in.config.AuthType - - // Create the credential if needed (only for client_secret auth) - // Both services use the same credential since they share the same auth_type - var credential azcore.TokenCredential - if authType == AuthTypeClientSecret { - credConfig := authConfig{ - AuthType: authType, - TenantID: in.config.TenantID, - ClientID: in.config.ClientID, - ClientSecret: in.config.ClientSecret, - AuthorityHost: in.config.AuthorityHost, - } - var err error - credential, err = newCredential(credConfig, authType, in.log) - if err != nil { - in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating credential: %s", err.Error())) - return fmt.Errorf("failed to create credential: %w", err) - } - } - - // Create the event hub consumerClient to receive events. - consumerClient, err := newEventHubConsumerClient( - eventHubClientConfig{ - Namespace: in.config.EventHubNamespace, - EventHubName: in.config.EventHubName, - ConsumerGroup: in.config.ConsumerGroup, - Credential: credential, - ConnectionString: in.config.ConnectionString, - }, - authType, - in.log, - ) + // Create the event hub consumer client + consumerClient, err := CreateEventHubConsumerClient(&in.config, in.log) if err != nil { + in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating consumer client: %s", err.Error())) return fmt.Errorf("failed to create consumer client: %w", err) } // Create the container client - containerClient, err := newStorageContainerClient( - storageContainerClientConfig{ - ConnectionString: in.config.SAConnectionString, - StorageAccount: in.config.SAName, - Container: in.config.SAContainer, - Credential: credential, - Cloud: getAzureCloud(in.config.AuthorityHost), - }, - authType, - in.log, - ) + containerClient, err := CreateStorageAccountContainerClient(&in.config, in.log) if err != nil { - in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating blob container client with credential: %s", err.Error())) - return fmt.Errorf("failed to create blob container client with credential: %w", err) + in.status.UpdateStatus(status.Failed, fmt.Sprintf("Setup failed on creating blob container client: %s", err.Error())) + return fmt.Errorf("failed to create blob container client: %w", err) } // The modern event hub SDK does not create the container From 296a37fd9df2e2877b7a6fc0010961cd755529e5 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 8 Dec 2025 17:38:05 -0700 Subject: [PATCH 17/21] remove unused code --- .../filebeat/filebeat-input-azure-eventhub.md | 81 ++++---- x-pack/filebeat/input/azureeventhub/auth.go | 163 ++++++++++++++++- .../input/azureeventhub/authenticator.go | 173 ------------------ .../filebeat/input/azureeventhub/clients.go | 143 --------------- 4 files changed, 193 insertions(+), 367 deletions(-) delete mode 100644 x-pack/filebeat/input/azureeventhub/authenticator.go delete mode 100644 x-pack/filebeat/input/azureeventhub/clients.go diff --git a/docs/reference/filebeat/filebeat-input-azure-eventhub.md b/docs/reference/filebeat/filebeat-input-azure-eventhub.md index b45981c30979..5fe7b1bed736 100644 --- a/docs/reference/filebeat/filebeat-input-azure-eventhub.md +++ b/docs/reference/filebeat/filebeat-input-azure-eventhub.md @@ -8,7 +8,6 @@ applies_to: # Azure eventhub input [filebeat-input-azure-eventhub] - Users can make use of the `azure-eventhub` input in order to read messages from an azure eventhub. The azure-eventhub input implementation is based on the the event processor host (EPH is intended to be run across multiple processes and machines while load balancing message consumers more on this here [https://github.com/Azure/azure-event-hubs-go#event-processor-host](https://github.com/Azure/azure-event-hubs-go#event-processor-host), [https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-event-processor-host](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-event-processor-host)). State such as leases on partitions and checkpoints in the event stream are shared between receivers using an Azure Storage container. For this reason, as a prerequisite to using this input, users will have to create or use an existing storage account. Users can enable internal logs tracing for this input by setting the environment variable `BEATS_AZURE_EVENTHUB_INPUT_TRACING_ENABLED: true`. When enabled, this input will log additional information to the logs. Additional information includes partition ownership, blob lease information, and other internal state. @@ -45,34 +44,18 @@ filebeat.inputs: processor_version: "v2" ``` - -## Configuration options [_configuration_options] - -The `azure-eventhub` input supports the following configuration: - - -## `eventhub` [_eventhub] - -The name of the eventhub users would like to read from, field required. - - -## `consumer_group` [_consumer_group] - -Optional, we recommend using a dedicated consumer group for the azure input. Reusing consumer groups among non-related consumers can cause unexpected behavior and possibly lost events. - - ## Authentication [_authentication] -The azure-eventhub input supports multiple authentication methods. The `auth_type` configuration option controls the authentication method used for both Event Hub and Storage Account. +The azure-eventhub input supports multiple authentication methods. The [`auth_type` configuration option](#_auth_type) controls the authentication method used for both Event Hub and Storage Account. -### Authentication Types +### Authentication types The following authentication types are supported: -- **`connection_string`** (default if `auth_type` is not specified): Uses Azure Event Hubs and Storage Account connection strings -- **`client_secret`**: Uses Azure Active Directory service principal with client secret credentials +- **`connection_string`** (default if `auth_type` is not specified): Uses Azure Event Hubs and Storage Account connection strings. +- {applies_to}`stack: ga 9.3.0` **`client_secret`**: Uses Azure Active Directory service principal with client secret credentials. -### Required Permissions +### Required permissions When using `client_secret` authentication, the service principal needs the following Azure RBAC permissions: @@ -98,15 +81,21 @@ For detailed instructions on how to set up an Azure AD service principal and con - [Azure Event Hubs authentication and authorization](https://learn.microsoft.com/en-us/azure/event-hubs/authorize-access-azure-active-directory) - [Authorize access to blobs using Azure Active Directory](https://learn.microsoft.com/en-us/azure/storage/blobs/authorize-access-azure-active-directory) -## `connection_string` [_connection_string] +## Configuration options [_configuration_options] -The connection string required to communicate with Event Hubs when using `connection_string` authentication, steps here [https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string). +The `azure-eventhub` input supports the following configuration options: -Required when `auth_type` is set to `connection_string` or when `auth_type` is not specified (defaults to `connection_string` for backwards compatibility). -A Blob Storage account is required in order to store/retrieve/update the offset or state of the eventhub messages. This means that after stopping filebeat it can start back up at the spot that it stopped processing messages. +### `eventhub` [_eventhub] + +The name of the eventhub users would like to read from, field required. -## `auth_type` [_auth_type] + +### `consumer_group` [_consumer_group] + +Optional, we recommend using a dedicated consumer group for the azure input. Reusing consumer groups among non-related consumers can cause unexpected behavior and possibly lost events. + +### `auth_type` [_auth_type] ```{applies_to} stack: ga 9.3.0 @@ -114,11 +103,23 @@ stack: ga 9.3.0 Specifies the authentication method to use for both Event Hub and Storage Account. If not specified, defaults to `connection_string` for backwards compatibility. -Valid values: -- `connection_string`: Uses connection string authentication (default) -- `client_secret`: Uses Azure Active Directory service principal with client secret credentials +Valid values include: +- `connection_string` (default): Uses connection string authentication. You _must_ provide a [`connection_string`](#_connection_string). +- `client_secret`: Uses Azure Active Directory service principal with client secret credentials. + +### `connection_string` [_connection_string] -## `eventhub_namespace` [_eventhub_namespace] +The connection string required to communicate with Event Hubs when using `connection_string` authentication. For more information, refer to [Get an Azure Event Hubs connection string](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string). + +This option is required if: + +* `auth_type` is set to `connection_string` +* `auth_type` is not specified (in which case it defaults to `connection_string` for backwards compatibility) + +A Blob Storage account is required to store, retrieve, or update the offset or state of the Event Hub messages. This means that after stopping Filebeat it can resume from where it stopped processing messages. + + +### `eventhub_namespace` [_eventhub_namespace] ```{applies_to} stack: ga 9.3.0 @@ -126,7 +127,7 @@ stack: ga 9.3.0 The fully qualified namespace for the Event Hub. Required when using `client_secret` authentication (`auth_type` is set to `client_secret`). Format: `your-eventhub-namespace.servicebus.windows.net` -## `tenant_id` [_tenant_id] +### `tenant_id` [_tenant_id] ```{applies_to} stack: ga 9.3.0 @@ -134,7 +135,7 @@ stack: ga 9.3.0 The Azure Active Directory tenant ID. Required when using `client_secret` authentication for Event Hub or Storage Account. -## `client_id` [_client_id] +### `client_id` [_client_id] ```{applies_to} stack: ga 9.3.0 @@ -142,7 +143,7 @@ stack: ga 9.3.0 The Azure Active Directory application (client) ID. Required when using `client_secret` authentication for Event Hub or Storage Account. -## `client_secret` [_client_secret] +### `client_secret` [_client_secret] ```{applies_to} stack: ga 9.3.0 @@ -150,7 +151,7 @@ stack: ga 9.3.0 The Azure Active Directory application client secret. Required when using `client_secret` authentication for Event Hub or Storage Account. -## `authority_host` [_authority_host] +### `authority_host` [_authority_host] ```{applies_to} stack: ga 9.3.0 @@ -164,22 +165,22 @@ Supported values: - `https://login.chinacloudapi.cn` (Azure China) -## `storage_account` [_storage_account] +### `storage_account` [_storage_account] The name of the storage account. Required. -## `storage_account_key` [_storage_account_key] +### `storage_account_key` [_storage_account_key] The storage account key, this key will be used to authorize access to data in your storage account, option is required. -## `storage_account_container` [_storage_account_container] +### `storage_account_container` [_storage_account_container] Optional, the name of the storage account container you would like to store the offset information in. -## `resource_manager_endpoint` [_resource_manager_endpoint] +### `resource_manager_endpoint` [_resource_manager_endpoint] Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. Ex: [https://management.chinacloudapi.cn/](https://management.chinacloudapi.cn/) for azure ChinaCloud [https://management.microsoftazure.de/](https://management.microsoftazure.de/) for azure GermanCloud [https://management.azure.com/](https://management.azure.com/) for azure PublicCloud [https://management.usgovcloudapi.net/](https://management.usgovcloudapi.net/) for azure USGovernmentCloud Users can also use this in case of a Hybrid Cloud model, where one may define their own endpoints. @@ -198,5 +199,3 @@ This input exposes metrics under the [HTTP monitoring endpoint](/reference/fileb | `sent_events_total` | Number of events that were sent successfully. | | `processing_time` | Histogram of the elapsed processing times in nanoseconds. | | `decode_errors_total` | Number of errors that occurred while decoding a message. | - - diff --git a/x-pack/filebeat/input/azureeventhub/auth.go b/x-pack/filebeat/input/azureeventhub/auth.go index fe6119c92c37..9c2334c362b8 100644 --- a/x-pack/filebeat/input/azureeventhub/auth.go +++ b/x-pack/filebeat/input/azureeventhub/auth.go @@ -10,6 +10,9 @@ import ( "fmt" "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/elastic/elastic-agent-libs/logp" ) @@ -39,18 +42,158 @@ type authConfig struct { AuthorityHost string } -// newCredential creates a new TokenCredential based on the configured auth type. -func newCredential(config authConfig, authType string, log *logp.Logger) (azcore.TokenCredential, error) { - switch authType { +// createCredential creates a TokenCredential if needed based on the authentication type. +// Returns nil for connection_string authentication (which doesn't use credentials). +func createCredential(cfg *azureInputConfig, log *logp.Logger) (azcore.TokenCredential, error) { + switch cfg.AuthType { case AuthTypeConnectionString: - // Connection string authentication doesn't use TokenCredential - // This is handled separately in the client creation - return nil, fmt.Errorf("connection_string authentication does not use TokenCredential") + // No credential needed for connection string authentication + return nil, nil case AuthTypeClientSecret: - // This function is not required right now for only supporting client_secret. - // But we will need it once we start supporting more auth types. - return newClientSecretCredential(config, log) + credConfig := authConfig{ + AuthType: cfg.AuthType, + TenantID: cfg.TenantID, + ClientID: cfg.ClientID, + ClientSecret: cfg.ClientSecret, + AuthorityHost: cfg.AuthorityHost, + } + credential, err := newClientSecretCredential(credConfig, log) + if err != nil { + return nil, fmt.Errorf("failed to create client secret credential: %w", err) + } + return credential, nil default: - return nil, fmt.Errorf("unknown auth_type: %s (valid values: connection_string, client_secret)", authType) + return nil, fmt.Errorf("invalid auth_type: %s", cfg.AuthType) } } + +// CreateEventHubConsumerClient creates an Event Hub consumer client +// using the configured authentication method from the provided config. +func CreateEventHubConsumerClient(cfg *azureInputConfig, log *logp.Logger) (*azeventhubs.ConsumerClient, error) { + switch cfg.AuthType { + case AuthTypeConnectionString: + // Use connection string authentication for Event Hub + // There is a mismatch between how the azure-eventhub input and the new + // Event Hub SDK expect the event hub name in the connection string. + // + // The azure-eventhub input was designed to work with the old Event Hub SDK, + // which worked using the event hub name in the connection string. + // + // The new Event Hub SDK expects clients to pass the event hub name as a + // parameter, or in the connection string as the entity path. + // + // We need to handle both cases. + connectionStringProperties, err := parseConnectionString(cfg.ConnectionString) + if err != nil { + return nil, fmt.Errorf("failed to parse connection string: %w", err) + } + + // Determine the event hub name to use + // If the connection string contains an entity path, we need to + // set the event hub name to an empty string. + // + // This is a requirement of the new Event Hub SDK. + // + // See: https://github.com/Azure/azure-sdk-for-go/blob/4ece3e50652223bba502f2b73e7f297de34a799c/sdk/messaging/azeventhubs/producer_client.go#L304-L306 + eventHubName := cfg.EventHubName + if connectionStringProperties.EntityPath != nil { + eventHubName = "" + } + + // Use connection string authentication + consumerClient, err := azeventhubs.NewConsumerClientFromConnectionString( + cfg.ConnectionString, + eventHubName, + cfg.ConsumerGroup, + nil, + ) + if err != nil { + return nil, fmt.Errorf("failed to create consumer client from connection string: %w", err) + } + return consumerClient, nil + + case AuthTypeClientSecret: + credential, err := createCredential(cfg, log) + if err != nil { + return nil, err + } + if credential == nil { + return nil, fmt.Errorf("credential cannot be empty when auth_type is client_secret") + } + + consumerClient, err := azeventhubs.NewConsumerClient( + cfg.EventHubNamespace, + cfg.EventHubName, + cfg.ConsumerGroup, + credential, + nil, + ) + if err != nil { + return nil, fmt.Errorf("failed to create consumer client with credential: %w", err) + } + + log.Infow("successfully created consumer client with credential authentication", + "namespace", cfg.EventHubNamespace, + "eventhub", cfg.EventHubName, + ) + + return consumerClient, nil + + default: + return nil, fmt.Errorf("invalid auth_type: %s", cfg.AuthType) + } +} + +// CreateStorageAccountContainerClient creates a Storage Account container client +// using the configured authentication method from the provided config. +func CreateStorageAccountContainerClient(cfg *azureInputConfig, log *logp.Logger) (*container.Client, error) { + switch cfg.AuthType { + case AuthTypeConnectionString: + // Use connection string authentication + cloudConfig := getAzureCloud(cfg.AuthorityHost) + if cloudConfig.ActiveDirectoryAuthorityHost == "" { + cloudConfig = cloud.AzurePublic + } + + containerClient, err := container.NewClientFromConnectionString( + cfg.SAConnectionString, + cfg.SAContainer, + &container.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Cloud: cloudConfig, + }, + }, + ) + if err != nil { + return nil, fmt.Errorf("failed to create container client from connection string: %w", err) + } + return containerClient, nil + + case AuthTypeClientSecret: + credential, err := createCredential(cfg, log) + if err != nil { + return nil, err + } + if credential == nil { + return nil, fmt.Errorf("credential cannot be empty when auth_type is client_secret") + } + + // Build the storage account URL + storageAccountURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s", cfg.SAName, cfg.SAContainer) + containerClient, err := container.NewClient(storageAccountURL, credential, nil) + if err != nil { + return nil, fmt.Errorf("failed to create container client with credential: %w", err) + } + + log.Infow("successfully created container client with credential authentication", + "storage_account", cfg.SAName, + "container", cfg.SAContainer, + ) + + return containerClient, nil + + default: + return nil, fmt.Errorf("invalid auth_type: %s", cfg.AuthType) + } +} + diff --git a/x-pack/filebeat/input/azureeventhub/authenticator.go b/x-pack/filebeat/input/azureeventhub/authenticator.go deleted file mode 100644 index 33346a6e0e27..000000000000 --- a/x-pack/filebeat/input/azureeventhub/authenticator.go +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -//go:build !aix - -package azureeventhub - -import ( - "fmt" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" - "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" - - "github.com/elastic/elastic-agent-libs/logp" -) - -// createCredential creates a TokenCredential if needed based on the authentication type. -// Returns nil for connection_string authentication (which doesn't use credentials). -func createCredential(cfg *azureInputConfig, log *logp.Logger) (azcore.TokenCredential, error) { - switch cfg.AuthType { - case AuthTypeConnectionString: - // No credential needed for connection string authentication - return nil, nil - case AuthTypeClientSecret: - credConfig := authConfig{ - AuthType: cfg.AuthType, - TenantID: cfg.TenantID, - ClientID: cfg.ClientID, - ClientSecret: cfg.ClientSecret, - AuthorityHost: cfg.AuthorityHost, - } - credential, err := newClientSecretCredential(credConfig, log) - if err != nil { - return nil, fmt.Errorf("failed to create client secret credential: %w", err) - } - return credential, nil - default: - return nil, fmt.Errorf("invalid auth_type: %s", cfg.AuthType) - } -} - -// CreateEventHubConsumerClient creates an Event Hub consumer client -// using the configured authentication method from the provided config. -func CreateEventHubConsumerClient(cfg *azureInputConfig, log *logp.Logger) (*azeventhubs.ConsumerClient, error) { - switch cfg.AuthType { - case AuthTypeConnectionString: - // Use connection string authentication for Event Hub - // There is a mismatch between how the azure-eventhub input and the new - // Event Hub SDK expect the event hub name in the connection string. - // - // The azure-eventhub input was designed to work with the old Event Hub SDK, - // which worked using the event hub name in the connection string. - // - // The new Event Hub SDK expects clients to pass the event hub name as a - // parameter, or in the connection string as the entity path. - // - // We need to handle both cases. - connectionStringProperties, err := parseConnectionString(cfg.ConnectionString) - if err != nil { - return nil, fmt.Errorf("failed to parse connection string: %w", err) - } - - // Determine the event hub name to use - // If the connection string contains an entity path, we need to - // set the event hub name to an empty string. - // - // This is a requirement of the new Event Hub SDK. - // - // See: https://github.com/Azure/azure-sdk-for-go/blob/4ece3e50652223bba502f2b73e7f297de34a799c/sdk/messaging/azeventhubs/producer_client.go#L304-L306 - eventHubName := cfg.EventHubName - if connectionStringProperties.EntityPath != nil { - eventHubName = "" - } - - // Use connection string authentication - consumerClient, err := azeventhubs.NewConsumerClientFromConnectionString( - cfg.ConnectionString, - eventHubName, - cfg.ConsumerGroup, - nil, - ) - if err != nil { - return nil, fmt.Errorf("failed to create consumer client from connection string: %w", err) - } - return consumerClient, nil - - case AuthTypeClientSecret: - credential, err := createCredential(cfg, log) - if err != nil { - return nil, err - } - if credential == nil { - return nil, fmt.Errorf("credential cannot be empty when auth_type is client_secret") - } - - consumerClient, err := azeventhubs.NewConsumerClient( - cfg.EventHubNamespace, - cfg.EventHubName, - cfg.ConsumerGroup, - credential, - nil, - ) - if err != nil { - return nil, fmt.Errorf("failed to create consumer client with credential: %w", err) - } - - log.Infow("successfully created consumer client with credential authentication", - "namespace", cfg.EventHubNamespace, - "eventhub", cfg.EventHubName, - ) - - return consumerClient, nil - - default: - return nil, fmt.Errorf("invalid auth_type: %s", cfg.AuthType) - } -} - -// CreateStorageAccountContainerClient creates a Storage Account container client -// using the configured authentication method from the provided config. -func CreateStorageAccountContainerClient(cfg *azureInputConfig, log *logp.Logger) (*container.Client, error) { - switch cfg.AuthType { - case AuthTypeConnectionString: - // Use connection string authentication - cloudConfig := getAzureCloud(cfg.AuthorityHost) - if cloudConfig.ActiveDirectoryAuthorityHost == "" { - cloudConfig = cloud.AzurePublic - } - - containerClient, err := container.NewClientFromConnectionString( - cfg.SAConnectionString, - cfg.SAContainer, - &container.ClientOptions{ - ClientOptions: azcore.ClientOptions{ - Cloud: cloudConfig, - }, - }, - ) - if err != nil { - return nil, fmt.Errorf("failed to create container client from connection string: %w", err) - } - return containerClient, nil - - case AuthTypeClientSecret: - credential, err := createCredential(cfg, log) - if err != nil { - return nil, err - } - if credential == nil { - return nil, fmt.Errorf("credential cannot be empty when auth_type is client_secret") - } - - // Build the storage account URL - storageAccountURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s", cfg.SAName, cfg.SAContainer) - containerClient, err := container.NewClient(storageAccountURL, credential, nil) - if err != nil { - return nil, fmt.Errorf("failed to create container client with credential: %w", err) - } - - log.Infow("successfully created container client with credential authentication", - "storage_account", cfg.SAName, - "container", cfg.SAContainer, - ) - - return containerClient, nil - - default: - return nil, fmt.Errorf("invalid auth_type: %s", cfg.AuthType) - } -} diff --git a/x-pack/filebeat/input/azureeventhub/clients.go b/x-pack/filebeat/input/azureeventhub/clients.go deleted file mode 100644 index e6db1132e3d4..000000000000 --- a/x-pack/filebeat/input/azureeventhub/clients.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -//go:build !aix - -package azureeventhub - -import ( - "fmt" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" - "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" - - "github.com/elastic/elastic-agent-libs/logp" -) - -// eventHubClientConfig holds configuration for creating an Event Hub consumer client. -type eventHubClientConfig struct { - Namespace string - EventHubName string - ConsumerGroup string - Credential azcore.TokenCredential - ConnectionString string -} - -// newEventHubConsumerClient creates a new Event Hub consumer client using the provided credential or connection string. -func newEventHubConsumerClient(config eventHubClientConfig, authType string, log *logp.Logger) (*azeventhubs.ConsumerClient, error) { - if authType == AuthTypeConnectionString { - // Use connection string authentication for Event Hub - // There is a mismatch between how the azure-eventhub input and the new - // Event Hub SDK expect the event hub name in the connection string. - // - // The azure-eventhub input was designed to work with the old Event Hub SDK, - // which worked using the event hub name in the connection string. - // - // The new Event Hub SDK expects clients to pass the event hub name as a - // parameter, or in the connection string as the entity path. - // - // We need to handle both cases. - connectionStringProperties, err := parseConnectionString(config.ConnectionString) - if err != nil { - return nil, fmt.Errorf("failed to parse connection string: %w", err) - } - if connectionStringProperties.EntityPath != nil { - // If the connection string contains an entity path, we need to - // set the event hub name to an empty string. - // - // This is a requirement of the new Event Hub SDK. - // - // See: https://github.com/Azure/azure-sdk-for-go/blob/4ece3e50652223bba502f2b73e7f297de34a799c/sdk/messaging/azeventhubs/producer_client.go#L304-L306 - config.EventHubName = "" - } - - // Use connection string authentication - consumerClient, err := azeventhubs.NewConsumerClientFromConnectionString( - config.ConnectionString, - config.EventHubName, - config.ConsumerGroup, - nil, - ) - if err != nil { - return nil, fmt.Errorf("failed to create consumer client from connection string: %w", err) - } - return consumerClient, nil - } - - // Use credential authentication - if config.Credential == nil { - return nil, fmt.Errorf("credential cannot be empty when auth_type is not connection_string") - } - - consumerClient, err := azeventhubs.NewConsumerClient( - config.Namespace, - config.EventHubName, - config.ConsumerGroup, - config.Credential, - nil, - ) - if err != nil { - return nil, fmt.Errorf("failed to create consumer client with credential: %w", err) - } - - log.Infow("successfully created consumer client with credential authentication", - "namespace", config.Namespace, - "eventhub", config.EventHubName, - ) - - return consumerClient, nil -} - -// storageContainerClientConfig holds configuration for creating a Storage container client. -type storageContainerClientConfig struct { - StorageAccount string - Container string - Credential azcore.TokenCredential - ConnectionString string - Cloud cloud.Configuration -} - -// newStorageContainerClient creates a new Storage container client using the provided credential or connection string. -func newStorageContainerClient(config storageContainerClientConfig, authType string, log *logp.Logger) (*container.Client, error) { - if authType == AuthTypeConnectionString { - // Use connection string authentication - if config.Cloud.ActiveDirectoryAuthorityHost == "" { - config.Cloud = cloud.AzurePublic - } - containerClient, err := container.NewClientFromConnectionString( - config.ConnectionString, - config.Container, - &container.ClientOptions{ - ClientOptions: azcore.ClientOptions{ - Cloud: config.Cloud, - }, - }, - ) - if err != nil { - return nil, fmt.Errorf("failed to create container client from connection string: %w", err) - } - return containerClient, nil - } - - // Use credential authentication - if config.Credential == nil { - return nil, fmt.Errorf("credential cannot be empty when auth_type is not connection_string") - } - - // Build the storage account URL - storageAccountURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s", config.StorageAccount, config.Container) - containerClient, err := container.NewClient(storageAccountURL, config.Credential, nil) - if err != nil { - return nil, fmt.Errorf("failed to create container client with credential: %w", err) - } - - log.Infow("successfully created container client with credential authentication", - "storage_account", config.StorageAccount, - "container", config.Container, - ) - - return containerClient, nil -} From 9deb52880e1f41791d0c9b7b6b713a4f96687bbd Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 8 Dec 2025 23:24:04 -0700 Subject: [PATCH 18/21] use azureInputConfig instead of authConfig --- .../filebeat/filebeat-input-azure-eventhub.md | 8 ------ x-pack/filebeat/input/azureeventhub/auth.go | 28 +------------------ .../input/azureeventhub/client_secret.go | 2 +- .../input/azureeventhub/config_test.go | 2 +- 4 files changed, 3 insertions(+), 37 deletions(-) diff --git a/docs/reference/filebeat/filebeat-input-azure-eventhub.md b/docs/reference/filebeat/filebeat-input-azure-eventhub.md index 5fe7b1bed736..95207d40e26e 100644 --- a/docs/reference/filebeat/filebeat-input-azure-eventhub.md +++ b/docs/reference/filebeat/filebeat-input-azure-eventhub.md @@ -85,12 +85,10 @@ For detailed instructions on how to set up an Azure AD service principal and con The `azure-eventhub` input supports the following configuration options: - ### `eventhub` [_eventhub] The name of the eventhub users would like to read from, field required. - ### `consumer_group` [_consumer_group] Optional, we recommend using a dedicated consumer group for the azure input. Reusing consumer groups among non-related consumers can cause unexpected behavior and possibly lost events. @@ -118,7 +116,6 @@ This option is required if: A Blob Storage account is required to store, retrieve, or update the offset or state of the Event Hub messages. This means that after stopping Filebeat it can resume from where it stopped processing messages. - ### `eventhub_namespace` [_eventhub_namespace] ```{applies_to} @@ -164,27 +161,22 @@ Supported values: - `https://login.microsoftonline.us` (Azure Government) - `https://login.chinacloudapi.cn` (Azure China) - ### `storage_account` [_storage_account] The name of the storage account. Required. - ### `storage_account_key` [_storage_account_key] The storage account key, this key will be used to authorize access to data in your storage account, option is required. - ### `storage_account_container` [_storage_account_container] Optional, the name of the storage account container you would like to store the offset information in. - ### `resource_manager_endpoint` [_resource_manager_endpoint] Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. Ex: [https://management.chinacloudapi.cn/](https://management.chinacloudapi.cn/) for azure ChinaCloud [https://management.microsoftazure.de/](https://management.microsoftazure.de/) for azure GermanCloud [https://management.azure.com/](https://management.azure.com/) for azure PublicCloud [https://management.usgovcloudapi.net/](https://management.usgovcloudapi.net/) for azure USGovernmentCloud Users can also use this in case of a Hybrid Cloud model, where one may define their own endpoints. - ## Metrics [_metrics_3] This input exposes metrics under the [HTTP monitoring endpoint](/reference/filebeat/http-endpoint.md). These metrics are exposed under the `/inputs` path. They can be used to observe the activity of the input. diff --git a/x-pack/filebeat/input/azureeventhub/auth.go b/x-pack/filebeat/input/azureeventhub/auth.go index 9c2334c362b8..d4e981689ae4 100644 --- a/x-pack/filebeat/input/azureeventhub/auth.go +++ b/x-pack/filebeat/input/azureeventhub/auth.go @@ -24,24 +24,6 @@ const ( AuthTypeClientSecret string = "client_secret" ) -// authConfig represents the authentication configuration. -type authConfig struct { - // AuthType specifies the authentication method to use. - // If not specified, will be inferred from other fields: - // - If connection_string is provided, defaults to connection_string - // - Otherwise, defaults to client_secret - AuthType string - - // Connection string authentication - ConnectionString string - - // Client secret authentication - TenantID string - ClientID string - ClientSecret string - AuthorityHost string -} - // createCredential creates a TokenCredential if needed based on the authentication type. // Returns nil for connection_string authentication (which doesn't use credentials). func createCredential(cfg *azureInputConfig, log *logp.Logger) (azcore.TokenCredential, error) { @@ -50,14 +32,7 @@ func createCredential(cfg *azureInputConfig, log *logp.Logger) (azcore.TokenCred // No credential needed for connection string authentication return nil, nil case AuthTypeClientSecret: - credConfig := authConfig{ - AuthType: cfg.AuthType, - TenantID: cfg.TenantID, - ClientID: cfg.ClientID, - ClientSecret: cfg.ClientSecret, - AuthorityHost: cfg.AuthorityHost, - } - credential, err := newClientSecretCredential(credConfig, log) + credential, err := newClientSecretCredential(cfg, log) if err != nil { return nil, fmt.Errorf("failed to create client secret credential: %w", err) } @@ -196,4 +171,3 @@ func CreateStorageAccountContainerClient(cfg *azureInputConfig, log *logp.Logger return nil, fmt.Errorf("invalid auth_type: %s", cfg.AuthType) } } - diff --git a/x-pack/filebeat/input/azureeventhub/client_secret.go b/x-pack/filebeat/input/azureeventhub/client_secret.go index 86827e797860..6cafc21f17b7 100644 --- a/x-pack/filebeat/input/azureeventhub/client_secret.go +++ b/x-pack/filebeat/input/azureeventhub/client_secret.go @@ -17,7 +17,7 @@ import ( ) // newClientSecretCredential creates a new client secret credential(Oauth2). -func newClientSecretCredential(config authConfig, log *logp.Logger) (azcore.TokenCredential, error) { +func newClientSecretCredential(config *azureInputConfig, log *logp.Logger) (azcore.TokenCredential, error) { log = log.Named("client_secret") if config.TenantID == "" { diff --git a/x-pack/filebeat/input/azureeventhub/config_test.go b/x-pack/filebeat/input/azureeventhub/config_test.go index 14032a11d7ed..1a1f6b3e1381 100644 --- a/x-pack/filebeat/input/azureeventhub/config_test.go +++ b/x-pack/filebeat/input/azureeventhub/config_test.go @@ -422,7 +422,7 @@ func TestConnectionStringConfigValidation(t *testing.T) { return c }(), expectError: true, - errorMsg: "storage_account_connection_string is required when using connection_string authentication with processor v2", + errorMsg: "no storage account connection string configured (config: storage_account_connection_string)", }, { name: "invalid auth_type", From 55ce7e81411ceacc56f653b2dd88fe4fbaeb9456 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 10 Dec 2025 17:24:13 -0700 Subject: [PATCH 19/21] support diff domains for the storage account url --- .../filebeat/filebeat-input-azure-eventhub.md | 64 +++++++++++++++---- x-pack/filebeat/input/azureeventhub/auth.go | 10 ++- 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/docs/reference/filebeat/filebeat-input-azure-eventhub.md b/docs/reference/filebeat/filebeat-input-azure-eventhub.md index 95207d40e26e..5c59846682d5 100644 --- a/docs/reference/filebeat/filebeat-input-azure-eventhub.md +++ b/docs/reference/filebeat/filebeat-input-azure-eventhub.md @@ -12,38 +12,78 @@ Users can make use of the `azure-eventhub` input in order to read messages from Users can enable internal logs tracing for this input by setting the environment variable `BEATS_AZURE_EVENTHUB_INPUT_TRACING_ENABLED: true`. When enabled, this input will log additional information to the logs. Additional information includes partition ownership, blob lease information, and other internal state. -Example configuration using Shared Access Key authentication: +## Example configurations + +### Connection string authentication (processor v1) + +Example configuration using connection string authentication with processor v1: ```yaml filebeat.inputs: - type: azure-eventhub eventhub: "insights-operational-logs" - consumer_group: "test" - connection_string: "Endpoint=sb://....." - storage_account: "azureeph" - storage_account_key: "....." - storage_account_container: "" - resource_manager_endpoint: "" + consumer_group: "$Default" + # Connection string authentication (default) + connection_string: "Endpoint=sb://your-namespace.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=your-shared-access-key" + # Storage account configuration + storage_account: "your-storage-account" + storage_account_key: "your-storage-account-key" + storage_account_container: "" # Optional: defaults to filebeat- + processor_version: "v1" + # Optional: for non-public Azure clouds + # resource_manager_endpoint: "https://management.usgovcloudapi.net/" # For Azure Government ``` -{applies_to}`stack: ga 9.3.0` Example configuration using client secret authentication: +### Connection string authentication (processor v2) + +Example configuration using connection string authentication with processor v2: ```yaml filebeat.inputs: - type: azure-eventhub eventhub: "insights-operational-logs" - consumer_group: "test" + consumer_group: "$Default" + # Connection string authentication (default) + auth_type: "connection_string" + connection_string: "Endpoint=sb://your-namespace.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=your-shared-access-key" + # Storage account configuration + storage_account: "your-storage-account" + storage_account_connection_string: "DefaultEndpointsProtocol=https;AccountName=your-storage-account;AccountKey=your-storage-account-key;EndpointSuffix=core.windows.net" + storage_account_container: "" # Optional: defaults to filebeat- + processor_version: "v2" + # Optional: for non-public Azure clouds + # resource_manager_endpoint: "https://management.usgovcloudapi.net/" # For Azure Government +``` + +{applies_to}`stack: ga 9.3.0` ### Client secret authentication (processor v2) + +Example configuration using Azure Active Directory service principal authentication with processor v2: + +```yaml +filebeat.inputs: +- type: azure-eventhub + eventhub: "insights-operational-logs" + consumer_group: "$Default" + # Client secret authentication auth_type: "client_secret" - eventhub_namespace: "your-eventhub-namespace.servicebus.windows.net" + eventhub_namespace: "your-namespace.servicebus.windows.net" tenant_id: "your-tenant-id" client_id: "your-client-id" client_secret: "your-client-secret" + # Optional: defaults to Azure Public Cloud authority_host: "https://login.microsoftonline.com" - storage_account: "azureeph" - storage_account_container: "" + # For Azure Government, use: "https://login.microsoftonline.us" + # For Azure China, use: "https://login.chinacloudapi.cn" + # Storage account configuration + storage_account: "your-storage-account" + storage_account_container: "" # Optional: defaults to filebeat- processor_version: "v2" + # Optional: for non-public Azure clouds + # resource_manager_endpoint: "https://management.usgovcloudapi.net/" # For Azure Government ``` +**Note:** When using `client_secret` authentication, the service principal must have the appropriate Azure RBAC permissions. See [Required permissions](#_required_permissions) for details. + ## Authentication [_authentication] The azure-eventhub input supports multiple authentication methods. The [`auth_type` configuration option](#_auth_type) controls the authentication method used for both Event Hub and Storage Account. diff --git a/x-pack/filebeat/input/azureeventhub/auth.go b/x-pack/filebeat/input/azureeventhub/auth.go index d4e981689ae4..733f5260ffa3 100644 --- a/x-pack/filebeat/input/azureeventhub/auth.go +++ b/x-pack/filebeat/input/azureeventhub/auth.go @@ -153,8 +153,14 @@ func CreateStorageAccountContainerClient(cfg *azureInputConfig, log *logp.Logger return nil, fmt.Errorf("credential cannot be empty when auth_type is client_secret") } - // Build the storage account URL - storageAccountURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s", cfg.SAName, cfg.SAContainer) + // Get the Azure environment to determine the correct storage endpoint suffix + env, err := getAzureEnvironment(cfg.OverrideEnvironment) + if err != nil { + return nil, fmt.Errorf("failed to get azure environment: %w", err) + } + + // Build the storage account URL using the correct endpoint suffix for the cloud environment + storageAccountURL := fmt.Sprintf("https://%s.blob.%s/%s", cfg.SAName, env.StorageEndpointSuffix, cfg.SAContainer) containerClient, err := container.NewClient(storageAccountURL, credential, nil) if err != nil { return nil, fmt.Errorf("failed to create container client with credential: %w", err) From 3ee197153966a2095bdaa9713f4daa01f30c7659 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 10 Dec 2025 22:21:22 -0700 Subject: [PATCH 20/21] remove config.Validate() --- .../filebeat/filebeat-input-azure-eventhub.md | 26 +++---------------- x-pack/filebeat/input/azureeventhub/input.go | 5 ---- 2 files changed, 4 insertions(+), 27 deletions(-) diff --git a/docs/reference/filebeat/filebeat-input-azure-eventhub.md b/docs/reference/filebeat/filebeat-input-azure-eventhub.md index 5c59846682d5..66a78f7f7c25 100644 --- a/docs/reference/filebeat/filebeat-input-azure-eventhub.md +++ b/docs/reference/filebeat/filebeat-input-azure-eventhub.md @@ -23,15 +23,11 @@ filebeat.inputs: - type: azure-eventhub eventhub: "insights-operational-logs" consumer_group: "$Default" - # Connection string authentication (default) connection_string: "Endpoint=sb://your-namespace.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=your-shared-access-key" - # Storage account configuration storage_account: "your-storage-account" storage_account_key: "your-storage-account-key" - storage_account_container: "" # Optional: defaults to filebeat- + storage_account_container: "your-storage-container" processor_version: "v1" - # Optional: for non-public Azure clouds - # resource_manager_endpoint: "https://management.usgovcloudapi.net/" # For Azure Government ``` ### Connection string authentication (processor v2) @@ -43,16 +39,11 @@ filebeat.inputs: - type: azure-eventhub eventhub: "insights-operational-logs" consumer_group: "$Default" - # Connection string authentication (default) auth_type: "connection_string" connection_string: "Endpoint=sb://your-namespace.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=your-shared-access-key" - # Storage account configuration storage_account: "your-storage-account" storage_account_connection_string: "DefaultEndpointsProtocol=https;AccountName=your-storage-account;AccountKey=your-storage-account-key;EndpointSuffix=core.windows.net" - storage_account_container: "" # Optional: defaults to filebeat- - processor_version: "v2" - # Optional: for non-public Azure clouds - # resource_manager_endpoint: "https://management.usgovcloudapi.net/" # For Azure Government + storage_account_container: "your-storage-container" ``` {applies_to}`stack: ga 9.3.0` ### Client secret authentication (processor v2) @@ -64,22 +55,13 @@ filebeat.inputs: - type: azure-eventhub eventhub: "insights-operational-logs" consumer_group: "$Default" - # Client secret authentication auth_type: "client_secret" eventhub_namespace: "your-namespace.servicebus.windows.net" tenant_id: "your-tenant-id" client_id: "your-client-id" client_secret: "your-client-secret" - # Optional: defaults to Azure Public Cloud - authority_host: "https://login.microsoftonline.com" - # For Azure Government, use: "https://login.microsoftonline.us" - # For Azure China, use: "https://login.chinacloudapi.cn" - # Storage account configuration storage_account: "your-storage-account" - storage_account_container: "" # Optional: defaults to filebeat- - processor_version: "v2" - # Optional: for non-public Azure clouds - # resource_manager_endpoint: "https://management.usgovcloudapi.net/" # For Azure Government + storage_account_container: "your-storage-container" ``` **Note:** When using `client_secret` authentication, the service principal must have the appropriate Azure RBAC permissions. See [Required permissions](#_required_permissions) for details. @@ -95,7 +77,7 @@ The following authentication types are supported: - **`connection_string`** (default if `auth_type` is not specified): Uses Azure Event Hubs and Storage Account connection strings. - {applies_to}`stack: ga 9.3.0` **`client_secret`**: Uses Azure Active Directory service principal with client secret credentials. -### Required permissions +### Required permissions [_required_permissions] When using `client_secret` authentication, the service principal needs the following Azure RBAC permissions: diff --git a/x-pack/filebeat/input/azureeventhub/input.go b/x-pack/filebeat/input/azureeventhub/input.go index 01b010175c47..a6a38d0e5318 100644 --- a/x-pack/filebeat/input/azureeventhub/input.go +++ b/x-pack/filebeat/input/azureeventhub/input.go @@ -85,11 +85,6 @@ func (m *eventHubInputManager) Create(cfg *conf.C) (v2.Input, error) { config.checkUnsupportedParams(m.log) - // Validate config - if err := config.Validate(); err != nil { - return nil, fmt.Errorf("config validation failed: %w", err) - } - switch config.ProcessorVersion { case processorV1: return newEventHubInputV1(config, m.log) From 5602b8cab949c8abdfd37aca0bb26a8e55e574b1 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 10 Dec 2025 22:35:39 -0700 Subject: [PATCH 21/21] add extra line back --- x-pack/filebeat/input/azureeventhub/input.go | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/filebeat/input/azureeventhub/input.go b/x-pack/filebeat/input/azureeventhub/input.go index a6a38d0e5318..664123c564ea 100644 --- a/x-pack/filebeat/input/azureeventhub/input.go +++ b/x-pack/filebeat/input/azureeventhub/input.go @@ -71,6 +71,7 @@ func (m *eventHubInputManager) Init(unison.Group) error { // Create creates a new azure-eventhub input based on the configuration. func (m *eventHubInputManager) Create(cfg *conf.C) (v2.Input, error) { + // Register the logs tracer only if the environment variable is // set to avoid the overhead of the tracer in environments where // it's not needed.