[bearertokenauthextension] Load token lazily for gRPC AUTH to fix token refresh issue (open-telemetry#36749)

<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description
For the gRPC client AUTH part of bearertokenauthextension, there's a
token refresh issue which causes all telemetry rejected by server after
a period of time. The old token is always used even if the token is
refreshed in filesystem. The root cause is the token value is cached
during the `PerRPCCredentials()` instead of retrieving the latest. This
PR aims to fix it by loading the token lazily in `GetRequestMetadata()`.

<!-- Issue number (e.g. open-telemetry#1234) or full URL to issue, if applicable. -->
#### Link to tracking issue
N/A.

<!--Describe what testing was performed and which tests were added.-->
#### Testing
After I built a new docker image with this fix and tested in my
environment, the token refresh issue didn't happen again after the first
token expired.

<!--Describe the documentation added.-->
#### Documentation
N/A.
<!--Please delete paragraphs that you did not use before submitting.-->

Author: ms-hujia

.chloggen/bearertokenauthextension-fix-grpc-token-refresh.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: bearertokenauthextension
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Load token lazily for gRPC AUTH to fix token refresh issue
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [36749]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

extension/bearertokenauthextension/bearertokenauth.go

@@ -23,12 +23,12 @@ var _ credentials.PerRPCCredentials = (*PerRPCAuth)(nil)
 
 // PerRPCAuth is a gRPC credentials.PerRPCCredentials implementation that returns an 'authorization' header.
 type PerRPCAuth struct {
-	metadata map[string]string
+	auth *BearerTokenAuth
 }
 
 // GetRequestMetadata returns the request metadata to be used with the RPC.
 func (c *PerRPCAuth) GetRequestMetadata(context.Context, ...string) (map[string]string, error) {
-	return c.metadata, nil
+	return map[string]string{"authorization": c.auth.authorizationValue()}, nil
 }
 
 // RequireTransportSecurity always returns true for this implementation. Passing bearer tokens in plain-text connections is a bad idea.
@@ -171,7 +171,7 @@ func (b *BearerTokenAuth) Shutdown(_ context.Context) error {
 // PerRPCCredentials returns PerRPCAuth an implementation of credentials.PerRPCCredentials that
 func (b *BearerTokenAuth) PerRPCCredentials() (credentials.PerRPCCredentials, error) {
 	return &PerRPCAuth{
-		metadata: map[string]string{"authorization": b.authorizationValue()},
+		auth: b,
 	}, nil
 }
 

extension/bearertokenauthextension/bearertokenauth_test.go

@@ -18,15 +18,19 @@ import (
 )
 
 func TestPerRPCAuth(t *testing.T) {
-	metadata := map[string]string{
-		"authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...",
-	}
+	cfg := createDefaultConfig().(*Config)
+	cfg.BearerToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9..."
 
 	// test meta data is properly
-	perRPCAuth := &PerRPCAuth{metadata: metadata}
+	bauth := newBearerTokenAuth(cfg, nil)
+	assert.NotNil(t, bauth)
+	perRPCAuth := &PerRPCAuth{auth: bauth}
 	md, err := perRPCAuth.GetRequestMetadata(context.Background())
 	assert.NoError(t, err)
-	assert.Equal(t, md, metadata)
+	expectedMetadata := map[string]string{
+		"authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...",
+	}
+	assert.Equal(t, expectedMetadata, md)
 
 	// always true
 	ok := perRPCAuth.RequireTransportSecurity()
@@ -202,6 +206,34 @@ func TestBearerTokenFileContentUpdate(t *testing.T) {
 	assert.Equal(t, authHeaderValue, fmt.Sprintf("%s %s", scheme, string(token)))
 }
 
+func TestBearerTokenUpdateForGrpc(t *testing.T) {
+	// prepare
+	cfg := createDefaultConfig().(*Config)
+	cfg.BearerToken = "1234"
+
+	bauth := newBearerTokenAuth(cfg, zaptest.NewLogger(t))
+	assert.NotNil(t, bauth)
+
+	perRPCAuth, err := bauth.PerRPCCredentials()
+	assert.NoError(t, err)
+
+	ctx := context.Background()
+	assert.NoError(t, bauth.Start(ctx, componenttest.NewNopHost()))
+
+	// initial token, OK
+	md, err := perRPCAuth.GetRequestMetadata(context.Background())
+	assert.NoError(t, err)
+	assert.Equal(t, map[string]string{"authorization": "Bearer " + "1234"}, md)
+
+	// update the token
+	bauth.setAuthorizationValue("5678")
+	md, err = perRPCAuth.GetRequestMetadata(context.Background())
+	assert.NoError(t, err)
+	assert.Equal(t, map[string]string{"authorization": "Bearer " + "5678"}, md)
+
+	assert.NoError(t, bauth.Shutdown(context.Background()))
+}
+
 func TestBearerServerAuthenticateWithScheme(t *testing.T) {
 	const token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9..." // #nosec
 	cfg := createDefaultConfig().(*Config)