Allow more retries for matching client polls (temporalio#8155)

## What changed?
Allow frontend->matching poll requests to retry up to their context
timeout instead of just once.

## Why?
On matching service deployments, a busy new matching node may hit its
persistence rps limit trying to acquire new task queues and be unable to
accept polls.

## How did you test it?
- [x] built
- [ ] run locally and tested manually
- [ ] covered by existing tests
- [ ] added new unit test(s)
- [ ] added new functional test(s)

Author: dnr

client/matching/retryable_client.go

@@ -10,14 +10,21 @@ var _ matchingservice.MatchingServiceClient = (*retryableClient)(nil)
 type retryableClient struct {
 	client      matchingservice.MatchingServiceClient
 	policy      backoff.RetryPolicy
+	pollPolicy  backoff.RetryPolicy
 	isRetryable backoff.IsRetryable
 }
 
 // NewRetryableClient creates a new instance of matchingservice.MatchingServiceClient with retry policy
-func NewRetryableClient(client matchingservice.MatchingServiceClient, policy backoff.RetryPolicy, isRetryable backoff.IsRetryable) matchingservice.MatchingServiceClient {
+func NewRetryableClient(
+	client matchingservice.MatchingServiceClient,
+	policy,
+	pollPolicy backoff.RetryPolicy,
+	isRetryable backoff.IsRetryable,
+) matchingservice.MatchingServiceClient {
 	return &retryableClient{
 		client:      client,
 		policy:      policy,
+		pollPolicy:  pollPolicy,
 		isRetryable: isRetryable,
 	}
 }

client/matching/retryable_client_gen.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"cmp"
 	"flag"
 	"fmt"
 	"io"
@@ -72,6 +73,11 @@ var (
 	largeTimeoutContext = map[string]bool{
 		"client.admin.GetReplicationMessages": true,
 	}
+	longPollRetryPolicy = map[string]string{
+		"retryableClient.matching.PollWorkflowTaskQueue": "pollPolicy",
+		"retryableClient.matching.PollActivityTaskQueue": "pollPolicy",
+		"retryableClient.matching.PollNexusTaskQueue":    "pollPolicy",
+	}
 	ignoreMethod = map[string]bool{
 		// TODO stream APIs are not supported. do not generate.
 		"client.admin.StreamWorkflowReplicationMessages":          true,
@@ -387,6 +393,7 @@ func writeTemplatedMethod(w io.Writer, service service, impl string, m reflect.M
 		"RequestType":  reqType.String(),
 		"ResponseType": respType.String(),
 		"MetricPrefix": fmt.Sprintf("%s%sClient", strings.ToUpper(service.name[:1]), service.name[1:]),
+		"RetryPolicy":  cmp.Or(longPollRetryPolicy[key], "policy"),
 	}
 	if longPollContext[key] {
 		fields["LongPoll"] = "LongPoll"
@@ -581,7 +588,7 @@ func (c *retryableClient) {{.Method}}(
 		resp, err = c.client.{{.Method}}(ctx, request, opts...)
 		return err
 	}
-	err := backoff.ThrottleRetryContext(ctx, op, c.policy, c.isRetryable)
+	err := backoff.ThrottleRetryContext(ctx, op, c.{{.RetryPolicy}}, c.isRetryable)
 	return resp, err
 }
 `)

cmd/tools/genrpcwrappers/main.go

@@ -272,6 +272,7 @@ func MatchingClientProvider(matchingRawClient MatchingRawClient) MatchingClient
 	return matching.NewRetryableClient(
 		matchingRawClient,
 		common.CreateMatchingClientRetryPolicy(),
+		common.CreateMatchingClientLongPollRetryPolicy(),
 		common.IsServiceClientTransientError,
 	)
 }

common/resource/fx.go

@@ -178,6 +178,12 @@ func CreateMatchingClientRetryPolicy() backoff.RetryPolicy {
 		WithMaximumAttempts(matchingClientRetryMaxAttempts)
 }
 
+// CreateMatchingClientLongPollRetryPolicy creates a retry policy for poll calls to matching service
+func CreateMatchingClientLongPollRetryPolicy() backoff.RetryPolicy {
+	// no maximum attempts, using default expiration interval of 1 minute
+	return backoff.NewExponentialRetryPolicy(matchingClientRetryInitialInterval)
+}
+
 // CreateFrontendHandlerRetryPolicy creates a retry policy for calls to frontend service
 func CreateFrontendHandlerRetryPolicy() backoff.RetryPolicy {
 	return backoff.NewExponentialRetryPolicy(frontendHandlerRetryInitialInterval).