Truncate service error message (#10136)

## What changed?
Truncate service error message to prevent grpc-go to return `RST_STREAM`
instead of the service error.

## Why?
grpc-go returns `RST_STREAM` error when the error message is too long,
masking the actual service error returned by Temporal.

## How did you test it?
- [x] built
- [ ] run locally and tested manually
- [ ] covered by existing tests
- [x] added new unit test(s)
- [ ] added new functional test(s)

## Potential risks

Author: rodrigozhou

common/rpc/interceptor/service_error_interceptor.go

@@ -6,7 +6,14 @@ import (
 
 	"go.temporal.io/api/serviceerror"
 	"go.temporal.io/server/common/persistence/serialization"
+	"go.temporal.io/server/common/util"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/status"
+)
+
+const (
+	maxMessageLength = 4000
+	truncatedSuffix  = "... <truncated>"
 )
 
 func ServiceErrorInterceptor(
@@ -24,5 +31,14 @@ func ServiceErrorInterceptor(
 	if errors.As(err, &deserializationError) || errors.As(err, &serializationError) {
 		err = serviceerror.NewDataLoss(err.Error())
 	}
-	return resp, serviceerror.ToStatus(err).Err()
+
+	// truncate message length if needed
+	st := serviceerror.ToStatus(err)
+	if len(st.Message()) > maxMessageLength {
+		p := st.Proto()
+		p.Message = util.TruncateUTF8(p.Message, maxMessageLength-len(truncatedSuffix)) + truncatedSuffix
+		st = status.FromProto(p)
+	}
+
+	return resp, st.Err()
 }

common/rpc/interceptor/service_error_interceptor_test.go

@@ -2,9 +2,10 @@ package interceptor
 
 import (
 	"context"
+	"strings"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	enumspb "go.temporal.io/api/enums/v1"
 	"go.temporal.io/api/serviceerror"
 	"go.temporal.io/server/common/persistence/serialization"
@@ -28,24 +29,24 @@ func (e *ErrorWithoutStatus) Error() string {
 // Error returns string message.
 func TestServiceErrorInterceptorUnknown(t *testing.T) {
 
-	_, err := ServiceErrorInterceptor(context.Background(), nil, nil,
+	_, err := ServiceErrorInterceptor(t.Context(), nil, nil,
 		func(ctx context.Context, req any) (any, error) {
 			return nil, status.Error(codes.InvalidArgument, "invalid argument")
 		})
 
-	assert.Error(t, err)
-	assert.Equal(t, codes.InvalidArgument, status.Code(err))
+	require.Error(t, err)
+	require.Equal(t, codes.InvalidArgument, status.Code(err))
 
-	_, err = ServiceErrorInterceptor(context.Background(), nil, nil,
+	_, err = ServiceErrorInterceptor(t.Context(), nil, nil,
 		func(ctx context.Context, req any) (any, error) {
 			errWithoutStatus := &ErrorWithoutStatus{
 				Message: "unknown error without status",
 			}
 			return nil, errWithoutStatus
 		})
 
-	assert.Error(t, err)
-	assert.Equal(t, codes.Unknown, status.Code(err))
+	require.Error(t, err)
+	require.Equal(t, codes.Unknown, status.Code(err))
 }
 
 func TestServiceErrorInterceptorSer(t *testing.T) {
@@ -54,10 +55,87 @@ func TestServiceErrorInterceptorSer(t *testing.T) {
 		serialization.NewSerializationError(enumspb.ENCODING_TYPE_PROTO3, nil),
 	}
 	for _, inErr := range serErrors {
-		_, err := ServiceErrorInterceptor(context.Background(), nil, nil,
+		_, err := ServiceErrorInterceptor(t.Context(), nil, nil,
 			func(_ context.Context, _ any) (any, error) {
 				return nil, inErr
 			})
-		assert.Equal(t, serviceerror.ToStatus(err).Code(), codes.DataLoss)
+		require.Equal(t, codes.DataLoss, serviceerror.ToStatus(err).Code())
 	}
 }
+
+func TestServiceErrorInterceptorTruncation(t *testing.T) {
+	t.Run("nil error is not affected", func(t *testing.T) {
+		_, err := ServiceErrorInterceptor(t.Context(), nil, nil,
+			func(_ context.Context, _ any) (any, error) {
+				return "ok", nil
+			})
+		require.NoError(t, err)
+	})
+
+	t.Run("short message is not truncated", func(t *testing.T) {
+		msg := "short error"
+		_, err := ServiceErrorInterceptor(t.Context(), nil, nil,
+			func(_ context.Context, _ any) (any, error) {
+				return nil, serviceerror.NewInternal(msg)
+			})
+		require.Error(t, err)
+		st := status.Convert(err)
+		require.Equal(t, msg, st.Message())
+	})
+
+	t.Run("message at exact limit is not truncated", func(t *testing.T) {
+		msg := strings.Repeat("a", maxMessageLength)
+		_, err := ServiceErrorInterceptor(t.Context(), nil, nil,
+			func(_ context.Context, _ any) (any, error) {
+				return nil, serviceerror.NewInternal(msg)
+			})
+		require.Error(t, err)
+		st := status.Convert(err)
+		require.Equal(t, msg, st.Message())
+	})
+
+	t.Run("message over limit is truncated", func(t *testing.T) {
+		msg := strings.Repeat("a", maxMessageLength+100)
+		_, err := ServiceErrorInterceptor(t.Context(), nil, nil,
+			func(_ context.Context, _ any) (any, error) {
+				return nil, serviceerror.NewInternal(msg)
+			})
+		require.Error(t, err)
+		st := status.Convert(err)
+		require.LessOrEqual(t, len(st.Message()), maxMessageLength)
+		require.True(t, strings.HasSuffix(st.Message(), truncatedSuffix))
+	})
+
+	t.Run("truncation preserves error code", func(t *testing.T) {
+		msg := strings.Repeat("x", maxMessageLength+500)
+		_, err := ServiceErrorInterceptor(t.Context(), nil, nil,
+			func(_ context.Context, _ any) (any, error) {
+				return nil, serviceerror.NewNotFound(msg)
+			})
+		require.Error(t, err)
+		require.Equal(t, codes.NotFound, status.Code(err))
+	})
+
+	t.Run("truncation respects multi-byte UTF-8 boundary", func(t *testing.T) {
+		// Fill up to near the limit with multi-byte characters (3 bytes each for '€')
+		// then push over the limit so truncation must split within the repeated chars.
+		euroCount := maxMessageLength / len("€") // each '€' is 3 bytes
+		msg := strings.Repeat("€", euroCount+100)
+		_, err := ServiceErrorInterceptor(t.Context(), nil, nil,
+			func(_ context.Context, _ any) (any, error) {
+				return nil, serviceerror.NewInternal(msg)
+			})
+		require.Error(t, err)
+		st := status.Convert(err)
+		require.LessOrEqual(t, len(st.Message()), maxMessageLength)
+		require.True(t, strings.HasSuffix(st.Message(), truncatedSuffix))
+		// Verify the truncated body (without suffix) is valid UTF-8 by checking
+		// that no partial rune was left behind — the full message should be valid.
+		require.True(t, strings.HasSuffix(st.Message(), truncatedSuffix))
+		body := strings.TrimSuffix(st.Message(), truncatedSuffix)
+		// Every character in body should be '€' (no partial runes).
+		for _, r := range body {
+			require.Equal(t, '€', r)
+		}
+	})
+}