Fix RespondWorkflowTaskFailed workflow termination (temporalio#8725)

## What changed?
- Properly terminate workflow in RespondWorkflowTaskFailed to flush
buffered events.

## Why?
- `workflow.TerminateWorkflow` util func performs failing started
workflow task, flushing buffered event and terminating workflow, but it
won't work properly if workflow task has already been failed.

## How did you test it?
- [ ] built
- [ ] run locally and tested manually
- [ ] covered by existing tests
- [ ] added new unit test(s)
- [x] added new functional test(s)

Author: yycptt

service/history/api/respondworkflowtaskfailed/api.go

@@ -75,19 +75,6 @@ func Invoke(
 				}, nil
 			}
 
-			if _, err := mutableState.AddWorkflowTaskFailedEvent(
-				workflowTask,
-				request.GetCause(),
-				request.GetFailure(),
-				request.GetIdentity(),
-				request.GetWorkerVersion(),
-				request.GetBinaryChecksum(),
-				"",
-				"",
-				0); err != nil {
-				return nil, err
-			}
-
 			metrics.FailedWorkflowTasksCounter.With(shardContext.GetMetricsHandler()).Record(
 				1,
 				metrics.OperationTag(metrics.HistoryRespondWorkflowTaskFailedScope),
@@ -112,6 +99,21 @@ func Invoke(
 				return api.UpdateWorkflowTerminate, nil
 			}
 
+			if _, err := mutableState.AddWorkflowTaskFailedEvent(
+				workflowTask,
+				request.GetCause(),
+				request.GetFailure(),
+				request.GetIdentity(),
+				//nolint:staticcheck
+				request.GetWorkerVersion(),
+				//nolint:staticcheck
+				request.GetBinaryChecksum(),
+				"",
+				"",
+				0); err != nil {
+				return nil, err
+			}
+
 			// TODO (alex-update): if it was speculative WT that failed, and there is nothing but pending updates,
 			//  new WT also should be create as speculative (or not?). Currently, it will be recreated as normal WT.
 			return &api.UpdateWorkflowAction{

tests/workflow_test.go

@@ -1021,6 +1021,17 @@ func (s *WorkflowTestSuite) TestTerminateWorkflowOnMessageTooLargeFailure() {
 	s.Logger.Info("PollWorkflowTaskQueue", tag.Error(err))
 	s.NoError(err)
 
+	// send a signal to test buffered event case.
+	_, err = s.FrontendClient().SignalWorkflowExecution(testContext, &workflowservice.SignalWorkflowExecutionRequest{
+		Namespace: s.Namespace().String(),
+		WorkflowExecution: &commonpb.WorkflowExecution{
+			WorkflowId: tv.WorkflowID(),
+			RunId:      we.RunId,
+		},
+		SignalName: "buffered-signal",
+	})
+	s.NoError(err)
+
 	// respond workflow task as failed with grpc message too large error
 	_, err = s.FrontendClient().RespondWorkflowTaskFailed(testContext, &workflowservice.RespondWorkflowTaskFailedRequest{
 		Namespace: s.Namespace().String(),
@@ -1034,12 +1045,14 @@ func (s *WorkflowTestSuite) TestTerminateWorkflowOnMessageTooLargeFailure() {
 		RunId:      we.RunId,
 	})
 
+	// verify that workflow is terminated and buffered signal is flushed
 	s.EqualHistoryEvents(`
   1 WorkflowExecutionStarted
   2 WorkflowTaskScheduled
   3 WorkflowTaskStarted
   4 WorkflowTaskFailed
-  5 WorkflowExecutionTerminated`, // verify that workflow is terminated
+  5 WorkflowExecutionSignaled
+  6 WorkflowExecutionTerminated`,
 		historyEvents)
 }