[security] Remove lock that guards the tsi_frame_protector. (grpc#39356)

This is believed to be safe following grpc#39308, but we are guarding the change with an experiment to err on the safe side. The new concurrency expectations are documented in grpc#39351.

Closes grpc#39356

COPYBARA_INTEGRATE_REVIEW=grpc#39356 from matthewstevenson88:remove-mu 40c0e02
PiperOrigin-RevId: 751505414

Author: matthewstevenson88

BUILD

@@ -2238,6 +2238,7 @@ grpc_cc_library(
         "//src/core:context",
         "//src/core:error",
         "//src/core:event_engine_memory_allocator",
+        "//src/core:experiments",
         "//src/core:gpr_atm",
         "//src/core:handshaker_factory",
         "//src/core:handshaker_registry",

bazel/experiments.bzl

@@ -39,6 +39,7 @@
 #include "absl/strings/string_view.h"
 #include "src/core/handshaker/security/secure_endpoint.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/experiments/experiments.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/error.h"
@@ -288,11 +289,17 @@ static void on_read(void* user_data, grpc_error_handle error) {
           size_t unprotected_buffer_size_written =
               static_cast<size_t>(end - cur);
           size_t processed_message_size = message_size;
-          gpr_mu_lock(&ep->protector_mu);
-          result = tsi_frame_protector_unprotect(
-              ep->protector, message_bytes, &processed_message_size, cur,
-              &unprotected_buffer_size_written);
-          gpr_mu_unlock(&ep->protector_mu);
+          if (grpc_core::IsTsiFrameProtectorWithoutLocksEnabled()) {
+            result = tsi_frame_protector_unprotect(
+                ep->protector, message_bytes, &processed_message_size, cur,
+                &unprotected_buffer_size_written);
+          } else {
+            gpr_mu_lock(&ep->protector_mu);
+            result = tsi_frame_protector_unprotect(
+                ep->protector, message_bytes, &processed_message_size, cur,
+                &unprotected_buffer_size_written);
+            gpr_mu_unlock(&ep->protector_mu);
+          }
           if (result != TSI_OK) {
             LOG(ERROR) << "Decryption error: " << tsi_result_to_string(result);
             break;
@@ -443,11 +450,17 @@ static void endpoint_write(grpc_endpoint* secure_ep, grpc_slice_buffer* slices,
         while (message_size > 0) {
           size_t protected_buffer_size_to_send = static_cast<size_t>(end - cur);
           size_t processed_message_size = message_size;
-          gpr_mu_lock(&ep->protector_mu);
-          result = tsi_frame_protector_protect(ep->protector, message_bytes,
-                                               &processed_message_size, cur,
-                                               &protected_buffer_size_to_send);
-          gpr_mu_unlock(&ep->protector_mu);
+          if (grpc_core::IsTsiFrameProtectorWithoutLocksEnabled()) {
+            result = tsi_frame_protector_protect(
+                ep->protector, message_bytes, &processed_message_size, cur,
+                &protected_buffer_size_to_send);
+          } else {
+            gpr_mu_lock(&ep->protector_mu);
+            result = tsi_frame_protector_protect(
+                ep->protector, message_bytes, &processed_message_size, cur,
+                &protected_buffer_size_to_send);
+            gpr_mu_unlock(&ep->protector_mu);
+          }
           if (result != TSI_OK) {
             LOG(ERROR) << "Encryption error: " << tsi_result_to_string(result);
             break;
@@ -466,11 +479,17 @@ static void endpoint_write(grpc_endpoint* secure_ep, grpc_slice_buffer* slices,
         size_t still_pending_size;
         do {
           size_t protected_buffer_size_to_send = static_cast<size_t>(end - cur);
-          gpr_mu_lock(&ep->protector_mu);
-          result = tsi_frame_protector_protect_flush(
-              ep->protector, cur, &protected_buffer_size_to_send,
-              &still_pending_size);
-          gpr_mu_unlock(&ep->protector_mu);
+          if (grpc_core::IsTsiFrameProtectorWithoutLocksEnabled()) {
+            result = tsi_frame_protector_protect_flush(
+                ep->protector, cur, &protected_buffer_size_to_send,
+                &still_pending_size);
+          } else {
+            gpr_mu_lock(&ep->protector_mu);
+            result = tsi_frame_protector_protect_flush(
+                ep->protector, cur, &protected_buffer_size_to_send,
+                &still_pending_size);
+            gpr_mu_unlock(&ep->protector_mu);
+          }
           if (result != TSI_OK) break;
           cur += protected_buffer_size_to_send;
           if (cur == end) {

src/core/handshaker/security/legacy_secure_endpoint.cc

@@ -41,6 +41,7 @@
 #include "absl/status/status.h"
 #include "absl/strings/string_view.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/experiments/experiments.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/error.h"
@@ -218,11 +219,17 @@ class FrameProtector : public RefCounted<FrameProtector> {
           size_t unprotected_buffer_size_written =
               static_cast<size_t>(end - cur);
           size_t processed_message_size = message_size;
-          protector_mu_.Lock();
-          result = tsi_frame_protector_unprotect(
-              protector_, message_bytes, &processed_message_size, cur,
-              &unprotected_buffer_size_written);
-          protector_mu_.Unlock();
+          if (IsTsiFrameProtectorWithoutLocksEnabled()) {
+            result = tsi_frame_protector_unprotect(
+                protector_, message_bytes, &processed_message_size, cur,
+                &unprotected_buffer_size_written);
+          } else {
+            protector_mu_.Lock();
+            result = tsi_frame_protector_unprotect(
+                protector_, message_bytes, &processed_message_size, cur,
+                &unprotected_buffer_size_written);
+            protector_mu_.Unlock();
+          }
           if (result != TSI_OK) {
             LOG(ERROR) << "Decryption error: " << tsi_result_to_string(result);
             break;
@@ -343,11 +350,17 @@ class FrameProtector : public RefCounted<FrameProtector> {
         while (message_size > 0) {
           size_t protected_buffer_size_to_send = static_cast<size_t>(end - cur);
           size_t processed_message_size = message_size;
-          protector_mu_.Lock();
-          result = tsi_frame_protector_protect(protector_, message_bytes,
-                                               &processed_message_size, cur,
-                                               &protected_buffer_size_to_send);
-          protector_mu_.Unlock();
+          if (IsTsiFrameProtectorWithoutLocksEnabled()) {
+            result = tsi_frame_protector_protect(
+                protector_, message_bytes, &processed_message_size, cur,
+                &protected_buffer_size_to_send);
+          } else {
+            protector_mu_.Lock();
+            result = tsi_frame_protector_protect(
+                protector_, message_bytes, &processed_message_size, cur,
+                &protected_buffer_size_to_send);
+            protector_mu_.Unlock();
+          }
           if (result != TSI_OK) {
             LOG(ERROR) << "Encryption error: " << tsi_result_to_string(result);
             break;
@@ -366,11 +379,17 @@ class FrameProtector : public RefCounted<FrameProtector> {
         size_t still_pending_size;
         do {
           size_t protected_buffer_size_to_send = static_cast<size_t>(end - cur);
-          protector_mu_.Lock();
-          result = tsi_frame_protector_protect_flush(
-              protector_, cur, &protected_buffer_size_to_send,
-              &still_pending_size);
-          protector_mu_.Unlock();
+          if (IsTsiFrameProtectorWithoutLocksEnabled()) {
+            result = tsi_frame_protector_protect_flush(
+                protector_, cur, &protected_buffer_size_to_send,
+                &still_pending_size);
+          } else {
+            protector_mu_.Lock();
+            result = tsi_frame_protector_protect_flush(
+                protector_, cur, &protected_buffer_size_to_send,
+                &still_pending_size);
+            protector_mu_.Unlock();
+          }
           if (result != TSI_OK) break;
           cur += protected_buffer_size_to_send;
           if (cur == end) {

src/core/handshaker/security/secure_endpoint.cc

@@ -287,6 +287,11 @@
   expiry: 2025/06/01
   owner: [email protected]
   test_tags: ["endpoint_test", "flow_control_test"]
+- name: tsi_frame_protector_without_locks
+  description: Do not hold locks while using the tsi_frame_protector.
+  expiry: 2025/09/03
+  owner: [email protected]
+  test_tags: []
 - name: unconstrained_max_quota_buffer_size
   description: Discard the cap on the max free pool size for one memory allocator
   expiry: 2025/09/03

src/core/lib/experiments/experiments.cc

@@ -104,5 +104,7 @@
   default: false
 - name: tcp_rcv_lowat
   default: false
+- name: tsi_frame_protector_without_locks
+  default: false
 - name: unconstrained_max_quota_buffer_size
   default: false