Remove linked project connection error metric

The message logged in RemoteConnectionStrategy
has proven sufficient for use in constructing
the dashboard and implementing an alert.
This PR is for removing the metric code to avoid
emitting extra unused APM data.

Relates: ES-12696

Author: JeremyDahlgren

server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java

@@ -65,7 +65,6 @@ public final class RemoteClusterService extends RemoteClusterAware
     private static final Logger logger = LogManager.getLogger(RemoteClusterService.class);
 
     public static final String REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME = "cluster:internal/remote_cluster/handshake";
-    public static final String CONNECTION_ATTEMPT_FAILURES_COUNTER_NAME = "es.projects.linked.connections.error.total";
 
     private final boolean isRemoteClusterClient;
     private final boolean isSearchNode;
@@ -102,13 +101,6 @@ public boolean isRemoteClusterServerEnabled() {
          *  the functionality to do it the right way is not yet ready -- replace this code when it's ready.
          */
         this.crossProjectEnabled = settings.getAsBoolean("serverless.cross_project.enabled", false);
-        // Since this counter may never be incremented we need to force an initial observed value of zero via add(0) so the metric will be
-        // in the mappings of the indices of the APM data stream, otherwise we will encounter 'not found' errors in dashboards and alerts.
-        // See observability-dev issue #3042.
-        transportService.getTelemetryProvider()
-            .getMeterRegistry()
-            .registerLongUpDownCounter(CONNECTION_ATTEMPT_FAILURES_COUNTER_NAME, "linked project connection attempt failure count", "count")
-            .add(0);
     }
 
     public RemoteClusterCredentialsManager getRemoteClusterCredentialsManager() {

server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java

@@ -21,8 +21,6 @@
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.telemetry.TelemetryProvider;
-import org.elasticsearch.telemetry.metric.LongUpDownCounter;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.Closeable;
@@ -33,7 +31,6 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -79,10 +76,6 @@ enum ConnectionAttempt {
         reconnect
     }
 
-    static final String linkedProjectIdLabel = "es_linked_project_id";
-    static final String linkedProjectAliasLabel = "es_linked_project_alias";
-    static final String connectionAtemptLabel = "es_linked_project_attempt";
-
     private final int maxPendingConnectionListeners;
 
     protected final Logger logger = LogManager.getLogger(getClass());
@@ -91,7 +84,6 @@ enum ConnectionAttempt {
     private final Object mutex = new Object();
     private List<ActionListener<Void>> listeners = new ArrayList<>();
     private final AtomicBoolean initialConnectionAttempted = new AtomicBoolean(false);
-    private final LongUpDownCounter connectionAttemptFailures;
 
     protected final TransportService transportService;
     protected final RemoteConnectionManager connectionManager;
@@ -106,17 +98,9 @@ enum ConnectionAttempt {
         this.transportService = transportService;
         this.connectionManager = connectionManager;
         this.maxPendingConnectionListeners = config.maxPendingConnectionListeners();
-        this.connectionAttemptFailures = lookupConnectionFailureMetric(transportService.getTelemetryProvider());
         connectionManager.addListener(this);
     }
 
-    private LongUpDownCounter lookupConnectionFailureMetric(TelemetryProvider telemetryProvider) {
-        final var meterRegistry = telemetryProvider == null ? null : telemetryProvider.getMeterRegistry();
-        return meterRegistry == null
-            ? null
-            : meterRegistry.getLongUpDownCounter(RemoteClusterService.CONNECTION_ATTEMPT_FAILURES_COUNTER_NAME);
-    }
-
     static ConnectionProfile buildConnectionProfile(LinkedProjectConfig config, String transportProfile) {
         ConnectionProfile.Builder builder = new ConnectionProfile.Builder().setConnectTimeout(config.transportConnectTimeout())
             .setHandshakeTimeout(config.transportConnectTimeout())
@@ -242,21 +226,7 @@ private void connectionAttemptCompleted(@Nullable Exception e) {
         if (e == null) {
             logger.debug(msgSupplier);
         } else {
-            final var isClosed = isClosed();
-            logger.log(isClosed ? Level.DEBUG : Level.WARN, msgSupplier, e);
-            if (isClosed == false && connectionAttemptFailures != null) {
-                connectionAttemptFailures.add(
-                    1,
-                    Map.of(
-                        linkedProjectIdLabel,
-                        linkedProjectId.toString(),
-                        linkedProjectAliasLabel,
-                        clusterAlias,
-                        connectionAtemptLabel,
-                        (isInitialAttempt ? ConnectionAttempt.initial : ConnectionAttempt.reconnect).toString()
-                    )
-                );
-            }
+            logger.log(isClosed() ? Level.DEBUG : Level.WARN, msgSupplier, e);
         }
     }
 

server/src/test/java/org/elasticsearch/transport/RemoteConnectionStrategyTests.java

@@ -20,8 +20,6 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.telemetry.InstrumentType;
-import org.elasticsearch.telemetry.RecordingMeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.EnumSerializationTestUtils;
 import org.elasticsearch.test.MockLog;
@@ -32,7 +30,6 @@
 
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 
 import static org.elasticsearch.test.MockLog.assertThatLogger;
@@ -42,7 +39,6 @@
 import static org.elasticsearch.transport.RemoteClusterSettings.SniffConnectionStrategySettings.REMOTE_CLUSTER_SEEDS;
 import static org.elasticsearch.transport.RemoteClusterSettings.toConfig;
 import static org.elasticsearch.transport.RemoteConnectionStrategy.buildConnectionProfile;
-import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
 
 public class RemoteConnectionStrategyTests extends ESTestCase {
@@ -219,11 +215,6 @@ public void testConnectionAttemptMetricsAndLogging() {
                 new ClusterConnectionManager(TestProfiles.LIGHT_PROFILE, mock(Transport.class), threadContext)
             )
         ) {
-            assert transportService.getTelemetryProvider() != null;
-            final var meterRegistry = transportService.getTelemetryProvider().getMeterRegistry();
-            assert meterRegistry instanceof RecordingMeterRegistry;
-            final var metricRecorder = ((RecordingMeterRegistry) meterRegistry).getRecorder();
-
             for (boolean shouldConnectFail : new boolean[] { true, false }) {
                 for (boolean isInitialConnectAttempt : new boolean[] { true, false }) {
                     final var strategy = new FakeConnectionStrategy(
@@ -265,34 +256,12 @@ public void testConnectionAttemptMetricsAndLogging() {
                             expectedLogMessage
                         )
                     );
-                    if (shouldConnectFail) {
-                        metricRecorder.collect();
-                        final var counterName = RemoteClusterService.CONNECTION_ATTEMPT_FAILURES_COUNTER_NAME;
-                        final var measurements = metricRecorder.getMeasurements(InstrumentType.LONG_UP_DOWN_COUNTER, counterName);
-                        assertFalse(measurements.isEmpty());
-                        final var measurement = measurements.getLast();
-                        assertThat(measurement.getLong(), equalTo(1L));
-                        final var attributes = measurement.attributes();
-                        final var keySet = Set.of(
-                            RemoteConnectionStrategy.linkedProjectIdLabel,
-                            RemoteConnectionStrategy.linkedProjectAliasLabel,
-                            RemoteConnectionStrategy.connectionAtemptLabel
-                        );
-                        final var expectedAttemptType = isInitialConnectAttempt
-                            ? RemoteConnectionStrategy.ConnectionAttempt.initial
-                            : RemoteConnectionStrategy.ConnectionAttempt.reconnect;
-                        assertThat(attributes.keySet(), equalTo(keySet));
-                        assertThat(attributes.get(RemoteConnectionStrategy.linkedProjectIdLabel), equalTo(linkedProjectId.toString()));
-                        assertThat(attributes.get(RemoteConnectionStrategy.linkedProjectAliasLabel), equalTo(alias));
-                        assertThat(attributes.get(RemoteConnectionStrategy.connectionAtemptLabel), equalTo(expectedAttemptType.toString()));
-                    }
                 }
             }
 
             // Now verify connection errors when closing (node shutting down) are logged at debug and not warn.
             final var strategy = new FakeConnectionStrategy(originProjectId, linkedProjectId, alias, transportService, connectionManager);
             waitForConnect(strategy);
-            metricRecorder.resetCalls();
             strategy.setShouldConnectFail(true);
             strategy.setWaitInConnect(true);
             final var expectedLogLevel = Level.DEBUG;
@@ -320,11 +289,6 @@ public void testConnectionAttemptMetricsAndLogging() {
                     expectedLogMessage
                 )
             );
-            // Expect no metric change if the strategy has been closed.
-            metricRecorder.collect();
-            final var counterName = RemoteClusterService.CONNECTION_ATTEMPT_FAILURES_COUNTER_NAME;
-            final var measurements = metricRecorder.getMeasurements(InstrumentType.LONG_UP_DOWN_COUNTER, counterName);
-            assertTrue(measurements.isEmpty());
         }
     }