sidecar: try to reconnect if channel closed (#1180)

Since the write is inline in user threads the write can get
interruped leading to the channel being closed. Before all
subsequent writes would fail, now it will detect a closed
channel and attempt to reconnect.

Author: brharrington

spectator-reg-sidecar/src/main/java/com/netflix/spectator/sidecar/UdpWriter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2022 Netflix, Inc.
+ * Copyright 2014-2025 Netflix, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,27 +15,48 @@
  */
 package com.netflix.spectator.sidecar;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.io.IOException;
 import java.net.SocketAddress;
 import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
 import java.nio.channels.DatagramChannel;
 import java.nio.charset.StandardCharsets;
 
 /** Writer that outputs data to UDP socket. */
 final class UdpWriter extends SidecarWriter {
 
-  private final DatagramChannel channel;
+  private static final Logger LOGGER = LoggerFactory.getLogger(UdpWriter.class);
+
+  private final SocketAddress address;
+  private DatagramChannel channel;
 
   /** Create a new instance. */
   UdpWriter(String location, SocketAddress address) throws IOException {
     super(location);
-    this.channel = DatagramChannel.open();
-    this.channel.connect(address);
+    this.address = address;
+    connect();
+  }
+
+  private void connect() throws IOException {
+    channel = DatagramChannel.open();
+    channel.connect(address);
   }
 
   @Override public void writeImpl(String line) throws IOException {
     ByteBuffer buffer = ByteBuffer.wrap(line.getBytes(StandardCharsets.UTF_8));
-    channel.write(buffer);
+    try {
+      channel.write(buffer);
+    } catch (ClosedChannelException e) {
+      try {
+        connect();
+      } catch (IOException ex) {
+        LOGGER.warn("channel closed, failed to reconnect", ex);
+      }
+      throw e;
+    }
   }
 
   @Override public void close() throws IOException {

spectator-reg-sidecar/src/test/java/com/netflix/spectator/sidecar/UdpWriterTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2022 Netflix, Inc.
+ * Copyright 2014-2025 Netflix, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -39,6 +39,23 @@ public void udp() throws IOException {
     }
   }
 
+  @Test
+  public void udpReconnectIfClosed() throws IOException {
+    try (UdpServer server = new UdpServer()) {
+      try (SidecarWriter w = SidecarWriter.create(server.address())) {
+        // Used to simulate close from something like an interrupt. The next write
+        // will fail and it should try to reconnect.
+        w.close();
+        w.write("1");
+
+        w.write("2");
+        Assertions.assertEquals("2", server.read());
+        w.write("3");
+        Assertions.assertEquals("3", server.read());
+      }
+    }
+  }
+
   // Disabled because it can have issues on CI
   @Test
   @Disabled