Handle DTLS close_notify shutdown (algesten#956)

* DTLS close_notify handling

* Formatting

* Assert on reciprocal close too

* Fix formatting

* When closing, only process DTLS messages and ignore media etc.

* Formatting nitpick

* Fix clippy issue

* Remove special closing logic

* A different approach to closing

* Remove duplicate comment

* Fix a clippy nag

* Let's try to fix Windows tests failing

* Add close_notify changelog entry

---------

Co-authored-by: Martin Algesten <martin@algesten.se>

Author: kvasilye

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Unreleased
 
+  * Send DTLS `close_notify` on `Rtc::close()` #956
   * Fix `Simulcast::add_recv_layer` to push to recv instead of send #968
   * Accept any non-empty SDP session name (`s=`), not just `s=-` (RFC 8866 section 5.3)
 

crates/proto/src/crypto/provider.rs

@@ -192,6 +192,9 @@ pub trait DtlsInstance: CryptoSafe {
 
     /// Return the negotiated DTLS protocol version. May return `None` before handshake completion.
     fn protocol_version(&self) -> Option<ProtocolVersion>;
+
+    /// Initiate graceful shutdown by sending a DTLS `close_notify` alert.
+    fn close(&mut self) -> Result<(), DtlsImplError>;
 }
 
 // ============================================================================

crypto/apple-crypto/src/dtls.rs

@@ -169,6 +169,10 @@ impl DtlsInstance for AppleCryptoDtlsInstance {
     fn protocol_version(&self) -> Option<ProtocolVersion> {
         self.dtls.protocol_version()
     }
+
+    fn close(&mut self) -> Result<(), DtlsImplError> {
+        self.dtls.close()
+    }
 }
 
 #[cfg(test)]

crypto/aws-lc-rs/src/dtls.rs

@@ -111,4 +111,8 @@ impl DtlsInstance for AwsLcRsDtlsInstance {
     fn protocol_version(&self) -> Option<ProtocolVersion> {
         self.dtls.protocol_version()
     }
+
+    fn close(&mut self) -> Result<(), DtlsImplError> {
+        self.dtls.close()
+    }
 }

crypto/openssl/src/dtls_dimpl.rs

@@ -112,4 +112,8 @@ impl DtlsInstance for DimplDtlsInstance {
     fn protocol_version(&self) -> Option<ProtocolVersion> {
         self.dtls.protocol_version()
     }
+
+    fn close(&mut self) -> Result<(), DtlsImplError> {
+        self.dtls.close()
+    }
 }

crypto/openssl/src/dtls_ossl.rs

@@ -434,6 +434,10 @@ pub(super) struct OsslDtlsInstance {
     queued_app_data: VecDeque<Vec<u8>>,
     next_timeout: Option<Instant>,
     connected_emitted: bool,
+    /// Close handling
+    close_notify_received: bool,
+    close_notify_emitted: bool,
+    close_notify_sent: bool,
 }
 
 impl std::fmt::Debug for OsslDtlsInstance {
@@ -454,6 +458,9 @@ impl OsslDtlsInstance {
             queued_app_data: VecDeque::new(),
             next_timeout: None,
             connected_emitted: false,
+            close_notify_received: false,
+            close_notify_emitted: false,
+            close_notify_sent: false,
         })
     }
 
@@ -498,7 +505,30 @@ impl DtlsInstance for OsslDtlsInstance {
     fn handle_packet(&mut self, packet: &[u8]) -> Result<(), DtlsImplError> {
         match self.inner.handle_receive(packet) {
             Ok(Some(data)) => {
-                self.pending_application_data.push_back(data);
+                if data.is_empty() {
+                    // Zero-length read — confirm it's a close_notify via SSL_get_shutdown
+                    let is_shutdown =
+                        if let State::Established(ref mut stream) = self.inner.tls.state {
+                            stream
+                                .get_shutdown()
+                                .contains(openssl::ssl::ShutdownState::RECEIVED)
+                        } else {
+                            false
+                        };
+                    if is_shutdown {
+                        self.close_notify_received = true;
+                        // Send reciprocal close_notify per RFC 5246 §7.2.1,
+                        // but only if we haven't already sent one ourselves.
+                        if !self.close_notify_sent {
+                            if let State::Established(ref mut stream) = self.inner.tls.state {
+                                let _ = stream.shutdown();
+                            }
+                            self.close_notify_sent = true;
+                        }
+                    }
+                } else {
+                    self.pending_application_data.push_back(data);
+                }
             }
             Ok(None) => {}
             Err(e) => {
@@ -568,6 +598,12 @@ impl DtlsInstance for OsslDtlsInstance {
             }
         }
 
+        // Return close_notify event (after packets so reciprocal alert goes out first)
+        if self.close_notify_received && !self.close_notify_emitted {
+            self.close_notify_emitted = true;
+            return DtlsOutput::CloseNotify;
+        }
+
         // Return application data
         if let Some(data) = self.pending_application_data.pop_front() {
             if data.len() <= buf.len() {
@@ -615,6 +651,15 @@ impl DtlsInstance for OsslDtlsInstance {
     fn protocol_version(&self) -> Option<ProtocolVersion> {
         Some(ProtocolVersion::DTLS1_2)
     }
+
+    fn close(&mut self) -> Result<(), DtlsImplError> {
+        if let State::Established(ref mut stream) = self.inner.tls.state {
+            let _ = stream.shutdown();
+            self.close_notify_sent = true;
+            self.collect_output();
+        }
+        Ok(())
+    }
 }
 
 // ============================================================================

crypto/rust-crypto/src/dtls.rs

@@ -111,4 +111,8 @@ impl DtlsInstance for RustCryptoDtlsInstance {
     fn protocol_version(&self) -> Option<ProtocolVersion> {
         self.dtls.protocol_version()
     }
+
+    fn close(&mut self) -> Result<(), DtlsImplError> {
+        self.dtls.close()
+    }
 }

crypto/wincrypto/src/dtls_dimpl.rs

@@ -102,6 +102,10 @@ impl DtlsInstance for WinCryptoDtlsInstance {
     fn protocol_version(&self) -> Option<ProtocolVersion> {
         self.dtls.protocol_version()
     }
+
+    fn close(&mut self) -> Result<(), DtlsImplError> {
+        self.dtls.close()
+    }
 }
 
 // ============================================================================

crypto/wincrypto/src/dtls_schannel.rs

@@ -252,4 +252,10 @@ impl DtlsInstance for WinCryptoDtlsInstance {
     fn protocol_version(&self) -> Option<ProtocolVersion> {
         Some(ProtocolVersion::DTLS1_2)
     }
+
+    fn close(&mut self) -> Result<(), DtlsImplError> {
+        Err(DtlsImplError::CryptoError(
+            "SChannel native DTLS does not support close_notify".into(),
+        ))
+    }
 }

crypto/wincrypto/src/sys/provider.rs

@@ -409,4 +409,10 @@ impl DtlsInstance for WinCryptoDtlsInstance {
     fn is_active(&self) -> bool {
         self.dtls.is_client().unwrap_or(false)
     }
+
+    fn close(&mut self) -> Result<(), DtlsImplError> {
+        Err(DtlsImplError::CryptoError(
+            "SChannel native DTLS does not support close_notify".into(),
+        ))
+    }
 }