Feature: refactor DTLS to merge it into tls_openssl.c

configure

@@ -3747,6 +3747,7 @@ wav_demuxer_select="riffdec"
 wav_muxer_select="riffenc"
 webm_chunk_muxer_select="webm_muxer"
 webm_dash_manifest_demuxer_select="matroska_demuxer"
+whip_muxer_deps_any="dtls_protocol"
 wtv_demuxer_select="mpegts_demuxer riffdec"
 wtv_muxer_select="mpegts_muxer riffenc"
 xmv_demuxer_select="riffdec"
@@ -3845,6 +3846,9 @@ srtp_protocol_select="rtp_protocol srtp"
 tcp_protocol_select="network"
 tls_protocol_deps_any="gnutls openssl schannel securetransport libtls mbedtls"
 tls_protocol_select="tcp_protocol"
+# TODO: Support libtls, mbedtls, and gnutls.
+dtls_protocol_deps_any="openssl"
+dtls_protocol_select="udp_protocol"
 udp_protocol_select="network"
 udplite_protocol_select="network"
 unix_protocol_deps="sys_un_h"
@@ -7192,6 +7196,14 @@ enabled rkmpp             && { require_pkg_config rkmpp rockchip_mpp  rockchip/r
                              }
 enabled vapoursynth       && require_headers "vapoursynth/VSScript4.h vapoursynth/VapourSynth4.h"
 
+enabled openssl            && {
+    enabled whip_muxer && {
+        $pkg_config --exists --print-errors "openssl >= 1.0.1k" ||
+        require_pkg_config openssl "openssl >= 1.0.1k" openssl/ssl.h SSL_library_init ||
+        require_pkg_config openssl "openssl >= 1.0.1k" openssl/ssl.h OPENSSL_init_ssl
+    }
+}
+
 
 if enabled gcrypt; then
     GCRYPT_CONFIG="${cross_prefix}libgcrypt-config"

doc/muxers.texi

@@ -3879,4 +3879,51 @@ ffmpeg -f webm_dash_manifest -i video1.webm \
        manifest.xml
 @end example
 
+@anchor{whip}
+@section whip
+
+WebRTC (Real-Time Communication) muxer that supports sub-second latency streaming according to
+the WHIP (WebRTC-HTTP ingestion protocol) specification.
+
+It uses HTTP as a signaling protocol to exchange SDP capabilities and ICE lite candidates. Then,
+it uses STUN binding requests and responses to establish a session over UDP. Subsequently, it
+initiates a DTLS handshake to exchange the SRTP encryption keys. Lastly, it splits video and
+audio frames into RTP packets and encrypts them using SRTP.
+
+Ensure that you use H.264 without B frames and Opus for the audio codec. For example, to convert
+an input file with @command{ffmpeg} to WebRTC:
+@example
+ffmpeg -re -i input.mp4 -acodec libopus -ar 48000 -ac 2 \
+  -vcodec libx264 -profile:v baseline -tune zerolatency -threads 1 -bf 0 \
+  -f whip "http://localhost:1985/rtc/v1/whip/?app=live&stream=livestream"
+@end example
+
+For this example, we have employed low latency options, resulting in an end-to-end latency of
+approximately 150ms.
+
+@subsection Options
+
+This muxer supports the following options:
+
+@table @option
+
+@item handshake_timeout @var{integer}
+Set the timeout in milliseconds for ICE and DTLS handshake.
+Default value is 5000.
+
+@item pkt_size @var{integer}
+Set the maximum size, in bytes, of RTP packets that send out.
+Default value is 1500.
+
+@item authorization @var{string}
+The optional Bearer token for WHIP Authorization.
+
+@item cert_file @var{string}
+The optional certificate file path for DTLS.
+
+@item key_file @var{string}
+The optional private key file path for DTLS.
+
+@end table
+
 @c man end MUXERS

libavformat/Makefile

@@ -637,6 +637,7 @@ OBJS-$(CONFIG_WEBM_CHUNK_MUXER)          += webm_chunk.o
 OBJS-$(CONFIG_WEBP_MUXER)                += webpenc.o
 OBJS-$(CONFIG_WEBVTT_DEMUXER)            += webvttdec.o subtitles.o
 OBJS-$(CONFIG_WEBVTT_MUXER)              += webvttenc.o
+OBJS-$(CONFIG_WHIP_MUXER)                += whip.o avc.o http.o srtp.o tls_openssl.o
 OBJS-$(CONFIG_WSAUD_DEMUXER)             += westwood_aud.o
 OBJS-$(CONFIG_WSAUD_MUXER)               += westwood_audenc.o
 OBJS-$(CONFIG_WSD_DEMUXER)               += wsddec.o rawdec.o

libavformat/allformats.c

@@ -517,6 +517,7 @@ extern const FFOutputFormat ff_webp_muxer;
 extern const FFInputFormat  ff_webvtt_demuxer;
 extern const FFOutputFormat ff_webvtt_muxer;
 extern const FFInputFormat  ff_wsaud_demuxer;
+extern const FFOutputFormat ff_whip_muxer;
 extern const FFOutputFormat ff_wsaud_muxer;
 extern const FFInputFormat  ff_wsd_demuxer;
 extern const FFInputFormat  ff_wsvqa_demuxer;

libavformat/avio.c

@@ -339,8 +339,9 @@ static const struct URLProtocol *url_find_protocol(const char *filename)
         }
     }
     av_freep(&protocols);
-    if (av_strstart(filename, "https:", NULL) || av_strstart(filename, "tls:", NULL))
-        av_log(NULL, AV_LOG_WARNING, "https protocol not found, recompile FFmpeg with "
+    if (av_strstart(filename, "https:", NULL) || av_strstart(filename, "tls:", NULL) ||
+        av_strstart(filename, "dtls:", NULL))
+        av_log(NULL, AV_LOG_WARNING, "https or dtls protocol not found, recompile FFmpeg with "
                                      "openssl, gnutls or securetransport enabled.\n");
 
     return NULL;

libavformat/http.c

@@ -562,6 +562,12 @@ int ff_http_averror(int status_code, int default_averror)
         return default_averror;
 }
 
+const char* ff_http_get_new_location(URLContext *h)
+{
+    HTTPContext *s = h->priv_data;
+    return s->new_location;
+}
+
 static int http_write_reply(URLContext* h, int status_code)
 {
     int ret, body = 0, reply_code, message_len;

libavformat/http.h

@@ -62,4 +62,6 @@ int ff_http_do_new_request2(URLContext *h, const char *uri, AVDictionary **optio
 
 int ff_http_averror(int status_code, int default_averror);
 
+const char* ff_http_get_new_location(URLContext *h);
+
 #endif /* AVFORMAT_HTTP_H */

libavformat/protocols.c

@@ -62,6 +62,7 @@ extern const URLProtocol ff_subfile_protocol;
 extern const URLProtocol ff_tee_protocol;
 extern const URLProtocol ff_tcp_protocol;
 extern const URLProtocol ff_tls_protocol;
+extern const URLProtocol ff_dtls_protocol;
 extern const URLProtocol ff_udp_protocol;
 extern const URLProtocol ff_udplite_protocol;
 extern const URLProtocol ff_unix_protocol;

libavformat/srtp.h

@@ -27,7 +27,7 @@
 struct AVAES;
 struct AVHMAC;
 
-struct SRTPContext {
+typedef struct SRTPContext {
     struct AVAES *aes;
     struct AVHMAC *hmac;
     int rtp_hmac_size, rtcp_hmac_size;
@@ -40,7 +40,7 @@ struct SRTPContext {
     uint32_t roc;
 
     uint32_t rtcp_index;
-};
+} SRTPContext;
 
 int ff_srtp_set_crypto(struct SRTPContext *s, const char *suite,
                        const char *params);

libavformat/tls.c

@@ -1,6 +1,7 @@
 /*
- * TLS/SSL Protocol
+ * TLS/DTLS/SSL Protocol
  * Copyright (c) 2011 Martin Storsjo
+ * Copyright (c) 2025 Jack Lau
  *
  * This file is part of FFmpeg.
  *
@@ -20,6 +21,7 @@
  */
 
 #include "avformat.h"
+#include "internal.h"
 #include "network.h"
 #include "os_support.h"
 #include "url.h"
@@ -93,7 +95,7 @@ int ff_tls_open_underlying(TLSShared *c, URLContext *parent, const char *uri, AV
             c->listen = 1;
     }
 
-    ff_url_join(buf, sizeof(buf), "tcp", NULL, c->underlying_host, port, "%s", p);
+    ff_url_join(buf, sizeof(buf), c->is_dtls ? "udp" : "tcp", NULL, c->underlying_host, port, "%s", p);
 
     hints.ai_flags = AI_NUMERICHOST;
     if (!getaddrinfo(c->underlying_host, NULL, &hints, &ai)) {
@@ -124,7 +126,65 @@ int ff_tls_open_underlying(TLSShared *c, URLContext *parent, const char *uri, AV
     }
 
     freeenv_utf8(env_http_proxy);
-    return ffurl_open_whitelist(&c->tcp, buf, AVIO_FLAG_READ_WRITE,
-                                &parent->interrupt_callback, options,
-                                parent->protocol_whitelist, parent->protocol_blacklist, parent);
+    if (c->is_dtls) {
+        av_dict_set_int(options, "connect", 1, 0);
+        av_dict_set_int(options, "fifo_size", 0, 0);
+        /* Set the max packet size to the buffer size. */
+        av_dict_set_int(options, "pkt_size", c->mtu, 0);
+    }
+    ret = ffurl_open_whitelist(c->is_dtls ? &c->udp : &c->tcp, buf, AVIO_FLAG_READ_WRITE,
+                               &parent->interrupt_callback, options,
+                               parent->protocol_whitelist, parent->protocol_blacklist, parent);
+    if (c->is_dtls) {
+        if (ret < 0) {
+            av_log(c, AV_LOG_ERROR, "WHIP: Failed to connect udp://%s:%d\n", c->underlying_host, port);
+            return ret;
+        }
+        /* Make the socket non-blocking, set to READ and WRITE mode after connected */
+        ff_socket_nonblock(ffurl_get_file_handle(c->udp), 1);
+        c->udp->flags |= AVIO_FLAG_READ | AVIO_FLAG_NONBLOCK;
+    }
+    return ret;
 }
+
+/**
+ * Read all data from the given URL url and store it in the given buffer bp.
+ */
+int ff_url_read_all(const char *url, AVBPrint *bp)
+{
+    int ret = 0;
+    AVDictionary *opts = NULL;
+    URLContext *uc = NULL;
+    char buf[MAX_URL_SIZE];
+
+    ret = ffurl_open_whitelist(&uc, url, AVIO_FLAG_READ, NULL, &opts, NULL, NULL, NULL);
+    if (ret < 0) {
+        av_log(NULL, AV_LOG_ERROR, "TLS: Failed to open url %s\n", url);
+        goto end;
+    }
+
+    while (1) {
+        ret = ffurl_read(uc, buf, sizeof(buf));
+        if (ret == AVERROR_EOF) {
+            /* Reset the error because we read all response as answer util EOF. */
+            ret = 0;
+            break;
+        }
+        if (ret <= 0) {
+            av_log(NULL, AV_LOG_ERROR, "TLS: Failed to read from url=%s, key is %s\n", url, bp->str);
+            goto end;
+        }
+
+        av_bprintf(bp, "%.*s", ret, buf);
+        if (!av_bprint_is_complete(bp)) {
+            av_log(NULL, AV_LOG_ERROR, "TLS: Exceed max size %.*s, %s\n", ret, buf, bp->str);
+            ret = AVERROR(EIO);
+            goto end;
+        }
+    }
+
+end:
+    ffurl_closep(&uc);
+    av_dict_free(&opts);
+    return ret;
+}

libavformat/tls.h

@@ -1,6 +1,7 @@
 /*
- * TLS/SSL Protocol
+ * TLS/DTLS/SSL Protocol
  * Copyright (c) 2011 Martin Storsjo
+ * Copyright (c) 2025 Jack Lau
  *
  * This file is part of FFmpeg.
  *
@@ -22,10 +23,27 @@
 #ifndef AVFORMAT_TLS_H
 #define AVFORMAT_TLS_H
 
+#include "libavutil/bprint.h"
 #include "libavutil/opt.h"
 
 #include "url.h"
 
+/**
+ * Maximum size limit of a certificate and private key size.
+ */
+#define MAX_CERTIFICATE_SIZE 8192
+
+enum DTLSState {
+    DTLS_STATE_NONE,
+
+    /* Whether DTLS handshake is finished. */
+    DTLS_STATE_FINISHED,
+    /* Whether DTLS session is closed. */
+    DTLS_STATE_CLOSED,
+    /* Whether DTLS handshake is failed. */
+    DTLS_STATE_FAILED,
+};
+
 typedef struct TLSShared {
     char *ca_file;
     int verify;
@@ -40,6 +58,25 @@ typedef struct TLSShared {
     int numerichost;
 
     URLContext *tcp;
+
+    int is_dtls;
+
+    enum DTLSState state;
+
+    int use_external_udp;
+    URLContext *udp;
+
+    /* The fingerprint of certificate, used in SDP offer. */
+    char *fingerprint;
+
+    /* The certificate and private key content used for DTLS handshake */
+    char* cert_buf;
+    char* key_buf;
+    /**
+     * The size of RTP packet, should generally be set to MTU.
+     * Note that pion requires a smaller value, for example, 1200.
+     */
+    int mtu;
 } TLSShared;
 
 #define TLS_OPTFL (AV_OPT_FLAG_DECODING_PARAM | AV_OPT_FLAG_ENCODING_PARAM)
@@ -51,10 +88,27 @@ typedef struct TLSShared {
     {"key_file",   "Private key file",                    offsetof(pstruct, options_field . key_file),  AV_OPT_TYPE_STRING, .flags = TLS_OPTFL }, \
     {"listen",     "Listen for incoming connections",     offsetof(pstruct, options_field . listen),    AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, .flags = TLS_OPTFL }, \
     {"verifyhost", "Verify against a specific hostname",  offsetof(pstruct, options_field . host),      AV_OPT_TYPE_STRING, .flags = TLS_OPTFL }, \
-    {"http_proxy", "Set proxy to tunnel through",         offsetof(pstruct, options_field . http_proxy), AV_OPT_TYPE_STRING, .flags = TLS_OPTFL }
+    {"http_proxy", "Set proxy to tunnel through",         offsetof(pstruct, options_field . http_proxy), AV_OPT_TYPE_STRING, .flags = TLS_OPTFL }, \
+    {"use_external_udp", "Use external UDP from muxer or demuxer", offsetof(pstruct, options_field . use_external_udp), AV_OPT_TYPE_INT, { .i64 = 0}, 0, 1, .flags = TLS_OPTFL }, \
+    {"mtu", "Maximum Transmission Unit", offsetof(pstruct, options_field . mtu), AV_OPT_TYPE_INT,  { .i64 = 0}, INT64_MIN, INT64_MAX, .flags = TLS_OPTFL}, \
+    {"fingerprint", "The optional fingerprint for DTLS", offsetof(pstruct, options_field . fingerprint), AV_OPT_TYPE_STRING, .flags = TLS_OPTFL}, \
+    {"cert_buf", "The optional certificate buffer for DTLS", offsetof(pstruct, options_field . cert_buf), AV_OPT_TYPE_STRING, .flags = TLS_OPTFL}, \
+    {"key_buf", "The optional private key buffer for DTLS", offsetof(pstruct, options_field . key_buf), AV_OPT_TYPE_STRING, .flags = TLS_OPTFL}
 
 int ff_tls_open_underlying(TLSShared *c, URLContext *parent, const char *uri, AVDictionary **options);
 
+int ff_url_read_all(const char *url, AVBPrint *bp);
+
+int ff_dtls_set_udp(URLContext *h, URLContext *udp);
+
+int ff_dtls_export_materials(URLContext *h, char *dtls_srtp_materials, size_t materials_sz);
+
+int ff_dtls_state(URLContext *h);
+
+int ff_ssl_read_key_cert(char *key_url, char *cert_url, char *key_buf, size_t key_sz, char *cert_buf, size_t cert_sz, char **fingerprint);
+
+int ff_ssl_gen_key_cert(char *key_buf, size_t key_sz, char *cert_buf, size_t cert_sz, char **fingerprint);
+
 void ff_gnutls_init(void);
 void ff_gnutls_deinit(void);