Add parser for SEI Timestamp metadata from H265 bitstream

go.mod

@@ -12,12 +12,12 @@ require (
 	github.com/livekit/mediatransportutil v0.0.0-20251128105421-19c7a7b81c22
 	github.com/livekit/protocol v1.43.2
 	github.com/magefile/mage v1.15.0
-	github.com/pion/dtls/v3 v3.0.7
-	github.com/pion/interceptor v0.1.42
+	github.com/pion/dtls/v3 v3.0.10
+	github.com/pion/interceptor v0.1.43
 	github.com/pion/rtcp v1.2.16
-	github.com/pion/rtp v1.8.25
-	github.com/pion/sdp/v3 v3.0.16
-	github.com/pion/webrtc/v4 v4.1.6
+	github.com/pion/rtp v1.10.0
+	github.com/pion/sdp/v3 v3.0.17
+	github.com/pion/webrtc/v4 v4.2.2
 	github.com/stretchr/testify v1.11.1
 	github.com/twitchtv/twirp v8.1.3+incompatible
 	go.uber.org/atomic v1.11.0
@@ -65,6 +65,7 @@ require (
 	github.com/opencontainers/runc v1.3.3 // indirect
 	github.com/opencontainers/runtime-spec v1.3.0 // indirect
 	github.com/opencontainers/selinux v1.13.1 // indirect
+	github.com/pion/transport/v4 v4.0.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/secure-systems-lab/go-securesystemslib v0.9.1 // indirect
@@ -117,16 +118,15 @@ require (
 	github.com/nats-io/nats.go v1.47.0 // indirect
 	github.com/nats-io/nkeys v0.4.12 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/pion/datachannel v1.5.10 // indirect
-	github.com/pion/ice/v4 v4.0.12 // indirect
+	github.com/pion/datachannel v1.6.0 // indirect
+	github.com/pion/ice/v4 v4.2.0 // indirect
 	github.com/pion/logging v0.2.4 // indirect
 	github.com/pion/mdns/v2 v2.1.0 // indirect
 	github.com/pion/randutil v0.1.0 // indirect
-	github.com/pion/sctp v1.8.41 // indirect
-	github.com/pion/srtp/v3 v3.0.9 // indirect
-	github.com/pion/stun/v3 v3.0.1 // indirect
-	github.com/pion/transport/v3 v3.1.1 // indirect
-	github.com/pion/turn/v4 v4.1.3 // indirect
+	github.com/pion/sctp v1.9.1 // indirect
+	github.com/pion/srtp/v3 v3.0.10 // indirect
+	github.com/pion/stun/v3 v3.1.1 // indirect
+	github.com/pion/turn/v4 v4.1.4 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect
 	github.com/redis/go-redis/v9 v9.17.2 // indirect

go.sum

@@ -213,14 +213,14 @@ github.com/ory/dockertest/v3 v3.11.0 h1:OiHcxKAvSDUwsEVh2BjxQQc/5EHz9n0va9awCtNG
 github.com/ory/dockertest/v3 v3.11.0/go.mod h1:VIPxS1gwT9NpPOrfD3rACs8Y9Z7yhzO4SB194iUDnUI=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
-github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o=
-github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M=
-github.com/pion/dtls/v3 v3.0.7 h1:bItXtTYYhZwkPFk4t1n3Kkf5TDrfj6+4wG+CZR8uI9Q=
-github.com/pion/dtls/v3 v3.0.7/go.mod h1:uDlH5VPrgOQIw59irKYkMudSFprY9IEFCqz/eTz16f8=
-github.com/pion/ice/v4 v4.0.12 h1:vuI3h9OD5M0Z+V304qLC1/o16ahHVnDgqNCWbONv6s0=
-github.com/pion/ice/v4 v4.0.12/go.mod h1:tAp574oAufhHRHr8EO1xgPmVKVDBROX+708WggYD6NE=
-github.com/pion/interceptor v0.1.42 h1:0/4tvNtruXflBxLfApMVoMubUMik57VZ+94U0J7cmkQ=
-github.com/pion/interceptor v0.1.42/go.mod h1:g6XYTChs9XyolIQFhRHOOUS+bGVGLRfgTCUzH29EfVU=
+github.com/pion/datachannel v1.6.0 h1:XecBlj+cvsxhAMZWFfFcPyUaDZtd7IJvrXqlXD/53i0=
+github.com/pion/datachannel v1.6.0/go.mod h1:ur+wzYF8mWdC+Mkis5Thosk+u/VOL287apDNEbFpsIk=
+github.com/pion/dtls/v3 v3.0.10 h1:k9ekkq1kaZoxnNEbyLKI8DI37j/Nbk1HWmMuywpQJgg=
+github.com/pion/dtls/v3 v3.0.10/go.mod h1:YEmmBYIoBsY3jmG56dsziTv/Lca9y4Om83370CXfqJ8=
+github.com/pion/ice/v4 v4.2.0 h1:jJC8S+CvXCCvIQUgx+oNZnoUpt6zwc34FhjWwCU4nlw=
+github.com/pion/ice/v4 v4.2.0/go.mod h1:EgjBGxDgmd8xB0OkYEVFlzQuEI7kWSCFu+mULqaisy4=
+github.com/pion/interceptor v0.1.43 h1:6hmRfnmjogSs300xfkR0JxYFZ9k5blTEvCD7wxEDuNQ=
+github.com/pion/interceptor v0.1.43/go.mod h1:BSiC1qKIJt1XVr3l3xQ2GEmCFStk9tx8fwtCZxxgR7M=
 github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8=
 github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so=
 github.com/pion/mdns/v2 v2.1.0 h1:3IJ9+Xio6tWYjhN6WwuY142P/1jA0D5ERaIqawg/fOY=
@@ -229,22 +229,24 @@ github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
 github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
 github.com/pion/rtcp v1.2.16 h1:fk1B1dNW4hsI78XUCljZJlC4kZOPk67mNRuQ0fcEkSo=
 github.com/pion/rtcp v1.2.16/go.mod h1:/as7VKfYbs5NIb4h6muQ35kQF/J0ZVNz2Z3xKoCBYOo=
-github.com/pion/rtp v1.8.25 h1:b8+y44GNbwOJTYWuVan7SglX/hMlicVCAtL50ztyZHw=
-github.com/pion/rtp v1.8.25/go.mod h1:rF5nS1GqbR7H/TCpKwylzeq6yDM+MM6k+On5EgeThEM=
-github.com/pion/sctp v1.8.41 h1:20R4OHAno4Vky3/iE4xccInAScAa83X6nWUfyc65MIs=
-github.com/pion/sctp v1.8.41/go.mod h1:2wO6HBycUH7iCssuGyc2e9+0giXVW0pyCv3ZuL8LiyY=
-github.com/pion/sdp/v3 v3.0.16 h1:0dKzYO6gTAvuLaAKQkC02eCPjMIi4NuAr/ibAwrGDCo=
-github.com/pion/sdp/v3 v3.0.16/go.mod h1:9tyKzznud3qiweZcD86kS0ff1pGYB3VX+Bcsmkx6IXo=
-github.com/pion/srtp/v3 v3.0.9 h1:lRGF4G61xxj+m/YluB3ZnBpiALSri2lTzba0kGZMrQY=
-github.com/pion/srtp/v3 v3.0.9/go.mod h1:E+AuWd7Ug2Fp5u38MKnhduvpVkveXJX6J4Lq4rxUYt8=
-github.com/pion/stun/v3 v3.0.1 h1:jx1uUq6BdPihF0yF33Jj2mh+C9p0atY94IkdnW174kA=
-github.com/pion/stun/v3 v3.0.1/go.mod h1:RHnvlKFg+qHgoKIqtQWMOJF52wsImCAf/Jh5GjX+4Tw=
+github.com/pion/rtp v1.10.0 h1:XN/xca4ho6ZEcijpdF2VGFbwuHUfiIMf3ew8eAAE43w=
+github.com/pion/rtp v1.10.0/go.mod h1:rF5nS1GqbR7H/TCpKwylzeq6yDM+MM6k+On5EgeThEM=
+github.com/pion/sctp v1.9.1 h1:ACiozSfsMXYjXOk2q0bBFzxqFZMmq+TalD2R5f9Rh4M=
+github.com/pion/sctp v1.9.1/go.mod h1:OTOlsQ5EDQ6mQ0z4MUGXt2CgQmKyafBEXhUVqLRB6G8=
+github.com/pion/sdp/v3 v3.0.17 h1:9SfLAW/fF1XC8yRqQ3iWGzxkySxup4k4V7yN8Fs8nuo=
+github.com/pion/sdp/v3 v3.0.17/go.mod h1:9tyKzznud3qiweZcD86kS0ff1pGYB3VX+Bcsmkx6IXo=
+github.com/pion/srtp/v3 v3.0.10 h1:tFirkpBb3XccP5VEXLi50GqXhv5SKPxqrdlhDCJlZrQ=
+github.com/pion/srtp/v3 v3.0.10/go.mod h1:3mOTIB0cq9qlbn59V4ozvv9ClW/BSEbRp4cY0VtaR7M=
+github.com/pion/stun/v3 v3.1.1 h1:CkQxveJ4xGQjulGSROXbXq94TAWu8gIX2dT+ePhUkqw=
+github.com/pion/stun/v3 v3.1.1/go.mod h1:qC1DfmcCTQjl9PBaMa5wSn3x9IPmKxSdcCsxBcDBndM=
 github.com/pion/transport/v3 v3.1.1 h1:Tr684+fnnKlhPceU+ICdrw6KKkTms+5qHMgw6bIkYOM=
 github.com/pion/transport/v3 v3.1.1/go.mod h1:+c2eewC5WJQHiAA46fkMMzoYZSuGzA/7E2FPrOYHctQ=
-github.com/pion/turn/v4 v4.1.3 h1:jVNW0iR05AS94ysEtvzsrk3gKs9Zqxf6HmnsLfRvlzA=
-github.com/pion/turn/v4 v4.1.3/go.mod h1:TD/eiBUf5f5LwXbCJa35T7dPtTpCHRJ9oJWmyPLVT3A=
-github.com/pion/webrtc/v4 v4.1.6 h1:srHH2HwvCGwPba25EYJgUzgLqCQoXl1VCUnrGQMSzUw=
-github.com/pion/webrtc/v4 v4.1.6/go.mod h1:wKecGRlkl3ox/As/MYghJL+b/cVXMEhoPMJWPuGQFhU=
+github.com/pion/transport/v4 v4.0.1 h1:sdROELU6BZ63Ab7FrOLn13M6YdJLY20wldXW2Cu2k8o=
+github.com/pion/transport/v4 v4.0.1/go.mod h1:nEuEA4AD5lPdcIegQDpVLgNoDGreqM/YqmEx3ovP4jM=
+github.com/pion/turn/v4 v4.1.4 h1:EU11yMXKIsK43FhcUnjLlrhE4nboHZq+TXBIi3QpcxQ=
+github.com/pion/turn/v4 v4.1.4/go.mod h1:ES1DXVFKnOhuDkqn9hn5VJlSWmZPaRJLyBXoOeO/BmQ=
+github.com/pion/webrtc/v4 v4.2.2 h1:Kx85S9QHckp9mb28Uwn12XGE7/y7Bd0TC0jbiiXzhaY=
+github.com/pion/webrtc/v4 v4.2.2/go.mod h1:NXRhsXD0sBvk1KHfB3Y9mAYFk6Omjh+d9LsxwCyn7gc=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=

readersampleprovider.go

@@ -73,6 +73,13 @@
 	AudioLevel          uint8
 	trackOpts           []LocalTrackOptions
 	h26xStreamingFormat H26xStreamingFormat
+	appendUserTimestamp bool
+
+	// When appendUserTimestamp is enabled, we will attempt to parse timestamps from
+	// H264 SEI user_data_unregistered NALs that precede frame NALs.
+	// We then stash the parsed timestamp and attach it to the next frame as an LKTS trailer.
+	pendingUserTimestampUs  int64
+	hasPendingUserTimestamp bool
 
 	// Allow various types of ingress
 	reader io.ReadCloser
@@ -139,6 +146,15 @@
 	}
 }
 
+// ReaderTrackWithUserTimestamp enables attaching the custom LKTS trailer
+// (timestamp_us + magic) to outgoing encoded frame payloads.
+// This currently supports H264.
+func ReaderTrackWithUserTimestamp(enabled bool) func(provider *ReaderSampleProvider) {
+	return func(provider *ReaderSampleProvider) {
+		provider.appendUserTimestamp = enabled
+	}
+}
+
 // NewLocalFileTrack creates an *os.File reader for NewLocalReaderTrack
 func NewLocalFileTrack(file string, options ...ReaderSampleProviderOption) (*LocalTrack, error) {
 	// File health check
@@ -252,10 +268,10 @@
 	switch p.Mime {
 	case webrtc.MimeTypeH264:
 		if p.h26xStreamingFormat == H26xStreamingFormatAnnexB {
-			p.h264reader, err = h264reader.NewReader(p.reader)
+			p.h264reader, err = h264reader.NewReaderWithOptions(p.reader, h264reader.WithIncludeSEI(true))
 		}
 	case webrtc.MimeTypeH265:
-		p.h265reader, err = h265reader.NewReader(p.reader)
+		p.h265reader, err = h265reader.NewReaderWithOptions(p.reader, h265reader.WithIncludeSEI(true))
 	case webrtc.MimeTypeVP8, webrtc.MimeTypeVP9:
 		var ivfHeader *ivfreader.IVFFileHeader
 		p.ivfReader, ivfHeader, err = ivfreader.NewWith(p.reader)
@@ -319,6 +335,20 @@
 			nalUnitData = nal.Data
 		}
 
+		if nalUnitType == h264reader.NalUnitTypeSEI {
+			if p.appendUserTimestamp {
+				if ts, ok := parseH264SEIUserTimestamp(nalUnitData); ok {
+					p.pendingUserTimestampUs = ts
+					p.hasPendingUserTimestamp = true
+				}
+			} else {
+				// If SEI, clear the data and do not return a frame (try next NAL)
+				sample.Data = nil
+				sample.Duration = 0
+				return sample, nil
+			}
+		}
+
 		isFrame := false
 		switch nalUnitType {
 		case h264reader.NalUnitTypeCodedSliceDataPartitionA,
@@ -334,6 +364,20 @@
 			// return it without duration
 			return sample, nil
 		}
+
+		// Attach the LKTS trailer to the encoded frame payload when enabled.
+		// If we didn't see a preceding timestamp, we still append a trailer with
+		// a zero timestamp.
+		if p.appendUserTimestamp {
+			ts := int64(0)
+			if p.hasPendingUserTimestamp {
+				ts = p.pendingUserTimestampUs
+				p.hasPendingUserTimestamp = false
+				p.pendingUserTimestampUs = 0
+			}
+			sample.Data = appendUserTimestampTrailer(sample.Data, ts)
+		}
+
 		sample.Duration = defaultH264FrameDuration
 
 	case webrtc.MimeTypeH265:
@@ -348,6 +392,27 @@
 				return sample, err
 			}
 
+			if nal.NalUnitType == 39 { // prefix SEI
+				if p.appendUserTimestamp {
+					if ts, ok := parseH265SEIUserTimestamp(nal.Data); ok {
+						p.pendingUserTimestampUs = ts
+						p.hasPendingUserTimestamp = true
+					}
+					continue
+				}
+				// If SEI, clear the data and do not return a frame (try next NAL)
+				sample.Data = nil
+				sample.Duration = 0
+				return sample, nil
+			}
+
+			if nal.NalUnitType == 40 { // suffix SEI
+				// Ignore suffix SEI entirely (do not parse or append).
+				sample.Data = nil
+				sample.Duration = 0
+				return sample, nil
+			}
+
 			// aggregate vps,sps,pps into a single AP packet (chrome requires this)
 			if nal.NalUnitType == 32 || nal.NalUnitType == 33 || nal.NalUnitType == 34 {
 				sample.Data = append(sample.Data, []byte{0, 0, 0, 1}...) // add NAL prefix
@@ -372,6 +437,19 @@
 				return sample, nil
 			}
 
+			// Attach the LKTS trailer to the encoded frame payload when enabled.
+			// If we didn't see a preceding timestamp, we still append a trailer with
+			// a zero timestamp.
+			if p.appendUserTimestamp {
+				ts := int64(0)
+				if p.hasPendingUserTimestamp {
+					ts = p.pendingUserTimestampUs
+					p.hasPendingUserTimestamp = false
+					p.pendingUserTimestampUs = 0
+				}
+				sample.Data = appendUserTimestampTrailer(sample.Data, ts)
+			}
+
 			sample.Duration = defaultH265FrameDuration
 			break
 		}

user_timestamp.go

@@ -0,0 +1,53 @@
+package lksdk
+
+import "encoding/binary"
+
+const (
+	// userTimestampMagic must remain consistent with kUserTimestampMagic
+	// and kUserTimestampTrailerSize in rust-sdks/webrtc-sys/include/livekit/user_timestamp.h.
+	userTimestampMagic       = "LKTS"
+	userTimestampTrailerSize = 8 + len(userTimestampMagic)
+)
+
+// appendUserTimestampTrailer returns a new slice containing data followed by
+// a user timestamp trailer:
+//   - 8-byte big-endian int64 timestamp in microseconds
+//   - 4-byte ASCII magic "LKTS"
+func appendUserTimestampTrailer(data []byte, userTimestampUs int64) []byte {
+	outLen := len(data) + userTimestampTrailerSize
+	out := make([]byte, outLen)
+	copy(out, data)
+
+	// Write timestamp (big-endian) just before the magic bytes.
+	tsOffset := len(data)
+	binary.BigEndian.PutUint64(out[tsOffset:tsOffset+8], uint64(userTimestampUs))
+
+	// Append magic bytes.
+	copy(out[tsOffset+8:], userTimestampMagic)
+
+	return out
+}
+
+// parseUserTimestampTrailer attempts to parse an LKTS trailer from the end
+// of the provided buffer. It returns the timestamp in microseconds and true
+// when a valid trailer is present.
+func parseUserTimestampTrailer(data []byte) (int64, bool) {
+	if len(data) < userTimestampTrailerSize {
+		return 0, false
+	}
+
+	// Check magic bytes at the very end.
+	magicStart := len(data) - len(userTimestampMagic)
+	if string(data[magicStart:]) != userTimestampMagic {
+		return 0, false
+	}
+
+	// Timestamp is placed immediately before the magic.
+	tsStart := magicStart - 8
+	if tsStart < 0 {
+		return 0, false
+	}
+
+	ts := int64(binary.BigEndian.Uint64(data[tsStart : tsStart+8]))
+	return ts, true
+}

user_timestamp_h264_parser.go

@@ -0,0 +1,100 @@
+package lksdk
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+)
+
+// userTimestampSEIUUID is the UUID of the user timestamp SEI NAL unit.
+var userTimestampSEIUUID = [16]byte{
+	0x3f, 0xa8, 0x5f, 0x64, 0x57, 0x17, 0x45, 0x62,
+	0xb3, 0xfc, 0x2c, 0x96, 0x3f, 0x66, 0xaf, 0xa6,
+}
+
+// parseH264SEIUserTimestamp parses H264 SEI NAL units (type 6) carrying
+// user_data_unregistered messages and returns a timestamp (microseconds) when detected.
+//
+// Expected payload format (after the NAL header byte):
+//
+//	payloadType  = 5 (user_data_unregistered)
+//	payloadSize  = 24
+//	UUID         = 16 bytes (3fa85f64-5717-4562-b3fc-2c963f66afa6)
+//	timestamp_us = 8 bytes, big-endian
+//	trailing     = 0x80 (stop bits + padding)
+func parseH264SEIUserTimestamp(nalData []byte) (int64, bool) {
+	if len(nalData) < 2 {
+		logger.Infow("H264 SEI user_data_unregistered: nal too short", "nal_len", len(nalData))
+		return 0, false
+	}
+
+	// Skip NAL header (first byte).
+	payload := nalData[1:]
+	i := 0
+
+	// Parse payloadType (can be extended with 0xFF bytes).
+	payloadType := 0
+	for i < len(payload) && payload[i] == 0xFF {
+		payloadType += 255
+		i++
+	}
+	if i >= len(payload) {
+		logger.Infow("H264 SEI user_data_unregistered: payloadType truncated", "payload_len", len(payload))
+		return 0, false
+	}
+	payloadType += int(payload[i])
+	i++
+
+	// We only care about user_data_unregistered (type 5).
+	if payloadType != 5 {
+		return 0, false
+	}
+
+	// Parse payloadSize (can be extended with 0xFF bytes).
+	payloadSize := 0
+	for i < len(payload) && payload[i] == 0xFF {
+		payloadSize += 255
+		i++
+	}
+	if i >= len(payload) {
+		logger.Infow("H264 SEI user_data_unregistered: payloadSize truncated", "payload_len", len(payload))
+		return 0, false
+	}
+	payloadSize += int(payload[i])
+	i++
+
+	if payloadSize < 24 || len(payload) < i+payloadSize {
+		// Not enough data for UUID (16) + timestamp (8).
+		logger.Infow(
+			"H264 SEI user_data_unregistered: insufficient data for UUID + timestamp",
+			"payloadSize", payloadSize,
+			"payload_len", len(payload),
+			"offset", i,
+		)
+		return 0, false
+	}
+
+	userData := payload[i : i+payloadSize]
+	uuidBytes := userData[:16]
+	tsBytes := userData[16:24]
+
+	// Validate the UUID matches the exact user timestamp UUID we expect.
+	if !bytes.Equal(uuidBytes, userTimestampSEIUUID[:]) {
+		return 0, false
+	}
+
+	timestampUS := binary.BigEndian.Uint64(tsBytes)
+
+	// Format UUID as 8-4-4-4-12 hex segments (for debug logs).
+	uuid := fmt.Sprintf("%x-%x-%x-%x-%x",
+		uuidBytes[0:4],
+		uuidBytes[4:6],
+		uuidBytes[6:8],
+		uuidBytes[8:10],
+		uuidBytes[10:16],
+	)
+
+	logger.Debugw("H264 SEI user_data_unregistered parsed", "uuid", uuid, "timestamp_us", timestampUS)
+
+	return int64(timestampUS), true
+}