diff --git a/Documentation/players.md b/Documentation/players.md index c7dd0f47a9..c68f31f158 100644 --- a/Documentation/players.md +++ b/Documentation/players.md @@ -8,9 +8,10 @@ Swiftfin offers two player options: **Swiftfin** (VLCKit) and **Native** (AVPlay | Feature | Swiftfin (VLCKit) | Native (AVPlayer) | |----------------------------|-------------------|----------------| +| **External Audio Tracks** | ❌ | ❌ | | **Framerate Matching** | ❌ | ✅ | | **HDR to SDR Tonemapping** | ✅ [1] | 🔶 [2] | -| **Player Controls** | - Speed adjustment
- Aspect Fill
- Chapter Support
- Subtitle Support
- Audio Track Selection
- Customizable UI | - Speed adjustment
- Aspect Fill | +| **Player Controls** | - Speed Adjustment
- Aspect Fill
- Chapter Support
- Subtitle Support
- Trickplay Support
- Audio Track Selection
- Customizable UI | - Speed Adjustment
- Aspect Fill | | **Picture-in-Picture** | ❌ | ✅ | | **TLS Support** | 1.1, 1.2 [3] | 1.1, 1.2, 1.3 | | **[Airplay Audio Output](https://support.apple.com/en-us/102357)** | 🔶 [4] | ✅ | @@ -99,7 +100,7 @@ Swiftfin offers two player options: **Swiftfin** (VLCKit) and **Native** (AVPlay | [FLV1](https://en.wikipedia.org/wiki/Sorenson_Spark) | ✅ | ❌ | | [H.261](https://en.wikipedia.org/wiki/H.261) | ✅ | ❌ | | [H.263](https://en.wikipedia.org/wiki/H.263) | ✅ | ❌ | -| [H.264](https://en.wikipedia.org/wiki/Advanced_Video_Coding) | ✅ | ✅ | +| [H.264/AVC](https://en.wikipedia.org/wiki/Advanced_Video_Coding) | ✅ | ✅ | | [H.265/HEVC](https://en.wikipedia.org/wiki/High_Efficiency_Video_Coding) | ✅ | ✅ [2] | | [H.266/VVC](https://en.wikipedia.org/wiki/Versatile_Video_Coding) | ❌ [3] | ❌ | | [MJPEG](https://en.wikipedia.org/wiki/Motion_JPEG) | ✅ | ✅ | @@ -136,12 +137,12 @@ Swiftfin offers two player options: **Swiftfin** (VLCKit) and **Native** (AVPlay |---------------------------------------------------------------------------------|-------------------|-------------------| | [ASS](https://en.wikipedia.org/wiki/SubStation_Alpha#Advanced_SubStation_Alpha) | ✅ | ❌ | | [CC_DEC](https://en.wikipedia.org/wiki/Closed_captioning) | ✅ | ✅ | -| [DVBSub](https://en.wikipedia.org/wiki/DVB_subtitles) | ✅ | 🔶 [1] | -| [DVDSub](https://en.wikipedia.org/wiki/VobSub) | ✅ | 🔶 [1] | +| [DVBSub](https://en.wikipedia.org/wiki/DVB_subtitles) | ✅ [1] | 🔶 [2] | +| [DVDSub](https://en.wikipedia.org/wiki/VobSub) | ✅ [1] | 🔶 [2] | | [JacoSub](https://en.wikipedia.org/wiki/JACOsub) | ✅ | ❌ | | [MOV_Text](https://en.wikipedia.org/wiki/MPEG-4_Part_17) | ✅ | ❌ | | [MPL2](https://en.wikipedia.org/wiki/MPL2) | ✅ | ❌ | -| [PGSSub](https://en.wikipedia.org/wiki/Presentation_Graphic_Stream) | ✅ | 🔶 [1] | +| [PGSSub](https://en.wikipedia.org/wiki/Presentation_Graphic_Stream) | ✅ [1] | 🔶 [2] | | [PJS](https://en.wikipedia.org/wiki/Phoenix_Subtitle) | ✅ | ❌ | | [RealText](https://en.wikipedia.org/wiki/RealText) | ✅ | ❌ | | [SAMI](https://en.wikipedia.org/wiki/SAMI) | ✅ | ❌ | @@ -154,11 +155,12 @@ Swiftfin offers two player options: **Swiftfin** (VLCKit) and **Native** (AVPlay | [TTML](https://en.wikipedia.org/wiki/Timed_Text_Markup_Language) | ✅ | ✅ | | [VPlayer](https://en.wikipedia.org/wiki/VPlayer) | ✅ | ❌ | | [VTT](https://en.wikipedia.org/wiki/WebVTT) | ✅ | ✅ | -| [XSub](https://en.wikipedia.org/wiki/XSUB) | ✅ | 🔶 [1] | +| [XSub](https://en.wikipedia.org/wiki/XSUB) | ✅ [1] | 🔶 [2] | **Notes:** -- [1] Subtitle format requires server-side encoding for Native (AVPlayer) playback. +- [1] Subtitle format can be played if embedded in the container (MKV) but requres server-side encoding for playback is the source is an external file. +- [2] Subtitle format requires server-side encoding for playback. - Subtitle track selection is not currently supported in Native (AVPlayer) due to issues with HLS file incompatibilities. @@ -192,51 +194,11 @@ Swiftfin offers two player options: **Swiftfin** (VLCKit) and **Native** (AVPlay --- -## Track Selection - -Swiftfin track selection is limited by compatibility with each player. In testing, as of Swiftfin 1.3, the following interactions have been tested. - -✅ Working correctly
-🔶 Partially working with limitations
-❌ Not working - -### Swiftfin Player - -| File Configuration | DirectPlay | Transcode | Notes | -|---------------------------------------------------------|------------|-----------|------------------------------------------------| -| Internal Audio | ✅ | ✅ | | -| Internal Audio + Internal Subtitles | ✅ | 🔶 | - Subtitles do not work if Non-External *(DVDSUB)* | -| Internal Audio + External Subtitles | ✅ | ✅ | | -| Internal Audio + Internal Subtitles + External Subtitles| ✅ | 🔶 | - Subtitles do not work if Non-External *(DVDSUB)* | -| Multiple Internal Audio + Multiple Internal Subtitles | ✅ | 🔶 | - Subtitles do not work if Non-External *(DVDSUB)* | -| Multiple Internal Audio + Multiple External Subtitles | ✅ | ✅ | | -| Multiple Internal Audio + Internal Subtitles + External Subtitles | ✅ | 🔶 | - Subtitles do not work if Non-External *(DVDSUB)* | -| External Audio + Internal Audio + External Subtitles | ✅ | ✅ | - Cannot play external audio track if transcoding is required
- Subtitles do not work if Non-External *(DVDSUB)* | -| External Audio + Internal Audio + Internal Subtitles | ✅ | ✅ | - Cannot play external audio track if transcoding is required
- Subtitles do not work if Non-External *(DVDSUB)* | -| External Audio + Internal Audio + Internal Subtitles + External Subtitles | ✅ | ✅ | - Cannot play external audio track if transcoding is required
- Subtitles do not work if Non-External *(DVDSUB)* | - -### Native Player - -| File Configuration | DirectPlay | Transcode | Notes | -|--------------------------------------------------------|------------|-----------|------------------------------------------------| -| Internal Audio | ✅ | ✅ | | -| Internal Audio + Internal Subtitles | 🔶 | ❌ | - The default audio track will played
- subtitles cannot be selected. | -| Internal Audio + External Subtitles | 🔶 | ❌ | - The default audio track will played
- subtitles cannot be selected. | -| Internal Audio + Internal Subtitles + External Subtitles | 🔶 | ❌ | - The default audio track will played
- subtitles cannot be selected. | -| Multiple Internal Audio + Multiple Internal Subtitles | 🔶 | ❌ | - The default audio track will played
- subtitles cannot be selected. | -| Multiple Internal Audio + Multiple External Subtitles | 🔶 | ❌ | - The default audio track will played
- subtitles cannot be selected. | -| Multiple Internal Audio + Internal Subtitles + External Subtitles | 🔶 | ❌ | - The default audio track will played
- subtitles cannot be selected. | -| External Audio + Internal Audio + External Subtitles | 🔶 | ❌ | - The default audio track will played
- subtitles cannot be selected. | -| External Audio + Internal Audio + Internal Subtitles | 🔶 | ❌ | - The default audio track will played
- subtitles cannot be selected. | -| External Audio + Internal Audio + Internal Subtitles + External Subtitles | 🔶 | ❌ | - The default audio track will played
- subtitles cannot be selected. | - ---- - ### Miscellaneous -| Feature | Swiftfin (VLCKit) | Native (AVPlayer) | Notes | -|-------------|-------------------|----------------|----------------| -| **External Display Support** | 🔶 | ✅ | Swiftfin Player can only be mirrored. As a result, the player will retain the source device dimensions. | -| **Energy Consumption** | 🔶 | ✅ | Swiftfin Player will use a software decoder if the media cannot be handled by iOS natively. This results in higher power consumption. | +| Feature | Swiftfin (VLCKit) | Native (AVPlayer) | Notes | +|------------------------------|-------------------|-------------------|---------------------------------------------------------------------------------------------------------------------------------------| +| **External Display Support** | 🔶 | ✅ | Swiftfin Player can only be mirrored. As a result, the player will retain the source device dimensions. | +| **Energy Consumption** | 🔶 | ✅ | Swiftfin Player will use a software decoder if the media cannot be handled by iOS natively. This results in higher power consumption. | --- diff --git a/Shared/Extensions/JellyfinAPI/DeviceProfile.swift b/Shared/Extensions/JellyfinAPI/DeviceProfile.swift index 2610d158b1..02d5bbf6b6 100644 --- a/Shared/Extensions/JellyfinAPI/DeviceProfile.swift +++ b/Shared/Extensions/JellyfinAPI/DeviceProfile.swift @@ -75,4 +75,43 @@ extension DeviceProfile { return deviceProfile } + + // MARK: - Playback Capability Queries + + /// Whether any `DirectPlayProfile` allows media with this audio codec in the given container to be played directly. + func canPlay(type: DlnaProfileType, audioCodec: String?, container: String?) -> Bool { + (directPlayProfiles ?? []).contains { profile in + profile.type == type + && profileContains(profile: profile.audioCodec, audioCodec) + && profileContains(profile: profile.container, container) + } + } + + /// Whether any `DirectPlayProfile` allows media with this video codec in the given container to be played directly. + func canPlay(type: DlnaProfileType, videoCodec: String?, container: String?) -> Bool { + (directPlayProfiles ?? []).contains { profile in + profile.type == type + && profileContains(profile: profile.videoCodec, videoCodec) + && profileContains(profile: profile.container, container) + } + } + + /// Whether any `SubtitleProfile` allows this format to be delivered via the given method. + func canPlay(subtitleFormat: String?, method: SubtitleDeliveryMethod) -> Bool { + guard let subtitleFormat = subtitleFormat?.lowercased() else { return false } + return (subtitleProfiles ?? []).contains { profile in + profile.method == method + && profile.format?.lowercased() == subtitleFormat + } + } + + /// Parse & check membership like this is CSV as that's the format we send to the server. + private func profileContains(profile: String?, _ candidate: String?) -> Bool { + guard let profile else { return true } + guard let candidate = candidate?.lowercased() else { return false } + return profile + .lowercased() + .split(separator: ",") + .contains { $0.trimmingCharacters(in: .whitespaces) == candidate } + } } diff --git a/Shared/Extensions/JellyfinAPI/MediaStream.swift b/Shared/Extensions/JellyfinAPI/MediaStream.swift index a5986e4304..0ace485bb3 100644 --- a/Shared/Extensions/JellyfinAPI/MediaStream.swift +++ b/Shared/Extensions/JellyfinAPI/MediaStream.swift @@ -209,55 +209,114 @@ extension MediaStream { extension [MediaStream] { - /// Adjusts track indexes for a full set of media streams. - /// For non-transcode stream types: - /// Internal tracks (non-external) are ordered as: Video, Audio, Subtitles, then any others. - /// Their relative order within each group is preserved and indexes start at 0. - /// For transcode stream type: - /// Only the first internal video track and the first internal audio track are included, in that order. - /// In both cases, external tracks are appended in their original order with indexes continuing after internal tracks. - func adjustedTrackIndexes(for playMethod: PlayMethod, selectedAudioStreamIndex: Int) -> [MediaStream] { - let internalTracks = self.filter { !($0.isExternal ?? false) } - let externalTracks = self.filter { $0.isExternal ?? false } - - var orderedInternal: [MediaStream] = [] + /// Text-based external subtitles loaded as sidecar files. Image-based subtitles are excluded because the player silently drops them. + var sidecarSubtitles: [MediaStream] { + filter { $0.deliveryMethod == .external && $0.deliveryURL != nil && $0.isTextSubtitleStream == true } + } - let subtitleInternal = internalTracks.filter { $0.type == .subtitle } + /// Builds a mapping from Jellyfin's global stream indexes to VLC's container-position indexes. + /// + /// Jellyfin assigns a single global index across all streams (video, audio, subtitle — internal and external). + /// VLC numbers tracks by their position within the actual media container, starting at 0. + /// This function produces `[JellyfinIndex: PlayerIndex]` so we can translate between the two. + /// + /// Called at init time — before VLC has loaded the media. Sidecar subtitle indexes are estimated here + /// but finalized later by `resolveIndexMap` once the player reports its actual track list. + /// + /// - `Transcode`: The HLS container has exactly 1 video (index 0) and 1 audio (index 1). + /// - `DirectPlay`: Jellyfin lists external tracks first, offsetting all internal container indexes by that count. + func buildIndexMap( + for playMethod: PlayMethod, + selectedAudioStreamIndex: Int + ) -> [Int: Int] { + var indexMap: [Int: Int] = [:] if playMethod == .transcode { - // Only include the first video and first audio track for transcode. - let videoInternal = internalTracks.filter { $0.type == .video } - let audioInternal = internalTracks.filter { $0.type == .audio } + var containerTracks: [MediaStream] = [] - if let firstVideo = videoInternal.first { - orderedInternal.append(firstVideo) + let videoTracks = filter { $0.type == .video && !($0.isExternal == true) } + let audioTracks = filter { $0.type == .audio && !($0.isExternal == true) } + + if let firstVideo = videoTracks.first { + containerTracks.append(firstVideo) + } + if let selectedAudio = audioTracks.first(where: { $0.index == selectedAudioStreamIndex }) { + containerTracks.append(selectedAudio) } - if let selectedAudio = audioInternal.first(where: { $0.index == selectedAudioStreamIndex }) { - orderedInternal.append(selectedAudio) + + for (newIndex, track) in containerTracks.enumerated() { + guard let oldIndex = track.index else { continue } + indexMap[oldIndex] = newIndex } - orderedInternal += subtitleInternal + let playbackChildStartIndex = containerTracks.count + let sidecarSubtitles = self.sidecarSubtitles + + for (offset, track) in sidecarSubtitles.enumerated() { + guard let oldIndex = track.index else { continue } + let playerIndex = playbackChildStartIndex + offset + indexMap[oldIndex] = playerIndex + } } else { - let videoInternal = internalTracks.filter { $0.type == .video } - let audioInternal = internalTracks.filter { $0.type == .audio } + let externalCount = self.count(where: { $0.isExternal == true }) + let internalTracks = self.filter { !($0.isExternal ?? false) } - orderedInternal = videoInternal + audioInternal + subtitleInternal + for track in internalTracks { + guard let oldIndex = track.index else { continue } + let playerIndex = oldIndex - externalCount + indexMap[oldIndex] = playerIndex + } } - var newInternalTracks: [MediaStream] = [] - for (index, var track) in orderedInternal.enumerated() { - track.index = index - newInternalTracks.append(track) - } + return indexMap + } - var newExternalTracks: [MediaStream] = [] - let startingIndexForExternal = newInternalTracks.count - for (offset, var track) in externalTracks.enumerated() { - track.index = startingIndexForExternal + offset - newExternalTracks.append(track) + /// Updates the index map with real player indexes for sidecar subtitles. + /// + /// Called after VLC reports its actual track list. Sidecar subtitles are loaded as "playback children" + /// at runtime, so their player-assigned indexes aren't known until the player is running. + /// This takes the existing map from `buildIndexMap` and fills in the sidecar entries. + /// + /// - `Transcode`: HLS has no embedded subtitles, so all reported subtitle tracks are sidecars — matched sequentially. + /// - `DirectPlay`: Container subtitles are already mapped. The remaining unmapped tracks are sidecars. + static func resolveIndexMap( + into indexMap: [Int: Int], + playbackChildren: [MediaStream], + subtitleTracks: [(index: Int, title: String)], + isTranscoding: Bool + ) -> [Int: Int] { + guard !playbackChildren.isEmpty else { return indexMap } + + var updatedMap = indexMap + + let playerIndexes = subtitleTracks + .map(\.index) + .filter { $0 >= 0 } + .sorted() + + if isTranscoding { + for (offset, playerIndex) in playerIndexes.enumerated() { + guard offset < playbackChildren.count, + let jellyfinIndex = playbackChildren[offset].index + else { continue } + updatedMap[jellyfinIndex] = playerIndex + } + } else { + let mappedIndexes = Set(indexMap.values) + let unmappedIndexes = playerIndexes + .filter { !mappedIndexes.contains($0) } + + let externalIndexes: [Int] = [Int](unmappedIndexes.suffix(playbackChildren.count)) + + for (offset, stream) in playbackChildren.enumerated() { + guard let jellyfinIndex = stream.index else { continue } + if offset < externalIndexes.count { + updatedMap[jellyfinIndex] = externalIndexes[offset] + } + } } - return newInternalTracks + newExternalTracks + return updatedMap } var has4KVideo: Bool { diff --git a/Shared/Objects/MediaPlayerManager/MediaPlayerItem/MediaPlayerItem+Build.swift b/Shared/Objects/MediaPlayerManager/MediaPlayerItem/MediaPlayerItem+Build.swift index 297277f4cd..be8f6c28fe 100644 --- a/Shared/Objects/MediaPlayerManager/MediaPlayerItem/MediaPlayerItem+Build.swift +++ b/Shared/Objects/MediaPlayerManager/MediaPlayerItem/MediaPlayerItem+Build.swift @@ -21,6 +21,8 @@ extension MediaPlayerItem { static func build( for initialItem: BaseItemDto, mediaSource _initialMediaSource: MediaSourceInfo? = nil, + audioStreamIndex: Int? = nil, + subtitleStreamIndex: Int? = nil, videoPlayerType: VideoPlayerType = Defaults[.VideoPlayer.videoPlayerType], requestedBitrate: PlaybackBitrate = Defaults[.VideoPlayer.Playback.appMaximumBitrate], compatibilityMode: PlaybackCompatibility = Defaults[.VideoPlayer.Playback.compatibilityMode], @@ -75,6 +77,8 @@ extension MediaPlayerItem { playbackInfo.liveStreamID = initialMediaSource.liveStreamID playbackInfo.maxStreamingBitrate = maxBitrate playbackInfo.userID = userSession.user.id + playbackInfo.audioStreamIndex = audioStreamIndex + playbackInfo.subtitleStreamIndex = subtitleStreamIndex if !item.isLiveStream { playbackInfo.mediaSourceID = initialMediaSource.id @@ -168,6 +172,8 @@ extension MediaPlayerItem { playSessionID: playSessionID, url: playbackURL, requestedBitrate: requestedBitrate, + initialAudioStreamIndex: audioStreamIndex, + initialSubtitleStreamIndex: subtitleStreamIndex, previewImageProvider: previewImageProvider, thumbnailProvider: item.getNowPlayingImage ) diff --git a/Shared/Objects/MediaPlayerManager/MediaPlayerItem/MediaPlayerItem.swift b/Shared/Objects/MediaPlayerManager/MediaPlayerItem/MediaPlayerItem.swift index 0182c98495..a5861eb68a 100644 --- a/Shared/Objects/MediaPlayerManager/MediaPlayerItem/MediaPlayerItem.swift +++ b/Shared/Objects/MediaPlayerManager/MediaPlayerItem/MediaPlayerItem.swift @@ -6,6 +6,7 @@ // Copyright (c) 2026 Jellyfin & Jellyfin Contributors // +import Defaults import JellyfinAPI import SwiftUI @@ -22,21 +23,23 @@ class MediaPlayerItem: ViewModel, MediaPlayerObserver { @Published var selectedAudioStreamIndex: Int? = nil { didSet { - if let proxy = manager?.proxy as? any VideoMediaPlayerProxy { - proxy.setAudioStream(.init(index: selectedAudioStreamIndex)) - } + guard let selectedAudioStreamIndex, selectedAudioStreamIndex != oldValue else { return } + manager?.setTrack(type: .audio, from: oldValue, to: selectedAudioStreamIndex) } } @Published var selectedSubtitleStreamIndex: Int? = nil { didSet { - if let proxy = manager?.proxy as? any VideoMediaPlayerProxy { - proxy.setSubtitleStream(.init(index: selectedSubtitleStreamIndex)) - } + guard selectedSubtitleStreamIndex != oldValue else { return } + manager?.setTrack(type: .subtitle, from: oldValue, to: selectedSubtitleStreamIndex) } } + private(set) var indexMap: [Int: Int] + + private var externalSubtitlesResolved = false + weak var manager: MediaPlayerManager? { didSet { for var o in observers { @@ -48,6 +51,7 @@ class MediaPlayerItem: ViewModel, MediaPlayerObserver { var observers: [any MediaPlayerObserver] = [] let baseItem: BaseItemDto + let deviceProfile: DeviceProfile = .init() let mediaSource: MediaSourceInfo let playSessionID: String let previewImageProvider: (any PreviewImageProvider)? @@ -68,6 +72,8 @@ class MediaPlayerItem: ViewModel, MediaPlayerObserver { playSessionID: String, url: URL, requestedBitrate: PlaybackBitrate = .max, + initialAudioStreamIndex: Int? = nil, + initialSubtitleStreamIndex: Int? = nil, previewImageProvider: (any PreviewImageProvider)? = nil, thumbnailProvider: ThumbnailProvider? = nil ) { @@ -79,24 +85,134 @@ class MediaPlayerItem: ViewModel, MediaPlayerObserver { self.thumbnailProvider = thumbnailProvider self.url = url - let adjustedMediaStreams = mediaSource.mediaStreams?.adjustedTrackIndexes( - for: mediaSource.transcodingURL == nil ? .directPlay : .transcode, - selectedAudioStreamIndex: mediaSource.defaultAudioStreamIndex ?? 0 - ) - - let audioStreams = adjustedMediaStreams?.filter { $0.type == .audio } ?? [] - let subtitleStreams = adjustedMediaStreams?.filter { $0.type == .subtitle } ?? [] - let videoStreams = adjustedMediaStreams?.filter { $0.type == .video } ?? [] - - self.audioStreams = audioStreams - self.subtitleStreams = subtitleStreams - self.videoStreams = videoStreams + let mediaStreams = mediaSource.mediaStreams + let isTranscoding = mediaSource.transcodingURL != nil + + // TODO: Fix External Audio Tracks & Re-Enable + self.audioStreams = mediaStreams?.filter { $0.type == .audio && $0.isExternal != true } ?? [] + self.subtitleStreams = mediaStreams?.filter { + $0.type == .subtitle + && $0.deliveryMethod != .drop + && !(Defaults[.VideoPlayer.Playback.compatibilityMode] == .directPlay + && $0.isExternal == true + && $0.isTextSubtitleStream != true) + } ?? [] + self.videoStreams = mediaStreams?.filter { $0.type == .video } ?? [] + + let resolvedAudioStreamIndex = initialAudioStreamIndex + ?? mediaSource.defaultAudioStreamIndex + ?? mediaSource.mediaStreams?.first(where: { $0.type == .audio })?.index ?? 0 + + self.indexMap = mediaStreams?.buildIndexMap( + for: isTranscoding ? .transcode : .directPlay, + selectedAudioStreamIndex: resolvedAudioStreamIndex + ) ?? [:] super.init() - selectedAudioStreamIndex = mediaSource.defaultAudioStreamIndex ?? -1 - selectedSubtitleStreamIndex = mediaSource.defaultSubtitleStreamIndex ?? -1 + selectedAudioStreamIndex = resolvedAudioStreamIndex + + selectedSubtitleStreamIndex = initialSubtitleStreamIndex + ?? mediaSource.defaultSubtitleStreamIndex + ?? -1 observers.append(MediaProgressObserver(item: self)) } + + /// Decides whether a track change can be performed by the player in place, or whether the server must produce a new stream. + func isRebuildRequired(type: MediaStreamType, from oldIndex: Int?, to newIndex: Int?) -> Bool { + let isTranscoding = mediaSource.transcodingURL != nil + + // Disabling a track is ALWAYS a local-only operation. + guard let newIndex, newIndex != -1 else { return false } + + switch type { + case .audio: + + // Transcodes contain a single audio track and MUST rebuild. + if isTranscoding { return true } + + guard let newStream = audioStreams.first(where: { $0.index == newIndex }) else { return true } + + // TODO: When audio playback exists then get the type dynamically. + return !deviceProfile.canPlay( + type: .video, + audioCodec: newStream.codec, + container: mediaSource.container + ) + + case .subtitle: + // Optional (do not guard) since this could be -1 for disabled. + let oldStream = oldIndex.flatMap { idx in subtitleStreams.first { $0.index == idx } } + + // Transitioning away from encoded subtitles always requires a rebuild so the server stops burning them into the video. + if oldStream?.deliveryMethod == .encode { return true } + + // Catch if the new stream doesn't exist. If non-existent this will fallback to -1 and disable locally. + guard let newStream = subtitleStreams.first(where: { $0.index == newIndex }) else { return false } + + if newStream.isExternal == true { + + // External subtitles can only be loaded as sidecars when the profile allows external or HLS delivery for the format. + // E.G, This should disable external PGS for VLC since VLC cannot play them. + return !(deviceProfile.canPlay(subtitleFormat: newStream.codec, method: .external) + || deviceProfile.canPlay(subtitleFormat: newStream.codec, method: .hls)) + } + + // Embedded subtitles are in the source container. + // Only reachable while direct-playing AND when the profile supports embed delivery. + return isTranscoding || !deviceProfile.canPlay(subtitleFormat: newStream.codec, method: .embed) + + default: + return false + } + } + + /// Switches an audio, subtitle, or lyric* track in the player without rebuilding the stream. + /// *Lyrics stubs are there but needs a `MediaPlayerLyricTrackConfigurable` + func switchTrack(type: MediaStreamType, index: Int?) { + let playerIndex: Int + if index == nil || index == -1 { + playerIndex = -1 + } else if let mapped = indexMap[index] { + playerIndex = mapped + } else { + return + } + + switch type { + case .audio: + guard let proxy = manager?.proxy as? any MediaPlayerAudioTrackConfigurable else { return } + proxy.setAudioStream(.init(index: playerIndex)) + case .subtitle: + guard let proxy = manager?.proxy as? any MediaPlayerSubtitleTrackConfigurable else { return } + proxy.setSubtitleStream(.init(index: playerIndex)) + case .lyric: + + // TODO: Enable for Audio Player when Lyrics are needed. + // guard let proxy = manager?.proxy as? any MediaPlayerLyricTrackConfigurable else { return } + // proxy.setLyricStream(.init(index: playerIndex)) + return + default: + return + } + } + + /// Get subtitle mapped subtitle track indexes from `playbackChildren` + func getSubtitleIndexes(subtitleTracks: [(index: Int, title: String)]) { + guard !externalSubtitlesResolved else { return } + externalSubtitlesResolved = true + + let playbackChildren = subtitleStreams.sidecarSubtitles + guard playbackChildren.isNotEmpty else { return } + + indexMap = [MediaStream].resolveIndexMap( + into: indexMap, + playbackChildren: playbackChildren, + subtitleTracks: subtitleTracks, + isTranscoding: mediaSource.transcodingURL != nil + ) + + switchTrack(type: .subtitle, index: selectedSubtitleStreamIndex) + } } diff --git a/Shared/Objects/MediaPlayerManager/MediaPlayerManager.swift b/Shared/Objects/MediaPlayerManager/MediaPlayerManager.swift index da70858ad5..eb3cd18701 100644 --- a/Shared/Objects/MediaPlayerManager/MediaPlayerManager.swift +++ b/Shared/Objects/MediaPlayerManager/MediaPlayerManager.swift @@ -54,8 +54,10 @@ final class MediaPlayerManager: ViewModel { case ended case error case playNewItem(provider: MediaPlayerItemProvider) + case setBitrate(bitrate: PlaybackBitrate) case setPlaybackRequestStatus(status: PlaybackRequestStatus) case setRate(rate: Float) + case setTrack(type: MediaStreamType, from: Int?, to: Int? = nil) case start case stop case togglePlayPause @@ -173,8 +175,6 @@ final class MediaPlayerManager: ViewModel { } } - private var itemBuildTask: AnyCancellable? - private var initialMediaPlayerItemProvider: MediaPlayerItemProvider? // MARK: init @@ -278,6 +278,16 @@ final class MediaPlayerManager: ViewModel { playbackItem = try await provider() } + @Function(\Action.Cases.setBitrate) + private func _setBitrate(_ requestedBitrate: PlaybackBitrate) async throws { + guard let currentItem = playbackItem else { return } + + try await updateMediaPlayerItem( + currentItem: currentItem, + requestedBitrate: requestedBitrate + ) + } + @Function(\Action.Cases.setPlaybackRequestStatus) private func set(_ status: PlaybackRequestStatus) { if self.playbackRequestStatus != status { @@ -299,6 +309,47 @@ final class MediaPlayerManager: ViewModel { } } + @Function(\Action.Cases.setTrack) + private func _setTrack(_ type: MediaStreamType, _ oldIndex: Int?, _ newIndex: Int?) async throws { + guard let playbackItem else { + logger.warning("MediaPlayerManager.SetTrack call with an invalid playbackItem") + return + } + + switch type { + case .audio: + guard playbackItem.audioStreams.contains(where: { $0.index == oldIndex }) else { + logger.warning("MediaPlayerManager.SetTrack call with an invalid audio track index") + return + } + + if playbackItem.isRebuildRequired(type: .audio, from: oldIndex, to: newIndex) { + try await updateMediaPlayerItem( + currentItem: playbackItem, + audioStreamIndex: newIndex + ) + } else { + playbackItem.switchTrack(type: .audio, index: newIndex) + } + case .subtitle: + guard playbackItem.subtitleStreams.contains(where: { $0.index == oldIndex }) else { + logger.warning("MediaPlayerManager.SetTrack call with an invalid subtitle track index") + return + } + + if playbackItem.isRebuildRequired(type: .subtitle, from: oldIndex, to: newIndex) { + try await updateMediaPlayerItem( + currentItem: playbackItem, + subtitleStreamIndex: newIndex + ) + } else { + playbackItem.switchTrack(type: .subtitle, index: newIndex) + } + default: + logger.warning("MediaPlayerManager.SetTrack called with unsupported type: \(String(describing: type))") + } + } + @Function(\Action.Cases.start) private func _start() async throws { guard let initialMediaPlayerItemProvider else { @@ -309,13 +360,12 @@ final class MediaPlayerManager: ViewModel { playbackItem = try await initialMediaPlayerItemProvider() } + // TODO: remove playback item? + // - check that observers would respond correctly to stopping @Function(\Action.Cases.stop) private func _stop() async throws { await self.cancel() - // TODO: remove playback item? - // - check that observers would respond correctly to stopping - itemBuildTask?.cancel() proxy?.stop() Container.shared.mediaPlayerManager.reset() } @@ -329,4 +379,56 @@ final class MediaPlayerManager: ViewModel { setPlaybackRequestStatus(status: .playing) } } + + /// Rebuilds the playback item with new stream indexes / bitrate. + /// Stops the current proxy, requests new playback info from the server, and starts playback with the new configuration. + /// + /// Rebuilds the current item + private func updateMediaPlayerItem( + currentItem: MediaPlayerItem, + audioStreamIndex: Int? = nil, + subtitleStreamIndex: Int? = nil, + requestedBitrate: PlaybackBitrate? = nil + ) async throws { + + // Capture the current playback position before stopping + let currentSeconds = self.seconds + + logger.info( + "Rebuilding Media Player Item", + metadata: [ + "audioIndex": "\(audioStreamIndex ?? -1)", + "subtitleIndex": "\(subtitleStreamIndex ?? -1)", + "currentSeconds": "\(currentSeconds)", + ] + ) + + proxy?.stop() + + let newItem = try await MediaPlayerItem.build( + for: currentItem.baseItem, + mediaSource: currentItem.mediaSource, + audioStreamIndex: audioStreamIndex ?? currentItem.selectedAudioStreamIndex, + subtitleStreamIndex: subtitleStreamIndex ?? currentItem.selectedSubtitleStreamIndex, + requestedBitrate: requestedBitrate ?? currentItem.requestedBitrate, + modifyItem: { item in + if item.userData == nil { + item.userData = UserItemDataDto() + } + item.userData?.playbackPositionTicks = currentSeconds.ticks + } + ) + + logger.info( + "Built new playback item", + metadata: [ + "playSessionID": "\(newItem.playSessionID)", + "isTranscoding": "\(newItem.mediaSource.transcodingURL != nil)", + "url": "\(newItem.url.absoluteString)", + ] + ) + + self.playbackItem = newItem + self.seconds = currentSeconds + } } diff --git a/Shared/Objects/MediaPlayerManager/MediaPlayerProxy/MediaPlayerProxy+VLC.swift b/Shared/Objects/MediaPlayerManager/MediaPlayerProxy/MediaPlayerProxy+VLC.swift index b83a80de92..ba14e2ef22 100644 --- a/Shared/Objects/MediaPlayerManager/MediaPlayerProxy/MediaPlayerProxy+VLC.swift +++ b/Shared/Objects/MediaPlayerManager/MediaPlayerProxy/MediaPlayerProxy+VLC.swift @@ -146,8 +146,17 @@ extension VLCMediaPlayerProxy { if !baseItem.isLiveStream { configuration.startSeconds = startSeconds - configuration.audioIndex = .absolute(mediaSource.defaultAudioStreamIndex ?? -1) - configuration.subtitleIndex = .absolute(mediaSource.defaultSubtitleStreamIndex ?? -1) + + let subtitleIndex = item.indexMap[item.selectedSubtitleStreamIndex] ?? -1 + + if mediaSource.transcodingURL != nil { + configuration.audioIndex = .auto + } else { + let audioIndex = item.indexMap[item.selectedAudioStreamIndex] ?? -1 + configuration.audioIndex = .absolute(audioIndex) + } + + configuration.subtitleIndex = .absolute(subtitleIndex) } configuration.subtitleSize = .absolute(25 - Defaults[.VideoPlayer.Subtitle.subtitleSize]) @@ -157,8 +166,7 @@ extension VLCMediaPlayerProxy { configuration.subtitleFont = .absolute(font) } - configuration.playbackChildren = item.subtitleStreams - .filter { $0.deliveryMethod == .external } + configuration.playbackChildren = item.subtitleStreams.sidecarSubtitles .compactMap(\.asVLCPlaybackChild) return configuration @@ -192,7 +200,7 @@ extension VLCMediaPlayerProxy { manager.proxy?.isBuffering.value = true case .ended: // Live streams will send stopped/ended events - guard !playbackItem.baseItem.isLiveStream else { return } + guard !(manager.playbackItem?.baseItem.isLiveStream ?? false) else { return } manager.proxy?.isBuffering.value = false manager.ended() case .stopped: () @@ -205,6 +213,9 @@ extension VLCMediaPlayerProxy { case .playing: manager.proxy?.isBuffering.value = false manager.setPlaybackRequestStatus(status: .playing) + + let tracks = info.subtitleTracks.map { (index: $0.index, title: $0.title) } + manager.playbackItem?.getSubtitleIndexes(subtitleTracks: tracks) case .paused: manager.setPlaybackRequestStatus(status: .paused) } diff --git a/Shared/Objects/MediaPlayerManager/MediaPlayerProxy/MediaPlayerProxy.swift b/Shared/Objects/MediaPlayerManager/MediaPlayerProxy/MediaPlayerProxy.swift index ad105958e6..f9c4cac2d0 100644 --- a/Shared/Objects/MediaPlayerManager/MediaPlayerProxy/MediaPlayerProxy.swift +++ b/Shared/Objects/MediaPlayerManager/MediaPlayerProxy/MediaPlayerProxy.swift @@ -31,7 +31,7 @@ protocol MediaPlayerProxy: ObservableObject, MediaPlayerObserver { } @MainActor -protocol VideoMediaPlayerProxy: MediaPlayerProxy { +protocol VideoMediaPlayerProxy: MediaPlayerProxy, MediaPlayerAudioTrackConfigurable, MediaPlayerSubtitleTrackConfigurable { associatedtype VideoPlayerBody: View @@ -41,14 +41,20 @@ protocol VideoMediaPlayerProxy: MediaPlayerProxy { // TODO: remove when container view handles aspect fill func setAspectFill(_ aspectFill: Bool) - func setAudioStream(_ stream: MediaStream) - func setSubtitleStream(_ stream: MediaStream) @ViewBuilder @MainActor var videoPlayerBody: Self.VideoPlayerBody { get } } +protocol MediaPlayerAudioTrackConfigurable { + func setAudioStream(_ stream: MediaStream) +} + +protocol MediaPlayerSubtitleTrackConfigurable { + func setSubtitleStream(_ stream: MediaStream) +} + protocol MediaPlayerOffsetConfigurable { func setAudioOffset(_ seconds: Duration) func setSubtitleOffset(_ seconds: Duration) diff --git a/Shared/Objects/MediaPlayerManager/MediaProgressObserver.swift b/Shared/Objects/MediaPlayerManager/MediaProgressObserver.swift index c64a61ceae..721cc35823 100644 --- a/Shared/Objects/MediaPlayerManager/MediaProgressObserver.swift +++ b/Shared/Objects/MediaPlayerManager/MediaProgressObserver.swift @@ -137,6 +137,7 @@ class MediaProgressObserver: ViewModel, MediaPlayerObserver { var info = PlaybackStopInfo() info.itemID = item.baseItem.id info.mediaSourceID = item.mediaSource.id + info.playSessionID = item.playSessionID info.positionTicks = seconds?.ticks info.sessionID = item.playSessionID diff --git a/Shared/Objects/VideoPlayerType/VideoPlayerType+Swiftfin.swift b/Shared/Objects/VideoPlayerType/VideoPlayerType+Swiftfin.swift index db8ae0c55b..f3a944506d 100644 --- a/Shared/Objects/VideoPlayerType/VideoPlayerType+Swiftfin.swift +++ b/Shared/Objects/VideoPlayerType/VideoPlayerType+Swiftfin.swift @@ -155,12 +155,9 @@ extension VideoPlayerType { SubtitleProfile.build(method: .external) { SubtitleFormat.ass - SubtitleFormat.dvbsub - SubtitleFormat.dvdsub SubtitleFormat.jacosub SubtitleFormat.libzvbi_teletextdec SubtitleFormat.mpl2 - SubtitleFormat.pgssub SubtitleFormat.pjs SubtitleFormat.realtext SubtitleFormat.sami @@ -172,6 +169,12 @@ extension VideoPlayerType { SubtitleFormat.ttml SubtitleFormat.vplayer SubtitleFormat.vtt + } + + SubtitleProfile.build(method: .encode) { + SubtitleFormat.dvbsub + SubtitleFormat.dvdsub + SubtitleFormat.pgssub SubtitleFormat.xsub } } diff --git a/Swiftfin.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Swiftfin.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 0f32055b27..c32b402b74 100644 --- a/Swiftfin.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Swiftfin.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -159,8 +159,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/kean/Nuke", "state" : { - "revision" : "1248465fc4e41ee2fdbe253b8b14d5225b04093b", - "version" : "13.0.2" + "revision" : "2c5f7a3ccac66ae7c94d6f75c2080533f6cca0ac", + "version" : "13.0.4" } }, {