diff --git a/AK/RedBlackTree.h b/AK/RedBlackTree.h index 60033af3ca207..64b500f808498 100644 --- a/AK/RedBlackTree.h +++ b/AK/RedBlackTree.h @@ -559,6 +559,25 @@ class RedBlackTree final : public BaseRedBlackTree { return temp; } + V remove_and_advance(Iterator& iterator) + { + VERIFY(!iterator.is_end()); + + auto* node = iterator.m_node; + auto* successor = static_cast(BaseTree::successor(node)); + iterator.m_node = successor; + + BaseTree::remove(node); + + V temp = move(node->value); + + node->right_child = nullptr; + node->left_child = nullptr; + delete node; + + return temp; + } + bool remove(K key) { auto* node = BaseTree::find(this->m_root, key); diff --git a/Libraries/LibMedia/CMakeLists.txt b/Libraries/LibMedia/CMakeLists.txt index f55cf0f414cce..e779e064ff2ed 100644 --- a/Libraries/LibMedia/CMakeLists.txt +++ b/Libraries/LibMedia/CMakeLists.txt @@ -2,8 +2,18 @@ include(audio) set(SOURCES Audio/AudioDevices.cpp + Codecs/FLAC.cpp + Codecs/Opus.cpp + Codecs/Vorbis.cpp + Containers/ConstantBitrateContainerNavigator.cpp + Containers/FLACNavigator.cpp + Containers/IndexedContainerNavigator.cpp + Containers/MP3Navigator.cpp + Containers/OggNavigator.cpp Containers/Matroska/MatroskaDemuxer.cpp Containers/Matroska/Reader.cpp + Containers/Matroska/SampleIterator.cpp + Containers/Matroska/Streamer.cpp GenericTimeProvider.cpp IncrementallyPopulatedStream.cpp PlaybackManager.cpp diff --git a/Libraries/LibMedia/Codecs/FLAC.cpp b/Libraries/LibMedia/Codecs/FLAC.cpp new file mode 100644 index 0000000000000..9cc194df94fea --- /dev/null +++ b/Libraries/LibMedia/Codecs/FLAC.cpp @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include + +namespace Media::Codecs { + +template +static Optional read_big_endian_value(MediaStreamCursor& cursor) +{ + auto value = cursor.read_value(AK::Endianness::Big); + if (value.is_error()) + return {}; + return value.release_value(); +} + +static Optional read_coded_number(MediaStreamCursor& cursor) +{ + auto first_byte = read_big_endian_value(cursor); + if (!first_byte.has_value()) + return {}; + u64 value = first_byte.value(); + + auto length = count_leading_zeroes_safe(static_cast(~value)); + if (length == 0) + return value; + if (length == 1) + return {}; + if (length > 7) + return {}; + + value &= (1 << (8 - (length + 1))) - 1; + while (--length > 0) { + auto continuation_byte = read_big_endian_value(cursor); + if (!continuation_byte.has_value()) + return {}; + if (continuation_byte.value() >> 6 != 0b10) + return {}; + value <<= 6; + value |= continuation_byte.value() & 0b111111; + } + + return value; +} + +static bool verify_header_crc(MediaStreamCursor& cursor, size_t header_start) +{ + constexpr auto lookup_table = [] { + Array result; + for (size_t i = 0; i < result.size(); i++) { + u8 value = i; + for (auto j = 0; j < 8; j++) + value = (value << 1) ^ (value & 0x80 ? 0x07 : 0); + result[i] = value; + } + return result; + }(); + + auto header_end = cursor.position(); + VERIFY(header_end > header_start); + auto header_size = header_end - header_start; + Array buffer; + if (header_size > buffer.size()) + return false; + + auto crc = read_big_endian_value(cursor); + if (!crc.has_value()) + return false; + + if (cursor.seek_to_position(header_start).is_error()) + return false; + auto header_data = buffer.span().trim(header_size); + if (cursor.read_until_filled(header_data).is_error()) + return false; + if (cursor.skip(1).is_error()) + return false; + + u8 actual_crc = 0; + for (auto const& byte : header_data) + actual_crc = lookup_table[actual_crc ^ byte]; + return crc.value() == actual_crc; +} + +bool FLAC::is_sync_code(u16 value) +{ + return (value >> 1) == 0b111111111111100; +} + +Optional FLAC::parse_frame_header(MediaStreamCursor& cursor, u16 sync_code, u16 fixed_block_size) +{ + auto header_start = cursor.position(); + + Array header; + if (cursor.read_until_filled(header).is_error()) + return {}; + + u16 maybe_sync_code = (static_cast(header[0]) << 8) | header[1]; + if (maybe_sync_code != sync_code) + return {}; + + bool variable_block_size = (sync_code & 1) != 0; + u8 block_size_bits = (header[2] >> 4) & 0x0F; + u8 sample_rate_bits = header[2] & 0x0F; + u8 channels_bits = (header[3] >> 4) & 0x0F; + u8 bit_depth_bits = (header[3] >> 1) & 0x07; + u8 reserved_bit = header[3] & 0x01; + + if (reserved_bit != 0) + return {}; + + auto coded_number = read_coded_number(cursor); + if (!coded_number.has_value()) + return {}; + + u16 block_size = 0; + if (block_size_bits == 0b0000) + return {}; + if (block_size_bits == 0b0001) { + block_size = 192; + } else if (block_size_bits <= 0b0101) { + block_size = 144 * (1 << block_size_bits); + } else if (block_size_bits == 0b0110) { + auto uncommon_block_size_minus_1 = read_big_endian_value(cursor); + if (!uncommon_block_size_minus_1.has_value()) + return {}; + block_size = uncommon_block_size_minus_1.value() + 1; + } else if (block_size_bits == 0b0111) { + auto uncommon_block_size_minus_1 = read_big_endian_value(cursor); + if (!uncommon_block_size_minus_1.has_value()) + return {}; + block_size = uncommon_block_size_minus_1.value() + 1; + } else if (block_size_bits >= 0b1000) { + block_size = 1 << block_size_bits; + } + + if (sample_rate_bits == 0b1100 && cursor.skip(1).is_error()) + return {}; + if ((sample_rate_bits == 0b1101 || sample_rate_bits == 0b1110) && cursor.skip(2).is_error()) + return {}; + if (sample_rate_bits == 0b1111) + return {}; + + if (channels_bits >= 0b1011) + return {}; + + if (bit_depth_bits == 0b011) + return {}; + + if (!verify_header_crc(cursor, header_start)) + return {}; + + u64 sample_number = coded_number.value(); + if (!variable_block_size) + sample_number *= fixed_block_size; + + return FrameInfo { sample_number, block_size }; +} + +} diff --git a/Libraries/LibMedia/Codecs/FLAC.h b/Libraries/LibMedia/Codecs/FLAC.h new file mode 100644 index 0000000000000..430587a68f596 --- /dev/null +++ b/Libraries/LibMedia/Codecs/FLAC.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include + +namespace Media::Codecs { + +class FLAC { +public: + struct FrameInfo { + u64 sample_number; + u16 block_size; + }; + + static bool is_sync_code(u16); + static Optional parse_frame_header(MediaStreamCursor&, u16 sync_code, u16 fixed_block_size); +}; + +} diff --git a/Libraries/LibMedia/Codecs/Opus.cpp b/Libraries/LibMedia/Codecs/Opus.cpp new file mode 100644 index 0000000000000..96d47c74fc6ec --- /dev/null +++ b/Libraries/LibMedia/Codecs/Opus.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include + +namespace Media::Codecs { + +DecoderErrorOr Opus::parse_frame_duration_in_samples(MediaStreamCursor& cursor, size_t frame_size) +{ + if (frame_size == 0) + return DecoderError::corrupted("Opus frame is too small"sv); + + // https://datatracker.ietf.org/doc/html/rfc6716#section-3.1 + auto toc_byte = TRY(cursor.read_value()); + auto configuration_number = (toc_byte >> 3) & 0b1'1111; + auto packet_code = toc_byte & 0b11; + + // clang-format off + constexpr Array frame_durations = { + 10000, 20000, 40000, 60000, // SILK-only NB + 10000, 20000, 40000, 60000, // SILK-only MB + 10000, 20000, 40000, 60000, // SILK-only WB + 10000, 20000, // Hybrid SWB + 10000, 20000, // Hybrid FB + 2500, 5000, 10000, 20000, // CELT-only NB + 2500, 5000, 10000, 20000, // CELT-only WB + 2500, 5000, 10000, 20000, // CELT-only SWB + 2500, 5000, 10000, 20000, // CELT-only FB + }; + // clang-format on + + auto frame_duration = frame_durations[configuration_number]; + auto packet_duration = TRY([&] -> DecoderErrorOr { + switch (packet_code) { + case 0: + return frame_duration; + case 1: + case 2: + return frame_duration * 2; + case 3: { + if (frame_size == 1) + return DecoderError::corrupted("Opus frame is too small"sv); + auto frame_count_byte = TRY(cursor.read_value()); + auto frame_count = frame_count_byte & 0b11'1111; + return frame_duration * frame_count; + } + default: + VERIFY_NOT_REACHED(); + } + }()); + + return packet_duration * 48 / 1000; +} + +DecoderErrorOr Opus::parse_frame_duration(MediaStreamCursor& cursor, size_t frame_size) +{ + return AK::Duration::from_time_units(TRY(parse_frame_duration_in_samples(cursor, frame_size)), 1, 48000); +} + +} diff --git a/Libraries/LibMedia/Codecs/Opus.h b/Libraries/LibMedia/Codecs/Opus.h new file mode 100644 index 0000000000000..93ac4f607a59f --- /dev/null +++ b/Libraries/LibMedia/Codecs/Opus.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include + +namespace Media::Codecs { + +class Opus { +public: + static DecoderErrorOr parse_frame_duration(MediaStreamCursor&, size_t frame_size); + static DecoderErrorOr parse_frame_duration_in_samples(MediaStreamCursor&, size_t frame_size); +}; + +} diff --git a/Libraries/LibMedia/Codecs/Vorbis.cpp b/Libraries/LibMedia/Codecs/Vorbis.cpp new file mode 100644 index 0000000000000..cfc8d0835f4a6 --- /dev/null +++ b/Libraries/LibMedia/Codecs/Vorbis.cpp @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include + +extern "C" { +#include +} + +namespace Media::Codecs { + +Optional Vorbis::Parser::create(ReadonlyBytes codec_initialization_data) +{ + if (codec_initialization_data.is_empty()) + return {}; + + auto* parser = av_vorbis_parse_init(codec_initialization_data.data(), AK::clamp_to(codec_initialization_data.size())); + if (!parser) + return {}; + return Parser { parser }; +} + +Vorbis::Parser::Parser(AVVorbisParseContext* context) + : m_context(context) +{ + VERIFY(m_context); +} + +Vorbis::Parser::Parser(Parser&& other) + : m_context(exchange(other.m_context, nullptr)) +{ +} + +Vorbis::Parser& Vorbis::Parser::operator=(Parser&& other) +{ + if (this == &other) + return *this; + + if (m_context) + av_vorbis_parse_free(&m_context); + m_context = exchange(other.m_context, nullptr); + return *this; +} + +Vorbis::Parser::~Parser() +{ + if (m_context) + av_vorbis_parse_free(&m_context); +} + +void Vorbis::Parser::reset() const +{ + av_vorbis_parse_reset(m_context); +} + +Optional Vorbis::Parser::parse_packet_duration_in_samples(ReadonlyBytes packet) const +{ + int flags = 0; + auto duration = av_vorbis_parse_frame_flags(m_context, packet.data(), AK::clamp_to(packet.size()), &flags); + if (duration < 0) + return {}; + return duration; +} + +Optional Vorbis::Parser::parse_packet_duration(ReadonlyBytes packet, u32 sample_rate) const +{ + auto duration = parse_packet_duration_in_samples(packet); + if (!duration.has_value()) + return {}; + return AK::Duration::from_time_units(duration.value(), 1, sample_rate); +} + +} diff --git a/Libraries/LibMedia/Codecs/Vorbis.h b/Libraries/LibMedia/Codecs/Vorbis.h new file mode 100644 index 0000000000000..f5e81a662d6ec --- /dev/null +++ b/Libraries/LibMedia/Codecs/Vorbis.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include + +struct AVVorbisParseContext; + +namespace Media::Codecs { + +class Vorbis { +public: + class Parser { + AK_MAKE_NONCOPYABLE(Parser); + + public: + static Optional create(ReadonlyBytes codec_initialization_data); + + Parser(Parser&&); + Parser& operator=(Parser&&); + ~Parser(); + + void reset() const; + Optional parse_packet_duration_in_samples(ReadonlyBytes packet) const; + Optional parse_packet_duration(ReadonlyBytes packet, u32 sample_rate) const; + + private: + explicit Parser(AVVorbisParseContext*); + + AVVorbisParseContext* m_context { nullptr }; + }; +}; + +} diff --git a/Libraries/LibMedia/Containers/ConstantBitrateContainerNavigator.cpp b/Libraries/LibMedia/Containers/ConstantBitrateContainerNavigator.cpp new file mode 100644 index 0000000000000..565e82a9c634d --- /dev/null +++ b/Libraries/LibMedia/Containers/ConstantBitrateContainerNavigator.cpp @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include + +#include "ConstantBitrateContainerNavigator.h" + +namespace Media { + +TimeRanges ConstantBitrateContainerNavigator::buffered_time_ranges(Vector const& byte_ranges) const +{ + if (byte_ranges.is_empty()) + return {}; + + TimeRanges ranges; + + for (auto const& byte_range : byte_ranges) { + if (byte_range.end <= m_first_sample_position) + continue; + + auto data_start = (byte_range.start > m_first_sample_position) ? byte_range.start - m_first_sample_position : 0; + auto data_end = byte_range.end - m_first_sample_position; + + auto time_start = AK::Duration::from_time_units(AK::clamp_to(data_start), 1, m_bytes_per_second); + auto time_end = AK::Duration::from_time_units(AK::clamp_to(data_end), 1, m_bytes_per_second); + + ranges.add_range(max(AK::Duration::zero(), time_start), time_end); + } + + return ranges; +} + +DecoderErrorOr ConstantBitrateContainerNavigator::seek_to_timestamp(AK::Duration timestamp) const +{ + timestamp = max(timestamp, AK::Duration::zero()); + auto byte_offset = timestamp.to_time_units(1, m_bytes_per_second); + byte_offset -= byte_offset % static_cast(m_block_align); + auto byte_position = static_cast(m_first_sample_position) + byte_offset; + auto resolved_timestamp = AK::Duration::from_time_units(byte_offset, 1, m_bytes_per_second); + return SeekedPosition { byte_position, resolved_timestamp }; +} + +} diff --git a/Libraries/LibMedia/Containers/ConstantBitrateContainerNavigator.h b/Libraries/LibMedia/Containers/ConstantBitrateContainerNavigator.h new file mode 100644 index 0000000000000..a9ef3b83e7b52 --- /dev/null +++ b/Libraries/LibMedia/Containers/ConstantBitrateContainerNavigator.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include "ContainerNavigator.h" + +namespace Media { + +class ConstantBitrateContainerNavigator final : public ContainerNavigator { +public: + ConstantBitrateContainerNavigator(size_t first_sample_position, u32 bytes_per_second, u32 block_align) + : m_first_sample_position(first_sample_position) + , m_bytes_per_second(bytes_per_second) + , m_block_align(block_align) + { + } + + TimeRanges buffered_time_ranges(Vector const& byte_ranges) const override; + virtual DecoderErrorOr seek_to_timestamp(AK::Duration timestamp) const override; + +private: + size_t m_first_sample_position; + size_t m_bytes_per_second; + size_t m_block_align; +}; + +} diff --git a/Libraries/LibMedia/Containers/ContainerNavigator.h b/Libraries/LibMedia/Containers/ContainerNavigator.h new file mode 100644 index 0000000000000..5f0fbf438246e --- /dev/null +++ b/Libraries/LibMedia/Containers/ContainerNavigator.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace Media { + +struct SeekSkipped { }; + +struct SeekedPosition { + i64 byte_position; + AK::Duration timestamp; +}; + +using SeekResult = Variant; + +class ContainerNavigator { +public: + virtual ~ContainerNavigator() = default; + virtual DecoderErrorOr seek_to_timestamp(AK::Duration) const { return Empty {}; } + virtual TimeRanges buffered_time_ranges(Vector const& byte_ranges) const = 0; +}; + +} diff --git a/Libraries/LibMedia/Containers/FLACNavigator.cpp b/Libraries/LibMedia/Containers/FLACNavigator.cpp new file mode 100644 index 0000000000000..0007c352f62af --- /dev/null +++ b/Libraries/LibMedia/Containers/FLACNavigator.cpp @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include + +#include "FLACNavigator.h" + +namespace Media { + +OwnPtr FLACNavigator::create(ReadonlyBytes first_frame, NonnullRefPtr cursor, u32 sample_rate) +{ + if (first_frame.size() < 2) + return {}; + u16 sync_code = (static_cast(first_frame[0]) << 8) | first_frame[1]; + if (!Codecs::FLAC::is_sync_code(sync_code)) + return {}; + + bool is_fixed_blocksize = (sync_code & 1) == 0; + + auto frame_cursor = make_ref_counted(first_frame); + auto navigator = adopt_own(*new (nothrow) FLACNavigator(move(cursor), sample_rate, sync_code, 0)); + auto frame_info = Codecs::FLAC::parse_frame_header(frame_cursor, sync_code, 0); + if (!frame_info.has_value()) + return {}; + + if (is_fixed_blocksize) + navigator->m_fixed_block_size = frame_info->block_size; + + return navigator; +} + +static constexpr size_t SCAN_CHUNK_SIZE = 4096; + +Optional FLACNavigator::find_first_frame(MediaStreamCursor& cursor, size_t search_start, size_t search_end) const +{ + VERIFY(search_start <= search_end); + + Array chunk; + auto chunk_start = search_start; + + while (chunk_start < search_end) { + if (cursor.seek_to_position(chunk_start).is_error()) + return {}; + auto read_result = cursor.read_into(chunk); + if (read_result.is_error()) + return {}; + auto bytes_read = read_result.value(); + if (bytes_read < 2) + return {}; + + for (size_t i = 0; i + 1 < bytes_read; i++) { + if (chunk[i] != 0xFF) + continue; + u16 maybe_sync_code = (static_cast(chunk[i]) << 8) | chunk[i + 1]; + if (maybe_sync_code != m_sync_code) + continue; + + auto sync_position = chunk_start + i; + if (cursor.seek_to_position(sync_position).is_error()) + return {}; + auto frame_info = Codecs::FLAC::parse_frame_header(cursor, m_sync_code, m_fixed_block_size); + if (frame_info.has_value()) + return frame_info; + } + + chunk_start += bytes_read - 1; + } + return {}; +} + +Optional FLACNavigator::find_last_frame(MediaStreamCursor& cursor, size_t search_start, size_t search_end) const +{ + VERIFY(search_start <= search_end); + + Array chunk; + auto chunk_end = search_end; + + while (chunk_end > search_start) { + auto chunk_start = chunk_end > SCAN_CHUNK_SIZE ? chunk_end - SCAN_CHUNK_SIZE : 0; + chunk_start = max(chunk_start, search_start); + VERIFY(chunk_start <= chunk_end); + auto chunk_size = chunk_end - chunk_start; + if (chunk_size < 2) + return {}; + + if (cursor.seek_to_position(chunk_start).is_error()) + return {}; + if (cursor.read_until_filled(chunk.span().trim(chunk_size)).is_error()) + return {}; + + for (size_t i = chunk_size - 1; i-- > 0;) { + if (chunk[i] != 0xFF) + continue; + u16 maybe_sync_code = (static_cast(chunk[i]) << 8) | chunk[i + 1]; + if (maybe_sync_code != m_sync_code) + continue; + + auto sync_position = chunk_start + i; + if (cursor.seek_to_position(sync_position).is_error()) + return {}; + auto frame_info = Codecs::FLAC::parse_frame_header(cursor, m_sync_code, m_fixed_block_size); + if (frame_info.has_value()) + return frame_info; + } + + chunk_end = chunk_start + 1; + } + + return {}; +} + +AK::Duration FLACNavigator::sample_to_duration(u64 sample) const +{ + return AK::Duration::from_time_units(static_cast(sample), 1, m_sample_rate); +} + +Optional FLACNavigator::scan_forward_for_timestamp(size_t search_start, size_t search_end) const +{ + auto first = find_first_frame(*m_cursor, search_start, search_end); + if (!first.has_value()) + return {}; + return sample_to_duration(first->sample_number); +} + +Optional FLACNavigator::scan_backward_for_end_timestamp(size_t search_start, size_t search_end) const +{ + auto last = find_last_frame(*m_cursor, search_start, search_end); + if (!last.has_value()) + return {}; + return sample_to_duration(last->sample_number + last->block_size); +} + +void FLACNavigator::on_cached_range_changed(FLACCachedRange& cached, CachedRangeChange change) const +{ + bool rescan_start = has_flag(change, CachedRangeChange::Start); + bool rescan_end = has_flag(change, CachedRangeChange::End); + + if (rescan_start && !cached.time_end.has_value()) + rescan_end = true; + if (rescan_end && !cached.time_start.has_value()) + rescan_start = true; + + if (rescan_start) + cached.time_start = scan_forward_for_timestamp(cached.byte_start, cached.byte_end); + if (rescan_end) + cached.time_end = scan_backward_for_end_timestamp(cached.byte_start, cached.byte_end); +} + +void FLACNavigator::append_time_range(FLACCachedRange const& cached_range, TimeRanges& to) +{ + if (!cached_range.time_start.has_value() || !cached_range.time_end.has_value()) + return; + + auto time_start = max(cached_range.time_start.value(), AK::Duration::zero()); + auto time_end = cached_range.time_end.value(); + if (time_start >= time_end) + return; + + to.add_range(time_start, time_end); +} + +} diff --git a/Libraries/LibMedia/Containers/FLACNavigator.h b/Libraries/LibMedia/Containers/FLACNavigator.h new file mode 100644 index 0000000000000..5c9200880c8d0 --- /dev/null +++ b/Libraries/LibMedia/Containers/FLACNavigator.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include + +#include "ScanningContainerNavigator.h" + +namespace Media { + +struct FLACCachedRange : CachedByteRange { + using CachedByteRange::CachedByteRange; + + // Empty means "we scanned but didn't find a valid frame at this boundary". + Optional time_start; + Optional time_end; +}; + +class FLACNavigator final : public ScanningContainerNavigator { +public: + static OwnPtr create(ReadonlyBytes first_frame, NonnullRefPtr cursor, u32 sample_rate); + + void on_cached_range_changed(FLACCachedRange& cached_range, CachedRangeChange change) const; + static void append_time_range(FLACCachedRange const& cached_range, TimeRanges& to); + + Optional scan_forward_for_timestamp(size_t search_start, size_t search_end) const; + Optional scan_backward_for_end_timestamp(size_t search_start, size_t search_end) const; + +private: + FLACNavigator(NonnullRefPtr cursor, u32 sample_rate, u16 sync_code, u16 fixed_block_size) + : m_cursor(move(cursor)) + , m_sample_rate(sample_rate) + , m_sync_code(sync_code) + , m_fixed_block_size(fixed_block_size) + { + } + + Optional find_first_frame(MediaStreamCursor& cursor, size_t search_start, size_t search_end) const; + Optional find_last_frame(MediaStreamCursor& cursor, size_t search_start, size_t search_end) const; + + AK::Duration sample_to_duration(u64 sample) const; + + NonnullRefPtr m_cursor; + u32 m_sample_rate; + u16 m_sync_code; + u16 m_fixed_block_size; +}; + +} diff --git a/Libraries/LibMedia/Containers/IndexEntry.h b/Libraries/LibMedia/Containers/IndexEntry.h new file mode 100644 index 0000000000000..49f560ad321f8 --- /dev/null +++ b/Libraries/LibMedia/Containers/IndexEntry.h @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include + +namespace Media { + +struct IndexEntry { + size_t position; + AK::Duration timestamp; +}; + +} diff --git a/Libraries/LibMedia/Containers/IndexedContainerNavigator.cpp b/Libraries/LibMedia/Containers/IndexedContainerNavigator.cpp new file mode 100644 index 0000000000000..6af916bb3d203 --- /dev/null +++ b/Libraries/LibMedia/Containers/IndexedContainerNavigator.cpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "IndexedContainerNavigator.h" + +namespace Media { + +size_t IndexedContainerNavigator::lower_bound(size_t target) const +{ + size_t lo = 0; + size_t hi = m_entries.size(); + while (lo < hi) { + auto mid = lo + (hi - lo) / 2; + if (m_entries[mid].position < target) + lo = mid + 1; + else + hi = mid; + } + return lo; +} + +TimeRanges IndexedContainerNavigator::buffered_time_ranges(Vector const& byte_ranges) const +{ + if (byte_ranges.is_empty()) + return {}; + + auto entry_count = m_entries.size(); + + TimeRanges ranges; + + for (auto const& byte_range : byte_ranges) { + // Find the first entry at or after the start, and the first entry at or after the end. + auto first = lower_bound(byte_range.start); + auto end = lower_bound(byte_range.end); + if (first >= end) + continue; + + auto time_start = max(AK::Duration::zero(), m_entries[first].timestamp); + auto time_end = (end < entry_count) ? m_entries[end].timestamp : m_duration; + ranges.add_range(time_start, time_end); + } + + return ranges; +} + +} diff --git a/Libraries/LibMedia/Containers/IndexedContainerNavigator.h b/Libraries/LibMedia/Containers/IndexedContainerNavigator.h new file mode 100644 index 0000000000000..c66e3cb8bbc57 --- /dev/null +++ b/Libraries/LibMedia/Containers/IndexedContainerNavigator.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include + +#include "ContainerNavigator.h" + +namespace Media { + +class IndexedContainerNavigator final : public ContainerNavigator { +public: + IndexedContainerNavigator(Vector&& entries, AK::Duration duration) + : m_entries(move(entries)) + , m_duration(duration) + { + } + + TimeRanges buffered_time_ranges(Vector const& byte_ranges) const override; + +private: + size_t lower_bound(size_t target) const; + + Vector m_entries; + AK::Duration m_duration; +}; + +} diff --git a/Libraries/LibMedia/Containers/MP3Navigator.cpp b/Libraries/LibMedia/Containers/MP3Navigator.cpp new file mode 100644 index 0000000000000..ef08cd37dcc62 --- /dev/null +++ b/Libraries/LibMedia/Containers/MP3Navigator.cpp @@ -0,0 +1,630 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "MP3Navigator.h" + +#include +#include +#include +#include + +namespace Media { + +static bool read_exact(MediaStreamCursor& cursor, Bytes buffer) +{ + auto result = cursor.read_into(buffer); + return !result.is_error() && result.value() == buffer.size(); +} + +template +static bool read(MediaStreamCursor& cursor, V& value) +{ + T read_value = 0; + bool result = read_exact(cursor, { &read_value, sizeof(read_value) }); + value = AK::convert_between_host_and_big_endian(read_value); + return result; +} + +template +static bool seek(MediaStreamCursor& cursor, T position) +{ + if (position > NumericLimits::max()) + return false; + auto result = cursor.seek(static_cast(position), SeekMode::SetPosition); + return !result.is_error(); +} + +static constexpr i16 BITRATES[2][3][16] = { + // Version 1 + { + { 0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, -1 }, // Layer III + { 0, 32, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384, -1 }, // Layer II + { 0, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448, -1 }, // Layer I + }, + // Version 2/2.5 + { + { 0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, -1 }, // Layer III + { 0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, -1 }, // Layer II + { 0, 32, 48, 56, 64, 80, 96, 112, 128, 144, 160, 176, 192, 224, 256, -1 }, // Layer I + } +}; + +static constexpr u16 SAMPLES_PER_FRAME[2][3] = { + // Version 1 + { + 1152, // Layer III + 1152, // Layer II + 384, // Layer I + }, + // Version 2/2.5 + { + 576, // Layer III + 1152, // Layer II + 384, // Layer I + }, +}; + +static constexpr u16 SAMPLING_RATES[4][4] = { + { 11025, 12000, 8000, 0 }, // Version 2.5 + { 0, 0, 0, 0 }, // Reserved + { 22050, 24000, 16000, 0 }, // Version 2 + { 44100, 48000, 32000, 0 }, // Version 1 +}; + +template +static bool has_sync_code(T value) +{ + constexpr auto all_bits = static_cast(-1); + constexpr auto shift = NumericLimits::digits() - 11; + constexpr auto sync_code = static_cast(all_bits << shift); + return (value & sync_code) == sync_code; +} + +static bool has_sync_code(ReadonlyBytes bytes, size_t start) +{ + if (bytes.size() < start + 2) + return false; + auto value = static_cast((bytes[start] << 8) | bytes[start + 1]); + return has_sync_code(value); +} + +template +static u8 read_field(T value, u8 offset, u8 size) +{ + constexpr auto bit_count = NumericLimits::digits(); + auto mask = (static_cast(1) << size) - 1; + return (value >> (bit_count - offset - size)) & mask; +} + +static constexpr u64 EXACT_MPEG_AUDIO_DURATION_TIMEBASE = 14'112'000; + +struct FrameInfo { + u16 frame_byte_size { 0 }; + u32 duration_in_ticks { 0 }; +}; + +static bool parse_frame_header(MediaStreamCursor& cursor, FrameInfo& frame_info) +{ + u32 frame_header_data = 0; + if (!read(cursor, frame_header_data)) + return false; + + if (!has_sync_code(frame_header_data)) + return false; + + u8 mpeg_version = read_field(frame_header_data, 11, 2); + if (mpeg_version == 0b01) + return false; + auto is_mpeg_version_2 = mpeg_version != 0b11; + + u8 layer_description = read_field(frame_header_data, 13, 2); + if (layer_description == 0b00) + return false; + auto is_layer_i = layer_description == 0b11; + + auto layer_description_index = layer_description - 1; + u16 frame_samples = SAMPLES_PER_FRAME[is_mpeg_version_2][layer_description_index]; + + u8 bitrate_description = read_field(frame_header_data, 16, 4); + i16 bitrate = BITRATES[is_mpeg_version_2][layer_description_index][bitrate_description]; + if (bitrate <= 0) + return false; + + u8 sampling_frequency_index = read_field(frame_header_data, 20, 2); + u16 sampling_frequency = SAMPLING_RATES[mpeg_version][sampling_frequency_index]; + if (sampling_frequency == 0) + return false; + + u8 padding_bit = read_field(frame_header_data, 22, 1); + + constexpr size_t bytes_per_kb = 1000 / 8; + size_t slot_size = is_layer_i ? 4 : 1; + auto slot_count = static_cast(frame_samples) * static_cast(bitrate) * bytes_per_kb / (sampling_frequency * slot_size); + frame_info.frame_byte_size = static_cast((slot_count + padding_bit) * slot_size); + + frame_info.duration_in_ticks = static_cast(frame_samples) * EXACT_MPEG_AUDIO_DURATION_TIMEBASE / sampling_frequency; + return true; +} + +static AK::Duration duration_from_ticks(u64 ticks) +{ + return AK::Duration::from_time_units(static_cast(ticks), 1, EXACT_MPEG_AUDIO_DURATION_TIMEBASE); +} + +static AK::Duration scale_duration_by_byte_ratio(AK::Duration duration, size_t numerator, size_t denominator) +{ + VERIFY(numerator <= denominator); + VERIFY(denominator > 0); + + auto denominator_bit_width = AK::log2(denominator) + 1; + if (denominator_bit_width > NumericLimits::digits()) { + auto shift = denominator_bit_width - NumericLimits::digits(); + numerator >>= shift; + denominator >>= shift; + } + + return duration.scaled_by(static_cast(numerator), static_cast(denominator)); +} + +struct ByteTimePoint { + size_t byte; + AK::Duration time; +}; + +struct EnclosingCachedRanges { + MP3Navigator::CachedRange before; + Optional after; +}; + +struct FrameScanResult { + ByteTimePoint scanned_end; + ByteTimePoint last_parsed_frame; + Optional frame_spanning_target; +}; + +static int compare_byte_range_end_to_position(MediaStream::ByteRange const& range, size_t position) +{ + return range.end <= position ? -1 : 1; +} + +static AK::Duration interpolate_timestamp_at_byte(size_t byte, ByteTimePoint const& left, ByteTimePoint const& right) +{ + if (byte <= left.byte) + return left.time; + if (byte >= right.byte) + return right.time; + if (right.time <= left.time) + return left.time; + + return left.time + scale_duration_by_byte_ratio(right.time - left.time, byte - left.byte, right.byte - left.byte); +} + +static size_t estimate_byte_for_timestamp(AK::Duration timestamp, ByteTimePoint const& left, ByteTimePoint const& right) +{ + auto estimated_byte = left.byte; + if (right.time > left.time && right.byte > left.byte) { + auto numerator = (timestamp - left.time).to_time_units(1, EXACT_MPEG_AUDIO_DURATION_TIMEBASE); + auto denominator = (right.time - left.time).to_time_units(1, EXACT_MPEG_AUDIO_DURATION_TIMEBASE); + if (denominator > 0) { + auto byte_span = static_cast(right.byte - left.byte); + if (Checked::multiplication_would_overflow(numerator, byte_span)) { + estimated_byte = right.byte; + } else { + estimated_byte = left.byte + static_cast(numerator * byte_span / denominator); + } + } + } + return estimated_byte; +} + +static constexpr int FRAME_VALIDATION_COUNT = 3; + +static bool has_valid_frame_sequence_at(MediaStreamCursor& cursor, size_t candidate, size_t upper_bound) +{ + auto position = candidate; + for (int i = 0; i < FRAME_VALIDATION_COUNT; i++) { + if (position + 4 > upper_bound) + return false; + if (!seek(cursor, position)) + return false; + FrameInfo frame_info; + if (!parse_frame_header(cursor, frame_info)) + return false; + position += frame_info.frame_byte_size; + } + return true; +} + +static constexpr size_t RESYNC_CHUNK_SIZE = 4096; + +static Optional find_frame_boundary_at_or_before(MediaStreamCursor& cursor, size_t target_byte, size_t lower_bound, size_t upper_bound) +{ + VERIFY(target_byte > lower_bound); + VERIFY(target_byte <= upper_bound); + + auto chunk_end = min(target_byte + 2, upper_bound); + auto chunk_start = chunk_end > RESYNC_CHUNK_SIZE ? chunk_end - RESYNC_CHUNK_SIZE : 0; + chunk_start = max(chunk_start, lower_bound); + auto chunk_size = chunk_end - chunk_start; + if (chunk_size < 2) + return {}; + + Array buffer; + if (!seek(cursor, chunk_start)) + return {}; + if (!read_exact(cursor, buffer.span().trim(chunk_size))) + return {}; + + auto bytes = buffer.span().trim(chunk_size); + for (size_t i = bytes.size(); i-- > 0;) { + if (!has_sync_code(bytes, i)) + continue; + auto candidate = chunk_start + i; + if (candidate > target_byte) + continue; + if (has_valid_frame_sequence_at(cursor, candidate, upper_bound)) + return candidate; + } + return {}; +} + +static Optional find_frame_boundary_at_or_after(MediaStreamCursor& cursor, size_t start_byte, size_t upper_bound) +{ + VERIFY(start_byte < upper_bound); + auto chunk_start = start_byte; + auto chunk_size = min(RESYNC_CHUNK_SIZE, upper_bound - chunk_start); + if (chunk_size < 2) + return {}; + + Array buffer; + if (!seek(cursor, chunk_start)) + return {}; + if (!read_exact(cursor, buffer.span().trim(chunk_size))) + return {}; + + auto bytes = buffer.span().trim(chunk_size); + for (size_t i = 0; i < bytes.size(); i++) { + if (!has_sync_code(bytes, i)) + continue; + auto candidate = chunk_start + i; + if (has_valid_frame_sequence_at(cursor, candidate, upper_bound)) + return candidate; + } + return {}; +} + +static Optional find_containing_byte_range(Vector const& byte_ranges, size_t byte_position) +{ + auto index = lower_bound_index(byte_ranges, byte_position, compare_byte_range_end_to_position); + if (index >= byte_ranges.size() || byte_position < byte_ranges[index].start) + return {}; + return byte_ranges[index]; +} + +static void scan_available_frames_for_cached_range(MediaStreamCursor& cursor, MP3Navigator::CachedRange& range, MediaStream::ByteRange const& byte_range) +{ + auto position = max(range.last_scanned_byte, range.byte_start); + auto upper_bound = byte_range.end; + + while (position + 4 <= upper_bound) { + if (!seek(cursor, position)) + return; + FrameInfo frame_info; + if (!parse_frame_header(cursor, frame_info)) + return; + if (position + frame_info.frame_byte_size > upper_bound) + return; + position += frame_info.frame_byte_size; + range.duration_in_ticks += frame_info.duration_in_ticks; + range.last_scanned_byte = position; + } +} + +static ByteTimePoint scanned_endpoint(MP3Navigator::CachedRange const& range) +{ + return { + max(range.last_scanned_byte, range.byte_start), + range.time_start + duration_from_ticks(range.duration_in_ticks), + }; +} + +static Optional file_endpoint_after(ByteTimePoint const& left, Optional file_size, AK::Duration total_duration) +{ + if (!file_size.has_value()) + return {}; + if (*file_size <= left.byte) + return {}; + if (total_duration <= left.time) + return {}; + return ByteTimePoint { static_cast(*file_size), total_duration }; +} + +static EnclosingCachedRanges find_enclosing_cached_ranges_for_timestamp(Vector const& cached_ranges, size_t first_frame_position, AK::Duration timestamp) +{ + EnclosingCachedRanges ranges { + MP3Navigator::CachedRange { + .byte_start = first_frame_position, + .last_scanned_byte = first_frame_position, + }, + {}, + }; + for (auto const& range : cached_ranges) { + if (range.time_start <= timestamp) { + ranges.before = range; + continue; + } + ranges.after = range; + break; + } + return ranges; +} + +static AK::Duration estimate_time_for_cached_range_start(Vector const& cached_ranges, size_t byte_start, size_t first_frame_position, Optional file_size, AK::Duration total_duration) +{ + ByteTimePoint left { first_frame_position, AK::Duration::zero() }; + for (auto const& range : cached_ranges) { + if (range.byte_start >= byte_start) + break; + left = scanned_endpoint(range); + } + + auto right = file_endpoint_after(left, file_size, total_duration); + if (!right.has_value()) + return left.time; + return interpolate_timestamp_at_byte(byte_start, left, *right); +} + +static bool insert_cached_range_sorted(Vector& cached_ranges, MP3Navigator::CachedRange new_range) +{ + size_t insert_index = cached_ranges.size(); + for (size_t i = 0; i < cached_ranges.size(); i++) { + if (cached_ranges[i].byte_start == new_range.byte_start) + return false; + if (cached_ranges[i].byte_start > new_range.byte_start) { + insert_index = i; + break; + } + } + + VERIFY(insert_index == 0 || cached_ranges[insert_index - 1].byte_start < new_range.byte_start); + VERIFY(insert_index == cached_ranges.size() || new_range.byte_start < cached_ranges[insert_index].byte_start); + cached_ranges.insert(insert_index, new_range); + return true; +} + +static FrameScanResult scan_available_seek_frames(MediaStreamCursor& cursor, MP3Navigator::CachedRange const& range, Optional const& next_range, AK::Duration target) +{ + VERIFY(range.time_start <= target); + + auto target_duration_in_ticks = (target - range.time_start).to_time_units(1, EXACT_MPEG_AUDIO_DURATION_TIMEBASE); + auto scan_limit_byte = NumericLimits::max(); + if (next_range.has_value()) { + VERIFY(range.byte_start < next_range->byte_start); + VERIFY(range.time_start <= next_range->time_start); + + scan_limit_byte = next_range->byte_start; + } + + auto position = range.byte_start; + auto last_frame_position = position; + auto last_frame_duration_in_ticks = 0ull; + u64 duration_in_ticks = 0; + while (position + 4 <= scan_limit_byte) { + if (!seek(cursor, position)) + break; + FrameInfo frame_info; + if (!parse_frame_header(cursor, frame_info)) + break; + if (static_cast(duration_in_ticks + frame_info.duration_in_ticks) > target_duration_in_ticks) { + auto frame_time = range.time_start + duration_from_ticks(duration_in_ticks); + return { + { position, frame_time }, + { last_frame_position, range.time_start + duration_from_ticks(last_frame_duration_in_ticks) }, + SeekedPosition { static_cast(position), frame_time }, + }; + } + last_frame_position = position; + last_frame_duration_in_ticks = duration_in_ticks; + position += frame_info.frame_byte_size; + duration_in_ticks += frame_info.duration_in_ticks; + } + + return { + { position, range.time_start + duration_from_ticks(duration_in_ticks) }, + { last_frame_position, range.time_start + duration_from_ticks(last_frame_duration_in_ticks) }, + {}, + }; +} + +static void scan_cached_range(MediaStreamCursor& cursor, MP3Navigator::CachedRange& range, MediaStream::ByteRange const& byte_range) +{ + if (range.last_scanned_byte > byte_range.end) { + range.last_scanned_byte = range.byte_start; + range.duration_in_ticks = 0; + } else if (range.last_scanned_byte == byte_range.end) { + return; + } + scan_available_frames_for_cached_range(cursor, range, byte_range); +} + +static bool insert_cached_range_for_byte_range(MediaStreamCursor& cursor, Vector& ranges, MediaStream::ByteRange const& byte_range, size_t first_frame_position, Optional file_size, AK::Duration total_duration) +{ + if (byte_range.end <= first_frame_position) + return false; + + auto search_start = max(byte_range.start, first_frame_position); + if (search_start >= byte_range.end) + return false; + + auto frame_boundary = find_frame_boundary_at_or_after(cursor, search_start, byte_range.end); + if (!frame_boundary.has_value()) + return false; + + MP3Navigator::CachedRange range { + .byte_start = *frame_boundary, + .time_start = estimate_time_for_cached_range_start(ranges, *frame_boundary, first_frame_position, file_size, total_duration), + .last_scanned_byte = *frame_boundary, + .duration_in_ticks = 0, + }; + scan_available_frames_for_cached_range(cursor, range, byte_range); + + return insert_cached_range_sorted(ranges, range); +} + +MP3Navigator::MP3Navigator(NonnullRefPtr stream, size_t first_frame_position, AK::Duration total_duration) + : m_stream(move(stream)) + , m_first_frame_position(first_frame_position) + , m_total_duration(total_duration) + , m_buffered_range_scanning_cursor(m_stream->create_cursor()) + , m_seek_range_scanning_cursor(m_stream->create_cursor()) + , m_seek_cursor(m_stream->create_cursor()) +{ + m_buffered_range_scanning_cursor->set_is_blocking(false); + m_seek_range_scanning_cursor->set_is_blocking(false); + m_seek_cursor->set_is_blocking(true); + + m_cached_ranges.append({ + .byte_start = m_first_frame_position, + .time_start = AK::Duration::zero(), + .last_scanned_byte = m_first_frame_position, + .duration_in_ticks = 0, + }); +} + +TimeRanges MP3Navigator::buffered_time_ranges(Vector const& byte_ranges) const +{ + Sync::MutexLocker locker { m_mutex }; + update_cached_ranges(byte_ranges, *m_buffered_range_scanning_cursor); + + TimeRanges result; + auto file_size = m_stream->expected_size(); + for (auto const& range : m_cached_ranges) { + auto byte_range = find_containing_byte_range(byte_ranges, range.byte_start); + if (!byte_range.has_value()) + continue; + auto time_start = max(range.time_start, AK::Duration::zero()); + auto time_end = range.time_start + duration_from_ticks(range.duration_in_ticks); + if (file_size.has_value() && byte_range->end == *file_size) + time_end = AK::Duration::max(); + result.add_range(time_start, time_end); + } + return result; +} + +DecoderErrorOr MP3Navigator::seek_to_timestamp(AK::Duration timestamp) const +{ + if (m_total_duration <= AK::Duration::zero()) + return SeekSkipped {}; + + timestamp = max(timestamp, AK::Duration::zero()); + + EnclosingCachedRanges enclosing_cached_ranges; + FrameScanResult scan_result; + { + Sync::MutexLocker locker { m_mutex }; + update_cached_ranges(m_stream->available_byte_ranges(), *m_seek_range_scanning_cursor); + enclosing_cached_ranges = find_enclosing_cached_ranges_for_timestamp(m_cached_ranges, m_first_frame_position, timestamp); + scan_result = scan_available_seek_frames(*m_seek_range_scanning_cursor, enclosing_cached_ranges.before, enclosing_cached_ranges.after, timestamp); + if (scan_result.frame_spanning_target.has_value()) + return *scan_result.frame_spanning_target; + } + + auto file_size = m_stream->expected_size(); + Optional boundary_after_target; + if (enclosing_cached_ranges.after.has_value()) + boundary_after_target = ByteTimePoint { enclosing_cached_ranges.after->byte_start, enclosing_cached_ranges.after->time_start }; + else + boundary_after_target = file_endpoint_after(scan_result.scanned_end, file_size, m_total_duration); + if (!boundary_after_target.has_value()) + return SeekedPosition { static_cast(scan_result.last_parsed_frame.byte), scan_result.last_parsed_frame.time }; + + auto estimated_byte = estimate_byte_for_timestamp(timestamp, scan_result.scanned_end, *boundary_after_target); + + auto resynced_frame_byte = find_frame_boundary_at_or_before(*m_seek_cursor, estimated_byte, m_first_frame_position, boundary_after_target->byte); + if (!resynced_frame_byte.has_value() && estimated_byte < boundary_after_target->byte) + resynced_frame_byte = find_frame_boundary_at_or_after(*m_seek_cursor, estimated_byte, boundary_after_target->byte); + if (!resynced_frame_byte.has_value()) + return DecoderError::format(DecoderErrorCategory::Corrupted, "MP3 resync failed"); + + if (*resynced_frame_byte <= scan_result.scanned_end.byte) + return SeekedPosition { static_cast(scan_result.last_parsed_frame.byte), scan_result.last_parsed_frame.time }; + + auto resolved_time = interpolate_timestamp_at_byte(*resynced_frame_byte, scan_result.scanned_end, *boundary_after_target); + return SeekedPosition { static_cast(*resynced_frame_byte), resolved_time }; +} + +void MP3Navigator::update_cached_ranges(Vector const& byte_ranges, MediaStreamCursor& cursor) const +{ + if (byte_ranges.is_empty()) { + m_cached_ranges.clear(); + m_cached_ranges.append({ + .byte_start = m_first_frame_position, + .time_start = AK::Duration::zero(), + .last_scanned_byte = m_first_frame_position, + .duration_in_ticks = 0, + }); + return; + } + + auto file_size = m_stream->expected_size(); + size_t cached_index = 0; + size_t byte_index = 0; + + while (byte_index < byte_ranges.size()) { + auto const& byte_range = byte_ranges[byte_index]; + VERIFY(byte_range.start < byte_range.end); + + if (cached_index < m_cached_ranges.size() && m_cached_ranges[cached_index].byte_start < byte_range.start) { + m_cached_ranges.remove(cached_index); + continue; + } + + if (cached_index >= m_cached_ranges.size() || m_cached_ranges[cached_index].byte_start >= byte_range.end) { + if (insert_cached_range_for_byte_range(cursor, m_cached_ranges, byte_range, m_first_frame_position, file_size, m_total_duration)) + cached_index++; + byte_index++; + continue; + } + + auto& cached_range = m_cached_ranges[cached_index]; + VERIFY(byte_range.start <= cached_range.byte_start); + VERIFY(cached_range.byte_start < byte_range.end); + + scan_cached_range(cursor, cached_range, byte_range); + + while (cached_index + 1 < m_cached_ranges.size() && m_cached_ranges[cached_index + 1].byte_start < byte_range.end) + m_cached_ranges.remove(cached_index + 1); + + cached_index++; + byte_index++; + } + + if (cached_index < m_cached_ranges.size()) + m_cached_ranges.remove(cached_index, m_cached_ranges.size() - cached_index); + + reproject_cached_range_times(); +} + +void MP3Navigator::reproject_cached_range_times() const +{ + if (m_cached_ranges.size() < 2) + return; + + auto file_size = m_stream->expected_size(); + for (size_t i = 1; i < m_cached_ranges.size(); i++) { + auto left = scanned_endpoint(m_cached_ranges[i - 1]); + auto right = file_endpoint_after(left, file_size, m_total_duration); + if (!right.has_value()) { + m_cached_ranges[i].time_start = left.time; + continue; + } + + auto projected_time = interpolate_timestamp_at_byte(m_cached_ranges[i].byte_start, left, *right); + m_cached_ranges[i].time_start = projected_time; + } +} + +} diff --git a/Libraries/LibMedia/Containers/MP3Navigator.h b/Libraries/LibMedia/Containers/MP3Navigator.h new file mode 100644 index 0000000000000..5ab048500343d --- /dev/null +++ b/Libraries/LibMedia/Containers/MP3Navigator.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace Media { + +class MP3Navigator final : public ContainerNavigator { +public: + MP3Navigator(NonnullRefPtr stream, size_t first_frame_position, AK::Duration total_duration); + + struct CachedRange { + size_t byte_start { 0 }; + AK::Duration time_start { AK::Duration::zero() }; + size_t last_scanned_byte { 0 }; + u64 duration_in_ticks { 0 }; + }; + + TimeRanges buffered_time_ranges(Vector const& byte_ranges) const override; + DecoderErrorOr seek_to_timestamp(AK::Duration timestamp) const override; + +private: + void update_cached_ranges(Vector const& byte_ranges, MediaStreamCursor&) const; + void reproject_cached_range_times() const; + + NonnullRefPtr m_stream; + size_t m_first_frame_position; + AK::Duration m_total_duration; + + NonnullRefPtr m_buffered_range_scanning_cursor; + NonnullRefPtr m_seek_range_scanning_cursor; + NonnullRefPtr m_seek_cursor; + + mutable Sync::Mutex m_mutex; + mutable Vector m_cached_ranges; +}; + +} diff --git a/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.cpp b/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.cpp index deaf26fec0926..d0f4a1c39bb75 100644 --- a/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.cpp +++ b/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.cpp @@ -25,8 +25,10 @@ DecoderErrorOr> MatroskaDemuxer::from_stream(Nonn MatroskaDemuxer::MatroskaDemuxer(NonnullRefPtr const& stream, Reader&& reader) : m_stream(stream) + , m_buffered_scan_cursor(stream->create_cursor()) , m_reader(move(reader)) { + m_buffered_scan_cursor->set_is_blocking(false); } MatroskaDemuxer::~MatroskaDemuxer() = default; @@ -177,12 +179,10 @@ DecoderErrorOr MatroskaDemuxer::total_duration() TimeRanges MatroskaDemuxer::buffered_time_ranges() const { - // FIXME: Scan the stream for buffered ranges. - TimeRanges ranges; - auto duration = m_reader.duration(); - if (duration.has_value()) - ranges.add_range(AK::Duration::zero(), duration.value()); - return ranges; + auto byte_ranges = m_stream->available_byte_ranges(); + if (byte_ranges.is_empty()) + return {}; + return m_reader.buffered_time_ranges(m_buffered_scan_cursor, byte_ranges); } DecoderErrorOr MatroskaDemuxer::duration_of_track(Track const&) diff --git a/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.h b/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.h index 3c57a3c8df558..df169bdafd60d 100644 --- a/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.h +++ b/Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.h @@ -63,6 +63,7 @@ class MEDIA_API MatroskaDemuxer final : public Demuxer { TrackStatus& get_track_status(Track const&); NonnullRefPtr m_stream; + NonnullRefPtr m_buffered_scan_cursor; Reader m_reader; mutable Sync::Mutex m_track_statuses_mutex; diff --git a/Libraries/LibMedia/Containers/Matroska/Reader.cpp b/Libraries/LibMedia/Containers/Matroska/Reader.cpp index e2918ca202e60..32e98f7b26aa8 100644 --- a/Libraries/LibMedia/Containers/Matroska/Reader.cpp +++ b/Libraries/LibMedia/Containers/Matroska/Reader.cpp @@ -1,6 +1,6 @@ /* * Copyright (c) 2021, Hunter Salyer - * Copyright (c) 2022-2025, Gregory Bertilson + * Copyright (c) 2022-2026, Gregory Bertilson * * SPDX-License-Identifier: BSD-2-Clause */ @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -644,7 +645,7 @@ DecoderErrorOr Reader::for_each_track_of_type(TrackEntry::TrackType type, }); } -DecoderErrorOr> Reader::track_for_track_number(u64 track_number) +DecoderErrorOr> Reader::track_for_track_number(u64 track_number) const { auto optional_track_entry = m_tracks.get(track_number); if (!optional_track_entry.has_value()) @@ -652,7 +653,7 @@ DecoderErrorOr> Reader::track_for_track_number(u64 tra return *optional_track_entry.release_value(); } -DecoderErrorOr Reader::track_count() +DecoderErrorOr Reader::track_count() const { return m_tracks.size(); } @@ -707,81 +708,6 @@ static AK::Duration block_timestamp_to_duration(AK::Duration cluster_timestamp, return cluster_timestamp + AK::Duration::from_nanoseconds(timestamp_offset_in_cluster_offset); } -DecoderErrorOr> SampleIterator::get_frames(Block block) -{ - Streamer streamer { m_stream_cursor }; - TRY(streamer.seek_to_position(block.data_position())); - Vector frames; - - if (block.lacing() == Block::Lacing::EBML) { - auto frames_start_position = streamer.position(); - auto frame_count = TRY(streamer.read_octet()) + 1; - Vector frame_sizes; - frame_sizes.ensure_capacity(frame_count); - - u64 frame_size_sum = 0; - u64 previous_frame_size; - auto first_frame_size = TRY(streamer.read_variable_size_integer()); - frame_sizes.append(first_frame_size); - frame_size_sum += first_frame_size; - previous_frame_size = first_frame_size; - - for (int i = 0; i < frame_count - 2; i++) { - auto frame_size_difference = TRY(streamer.read_variable_size_signed_integer()); - u64 frame_size; - // FIXME: x - (-y) == x + y? - if (frame_size_difference < 0) - frame_size = previous_frame_size - (-frame_size_difference); - else - frame_size = previous_frame_size + frame_size_difference; - frame_sizes.append(frame_size); - frame_size_sum += frame_size; - previous_frame_size = frame_size; - } - frame_sizes.append(block.data_size() - frame_size_sum - (streamer.position() - frames_start_position)); - - for (int i = 0; i < frame_count; i++) { - // FIXME: ReadonlyBytes instead of copying the frame data? - auto current_frame_size = frame_sizes.at(i); - frames.append(TRY(streamer.read_raw_octets(current_frame_size))); - } - } else if (block.lacing() == Block::Lacing::FixedSize) { - auto frame_count = TRY(streamer.read_octet()) + 1; - auto frames_data_size = block.data_size() - 1; - if ((frames_data_size % frame_count) != 0) - return DecoderError::corrupted("Block with fixed-size frames has non-divisible size"sv); - auto individual_frame_size = frames_data_size / frame_count; - for (int i = 0; i < frame_count; i++) - frames.append(TRY(streamer.read_raw_octets(individual_frame_size))); - } else if (block.lacing() == Block::Lacing::XIPH) { - auto frames_start_position = streamer.position(); - - auto frame_count_minus_one = TRY(streamer.read_octet()); - frames.ensure_capacity(frame_count_minus_one + 1); - - auto frame_sizes = Vector(); - frame_sizes.ensure_capacity(frame_count_minus_one); - for (auto i = 0; i < frame_count_minus_one; i++) { - auto frame_size = 0; - while (true) { - auto octet = TRY(streamer.read_octet()); - frame_size += octet; - if (octet < 255) - break; - } - frame_sizes.append(frame_size); - } - - for (auto i = 0; i < frame_count_minus_one; i++) - frames.append(TRY(streamer.read_raw_octets(frame_sizes[i]))); - frames.append(TRY(streamer.read_raw_octets(block.data_size() - (streamer.position() - frames_start_position)))); - } else { - frames.append(TRY(streamer.read_raw_octets(block.data_size()))); - } - - return frames; -} - static void set_block_duration_to_default(Block& block, TrackBlockContext const& context) { if (context.default_duration != 0) @@ -794,47 +720,8 @@ static DecoderErrorOr maybe_parse_opus_frame_duration(Streamer& streamer, return {}; if (codec_id_from_matroska_id_string(context.codec_id) != CodecID::Opus) return {}; - if (block.data_size() == 0) - return DecoderError::corrupted("Opus frame is too small"sv); - - // https://datatracker.ietf.org/doc/html/rfc6716#section-3.1 - auto toc_byte = TRY(streamer.read_octet()); - auto configuration_number = (toc_byte >> 3) & 0b1'1111; - auto packet_code = toc_byte & 0b11; - // clang-format off - constexpr Array frame_durations = { - 10000, 20000, 40000, 60000, // SILK-only NB - 10000, 20000, 40000, 60000, // SILK-only MB - 10000, 20000, 40000, 60000, // SILK-only WB - 10000, 20000, // Hybrid SWB - 10000, 20000, // Hybrid FB - 2500, 5000, 10000, 20000, // CELT-only NB - 2500, 5000, 10000, 20000, // CELT-only WB - 2500, 5000, 10000, 20000, // CELT-only SWB - 2500, 5000, 10000, 20000, // CELT-only FB - }; - // clang-format on - - auto frame_duration = frame_durations[configuration_number]; - auto block_duration = TRY([&] -> DecoderErrorOr { - switch (packet_code) { - case 0: - return frame_duration; - case 1: - case 2: - return frame_duration * 2; - case 3: { - if (block.data_size() == 1) - return DecoderError::corrupted("Opus frame is too small"sv); - auto frame_count_byte = TRY(streamer.read_octet()); - auto frame_count = frame_count_byte & 0b11'1111; - return frame_duration * frame_count; - } - default: - VERIFY_NOT_REACHED(); - } - }()); - block.set_duration(AK::Duration::from_microseconds(block_duration)); + + block.set_duration(TRY(Codecs::Opus::parse_frame_duration(streamer.cursor(), block.data_size()))); return {}; } @@ -939,13 +826,44 @@ DecoderErrorOr Reader::parse_block_group(Streamer& streamer, AK::Duration return block; } -DecoderErrorOr Reader::create_sample_iterator(NonnullRefPtr const& stream_consumer, u64 track_number) +DecoderErrorOr Reader::create_sample_iterator(NonnullRefPtr const& cursor, Optional track_number) const +{ + return create_sample_iterator_at_byte_position(cursor, 0, track_number); +} + +DecoderErrorOr Reader::create_sample_iterator_at_byte_position(NonnullRefPtr const& cursor, size_t position, Optional track_number) const { - dbgln_if(MATROSKA_DEBUG, "Creating sample iterator starting at {} relative to segment at {}", m_first_cluster_position, m_segment_contents_position); - auto track_entry = TRY(track_for_track_number(track_number)); + Optional cluster_position; + + if (m_first_cluster_position >= position) { + cluster_position = m_first_cluster_position; + } else { + for (auto const& [track_number_with_cues, cue_points] : m_cues) { + if (track_number.has_value() && track_number != track_number_with_cues) + continue; + for (auto const& cue_point : cue_points) { + auto cue_cluster_position = m_segment_contents_position + cue_point.position.cluster_position(); + if (cue_cluster_position < position) + continue; + if (!cluster_position.has_value() || cue_cluster_position < cluster_position.value()) + cluster_position = cue_cluster_position; + } + } + } + + if (!cluster_position.has_value()) + return DecoderError::format(DecoderErrorCategory::EndOfStream, "Could not find a Cluster element after {}", position); + + dbgln_if(MATROSKA_DEBUG, "Creating sample iterator starting at {} relative to segment at {}", cluster_position, m_segment_contents_position); TrackBlockContexts track_contexts; - track_contexts.set(track_number, TrackBlockContext::from_track_entry(track_entry)); - return SampleIterator(stream_consumer, track_number, move(track_contexts), m_segment_information.timestamp_scale(), m_segment_contents_position, m_first_cluster_position); + if (track_number.has_value()) { + auto track = TRY(track_for_track_number(track_number.value())); + track_contexts.set(track_number.value(), TrackBlockContext::from_track_entry(*track)); + } else { + for (auto const& [number, track_entry] : m_tracks) + track_contexts.set(number, TrackBlockContext::from_track_entry(*track_entry)); + } + return SampleIterator(cursor, track_number, move(track_contexts), m_segment_information.timestamp_scale(), m_segment_contents_position, cluster_position.value()); } static DecoderErrorOr parse_cue_track_position(Streamer& streamer) @@ -1104,7 +1022,7 @@ size_t Reader::find_cue_point_index_at_or_before(Vector const& cu return index; } -DecoderErrorOr Reader::seek_to_cue_for_timestamp(SampleIterator& iterator, AK::Duration const& timestamp, Vector const& cue_points, CuePointTarget target) +DecoderErrorOr Reader::seek_to_cue_for_timestamp(SampleIterator& iterator, AK::Duration const& timestamp, Vector const& cue_points, CuePointTarget target) const { auto index = find_cue_point_index_at_or_before(cue_points, m_segment_information.duration(), timestamp); TRY(iterator.seek_to_cue_point(cue_points[index], target)); @@ -1151,17 +1069,14 @@ static DecoderErrorOr search_clusters_for_keyframe_before_timestamp(Sample return {}; } -bool Reader::has_cues_for_track(u64 track_number) +DecoderErrorOr Reader::seek_to_random_access_point(SampleIterator iterator, AK::Duration timestamp) const { - return m_cues.contains(track_number); -} - -DecoderErrorOr Reader::seek_to_random_access_point(SampleIterator iterator, AK::Duration timestamp) -{ - auto seek_pre_roll = iterator.m_track_block_contexts.get(iterator.m_track_number)->seek_pre_roll; + VERIFY(iterator.m_track_number.has_value()); + auto track_number = iterator.m_track_number.value(); + auto seek_pre_roll = iterator.m_track_block_contexts.get(track_number)->seek_pre_roll; timestamp -= AK::Duration::from_nanoseconds(AK::clamp_to(seek_pre_roll)); - auto cue_points = cue_points_for_track(iterator.m_track_number); + auto cue_points = cue_points_for_track(track_number); auto seek_target = CuePointTarget::Block; // If no cues are present for the track, use the first track's cues. @@ -1189,276 +1104,136 @@ DecoderErrorOr Reader::seek_to_random_access_point(SampleIterato return iterator; } -Optional const&> Reader::cue_points_for_track(u64 track_number) +Optional const&> Reader::cue_points_for_track(u64 track_number) const { return m_cues.get(track_number); } -DecoderErrorOr SampleIterator::next_block() +TimeRanges Reader::buffered_time_ranges(NonnullRefPtr const& cursor, Vector const& byte_ranges) const { - Streamer streamer { m_stream_cursor }; - TRY(streamer.seek_to_position(m_position)); - - Optional block; - - while (true) { -#if MATROSKA_TRACE_DEBUG - auto element_position = streamer.position(); -#endif - auto element_id = TRY(streamer.read_element_id()); -#if MATROSKA_TRACE_DEBUG - dbgln("Iterator found element with ID {:#010x} at offset {} within the segment.", element_id, element_position); -#endif - - if (element_id == CLUSTER_ELEMENT_ID) { - dbgln_if(MATROSKA_DEBUG, " Iterator is parsing new cluster."); - m_current_cluster = TRY(Reader::parse_cluster_element(streamer, m_segment_timestamp_scale)); - } else if (element_id == SIMPLE_BLOCK_ID) { - dbgln_if(MATROSKA_TRACE_DEBUG, " Iterator is parsing a new simple block."); - auto candidate_block = TRY(Reader::parse_simple_block(streamer, m_current_cluster->timestamp(), m_segment_timestamp_scale, m_track_block_contexts)); - if (candidate_block.track_number() == m_track_number) - block = move(candidate_block); - } else if (element_id == BLOCK_GROUP_ID) { - dbgln_if(MATROSKA_TRACE_DEBUG, " Iterator is parsing a new block group."); - auto candidate_block = TRY(Reader::parse_block_group(streamer, m_current_cluster->timestamp(), m_segment_timestamp_scale, m_track_block_contexts)); - if (candidate_block.track_number() == m_track_number) - block = move(candidate_block); - } else if (element_id == SEGMENT_ELEMENT_ID) { - dbgln("Malformed file, found a segment element within the root segment element. Jumping into it."); - [[maybe_unused]] auto segment_size = TRY(streamer.read_variable_size_integer()); - } else { - dbgln_if(MATROSKA_TRACE_DEBUG, " Iterator is skipping unknown element with ID {:#010x}.", element_id); - TRY(streamer.read_unknown_element()); - } + auto create_iterator = [&](size_t position) -> Optional { + auto iterator = create_sample_iterator_at_byte_position(cursor, position); + if (iterator.is_error()) + return {}; + return iterator.release_value(); + }; - m_position = streamer.position(); - if (block.has_value()) { - m_last_timestamp = block->timestamp(); - return block.release_value(); + size_t cached_range_index = 0; + size_t byte_range_index = 0; + while (byte_range_index < byte_ranges.size()) { + auto cached_range = m_buffered_ranges.get(cached_range_index); + auto const& byte_range = byte_ranges[byte_range_index]; + VERIFY(byte_range.start < byte_range.end); + + auto previous_byte_range = byte_ranges.get(byte_range_index - 1); + if (previous_byte_range.has_value()) + VERIFY(previous_byte_range->end < byte_range.start); + + // If the current byte range precedes the current cached range, insert a new one for that byte range. + // Restart the loop with the same cached range. + if (!cached_range.has_value() || byte_range.start < cached_range->start) { + auto new_cached_range = BufferedRange { + .start = byte_range.start, + .end = byte_range.end, + .iterator = create_iterator(byte_range.start), + }; + m_buffered_ranges.insert(cached_range_index, move(new_cached_range)); + cached_range_index++; + byte_range_index++; + continue; } - } - - VERIFY_NOT_REACHED(); -} - -SampleIterator::SampleIterator(NonnullRefPtr const& stream_cursor, u64 track_number, TrackBlockContexts&& track_contexts, u64 timestamp_scale, size_t segment_contents_position, size_t position) - : m_stream_cursor(stream_cursor) - , m_track_number(track_number) - , m_track_block_contexts(move(track_contexts)) - , m_segment_timestamp_scale(timestamp_scale) - , m_segment_contents_position(segment_contents_position) - , m_position(position) -{ -} - -SampleIterator::~SampleIterator() = default; - -DecoderErrorOr SampleIterator::seek_to_cue_point(TrackCuePoint const& cue_point, CuePointTarget target) -{ - // This is a private function. The position getter can return optional, but the caller should already know that this track has a position. - auto const& cue_position = cue_point.position; - Streamer streamer { m_stream_cursor }; - TRY(streamer.seek_to_position(m_segment_contents_position + cue_position.cluster_position())); - - auto element_id = TRY(streamer.read_element_id()); - if (element_id != CLUSTER_ELEMENT_ID) - return DecoderError::corrupted("Cue point's cluster position didn't point to a cluster"sv); - - m_current_cluster = TRY(Reader::parse_cluster_element(streamer, m_segment_timestamp_scale)); - dbgln_if(MATROSKA_DEBUG, "SampleIterator set to cue point at timestamp {}ms", m_current_cluster->timestamp().to_milliseconds()); - - if (target == CuePointTarget::Cluster) { - m_position = streamer.position(); - m_last_timestamp = m_current_cluster->timestamp(); - } else { - m_position = streamer.position() + cue_position.block_offset(); - m_last_timestamp = cue_point.timestamp; - } - return {}; -} - -Streamer::Streamer(NonnullRefPtr const& stream_cursor) - : m_stream_cursor(stream_cursor) -{ -} - -Streamer::~Streamer() = default; -DecoderErrorOr Streamer::read_string() -{ - auto string_length = TRY(read_variable_size_integer()); - auto string_data = TRY(read_raw_octets(string_length)); - auto const* string_data_raw = reinterpret_cast(string_data.data()); - auto string_value = String::from_utf8(ReadonlyBytes(string_data.data(), strnlen(string_data_raw, string_length))); - if (string_value.is_error()) - return DecoderError::format(DecoderErrorCategory::Invalid, "String is not valid UTF-8"); - return string_value.release_value(); -} + VERIFY(cached_range.has_value()); -DecoderErrorOr Streamer::read_octet() -{ - u8 result; - Bytes bytes { &result, 1 }; - TRY(m_stream_cursor->read_into(bytes)); - return bytes[0]; -} + // If the current range is an exact match to the byte range, we can just update the end byte and advance. + if (byte_range.start == cached_range->start) { + cached_range->end = byte_range.end; + cached_range_index++; + byte_range_index++; + continue; + } -DecoderErrorOr Streamer::read_i16() -{ - return (TRY(read_octet()) << 8) | TRY(read_octet()); -} + auto previous_cached_range = m_buffered_ranges.get(cached_range_index - 1); + if (previous_cached_range.has_value()) { + VERIFY(previous_cached_range->start < cached_range->start); -DecoderErrorOr Streamer::read_element_id() -{ - auto length_byte = TRY(read_octet()); - auto length = count_leading_zeroes_safe(length_byte) + 1; - if (length > 4) - return DecoderError::corrupted("Element ID must be 4 bytes or less"sv); - u32 result = length_byte; - auto bytes_left = length; - while (--bytes_left > 0) { - result <<= 8; - result |= TRY(read_octet()); - } + // If the current cached range is entirely encompassed by the previous one, then remove that range. We'll + // need to rescan its contents. + if (previous_cached_range->end >= cached_range->end) { + m_buffered_ranges.remove(cached_range_index); + continue; + } + } - return result; -} + // The range has shifted forward. We'll need to re-read from the new start position. + auto new_iterator = create_iterator(byte_range.start); + + // If the cached range's last read is still contained in the new byte range, we can keep using its end time. + // Just grab the first frame at the new byte range's start and update the cached range's start from it. + auto& cached_iterator = cached_range->iterator; + if (cached_iterator.has_value() && new_iterator.has_value()) { + auto last_cached_position = cached_iterator->position(); + if (byte_range.start <= last_cached_position && last_cached_position <= byte_range.end) { + auto first_block = new_iterator->next_block(); + + if (!first_block.is_error() && first_block.value().timestamp().has_value()) { + cached_range->start = byte_range.start; + cached_range->end = byte_range.end; + cached_range->time_start = first_block.value().timestamp().value(); + cached_range_index++; + byte_range_index++; + continue; + } + } + } -DecoderErrorOr> Streamer::read_element_size() -{ - auto length_byte = TRY(read_octet()); - auto length = count_leading_zeroes_safe(length_byte) + 1; - if (length > 8) - return DecoderError::corrupted("Element Data Size 8 bytes or less"sv); - - size_t result = length_byte; - auto bytes_left = length; - while (--bytes_left > 0) { - result <<= 8; - result |= TRY(read_octet()); + // Otherwise, we have to reset everything for this range. + *cached_range = { + .start = byte_range.start, + .end = byte_range.end, + .iterator = move(new_iterator), + }; + cached_range_index++; + byte_range_index++; } - auto mask = 0xFFFFFFFFFFFFFFFF >> (64 - (7 * length)); - result &= mask; + // Remove any leftover ranges. We should be left with only the exact ranges provided to us. + m_buffered_ranges.remove(cached_range_index, m_buffered_ranges.size() - cached_range_index); - if (result == mask) - return OptionalNone(); + // All previously known buffered ranges are now matched up or discarded. Iterate the blocks to update the ranges' + // end times and append the ranges. + VERIFY(m_buffered_ranges.size() == byte_ranges.size()); + TimeRanges result; - return result; -} + for (size_t i = 0; i < byte_ranges.size(); i++) { + auto& cached_range = m_buffered_ranges[i]; + auto const& byte_range = byte_ranges[i]; + VERIFY(cached_range.start == byte_range.start); + VERIFY(cached_range.end == byte_range.end); -DecoderErrorOr Streamer::read_variable_size_integer() -{ - auto length_byte = TRY(read_octet()); - auto length = count_leading_zeroes_safe(length_byte) + 1; - if (length > 8) - return DecoderError::corrupted("VINT is too large"sv); - - auto bytes_left = length; - auto result = length_byte & (0xFF >> length); - while (--bytes_left > 0) { - result <<= 8; - result |= TRY(read_octet()); - } - return result; -} + if (cached_range.iterator.has_value() && cached_range.iterator->position() < cached_range.end) { + auto& iterator = cached_range.iterator; -DecoderErrorOr Streamer::read_variable_size_signed_integer() -{ - auto length_byte = TRY(read_octet()); - auto length = count_leading_zeroes_safe(length_byte) + 1; - if (length > 8) - return DecoderError::corrupted("VINT is too large"sv); - - auto bytes_left = length; - i64 result = length_byte & (0xFF >> length); - while (--bytes_left > 0) { - result <<= 8; - result |= TRY(read_octet()); - } - result -= AK::exp2((length * 7) - 1) - 1; - return result; -} + while (iterator->position() < byte_range.end) { + auto block_or_error = iterator->next_block(); + if (block_or_error.is_error()) + break; + auto block = block_or_error.release_value(); + if (block.timestamp().has_value() && block.duration().has_value()) { + if (!cached_range.time_start.has_value()) + cached_range.time_start = block.timestamp().value(); -DecoderErrorOr Streamer::read_raw_octets(size_t num_octets) -{ - auto result = MUST(ByteBuffer::create_uninitialized(num_octets)); - auto bytes = result.bytes(); - TRY(m_stream_cursor->read_into(bytes)); - return result; -} + auto block_end = block.timestamp().value() + block.duration().value(); + cached_range.time_end = block_end; + } + } + } -DecoderErrorOr Streamer::read_u64() -{ - auto integer_length = TRY(read_variable_size_integer()); - if (integer_length == 0) - return 0; - if (integer_length > 8) - return DecoderError::corrupted("Integer Element is too large"sv); - u64 result = 0; - for (size_t i = 0; i < integer_length; i++) { - result <<= 8; - result |= TRY(read_octet()); + if (cached_range.time_start.has_value()) + result.add_range(max(AK::Duration::zero(), cached_range.time_start.value()), cached_range.time_end); } - return result; -} -DecoderErrorOr Streamer::read_i64() -{ - auto integer_length = TRY(read_variable_size_integer()); - if (integer_length == 0) - return 0; - if (integer_length > 8) - return DecoderError::corrupted("Signed Integer Element is too large"sv); - i64 result = 0; - for (size_t i = 0; i < integer_length; i++) { - result <<= 8; - result |= TRY(read_octet()); - } - auto shift = 64 - (static_cast(integer_length) * 8); - result <<= shift; - result >>= shift; return result; } -DecoderErrorOr Streamer::read_float() -{ - auto length = TRY(read_variable_size_integer()); - if (length == 0) - return 0; - if (length != 4u && length != 8u) - return DecoderError::format(DecoderErrorCategory::Invalid, "Float size must be 4 or 8 bytes"); - - union { - u64 value; - float float_value; - double double_value; - } read_data; - read_data.value = 0; - for (size_t i = 0; i < length; i++) { - read_data.value = (read_data.value << 8u) + TRY(read_octet()); - } - if (length == 4u) - return read_data.float_value; - return read_data.double_value; -} - -DecoderErrorOr Streamer::read_unknown_element() -{ - auto element_length = TRY(read_variable_size_integer()); - dbgln_if(MATROSKA_TRACE_DEBUG, "Skipping unknown element of size {}.", element_length); - TRY(m_stream_cursor->seek(element_length, AK::SeekMode::FromCurrentPosition)); - return {}; -} - -size_t Streamer::position() const -{ - return m_stream_cursor->position(); -} - -DecoderErrorOr Streamer::seek_to_position(size_t position) -{ - return m_stream_cursor->seek(position, AK::SeekMode::SetPosition); -} - } diff --git a/Libraries/LibMedia/Containers/Matroska/Reader.h b/Libraries/LibMedia/Containers/Matroska/Reader.h index 0265a918db636..5814ae5d362e6 100644 --- a/Libraries/LibMedia/Containers/Matroska/Reader.h +++ b/Libraries/LibMedia/Containers/Matroska/Reader.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2021, Hunter Salyer - * Copyright (c) 2022, Gregory Bertilson + * Copyright (c) 2022-2026, Gregory Bertilson * * SPDX-License-Identifier: BSD-2-Clause */ @@ -13,14 +13,15 @@ #include #include #include +#include +#include #include "Document.h" +#include "SampleIterator.h" +#include "Streamer.h" namespace Media::Matroska { -class SampleIterator; -class Streamer; - enum class ElementIterationDecision : u8 { Continue, BreakHere, @@ -28,16 +29,6 @@ enum class ElementIterationDecision : u8 { EndOfElement, }; -struct TrackCuePoint { - AK::Duration timestamp; - CueTrackPosition position; -}; - -enum class CuePointTarget : u8 { - Cluster, - Block, -}; - class MEDIA_API Reader { public: typedef Function(TrackEntry const&)> TrackEntryCallback; @@ -58,16 +49,19 @@ class MEDIA_API Reader { DecoderErrorOr for_each_track(TrackEntryCallback); DecoderErrorOr for_each_track_of_type(TrackEntry::TrackType, TrackEntryCallback); - DecoderErrorOr> track_for_track_number(u64); - DecoderErrorOr track_count(); + DecoderErrorOr> track_for_track_number(u64) const; + DecoderErrorOr track_count() const; - DecoderErrorOr create_sample_iterator(NonnullRefPtr const& stream_consumer, u64 track_number); - DecoderErrorOr seek_to_random_access_point(SampleIterator, AK::Duration); + DecoderErrorOr create_sample_iterator(NonnullRefPtr const& cursor, Optional track_number = {}) const; + DecoderErrorOr create_sample_iterator_at_byte_position(NonnullRefPtr const& cursor, size_t position, Optional track_number = {}) const; + DecoderErrorOr seek_to_random_access_point(SampleIterator, AK::Duration) const; - Optional const&> cue_points_for_track(u64 track_number); + Optional const&> cue_points_for_track(u64 track_number) const; static size_t find_cue_point_index_at_or_before(Vector const&, Optional total_duration, AK::Duration target); + TimeRanges buffered_time_ranges(NonnullRefPtr const&, Vector const& byte_ranges) const; + private: Reader() = default; @@ -83,8 +77,7 @@ class MEDIA_API Reader { DecoderErrorOr parse_cues(Streamer&); - bool has_cues_for_track(u64 track_number); - DecoderErrorOr seek_to_cue_for_timestamp(SampleIterator&, AK::Duration const&, Vector const&, CuePointTarget); + DecoderErrorOr seek_to_cue_for_timestamp(SampleIterator&, AK::Duration const&, Vector const&, CuePointTarget) const; Optional m_header; @@ -102,72 +95,15 @@ class MEDIA_API Reader { // The vectors must be sorted by timestamp at all times. HashMap> m_cues; -}; - -class MEDIA_API SampleIterator { - AK_MAKE_DEFAULT_MOVABLE(SampleIterator); - AK_MAKE_DEFAULT_COPYABLE(SampleIterator); - -public: - ~SampleIterator(); - - DecoderErrorOr next_block(); - DecoderErrorOr> get_frames(Block); - Cluster const& current_cluster() const { return *m_current_cluster; } - Optional const& last_timestamp() const { return m_last_timestamp; } - MediaStreamCursor& cursor() { return m_stream_cursor; } - -private: - friend class Reader; - - SampleIterator(NonnullRefPtr const& stream_cursor, u64 track_number, TrackBlockContexts&&, u64 timestamp_scale, size_t segment_contents_position, size_t position); - - DecoderErrorOr seek_to_cue_point(TrackCuePoint const& cue_point, CuePointTarget); - - NonnullRefPtr m_stream_cursor; - u64 m_track_number; - TrackBlockContexts m_track_block_contexts; - u64 m_segment_timestamp_scale { 0 }; - size_t m_segment_contents_position { 0 }; - // Must always point to an element ID or the end of the stream. - size_t m_position { 0 }; - - Optional m_last_timestamp; - - Optional m_current_cluster; -}; - -class MEDIA_API Streamer { -public: - Streamer(NonnullRefPtr const& stream_cursor); - ~Streamer(); - - DecoderErrorOr read_octet(); - - DecoderErrorOr read_i16(); - - DecoderErrorOr read_element_id(); - DecoderErrorOr> read_element_size(); - DecoderErrorOr read_variable_size_integer(); - DecoderErrorOr read_variable_size_signed_integer(); - - DecoderErrorOr read_u64(); - DecoderErrorOr read_i64(); - DecoderErrorOr read_float(); - - DecoderErrorOr read_string(); - - DecoderErrorOr read_unknown_element(); - - DecoderErrorOr read_raw_octets(size_t num_octets); - - size_t position() const; - - DecoderErrorOr seek_to_position(size_t position); - -private: - NonnullRefPtr m_stream_cursor; + struct BufferedRange { + size_t start { 0 }; + size_t end { 0 }; + Optional iterator; + Optional time_start { OptionalNone() }; + AK::Duration time_end { AK::Duration::zero() }; + }; + mutable Vector m_buffered_ranges; }; } diff --git a/Libraries/LibMedia/Containers/Matroska/SampleIterator.cpp b/Libraries/LibMedia/Containers/Matroska/SampleIterator.cpp new file mode 100644 index 0000000000000..d8dd7fcd5ee4f --- /dev/null +++ b/Libraries/LibMedia/Containers/Matroska/SampleIterator.cpp @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2021, Hunter Salyer + * Copyright (c) 2022-2026, Gregory Bertilson + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include + +namespace Media::Matroska { + +SampleIterator::SampleIterator(NonnullRefPtr const& stream_cursor, Optional track_number, TrackBlockContexts&& track_contexts, u64 timestamp_scale, size_t segment_contents_position, size_t position) + : m_stream_cursor(stream_cursor) + , m_track_number(track_number) + , m_track_block_contexts(move(track_contexts)) + , m_segment_timestamp_scale(timestamp_scale) + , m_segment_contents_position(segment_contents_position) + , m_position(position) +{ +} + +SampleIterator::~SampleIterator() = default; + +DecoderErrorOr SampleIterator::next_block() +{ + Streamer streamer { m_stream_cursor }; + TRY(streamer.seek_to_position(m_position)); + + Optional block; + + while (true) { +#if MATROSKA_TRACE_DEBUG + auto element_position = streamer.position(); +#endif + auto element_id = TRY(streamer.read_element_id()); +#if MATROSKA_TRACE_DEBUG + dbgln("Iterator found element with ID {:#010x} at offset {} within the segment.", element_id, element_position); +#endif + + auto maybe_set_block = [&](Block&& candidate_block) { + if (m_track_number.has_value() && candidate_block.track_number() != m_track_number) + return; + block = move(candidate_block); + }; + + if (element_id == CLUSTER_ELEMENT_ID) { + dbgln_if(MATROSKA_DEBUG, " Iterator is parsing new cluster."); + m_current_cluster = TRY(Reader::parse_cluster_element(streamer, m_segment_timestamp_scale)); + } else if (element_id == SIMPLE_BLOCK_ID) { + if (!m_current_cluster.has_value()) { + dbgln(" Iterator encountered a simple block before parsing a Cluster."); + TRY(streamer.read_unknown_element()); + } else { + dbgln_if(MATROSKA_TRACE_DEBUG, " Iterator is parsing a new simple block."); + auto candidate_block = TRY(Reader::parse_simple_block(streamer, m_current_cluster->timestamp(), m_segment_timestamp_scale, m_track_block_contexts)); + maybe_set_block(move(candidate_block)); + } + } else if (element_id == BLOCK_GROUP_ID) { + if (!m_current_cluster.has_value()) { + dbgln(" Iterator encountered a BlockGroup before parsing a Cluster."); + TRY(streamer.read_unknown_element()); + } else { + dbgln_if(MATROSKA_TRACE_DEBUG, " Iterator is parsing a new block group."); + auto candidate_block = TRY(Reader::parse_block_group(streamer, m_current_cluster->timestamp(), m_segment_timestamp_scale, m_track_block_contexts)); + maybe_set_block(move(candidate_block)); + } + } else if (element_id == SEGMENT_ELEMENT_ID) { + dbgln("Malformed file, found a segment element within the root segment element. Jumping into it."); + [[maybe_unused]] auto segment_size = TRY(streamer.read_variable_size_integer()); + } else { + dbgln_if(MATROSKA_TRACE_DEBUG, " Iterator is skipping unknown element with ID {:#010x}.", element_id); + TRY(streamer.read_unknown_element()); + } + + m_position = streamer.position(); + if (block.has_value()) { + m_last_timestamp = block->timestamp(); + return block.release_value(); + } + } + + VERIFY_NOT_REACHED(); +} + +DecoderErrorOr> SampleIterator::get_frames(Block block) +{ + Streamer streamer { m_stream_cursor }; + TRY(streamer.seek_to_position(block.data_position())); + Vector frames; + + if (block.lacing() == Block::Lacing::EBML) { + auto frames_start_position = streamer.position(); + auto frame_count = TRY(streamer.read_octet()) + 1; + Vector frame_sizes; + frame_sizes.ensure_capacity(frame_count); + + u64 frame_size_sum = 0; + u64 previous_frame_size; + auto first_frame_size = TRY(streamer.read_variable_size_integer()); + frame_sizes.append(first_frame_size); + frame_size_sum += first_frame_size; + previous_frame_size = first_frame_size; + + for (int i = 0; i < frame_count - 2; i++) { + auto frame_size_difference = TRY(streamer.read_variable_size_signed_integer()); + u64 frame_size; + // FIXME: x - (-y) == x + y? + if (frame_size_difference < 0) + frame_size = previous_frame_size - (-frame_size_difference); + else + frame_size = previous_frame_size + frame_size_difference; + frame_sizes.append(frame_size); + frame_size_sum += frame_size; + previous_frame_size = frame_size; + } + frame_sizes.append(block.data_size() - frame_size_sum - (streamer.position() - frames_start_position)); + + for (int i = 0; i < frame_count; i++) { + // FIXME: ReadonlyBytes instead of copying the frame data? + auto current_frame_size = frame_sizes.at(i); + frames.append(TRY(streamer.read_raw_octets(current_frame_size))); + } + } else if (block.lacing() == Block::Lacing::FixedSize) { + auto frame_count = TRY(streamer.read_octet()) + 1; + auto frames_data_size = block.data_size() - 1; + if ((frames_data_size % frame_count) != 0) + return DecoderError::corrupted("Block with fixed-size frames has non-divisible size"sv); + auto individual_frame_size = frames_data_size / frame_count; + for (int i = 0; i < frame_count; i++) + frames.append(TRY(streamer.read_raw_octets(individual_frame_size))); + } else if (block.lacing() == Block::Lacing::XIPH) { + auto frames_start_position = streamer.position(); + + auto frame_count_minus_one = TRY(streamer.read_octet()); + frames.ensure_capacity(frame_count_minus_one + 1); + + auto frame_sizes = Vector(); + frame_sizes.ensure_capacity(frame_count_minus_one); + for (auto i = 0; i < frame_count_minus_one; i++) { + auto frame_size = 0; + while (true) { + auto octet = TRY(streamer.read_octet()); + frame_size += octet; + if (octet < 255) + break; + } + frame_sizes.append(frame_size); + } + + for (auto i = 0; i < frame_count_minus_one; i++) + frames.append(TRY(streamer.read_raw_octets(frame_sizes[i]))); + frames.append(TRY(streamer.read_raw_octets(block.data_size() - (streamer.position() - frames_start_position)))); + } else { + frames.append(TRY(streamer.read_raw_octets(block.data_size()))); + } + + return frames; +} + +DecoderErrorOr SampleIterator::seek_to_cue_point(TrackCuePoint const& cue_point, CuePointTarget target) +{ + // This is a private function. The position getter can return optional, but the caller should already know that this track has a position. + auto const& cue_position = cue_point.position; + Streamer streamer { m_stream_cursor }; + TRY(streamer.seek_to_position(m_segment_contents_position + cue_position.cluster_position())); + + auto element_id = TRY(streamer.read_element_id()); + if (element_id != CLUSTER_ELEMENT_ID) + return DecoderError::corrupted("Cue point's cluster position didn't point to a cluster"sv); + + m_current_cluster = TRY(Reader::parse_cluster_element(streamer, m_segment_timestamp_scale)); + dbgln_if(MATROSKA_DEBUG, "SampleIterator set to cue point at timestamp {}ms", m_current_cluster->timestamp().to_milliseconds()); + + if (target == CuePointTarget::Cluster) { + m_position = streamer.position(); + m_last_timestamp = m_current_cluster->timestamp(); + } else { + m_position = streamer.position() + cue_position.block_offset(); + m_last_timestamp = cue_point.timestamp; + } + return {}; +} + +} diff --git a/Libraries/LibMedia/Containers/Matroska/SampleIterator.h b/Libraries/LibMedia/Containers/Matroska/SampleIterator.h new file mode 100644 index 0000000000000..0f12e6fcba772 --- /dev/null +++ b/Libraries/LibMedia/Containers/Matroska/SampleIterator.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2021, Hunter Salyer + * Copyright (c) 2022-2026, Gregory Bertilson + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "Document.h" + +namespace Media::Matroska { + +class Reader; + +struct TrackCuePoint { + AK::Duration timestamp; + CueTrackPosition position; +}; + +enum class CuePointTarget : u8 { + Cluster, + Block, +}; + +class MEDIA_API SampleIterator { + AK_MAKE_DEFAULT_MOVABLE(SampleIterator); + AK_MAKE_DEFAULT_COPYABLE(SampleIterator); + +public: + ~SampleIterator(); + + DecoderErrorOr next_block(); + DecoderErrorOr> get_frames(Block); + Cluster const& current_cluster() const { return *m_current_cluster; } + Optional const& last_timestamp() const { return m_last_timestamp; } + MediaStreamCursor& cursor() { return m_stream_cursor; } + size_t position() const { return m_position; } + +private: + friend class Reader; + + SampleIterator(NonnullRefPtr const& stream_cursor, Optional track_number, TrackBlockContexts&&, u64 timestamp_scale, size_t segment_contents_position, size_t position); + + DecoderErrorOr seek_to_cue_point(TrackCuePoint const& cue_point, CuePointTarget); + + NonnullRefPtr m_stream_cursor; + Optional m_track_number; + TrackBlockContexts m_track_block_contexts; + u64 m_segment_timestamp_scale { 0 }; + size_t m_segment_contents_position { 0 }; + + // Must always point to an element ID or the end of the stream. + size_t m_position { 0 }; + + Optional m_last_timestamp; + + Optional m_current_cluster; +}; + +} diff --git a/Libraries/LibMedia/Containers/Matroska/Streamer.cpp b/Libraries/LibMedia/Containers/Matroska/Streamer.cpp new file mode 100644 index 0000000000000..3768957e63c64 --- /dev/null +++ b/Libraries/LibMedia/Containers/Matroska/Streamer.cpp @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2021, Hunter Salyer + * Copyright (c) 2022-2026, Gregory Bertilson + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include + +#include "Streamer.h" + +namespace Media::Matroska { + +Streamer::Streamer(NonnullRefPtr const& stream_cursor) + : m_stream_cursor(stream_cursor) +{ +} + +Streamer::~Streamer() = default; + +DecoderErrorOr Streamer::read_string() +{ + auto string_length = TRY(read_variable_size_integer()); + auto string_data = TRY(read_raw_octets(string_length)); + auto const* string_data_raw = reinterpret_cast(string_data.data()); + auto string_value = String::from_utf8(ReadonlyBytes(string_data.data(), strnlen(string_data_raw, string_length))); + if (string_value.is_error()) + return DecoderError::format(DecoderErrorCategory::Invalid, "String is not valid UTF-8"); + return string_value.release_value(); +} + +DecoderErrorOr Streamer::read_octet() +{ + u8 result; + Bytes bytes { &result, 1 }; + auto bytes_read = TRY(m_stream_cursor->read_into(bytes)); + if (bytes_read != 1) + return DecoderError::corrupted("Failed to read octet"sv); + return bytes[0]; +} + +DecoderErrorOr Streamer::read_i16() +{ + return (TRY(read_octet()) << 8) | TRY(read_octet()); +} + +DecoderErrorOr Streamer::read_element_id() +{ + auto length_byte = TRY(read_octet()); + auto length = count_leading_zeroes_safe(length_byte) + 1; + if (length > 4) + return DecoderError::corrupted("Element ID must be 4 bytes or less"sv); + u32 result = length_byte; + auto bytes_left = length; + while (--bytes_left > 0) { + result <<= 8; + result |= TRY(read_octet()); + } + + return result; +} + +DecoderErrorOr> Streamer::read_element_size() +{ + auto length_byte = TRY(read_octet()); + auto length = count_leading_zeroes_safe(length_byte) + 1; + if (length > 8) + return DecoderError::corrupted("Element Data Size 8 bytes or less"sv); + + size_t result = length_byte; + auto bytes_left = length; + while (--bytes_left > 0) { + result <<= 8; + result |= TRY(read_octet()); + } + + auto mask = 0xFFFFFFFFFFFFFFFF >> (64 - (7 * length)); + result &= mask; + + if (result == mask) + return OptionalNone(); + + return result; +} + +DecoderErrorOr Streamer::read_variable_size_integer() +{ + auto length_byte = TRY(read_octet()); + auto length = count_leading_zeroes_safe(length_byte) + 1; + if (length > 8) + return DecoderError::corrupted("VINT is too large"sv); + + auto bytes_left = length; + auto result = length_byte & (0xFF >> length); + while (--bytes_left > 0) { + result <<= 8; + result |= TRY(read_octet()); + } + return result; +} + +DecoderErrorOr Streamer::read_variable_size_signed_integer() +{ + auto length_byte = TRY(read_octet()); + auto length = count_leading_zeroes_safe(length_byte) + 1; + if (length > 8) + return DecoderError::corrupted("VINT is too large"sv); + + auto bytes_left = length; + i64 result = length_byte & (0xFF >> length); + while (--bytes_left > 0) { + result <<= 8; + result |= TRY(read_octet()); + } + result -= AK::exp2((length * 7) - 1) - 1; + return result; +} + +DecoderErrorOr Streamer::read_raw_octets(size_t num_octets) +{ + auto result = MUST(ByteBuffer::create_uninitialized(num_octets)); + auto bytes = result.bytes(); + TRY(m_stream_cursor->read_into(bytes)); + return result; +} + +DecoderErrorOr Streamer::read_u64() +{ + auto integer_length = TRY(read_variable_size_integer()); + if (integer_length == 0) + return 0; + if (integer_length > 8) + return DecoderError::corrupted("Integer Element is too large"sv); + u64 result = 0; + for (size_t i = 0; i < integer_length; i++) { + result <<= 8; + result |= TRY(read_octet()); + } + return result; +} + +DecoderErrorOr Streamer::read_i64() +{ + auto integer_length = TRY(read_variable_size_integer()); + if (integer_length == 0) + return 0; + if (integer_length > 8) + return DecoderError::corrupted("Signed Integer Element is too large"sv); + i64 result = 0; + for (size_t i = 0; i < integer_length; i++) { + result <<= 8; + result |= TRY(read_octet()); + } + auto shift = 64 - (static_cast(integer_length) * 8); + result <<= shift; + result >>= shift; + return result; +} + +DecoderErrorOr Streamer::read_float() +{ + auto length = TRY(read_variable_size_integer()); + if (length == 0) + return 0; + if (length != 4u && length != 8u) + return DecoderError::format(DecoderErrorCategory::Invalid, "Float size must be 4 or 8 bytes"); + + union { + u64 value; + float float_value; + double double_value; + } read_data; + read_data.value = 0; + for (size_t i = 0; i < length; i++) { + read_data.value = (read_data.value << 8u) + TRY(read_octet()); + } + if (length == 4u) + return read_data.float_value; + return read_data.double_value; +} + +DecoderErrorOr Streamer::read_unknown_element() +{ + auto element_length = TRY(read_variable_size_integer()); + dbgln_if(MATROSKA_TRACE_DEBUG, "Skipping unknown element of size {}.", element_length); + TRY(m_stream_cursor->seek(element_length, AK::SeekMode::FromCurrentPosition)); + return {}; +} + +size_t Streamer::position() const +{ + return m_stream_cursor->position(); +} + +DecoderErrorOr Streamer::seek_to_position(size_t position) +{ + return m_stream_cursor->seek(position, AK::SeekMode::SetPosition); +} + +} diff --git a/Libraries/LibMedia/Containers/Matroska/Streamer.h b/Libraries/LibMedia/Containers/Matroska/Streamer.h new file mode 100644 index 0000000000000..c74138be2ed14 --- /dev/null +++ b/Libraries/LibMedia/Containers/Matroska/Streamer.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2021, Hunter Salyer + * Copyright (c) 2022-2026, Gregory Bertilson + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include + +namespace Media::Matroska { + +class MEDIA_API Streamer { +public: + Streamer(NonnullRefPtr const& stream_cursor); + ~Streamer(); + + DecoderErrorOr read_octet(); + + DecoderErrorOr read_i16(); + + DecoderErrorOr read_element_id(); + DecoderErrorOr> read_element_size(); + DecoderErrorOr read_variable_size_integer(); + DecoderErrorOr read_variable_size_signed_integer(); + + DecoderErrorOr read_u64(); + DecoderErrorOr read_i64(); + DecoderErrorOr read_float(); + + DecoderErrorOr read_string(); + + DecoderErrorOr read_unknown_element(); + + DecoderErrorOr read_raw_octets(size_t num_octets); + + size_t position() const; + + DecoderErrorOr seek_to_position(size_t position); + + MediaStreamCursor& cursor() { return *m_stream_cursor; } + +private: + NonnullRefPtr m_stream_cursor; +}; + +} diff --git a/Libraries/LibMedia/Containers/OggNavigator.cpp b/Libraries/LibMedia/Containers/OggNavigator.cpp new file mode 100644 index 0000000000000..cfd7e22a63e8d --- /dev/null +++ b/Libraries/LibMedia/Containers/OggNavigator.cpp @@ -0,0 +1,401 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "OggNavigator.h" + +#include +#include +#include +#include + +namespace Media { + +static constexpr StringView CAPTURE_PATTERN = "OggS"sv; +static constexpr size_t CAPTURE_PATTERN_SIZE = CAPTURE_PATTERN.length(); +static constexpr size_t PAGE_HEADER_SIZE = 27; +static constexpr size_t CHUNK_SIZE = 4096; + +static constexpr u8 CONTINUED_PACKET_FLAG = 1 << 0; +static constexpr i64 UNSET_GRANULE_POSITION = -1; + +OggNavigator::OggNavigator(NonnullRefPtr cursor, CodecData codec_data) + : m_cursor(move(cursor)) + , m_codec_data(move(codec_data)) +{ +} + +OwnPtr OggNavigator::create(ReadonlyBytes first_packet, NonnullRefPtr cursor, CodecID codec_id, u32 time_base_numerator, u32 time_base_denominator, u32 sample_rate, ReadonlyBytes codec_initialization_data) +{ + VERIFY(time_base_numerator > 0); + VERIFY(time_base_denominator > 0); + + switch (codec_id) { + case CodecID::Opus: + return adopt_own(*new (nothrow) OggNavigator(move(cursor), Opus { time_base_numerator, time_base_denominator })); + case CodecID::FLAC: { + if (sample_rate == 0 || first_packet.size() < 2) + return {}; + + u16 sync_code = (static_cast(first_packet[0]) << 8) | first_packet[1]; + if (!Codecs::FLAC::is_sync_code(sync_code)) + return {}; + + auto frame_cursor = make_ref_counted(first_packet); + auto frame_info = Codecs::FLAC::parse_frame_header(frame_cursor, sync_code, 0); + if (!frame_info.has_value()) + return {}; + + u16 fixed_block_size = 0; + if ((sync_code & 1) == 0) + fixed_block_size = frame_info->block_size; + + return adopt_own(*new (nothrow) OggNavigator(move(cursor), FLAC { sync_code, fixed_block_size, sample_rate })); + } + case CodecID::Vorbis: { + if (sample_rate == 0) + return {}; + if (codec_initialization_data.is_empty()) + return {}; + + auto parser = Codecs::Vorbis::Parser::create(codec_initialization_data); + if (!parser.has_value()) + return {}; + + return adopt_own(*new (nothrow) OggNavigator(move(cursor), Vorbis { parser.release_value(), sample_rate })); + } + default: + return {}; + } +} + +AK::Duration OggNavigator::granule_to_time(i64 granule_position) const +{ + return m_codec_data.visit( + [&](Opus const& codec) { + return AK::Duration::from_time_units(max(0, granule_position), codec.time_base_numerator, codec.time_base_denominator); + }, + [&](FLAC const& codec) { + return AK::Duration::from_time_units(max(0, granule_position), 1, codec.sample_rate); + }, + [&](Vorbis const& codec) { + return AK::Duration::from_time_units(max(0, granule_position), 1, codec.sample_rate); + }); +} + +void OggNavigator::reset_packet_parser_state() const +{ + m_codec_data.visit( + [](Opus const&) {}, + [](FLAC const&) {}, + [](Vorbis const& codec) { + codec.parser.reset(); + }); +} + +static bool is_capture_pattern(ReadonlyBytes bytes) +{ + return bytes.starts_with(CAPTURE_PATTERN.bytes()); +} + +static bool append_bytes(ByteBuffer& buffer, ReadonlyBytes bytes) +{ + auto original_size = buffer.size(); + if (buffer.try_resize(original_size + bytes.size()).is_error()) + return false; + bytes.copy_to(buffer.span().slice(original_size)); + return true; +} + +static bool verify_page_crc(ReadonlyBytes page) +{ + constexpr size_t checksum_start = 22; + constexpr size_t checksum_end = checksum_start + 4; + VERIFY(page.size() >= checksum_end); + + auto expected_crc = static_cast(page[checksum_start]) + | (static_cast(page[checksum_start + 1]) << 8) + | (static_cast(page[checksum_start + 2]) << 16) + | (static_cast(page[checksum_start + 3]) << 24); + + constexpr auto lookup_table = [] { + Array result; + for (size_t i = 0; i < result.size(); i++) { + u32 value = i << 24; + for (auto bit = 0; bit < 8; bit++) + value = (value << 1) ^ (value & 0x80000000 ? 0x04C11DB7 : 0); + result[i] = value; + } + return result; + }(); + + u32 actual_crc = 0; + for (size_t i = 0; i < page.size(); i++) { + u8 byte = 0; + if (i < checksum_start || i >= checksum_end) + byte = page[i]; + actual_crc = (actual_crc << 8) ^ lookup_table[((actual_crc >> 24) & 0xFF) ^ byte]; + } + + return expected_crc == actual_crc; +} + +Optional OggNavigator::read_packet_duration_in_granules(ReadonlyBytes packet) const +{ + auto packet_cursor = make_ref_counted(packet); + return m_codec_data.visit( + [&](Opus const&) -> Optional { + auto parsed_duration = Codecs::Opus::parse_frame_duration_in_samples(packet_cursor, packet.size()); + if (parsed_duration.is_error()) + return {}; + return parsed_duration.release_value(); + }, + [&](FLAC const& codec) -> Optional { + auto frame = Codecs::FLAC::parse_frame_header(packet_cursor, codec.sync_code, codec.fixed_block_size); + if (!frame.has_value()) + return {}; + return frame->block_size; + }, + [&](Vorbis const& codec) -> Optional { + auto packet_duration = codec.parser.parse_packet_duration_in_samples(packet); + if (!packet_duration.has_value()) + return {}; + return packet_duration.value(); + }); +} + +Optional OggNavigator::parse_page_at(size_t page_start, size_t search_end, ByteBuffer* continued_packet) const +{ + if (page_start + PAGE_HEADER_SIZE > search_end) + return {}; + if (m_cursor->seek_to_position(page_start).is_error()) + return {}; + + auto resize_page_buffer = [&](size_t size) { + m_page_buffer.ensure_capacity(size); + m_page_buffer.set_size(size); + }; + + resize_page_buffer(PAGE_HEADER_SIZE); + if (m_cursor->read_until_filled(m_page_buffer.span()).is_error()) + return {}; + if (!is_capture_pattern(m_page_buffer)) + return {}; + auto version = m_page_buffer[4]; + if (version != 0) + return {}; + auto header_type_flag = m_page_buffer[5]; + bool begins_with_continued_packet = (header_type_flag & CONTINUED_PACKET_FLAG) != 0; + + u64 granule_position_bits { 0 }; + for (size_t i = 0; i < sizeof(granule_position_bits); i++) + granule_position_bits |= static_cast(m_page_buffer[6 + i]) << (8 * i); + auto granule_position = bit_cast(granule_position_bits); + + auto page_segments = m_page_buffer[26]; + if (page_start + PAGE_HEADER_SIZE + page_segments > search_end) + return {}; + + resize_page_buffer(PAGE_HEADER_SIZE + page_segments); + auto segment_table = m_page_buffer.span().slice(PAGE_HEADER_SIZE); + if (m_cursor->read_until_filled(segment_table).is_error()) + return {}; + + size_t payload_size = 0; + for (auto segment_size : segment_table) + payload_size += segment_size; + auto payload_start = page_start + PAGE_HEADER_SIZE + page_segments; + auto payload_end = payload_start + payload_size; + if (payload_end > search_end) + return {}; + + auto page_size = PAGE_HEADER_SIZE + page_segments + payload_size; + resize_page_buffer(page_size); + segment_table = m_page_buffer.span().slice(PAGE_HEADER_SIZE, page_segments); + auto payload = m_page_buffer.span().slice(PAGE_HEADER_SIZE + page_segments); + if (m_cursor->read_until_filled(payload).is_error()) + return {}; + if (!verify_page_crc(m_page_buffer)) + return {}; + + u64 parsed_packet_duration_in_granules { 0 }; + size_t packet_size = 0; + size_t packet_start = 0; + bool packet_started_before_page = begins_with_continued_packet; + for (auto segment_size : segment_table) { + packet_size += segment_size; + if (segment_size == 255) + continue; + + auto packet = payload.slice(packet_start, packet_size); + + if (packet_started_before_page) { + if (continued_packet && !continued_packet->is_empty()) { + if (!append_bytes(*continued_packet, packet)) + return {}; + + auto packet_duration = read_packet_duration_in_granules(continued_packet->span()); + continued_packet->clear(); + if (!packet_duration.has_value()) + return {}; + parsed_packet_duration_in_granules += packet_duration.value(); + } + packet_started_before_page = false; + packet_start += packet_size; + packet_size = 0; + continue; + } + + auto packet_duration = read_packet_duration_in_granules(packet); + if (!packet_duration.has_value()) + return {}; + parsed_packet_duration_in_granules += packet_duration.value(); + packet_start += packet_size; + packet_size = 0; + } + + if (continued_packet && packet_size > 0 && (!packet_started_before_page || !continued_packet->is_empty())) { + if (!append_bytes(*continued_packet, payload.slice(packet_start, packet_size))) + return {}; + } + + if (granule_position == UNSET_GRANULE_POSITION) + return PageScanResult { {}, parsed_packet_duration_in_granules, page_start, payload_end }; + + return PageScanResult { granule_position, parsed_packet_duration_in_granules, page_start, payload_end }; +} + +Optional OggNavigator::find_start_timestamp(size_t search_start, size_t search_end) const +{ + VERIFY(search_start <= search_end); + + Array chunk; + auto chunk_start = search_start; + + while (chunk_start + CAPTURE_PATTERN_SIZE <= search_end) { + auto chunk_size = min(chunk.size(), search_end - chunk_start); + if (m_cursor->seek_to_position(chunk_start).is_error()) + return {}; + auto read_result = m_cursor->read_into(chunk.span().trim(chunk_size)); + if (read_result.is_error()) + return {}; + auto bytes_read = read_result.value(); + if (bytes_read < CAPTURE_PATTERN_SIZE) + return {}; + + for (size_t i = 0; i + CAPTURE_PATTERN_SIZE <= bytes_read; i++) { + if (!is_capture_pattern(chunk.span().slice(i))) + continue; + reset_packet_parser_state(); + auto page = parse_page_at(chunk_start + i, search_end); + if (!page.has_value()) + continue; + if (!page->granule_position.has_value()) + continue; + auto parsed_packet_duration_in_granules = static_cast(page->parsed_packet_duration_in_granules); + if (page->granule_position.value() <= parsed_packet_duration_in_granules) + return AK::Duration::zero(); + return granule_to_time(page->granule_position.value() - parsed_packet_duration_in_granules); + } + + chunk_start += bytes_read - (CAPTURE_PATTERN_SIZE - 1); + } + + return {}; +} + +Optional OggNavigator::find_last_page_with_valid_granule_position(size_t search_start, size_t search_end) const +{ + VERIFY(search_start <= search_end); + + auto chunk_end = search_end; + Array chunk; + while (chunk_end >= search_start + CAPTURE_PATTERN_SIZE) { + auto chunk_start = chunk_end > chunk.size() ? chunk_end - chunk.size() : 0; + chunk_start = max(chunk_start, search_start); + auto chunk_size = chunk_end - chunk_start; + + if (m_cursor->seek_to_position(chunk_start).is_error()) + return {}; + if (m_cursor->read_until_filled(chunk.span().trim(chunk_size)).is_error()) + return {}; + + for (size_t i = chunk_size - CAPTURE_PATTERN_SIZE + 1; i-- > 0;) { + if (!is_capture_pattern(chunk.span().slice(i))) + continue; + reset_packet_parser_state(); + auto page = parse_page_at(chunk_start + i, search_end); + if (page.has_value() && page->granule_position.has_value()) + return page; + } + + if (chunk_start == search_start) + break; + chunk_end = chunk_start + CAPTURE_PATTERN_SIZE - 1; + } + + return {}; +} + +Optional OggNavigator::find_end_timestamp(size_t search_start, size_t search_end) const +{ + auto last_complete_page = find_last_page_with_valid_granule_position(search_start, search_end); + if (!last_complete_page.has_value()) + return {}; + + ByteBuffer continued_packet; + reset_packet_parser_state(); + auto page = parse_page_at(last_complete_page->byte_start, search_end, &continued_packet); + if (!page.has_value() || !page->granule_position.has_value()) + return granule_to_time(last_complete_page->granule_position.value()); + + auto end_granule_position = page->granule_position.value(); + auto page_start = page->byte_end; + while (page_start + CAPTURE_PATTERN_SIZE <= search_end) { + page = parse_page_at(page_start, search_end, &continued_packet); + if (!page.has_value()) + break; + if (page->granule_position.has_value()) + end_granule_position = page->granule_position.value(); + else + end_granule_position += static_cast(page->parsed_packet_duration_in_granules); + if (page->byte_end >= search_end) + break; + page_start = page->byte_end; + } + + return granule_to_time(end_granule_position); +} + +void OggNavigator::on_cached_range_changed(OggCachedRange& cached, CachedRangeChange change) const +{ + bool rescan_start = has_flag(change, CachedRangeChange::Start); + bool rescan_end = has_flag(change, CachedRangeChange::End); + + if (rescan_start && !cached.time_end.has_value()) + rescan_end = true; + if (rescan_end && !cached.time_start.has_value()) + rescan_start = true; + + if (rescan_start) + cached.time_start = find_start_timestamp(cached.byte_start, cached.byte_end); + if (rescan_end) + cached.time_end = find_end_timestamp(cached.byte_start, cached.byte_end); +} + +void OggNavigator::append_time_range(OggCachedRange const& cached_range, TimeRanges& to) +{ + if (!cached_range.time_start.has_value() || !cached_range.time_end.has_value()) + return; + + auto time_start = max(cached_range.time_start.value(), AK::Duration::zero()); + auto time_end = cached_range.time_end.value(); + if (time_start >= time_end) + return; + + to.add_range(time_start, time_end); +} + +} diff --git a/Libraries/LibMedia/Containers/OggNavigator.h b/Libraries/LibMedia/Containers/OggNavigator.h new file mode 100644 index 0000000000000..4409aa92aae31 --- /dev/null +++ b/Libraries/LibMedia/Containers/OggNavigator.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ScanningContainerNavigator.h" + +namespace Media { + +struct OggCachedRange : CachedByteRange { + using CachedByteRange::CachedByteRange; + + Optional time_start; + Optional time_end; +}; + +class OggNavigator final : public ScanningContainerNavigator { +public: + static OwnPtr create(ReadonlyBytes first_packet, NonnullRefPtr, CodecID, u32 time_base_numerator, u32 time_base_denominator, u32 sample_rate, ReadonlyBytes codec_initialization_data); + + void on_cached_range_changed(OggCachedRange& cached_range, CachedRangeChange change) const; + static void append_time_range(OggCachedRange const& cached_range, TimeRanges& to); + +private: + struct Opus { + u32 time_base_numerator { 1 }; + u32 time_base_denominator { 1 }; + }; + + struct FLAC { + u16 sync_code { 0 }; + u16 fixed_block_size { 0 }; + u32 sample_rate { 1 }; + }; + + struct Vorbis { + Codecs::Vorbis::Parser parser; + u32 sample_rate { 1 }; + }; + + using CodecData = Variant; + + struct PageScanResult { + Optional granule_position; + u64 parsed_packet_duration_in_granules; + size_t byte_start; + size_t byte_end; + }; + + OggNavigator(NonnullRefPtr, CodecData); + + Optional find_start_timestamp(size_t search_start, size_t search_end) const; + Optional find_last_page_with_valid_granule_position(size_t search_start, size_t search_end) const; + Optional find_end_timestamp(size_t search_start, size_t search_end) const; + Optional parse_page_at(size_t page_start, size_t search_end, ByteBuffer* continued_packet = nullptr) const; + Optional read_packet_duration_in_granules(ReadonlyBytes) const; + void reset_packet_parser_state() const; + + AK::Duration granule_to_time(i64 granule_position) const; + + NonnullRefPtr m_cursor; + CodecData m_codec_data; + mutable ByteBuffer m_page_buffer; +}; + +} diff --git a/Libraries/LibMedia/Containers/ScanningContainerNavigator.h b/Libraries/LibMedia/Containers/ScanningContainerNavigator.h new file mode 100644 index 0000000000000..41f129cabceaa --- /dev/null +++ b/Libraries/LibMedia/Containers/ScanningContainerNavigator.h @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include + +#include "ContainerNavigator.h" + +namespace Media { + +enum class CachedRangeChange : u8 { + Start = 1 << 0, + End = 1 << 1, +}; +AK_ENUM_BITWISE_OPERATORS(CachedRangeChange); + +struct CachedByteRange { + size_t byte_start { 0 }; + size_t byte_end { 0 }; + + constexpr CachedByteRange(size_t start, size_t end) + : byte_start(start) + , byte_end(end) + { + } +}; + +template +class ScanningContainerNavigator : public ContainerNavigator { +public: + TimeRanges buffered_time_ranges(Vector const& byte_ranges) const override + { + auto const& self = *static_cast(this); + + // Reconcile incoming byte ranges with cached ranges in lockstep. Both are sorted by position. + size_t cached_index = 0; + size_t byte_index = 0; + + while (byte_index < byte_ranges.size()) { + auto const& byte_range = byte_ranges[byte_index]; + + if (cached_index < m_cached_ranges.size()) { + auto& cached = m_cached_ranges[cached_index]; + + // If the cached range doesn't have the same start byte as the incoming range, then we need to update + // the start of the cached range if possible. If that's not possible, then we discard the cached range + // if it's earlier, or insert a new one if it's later. + if (cached.byte_start != byte_range.start) { + if (cached.byte_end == byte_range.end) { + cached.byte_start = byte_range.start; + self.on_cached_range_changed(cached, CachedRangeChange::Start); + cached_index++; + byte_index++; + continue; + } + + if (cached.byte_end < byte_range.end) { + cached = CachedRange { byte_range.start, byte_range.end }; + self.on_cached_range_changed(cached, CachedRangeChange::Start | CachedRangeChange::End); + cached_index++; + byte_index++; + continue; + } + + if (cached.byte_start < byte_range.start) { + m_cached_ranges.remove(cached_index); + } else { + m_cached_ranges.insert(cached_index, CachedRange { byte_range.start, byte_range.end }); + auto& inserted = m_cached_ranges[cached_index]; + self.on_cached_range_changed(inserted, CachedRangeChange::Start | CachedRangeChange::End); + cached_index++; + byte_index++; + } + continue; + } + + // If we're here, the cached range's start matches the incoming one, so we only need to update the end. + if (cached.byte_end != byte_range.end) { + cached.byte_end = byte_range.end; + self.on_cached_range_changed(cached, CachedRangeChange::End); + } + + cached_index++; + byte_index++; + } else { + // We have new byte ranges on the end with no equivalent cached ranges, append them. + m_cached_ranges.append(CachedRange { byte_range.start, byte_range.end }); + auto& appended = m_cached_ranges.last(); + self.on_cached_range_changed(appended, CachedRangeChange::Start | CachedRangeChange::End); + cached_index++; + byte_index++; + } + } + + // Remove any leftover cached ranges past the end. + if (cached_index < m_cached_ranges.size()) + m_cached_ranges.remove(cached_index, m_cached_ranges.size() - cached_index); + + TimeRanges result; + for (auto const& cached : m_cached_ranges) + self.append_time_range(cached, result); + return result; + } + +protected: + mutable Vector m_cached_ranges; +}; + +} diff --git a/Libraries/LibMedia/FFmpeg/FFmpegDemuxer.cpp b/Libraries/LibMedia/FFmpeg/FFmpegDemuxer.cpp index 238258a7bcc0b..4cf097fd928c8 100644 --- a/Libraries/LibMedia/FFmpeg/FFmpegDemuxer.cpp +++ b/Libraries/LibMedia/FFmpeg/FFmpegDemuxer.cpp @@ -9,6 +9,11 @@ #include #include #include +#include +#include +#include +#include +#include #include #include #include @@ -39,7 +44,16 @@ static DecoderErrorOr initialize_format_context(AVFormatContext*& format_c if (format_context == nullptr) return DecoderError::with_description(DecoderErrorCategory::Memory, "Failed to allocate format context"sv); format_context->pb = &io_context; - if (avformat_open_input(&format_context, nullptr, nullptr, nullptr) < 0) + format_context->flags |= AVFMT_FLAG_FAST_SEEK; + + AVDictionary* options = nullptr; + ScopeGuard free_options = [&] { av_dict_free(&options); }; + + // Reduce the maximum packet size for the WAV demuxer, so that playback begins sooner. + av_dict_set(&options, "max_size", "4096", 0); + + auto open_result = avformat_open_input(&format_context, nullptr, nullptr, &options); + if (open_result < 0) return DecoderError::with_description(DecoderErrorCategory::Corrupted, "Failed to open input for format parsing"sv); // Read stream info; doing this is required for headerless formats like MPEG @@ -171,10 +185,147 @@ DecoderErrorOr> FFmpegDemuxer::from_stream(NonnullR demuxer->m_preferred_track_for_type[type_index] = static_cast(i); } + demuxer->m_container_navigator = create_container_navigator(*format_context, demuxer->m_total_duration, stream); + avformat_close_input(&format_context); return demuxer; } +static inline AK::Duration time_units_to_duration(i64 time_units, AVRational const& time_base) +{ + VERIFY(time_base.num > 0); + VERIFY(time_base.den > 0); + return AK::Duration::from_time_units(time_units, time_base.num, time_base.den); +} + +static inline i64 duration_to_time_units(AK::Duration duration, AVRational const& time_base) +{ + VERIFY(time_base.num > 0); + VERIFY(time_base.den > 0); + return duration.to_time_units(time_base.num, time_base.den); +} + +OwnPtr FFmpegDemuxer::create_container_navigator(AVFormatContext& context, AK::Duration total_duration, NonnullRefPtr const& stream) +{ + auto format_name = StringView(context.iformat->name, strlen(context.iformat->name)); + + if (format_name == "flac"sv && context.nb_streams == 1) { + auto& av_stream = *context.streams[0]; + if (av_stream.codecpar->sample_rate > 0) { + auto sample_rate = static_cast(av_stream.codecpar->sample_rate); + AVPacket* packet = av_packet_alloc(); + ScopeGuard free_packet = [&] { av_packet_free(&packet); }; + + if (av_read_frame(&context, packet) >= 0) { + VERIFY(packet->size >= 0); + auto cursor = stream->create_cursor(); + cursor->set_is_blocking(false); + return FLACNavigator::create({ packet->data, static_cast(packet->size) }, move(cursor), sample_rate); + } + } + } + + if (format_name == "wav"sv && context.nb_streams > 0) { + auto* stream = context.streams[0]; + auto* codec_par = stream->codecpar; + if (codec_par->block_align <= 0) + return nullptr; + if (codec_par->sample_rate <= 0) + return nullptr; + if (Checked::multiplication_would_overflow(static_cast(codec_par->block_align), codec_par->sample_rate)) + return nullptr; + auto bytes_per_second = static_cast(codec_par->block_align) * codec_par->sample_rate; + auto entry_count = avformat_index_get_entries_count(stream); + if (entry_count <= 0) + return nullptr; + auto data_offset = avformat_index_get_entry(stream, 0)->pos; + return make(data_offset, bytes_per_second, codec_par->block_align); + } + + if (format_name == "mp3"sv && context.nb_streams == 1) { + AVPacket* packet = av_packet_alloc(); + ScopeGuard free_packet = [&] { av_packet_free(&packet); }; + + if (av_read_frame(&context, packet) >= 0 && packet->pos >= 0) + return make(stream, static_cast(packet->pos), total_duration); + } + + if (format_name == "ogg"sv) { + if (context.nb_streams != 1) + return nullptr; + + auto& av_stream = *context.streams[0]; + if (av_stream.time_base.num <= 0 || av_stream.time_base.den <= 0) + return nullptr; + + auto cursor = stream->create_cursor(); + cursor->set_is_blocking(false); + + auto codec_id = FFmpeg::media_codec_id_from_ffmpeg_codec_id(av_stream.codecpar->codec_id); + ReadonlyBytes first_packet; + AVPacket* packet = nullptr; + ScopeGuard free_packet = [&] { av_packet_free(&packet); }; + + if (codec_id == CodecID::FLAC) { + if (av_stream.codecpar->sample_rate <= 0) + return nullptr; + + packet = av_packet_alloc(); + VERIFY(packet); + if (av_read_frame(&context, packet) >= 0 && packet->size >= 0) + first_packet = { packet->data, static_cast(packet->size) }; + } + + auto sample_rate = av_stream.codecpar->sample_rate > 0 ? static_cast(av_stream.codecpar->sample_rate) : 0; + auto codec_initialization_data = ReadonlyBytes { av_stream.codecpar->extradata, static_cast(av_stream.codecpar->extradata_size) }; + return OggNavigator::create(first_packet, move(cursor), codec_id, static_cast(av_stream.time_base.num), static_cast(av_stream.time_base.den), sample_rate, codec_initialization_data); + } + + return create_container_navigator_from_index(context); +} + +OwnPtr FFmpegDemuxer::create_container_navigator_from_index(AVFormatContext& context) +{ + Vector entries; + for (u32 i = 0; i < context.nb_streams; i++) { + auto* stream = context.streams[i]; + auto entry_count = avformat_index_get_entries_count(stream); + if (entry_count <= 0) + continue; + + MUST(entries.try_ensure_capacity(entries.size() + entry_count)); + for (int j = 0; j < entry_count; j++) { + auto const* entry = avformat_index_get_entry(stream, j); + entries.unchecked_append({ + .position = static_cast(entry->pos), + .timestamp = time_units_to_duration(entry->timestamp, stream->time_base), + }); + } + } + + if (entries.is_empty()) + return nullptr; + + // Sort and ensure monotonic ordering of both positions and timestamps. + for (size_t i = 1; i < entries.size(); i++) { + size_t j = i; + for (; j > 0; --j) { + if (entries[j - 1].position == entries[j].position) + return nullptr; + if (entries[j - 1].position < entries[j].position) + break; + swap(entries[j], entries[j - 1]); + } + if (j > 0 && entries[j - 1].timestamp > entries[j].timestamp) + return nullptr; + if (j < i && entries[j].timestamp > entries[j + 1].timestamp) + return nullptr; + } + + auto duration = AK::Duration::from_time_units(context.duration, 1, AV_TIME_BASE); + return make(move(entries), duration); +} + DecoderErrorOr FFmpegDemuxer::create_context_for_track(Track const& track) { auto cursor = m_stream->create_cursor(); @@ -203,20 +354,6 @@ FFmpegDemuxer::TrackContext& FFmpegDemuxer::get_track_context(Track const& track return *m_track_contexts.get(track).release_value(); } -static inline AK::Duration time_units_to_duration(i64 time_units, AVRational const& time_base) -{ - VERIFY(time_base.num > 0); - VERIFY(time_base.den > 0); - return AK::Duration::from_time_units(time_units, time_base.num, time_base.den); -} - -static inline i64 duration_to_time_units(AK::Duration duration, AVRational const& time_base) -{ - VERIFY(time_base.num > 0); - VERIFY(time_base.den > 0); - return duration.to_time_units(time_base.num, time_base.den); -} - DecoderErrorOr FFmpegDemuxer::total_duration() { return m_total_duration; @@ -229,11 +366,15 @@ Optional FFmpegDemuxer::start_time_realtime() const TimeRanges FFmpegDemuxer::buffered_time_ranges() const { - // FIXME: Use the format context's index to determine the buffered ranges from the underlying stream. - TimeRanges ranges; - if (!m_total_duration.is_zero()) - ranges.add_range(AK::Duration::zero(), m_total_duration); - return ranges; + if (!m_container_navigator) { + TimeRanges ranges; + if (!m_total_duration.is_zero()) + ranges.add_range(AK::Duration::zero(), m_total_duration); + return ranges; + } + + auto byte_ranges = m_stream->available_byte_ranges(); + return m_container_navigator->buffered_time_ranges(byte_ranges); } DecoderErrorOr FFmpegDemuxer::duration_of_track(Track const& track) @@ -286,7 +427,32 @@ DecoderErrorOr FFmpegDemuxer::seek_to_most_recent_keyframe(Tr auto av_timestamp = duration_to_time_units(timestamp, stream.time_base); auto seek_succeeded = false; - if (track_context.is_seekable && av_seek_frame(&format_context, stream.index, av_timestamp, AVSEEK_FLAG_BACKWARD) >= 0) + + // AVIOContext can skip calling through to the underlying seek callback if the new position lands in its buffer, + // leaving us in EOF/error, so we need to clear these here. + format_context.pb->eof_reached = 0; + format_context.pb->error = 0; + + if (m_container_navigator) { + auto seek_result = TRY(m_container_navigator->seek_to_timestamp(timestamp)); + if (seek_result.has()) { + return DemuxerSeekResult::KeptCurrentPosition; + } + if (auto const* seeked = seek_result.get_pointer()) { + if (av_seek_frame(&format_context, stream.index, seeked->byte_position, AVSEEK_FLAG_BYTE) >= 0) { + seek_succeeded = true; + track_context.pending_timestamp_offset = seeked->timestamp; + track_context.timestamp_offset = AK::Duration::zero(); + } + } + } + + if (!seek_succeeded) { + track_context.pending_timestamp_offset.clear(); + track_context.timestamp_offset = AK::Duration::zero(); + } + + if (!seek_succeeded && track_context.is_seekable && av_seek_frame(&format_context, stream.index, av_timestamp, AVSEEK_FLAG_BACKWARD) >= 0) seek_succeeded = true; if (!seek_succeeded) { track_context.is_seekable = false; @@ -353,9 +519,12 @@ DecoderErrorOr FFmpegDemuxer::get_next_sample_for_track(Track const& // to wipe the packet afterwards. auto packet_data = DECODER_TRY_ALLOC(ByteBuffer::copy(packet.data, packet.size)); + if (track_context.pending_timestamp_offset.has_value() && packet.pts == 0) + track_context.timestamp_offset = track_context.pending_timestamp_offset.release_value(); + auto flags = (packet.flags & AV_PKT_FLAG_KEY) != 0 ? FrameFlags::Keyframe : FrameFlags::None; auto sample = CodedFrame( - time_units_to_duration(packet.pts, stream.time_base), + track_context.timestamp_offset + time_units_to_duration(packet.pts, stream.time_base), time_units_to_duration(packet.duration, stream.time_base), flags, move(packet_data), diff --git a/Libraries/LibMedia/FFmpeg/FFmpegDemuxer.h b/Libraries/LibMedia/FFmpeg/FFmpegDemuxer.h index f1eff10549f6c..2d70398e4478e 100644 --- a/Libraries/LibMedia/FFmpeg/FFmpegDemuxer.h +++ b/Libraries/LibMedia/FFmpeg/FFmpegDemuxer.h @@ -11,11 +11,13 @@ #include #include #include +#include #include #include #include #include #include +#include namespace Media::FFmpeg { @@ -74,10 +76,15 @@ class MEDIA_API FFmpegDemuxer : public Demuxer { AVPacket* packet { nullptr }; bool is_seekable { true }; bool peeked_packet_already { false }; + Optional pending_timestamp_offset; + AK::Duration timestamp_offset; }; FFmpegDemuxer(NonnullRefPtr const&); + static OwnPtr create_container_navigator(AVFormatContext&, AK::Duration, NonnullRefPtr const&); + static OwnPtr create_container_navigator_from_index(AVFormatContext&); + StreamInfo const& get_track_info(Track const&) const; TrackContext& get_track_context(Track const&); @@ -85,6 +92,7 @@ class MEDIA_API FFmpegDemuxer : public Demuxer { AK::Duration m_total_duration; Optional m_start_time_realtime; Vector m_stream_info; + OwnPtr m_container_navigator; Array m_preferred_track_for_type; HashMap> m_track_contexts; diff --git a/Libraries/LibMedia/Forward.h b/Libraries/LibMedia/Forward.h index 5dbccd4967ce0..5256d04b239a6 100644 --- a/Libraries/LibMedia/Forward.h +++ b/Libraries/LibMedia/Forward.h @@ -15,6 +15,7 @@ class AudioPlaybackSink; class AudioProducer; class AudioSink; class CodedFrame; +class ContainerNavigator; class DecodedAudioProducer; class DecodedVideoProducer; class DecoderError; diff --git a/Libraries/LibMedia/FrameFlags.h b/Libraries/LibMedia/FrameFlags.h index 5d0d7880bedd8..e025a11480535 100644 --- a/Libraries/LibMedia/FrameFlags.h +++ b/Libraries/LibMedia/FrameFlags.h @@ -15,7 +15,6 @@ enum class FrameFlags : u8 { None = 0, Keyframe = 1 << 0, }; +AK_ENUM_BITWISE_OPERATORS(FrameFlags); } - -AK_ENUM_BITWISE_OPERATORS(Media::FrameFlags); diff --git a/Libraries/LibMedia/IncrementallyPopulatedStream.cpp b/Libraries/LibMedia/IncrementallyPopulatedStream.cpp index 66885b9f52213..d9fef532f3697 100644 --- a/Libraries/LibMedia/IncrementallyPopulatedStream.cpp +++ b/Libraries/LibMedia/IncrementallyPopulatedStream.cpp @@ -62,45 +62,77 @@ void IncrementallyPopulatedStream::add_chunk_at(u64 offset, ReadonlyBytes data) auto previous_chunk_iter = m_chunks.find_largest_not_above_iterator(offset); - // Add a new chunk to the collection if there are none. + // Add a new chunk to the collection if there's no overlapping previous chunk. if (previous_chunk_iter.is_end() || previous_chunk_iter->end() < offset) { DataChunk new_chunk { offset, MUST(ByteBuffer::copy(data)) }; m_chunks.insert(offset, move(new_chunk)); - m_state_changed.broadcast(); + previous_chunk_iter = m_chunks.find_largest_not_above_iterator(offset); + } else if (previous_chunk_iter->end() >= new_chunk_end) { + // The chunk is fully covered by the existing chunk, skip until after it. + begin_new_request_while_locked(previous_chunk_iter->end()); return; } auto& chunk = *previous_chunk_iter; auto& buffer = chunk.data(); - if (chunk.end() >= new_chunk_end) { - // The chunk is fully covered by the existing chunk, skip until after it. - begin_new_request_while_locked(chunk.end()); - return; - } - // Expand the existing chunk to contain this new data. buffer.resize(new_chunk_end - chunk.offset()); data.copy_to(buffer.bytes().slice(offset - chunk.offset())); - // Join the chunk to the next one if they intersect. + // Join the chunk to any following intersecting chunks. auto next_chunk_iter = previous_chunk_iter; ++next_chunk_iter; - if (!next_chunk_iter.is_end() && next_chunk_iter->offset() <= previous_chunk_iter->end()) { - auto& next_chunk = *next_chunk_iter; - - buffer.resize(next_chunk.end() - chunk.offset()); + while (!next_chunk_iter.is_end() && next_chunk_iter->offset() <= chunk.end()) { + auto next_chunk = m_chunks.remove_and_advance(next_chunk_iter); + auto joined_chunk_end = max(chunk.end(), next_chunk.end()); + buffer.resize(joined_chunk_end - chunk.offset()); next_chunk.data().bytes().copy_to(buffer.bytes().slice(next_chunk.offset() - chunk.offset())); + } - VERIFY(m_chunks.remove(next_chunk.offset())); + begin_new_request_while_locked(chunk.end()); + m_state_changed.broadcast(); +} + +void IncrementallyPopulatedStream::remove_byte_range(u64 start, u64 end) +{ + VERIFY(start < end); + + Sync::MutexLocker locker { m_mutex }; + + auto chunk_iterator = m_chunks.find_largest_not_above_iterator(start); + if (chunk_iterator.is_end() || chunk_iterator->end() <= start) + chunk_iterator = m_chunks.find_smallest_not_below_iterator(start); + + while (!chunk_iterator.is_end() && chunk_iterator->offset() < end) { + auto chunk = m_chunks.remove_and_advance(chunk_iterator); + auto chunk_start = chunk.offset(); + auto chunk_end = chunk.end(); + + if (chunk_start < start) { + auto left_size = start - chunk_start; + m_chunks.insert(chunk_start, DataChunk { chunk_start, MUST(ByteBuffer::copy(chunk.data().bytes().trim(left_size))) }); + } - begin_new_request_while_locked(chunk.end()); + if (chunk_end > end) { + auto right_offset_in_chunk = end - chunk_start; + m_chunks.insert(end, DataChunk { end, MUST(ByteBuffer::copy(chunk.data().bytes().slice(right_offset_in_chunk))) }); + } } m_state_changed.broadcast(); } +Vector IncrementallyPopulatedStream::available_byte_ranges() const +{ + Sync::MutexLocker locker { m_mutex }; + Vector ranges; + for (auto it = m_chunks.begin(); it != m_chunks.end(); ++it) + ranges.empend(it->offset(), it->end()); + return ranges; +} + void IncrementallyPopulatedStream::close() { Sync::MutexLocker locker { m_mutex }; @@ -132,6 +164,8 @@ Optional IncrementallyPopulatedStream::expected_size() const void IncrementallyPopulatedStream::begin_new_request_while_locked(u64 position) { + if (!m_callback_event_loop) + return; if (position == m_currently_requested_position) return; @@ -160,7 +194,7 @@ static u64 adjust_request_position(u64 position) return 0; } -bool IncrementallyPopulatedStream::check_if_data_is_available_or_begin_request_while_locked(MonotonicTime now, u64 position, u64 length) +bool IncrementallyPopulatedStream::check_if_data_is_available_or_begin_request_while_locked(Cursor& cursor, u64 position, u64 length) { auto* chunk = m_chunks.find_largest_not_above(position); if (!chunk) @@ -168,23 +202,30 @@ bool IncrementallyPopulatedStream::check_if_data_is_available_or_begin_request_w VERIFY(position >= chunk->offset()); - auto potential_request_position = adjust_request_position(position); - potential_request_position = max(chunk->end(), position); - for (size_t i = 0; i < m_cursors.size(); i++) { - auto const& other_cursor = m_cursors[i]; - if (now >= other_cursor.m_active_timeout && !other_cursor.m_blocked) - continue; - if (other_cursor.m_position < potential_request_position) { - auto* other_cursor_chunk = m_chunks.find_largest_not_above(other_cursor.m_position); - if (other_cursor_chunk && other_cursor_chunk->end() >= other_cursor.m_position) { - potential_request_position = other_cursor_chunk->end(); + if (cursor.m_is_blocking) { + auto now = MonotonicTime::now_coarse(); + cursor.m_active_timeout = now + CURSOR_ACTIVE_TIME; + + auto potential_request_position = adjust_request_position(position); + potential_request_position = max(chunk->end(), potential_request_position); + for (size_t i = 0; i < m_cursors.size(); i++) { + auto const& other_cursor = m_cursors[i]; + if (!other_cursor.m_is_blocking) + continue; + if (now >= other_cursor.m_active_timeout && !other_cursor.m_blocked) continue; + if (other_cursor.m_position < potential_request_position) { + auto* other_cursor_chunk = m_chunks.find_largest_not_above(other_cursor.m_position); + if (other_cursor_chunk && other_cursor_chunk->end() >= other_cursor.m_position) { + potential_request_position = other_cursor_chunk->end(); + continue; + } + potential_request_position = other_cursor.m_position; } - potential_request_position = other_cursor.m_position; } + if (m_currently_requested_position > potential_request_position || potential_request_position > m_last_chunk_end + FORWARD_REQUEST_THRESHOLD) + begin_new_request_while_locked(potential_request_position); } - if (m_currently_requested_position > potential_request_position || potential_request_position > m_last_chunk_end + FORWARD_REQUEST_THRESHOLD) - begin_new_request_while_locked(potential_request_position); u64 end = position + length; if (m_closed && end > m_expected_size.value()) @@ -195,14 +236,17 @@ bool IncrementallyPopulatedStream::check_if_data_is_available_or_begin_request_w size_t IncrementallyPopulatedStream::read_from_chunks_while_locked(u64 position, Bytes& bytes) const { auto chunk_iterator = m_chunks.find_largest_not_above_iterator(position); - VERIFY(!chunk_iterator.is_end()); + if (chunk_iterator.is_end()) + return 0; + auto const& chunk = *chunk_iterator; + if (position >= chunk.end()) + return 0; + VERIFY(position >= chunk.offset()); auto end = position + bytes.size(); auto copy_size = bytes.size(); if (end > chunk.end()) { - VERIFY(m_expected_size.has_value()); - VERIFY(chunk.end() == m_expected_size.value()); end = chunk.end(); copy_size = end - position; } @@ -217,11 +261,8 @@ DecoderErrorOr IncrementallyPopulatedStream::read_at(Cursor& cursor, siz { Sync::MutexLocker locker { m_mutex }; - auto now = MonotonicTime::now_coarse(); - cursor.m_active_timeout = now + CURSOR_ACTIVE_TIME; - - while (!cursor.m_aborted) { - if (check_if_data_is_available_or_begin_request_while_locked(now, position, bytes.size())) + while (!cursor.m_aborted && cursor.m_is_blocking) { + if (check_if_data_is_available_or_begin_request_while_locked(cursor, position, bytes.size())) break; cursor.m_blocked = true; @@ -259,6 +300,11 @@ IncrementallyPopulatedStream::Cursor::~Cursor() VERIFY(m_stream->m_cursors.remove_first_matching([&](Cursor const& cursor) { return this == &cursor; })); } +void IncrementallyPopulatedStream::Cursor::set_is_blocking(bool blocking) +{ + m_is_blocking = blocking; +} + DecoderErrorOr IncrementallyPopulatedStream::Cursor::seek(i64 offset, AK::SeekMode mode) { switch (mode) { diff --git a/Libraries/LibMedia/IncrementallyPopulatedStream.h b/Libraries/LibMedia/IncrementallyPopulatedStream.h index 93bc3dcc230d8..2480e0a7365ee 100644 --- a/Libraries/LibMedia/IncrementallyPopulatedStream.h +++ b/Libraries/LibMedia/IncrementallyPopulatedStream.h @@ -39,18 +39,23 @@ class MEDIA_API IncrementallyPopulatedStream : public MediaStream { void set_data_request_callback(DataRequestCallback); void add_chunk_at(u64 offset, ReadonlyBytes); + void remove_byte_range(u64 start, u64 end); u64 next_chunk_start() const { return m_last_chunk_end; } void close(); + virtual Vector available_byte_ranges() const override; + u64 size(); void set_expected_size(u64); - Optional expected_size() const; + virtual Optional expected_size() const override; class MEDIA_API Cursor : public MediaStreamCursor { public: ~Cursor(); + virtual void set_is_blocking(bool) override; + virtual DecoderErrorOr seek(i64 offset, AK::SeekMode mode) override; virtual DecoderErrorOr read_into(Bytes bytes) override; @@ -69,6 +74,7 @@ class MEDIA_API IncrementallyPopulatedStream : public MediaStream { Cursor(NonnullRefPtr const& stream); NonnullRefPtr m_stream; + bool m_is_blocking { true }; size_t m_position { 0 }; bool m_aborted { false }; Atomic m_blocked { false }; @@ -106,7 +112,7 @@ class MEDIA_API IncrementallyPopulatedStream : public MediaStream { DecoderErrorOr read_at(Cursor&, size_t position, Bytes&); void begin_new_request_while_locked(u64 position); - bool check_if_data_is_available_or_begin_request_while_locked(MonotonicTime now, u64 position, u64 length); + bool check_if_data_is_available_or_begin_request_while_locked(Cursor&, u64 position, u64 length); size_t read_from_chunks_while_locked(u64 position, Bytes& bytes) const; mutable Sync::Mutex m_mutex; diff --git a/Libraries/LibMedia/MediaStream.h b/Libraries/LibMedia/MediaStream.h index 8b5a82536aab6..1ab532e9badb2 100644 --- a/Libraries/LibMedia/MediaStream.h +++ b/Libraries/LibMedia/MediaStream.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025, Gregory Bertilson + * Copyright (c) 2025-2026, Gregory Bertilson * * SPDX-License-Identifier: BSD-2-Clause */ @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include #include @@ -17,11 +18,49 @@ class MediaStreamCursor : public AtomicRefCounted { public: virtual ~MediaStreamCursor() = default; + virtual void set_is_blocking(bool) = 0; + virtual DecoderErrorOr seek(i64 offset, AK::SeekMode) = 0; virtual DecoderErrorOr read_into(Bytes) = 0; virtual size_t position() const = 0; virtual size_t size() const = 0; + DecoderErrorOr read_until_filled(Bytes buffer) + { + auto bytes_read = TRY(read_into(buffer)); + if (bytes_read != buffer.size()) + return DecoderError::corrupted("Unexpected end of stream"sv); + return {}; + } + + template + DecoderErrorOr read_value(AK::Endianness endianness = AK::Endianness::Host) + { + T value = 0; + TRY(read_until_filled({ &value, sizeof(value) })); + switch (endianness) { + case AK::Endianness::Host: + return value; + case AK::Endianness::Big: + return AK::convert_between_host_and_big_endian(value); + case AK::Endianness::Little: + return AK::convert_between_host_and_little_endian(value); + } + VERIFY_NOT_REACHED(); + } + + DecoderErrorOr seek_to_position(size_t position) + { + if (position > NumericLimits::max()) + return DecoderError::corrupted("Seek position is too large"sv); + return seek(static_cast(position), AK::SeekMode::SetPosition); + } + + DecoderErrorOr skip(i64 bytes) + { + return seek(bytes, AK::SeekMode::FromCurrentPosition); + } + virtual void abort() { } virtual void reset_abort() { } virtual bool is_aborted() const { return false; } @@ -30,9 +69,18 @@ class MediaStreamCursor : public AtomicRefCounted { class MediaStream : public AtomicRefCounted { public: + struct ByteRange { + size_t start { 0 }; + size_t end { 0 }; + }; + virtual ~MediaStream() = default; virtual NonnullRefPtr create_cursor() = 0; + + virtual Vector available_byte_ranges() const = 0; + + virtual Optional expected_size() const = 0; }; } diff --git a/Libraries/LibMedia/ReadonlyBytesCursor.h b/Libraries/LibMedia/ReadonlyBytesCursor.h index f6e48bfa623f6..7c5d96a74e147 100644 --- a/Libraries/LibMedia/ReadonlyBytesCursor.h +++ b/Libraries/LibMedia/ReadonlyBytesCursor.h @@ -19,6 +19,8 @@ class ReadonlyBytesCursor final : public MediaStreamCursor { { } + virtual void set_is_blocking(bool) override { } + virtual DecoderErrorOr seek(i64 offset, AK::SeekMode mode) override { auto target_position = [&] -> size_t { diff --git a/Libraries/LibWeb/HTML/HTMLMediaElement.cpp b/Libraries/LibWeb/HTML/HTMLMediaElement.cpp index 93286e4efaddb..c4347eab64ee4 100644 --- a/Libraries/LibWeb/HTML/HTMLMediaElement.cpp +++ b/Libraries/LibWeb/HTML/HTMLMediaElement.cpp @@ -547,17 +547,6 @@ void HTMLMediaElement::pause() pause_element(); } -void HTMLMediaElement::toggle_playback() -{ - // AD-HOC: An execution context is required for Promise creation hooks. - TemporaryExecutionContext execution_context { realm() }; - - if (potentially_playing()) - pause(); - else - play(); -} - // https://html.spec.whatwg.org/multipage/media.html#dom-media-volume WebIDL::ExceptionOr HTMLMediaElement::set_volume(double volume) { @@ -1926,12 +1915,28 @@ void HTMLMediaElement::process_media_data(FetchingStatus fetching_status) // Set the networkState to NETWORK_IDLE and fire an event named suspend at the media element. m_network_state = NetworkState::Idle; dispatch_event(DOM::Event::create(realm, HTML::EventNames::suspend)); + + update_ready_state(); } else if (fetching_status == FetchingStatus::Ongoing) { - // If the user agent ever discards any media data and then needs to resume the network activity to obtain it again, then it must queue a media - // element task given the media element to set the networkState to NETWORK_LOADING. + // If the user agent ever discards any media data and then needs to resume the network activity to obtain it + // again, then it must queue a media element task given the media element to set the networkState to NETWORK_LOADING. queue_a_media_element_task(GC::weak_callback(*this, [](auto& self) { self.m_network_state = NetworkState::Loading; })); + + // While the load is not suspended (see below), every 350ms (±200ms) or for every byte received, whichever is + // least frequent, queue a media element task given the media element to: + auto now = MonotonicTime::now(); + if (!m_last_progress_event_time.has_value() || now - m_last_progress_event_time.value() > AK::Duration::from_milliseconds(350)) { + m_last_progress_event_time = now; + queue_a_media_element_task(GC::weak_callback(*this, [](auto& self) { + // FIXME: 1. Set the element's is currently stalled to false. + // 2. Fire an event named progress at the element. + self.dispatch_event(DOM::Event::create(self.realm(), HTML::EventNames::progress)); + })); + + update_ready_state(); + } } // -> If the connection is interrupted after some media data has been received, causing the user agent to give up trying diff --git a/Libraries/LibWeb/HTML/HTMLMediaElement.h b/Libraries/LibWeb/HTML/HTMLMediaElement.h index 40bc4dc37cddc..13b432b354fdd 100644 --- a/Libraries/LibWeb/HTML/HTMLMediaElement.h +++ b/Libraries/LibWeb/HTML/HTMLMediaElement.h @@ -125,7 +125,6 @@ class HTMLMediaElement : public HTMLElement { bool potentially_playing() const; GC::Ref play(); void pause(); - void toggle_playback(); double volume() const { return m_volume; } WebIDL::ExceptionOr set_volume(double); @@ -370,6 +369,8 @@ class HTMLMediaElement : public HTMLElement { bool m_running_time_update_event_handler { false }; Optional m_last_time_update_event_time; + Optional m_last_progress_event_time; + GC::Ptr m_document_observer; GC::Ptr m_source_element_selector; diff --git a/Libraries/LibWeb/HTML/MediaControls.cpp b/Libraries/LibWeb/HTML/MediaControls.cpp index 045b75e7c9fa7..864a754eca72a 100644 --- a/Libraries/LibWeb/HTML/MediaControls.cpp +++ b/Libraries/LibWeb/HTML/MediaControls.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -437,7 +439,10 @@ void MediaControls::toggle_playback() { if (m_scrubbing_timeline != Scrubbing::No) return; - m_media_element->toggle_playback(); + if (m_media_element->paused()) + m_media_element->play(); + else + m_media_element->pause(); show_controls(); } @@ -487,18 +492,58 @@ void MediaControls::update_play_pause_icon() void MediaControls::update_timeline() { VERIFY(m_media_element); + VERIFY(m_dom->timeline_track); VERIFY(m_dom->timeline_fill); + auto format_percent = [](double value) { + return MUST(String::formatted("{}%", value * 100)); + }; + auto duration = m_media_element->duration(); - double percentage = 0.0; + double progress = 0.0; if (!isnan(duration) && duration > 0.0) - percentage = (m_media_element->current_time() / duration) * 100.0; + progress = (m_media_element->current_time() / duration); - if (m_last_timeline_percentage == percentage) - return; + if (m_last_timeline_progress != progress) { + MUST(m_dom->timeline_fill->style_for_bindings()->set_property(CSS::PropertyID::Width, format_percent(progress))); + m_last_timeline_progress = progress; + } + + auto buffered = m_media_element->buffered(); + auto range_count = buffered->length(); + if (isnan(duration) || duration <= 0.0) + range_count = 0; - MUST(m_dom->timeline_fill->style_for_bindings()->set_property(CSS::PropertyID::Width, MUST(String::formatted("{}%", percentage)))); - m_last_timeline_percentage = percentage; + while (m_buffered_ranges.size() > range_count) { + auto range_div = m_buffered_ranges.take_last(); + VERIFY(range_div.element); + range_div.element->remove(); + } + + while (m_buffered_ranges.size() < range_count) { + auto range = MUST(DOM::create_element(m_media_element->document(), HTML::TagNames::div, Namespace::HTML)); + static auto s_timeline_buffered_class = "timeline-buffered"_string; + MUST(range->class_list()->toggle(s_timeline_buffered_class, true)); + MUST(range->style_for_bindings()->set_property(CSS::PropertyID::Display, "block"sv)); + m_dom->timeline_track->insert_before(range, nullptr); + m_buffered_ranges.empend(*range); + } + + for (size_t i = 0; i < range_count; i++) { + auto& range = m_buffered_ranges[i]; + auto range_start = MUST(buffered->start(i)); + auto range_duration = MUST(buffered->end(i)) - range_start; + auto left = range_start / duration; + auto width = range_duration / duration; + if (left == range.left && width == range.width) + continue; + range.left = left; + range.width = width; + + auto style = range.element->style_for_bindings(); + MUST(style->set_property(CSS::PropertyID::Left, format_percent(left))); + MUST(style->set_property(CSS::PropertyID::Width, format_percent(width))); + } } void MediaControls::request_timeline_update() @@ -572,13 +617,11 @@ void MediaControls::update_volume_and_mute_indicator() m_mute_icon_state = new_volume_icon_state; } - static Vector s_muted_class = { "muted"_string }; if (muted != m_was_muted) { MUST(m_dom->mute_button->class_list()->toggle("muted"_string, muted)); m_was_muted = muted; } - static Vector s_hidden_class = { "hidden"_string }; if (has_audio != m_had_audio) { MUST(m_dom->volume_area->class_list()->toggle("hidden"_string, !has_audio)); m_had_audio = has_audio; diff --git a/Libraries/LibWeb/HTML/MediaControls.css b/Libraries/LibWeb/HTML/MediaControls.css index 434803ca6cea0..f1d87b3ab2914 100644 --- a/Libraries/LibWeb/HTML/MediaControls.css +++ b/Libraries/LibWeb/HTML/MediaControls.css @@ -48,16 +48,28 @@ .timeline { z-index: 1; - height: 5px; padding: 5px 0; margin: -5px 0; - background: var(--track-color); - background-clip: content-box; cursor: pointer; flex-shrink: 0; text-align: left; } +.timeline-track { + position: relative; + height: 5px; + background: var(--track-color); +} + +.timeline-buffered { + position: absolute; + top: 0; + height: 100%; + background: rgba(255, 255, 255, 0.3); + mix-blend-mode: hard-light; + pointer-events: none; +} + .timeline-fill { height: 100%; background: var(--accent-color); diff --git a/Libraries/LibWeb/HTML/MediaControls.h b/Libraries/LibWeb/HTML/MediaControls.h index aa1a174f8595a..af6ff4f24b401 100644 --- a/Libraries/LibWeb/HTML/MediaControls.h +++ b/Libraries/LibWeb/HTML/MediaControls.h @@ -100,7 +100,14 @@ class MediaControls { }; MuteIconState m_mute_icon_state { MuteIconState::Empty }; - double m_last_timeline_percentage { 0.0 }; + double m_last_timeline_progress { 0.0 }; + + struct BufferedRange { + GC::Weak element; + double left { 0 }; + double width { 0 }; + }; + Vector m_buffered_ranges; }; } diff --git a/Libraries/LibWeb/HTML/MediaControls.html b/Libraries/LibWeb/HTML/MediaControls.html index 97a9fda95238f..28609883efc65 100644 --- a/Libraries/LibWeb/HTML/MediaControls.html +++ b/Libraries/LibWeb/HTML/MediaControls.html @@ -9,7 +9,9 @@
-
+
+
+