diff --git a/src/torchcodec/_core/BetaCudaDeviceInterface.cpp b/src/torchcodec/_core/BetaCudaDeviceInterface.cpp
index 7f47d327e..82ece2b7b 100644
--- a/src/torchcodec/_core/BetaCudaDeviceInterface.cpp
+++ b/src/torchcodec/_core/BetaCudaDeviceInterface.cpp
@@ -752,7 +752,8 @@ UniqueAVFrame BetaCudaDeviceInterface::convert_cuda_frame_to_av_frame(
   // Note that we used to rely on videoFormat_.frame_rate for this, but that
   // proved less accurate than FFmpeg.
   set_duration(
-      av_frame, compute_safe_duration(frame_rate_avg_from_ffmpeg_, time_base_));
+      *av_frame,
+      compute_safe_duration(frame_rate_avg_from_ffmpeg_, time_base_));
 
   // We need to assign the frame colorspace. This is crucial for proper color
   // conversion. NVCUVID stores that in the matrix_coefficients field, but
@@ -824,7 +825,7 @@ void BetaCudaDeviceInterface::flush() {
 }
 
 UniqueAVFrame BetaCudaDeviceInterface::transfer_cpu_frame_to_gpu(
-    UniqueAVFrame& cpu_frame,
+    const AVFrame& cpu_frame,
     AVPixelFormat target_pix_fmt) {
   // This is called in the context of the CPU fallback: the frame was decoded on
   // the CPU, and in this function we convert that frame into NV12 or P016
@@ -838,15 +839,14 @@ UniqueAVFrame BetaCudaDeviceInterface::transfer_cpu_frame_to_gpu(
   // (rounded up) width and height, even if the original CPU frame had odd
   // dimensions.
 
-  STD_TORCH_CHECK(cpu_frame != nullptr, "CPU frame cannot be null");
   // NV12 = 1 byte per sample, P016 = 2 bytes per sample
   STD_TORCH_CHECK(
       target_pix_fmt == AV_PIX_FMT_NV12 || target_pix_fmt == AV_PIX_FMT_P016LE,
       "targetPixFmt must be NV12 or P016LE");
   int bytes_per_sample = (target_pix_fmt == AV_PIX_FMT_P016LE) ? 2 : 1;
 
-  int width = cpu_frame->width;
-  int height = cpu_frame->height;
+  int width = cpu_frame.width;
+  int height = cpu_frame.height;
   int even_width = round_up_to_even(width);
   int even_height = round_up_to_even(height);
 
@@ -868,8 +868,8 @@ UniqueAVFrame BetaCudaDeviceInterface::transfer_cpu_frame_to_gpu(
   SwsConfig sws_config(
       width,
       height,
-      static_cast<AVPixelFormat>(cpu_frame->format),
-      cpu_frame->colorspace,
+      static_cast<AVPixelFormat>(cpu_frame.format),
+      cpu_frame.colorspace,
       even_width,
       even_height,
       target_pix_fmt);
@@ -881,8 +881,8 @@ UniqueAVFrame BetaCudaDeviceInterface::transfer_cpu_frame_to_gpu(
 
   int converted_height = sws_scale(
       sws_context_.get(),
-      cpu_frame->data,
-      cpu_frame->linesize,
+      cpu_frame.data,
+      cpu_frame.linesize,
       0,
       height,
       intermediate_cpu_frame->data,
@@ -944,7 +944,7 @@ UniqueAVFrame BetaCudaDeviceInterface::transfer_cpu_frame_to_gpu(
       "Failed to copy UV plane to GPU: ",
       cudaGetErrorString(err));
 
-  ret = av_frame_copy_props(gpu_frame.get(), cpu_frame.get());
+  ret = av_frame_copy_props(gpu_frame.get(), &cpu_frame);
   STD_TORCH_CHECK(
       ret >= 0,
       "Failed to copy frame properties: ",
@@ -967,7 +967,7 @@ UniqueAVFrame BetaCudaDeviceInterface::transfer_cpu_frame_to_gpu(
 }
 
 void BetaCudaDeviceInterface::convert_av_frame_to_frame_output(
-    UniqueAVFrame& av_frame,
+    const AVFrame& av_frame,
     FrameOutput& frame_output,
     std::optional<torch::stable::Tensor> pre_allocated_output_tensor) {
   if (cpu_fallback_) {
@@ -979,7 +979,7 @@ void BetaCudaDeviceInterface::convert_av_frame_to_frame_output(
     // do the color conversion on the CPU and then send the full RGB frame to
     // the GPU.
     const AVPixFmtDescriptor* desc =
-        av_pix_fmt_desc_get(static_cast<AVPixelFormat>(av_frame->format));
+        av_pix_fmt_desc_get(static_cast<AVPixelFormat>(av_frame.format));
     bool is444 = desc && desc->log2_chroma_w == 0 && desc->log2_chroma_h == 0;
     if (is444) {
       FrameOutput cpu_frame_output;
@@ -1001,28 +1001,29 @@ void BetaCudaDeviceInterface::convert_av_frame_to_frame_output(
 
   // Capture original dimensions before transferCpuFrameToGpu()
   // may round them up to even.
-  FrameDims original_dims(av_frame->height, av_frame->width);
+  FrameDims original_dims(av_frame.height, av_frame.width);
 
-  UniqueAVFrame gpu_frame;
+  // On the CPU fallback we own the GPU frame we just created; otherwise the
+  // input frame is already what we need, and we only observe it.
+  UniqueAVFrame transferred_frame;
   if (cpu_fallback_) {
     AVPixelFormat target_pix_fmt = (output_dtype_ == OutputDtype::FLOAT32)
         ? AV_PIX_FMT_P016LE
         : AV_PIX_FMT_NV12;
-    gpu_frame = transfer_cpu_frame_to_gpu(av_frame, target_pix_fmt);
-  } else {
-    gpu_frame = std::move(av_frame);
+    transferred_frame = transfer_cpu_frame_to_gpu(av_frame, target_pix_fmt);
   }
+  const AVFrame& gpu_frame = cpu_fallback_ ? *transferred_frame : av_frame;
 
   STD_TORCH_CHECK(
-      gpu_frame->format == AV_PIX_FMT_NV12 ||
-          gpu_frame->format == AV_PIX_FMT_P016LE,
+      gpu_frame.format == AV_PIX_FMT_NV12 ||
+          gpu_frame.format == AV_PIX_FMT_P016LE,
       "Expected NV12 or P016LE format frame");
 
   cudaStream_t nvdec_stream = get_current_cuda_stream(device_.index());
 
   auto convert_frame = [&](std::optional<torch::stable::Tensor> pre_alloc)
       -> torch::stable::Tensor {
-    bool is_p016 = (gpu_frame->format == AV_PIX_FMT_P016LE);
+    bool is_p016 = (gpu_frame.format == AV_PIX_FMT_P016LE);
     int bit_depth = 8;
     if (is_p016) {
       bit_depth = cpu_fallback_
diff --git a/src/torchcodec/_core/BetaCudaDeviceInterface.h b/src/torchcodec/_core/BetaCudaDeviceInterface.h
index 1812a341c..e2d0e3518 100644
--- a/src/torchcodec/_core/BetaCudaDeviceInterface.h
+++ b/src/torchcodec/_core/BetaCudaDeviceInterface.h
@@ -52,7 +52,7 @@ class BetaCudaDeviceInterface : public DeviceInterface {
       OutputDtype requested_dtype) const override;
 
   void convert_av_frame_to_frame_output(
-      UniqueAVFrame& av_frame,
+      const AVFrame& av_frame,
       FrameOutput& frame_output,
       std::optional<torch::stable::Tensor> pre_allocated_output_tensor)
       override;
@@ -92,7 +92,7 @@ class BetaCudaDeviceInterface : public DeviceInterface {
       const CUVIDPARSERDISPINFO& disp_info);
 
   UniqueAVFrame transfer_cpu_frame_to_gpu(
-      UniqueAVFrame& cpu_frame,
+      const AVFrame& cpu_frame,
       AVPixelFormat target_pix_fmt);
 
   void apply_rotation(
diff --git a/src/torchcodec/_core/ColorConverter.cpp b/src/torchcodec/_core/ColorConverter.cpp
index 6263af1cd..2e06dd617 100644
--- a/src/torchcodec/_core/ColorConverter.cpp
+++ b/src/torchcodec/_core/ColorConverter.cpp
@@ -45,7 +45,7 @@ ColorConverter::ColorConverter(
       /*resized_output_dims=*/std::nullopt);
 }
 
-torch::stable::Tensor ColorConverter::convert(UniqueAVFrame& av_frame) {
+torch::stable::Tensor ColorConverter::convert(const AVFrame& av_frame) {
   FrameOutput frame_output;
   device_interface_->convert_av_frame_to_frame_output(
       av_frame, frame_output, std::nullopt);
diff --git a/src/torchcodec/_core/ColorConverter.h b/src/torchcodec/_core/ColorConverter.h
index cf6368735..d06f2d4a2 100644
--- a/src/torchcodec/_core/ColorConverter.h
+++ b/src/torchcodec/_core/ColorConverter.h
@@ -21,7 +21,7 @@ class FORCE_PUBLIC_VISIBILITY ColorConverter {
       const StableDevice& device = StableDevice(kStableCPU),
       std::string_view device_variant = "default");
 
-  torch::stable::Tensor convert(UniqueAVFrame& av_frame);
+  torch::stable::Tensor convert(const AVFrame& av_frame);
 
  private:
   std::unique_ptr<DeviceInterface> device_interface_;
diff --git a/src/torchcodec/_core/CpuDeviceInterface.cpp b/src/torchcodec/_core/CpuDeviceInterface.cpp
index 46731ea0a..b6268adbd 100644
--- a/src/torchcodec/_core/CpuDeviceInterface.cpp
+++ b/src/torchcodec/_core/CpuDeviceInterface.cpp
@@ -200,7 +200,7 @@ ColorConversionLibrary CpuDeviceInterface::get_color_conversion_library(
 }
 
 void CpuDeviceInterface::convert_av_frame_to_frame_output(
-    UniqueAVFrame& av_frame,
+    const AVFrame& av_frame,
     FrameOutput& frame_output,
     std::optional<torch::stable::Tensor> pre_allocated_output_tensor) {
   STD_TORCH_CHECK(initialized_, "CpuDeviceInterface was not initialized.");
@@ -223,7 +223,7 @@ void CpuDeviceInterface::convert_av_frame_to_frame_output(
 // Dimension order of the preAllocatedOutputTensor must be HWC, regardless of
 // `dimension_order` parameter. It's up to callers to re-shape it if needed.
 void CpuDeviceInterface::convert_video_av_frame_to_frame_output(
-    UniqueAVFrame& av_frame,
+    const AVFrame& av_frame,
     FrameOutput& frame_output,
     std::optional<torch::stable::Tensor> pre_allocated_output_tensor) {
   // Note that we ignore the dimensions from the metadata; we don't even bother
@@ -239,7 +239,7 @@ void CpuDeviceInterface::convert_video_av_frame_to_frame_output(
   // Both cases cause problems for our batch APIs, as we allocate
   // FrameBatchOutputs based on the the stream metadata. But single-frame APIs
   // can still work in such situations, so they should.
-  auto input_dims = FrameDims(av_frame->height, av_frame->width);
+  auto input_dims = FrameDims(av_frame.height, av_frame.width);
   auto output_dims = resized_output_dims_.value_or(input_dims);
 
   if (pre_allocated_output_tensor.has_value()) {
@@ -264,12 +264,12 @@ void CpuDeviceInterface::convert_video_av_frame_to_frame_output(
         pre_allocated_output_tensor.value_or(allocate_empty_hwc_tensor(
             output_dims, kStableCPU, video_stream_options_.output_dtype));
 
-    auto av_frame_format = static_cast<AVPixelFormat>(av_frame->format);
+    auto av_frame_format = static_cast<AVPixelFormat>(av_frame.format);
     SwsConfig sws_config(
-        av_frame->width,
-        av_frame->height,
+        av_frame.width,
+        av_frame.height,
         av_frame_format,
-        av_frame->colorspace,
+        av_frame.colorspace,
         output_dims.width,
         output_dims.height,
         output_pixel_format_);
@@ -326,15 +326,15 @@ void CpuDeviceInterface::convert_video_av_frame_to_frame_output(
 
 torch::stable::Tensor
 CpuDeviceInterface::convert_av_frame_to_tensor_using_filter_graph(
-    const UniqueAVFrame& av_frame,
+    const AVFrame& av_frame,
     const FrameDims& output_dims) {
-  auto av_frame_format = static_cast<AVPixelFormat>(av_frame->format);
+  auto av_frame_format = static_cast<AVPixelFormat>(av_frame.format);
 
   FiltersConfig filters_config(
-      av_frame->width,
-      av_frame->height,
+      av_frame.width,
+      av_frame.height,
       av_frame_format,
-      av_frame->sample_aspect_ratio,
+      av_frame.sample_aspect_ratio,
       output_dims.width,
       output_dims.height,
       output_pixel_format_,
@@ -346,17 +346,17 @@ CpuDeviceInterface::convert_av_frame_to_tensor_using_filter_graph(
         std::make_unique<FilterGraph>(filters_config, video_stream_options_);
     prev_filters_config_ = std::move(filters_config);
   }
-  return rgb_av_frame_to_tensor(filter_graph_->convert(av_frame));
+  return rgb_av_frame_to_tensor(*filter_graph_->convert(av_frame));
 }
 
 void CpuDeviceInterface::convert_audio_av_frame_to_frame_output(
-    UniqueAVFrame& src_av_frame,
+    const AVFrame& src_av_frame,
     FrameOutput& frame_output) {
   AVSampleFormat src_sample_format =
-      static_cast<AVSampleFormat>(src_av_frame->format);
+      static_cast<AVSampleFormat>(src_av_frame.format);
   AVSampleFormat out_sample_format = AV_SAMPLE_FMT_FLTP;
 
-  int src_sample_rate = src_av_frame->sample_rate;
+  int src_sample_rate = src_av_frame.sample_rate;
   int out_sample_rate =
       audio_stream_options_.sample_rate.value_or(src_sample_rate);
 
@@ -397,10 +397,9 @@ void CpuDeviceInterface::convert_audio_av_frame_to_frame_output(
         out_sample_rate,
         out_num_channels);
   }
-  const UniqueAVFrame& av_frame =
-      must_convert ? converted_av_frame : src_av_frame;
+  const AVFrame& av_frame = must_convert ? *converted_av_frame : src_av_frame;
 
-  AVSampleFormat format = static_cast<AVSampleFormat>(av_frame->format);
+  AVSampleFormat format = static_cast<AVSampleFormat>(av_frame.format);
   STD_TORCH_CHECK(
       format == out_sample_format,
       "Something went wrong, the frame didn't get converted to the desired format. ",
@@ -419,7 +418,7 @@ void CpuDeviceInterface::convert_audio_av_frame_to_frame_output(
       num_channels,
       " instead.");
 
-  auto num_samples = av_frame->nb_samples;
+  auto num_samples = av_frame.nb_samples;
 
   frame_output.data = torch::stable::empty({num_channels, num_samples});
 
@@ -431,7 +430,7 @@ void CpuDeviceInterface::convert_audio_av_frame_to_frame_output(
          ++channel, output_channel_data += num_bytes_per_channel) {
       std::memcpy(
           output_channel_data,
-          av_frame->extended_data[channel],
+          av_frame.extended_data[channel],
           num_bytes_per_channel);
     }
   }
diff --git a/src/torchcodec/_core/CpuDeviceInterface.h b/src/torchcodec/_core/CpuDeviceInterface.h
index 1a3166e66..2d57eaf7b 100644
--- a/src/torchcodec/_core/CpuDeviceInterface.h
+++ b/src/torchcodec/_core/CpuDeviceInterface.h
@@ -41,7 +41,7 @@ class CpuDeviceInterface : public DeviceInterface {
       override;
 
   void convert_av_frame_to_frame_output(
-      UniqueAVFrame& av_frame,
+      const AVFrame& av_frame,
       FrameOutput& frame_output,
       std::optional<torch::stable::Tensor> pre_allocated_output_tensor)
       override;
@@ -59,16 +59,16 @@ class CpuDeviceInterface : public DeviceInterface {
 
  private:
   void convert_audio_av_frame_to_frame_output(
-      UniqueAVFrame& src_av_frame,
+      const AVFrame& src_av_frame,
       FrameOutput& frame_output);
 
   void convert_video_av_frame_to_frame_output(
-      UniqueAVFrame& av_frame,
+      const AVFrame& av_frame,
       FrameOutput& frame_output,
       std::optional<torch::stable::Tensor> pre_allocated_output_tensor);
 
   torch::stable::Tensor convert_av_frame_to_tensor_using_filter_graph(
-      const UniqueAVFrame& av_frame,
+      const AVFrame& av_frame,
       const FrameDims& output_dims);
 
   ColorConversionLibrary get_color_conversion_library(
diff --git a/src/torchcodec/_core/CudaDeviceInterface.cpp b/src/torchcodec/_core/CudaDeviceInterface.cpp
index d1962ac5a..1e8c2c34f 100644
--- a/src/torchcodec/_core/CudaDeviceInterface.cpp
+++ b/src/torchcodec/_core/CudaDeviceInterface.cpp
@@ -146,7 +146,7 @@ void CudaDeviceInterface::register_hardware_device_with_codec(
 }
 
 UniqueAVFrame CudaDeviceInterface::maybe_convert_av_frame_to_nv12_or_rgb24(
-    UniqueAVFrame& av_frame) {
+    const AVFrame& av_frame) {
   // We need FFmpeg filters to handle those conversion cases which are not
   // directly implemented in CUDA or CPU device interface (in case of a
   // fallback).
@@ -154,12 +154,12 @@ UniqueAVFrame CudaDeviceInterface::maybe_convert_av_frame_to_nv12_or_rgb24(
   // Input frame is on CPU, we will just pass it to CPU device interface, so
   // skipping filters context as CPU device interface will handle everything for
   // us.
-  if (av_frame->format != AV_PIX_FMT_CUDA) {
-    return std::move(av_frame);
+  if (av_frame.format != AV_PIX_FMT_CUDA) {
+    return UniqueAVFrame{};
   }
 
   auto hw_frames_ctx =
-      reinterpret_cast<AVHWFramesContext*>(av_frame->hw_frames_ctx->data);
+      reinterpret_cast<AVHWFramesContext*>(av_frame.hw_frames_ctx->data);
   STD_TORCH_CHECK(
       hw_frames_ctx != nullptr,
       "The AVFrame does not have a hw_frames_ctx. "
@@ -169,7 +169,7 @@ UniqueAVFrame CudaDeviceInterface::maybe_convert_av_frame_to_nv12_or_rgb24(
 
   // If the frame is already in NV12 format, we don't need to do anything.
   if (actual_format == AV_PIX_FMT_NV12) {
-    return std::move(av_frame);
+    return UniqueAVFrame{};
   }
 
   AVPixelFormat output_format;
@@ -198,19 +198,19 @@ UniqueAVFrame CudaDeviceInterface::maybe_convert_av_frame_to_nv12_or_rgb24(
   }
 
   enum AVPixelFormat frame_format =
-      static_cast<enum AVPixelFormat>(av_frame->format);
+      static_cast<enum AVPixelFormat>(av_frame.format);
 
   auto new_config = std::make_unique<FiltersConfig>(
-      av_frame->width,
-      av_frame->height,
+      av_frame.width,
+      av_frame.height,
       frame_format,
-      av_frame->sample_aspect_ratio,
-      av_frame->width,
-      av_frame->height,
+      av_frame.sample_aspect_ratio,
+      av_frame.width,
+      av_frame.height,
       output_format,
       filters.str(),
       time_base_,
-      av_buffer_ref(av_frame->hw_frames_ctx));
+      av_buffer_ref(av_frame.hw_frames_ctx));
 
   if (!nv12_conversion_ || *nv12_conversion_config_ != *new_config) {
     nv12_conversion_ =
@@ -237,20 +237,23 @@ UniqueAVFrame CudaDeviceInterface::maybe_convert_av_frame_to_nv12_or_rgb24(
 }
 
 void CudaDeviceInterface::convert_av_frame_to_frame_output(
-    UniqueAVFrame& av_frame,
+    const AVFrame& input_av_frame,
     FrameOutput& frame_output,
     std::optional<torch::stable::Tensor> pre_allocated_output_tensor) {
   validate_pre_allocated_tensor_shape(
       pre_allocated_output_tensor,
-      FrameDims(av_frame->height, av_frame->width));
+      FrameDims(input_av_frame.height, input_av_frame.width));
 
   has_decoded_frame_ = true;
 
   // All of our CUDA decoding assumes NV12 format. We handle non-NV12 formats by
   // converting them to NV12.
-  av_frame = maybe_convert_av_frame_to_nv12_or_rgb24(av_frame);
+  UniqueAVFrame converted_av_frame =
+      maybe_convert_av_frame_to_nv12_or_rgb24(input_av_frame);
+  const AVFrame& av_frame =
+      converted_av_frame ? *converted_av_frame : input_av_frame;
 
-  if (av_frame->format != AV_PIX_FMT_CUDA) {
+  if (av_frame.format != AV_PIX_FMT_CUDA) {
     // The frame's format is AV_PIX_FMT_CUDA if and only if its content is on
     // the GPU. In this branch, the frame is on the CPU. There are two possible
     // reasons:
@@ -266,7 +269,7 @@ void CudaDeviceInterface::convert_av_frame_to_frame_output(
     // CUDA device when we're done.
 
     enum AVPixelFormat frame_format =
-        static_cast<enum AVPixelFormat>(av_frame->format);
+        static_cast<enum AVPixelFormat>(av_frame.format);
 
     FrameOutput cpu_frame_output;
     if (frame_format == AV_PIX_FMT_RGB24) {
@@ -302,10 +305,10 @@ void CudaDeviceInterface::convert_av_frame_to_frame_output(
   // because this is what our color conversion kernel expects. This SHOULD
   // be enforced by our call to maybeConvertAVFrameToNV12OrRGB24() above.
   STD_TORCH_CHECK(
-      av_frame->hw_frames_ctx != nullptr,
+      av_frame.hw_frames_ctx != nullptr,
       "The AVFrame does not have a hw_frames_ctx. This should never happen");
   AVHWFramesContext* hw_frames_ctx =
-      reinterpret_cast<AVHWFramesContext*>(av_frame->hw_frames_ctx->data);
+      reinterpret_cast<AVHWFramesContext*>(av_frame.hw_frames_ctx->data);
   STD_TORCH_CHECK(
       hw_frames_ctx != nullptr,
       "The AVFrame does not have a valid hw_frames_ctx. This should never happen");
@@ -338,7 +341,7 @@ void CudaDeviceInterface::convert_av_frame_to_frame_output(
       device_,
       nvdec_stream,
       pre_allocated_output_tensor,
-      FrameDims(av_frame->height, av_frame->width),
+      FrameDims(av_frame.height, av_frame.width),
       /*isP016=*/false,
       /*bitDepth=*/8,
       cached_color_matrix_);
diff --git a/src/torchcodec/_core/CudaDeviceInterface.h b/src/torchcodec/_core/CudaDeviceInterface.h
index c77870230..d449ccb97 100644
--- a/src/torchcodec/_core/CudaDeviceInterface.h
+++ b/src/torchcodec/_core/CudaDeviceInterface.h
@@ -41,7 +41,7 @@ class CudaDeviceInterface : public DeviceInterface {
       AVCodecContext* codec_context) override;
 
   void convert_av_frame_to_frame_output(
-      UniqueAVFrame& av_frame,
+      const AVFrame& av_frame,
       FrameOutput& frame_output,
       std::optional<torch::stable::Tensor> pre_allocated_output_tensor)
       override;
@@ -63,9 +63,10 @@ class CudaDeviceInterface : public DeviceInterface {
  private:
   // Our CUDA decoding code assumes NV12 format. In order to handle other
   // kinds of input, we need to convert them to NV12. Our current implementation
-  // does this using filtergraph.
+  // does this using filtergraph. Returns a null frame when no conversion is
+  // needed, i.e. when the input frame can be used as-is.
   UniqueAVFrame maybe_convert_av_frame_to_nv12_or_rgb24(
-      UniqueAVFrame& av_frame);
+      const AVFrame& av_frame);
 
   // We sometimes encounter frames that cannot be decoded on the CUDA device.
   // Rather than erroring out, we decode them on the CPU.
diff --git a/src/torchcodec/_core/Demuxer.cpp b/src/torchcodec/_core/Demuxer.cpp
index 4f4ff4c6e..ba46d6450 100644
--- a/src/torchcodec/_core/Demuxer.cpp
+++ b/src/torchcodec/_core/Demuxer.cpp
@@ -68,12 +68,12 @@ Demuxer::Demuxer(
   }
 }
 
-AVPacket* Demuxer::next_packet() {
+UniqueAVPacket Demuxer::next_packet() {
   ReferenceAVPacket packet(auto_packet_);
   int status =
       read_next_packet(format_context_.get(), active_stream_index_, packet);
   if (status == AVERROR_EOF) {
-    return nullptr;
+    return UniqueAVPacket{};
   }
   STD_TORCH_CHECK(
       status >= AVSUCCESS,
@@ -82,9 +82,9 @@ AVPacket* Demuxer::next_packet() {
 
   // Move the reference out into a fresh, independent packet the caller owns.
   // This is what makes the packet safe to hand to another thread.
-  AVPacket* owned = av_packet_alloc();
+  UniqueAVPacket owned(av_packet_alloc());
   STD_TORCH_CHECK(owned != nullptr, "Failed to allocate AVPacket");
-  av_packet_move_ref(owned, packet.get());
+  av_packet_move_ref(owned.get(), packet.get());
   return owned;
 }
 
diff --git a/src/torchcodec/_core/Demuxer.h b/src/torchcodec/_core/Demuxer.h
index e0cfbed3e..204d08673 100644
--- a/src/torchcodec/_core/Demuxer.h
+++ b/src/torchcodec/_core/Demuxer.h
@@ -31,10 +31,9 @@ class FORCE_PUBLIC_VISIBILITY Demuxer {
       const std::string& file_path,
       std::optional<int> stream_index = std::nullopt);
 
-  // Returns the next packet for the active stream as a freshly-allocated,
-  // owning AVPacket (the caller takes ownership and must av_packet_free it), or
-  // nullptr at end of stream.
-  AVPacket* next_packet();
+  // Returns the next packet for the active stream as a freshly-allocated
+  // packet, or a null packet at end of stream.
+  UniqueAVPacket next_packet();
 
   AVStream* active_stream() const {
     return stream_;
diff --git a/src/torchcodec/_core/DeviceInterface.cpp b/src/torchcodec/_core/DeviceInterface.cpp
index 4ad7e8fe1..484b86728 100644
--- a/src/torchcodec/_core/DeviceInterface.cpp
+++ b/src/torchcodec/_core/DeviceInterface.cpp
@@ -116,16 +116,16 @@ std::unique_ptr<DeviceInterface> create_device_interface(
       "'");
 }
 
-torch::stable::Tensor rgb_av_frame_to_tensor(const UniqueAVFrame& av_frame) {
-  auto format = static_cast<AVPixelFormat>(av_frame->format);
+torch::stable::Tensor rgb_av_frame_to_tensor(const AVFrame& av_frame) {
+  auto format = static_cast<AVPixelFormat>(av_frame.format);
   STD_TORCH_CHECK(
       format == AV_PIX_FMT_RGB24 || format == AV_PIX_FMT_RGB48,
       "Expected RGB24 or RGB48 format, got ",
       (av_get_pix_fmt_name(format) ? av_get_pix_fmt_name(format) : "unknown"));
 
-  int height = av_frame->height;
-  int width = av_frame->width;
-  AVFrame* cloned_av_frame = av_frame_clone(av_frame.get());
+  int height = av_frame.height;
+  int width = av_frame.width;
+  AVFrame* cloned_av_frame = av_frame_clone(&av_frame);
 
   auto deleter = [cloned_av_frame](void*) {
     UniqueAVFrame av_frame_to_delete(cloned_av_frame);
diff --git a/src/torchcodec/_core/DeviceInterface.h b/src/torchcodec/_core/DeviceInterface.h
index e90915e20..ca976fd36 100644
--- a/src/torchcodec/_core/DeviceInterface.h
+++ b/src/torchcodec/_core/DeviceInterface.h
@@ -99,7 +99,7 @@ class DeviceInterface {
   }
 
   virtual void convert_av_frame_to_frame_output(
-      UniqueAVFrame& av_frame,
+      const AVFrame& av_frame,
       FrameOutput& frame_output,
       std::optional<torch::stable::Tensor> pre_allocated_output_tensor =
           std::nullopt) = 0;
@@ -207,6 +207,6 @@ create_device_interface(
     const StableDevice& device,
     const std::string_view variant = "default");
 
-torch::stable::Tensor rgb_av_frame_to_tensor(const UniqueAVFrame& av_frame);
+torch::stable::Tensor rgb_av_frame_to_tensor(const AVFrame& av_frame);
 
 } // namespace facebook::torchcodec
diff --git a/src/torchcodec/_core/Encoder.cpp b/src/torchcodec/_core/Encoder.cpp
index e3665574c..11808e091 100644
--- a/src/torchcodec/_core/Encoder.cpp
+++ b/src/torchcodec/_core/Encoder.cpp
@@ -794,7 +794,7 @@ UniqueAVFrame MultiStreamEncoder::maybe_convert_audio_av_frame(
     AudioStream& audio_stream) {
   if (static_cast<AVSampleFormat>(av_frame->format) ==
           audio_stream.av_codec_context->sample_fmt &&
-      get_num_channels(av_frame) == audio_stream.out_num_channels &&
+      get_num_channels(*av_frame) == audio_stream.out_num_channels &&
       av_frame->sample_rate == audio_stream.out_sample_rate) {
     // Note: the clone references the same underlying data, it's a cheap copy.
     return UniqueAVFrame(av_frame_clone(av_frame.get()));
@@ -806,7 +806,7 @@ UniqueAVFrame MultiStreamEncoder::maybe_convert_audio_av_frame(
         audio_stream.av_codec_context->sample_fmt,
         av_frame->sample_rate,
         audio_stream.out_sample_rate,
-        av_frame,
+        *av_frame,
         audio_stream.out_num_channels));
   }
   // convertAudioAVFrameSamples uses avFrame's extended_data field, so we ensure
@@ -817,7 +817,7 @@ UniqueAVFrame MultiStreamEncoder::maybe_convert_audio_av_frame(
       "Codec context data and extended_data pointers differ, this is unexpected.");
   UniqueAVFrame converted_av_frame = convert_audio_av_frame_samples(
       audio_stream.swr_context,
-      av_frame,
+      *av_frame,
       audio_stream.av_codec_context->sample_fmt,
       audio_stream.out_sample_rate,
       audio_stream.out_num_channels);
diff --git a/src/torchcodec/_core/FFMPEGCommon.cpp b/src/torchcodec/_core/FFMPEGCommon.cpp
index b7b6c1550..3726131eb 100644
--- a/src/torchcodec/_core/FFMPEGCommon.cpp
+++ b/src/torchcodec/_core/FFMPEGCommon.cpp
@@ -54,11 +54,11 @@ std::string get_ffmpeg_error_string_from_error_code(int error_code) {
   return std::string(error_buffer);
 }
 
-int64_t get_duration(const UniqueAVFrame& av_frame) {
-#if LIBAVUTIL_VERSION_MAJOR < 58
-  return av_frame->pkt_duration;
+int64_t get_duration(const AVFrame& av_frame) {
+#if FFMPEG_HAS_FRAME_DURATION
+  return av_frame.duration;
 #else
-  return av_frame->duration;
+  return av_frame.pkt_duration;
 #endif
 }
 
@@ -71,21 +71,21 @@ int64_t get_pts_or_dts(ReferenceAVPacket& packet) {
   return packet->pts == INT64_MIN ? packet->dts : packet->pts;
 }
 
-int64_t get_pts_or_dts(const UniqueAVFrame& av_frame) {
-  return av_frame->pts == INT64_MIN ? av_frame->pkt_dts : av_frame->pts;
+int64_t get_pts_or_dts(const AVFrame& av_frame) {
+  return av_frame.pts == INT64_MIN ? av_frame.pkt_dts : av_frame.pts;
 }
 
-void set_duration(const UniqueAVFrame& av_frame, int64_t duration) {
-#if LIBAVUTIL_VERSION_MAJOR < 58
-  av_frame->pkt_duration = duration;
+void set_duration(AVFrame& av_frame, int64_t duration) {
+#if FFMPEG_HAS_FRAME_DURATION
+  av_frame.duration = duration;
 #else
-  av_frame->duration = duration;
+  av_frame.pkt_duration = duration;
 #endif
 }
 
 const int* get_supported_sample_rates(const AVCodec& av_codec) {
   const int* supported_sample_rates = nullptr;
-#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(61, 13, 100) // FFmpeg >= 7.1
+#if FFMPEG_HAS_SUPPORTED_CONFIG
   int num_sample_rates = 0;
   int ret = avcodec_get_supported_config(
       nullptr,
@@ -106,7 +106,7 @@ const int* get_supported_sample_rates(const AVCodec& av_codec) {
 
 const AVPixelFormat* get_supported_pixel_formats(const AVCodec& av_codec) {
   const AVPixelFormat* supported_pixel_formats = nullptr;
-#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(61, 13, 100) // FFmpeg >= 7.1
+#if FFMPEG_HAS_SUPPORTED_CONFIG
   int num_pixel_formats = 0;
   int ret = avcodec_get_supported_config(
       nullptr,
@@ -128,7 +128,7 @@ const AVPixelFormat* get_supported_pixel_formats(const AVCodec& av_codec) {
 const AVSampleFormat* get_supported_output_sample_formats(
     const AVCodec& av_codec) {
   const AVSampleFormat* supported_sample_formats = nullptr;
-#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(61, 13, 100) // FFmpeg >= 7.1
+#if FFMPEG_HAS_SUPPORTED_CONFIG
   int num_sample_formats = 0;
   int ret = avcodec_get_supported_config(
       nullptr,
@@ -148,28 +148,32 @@ const AVSampleFormat* get_supported_output_sample_formats(
   return supported_sample_formats;
 }
 
-int get_num_channels(const UniqueAVFrame& av_frame) {
-#if LIBAVFILTER_VERSION_MAJOR > 8 || \
-    (LIBAVFILTER_VERSION_MAJOR == 8 && LIBAVFILTER_VERSION_MINOR >= 44)
-  return av_frame->ch_layout.nb_channels;
+#if !FFMPEG_HAS_CH_LAYOUT
+// FFmpeg 4 leaves channel_layout unset (0) on some decoded frames even though
+// .channels is correct. Everything we feed to swresample needs a real layout,
+// so fall back to the default layout for that channel count.
+int64_t get_channel_layout(const AVFrame& av_frame) {
+  if (av_frame.channel_layout != 0 || av_frame.channels <= 0) {
+    return static_cast<int64_t>(av_frame.channel_layout);
+  }
+  return av_get_default_channel_layout(av_frame.channels);
+}
+#endif
+
+int get_num_channels(const AVFrame& av_frame) {
+#if FFMPEG_HAS_CH_LAYOUT
+  return av_frame.ch_layout.nb_channels;
 #else
-  int num_channels =
-      av_get_channel_layout_nb_channels(av_frame->channel_layout);
-  // Handle FFmpeg 4 bug where channel_layout and num_channels are 0 or unset
-  // Set values based on av_frame->channels which appears to be correct
-  // to allow successful initialization of SwrContext
-  if (num_channels == 0 && av_frame->channels > 0) {
-    av_frame->channel_layout =
-        av_get_default_channel_layout(av_frame->channels);
-    num_channels = av_frame->channels;
+  int num_channels = av_get_channel_layout_nb_channels(av_frame.channel_layout);
+  if (num_channels == 0 && av_frame.channels > 0) {
+    num_channels = av_frame.channels;
   }
   return num_channels;
 #endif
 }
 
 int get_num_channels(const SharedAVCodecContext& av_codec_context) {
-#if LIBAVFILTER_VERSION_MAJOR > 8 || \
-    (LIBAVFILTER_VERSION_MAJOR == 8 && LIBAVFILTER_VERSION_MINOR >= 44)
+#if FFMPEG_HAS_CH_LAYOUT
   return av_codec_context->ch_layout.nb_channels;
 #else
   return av_codec_context->channels;
@@ -178,8 +182,7 @@ int get_num_channels(const SharedAVCodecContext& av_codec_context) {
 
 int get_num_channels(const AVCodecParameters* codecpar) {
   STD_TORCH_CHECK(codecpar != nullptr, "codecpar is null");
-#if LIBAVFILTER_VERSION_MAJOR > 8 || \
-    (LIBAVFILTER_VERSION_MAJOR == 8 && LIBAVFILTER_VERSION_MINOR >= 44)
+#if FFMPEG_HAS_CH_LAYOUT
   return codecpar->ch_layout.nb_channels;
 #else
   return codecpar->channels;
@@ -189,7 +192,7 @@ int get_num_channels(const AVCodecParameters* codecpar) {
 void set_default_channel_layout(
     UniqueAVCodecContext& av_codec_context,
     int num_channels) {
-#if LIBAVFILTER_VERSION_MAJOR > 7 // FFmpeg > 4
+#if FFMPEG_HAS_CH_LAYOUT
   AVChannelLayout channel_layout;
   av_channel_layout_default(&channel_layout, num_channels);
   av_codec_context->ch_layout = channel_layout;
@@ -200,20 +203,20 @@ void set_default_channel_layout(
 #endif
 }
 
-void set_default_channel_layout(UniqueAVFrame& av_frame, int num_channels) {
-#if LIBAVFILTER_VERSION_MAJOR > 7 // FFmpeg > 4
+void set_default_channel_layout(AVFrame& av_frame, int num_channels) {
+#if FFMPEG_HAS_CH_LAYOUT
   AVChannelLayout channel_layout;
   av_channel_layout_default(&channel_layout, num_channels);
-  av_frame->ch_layout = channel_layout;
+  av_frame.ch_layout = channel_layout;
 #else
   uint64_t channel_layout = av_get_default_channel_layout(num_channels);
-  av_frame->channel_layout = channel_layout;
-  av_frame->channels = num_channels;
+  av_frame.channel_layout = channel_layout;
+  av_frame.channels = num_channels;
 #endif
 }
 
 void validate_num_channels(const AVCodec& av_codec, int num_channels) {
-#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(61, 13, 100) // FFmpeg >= 7.1
+#if FFMPEG_HAS_SUPPORTED_CONFIG
   std::stringstream supported_num_channels;
   const AVChannelLayout* supported_layouts = nullptr;
   int num_layouts = 0;
@@ -238,7 +241,7 @@ void validate_num_channels(const AVCodec& av_codec, int num_channels) {
       return;
     }
   }
-#elif LIBAVFILTER_VERSION_MAJOR > 7 // FFmpeg > 4
+#elif FFMPEG_HAS_CH_LAYOUT
   if (av_codec.ch_layouts == nullptr) {
     // If we can't validate, we must assume it'll be fine. If not, FFmpeg will
     // eventually raise.
@@ -293,17 +296,17 @@ void validate_num_channels(const AVCodec& av_codec, int num_channels) {
 }
 
 namespace {
-#if LIBAVFILTER_VERSION_MAJOR > 7 // FFmpeg > 4
+#if FFMPEG_HAS_CH_LAYOUT
 
 // Returns:
 // - the src_av_frame's channel layout if src_av_frame has out_num_channels
 // - the default channel layout with out_num_channels otherwise.
 AVChannelLayout get_output_channel_layout(
     int out_num_channels,
-    const UniqueAVFrame& src_av_frame) {
+    const AVFrame& src_av_frame) {
   AVChannelLayout out_layout;
   if (out_num_channels == get_num_channels(src_av_frame)) {
-    out_layout = src_av_frame->ch_layout;
+    out_layout = src_av_frame.ch_layout;
   } else {
     av_channel_layout_default(&out_layout, out_num_channels);
   }
@@ -315,10 +318,10 @@ AVChannelLayout get_output_channel_layout(
 // Same as above
 int64_t get_output_channel_layout(
     int out_num_channels,
-    const UniqueAVFrame& src_av_frame) {
+    const AVFrame& src_av_frame) {
   int64_t out_layout;
   if (out_num_channels == get_num_channels(src_av_frame)) {
-    out_layout = src_av_frame->channel_layout;
+    out_layout = get_channel_layout(src_av_frame);
   } else {
     out_layout = av_get_default_channel_layout(out_num_channels);
   }
@@ -330,21 +333,21 @@ int64_t get_output_channel_layout(
 // Sets dst_av_frame' channel layout to get_output_channel_layout(): see doc
 // above
 void set_channel_layout(
-    UniqueAVFrame& dst_av_frame,
-    const UniqueAVFrame& src_av_frame,
+    AVFrame& dst_av_frame,
+    const AVFrame& src_av_frame,
     int out_num_channels) {
-#if LIBAVFILTER_VERSION_MAJOR > 7 // FFmpeg > 4
+#if FFMPEG_HAS_CH_LAYOUT
   AVChannelLayout out_layout =
       get_output_channel_layout(out_num_channels, src_av_frame);
-  auto status = av_channel_layout_copy(&dst_av_frame->ch_layout, &out_layout);
+  auto status = av_channel_layout_copy(&dst_av_frame.ch_layout, &out_layout);
   STD_TORCH_CHECK(
       status == AVSUCCESS,
       "Couldn't copy channel layout to av_frame: ",
       get_ffmpeg_error_string_from_error_code(status));
 #else
-  dst_av_frame->channel_layout =
+  dst_av_frame.channel_layout =
       get_output_channel_layout(out_num_channels, src_av_frame);
-  dst_av_frame->channels = out_num_channels;
+  dst_av_frame.channels = out_num_channels;
 #endif
 }
 
@@ -358,7 +361,7 @@ UniqueAVFrame allocate_av_frame(
 
   av_frame->nb_samples = num_samples;
   av_frame->sample_rate = sample_rate;
-  set_default_channel_layout(av_frame, num_channels);
+  set_default_channel_layout(*av_frame, num_channels);
   av_frame->format = sample_format;
   auto status = av_frame_get_buffer(av_frame.get(), 0);
 
@@ -380,11 +383,11 @@ SwrContext* create_swr_context(
     AVSampleFormat out_sample_format,
     int src_sample_rate,
     int out_sample_rate,
-    const UniqueAVFrame& src_av_frame,
+    const AVFrame& src_av_frame,
     int out_num_channels) {
   SwrContext* swr_context = nullptr;
   int status = AVSUCCESS;
-#if LIBAVFILTER_VERSION_MAJOR > 7 // FFmpeg > 4
+#if FFMPEG_HAS_CH_LAYOUT
   AVChannelLayout out_layout =
       get_output_channel_layout(out_num_channels, src_av_frame);
   status = swr_alloc_set_opts2(
@@ -392,7 +395,10 @@ SwrContext* create_swr_context(
       &out_layout,
       out_sample_format,
       out_sample_rate,
-      &src_av_frame->ch_layout,
+      // swr_alloc_set_opts2() only became const-correct in FFmpeg 6
+      // (libswresample 4.12): before that it asks for a non-const layout that
+      // it doesn't modify.
+      const_cast<AVChannelLayout*>(&src_av_frame.ch_layout),
       src_sample_format,
       src_sample_rate,
       0,
@@ -410,7 +416,7 @@ SwrContext* create_swr_context(
       out_layout,
       out_sample_format,
       out_sample_rate,
-      src_av_frame->channel_layout,
+      get_channel_layout(src_av_frame),
       src_sample_format,
       src_sample_rate,
       0,
@@ -493,7 +499,7 @@ AVFilterContext* create_av_filter_context_with_options(
 
 UniqueAVFrame convert_audio_av_frame_samples(
     const UniqueSwrContext& swr_context,
-    const UniqueAVFrame& src_av_frame,
+    const AVFrame& src_av_frame,
     AVSampleFormat out_sample_format,
     int out_sample_rate,
     int out_num_channels) {
@@ -502,11 +508,11 @@ UniqueAVFrame convert_audio_av_frame_samples(
       converted_av_frame,
       "Could not allocate frame for sample format conversion.");
 
-  converted_av_frame->pts = src_av_frame->pts;
+  converted_av_frame->pts = src_av_frame.pts;
   converted_av_frame->format = static_cast<int>(out_sample_format);
 
   converted_av_frame->sample_rate = out_sample_rate;
-  int src_sample_rate = src_av_frame->sample_rate;
+  int src_sample_rate = src_av_frame.sample_rate;
   if (src_sample_rate != out_sample_rate) {
     // Note that this is an upper bound on the number of output samples.
     // `swr_convert()` will likely not fill convertedAVFrame with that many
@@ -518,15 +524,15 @@ UniqueAVFrame convert_audio_av_frame_samples(
     // tighter bound.
     converted_av_frame->nb_samples = av_rescale_rnd(
         swr_get_delay(swr_context.get(), src_sample_rate) +
-            src_av_frame->nb_samples,
+            src_av_frame.nb_samples,
         out_sample_rate,
         src_sample_rate,
         AV_ROUND_UP);
   } else {
-    converted_av_frame->nb_samples = src_av_frame->nb_samples;
+    converted_av_frame->nb_samples = src_av_frame.nb_samples;
   }
 
-  set_channel_layout(converted_av_frame, src_av_frame, out_num_channels);
+  set_channel_layout(*converted_av_frame, src_av_frame, out_num_channels);
 
   auto status = av_frame_get_buffer(converted_av_frame.get(), 0);
   STD_TORCH_CHECK(
@@ -543,8 +549,8 @@ UniqueAVFrame convert_audio_av_frame_samples(
       converted_av_frame->extended_data,
       converted_av_frame->nb_samples,
       static_cast<const uint8_t**>(
-          const_cast<const uint8_t**>(src_av_frame->extended_data)),
-      src_av_frame->nb_samples);
+          const_cast<const uint8_t**>(src_av_frame.extended_data)),
+      src_av_frame.nb_samples);
   // numConvertedSamples can be 0 if we're downsampling by a great factor and
   // the first frame doesn't contain a lot of samples. It should be handled
   // properly by the caller.
diff --git a/src/torchcodec/_core/FFMPEGCommon.h b/src/torchcodec/_core/FFMPEGCommon.h
index 88bb04ceb..86381bb9f 100644
--- a/src/torchcodec/_core/FFMPEGCommon.h
+++ b/src/torchcodec/_core/FFMPEGCommon.h
@@ -31,6 +31,33 @@ extern "C" {
 #include <libswscale/swscale.h>
 }
 
+// FFmpeg 5.1 replaced the .channels + .channel_layout pair on AVFrame and
+// AVCodecContext with a single AVChannelLayout .ch_layout, and added the
+// av_channel_layout_* / swr_alloc_set_opts2() APIs that go with it.
+// libavutil 57.24 is the real marker, but libavfilter 8.44 is the equivalent
+// and is what this codebase has always tested against.
+#if LIBAVFILTER_VERSION_MAJOR > 8 || \
+    (LIBAVFILTER_VERSION_MAJOR == 8 && LIBAVFILTER_VERSION_MINOR >= 44)
+#define FFMPEG_HAS_CH_LAYOUT 1
+#else
+#define FFMPEG_HAS_CH_LAYOUT 0
+#endif
+
+// FFmpeg 7.1 added avcodec_get_supported_config(), replacing the codec's
+// pix_fmts / sample_fmts / supported_samplerates / ch_layouts arrays.
+#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(61, 13, 100)
+#define FFMPEG_HAS_SUPPORTED_CONFIG 1
+#else
+#define FFMPEG_HAS_SUPPORTED_CONFIG 0
+#endif
+
+// FFmpeg 6 renamed AVFrame.pkt_duration to AVFrame.duration.
+#if LIBAVUTIL_VERSION_MAJOR < 58
+#define FFMPEG_HAS_FRAME_DURATION 0
+#else
+#define FFMPEG_HAS_FRAME_DURATION 1
+#endif
+
 namespace facebook::torchcodec {
 
 // FFMPEG uses special delete functions for some structures. These template
@@ -83,6 +110,8 @@ inline SharedAVCodecContext make_shared_av_codec_context(AVCodecContext* ctx) {
 
 using UniqueAVFrame =
     std::unique_ptr<AVFrame, Deleterp<AVFrame, void, av_frame_free>>;
+using UniqueAVPacket =
+    std::unique_ptr<AVPacket, Deleterp<AVPacket, void, av_packet_free>>;
 using UniqueAVFilterGraph = std::unique_ptr<
     AVFilterGraph,
     Deleterp<AVFilterGraph, void, avfilter_graph_free>>;
@@ -206,20 +235,20 @@ std::string get_ffmpeg_error_string_from_error_code(int error_code);
 // Returns duration from the frame. Abstracted into a function because the
 // struct member representing duration has changed across the versions we
 // support.
-int64_t get_duration(const UniqueAVFrame& frame);
-void set_duration(const UniqueAVFrame& frame, int64_t duration);
+int64_t get_duration(const AVFrame& frame);
+void set_duration(AVFrame& frame, int64_t duration);
 
 // pts accessors that fall back to dts when pts is unset (INT64_MIN). See the
 // definitions for details.
 int64_t get_pts_or_dts(ReferenceAVPacket& packet);
-int64_t get_pts_or_dts(const UniqueAVFrame& av_frame);
+int64_t get_pts_or_dts(const AVFrame& av_frame);
 
 const int* get_supported_sample_rates(const AVCodec& av_codec);
 const AVSampleFormat* get_supported_output_sample_formats(
     const AVCodec& av_codec);
 const AVPixelFormat* get_supported_pixel_formats(const AVCodec& av_codec);
 
-int get_num_channels(const UniqueAVFrame& av_frame);
+int get_num_channels(const AVFrame& av_frame);
 int get_num_channels(const SharedAVCodecContext& av_codec_context);
 int get_num_channels(const AVCodecParameters* codecpar);
 
@@ -227,13 +256,13 @@ void set_default_channel_layout(
     UniqueAVCodecContext& av_codec_context,
     int num_channels);
 
-void set_default_channel_layout(UniqueAVFrame& av_frame, int num_channels);
+void set_default_channel_layout(AVFrame& av_frame, int num_channels);
 
 void validate_num_channels(const AVCodec& av_codec, int num_channels);
 
 void set_channel_layout(
-    UniqueAVFrame& dst_av_frame,
-    const UniqueAVFrame& src_av_frame,
+    AVFrame& dst_av_frame,
+    const AVFrame& src_av_frame,
     int desired_num_channels);
 
 UniqueAVFrame allocate_av_frame(
@@ -247,7 +276,7 @@ SwrContext* create_swr_context(
     AVSampleFormat desired_sample_format,
     int src_sample_rate,
     int desired_sample_rate,
-    const UniqueAVFrame& src_av_frame,
+    const AVFrame& src_av_frame,
     int desired_num_channels);
 
 // Converts, if needed:
@@ -257,7 +286,7 @@ SwrContext* create_swr_context(
 // createSwrContext must have been previously called with matching parameters.
 UniqueAVFrame convert_audio_av_frame_samples(
     const UniqueSwrContext& swr_context,
-    const UniqueAVFrame& src_av_frame,
+    const AVFrame& src_av_frame,
     AVSampleFormat desired_sample_format,
     int desired_sample_rate,
     int desired_num_channels);
diff --git a/src/torchcodec/_core/FilterGraph.cpp b/src/torchcodec/_core/FilterGraph.cpp
index 36c3fa4ae..f3b8d05be 100644
--- a/src/torchcodec/_core/FilterGraph.cpp
+++ b/src/torchcodec/_core/FilterGraph.cpp
@@ -150,8 +150,8 @@ FilterGraph::FilterGraph(
       ", provided filters: " + filters_config.filtergraph_str);
 }
 
-UniqueAVFrame FilterGraph::convert(const UniqueAVFrame& av_frame) {
-  int status = av_buffersrc_write_frame(source_context_, av_frame.get());
+UniqueAVFrame FilterGraph::convert(const AVFrame& av_frame) {
+  int status = av_buffersrc_write_frame(source_context_, &av_frame);
   STD_TORCH_CHECK(
       status >= AVSUCCESS, "Failed to add frame to buffer source context");
 
diff --git a/src/torchcodec/_core/FilterGraph.h b/src/torchcodec/_core/FilterGraph.h
index 650ea584d..356122e27 100644
--- a/src/torchcodec/_core/FilterGraph.h
+++ b/src/torchcodec/_core/FilterGraph.h
@@ -48,7 +48,7 @@ class FilterGraph {
       const FiltersConfig& filters_config,
       const VideoStreamOptions& video_stream_options);
 
-  UniqueAVFrame convert(const UniqueAVFrame& av_frame);
+  UniqueAVFrame convert(const AVFrame& av_frame);
 
  private:
   UniqueAVFilterGraph filter_graph_;
diff --git a/src/torchcodec/_core/SingleStreamDecoder.cpp b/src/torchcodec/_core/SingleStreamDecoder.cpp
index 28eac5e75..f6a770dbd 100644
--- a/src/torchcodec/_core/SingleStreamDecoder.cpp
+++ b/src/torchcodec/_core/SingleStreamDecoder.cpp
@@ -677,12 +677,11 @@ FrameOutput SingleStreamDecoder::get_next_frame() {
 FrameOutput SingleStreamDecoder::get_next_frame_internal(
     std::optional<torch::stable::Tensor> pre_allocated_output_tensor) {
   validate_active_stream();
-  UniqueAVFrame av_frame =
-      decode_av_frame([this](const UniqueAVFrame& av_frame) {
-        return get_pts_or_dts(av_frame) >= cursor_;
-      });
+  UniqueAVFrame av_frame = decode_av_frame([this](const AVFrame& av_frame) {
+    return get_pts_or_dts(av_frame) >= cursor_;
+  });
   return convert_av_frame_to_frame_output(
-      av_frame, pre_allocated_output_tensor);
+      *av_frame, pre_allocated_output_tensor);
 }
 
 FrameOutput SingleStreamDecoder::get_frame_at_index(int64_t frame_index) {
@@ -867,7 +866,7 @@ FrameOutput SingleStreamDecoder::get_frame_played_at(double seconds) {
 
   set_cursor_pts_in_seconds(seconds);
   UniqueAVFrame av_frame =
-      decode_av_frame([seconds, this](const UniqueAVFrame& av_frame) {
+      decode_av_frame([seconds, this](const AVFrame& av_frame) {
         StreamInfo& stream_info = stream_infos_[active_stream_index_];
         double frame_start_time =
             pts_to_seconds(get_pts_or_dts(av_frame), stream_info.time_base);
@@ -888,7 +887,7 @@ FrameOutput SingleStreamDecoder::get_frame_played_at(double seconds) {
       });
 
   // Convert the frame to tensor.
-  FrameOutput frame_output = convert_av_frame_to_frame_output(av_frame);
+  FrameOutput frame_output = convert_av_frame_to_frame_output(*av_frame);
   frame_output.data = maybe_permute_and_convert_to_float32(frame_output.data);
   return frame_output;
 }
@@ -1241,12 +1240,12 @@ AudioFramesOutput SingleStreamDecoder::get_frames_played_in_range_audio(
   while (!finished) {
     try {
       UniqueAVFrame av_frame =
-          decode_av_frame([start_pts, stop_pts](const UniqueAVFrame& av_frame) {
+          decode_av_frame([start_pts, stop_pts](const AVFrame& av_frame) {
             return start_pts <
                 get_pts_or_dts(av_frame) + get_duration(av_frame) &&
                 stop_pts > get_pts_or_dts(av_frame);
           });
-      auto frame_output = convert_av_frame_to_frame_output(av_frame);
+      auto frame_output = convert_av_frame_to_frame_output(*av_frame);
       if (!first_frame_pts_seconds.has_value()) {
         first_frame_pts_seconds = frame_output.pts_seconds;
       }
@@ -1458,7 +1457,7 @@ void SingleStreamDecoder::maybe_seek_to_before_desired_pts() {
 // --------------------------------------------------------------------------
 
 UniqueAVFrame SingleStreamDecoder::decode_av_frame(
-    std::function<bool(const UniqueAVFrame&)> filter_function) {
+    std::function<bool(const AVFrame&)> filter_function) {
   validate_active_stream();
 
   reset_decode_stats();
@@ -1484,7 +1483,7 @@ UniqueAVFrame SingleStreamDecoder::decode_av_frame(
 
     decode_stats_.num_frames_received_by_decoder++;
     // Is this the kind of frame we're looking for?
-    if (status == AVSUCCESS && filter_function(av_frame)) {
+    if (status == AVSUCCESS && filter_function(*av_frame)) {
       // Yes, this is the frame we'll return; break out of the decoding loop.
       break;
     } else if (status == AVSUCCESS) {
@@ -1562,8 +1561,8 @@ UniqueAVFrame SingleStreamDecoder::decode_av_frame(
   // received as frames. Eventually we will either hit AVERROR_EOF from
   // av_receive_frame() or the user will have seeked to a different location
   // in the file and that will flush the decoder.
-  last_decoded_av_frame_pts_ = get_pts_or_dts(av_frame);
-  last_decoded_av_frame_duration_ = get_duration(av_frame);
+  last_decoded_av_frame_pts_ = get_pts_or_dts(*av_frame);
+  last_decoded_av_frame_duration_ = get_duration(*av_frame);
 
   return av_frame;
 }
@@ -1573,7 +1572,7 @@ UniqueAVFrame SingleStreamDecoder::decode_av_frame(
 // --------------------------------------------------------------------------
 
 FrameOutput SingleStreamDecoder::convert_av_frame_to_frame_output(
-    UniqueAVFrame& av_frame,
+    const AVFrame& av_frame,
     std::optional<torch::stable::Tensor> pre_allocated_output_tensor) {
   // Convert the frame to tensor.
   FrameOutput frame_output;
diff --git a/src/torchcodec/_core/SingleStreamDecoder.h b/src/torchcodec/_core/SingleStreamDecoder.h
index e14a31661..c1ef2a1bd 100644
--- a/src/torchcodec/_core/SingleStreamDecoder.h
+++ b/src/torchcodec/_core/SingleStreamDecoder.h
@@ -270,7 +270,7 @@ class FORCE_PUBLIC_VISIBILITY SingleStreamDecoder {
   void maybe_seek_to_before_desired_pts();
 
   UniqueAVFrame decode_av_frame(
-      std::function<bool(const UniqueAVFrame&)> filter_function);
+      std::function<bool(const AVFrame&)> filter_function);
 
   FrameOutput get_next_frame_internal(
       std::optional<torch::stable::Tensor> pre_allocated_output_tensor =
@@ -282,7 +282,7 @@ class FORCE_PUBLIC_VISIBILITY SingleStreamDecoder {
       torch::stable::Tensor& tensor);
 
   FrameOutput convert_av_frame_to_frame_output(
-      UniqueAVFrame& av_frame,
+      const AVFrame& av_frame,
       std::optional<torch::stable::Tensor> pre_allocated_output_tensor =
           std::nullopt);
 
diff --git a/src/torchcodec/_core/SwScale.cpp b/src/torchcodec/_core/SwScale.cpp
index 48be27043..f0e585eba 100644
--- a/src/torchcodec/_core/SwScale.cpp
+++ b/src/torchcodec/_core/SwScale.cpp
@@ -52,7 +52,7 @@ SwScale::SwScale(const SwsConfig& config, int sws_flags)
 }
 
 int SwScale::convert(
-    const UniqueAVFrame& av_frame,
+    const AVFrame& av_frame,
     torch::stable::Tensor& output_tensor) {
   // When resizing is needed, we do sws_scale twice: first convert to output
   // RGB at original resolution, then resize in output RGB space. This ensures
@@ -85,19 +85,19 @@ int SwScale::convert(
 
   int color_converted_height = sws_scale(
       color_conversion_sws_context_.get(),
-      av_frame->data,
-      av_frame->linesize,
+      av_frame.data,
+      av_frame.linesize,
       0,
-      av_frame->height,
+      av_frame.height,
       color_converted_pointers,
       color_converted_linesizes);
 
   STD_TORCH_CHECK(
-      color_converted_height == av_frame->height,
+      color_converted_height == av_frame.height,
       "Color conversion swscale pass failed: colorConvertedHeight != avFrame->height: ",
       color_converted_height,
       " != ",
-      av_frame->height);
+      av_frame.height);
 
   if (needs_resize_) {
     uint8_t* src_pointers[4] = {
diff --git a/src/torchcodec/_core/SwScale.h b/src/torchcodec/_core/SwScale.h
index b76d743f4..3b307d579 100644
--- a/src/torchcodec/_core/SwScale.h
+++ b/src/torchcodec/_core/SwScale.h
@@ -29,9 +29,7 @@ class SwScale {
   // >8-bit.
   SwScale(const SwsConfig& config, int sws_flags = SWS_BILINEAR);
 
-  int convert(
-      const UniqueAVFrame& av_frame,
-      torch::stable::Tensor& output_tensor);
+  int convert(const AVFrame& av_frame, torch::stable::Tensor& output_tensor);
 
   const SwsConfig& get_config() const {
     return config_;
diff --git a/src/torchcodec/_core/color_conversion.cpp b/src/torchcodec/_core/color_conversion.cpp
index 22b9729af..0efa6d699 100644
--- a/src/torchcodec/_core/color_conversion.cpp
+++ b/src/torchcodec/_core/color_conversion.cpp
@@ -189,7 +189,7 @@ void compute_rgb_to_yuv_matrix(
 }
 
 torch::stable::Tensor convert_yuv_frame_to_rgb(
-    UniqueAVFrame& av_frame,
+    const AVFrame& av_frame,
     const StableDevice& device,
     cudaStream_t nvdec_stream,
     std::optional<torch::stable::Tensor> pre_allocated_output_tensor,
@@ -203,8 +203,8 @@ torch::stable::Tensor convert_yuv_frame_to_rgb(
   // Dimensions may be odd (NVDEC display area for VP9 etc.). NV12/P016
   // color conversion requires even dimensions, so we round up to even
   // for the kernel, then crop to outputDims.
-  int even_height = round_up_to_even(av_frame->height);
-  int even_width = round_up_to_even(av_frame->width);
+  int even_height = round_up_to_even(av_frame.height);
+  int even_width = round_up_to_even(av_frame.width);
 
   int out_height = output_dims.height;
   int out_width = output_dims.width;
@@ -227,33 +227,33 @@ torch::stable::Tensor convert_yuv_frame_to_rgb(
 
   maybe_update_color_matrix(
       cached_color_matrix,
-      av_frame->colorspace,
-      av_frame->color_range,
+      av_frame.colorspace,
+      av_frame.color_range,
       bit_depth,
       out_scale);
 
   if (is_p016) {
     launch_p016_to_rgb16_kernel(
-        reinterpret_cast<const uint16_t*>(av_frame->data[0]),
-        reinterpret_cast<const uint16_t*>(av_frame->data[1]),
+        reinterpret_cast<const uint16_t*>(av_frame.data[0]),
+        reinterpret_cast<const uint16_t*>(av_frame.data[1]),
         dst.mutable_data_ptr<uint16_t>(),
         even_width,
         even_height,
-        av_frame->linesize[0],
-        av_frame->linesize[1],
+        av_frame.linesize[0],
+        av_frame.linesize[1],
         validate_int64_to_int(dst.stride(0) * 2, "dst.stride(0)*2"),
         bit_depth,
         cached_color_matrix.matrix,
         stream);
   } else {
     launch_nv12_to_rgb_kernel(
-        av_frame->data[0],
-        av_frame->data[1],
+        av_frame.data[0],
+        av_frame.data[1],
         dst.mutable_data_ptr<uint8_t>(),
         even_width,
         even_height,
-        av_frame->linesize[0],
-        av_frame->linesize[1],
+        av_frame.linesize[0],
+        av_frame.linesize[1],
         validate_int64_to_int(dst.stride(0), "dst.stride(0)"),
         cached_color_matrix.matrix,
         stream);
diff --git a/src/torchcodec/_core/color_conversion.h b/src/torchcodec/_core/color_conversion.h
index 74c940b1e..ad3ae29ef 100644
--- a/src/torchcodec/_core/color_conversion.h
+++ b/src/torchcodec/_core/color_conversion.h
@@ -84,7 +84,7 @@ void launch_p016_to_rgb16_kernel(
 // outputDims: desired output size; if the frame was rounded up to even
 //   dimensions, the result is cropped back to outputDims.
 torch::stable::Tensor convert_yuv_frame_to_rgb(
-    UniqueAVFrame& av_frame,
+    const AVFrame& av_frame,
     const StableDevice& device,
     cudaStream_t nvdec_stream,
     std::optional<torch::stable::Tensor> pre_allocated_output_tensor,
diff --git a/src/torchcodec/_core/custom_ops.cpp b/src/torchcodec/_core/custom_ops.cpp
index 76634bc21..73351c99c 100644
--- a/src/torchcodec/_core/custom_ops.cpp
+++ b/src/torchcodec/_core/custom_ops.cpp
@@ -182,13 +182,17 @@ SingleStreamDecoder* unwrap_tensor_to_get_decoder(
 }
 
 // Generic pointer<->tensor laundering for the building-block handle types
-// (Demuxer / PacketDecoder / ColorConverter). Same trick as
-// wrap_decoder_pointer_to_tensor: the tensor's data pointer IS the raw pointer,
-// with a deleter that deletes the owned object when the handle is dropped.
-template <typename T>
-torch::stable::Tensor wrap_pointer_to_tensor(std::unique_ptr<T> ptr) {
+// (Demuxer / PacketDecoder / ColorConverter / AVPacket / AVFrame). Same trick
+// as wrap_decoder_pointer_to_tensor: the tensor's data pointer IS the raw
+// pointer, and the tensor is the owner. Taking the unique_ptr by value is what
+// makes the ownership transfer explicit at the call site; the unique_ptr's own
+// deleter is what frees the object, so FFmpeg types work here as long as they
+// arrive in their UniqueAVXxx alias.
+template <typename T, typename D>
+torch::stable::Tensor wrap_pointer_to_tensor(std::unique_ptr<T, D> ptr) {
+  D object_deleter = ptr.get_deleter();
   T* raw = ptr.release();
-  auto deleter = [raw](void*) { delete raw; };
+  auto deleter = [raw, object_deleter](void*) { object_deleter(raw); };
   int64_t sizes[] = {static_cast<int64_t>(sizeof(T*))};
   int64_t strides[] = {1};
   return torch::stable::from_blob(
@@ -206,56 +210,6 @@ T* unwrap_tensor_to_pointer(torch::stable::Tensor& tensor) {
   return static_cast<T*>(tensor.mutable_data_ptr());
 }
 
-// Opaque packet/frame handles: launder a raw AVPacket*/AVFrame* through a [1]
-// int64 CPU tensor whose data pointer IS the raw pointer, with a deleter that
-// frees it. Thread-movable, process-local.
-torch::stable::Tensor wrap_packet_pointer_to_tensor(AVPacket* packet) {
-  auto deleter = [packet](void*) {
-    AVPacket* p = packet;
-    av_packet_free(&p);
-  };
-  int64_t sizes[] = {1};
-  int64_t strides[] = {1};
-  return torch::stable::from_blob(
-      packet,
-      {sizes, 1},
-      {strides, 1},
-      StableDevice(kStableCPU),
-      kStableInt64,
-      deleter);
-}
-
-AVPacket* unwrap_tensor_to_packet(torch::stable::Tensor& tensor) {
-  STD_TORCH_CHECK(tensor.is_contiguous(), "packet handle must be contiguous");
-  return static_cast<AVPacket*>(tensor.mutable_data_ptr());
-}
-
-torch::stable::Tensor wrap_frame_pointer_to_tensor(AVFrame* frame) {
-  // Owning handle: the frame is freed when the handle tensor's refcount drops.
-  // ColorConverter borrows the frame during conversion (on CPU it does not free
-  // it), so the handle stays the sole owner and there is no leak even if a
-  // frame is never converted. (GPU conversion would consume the frame; GPU is
-  // not exposed through these ops yet.)
-  auto deleter = [frame](void*) {
-    AVFrame* f = frame;
-    av_frame_free(&f);
-  };
-  int64_t sizes[] = {1};
-  int64_t strides[] = {1};
-  return torch::stable::from_blob(
-      frame,
-      {sizes, 1},
-      {strides, 1},
-      StableDevice(kStableCPU),
-      kStableInt64,
-      deleter);
-}
-
-AVFrame* unwrap_tensor_to_frame(torch::stable::Tensor& tensor) {
-  STD_TORCH_CHECK(tensor.is_contiguous(), "frame handle must be contiguous");
-  return static_cast<AVFrame*>(tensor.mutable_data_ptr());
-}
-
 torch::stable::Tensor wrap_multi_stream_encoder_pointer_to_tensor(
     std::unique_ptr<MultiStreamEncoder> unique_encoder) {
   MultiStreamEncoder* encoder = unique_encoder.release();
@@ -850,11 +804,11 @@ using OpsPacketOutput = std::tuple<torch::stable::Tensor, bool>;
 
 OpsPacketOutput _blocks_demuxer_next_packet(torch::stable::Tensor& demuxer) {
   Demuxer* demuxer_ptr = unwrap_tensor_to_pointer<Demuxer>(demuxer);
-  AVPacket* packet = demuxer_ptr->next_packet();
+  UniqueAVPacket packet = demuxer_ptr->next_packet();
   if (packet == nullptr) {
     return std::make_tuple(torch::stable::full({1}, 0, kStableInt64), true);
   }
-  return std::make_tuple(wrap_packet_pointer_to_tensor(packet), false);
+  return std::make_tuple(wrap_pointer_to_tensor(std::move(packet)), false);
 }
 
 torch::stable::Tensor _blocks_create_packet_decoder(
@@ -877,7 +831,7 @@ int64_t _blocks_packet_decoder_send_packet(
     torch::stable::Tensor& decoder,
     torch::stable::Tensor& packet) {
   PacketDecoder* decoder_ptr = unwrap_tensor_to_pointer<PacketDecoder>(decoder);
-  AVPacket* raw_packet = unwrap_tensor_to_packet(packet);
+  AVPacket* raw_packet = unwrap_tensor_to_pointer<AVPacket>(packet);
   return static_cast<int64_t>(decoder_ptr->send_packet(raw_packet));
 }
 
@@ -908,11 +862,10 @@ OpsReceiveFrameOutput _blocks_packet_decoder_receive_frame(
         0.0);
   }
   AVRational time_base = decoder_ptr->time_base();
-  double pts_seconds = pts_to_seconds(get_pts_or_dts(av_frame), time_base);
-  double duration_seconds = pts_to_seconds(get_duration(av_frame), time_base);
-  AVFrame* raw_frame = av_frame.release();
+  double pts_seconds = pts_to_seconds(get_pts_or_dts(*av_frame), time_base);
+  double duration_seconds = pts_to_seconds(get_duration(*av_frame), time_base);
   return std::make_tuple(
-      wrap_frame_pointer_to_tensor(raw_frame),
+      wrap_pointer_to_tensor(std::move(av_frame)),
       static_cast<int64_t>(0),
       pts_seconds,
       duration_seconds);
@@ -932,13 +885,7 @@ torch::stable::Tensor _blocks_convert_frame(
     torch::stable::Tensor& frame) {
   ColorConverter* converter_ptr =
       unwrap_tensor_to_pointer<ColorConverter>(converter);
-  AVFrame* raw_frame = unwrap_tensor_to_frame(frame);
-  // Borrow the frame for conversion, then release() so the handle keeps
-  // ownership and frees it when its tensor is dropped (CPU path).
-  UniqueAVFrame borrowed(raw_frame);
-  torch::stable::Tensor data = converter_ptr->convert(borrowed);
-  borrowed.release();
-  return data;
+  return converter_ptr->convert(*unwrap_tensor_to_pointer<AVFrame>(frame));
 }
 
 // For testing only. We need to implement this operation as a core library
diff --git a/test/third-party-interface/ThirdPartyInterfaceTest.cpp b/test/third-party-interface/ThirdPartyInterfaceTest.cpp
index 8960ac7b8..a8c341f46 100644
--- a/test/third-party-interface/ThirdPartyInterfaceTest.cpp
+++ b/test/third-party-interface/ThirdPartyInterfaceTest.cpp
@@ -24,7 +24,7 @@ class DummyDeviceInterface : public DeviceInterface {
   }
 
   void convert_av_frame_to_frame_output(
-      UniqueAVFrame& av_frame,
+      const AVFrame& av_frame,
       FrameOutput& frame_output,
       std::optional<torch::stable::Tensor> pre_allocated_output_tensor =
           std::nullopt) override {}