Add H265 support to sink bin

Author: gBillal

lib/membrane_http_adaptive_stream/hls.ex

@@ -237,6 +237,9 @@ defmodule Membrane.HTTPAdaptiveStream.HLS do
     |> String.downcase()
   end
 
+  defp serialize_codec({:hvc1, %{profile: profile, level: level}}),
+    do: "hvc1.#{profile}.4.L#{level}.B0"
+
   defp serialize_codec({:mp4a, %{aot_id: aot_id}}), do: String.downcase("mp4a.40.#{aot_id}")
 
   defp serialize_codec(_other), do: ""

lib/membrane_http_adaptive_stream/sink_bin.ex

@@ -4,15 +4,15 @@ defmodule Membrane.HTTPAdaptiveStream.SinkBin do
   to eventually store them using provided storage configuration.
 
   ## Input streams
-  Parsed H264 or AAC video or audio streams are expected to be connected via the `:input` pad.
+  Parsed H264, H265 or AAC video or audio streams are expected to be connected via the `:input` pad.
   The type of stream has to be specified via the pad's `:encoding` option.
 
   ## Output
   Specify one of `Membrane.HTTPAdaptiveStream.Storages` as `:storage` to configure the sink.
   """
   use Membrane.Bin
 
-  alias Membrane.{AAC, H264, MP4, Time}
+  alias Membrane.{AAC, H264, H265, MP4, Time}
   alias Membrane.HTTPAdaptiveStream.{Manifest, Sink, Storage}
 
   def_options manifest_name: [
@@ -104,12 +104,13 @@ defmodule Membrane.HTTPAdaptiveStream.SinkBin do
     accepted_format:
       any_of(
         Membrane.AAC,
-        Membrane.H264
+        Membrane.H264,
+        Membrane.H265
       ),
     availability: :on_request,
     options: [
       encoding: [
-        spec: :AAC | :H264,
+        spec: :AAC | :H264 | :H265,
         description: """
         Encoding type determining which parser will be used for the given stream.
         """
@@ -198,10 +199,15 @@ defmodule Membrane.HTTPAdaptiveStream.SinkBin do
     {[spec: spec], state}
   end
 
-  defp do_handle_pad_added(pad, %{encoding: :H264} = pad_options, ctx, %{mode: :muxed_av} = state)
-       when is_map_key(ctx.children, :audio_tee) do
+  defp do_handle_pad_added(
+         pad,
+         %{encoding: encoding} = pad_options,
+         ctx,
+         %{mode: :muxed_av} = state
+       )
+       when encoding in [:H264, :H265] and is_map_key(ctx.children, :audio_tee) do
     Pad.ref(:input, ref) = pad
-    parser = get_parser(:H264, state)
+    parser = get_parser(encoding, state)
     muxer = cmaf_child_definiton(pad_options)
 
     spec = [
@@ -218,7 +224,8 @@ defmodule Membrane.HTTPAdaptiveStream.SinkBin do
     {[spec: spec], state}
   end
 
-  defp do_handle_pad_added(_pad, %{encoding: :H264}, _ctx, %{mode: :muxed_av} = state) do
+  defp do_handle_pad_added(_pad, %{encoding: encoding}, _ctx, %{mode: :muxed_av} = state)
+       when encoding in [:H264, :H265] do
     state = increment_streams_counters(state)
     {[], state}
   end
@@ -229,7 +236,7 @@ defmodule Membrane.HTTPAdaptiveStream.SinkBin do
 
     postponed_cmaf_muxers =
       Map.values(ctx.pads)
-      |> Enum.filter(&(&1.direction == :input and &1.options[:encoding] == :H264))
+      |> Enum.filter(&(&1.direction == :input and &1.options[:encoding] in [:H264, :H265]))
       |> Enum.map(fn pad_data ->
         Pad.ref(:input, cmaf_ref) = pad_data.ref
         muxer = cmaf_child_definiton(pad_options)
@@ -336,11 +343,15 @@ defmodule Membrane.HTTPAdaptiveStream.SinkBin do
         metadata.options.encoding == :AAC
       end)
 
-  defp get_parser(encoding, state) do
-    if encoding == :AAC,
-      do: %AAC.Parser{output_config: :esds, out_encapsulation: :none},
-      else: %H264.Parser{
-        output_stream_structure: if(state.mp4_parameters_in_band?, do: :avc3, else: :avc1)
-      }
-  end
+  defp get_parser(:AAC, _state), do: %AAC.Parser{output_config: :esds, out_encapsulation: :none}
+
+  defp get_parser(:H264, state),
+    do: %H264.Parser{
+      output_stream_structure: if(state.mp4_parameters_in_band?, do: :avc3, else: :avc1)
+    }
+
+  defp get_parser(:H265, state),
+    do: %H265.Parser{
+      output_stream_structure: if(state.mp4_parameters_in_band?, do: :hev1, else: :hvc1)
+    }
 end

mix.lock

@@ -1,22 +1,22 @@
 %{
   "bimap": {:hex, :bimap, "1.3.0", "3ea4832e58dc83a9b5b407c6731e7bae87458aa618e6d11d8e12114a17afa4b3", [:mix], [], "hexpm", "bf5a2b078528465aa705f405a5c638becd63e41d280ada41e0f77e6d255a10b4"},
   "bunch": {:hex, :bunch, "1.6.1", "5393d827a64d5f846092703441ea50e65bc09f37fd8e320878f13e63d410aec7", [:mix], [], "hexpm", "286cc3add551628b30605efbe2fca4e38cc1bea89bcd0a1a7226920b3364fe4a"},
-  "bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"},
+  "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
   "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"},
   "coerce": {:hex, :coerce, "1.0.1", "211c27386315dc2894ac11bc1f413a0e38505d808153367bd5c6e75a4003d096", [:mix], [], "hexpm", "b44a691700f7a1a15b4b7e2ff1fa30bebd669929ac8aa43cffe9e2f8bf051cf1"},
-  "credo": {:hex, :credo, "1.7.1", "6e26bbcc9e22eefbff7e43188e69924e78818e2fe6282487d0703652bc20fd62", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "e9871c6095a4c0381c89b6aa98bc6260a8ba6addccf7f6a53da8849c748a58a2"},
-  "dialyxir": {:hex, :dialyxir, "1.4.2", "764a6e8e7a354f0ba95d58418178d486065ead1f69ad89782817c296d0d746a5", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "516603d8067b2fd585319e4b13d3674ad4f314a5902ba8130cd97dc902ce6bbd"},
+  "credo": {:hex, :credo, "1.7.5", "643213503b1c766ec0496d828c90c424471ea54da77c8a168c725686377b9545", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "f799e9b5cd1891577d8c773d245668aa74a2fcd15eb277f51a0131690ebfb3fd"},
+  "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"},
   "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"},
   "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},
-  "ex_doc": {:hex, :ex_doc, "0.31.0", "06eb1dfd787445d9cab9a45088405593dd3bb7fe99e097eaa71f37ba80c7a676", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "5350cafa6b7f77bdd107aa2199fe277acf29d739aba5aee7e865fc680c62a110"},
-  "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"},
+  "ex_doc": {:hex, :ex_doc, "0.31.1", "8a2355ac42b1cc7b2379da9e40243f2670143721dd50748bf6c3b1184dae2089", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "3178c3a407c557d8343479e1ff117a96fd31bafe52a039079593fb0524ef61b0"},
+  "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"},
   "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
   "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
   "logger_backends": {:hex, :logger_backends, "1.0.0", "09c4fad6202e08cb0fbd37f328282f16539aca380f512523ce9472b28edc6bdf", [:mix], [], "hexpm", "1faceb3e7ec3ef66a8f5746c5afd020e63996df6fd4eb8cdb789e5665ae6c9ce"},
   "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"},
-  "makeup_erlang": {:hex, :makeup_erlang, "0.1.3", "d684f4bac8690e70b06eb52dad65d26de2eefa44cd19d64a8095e1417df7c8fd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "b78dc853d2e670ff6390b605d807263bf606da3c82be37f9d7f68635bd886fc9"},
+  "makeup_erlang": {:hex, :makeup_erlang, "0.1.5", "e0ff5a7c708dda34311f7522a8758e23bfcd7d8d8068dc312b5eb41c6fd76eba", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "94d2e986428585a21516d7d7149781480013c56e30c6a233534bedf38867a59a"},
   "membrane_aac_format": {:hex, :membrane_aac_format, "0.8.0", "515631eabd6e584e0e9af2cea80471fee6246484dbbefc4726c1d93ece8e0838", [:mix], [{:bimap, "~> 1.1", [hex: :bimap, repo: "hexpm", optional: false]}], "hexpm", "a30176a94491033ed32be45e51d509fc70a5ee6e751f12fd6c0d60bd637013f6"},
   "membrane_aac_plugin": {:hex, :membrane_aac_plugin, "0.18.1", "30433bffd4d5d773f79448dd9afd55d77338721688f09a89b20d742a68cc2c3d", [:mix], [{:bunch, "~> 1.0", [hex: :bunch, repo: "hexpm", optional: false]}, {:membrane_aac_format, "~> 0.8.0", [hex: :membrane_aac_format, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "8fd048c47d5d2949eb557e19f43f62d534d3af5096187f1a1a3a1694d14b772c"},
   "membrane_cmaf_format": {:hex, :membrane_cmaf_format, "0.7.1", "9ea858faefdcb181cdfa8001be827c35c5f854e9809ad57d7062cff1f0f703fd", [:mix], [], "hexpm", "3c7b4ed2a986e27f6f336d2f19e9442cb31d93b3142fc024c019572faca54a73"},

test/membrane_http_adaptive_stream/integration_test/fixtures/audio_video_hevc_tracks/audio_header_audio_track_part_0.mp4

@@ -0,0 +1,19 @@
+#EXTM3U
+#EXT-X-VERSION:7
+#EXT-X-TARGETDURATION:2
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-DISCONTINUITY-SEQUENCE:0
+#EXT-X-MAP:URI="audio_header_audio_track_part_0.mp4"
+#EXTINF:1.9969161,
+audio_segment_0_audio_track.m4s
+#EXTINF:1.9969161,
+audio_segment_1_audio_track.m4s
+#EXTINF:1.996916099,
+audio_segment_2_audio_track.m4s
+#EXTINF:1.9969161,
+audio_segment_3_audio_track.m4s
+#EXTINF:1.9969161,
+audio_segment_4_audio_track.m4s
+#EXTINF:0.04643991,
+audio_segment_5_audio_track.m4s
+#EXT-X-ENDLIST

test/membrane_http_adaptive_stream/integration_test/fixtures/audio_video_hevc_tracks/audio_segment_0_audio_track.m4s

@@ -0,0 +1,6 @@
+#EXTM3U
+#EXT-X-VERSION:7
+#EXT-X-INDEPENDENT-SEGMENTS
+#EXT-X-MEDIA:TYPE=AUDIO,NAME="audio_default_name",GROUP-ID="audio_default_id",AUTOSELECT=YES,DEFAULT=YES,URI="audio_track.m3u8",CODECS="mp4a.40.2"
+#EXT-X-STREAM-INF:BANDWIDTH=136392,AVERAGE-BANDWIDTH=108637,RESOLUTION=1280x720,FRAME-RATE=25,CODECS="hvc1.1.4.L93.B0",AUDIO="audio_default_id"
+video_track.m3u8

test/membrane_http_adaptive_stream/integration_test/fixtures/audio_video_hevc_tracks/audio_segment_1_audio_track.m4s

@@ -0,0 +1,15 @@
+#EXTM3U
+#EXT-X-VERSION:7
+#EXT-X-TARGETDURATION:3
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-DISCONTINUITY-SEQUENCE:0
+#EXT-X-MAP:URI="video_header_video_track_part_0.mp4"
+#EXTINF:3.0,
+video_segment_0_video_track.m4s
+#EXTINF:3.0,
+video_segment_1_video_track.m4s
+#EXTINF:3.0,
+video_segment_2_video_track.m4s
+#EXTINF:1.08,
+video_segment_3_video_track.m4s
+#EXT-X-ENDLIST

test/membrane_http_adaptive_stream/integration_test/fixtures/audio_video_hevc_tracks/audio_segment_2_audio_track.m4s

@@ -30,6 +30,14 @@ defmodule Membrane.HTTPAdaptiveStream.SinkBinIntegrationTest do
   @live_stream_ref_path "./test/membrane_http_adaptive_stream/integration_test/fixtures/live/"
   @persisted_stream_ref_path "./test/membrane_http_adaptive_stream/integration_test/fixtures/persisted/"
 
+  @audio_video_hevc_tracks_source [
+    {"http://raw.githubusercontent.com/membraneframework/static/gh-pages/samples/test-audio.aac",
+     :AAC, :LC, "audio_track"},
+    {"test/membrane_http_adaptive_stream/integration_test/fixtures/ffmpeg-testsrc.hevc", :H265,
+     :main, "video_track"}
+  ]
+  @audio_video_hevc_tracks_ref_path "./test/membrane_http_adaptive_stream/integration_test/fixtures/audio_video_hevc_tracks/"
+
   @audio_multiple_video_tracks_sources [
     {"http://raw.githubusercontent.com/membraneframework/static/gh-pages/samples/big-buck-bunny/bun33s.aac",
      :AAC, :LC, "audio_track"},
@@ -108,6 +116,12 @@ defmodule Membrane.HTTPAdaptiveStream.SinkBinIntegrationTest do
                   generate_best_effort_timestamps: %{framerate: {25, 1}}
                 }
 
+              {:H265, _profile} ->
+                %Membrane.H265.Parser{
+                  output_alignment: :au,
+                  generate_best_effort_timestamps: %{framerate: {25, 1}}
+                }
+
               {:AAC, _profile} ->
                 %Membrane.AAC.Parser{
                   out_encapsulation: :none
@@ -132,7 +146,7 @@ defmodule Membrane.HTTPAdaptiveStream.SinkBinIntegrationTest do
                 segment_duration: segment_duration_for(encoding),
                 partial_segment_duration:
                   if(partial_segments, do: partial_segment_duration_for(encoding), else: nil),
-                max_framerate: if(encoding == :H264, do: 25, else: nil)
+                max_framerate: if(encoding in [:H264, :H265], do: 25, else: nil)
               ]
             )
             |> get_child(:sink_bin)
@@ -144,13 +158,13 @@ defmodule Membrane.HTTPAdaptiveStream.SinkBinIntegrationTest do
     defp segment_duration_for(:AAC),
       do: Time.milliseconds(2000)
 
-    defp segment_duration_for(:H264),
+    defp segment_duration_for(codec) when codec in [:H264, :H265],
       do: Time.milliseconds(2000)
 
     defp partial_segment_duration_for(:AAC),
       do: Time.milliseconds(500)
 
-    defp partial_segment_duration_for(:H264),
+    defp partial_segment_duration_for(codec) when codec in [:H264, :H265],
       do: Time.milliseconds(500)
   end
 
@@ -168,6 +182,15 @@ defmodule Membrane.HTTPAdaptiveStream.SinkBinIntegrationTest do
       )
     end
 
+    @tag :tmp_dir
+    test "audio and hevc video tracks", %{tmp_dir: tmp_dir} do
+      test_pipeline(
+        @audio_video_hevc_tracks_source,
+        @audio_video_hevc_tracks_ref_path,
+        tmp_dir
+      )
+    end
+
     @tag :tmp_dir
     test "audio and multiple video tracks", %{tmp_dir: tmp_dir} do
       test_pipeline(
@@ -498,8 +521,14 @@ defmodule Membrane.HTTPAdaptiveStream.SinkBinIntegrationTest do
     hackney_sources =
       sources
       |> Enum.map(fn {path, encoding, profile, name} ->
-        {%Membrane.Hackney.Source{location: path, hackney_opts: [follow_redirect: true]},
-         encoding, profile, name}
+        case path do
+          "http" <> _rest ->
+            {%Membrane.Hackney.Source{location: path, hackney_opts: [follow_redirect: true]},
+             encoding, profile, name}
+
+          _path ->
+            {%Membrane.File.Source{location: path}, encoding, profile, name}
+        end
       end)
 
     if @create_fixtures do