diff --git a/lib/ex_webrtc/rtp/vp8_depayloader.ex b/lib/ex_webrtc/rtp/vp8_depayloader.ex new file mode 100644 index 00000000..d6f6fa32 --- /dev/null +++ b/lib/ex_webrtc/rtp/vp8_depayloader.ex @@ -0,0 +1,72 @@ +defmodule ExWebRTC.RTP.VP8Depayloader do + @moduledoc """ + Reassembles VP8 frames from RTP packets. + + Based on [RFC 7741: RTP Payload Format for VP8 Video](https://datatracker.ietf.org/doc/html/rfc7741) + """ + require Logger + + alias ExWebRTC.RTP.VP8Payload + + @opaque t() :: %__MODULE__{ + current_frame: nil, + current_timestamp: nil + } + + defstruct [:current_frame, :current_timestamp] + + @spec new() :: t() + def new() do + %__MODULE__{} + end + + @spec write(t(), ExRTP.Packet.t()) :: {:ok, t()} | {:ok, binary(), t()} + def write(depayloader, packet) do + with {:ok, vp8_payload} <- VP8Payload.parse(packet.payload) do + depayloader = + case {depayloader.current_frame, vp8_payload} do + {nil, %VP8Payload{s: 1, pid: 0}} -> + %{ + depayloader + | current_frame: vp8_payload.payload, + current_timestamp: packet.timestamp + } + + {nil, _vp8_payload} -> + Logger.debug("Dropping vp8 payload as it doesn't start a new frame") + depayloader + + {_current_frame, %VP8Payload{s: 1, pid: 0}} -> + Logger.debug(""" + Received packet that starts a new frame without finishing the previous frame. \ + Droping previous frame.\ + """) + + %{ + depayloader + | current_frame: vp8_payload.payload, + current_timestamp: packet.timestamp + } + + _ when packet.timestamp != depayloader.current_timestamp -> + Logger.debug(""" + Received packet with timestamp from a new frame that is not a beginning of this frame \ + and without finishing the previous frame. Droping both.\ + """) + + %{depayloader | current_frame: nil, current_timestamp: nil} + + {current_frame, vp8_payload} -> + %{depayloader | current_frame: current_frame <> vp8_payload.payload} + end + + case {depayloader.current_frame, packet.marker} do + {current_frame, true} when current_frame != nil -> + {:ok, current_frame, %{depayloader | current_frame: nil, current_timestamp: nil}} + + _ -> + {:ok, depayloader} + end + end + end +end diff --git a/lib/ex_webrtc/rtp/vp8_payload.ex b/lib/ex_webrtc/rtp/vp8_payload.ex new file mode 100644 index 00000000..ff8580d3 --- /dev/null +++ b/lib/ex_webrtc/rtp/vp8_payload.ex @@ -0,0 +1,151 @@ +defmodule ExWebRTC.RTP.VP8Payload do + @moduledoc """ + Defines VP8 payload structure stored in RTP packet payload. + + Based on [RFC 7741: RTP Payload Format for VP8 Video](https://datatracker.ietf.org/doc/html/rfc7741) + """ + + @type t() :: %__MODULE__{ + n: 0 | 1, + s: 0 | 1, + pid: non_neg_integer(), + picture_id: non_neg_integer() | nil, + tl0picidx: non_neg_integer() | nil, + tid: non_neg_integer() | nil, + y: 0 | 1 | nil, + keyidx: non_neg_integer() | nil, + payload: binary() + } + + @enforce_keys [:n, :s, :pid, :payload] + defstruct @enforce_keys ++ [:picture_id, :tl0picidx, :tid, :y, :keyidx] + + @doc """ + Parses RTP payload as VP8 payload. + """ + @spec parse(binary()) :: {:ok, t()} | {:error, :invalid_packet} + def parse(rtp_payload) + + def parse(<<>>), do: {:error, :invalid_packet} + + def parse(<<0::1, 0::1, n::1, s::1, 0::1, pid::3, payload::binary>>) do + if payload == <<>> do + {:error, :invalid_packet} + else + {:ok, + %__MODULE__{ + n: n, + s: s, + pid: pid, + payload: payload + }} + end + end + + def parse(<<1::1, 0::1, n::1, s::1, 0::1, pid::3, i::1, l::1, t::1, k::1, 0::4, rest::binary>>) do + with {:ok, picture_id, rest} <- parse_picture_id(i, rest), + {:ok, tl0picidx, rest} <- parse_tl0picidx(l, rest), + {:ok, tid, y, keyidx, rest} <- parse_tidykeyidx(t, k, rest) do + if rest == <<>> do + {:error, :invalid_packet} + else + {:ok, + %__MODULE__{ + n: n, + s: s, + pid: pid, + picture_id: picture_id, + tl0picidx: tl0picidx, + tid: tid, + y: y, + keyidx: keyidx, + payload: rest + }} + end + end + end + + def parse(_), do: {:error, :invalid_packet} + + defp parse_picture_id(0, rest), + do: {:ok, nil, rest} + + defp parse_picture_id(1, <<0::1, picture_id::7, rest::binary>>), do: {:ok, picture_id, rest} + defp parse_picture_id(1, <<1::1, picture_id::15, rest::binary>>), do: {:ok, picture_id, rest} + defp parse_picture_id(_, _), do: {:error, :invalid_packet} + + defp parse_tl0picidx(0, rest), do: {:ok, nil, rest} + defp parse_tl0picidx(1, <>), do: {:ok, tl0picidx, rest} + defp parse_tl0picidx(_, _), do: {:error, :invalid_packet} + + defp parse_tidykeyidx(0, 0, rest), do: {:ok, nil, nil, nil, rest} + + defp parse_tidykeyidx(1, 0, <>), + do: {:ok, tid, y, nil, rest} + + # note that both pion and web browser always set y bit to 0 in this case + # but RFC 7741, sec. 4.2 (definition for Y bit) explicitly states that Y bit + # can be set when T is 0 and K is 1 + defp parse_tidykeyidx(0, 1, <<_tid::2, y::1, keyidx::5, rest::binary>>), + do: {:ok, nil, y, keyidx, rest} + + defp parse_tidykeyidx(1, 1, <>), + do: {:ok, tid, y, keyidx, rest} + + defp parse_tidykeyidx(_, _, _), do: {:error, :invalid_packet} + + @spec serialize(t()) :: binary() + def serialize( + %__MODULE__{ + picture_id: nil, + tl0picidx: nil, + tid: nil, + y: nil, + keyidx: nil + } = vp8_payload + ) do + p = vp8_payload + <<0::1, 0::1, p.n::1, p.s::1, 0::1, p.pid::3, p.payload::binary>> + end + + def serialize(vp8_payload) do + p = vp8_payload + i = if p.picture_id, do: 1, else: 0 + l = if p.tl0picidx, do: 1, else: 0 + t = if p.tid, do: 1, else: 0 + k = if p.keyidx, do: 1, else: 0 + + payload = + <<1::1, 0::1, p.n::1, p.s::1, 0::1, p.pid::3, i::1, l::1, t::1, k::1, 0::4>> + |> add_picture_id(p.picture_id) + |> add_tl0picidx(p.tl0picidx) + |> add_tidykeyidx(p.tid, p.y, p.keyidx) + + <> + end + + defp add_picture_id(payload, nil), do: payload + + defp add_picture_id(payload, picture_id) when picture_id in 0..127 do + <> + end + + defp add_picture_id(payload, picture_id) when picture_id in 128..32_767 do + <> + end + + defp add_tl0picidx(payload, nil), do: payload + + defp add_tl0picidx(payload, tl0picidx) do + <> + end + + defp add_tidykeyidx(payload, nil, nil, nil), do: payload + + defp add_tidykeyidx(_payload, tid, nil, _keyidx) when tid != nil, + do: raise("VP8 Y bit has to be set when TID is set") + + defp add_tidykeyidx(payload, tid, y, keyidx) do + <> + end +end diff --git a/lib/ex_webrtc/rtp/vp8_payloader.ex b/lib/ex_webrtc/rtp/vp8_payloader.ex index c8a65f8f..5501f83a 100644 --- a/lib/ex_webrtc/rtp/vp8_payloader.ex +++ b/lib/ex_webrtc/rtp/vp8_payloader.ex @@ -2,6 +2,8 @@ defmodule ExWebRTC.RTP.VP8Payloader do @moduledoc """ Encapsulates VP8 video frames into RTP packets. + Based on [RFC 7741: RTP Payload Format for VP8 Video](https://datatracker.ietf.org/doc/html/rfc7741) + It does not support `X` bit right now, in particular it does not pay attention to VP8 partion boundaries (see RFC 7741 sec. 4.4). """ diff --git a/test/ex_webrtc/rtp/vp8_depayloader_test.exs b/test/ex_webrtc/rtp/vp8_depayloader_test.exs new file mode 100644 index 00000000..02158904 --- /dev/null +++ b/test/ex_webrtc/rtp/vp8_depayloader_test.exs @@ -0,0 +1,57 @@ +defmodule ExWebRTC.RTP.VP8DepayloaderTest do + use ExUnit.Case, async: true + + alias ExWebRTC.RTP.{VP8Payload, VP8Depayloader} + + test "write/2" do + depayloader = VP8Depayloader.new() + # random vp8 data, not necessarily correct + data = <<0, 1, 2, 3>> + + # packet with entire frame + vp8_payload = %VP8Payload{n: 0, s: 1, pid: 0, payload: data} + vp8_payload = VP8Payload.serialize(vp8_payload) + + packet = ExRTP.Packet.new(vp8_payload, 0, 0, 0, 0, marker: true) + + assert {:ok, ^data, %{current_frame: nil, current_timestamp: nil} = depayloader} = + VP8Depayloader.write(depayloader, packet) + + # packet that doesn't start a new frame + vp8_payload = %VP8Payload{n: 0, s: 0, pid: 0, payload: data} + vp8_payload = VP8Payload.serialize(vp8_payload) + + packet = ExRTP.Packet.new(vp8_payload, 0, 0, 0, 0) + + assert {:ok, %{current_frame: nil, current_timestamp: nil} = depayloader} = + VP8Depayloader.write(depayloader, packet) + + # packet that starts a new frame without finishing the previous one + vp8_payload = %VP8Payload{n: 0, s: 1, pid: 0, payload: data} + vp8_payload = VP8Payload.serialize(vp8_payload) + + packet = ExRTP.Packet.new(vp8_payload, 0, 0, 0, 0) + + assert {:ok, %{current_frame: ^data, current_timestamp: 0} = depayloader} = + VP8Depayloader.write(depayloader, packet) + + data2 = data <> <<0>> + vp8_payload = %VP8Payload{n: 0, s: 1, pid: 0, payload: data2} + vp8_payload = VP8Payload.serialize(vp8_payload) + + packet = ExRTP.Packet.new(vp8_payload, 0, 0, 3000, 0) + + assert {:ok, %{current_frame: ^data2, current_timestamp: 3000} = depayloader} = + VP8Depayloader.write(depayloader, packet) + + # packet with timestamp from a new frame that is not a beginning of this frame + data2 = data + vp8_payload = %VP8Payload{n: 0, s: 0, pid: 0, payload: data2} + vp8_payload = VP8Payload.serialize(vp8_payload) + + packet = ExRTP.Packet.new(vp8_payload, 0, 0, 6000, 0) + + assert {:ok, %{current_frame: nil, current_timestamp: nil}} = + VP8Depayloader.write(depayloader, packet) + end +end diff --git a/test/ex_webrtc/rtp/vp8_payload_test.exs b/test/ex_webrtc/rtp/vp8_payload_test.exs new file mode 100644 index 00000000..c0781b8a --- /dev/null +++ b/test/ex_webrtc/rtp/vp8_payload_test.exs @@ -0,0 +1,96 @@ +defmodule ExWebrtc.Rtp.Vp8PayloadTest do + use ExUnit.Case, async: true + + alias ExWebRTC.RTP.VP8Payload + + test "parse/1 and serialize/1" do + # test vectors are based on RFC 7741, sec. 4.6 + + # random vp8 data, not necessarily correct + vp8_payload = <<0, 1, 2, 3>> + + # X=1, S=1, PID=0, I=1, pciture_id=17 + frame = + <<1::1, 0::1, 0::1, 1::1, 0::1, 0::3, 1::1, 0::7, 0::1, 17::7, vp8_payload::binary>> + + parsed_frame = + %VP8Payload{ + n: 0, + s: 1, + pid: 0, + picture_id: 17, + tl0picidx: nil, + tid: nil, + y: nil, + keyidx: nil, + payload: vp8_payload + } + + assert {:ok, parsed_frame} == VP8Payload.parse(frame) + assert frame == VP8Payload.serialize(parsed_frame) + + # X=0, S=1, PID=0 + frame = <<0::1, 0::1, 0::1, 1::1, 0::1, 0::3, vp8_payload::binary>> + + parsed_frame = %VP8Payload{ + n: 0, + s: 1, + pid: 0, + picture_id: nil, + tl0picidx: nil, + tid: nil, + y: nil, + keyidx: nil, + payload: vp8_payload + } + + assert {:ok, parsed_frame} == VP8Payload.parse(frame) + assert frame == VP8Payload.serialize(parsed_frame) + + # X=1, S=1, I=1, L=1, T=1, K=1, M=1, picture_id=4711 + frame = + <<1::1, 0::1, 0::1, 1::1, 0::1, 0::3, 1::1, 1::1, 1::1, 1::1, 0::4, 1::1, 4711::15, 1::8, + 1::2, 1::1, 1::5, vp8_payload::binary>> + + parsed_frame = %VP8Payload{ + n: 0, + s: 1, + pid: 0, + picture_id: 4711, + tl0picidx: 1, + tid: 1, + y: 1, + keyidx: 1, + payload: vp8_payload + } + + assert {:ok, parsed_frame} == VP8Payload.parse(frame) + assert frame == VP8Payload.serialize(parsed_frame) + + assert {:error, :invalid_packet} = VP8Payload.parse(<<>>) + + # X=0 and no vp8_payload + assert {:error, :invalid_packet} = + VP8Payload.parse(<<0::1, 0::1, 0::1, 1::1, 0::1, 0::3>>) + + # X=1, I=1 picture_id=1 and no vp8_payload + frame = <<1::1, 0::1, 0::1, 1::1, 0::1, 0::3, 1::1, 0::7, 0::1, 1::7>> + assert {:error, :invalid_packet} = VP8Payload.parse(frame) + + # invalid reserved bit + assert {:error, :invalid_packet} = + VP8Payload.parse(<<0::1, 1::1, 0::1, 1::1, 1::1, 0::3>>) + + # missing picture id + missing_picture_id = <<1::1, 0::1, 0::1, 1::1, 0::1, 0::3, 1::1, 0::7>> + assert {:error, :invalid_packet} = VP8Payload.parse(missing_picture_id) + + # missing tl0picidx + missing_tl0picidx = <<1::1, 0::1, 0::1, 1::1, 0::1, 0::3, 0::1, 1::1, 0::6>> + assert {:error, :invalid_packet} = VP8Payload.parse(missing_tl0picidx) + + # missing tidykeyidx + missing_tidykeyidx = <<1::1, 0::1, 0::1, 1::1, 0::1, 0::3, 0::2, 1::1, 0::1, 0::4>> + assert {:error, :invalid_packet} = VP8Payload.parse(missing_tidykeyidx) + end +end