Add VP8 depayloader

lib/ex_webrtc/rtp/vp8_depayloader.ex

@@ -0,0 +1,72 @@
+defmodule ExWebRTC.RTP.VP8Depayloader do
+  @moduledoc """
+  Reassembles VP8 frames from RTP packets.
+
+  Based on [RFC 7741: RTP Payload Format for VP8 Video](https://datatracker.ietf.org/doc/html/rfc7741)
+  """
+  require Logger
+
+  alias ExWebRTC.RTP.VP8Payload
+
+  @opaque t() :: %__MODULE__{
+            current_frame: nil,
+            current_timestamp: nil
+          }
+
+  defstruct [:current_frame, :current_timestamp]
+
+  @spec new() :: t()
+  def new() do
+    %__MODULE__{}
+  end
+
+  @spec write(t(), ExRTP.Packet.t()) :: {:ok, t()} | {:ok, binary(), t()}
+  def write(depayloader, packet) do
+    with {:ok, vp8_payload} <- VP8Payload.parse(packet.payload) do
+      depayloader =
+        case {depayloader.current_frame, vp8_payload} do
+          {nil, %VP8Payload{s: 1, pid: 0}} ->
+            %{
+              depayloader
+              | current_frame: vp8_payload.payload,
+                current_timestamp: packet.timestamp
+            }
+
+          {nil, _vp8_payload} ->
+            Logger.debug("Dropping vp8 payload as it doesn't start a new frame")
+            depayloader
+
+          {_current_frame, %VP8Payload{s: 1, pid: 0}} ->
+            Logger.debug("""
+            Received packet that starts a new frame without finishing the previous frame. \
+            Droping previous frame.\
+            """)
+
+            %{
+              depayloader
+              | current_frame: vp8_payload.payload,
+                current_timestamp: packet.timestamp
+            }
+
+          _ when packet.timestamp != depayloader.current_timestamp ->
+            Logger.debug("""
+            Received packet with timestamp from a new frame that is not a beginning of this frame \
+            and without finishing the previous frame. Droping both.\
+            """)
+
+            %{depayloader | current_frame: nil, current_timestamp: nil}
+
+          {current_frame, vp8_payload} ->
+            %{depayloader | current_frame: current_frame <> vp8_payload.payload}
+        end
+
+      case {depayloader.current_frame, packet.marker} do
+        {current_frame, true} when current_frame != nil ->
+          {:ok, current_frame, %{depayloader | current_frame: nil, current_timestamp: nil}}
+
+        _ ->
+          {:ok, depayloader}
+      end
+    end
+  end
+end

lib/ex_webrtc/rtp/vp8_payload.ex

@@ -0,0 +1,151 @@
+defmodule ExWebRTC.RTP.VP8Payload do
+  @moduledoc """
+  Defines VP8 payload structure stored in RTP packet payload.
+
+  Based on [RFC 7741: RTP Payload Format for VP8 Video](https://datatracker.ietf.org/doc/html/rfc7741)
+  """
+
+  @type t() :: %__MODULE__{
+          n: 0 | 1,
+          s: 0 | 1,
+          pid: non_neg_integer(),
+          picture_id: non_neg_integer() | nil,
+          tl0picidx: non_neg_integer() | nil,
+          tid: non_neg_integer() | nil,
+          y: 0 | 1 | nil,
+          keyidx: non_neg_integer() | nil,
+          payload: binary()
+        }
+
+  @enforce_keys [:n, :s, :pid, :payload]
+  defstruct @enforce_keys ++ [:picture_id, :tl0picidx, :tid, :y, :keyidx]
+
+  @doc """
+  Parses RTP payload as VP8 payload.
+  """
+  @spec parse(binary()) :: {:ok, t()} | {:error, :invalid_packet}
+  def parse(rtp_payload)
+
+  def parse(<<>>), do: {:error, :invalid_packet}
+
+  def parse(<<0::1, 0::1, n::1, s::1, 0::1, pid::3, payload::binary>>) do
+    if payload == <<>> do
+      {:error, :invalid_packet}
+    else
+      {:ok,
+       %__MODULE__{
+         n: n,
+         s: s,
+         pid: pid,
+         payload: payload
+       }}
+    end
+  end
+
+  def parse(<<1::1, 0::1, n::1, s::1, 0::1, pid::3, i::1, l::1, t::1, k::1, 0::4, rest::binary>>) do
+    with {:ok, picture_id, rest} <- parse_picture_id(i, rest),
+         {:ok, tl0picidx, rest} <- parse_tl0picidx(l, rest),
+         {:ok, tid, y, keyidx, rest} <- parse_tidykeyidx(t, k, rest) do
+      if rest == <<>> do
+        {:error, :invalid_packet}
+      else
+        {:ok,
+         %__MODULE__{
+           n: n,
+           s: s,
+           pid: pid,
+           picture_id: picture_id,
+           tl0picidx: tl0picidx,
+           tid: tid,
+           y: y,
+           keyidx: keyidx,
+           payload: rest
+         }}
+      end
+    end
+  end
+
+  def parse(_), do: {:error, :invalid_packet}
+
+  defp parse_picture_id(0, rest),
+    do: {:ok, nil, rest}
+
+  defp parse_picture_id(1, <<0::1, picture_id::7, rest::binary>>), do: {:ok, picture_id, rest}
+  defp parse_picture_id(1, <<1::1, picture_id::15, rest::binary>>), do: {:ok, picture_id, rest}
+  defp parse_picture_id(_, _), do: {:error, :invalid_packet}
+
+  defp parse_tl0picidx(0, rest), do: {:ok, nil, rest}
+  defp parse_tl0picidx(1, <<tl0picidx, rest::binary>>), do: {:ok, tl0picidx, rest}
+  defp parse_tl0picidx(_, _), do: {:error, :invalid_packet}
+
+  defp parse_tidykeyidx(0, 0, rest), do: {:ok, nil, nil, nil, rest}
+
+  defp parse_tidykeyidx(1, 0, <<tid::2, y::1, _keyidx::5, rest::binary>>),
+    do: {:ok, tid, y, nil, rest}
+
+  # note that both pion and web browser always set y bit to 0 in this case
+  # but RFC 7741, sec. 4.2 (definition for Y bit) explicitly states that Y bit
+  # can be set when T is 0 and K is 1
+  defp parse_tidykeyidx(0, 1, <<_tid::2, y::1, keyidx::5, rest::binary>>),
+    do: {:ok, nil, y, keyidx, rest}
+
+  defp parse_tidykeyidx(1, 1, <<tid::2, y::1, keyidx::5, rest::binary>>),
+    do: {:ok, tid, y, keyidx, rest}
+
+  defp parse_tidykeyidx(_, _, _), do: {:error, :invalid_packet}
+
+  @spec serialize(t()) :: binary()
+  def serialize(
+        %__MODULE__{
+          picture_id: nil,
+          tl0picidx: nil,
+          tid: nil,
+          y: nil,
+          keyidx: nil
+        } = vp8_payload
+      ) do
+    p = vp8_payload
+    <<0::1, 0::1, p.n::1, p.s::1, 0::1, p.pid::3, p.payload::binary>>
+  end
+
+  def serialize(vp8_payload) do
+    p = vp8_payload
+    i = if p.picture_id, do: 1, else: 0
+    l = if p.tl0picidx, do: 1, else: 0
+    t = if p.tid, do: 1, else: 0
+    k = if p.keyidx, do: 1, else: 0
+
+    payload =
+      <<1::1, 0::1, p.n::1, p.s::1, 0::1, p.pid::3, i::1, l::1, t::1, k::1, 0::4>>
+      |> add_picture_id(p.picture_id)
+      |> add_tl0picidx(p.tl0picidx)
+      |> add_tidykeyidx(p.tid, p.y, p.keyidx)
+
+    <<payload::binary, vp8_payload.payload::binary>>
+  end
+
+  defp add_picture_id(payload, nil), do: payload
+
+  defp add_picture_id(payload, picture_id) when picture_id in 0..127 do
+    <<payload::binary, 0::1, picture_id::7>>
+  end
+
+  defp add_picture_id(payload, picture_id) when picture_id in 128..32_767 do
+    <<payload::binary, 1::1, picture_id::15>>
+  end
+
+  defp add_tl0picidx(payload, nil), do: payload
+
+  defp add_tl0picidx(payload, tl0picidx) do
+    <<payload::binary, tl0picidx>>
+  end
+
+  defp add_tidykeyidx(payload, nil, nil, nil), do: payload
+
+  defp add_tidykeyidx(_payload, tid, nil, _keyidx) when tid != nil,
+    do: raise("VP8 Y bit has to be set when TID is set")
+
+  defp add_tidykeyidx(payload, tid, y, keyidx) do
+    <<payload::binary, tid || 0::2, y || 0::1, keyidx || 0::5>>
+  end
+end

lib/ex_webrtc/rtp/vp8_payloader.ex

@@ -2,6 +2,8 @@ defmodule ExWebRTC.RTP.VP8Payloader do
   @moduledoc """
   Encapsulates VP8 video frames into RTP packets.
 
+  Based on [RFC 7741: RTP Payload Format for VP8 Video](https://datatracker.ietf.org/doc/html/rfc7741)
+
   It does not support `X` bit right now, in particular it
   does not pay attention to VP8 partion boundaries (see RFC 7741 sec. 4.4).
   """

test/ex_webrtc/rtp/vp8_depayloader_test.exs

@@ -0,0 +1,57 @@
+defmodule ExWebRTC.RTP.VP8DepayloaderTest do
+  use ExUnit.Case, async: true
+
+  alias ExWebRTC.RTP.{VP8Payload, VP8Depayloader}
+
+  test "write/2" do
+    depayloader = VP8Depayloader.new()
+    # random vp8 data, not necessarily correct
+    data = <<0, 1, 2, 3>>
+
+    # packet with entire frame
+    vp8_payload = %VP8Payload{n: 0, s: 1, pid: 0, payload: data}
+    vp8_payload = VP8Payload.serialize(vp8_payload)
+
+    packet = ExRTP.Packet.new(vp8_payload, 0, 0, 0, 0, marker: true)
+
+    assert {:ok, ^data, %{current_frame: nil, current_timestamp: nil} = depayloader} =
+             VP8Depayloader.write(depayloader, packet)
+
+    # packet that doesn't start a new frame
+    vp8_payload = %VP8Payload{n: 0, s: 0, pid: 0, payload: data}
+    vp8_payload = VP8Payload.serialize(vp8_payload)
+
+    packet = ExRTP.Packet.new(vp8_payload, 0, 0, 0, 0)
+
+    assert {:ok, %{current_frame: nil, current_timestamp: nil} = depayloader} =
+             VP8Depayloader.write(depayloader, packet)
+
+    # packet that starts a new frame without finishing the previous one
+    vp8_payload = %VP8Payload{n: 0, s: 1, pid: 0, payload: data}
+    vp8_payload = VP8Payload.serialize(vp8_payload)
+
+    packet = ExRTP.Packet.new(vp8_payload, 0, 0, 0, 0)
+
+    assert {:ok, %{current_frame: ^data, current_timestamp: 0} = depayloader} =
+             VP8Depayloader.write(depayloader, packet)
+
+    data2 = data <> <<0>>
+    vp8_payload = %VP8Payload{n: 0, s: 1, pid: 0, payload: data2}
+    vp8_payload = VP8Payload.serialize(vp8_payload)
+
+    packet = ExRTP.Packet.new(vp8_payload, 0, 0, 3000, 0)
+
+    assert {:ok, %{current_frame: ^data2, current_timestamp: 3000} = depayloader} =
+             VP8Depayloader.write(depayloader, packet)
+
+    # packet with timestamp from a new frame that is not a beginning of this frame
+    data2 = data
+    vp8_payload = %VP8Payload{n: 0, s: 0, pid: 0, payload: data2}
+    vp8_payload = VP8Payload.serialize(vp8_payload)
+
+    packet = ExRTP.Packet.new(vp8_payload, 0, 0, 6000, 0)
+
+    assert {:ok, %{current_frame: nil, current_timestamp: nil}} =
+             VP8Depayloader.write(depayloader, packet)
+  end
+end

test/ex_webrtc/rtp/vp8_payload_test.exs

@@ -0,0 +1,96 @@
+defmodule ExWebrtc.Rtp.Vp8PayloadTest do
+  use ExUnit.Case, async: true
+
+  alias ExWebRTC.RTP.VP8Payload
+
+  test "parse/1 and serialize/1" do
+    # test vectors are based on RFC 7741, sec. 4.6
+
+    # random vp8 data, not necessarily correct
+    vp8_payload = <<0, 1, 2, 3>>
+
+    # X=1, S=1, PID=0, I=1, pciture_id=17
+    frame =
+      <<1::1, 0::1, 0::1, 1::1, 0::1, 0::3, 1::1, 0::7, 0::1, 17::7, vp8_payload::binary>>
+
+    parsed_frame =
+      %VP8Payload{
+        n: 0,
+        s: 1,
+        pid: 0,
+        picture_id: 17,
+        tl0picidx: nil,
+        tid: nil,
+        y: nil,
+        keyidx: nil,
+        payload: vp8_payload
+      }
+
+    assert {:ok, parsed_frame} == VP8Payload.parse(frame)
+    assert frame == VP8Payload.serialize(parsed_frame)
+
+    # X=0, S=1, PID=0
+    frame = <<0::1, 0::1, 0::1, 1::1, 0::1, 0::3, vp8_payload::binary>>
+
+    parsed_frame = %VP8Payload{
+      n: 0,
+      s: 1,
+      pid: 0,
+      picture_id: nil,
+      tl0picidx: nil,
+      tid: nil,
+      y: nil,
+      keyidx: nil,
+      payload: vp8_payload
+    }
+
+    assert {:ok, parsed_frame} == VP8Payload.parse(frame)
+    assert frame == VP8Payload.serialize(parsed_frame)
+
+    # X=1, S=1, I=1, L=1, T=1, K=1, M=1, picture_id=4711
+    frame =
+      <<1::1, 0::1, 0::1, 1::1, 0::1, 0::3, 1::1, 1::1, 1::1, 1::1, 0::4, 1::1, 4711::15, 1::8,
+        1::2, 1::1, 1::5, vp8_payload::binary>>
+
+    parsed_frame = %VP8Payload{
+      n: 0,
+      s: 1,
+      pid: 0,
+      picture_id: 4711,
+      tl0picidx: 1,
+      tid: 1,
+      y: 1,
+      keyidx: 1,
+      payload: vp8_payload
+    }
+
+    assert {:ok, parsed_frame} == VP8Payload.parse(frame)
+    assert frame == VP8Payload.serialize(parsed_frame)
+
+    assert {:error, :invalid_packet} = VP8Payload.parse(<<>>)
+
+    # X=0 and no vp8_payload
+    assert {:error, :invalid_packet} =
+             VP8Payload.parse(<<0::1, 0::1, 0::1, 1::1, 0::1, 0::3>>)
+
+    # X=1, I=1 picture_id=1 and no vp8_payload
+    frame = <<1::1, 0::1, 0::1, 1::1, 0::1, 0::3, 1::1, 0::7, 0::1, 1::7>>
+    assert {:error, :invalid_packet} = VP8Payload.parse(frame)
+
+    # invalid reserved bit
+    assert {:error, :invalid_packet} =
+             VP8Payload.parse(<<0::1, 1::1, 0::1, 1::1, 1::1, 0::3>>)
+
+    # missing picture id
+    missing_picture_id = <<1::1, 0::1, 0::1, 1::1, 0::1, 0::3, 1::1, 0::7>>
+    assert {:error, :invalid_packet} = VP8Payload.parse(missing_picture_id)
+
+    # missing tl0picidx
+    missing_tl0picidx = <<1::1, 0::1, 0::1, 1::1, 0::1, 0::3, 0::1, 1::1, 0::6>>
+    assert {:error, :invalid_packet} = VP8Payload.parse(missing_tl0picidx)
+
+    # missing tidykeyidx
+    missing_tidykeyidx = <<1::1, 0::1, 0::1, 1::1, 0::1, 0::3, 0::2, 1::1, 0::1, 0::4>>
+    assert {:error, :invalid_packet} = VP8Payload.parse(missing_tidykeyidx)
+  end
+end