From 6cd369332479f7543fae82540c11e6dd1de7044d Mon Sep 17 00:00:00 2001 From: Jakub Pisarek <99591440+sgfn@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:51:21 +0100 Subject: [PATCH] [WIP] TCP ICE --- .tool-versions | 2 - example/peer.exs | 8 +- lib/ex_ice/candidate.ex | 98 +++++-- lib/ex_ice/ice_agent.ex | 20 +- lib/ex_ice/priv/candidate.ex | 114 +++++--- lib/ex_ice/priv/candidate/host.ex | 3 + lib/ex_ice/priv/candidate/prflx.ex | 3 + lib/ex_ice/priv/candidate/relay.ex | 3 + lib/ex_ice/priv/candidate/srflx.ex | 3 + lib/ex_ice/priv/candidate_base.ex | 21 +- lib/ex_ice/priv/checklist.ex | 2 + lib/ex_ice/priv/gatherer.ex | 50 +++- lib/ex_ice/priv/ice_agent.ex | 56 +++- lib/ex_ice/priv/transport.ex | 15 +- lib/ex_ice/priv/transport/tcp_client.ex | 330 ++++++++++++++++++++++++ lib/ex_ice/priv/transport/udp.ex | 12 +- mix.lock | 4 +- 17 files changed, 641 insertions(+), 103 deletions(-) delete mode 100644 .tool-versions create mode 100644 lib/ex_ice/priv/transport/tcp_client.ex diff --git a/.tool-versions b/.tool-versions deleted file mode 100644 index 537f99f..0000000 --- a/.tool-versions +++ /dev/null @@ -1,2 +0,0 @@ -erlang 27.3.1 -elixir 1.18.3-otp-27 diff --git a/example/peer.exs b/example/peer.exs index cf0a3bb..f7eda5c 100644 --- a/example/peer.exs +++ b/example/peer.exs @@ -1,7 +1,7 @@ Mix.install([{:gun, "~> 2.0.1"}, {:ex_ice, path: "../", force: true}, {:jason, "~> 1.4.0"}]) require Logger -Logger.configure(level: :info) +Logger.configure(level: :debug) defmodule Peer do use GenServer @@ -95,12 +95,14 @@ defmodule Peer do role = String.to_atom(role) {:ok, pid} = - ICEAgent.start_link(role, + ICEAgent.start_link( + role: role, ip_filter: fn {_, _, _, _} -> true {_, _, _, _, _, _, _, _} -> false end, - ice_servers: [%{urls: "stun:stun.l.google.com:19302"}] + ice_servers: [%{urls: "stun:stun.nextcloud.com"}], + transport: :tcp ) {:ok, ufrag, passwd} = ICEAgent.get_local_credentials(pid) diff --git a/lib/ex_ice/candidate.ex b/lib/ex_ice/candidate.ex index 4113b6a..be158dc 100644 --- a/lib/ex_ice/candidate.ex +++ b/lib/ex_ice/candidate.ex @@ -3,9 +3,10 @@ defmodule ExICE.Candidate do ICE candidate representation. """ - @type type() :: :host | :srflx | :prflx | :relay + @type type :: :host | :srflx | :prflx | :relay + @type tcp_type :: :active | :passive | :so - @type t() :: %__MODULE__{ + @type t :: %__MODULE__{ id: integer(), type: type(), address: :inet.ip_address() | String.t(), @@ -14,7 +15,8 @@ defmodule ExICE.Candidate do foundation: integer(), port: :inet.port_number(), priority: integer(), - transport: :udp | :tcp + transport: :udp | :tcp, + tcp_type: tcp_type() | nil } @enforce_keys [ @@ -24,7 +26,8 @@ defmodule ExICE.Candidate do :port, :foundation, :priority, - :transport + :transport, + :tcp_type ] defstruct @enforce_keys ++ [:base_address, :base_port] @@ -38,7 +41,8 @@ defmodule ExICE.Candidate do priority: priority, address: address, port: port, - type: type + type: type, + tcp_type: tcp_type } = cand # This is based on RFC 8839 sec. 5.1. @@ -54,31 +58,45 @@ defmodule ExICE.Candidate do transport = transport_to_string(transport) address = address_to_string(address) - - "#{foundation} #{component_id} #{transport} #{priority} #{address} #{port} typ #{type} #{related_addr}" - |> String.trim() + tcp_type = tcp_type_to_string(tcp_type) + + [ + foundation, + component_id, + transport, + priority, + address, + port, + "typ", + type, + related_addr, + tcp_type + ] + |> Enum.reject(&(&1 == "")) + |> Enum.join(" ") end @spec unmarshal(String.t()) :: {:ok, t()} | {:error, term()} def unmarshal(string) do - with [f_str, c_str, tr_str, pr_str, a_str, po_str, "typ", ty_str] <- - String.split(string, " ", parts: 8), + with [f_str, c_str, tr_str, pr_str, a_str, po_str, "typ", ty_str | rest] <- + String.split(string, " "), {foundation, ""} <- Integer.parse(f_str), {_component_id, ""} <- Integer.parse(c_str), {:ok, transport} <- parse_transport(String.downcase(tr_str)), {priority, ""} <- Integer.parse(pr_str), {:ok, address} <- parse_address(a_str), {port, ""} <- Integer.parse(po_str), - {:ok, type} <- parse_type(ty_str) do - {:ok, - new( - type, - address: address, - port: port, - priority: priority, - foundation: foundation, - transport: transport - )} + {:ok, type} <- parse_type(ty_str), + {:ok, extra_config} <- parse_optional_attributes(rest) do + config = [ + address: address, + port: port, + priority: priority, + foundation: foundation, + transport: transport + ] + + {:ok, new(type, config ++ extra_config)} else err when is_list(err) -> {:error, :invalid_candidate} err -> err @@ -89,12 +107,16 @@ defmodule ExICE.Candidate do def family(%__MODULE__{address: {_, _, _, _}}), do: :ipv4 def family(%__MODULE__{address: {_, _, _, _, _, _, _, _}}), do: :ipv6 + def tcp_type(%__MODULE__{tcp_type: tt}), do: tt + @doc false @spec new(type(), Keyword.t()) :: t() def new(type, config) when type in [:host, :srflx, :prflx, :relay] do transport = Keyword.get(config, :transport, :udp) address = Keyword.fetch!(config, :address) + tcp_type = if transport == :tcp, do: Keyword.fetch!(config, :tcp_type) + %__MODULE__{ id: ExICE.Priv.Utils.id(), address: address, @@ -104,7 +126,8 @@ defmodule ExICE.Candidate do port: Keyword.fetch!(config, :port), priority: Keyword.fetch!(config, :priority), transport: transport, - type: type + type: type, + tcp_type: tcp_type } end @@ -112,8 +135,13 @@ defmodule ExICE.Candidate do defp address_to_string(address), do: :inet.ntoa(address) defp transport_to_string(:udp), do: "UDP" + defp transport_to_string(:tcp), do: "TCP" + + defp tcp_type_to_string(nil), do: "" + defp tcp_type_to_string(type), do: "tcptype #{type}" defp parse_transport("udp"), do: {:ok, :udp} + defp parse_transport("tcp"), do: {:ok, :tcp} defp parse_transport(_other), do: {:error, :invalid_transport} defp parse_address(address) do @@ -124,9 +152,29 @@ defmodule ExICE.Candidate do end end - defp parse_type("host" <> _rest), do: {:ok, :host} - defp parse_type("srflx" <> _rest), do: {:ok, :srflx} - defp parse_type("prflx" <> _rest), do: {:ok, :prflx} - defp parse_type("relay" <> _rest), do: {:ok, :relay} + defp parse_type("host"), do: {:ok, :host} + defp parse_type("srflx"), do: {:ok, :srflx} + defp parse_type("prflx"), do: {:ok, :prflx} + defp parse_type("relay"), do: {:ok, :relay} defp parse_type(_other), do: {:error, :invalid_type} + + defp parse_optional_attributes(list, config \\ []) + defp parse_optional_attributes([], config), do: {:ok, config} + + defp parse_optional_attributes(["raddr", _2, _3, _4 | rest], config), + do: parse_optional_attributes(rest, config) + + defp parse_optional_attributes(["tcptype", tcp_type | rest], config) do + case parse_tcp_type(tcp_type) do + {:ok, tcp_type} -> parse_optional_attributes(rest, config ++ [tcp_type: tcp_type]) + err -> err + end + end + + defp parse_optional_attributes(_other, config), do: {:ok, config} + + defp parse_tcp_type("active"), do: {:ok, :active} + defp parse_tcp_type("passive"), do: {:ok, :passive} + defp parse_tcp_type("so"), do: {:ok, :so} + defp parse_tcp_type(_other), do: {:error, :invalid_tcp_type} end diff --git a/lib/ex_ice/ice_agent.ex b/lib/ex_ice/ice_agent.ex index 38338a5..19988d8 100644 --- a/lib/ex_ice/ice_agent.ex +++ b/lib/ex_ice/ice_agent.ex @@ -107,7 +107,8 @@ defmodule ExICE.ICEAgent do on_connection_state_change: pid() | nil, on_data: pid() | nil, on_new_candidate: pid() | nil, - host_to_srflx_ip_mapper: host_to_srflx_ip_mapper() | nil + host_to_srflx_ip_mapper: host_to_srflx_ip_mapper() | nil, + transport: :udp | :tcp ] @doc """ @@ -321,6 +322,15 @@ defmodule ExICE.ICEAgent do @impl true def init(opts) do + # TODO: this is ugly, and will not allow us to run more than two TCP ICE agents at the same time + opts = + if opts[:transport] == :tcp do + {:ok, _pid} = ExICE.Priv.Transport.TCP.Client.start_link() + opts ++ [transport_module: ExICE.Priv.Transport.TCP.Client] + else + opts + end + ice_agent = ExICE.Priv.ICEAgent.new(opts) {:ok, %{ice_agent: ice_agent, pending_eoc: false, pending_remote_cands: MapSet.new()}} end @@ -478,6 +488,14 @@ defmodule ExICE.ICEAgent do {:noreply, %{state | ice_agent: ice_agent}} end + @impl true + def handle_info({:tcp, _socket, _packet}, state) do + # TODO: consider receiving TCP data in the ICE Agent process + # ice_agent = ExICE.Priv.ICEAgent.handle_tcp(state.ice_agent, socket, packet) + # {:noreply, %{state | ice_agent: ice_agent}} + {:noreply, state} + end + @impl true def handle_info({:ex_turn, ref, msg}, state) do ice_agent = ExICE.Priv.ICEAgent.handle_ex_turn_msg(state.ice_agent, ref, msg) diff --git a/lib/ex_ice/priv/candidate.ex b/lib/ex_ice/priv/candidate.ex index ab053b7..b305251 100644 --- a/lib/ex_ice/priv/candidate.ex +++ b/lib/ex_ice/priv/candidate.ex @@ -1,9 +1,10 @@ defmodule ExICE.Priv.Candidate do @moduledoc false - @type type() :: :host | :srflx | :prflx | :relay + @type type :: :host | :srflx | :prflx | :relay + @type tcp_type :: :active | :passive | :so | nil - @type t() :: struct() + @type t :: struct() @type config :: [ address: :inet.ip_address() | String.t(), @@ -13,7 +14,8 @@ defmodule ExICE.Priv.Candidate do socket: :inet.socket(), priority: integer(), foundation: integer(), - transport: :udp | :tcp + transport: :udp | :tcp, + tcp_type: tcp_type() ] @callback new(config()) :: t() @@ -22,52 +24,104 @@ defmodule ExICE.Priv.Candidate do @callback family(t()) :: :ipv4 | :ipv6 + @callback tcp_type(t()) :: tcp_type() + @callback to_candidate(t()) :: ExICE.Candidate.t() @callback send_data(t(), :inet.ip_address(), :inet.port_number(), binary()) :: {:ok, t()} | {:error, term(), t()} - @spec priority!(%{:inet.ip_address() => non_neg_integer()}, :inet.ip_address(), type()) :: + @spec priority!( + %{:inet.ip_address() => non_neg_integer()}, + :inet.ip_address(), + type(), + tcp_type() + ) :: non_neg_integer() - def priority!(local_preferences, base_address, type) do - local_preference = Map.fetch!(local_preferences, base_address) - do_priority(local_preference, type) + def priority!(local_preferences, base_address, type, tcp_type) do + other_preference = Map.fetch!(local_preferences, base_address) + do_priority(other_preference, type, tcp_type) end - @spec priority(%{:inet.ip_address() => non_neg_integer()}, :inet.ip_address(), type()) :: + @spec priority( + %{:inet.ip_address() => non_neg_integer()}, + :inet.ip_address(), + type(), + tcp_type() + ) :: {%{:inet.ip_address() => non_neg_integer()}, non_neg_integer()} - def priority(local_preferences, base_address, type) do - local_preference = - Map.get(local_preferences, base_address) || generate_local_preference(local_preferences) + def priority(local_preferences, base_address, type, tcp_type) do + other_preference = + Map.get(local_preferences, base_address) || generate_other_preference(local_preferences) - local_preferences = Map.put(local_preferences, base_address, local_preference) + local_preferences = Map.put(local_preferences, base_address, other_preference) - {local_preferences, do_priority(local_preference, type)} + {local_preferences, do_priority(other_preference, type, tcp_type)} end - defp do_priority(local_preference, type) do - type_preference = - case type do - :host -> 126 - :prflx -> 110 - :srflx -> 100 - :relay -> 0 - end + defp do_priority(other_preference, type, tcp_type) do + type_preference = type_preference(type, tcp_type) + direction_preference = direction_preference(type, tcp_type) + + local_preference = 2 ** 13 * direction_preference + other_preference 2 ** 24 * type_preference + 2 ** 8 * local_preference + 2 ** 0 * (256 - 1) end - defp generate_local_preference(local_preferences, attempts \\ 200) + # TODO: revisit these when implementing UDP+TCP support at the same time + # UDP + defp type_preference(type, nil) do + case type do + :host -> 126 + :prflx -> 110 + :srflx -> 100 + :relay -> 10 + end + end + + # TCP + defp type_preference(type, _tcp_type) do + case type do + :host -> 80 + :prflx -> 70 + # :nat_assisted -> 65 + :srflx -> 60 + # :udp_tunneled -> 45 + :relay -> 0 + end + end + + # UDP + defp direction_preference(_type, nil), do: 7 + + # TCP + defp direction_preference(type, tcp_type) when type in [:host, :udp_tunneled, :relay] do + case tcp_type do + :active -> 6 + :passive -> 4 + :so -> 2 + end + end + + defp direction_preference(_type, tcp_type) do + case tcp_type do + :so -> 6 + :active -> 4 + :passive -> 2 + end + end + + defp generate_other_preference(local_preferences, attempts \\ 200) - defp generate_local_preference(_local_preferences, 0), + defp generate_other_preference(_local_preferences, 0), do: raise("Couldn't generate local preference") - defp generate_local_preference(local_preferences, attempts) do - # this should give us a number from 0 to 2**16-1 - <> = :crypto.strong_rand_bytes(2) + defp generate_other_preference(local_preferences, attempts) do + # 0..8191 + <> = :crypto.strong_rand_bytes(2) - if Map.has_key?(local_preferences, pref) do - generate_local_preference(local_preferences, attempts - 1) + if local_preferences |> Map.values() |> Enum.member?(pref) do + generate_other_preference(local_preferences, attempts - 1) else pref end @@ -76,8 +130,6 @@ defmodule ExICE.Priv.Candidate do @spec foundation(type(), :inet.ip_address() | String.t(), :inet.ip_address() | nil, atom()) :: integer() def foundation(type, ip, stun_turn_ip, transport) do - {type, ip, stun_turn_ip, transport} - |> then(&inspect(&1)) - |> then(&:erlang.crc32(&1)) + {type, ip, stun_turn_ip, transport} |> inspect() |> :erlang.crc32() end end diff --git a/lib/ex_ice/priv/candidate/host.ex b/lib/ex_ice/priv/candidate/host.ex index 606af10..e166ade 100644 --- a/lib/ex_ice/priv/candidate/host.ex +++ b/lib/ex_ice/priv/candidate/host.ex @@ -20,6 +20,9 @@ defmodule ExICE.Priv.Candidate.Host do @impl true def family(cand), do: CandidateBase.family(cand.base) + @impl true + def tcp_type(cand), do: CandidateBase.tcp_type(cand.base) + @impl true def to_candidate(cand), do: CandidateBase.to_candidate(cand.base) diff --git a/lib/ex_ice/priv/candidate/prflx.ex b/lib/ex_ice/priv/candidate/prflx.ex index 34db343..25c8969 100644 --- a/lib/ex_ice/priv/candidate/prflx.ex +++ b/lib/ex_ice/priv/candidate/prflx.ex @@ -20,6 +20,9 @@ defmodule ExICE.Priv.Candidate.Prflx do @impl true def family(cand), do: CandidateBase.family(cand.base) + @impl true + def tcp_type(cand), do: CandidateBase.tcp_type(cand.base) + @impl true def to_candidate(cand), do: CandidateBase.to_candidate(cand.base) diff --git a/lib/ex_ice/priv/candidate/relay.ex b/lib/ex_ice/priv/candidate/relay.ex index aea70f2..2c4be57 100644 --- a/lib/ex_ice/priv/candidate/relay.ex +++ b/lib/ex_ice/priv/candidate/relay.ex @@ -20,6 +20,9 @@ defmodule ExICE.Priv.Candidate.Relay do @impl true def family(cand), do: CandidateBase.family(cand.base) + @impl true + def tcp_type(cand), do: CandidateBase.tcp_type(cand.base) + @impl true def to_candidate(cand), do: CandidateBase.to_candidate(cand.base) diff --git a/lib/ex_ice/priv/candidate/srflx.ex b/lib/ex_ice/priv/candidate/srflx.ex index f32f402..3b61a8c 100644 --- a/lib/ex_ice/priv/candidate/srflx.ex +++ b/lib/ex_ice/priv/candidate/srflx.ex @@ -20,6 +20,9 @@ defmodule ExICE.Priv.Candidate.Srflx do @impl true def family(cand), do: CandidateBase.family(cand.base) + @impl true + def tcp_type(cand), do: CandidateBase.tcp_type(cand.base) + @impl true def to_candidate(cand), do: CandidateBase.to_candidate(cand.base) diff --git a/lib/ex_ice/priv/candidate_base.ex b/lib/ex_ice/priv/candidate_base.ex index 1ac18fd..e1eda2c 100644 --- a/lib/ex_ice/priv/candidate_base.ex +++ b/lib/ex_ice/priv/candidate_base.ex @@ -2,7 +2,7 @@ defmodule ExICE.Priv.CandidateBase do @moduledoc false alias ExICE.Priv.{Candidate, Utils} - @type t() :: %__MODULE__{ + @type t :: %__MODULE__{ id: integer(), address: :inet.ip_address() | String.t(), base_address: :inet.ip_address() | nil, @@ -10,10 +10,11 @@ defmodule ExICE.Priv.CandidateBase do foundation: integer(), port: :inet.port_number(), priority: integer(), - transport: :udp, + transport: :udp | :tcp, transport_module: module(), socket: :inet.socket() | nil, type: Candidate.type(), + tcp_type: Candidate.tcp_type(), closed?: boolean() } @@ -28,11 +29,12 @@ defmodule ExICE.Priv.CandidateBase do :transport_module, :type ] - defstruct @enforce_keys ++ [:base_address, :base_port, :socket, closed?: false] + defstruct @enforce_keys ++ [:base_address, :base_port, :socket, :tcp_type, closed?: false] @spec new(Candidate.type(), Keyword.t()) :: t() def new(type, config) do - transport = :udp + transport_module = Keyword.fetch!(config, :transport_module) + transport = transport_module.transport() address = Keyword.fetch!(config, :address) %__MODULE__{ @@ -44,9 +46,10 @@ defmodule ExICE.Priv.CandidateBase do port: Keyword.fetch!(config, :port), priority: Keyword.fetch!(config, :priority), transport: transport, - transport_module: Keyword.get(config, :transport_module, ExICE.Priv.Transport.UDP), + transport_module: transport_module, socket: Keyword.fetch!(config, :socket), - type: type + type: type, + tcp_type: config[:tcp_type] } end @@ -57,6 +60,9 @@ defmodule ExICE.Priv.CandidateBase do def family(%__MODULE__{address: {_, _, _, _}}), do: :ipv4 def family(%__MODULE__{address: {_, _, _, _, _, _, _, _}}), do: :ipv6 + @spec tcp_type(t()) :: Candidate.tcp_type() + def tcp_type(%__MODULE__{tcp_type: tt}), do: tt + @spec to_candidate(t()) :: ExICE.Candidate.t() def to_candidate(cand) do ExICE.Candidate.new(cand.type, @@ -66,7 +72,8 @@ defmodule ExICE.Priv.CandidateBase do base_port: cand.base_port, foundation: cand.foundation, transport: cand.transport, - priority: cand.priority + priority: cand.priority, + tcp_type: cand.tcp_type ) end end diff --git a/lib/ex_ice/priv/checklist.ex b/lib/ex_ice/priv/checklist.ex index f5cc157..7ca5f41 100644 --- a/lib/ex_ice/priv/checklist.ex +++ b/lib/ex_ice/priv/checklist.ex @@ -65,6 +65,8 @@ defmodule ExICE.Priv.Checklist do @spec prune(t()) :: t() def prune(checklist) do + # TODO: prune pairs where the local TCP candidate is passive, as per RFC 6544, sec. 6.2. + # This is done according to RFC 8838 sec. 10 {waiting, in_flight_or_done} = Enum.split_with(checklist, fn {_id, p} -> p.state in [:waiting, :frozen] end) diff --git a/lib/ex_ice/priv/gatherer.ex b/lib/ex_ice/priv/gatherer.ex index 430f5ba..8780559 100644 --- a/lib/ex_ice/priv/gatherer.ex +++ b/lib/ex_ice/priv/gatherer.ex @@ -41,14 +41,16 @@ defmodule ExICE.Priv.Gatherer do |> Stream.reject(&unsupported_ipv6?(&1)) |> Enum.to_list() - ips - |> Enum.map(&open_socket(gatherer, &1)) + for transport_opts <- gatherer.transport_module.socket_configs(), + ip <- ips do + open_socket(gatherer, ip, transport_opts) + end |> Enum.reject(&(&1 == nil)) |> then(&{:ok, &1}) end end - defp open_socket(gatherer, ip) do + defp open_socket(gatherer, ip, transport_opts) do inet = case ip do {_, _, _, _} -> :inet @@ -56,15 +58,16 @@ defmodule ExICE.Priv.Gatherer do end socket_opts = [ - {:inet_backend, :socket}, - {:ip, ip}, + # We're using the :inet` backend, as `:socket` has issues + # with the `{:reuseport, true}` option added by the TCP Client + {:inet_backend, :inet}, {:active, true}, :binary, inet ] Enum.reduce_while(gatherer.ports, nil, fn port, _ -> - case gatherer.transport_module.open(port, socket_opts) do + case gatherer.transport_module.setup_socket(ip, port, socket_opts, transport_opts) do {:ok, socket} -> {:ok, {^ip, sock_port}} = gatherer.transport_module.sockname(socket) @@ -72,7 +75,7 @@ defmodule ExICE.Priv.Gatherer do "Successfully opened socket for: #{inspect(ip)}:#{sock_port}, socket: #{inspect(socket)}" ) - {:halt, socket} + {:halt, %{socket: socket, transport_opts: transport_opts}} {:error, :eaddrinuse} -> Logger.debug("Address #{inspect(ip)}:#{inspect(port)} in use. Trying next port.") @@ -86,7 +89,7 @@ defmodule ExICE.Priv.Gatherer do end @spec gather_host_candidates(t(), %{:inet.ip_address() => non_neg_integer()}, [ - Transport.socket() + {Transport.socket(), map()} ]) :: [Candidate.t()] def gather_host_candidates(gatherer, local_preferences, sockets) do {local_preferences, cands} = @@ -121,7 +124,13 @@ defmodule ExICE.Priv.Gatherer do stun_family = Utils.family(ip) if cand_family == stun_family do - gatherer.transport_module.send(socket, {ip, port}, binding_request) + # Communication with STUN servers should be handled differently + # than the rest of the TCP traffic: the messages are not RFC 4571-framed, + # and we want to issue connection attempts from passive candidates as well. + gatherer.transport_module.send(socket, {ip, port}, binding_request, + frame?: false, + connect?: true + ) else Logger.debug(""" Not gathering srflx candidate because of incompatible ip address families. @@ -178,7 +187,12 @@ defmodule ExICE.Priv.Gatherer do if valid_external_ip?(external_ip, host_cand.base.address, external_ips) do priority = - Candidate.priority!(local_preferences, host_cand.base.address, :srflx) + Candidate.priority!( + local_preferences, + host_cand.base.address, + :srflx, + host_cand.base.tcp_type + ) cand = Candidate.Srflx.new( @@ -188,7 +202,8 @@ defmodule ExICE.Priv.Gatherer do base_port: host_cand.base.port, priority: priority, transport_module: host_cand.base.transport_module, - socket: host_cand.base.socket + socket: host_cand.base.socket, + tcp_type: host_cand.base.tcp_type ) Logger.debug("New srflx candidate from NAT mapping: #{inspect(cand)}") @@ -262,10 +277,16 @@ defmodule ExICE.Priv.Gatherer do Keyword.get_values(int, :addr) end - defp create_new_host_candidate(gatherer, local_preferences, socket) do + defp create_new_host_candidate(gatherer, local_preferences, %{ + socket: socket, + transport_opts: transport_opts + }) do {:ok, {sock_ip, sock_port}} = gatherer.transport_module.sockname(socket) - {local_preferences, priority} = Candidate.priority(local_preferences, sock_ip, :host) + tcp_type = transport_opts[:tcp_type] + + {local_preferences, priority} = + Candidate.priority(local_preferences, sock_ip, :host, tcp_type) cand = Candidate.Host.new( @@ -275,7 +296,8 @@ defmodule ExICE.Priv.Gatherer do base_port: sock_port, priority: priority, transport_module: gatherer.transport_module, - socket: socket + socket: socket, + tcp_type: tcp_type ) Logger.debug("New candidate: #{inspect(cand)}") diff --git a/lib/ex_ice/priv/ice_agent.ex b/lib/ex_ice/priv/ice_agent.ex index f854bd3..79d4034 100644 --- a/lib/ex_ice/priv/ice_agent.ex +++ b/lib/ex_ice/priv/ice_agent.ex @@ -358,7 +358,7 @@ defmodule ExICE.Priv.ICEAgent do %{ ice_agent - | sockets: sockets, + | sockets: Enum.map(sockets, fn %{socket: socket} -> socket end), gathering_transactions: gathering_transactions } |> update_gathering_state() @@ -993,7 +993,7 @@ defmodule ExICE.Priv.ICEAgent do ## PRIV API defp create_srflx_gathering_transactions(stun_servers, sockets) do - for stun_server <- stun_servers, socket <- sockets, into: %{} do + for stun_server <- stun_servers, %{socket: socket} <- sockets, into: %{} do <> = :crypto.strong_rand_bytes(12) t = %{ @@ -1010,7 +1010,7 @@ defmodule ExICE.Priv.ICEAgent do defp create_relay_gathering_transactions(ice_agent, turn_servers, sockets) do # TODO revisit this - for turn_server <- turn_servers, socket <- sockets do + for turn_server <- turn_servers, %{socket: socket} <- sockets do with {:ok, client} <- ExTURN.Client.new(turn_server.url, turn_server.username, turn_server.credential), {:ok, {sock_ip, _sock_port}} <- ice_agent.transport_module.sockname(socket), @@ -1217,8 +1217,9 @@ defmodule ExICE.Priv.ICEAgent do # In other case, we might get duplicates. {:ok, {sock_addr, _sock_port}} = ice_agent.transport_module.sockname(tr.socket) + # TODO: set correct tcp_type here {local_preferences, priority} = - Candidate.priority(ice_agent.local_preferences, sock_addr, :relay) + Candidate.priority(ice_agent.local_preferences, sock_addr, :relay, nil) ice_agent = %{ ice_agent @@ -1913,7 +1914,15 @@ defmodule ExICE.Priv.ICEAgent do nil -> {:ok, {base_addr, base_port}} = ice_agent.transport_module.sockname(tr.socket) - priority = Candidate.priority!(ice_agent.local_preferences, base_addr, :srflx) + host_cand = find_host_cand(Map.values(ice_agent.local_cands), tr.socket) + + priority = + Candidate.priority!( + ice_agent.local_preferences, + base_addr, + :srflx, + host_cand.base.tcp_type + ) cand = Candidate.Srflx.new( @@ -1923,7 +1932,8 @@ defmodule ExICE.Priv.ICEAgent do base_port: base_port, priority: priority, transport_module: ice_agent.transport_module, - socket: tr.socket + socket: tr.socket, + tcp_type: host_cand.base.tcp_type ) Logger.debug("New srflx candidate: #{inspect(cand)}") @@ -2208,16 +2218,24 @@ defmodule ExICE.Priv.ICEAgent do defp get_matching_candidates_local(candidates, %c_mod{} = cand) do Enum.filter(candidates, fn c -> - ExICE.Candidate.family(c) == c_mod.family(cand) + ExICE.Candidate.family(c) == c_mod.family(cand) and + tcp_types_ok?(ExICE.Candidate.tcp_type(c), c_mod.tcp_type(cand)) end) end defp get_matching_candidates_remote(candidates, cand) do Enum.filter(candidates, fn %c_mod{} = c -> - c_mod.family(c) == ExICE.Candidate.family(cand) + c_mod.family(c) == ExICE.Candidate.family(cand) and + tcp_types_ok?(c_mod.tcp_type(c), ExICE.Candidate.tcp_type(cand)) end) end + defp tcp_types_ok?(nil, nil), do: true + defp tcp_types_ok?(:so, :so), do: true + defp tcp_types_ok?(:active, :passive), do: true + defp tcp_types_ok?(:passive, :active), do: true + defp tcp_types_ok?(_, _), do: false + defp symmetric?(ice_agent, socket, response_src, conn_check_pair) do local_cand = Map.fetch!(ice_agent.local_cands, conn_check_pair.local_cand_id) remote_cand = Map.fetch!(ice_agent.remote_cands, conn_check_pair.remote_cand_id) @@ -2275,7 +2293,12 @@ defmodule ExICE.Priv.ICEAgent do local_cand = conn_check_local_cand priority = - Candidate.priority!(ice_agent.local_preferences, local_cand.base.base_address, :prflx) + Candidate.priority!( + ice_agent.local_preferences, + local_cand.base.base_address, + :prflx, + local_cand.base.tcp_type + ) cand = Candidate.Prflx.new( @@ -2285,7 +2308,8 @@ defmodule ExICE.Priv.ICEAgent do base_port: local_cand.base.base_port, priority: priority, transport_module: ice_agent.transport_module, - socket: local_cand.base.socket + socket: local_cand.base.socket, + tcp_type: local_cand.base.tcp_type ) Logger.debug("Adding new local prflx candidate: #{inspect(cand)}") @@ -3094,12 +3118,18 @@ defmodule ExICE.Priv.ICEAgent do {:ok, {sock_addr, _sock_port}} = ice_agent.transport_module.sockname(local_candidate.base.socket) - Candidate.priority!(ice_agent.local_preferences, sock_addr, :prflx) + Candidate.priority!( + ice_agent.local_preferences, + sock_addr, + :prflx, + local_candidate.base.tcp_type + ) else Candidate.priority!( ice_agent.local_preferences, local_candidate.base.base_address, - :prflx + :prflx, + local_candidate.base.tcp_type ) end @@ -3130,7 +3160,7 @@ defmodule ExICE.Priv.ICEAgent do # we get an eperm error but retrying seems to help ¯\_(ツ)_/¯ Logger.debug(""" Couldn't send data to: #{inspect(dst_ip)}:#{dst_port}, reason: #{reason}, cand: #{inspect(local_cand)}. \ - Retyring...\ + Retrying...\ """) do_send(ice_agent, local_cand, dst, data, false) diff --git a/lib/ex_ice/priv/transport.ex b/lib/ex_ice/priv/transport.ex index bda06a4..de4468f 100644 --- a/lib/ex_ice/priv/transport.ex +++ b/lib/ex_ice/priv/transport.ex @@ -1,15 +1,24 @@ defmodule ExICE.Priv.Transport do @moduledoc false - @type socket() :: term() + @type socket :: term() + @type open_option :: + :inet.inet_backend() + | :inet.address_family() + | {:ip, :inet.socket_address()} + | :inet.socket_setopt() - @callback open(:inet.port_number(), [:gen_udp.open_option()]) :: + @callback transport() :: atom() + + @callback socket_configs() :: [map()] + + @callback setup_socket(:inet.ip_address(), :inet.port_number(), [open_option()], map()) :: {:ok, socket()} | {:error, term()} @callback sockname(socket()) :: {:ok, {:inet.ip_address(), :inet.port_number()}} | {:error, term()} - @callback send(socket(), {:inet.ip_address(), :inet.port_number()}, binary()) :: + @callback send(socket(), {:inet.ip_address(), :inet.port_number()}, binary(), Keyword.t()) :: :ok | {:error, term()} @callback close(socket()) :: :ok diff --git a/lib/ex_ice/priv/transport/tcp_client.ex b/lib/ex_ice/priv/transport/tcp_client.ex new file mode 100644 index 0000000..dc02759 --- /dev/null +++ b/lib/ex_ice/priv/transport/tcp_client.ex @@ -0,0 +1,330 @@ +defmodule ExICE.Priv.Transport.TCP.Client do + @moduledoc false + @behaviour ExICE.Priv.Transport + + use GenServer + require Logger + + @spec start_link(Keyword.t()) :: GenServer.on_start() + def start_link(opts \\ []) do + GenServer.start_link(__MODULE__, opts ++ [ice_agent: self()], name: __MODULE__) + end + + @impl ExICE.Priv.Transport + def transport, do: :tcp + + # Obtaining three candidates, as per RFC 6544, sec. 5.1. + @impl ExICE.Priv.Transport + def socket_configs, + do: [ + %{tcp_type: :passive}, + %{tcp_type: :so}, + %{tcp_type: :active} + ] + + @impl ExICE.Priv.Transport + def setup_socket(ip, port, socket_opts, tp_config) do + GenServer.call(__MODULE__, {:setup_socket, ip, port, socket_opts, tp_config}) + end + + @impl ExICE.Priv.Transport + defdelegate sockname(socket), to: :inet + + # HACK: using listen sockets here is ugly, but was easier to fit into the existing ICE Agent implementation. + # This should be changed, especially because we're going to want to close the listen sockets + # after the connection is successfully established. + @impl ExICE.Priv.Transport + def send(listen_socket, dest, packet, tp_opts \\ []) do + GenServer.call(__MODULE__, {:send, listen_socket, dest, packet, tp_opts}) + end + + @impl ExICE.Priv.Transport + def close(listen_socket) do + GenServer.call(__MODULE__, {:close, listen_socket}) + end + + @impl GenServer + def init(opts) do + {:ok, %{ref: make_ref(), ice_agent: Keyword.fetch!(opts, :ice_agent), sockets: %{}}} + end + + # This protects us from reusing ports ONLY within the same TCP client + # See below for more details + @impl GenServer + def handle_call({:setup_socket, ip, port, _, _}, _from, %{sockets: sockets} = state) + when is_map_key(sockets, {ip, port}) do + # TODO: Consider using another (custom) reason to distinguish from POSIX EADDRINUSE + {:reply, {:error, :eaddrinuse}, state} + end + + @impl GenServer + def handle_call({:setup_socket, ip, port, socket_opts, %{tcp_type: tcp_type}}, _from, state) do + # * For TCP ICE to work, in certain cases we need to be able to both listen + # and make connection attempts from the same socket. + # * The OS will not allow that unless we set SO_REUSEADDR/SO_REUSEPORT, which we do here. + # * However, this means the OS will no longer protect us from binding to the same address+port + # when we DON'T want that to happen. + # * If `ice_agent.ports == [0]`, this has no impact: the OS will use a different ephemeral port every time. + # * Otherwise, we run into a problem: the user has specified a port range (i.e. `50000..50500`), + # and we'd need to explicitly ask the OS to list currently bound sockets to determine + # whether we can use a certain port number from that range. + # * We try to alleviate this issue to an extent by checking against the sockets which we know of, + # those created in the past by this GenServer. + # * This, however, does not protect us from binding to sockets opened with SO_REUSEPORT, + # that are in use by another OS process with the same effective UID as this one, + # which can happen i.e. when we run ICE Agents in two separate VM instances. + # + # TODO: warn about this in the documentation of ICE Agent's `:ports` option + socket_opts = socket_opts ++ [ip: ip, reuseport: true] + + case :gen_tcp.listen(port, socket_opts) do + {:ok, listen_socket} -> + {:ok, {^ip, _sock_port} = local} = :inet.sockname(listen_socket) + + # Always claim the port, but don't accept incoming connections in :active mode + if tcp_type in [:passive, :so] do + pid = self() + spawn_link(fn -> acceptor_loop(listen_socket, pid) end) + end + + state = + put_in(state, [:sockets, local], %{ + listen_socket: listen_socket, + socket_opts: socket_opts, + tcp_type: tcp_type, + connections: %{} + }) + + {:reply, {:ok, listen_socket}, state} + + {:error, _reason} = err -> + {:reply, err, state} + end + end + + @impl GenServer + def handle_call({:send, listen_socket, dest, packet, tp_opts}, _from, state) do + {:ok, src} = :inet.sockname(listen_socket) + sock_state = state.sockets[src] + + state = + if sock_state.connections[dest] == nil and + Keyword.get(tp_opts, :connect?, sock_state.tcp_type in [:active, :so]) do + try_connect(state, src, dest, tp_opts) + else + state + end + + case state.sockets[src][:connections][dest] do + %{socket: socket, frame?: frame?} -> + {:reply, do_send(socket, packet, frame?), state} + + nil -> + if sock_state.tcp_type == :passive do + Logger.debug("Not sending data from a passive candidate that isn't connected") + # We're lying here to make the rest of the logic (kinda) work + {:reply, :ok, state} + else + {:reply, {:error, :enotconn}, state} + end + end + end + + @impl GenServer + def handle_call({:close, listen_socket}, _from, state) do + {:ok, local} = :inet.sockname(listen_socket) + + {sock_state, state} = pop_in(state, [:sockets, local]) + + case sock_state do + nil -> + Logger.debug("Socket already closed") + + %{listen_socket: listen_socket, connections: conn_states} -> + # TODO: revisit the closing logic + :gen_tcp.close(listen_socket) + Enum.each(conn_states, fn {_, %{socket: socket}} -> :gen_tcp.close(socket) end) + end + + {:reply, :ok, state} + end + + @impl GenServer + def handle_info({:connected, _listen_socket, socket}, state) do + {:ok, local} = :inet.sockname(socket) + {:ok, remote} = :inet.peername(socket) + + conn_state = %{ + socket: socket, + recv_buffer: <<>>, + frame?: true + } + + # TODO: we should probably ensure `local` is key in `state.sockets` + state = put_in(state, [:sockets, local, :connections, remote], conn_state) + + {:noreply, state} + end + + @impl GenServer + def handle_info({:tcp, socket, packet}, state) do + {:ok, local} = :inet.sockname(socket) + {:ok, {src_ip, src_port} = remote} = :inet.peername(socket) + + sock_state = state.sockets[local] + conn_state = sock_state.connections[remote] + + cond do + is_nil(conn_state) -> + # FIXME: this occasionally happens, and it shouldn't + Logger.warning("Received TCP data on unknown connection, dropping") + {:noreply, state} + + conn_state.frame? -> + # Framing according to RFC 4571 + previous = + case conn_state.recv_buffer do + nil -> <<>> + data -> data + end + + case previous <> packet do + <> -> + # HACK: this is dirty and means that, with framing, we're miscalculating + # the bytes_sent and bytes_received counters + send(state.ice_agent, {:udp, sock_state.listen_socket, src_ip, src_port, data}) + state = put_in(state, [:sockets, local, :connections, remote, :recv_buffer], <<>>) + + if rest != <<>> do + handle_info({:tcp, socket, rest}, state) + else + {:noreply, state} + end + + data -> + state = put_in(state, [:sockets, local, :connections, remote, :recv_buffer], data) + {:noreply, state} + end + + true -> + send(state.ice_agent, {:udp, sock_state.listen_socket, src_ip, src_port, packet}) + {:noreply, state} + end + end + + @impl GenServer + def handle_info({:tcp_closed, socket}, state) do + {src, dst} = find_by_socket(socket, state) + + {_, state} = pop_in(state, [:sockets, src, :connections, dst]) + + {:noreply, state} + end + + defp find_by_socket(socket, state) do + for {src, sock_state} <- state.sockets, + {dst, %{socket: connected_socket}} <- sock_state.connections do + {src, dst, connected_socket} + end + |> Enum.find_value(fn + {src, dst, ^socket} -> {src, dst} + _other -> nil + end) + end + + defp try_connect(state, local, remote, tp_opts) do + %{socket_opts: socket_opts, listen_socket: listen_socket} = state.sockets[local] + + {local_ip, local_port} = local + {remote_ip, remote_port} = remote + + # TODO: determine how big of a timeout we should use here + case :gen_tcp.connect(remote_ip, remote_port, socket_opts ++ [port: local_port], 500) do + {:ok, socket} -> + Logger.debug(""" + Successfully initiated new connection. + Local: #{inspect(local_ip)}:#{inspect(local_port)} + Remote: #{inspect(remote_ip)}:#{inspect(remote_port)} + Socket: #{inspect(socket)} + """) + + conn_state = %{ + socket: socket, + recv_buffer: <<>>, + frame?: Keyword.get(tp_opts, :frame?, true) + } + + put_in(state, [:sockets, local, :connections, remote], conn_state) + + {:error, :eaddrinuse} -> + # This happens with SO candidates, when the acceptor loop accepted the incoming connection already, + # but we have yet to process the relevant message + Logger.debug("Unable to initiate connection, we're already connected") + + receive do + {:connected, ^listen_socket, _} = msg -> + {:noreply, state} = handle_info(msg, state) + state + after + 50 -> state + end + + other -> + Logger.debug("Unable to initiate connection, reason: #{inspect(other)}") + state + end + end + + defp acceptor_loop(listen_socket, pid) do + {:ok, {sock_ip, sock_port}} = :inet.sockname(listen_socket) + + case :gen_tcp.accept(listen_socket) do + {:ok, socket} -> + :ok = :gen_tcp.controlling_process(socket, pid) + send(pid, {:connected, listen_socket, socket}) + + {:ok, {peer_ip, peer_port}} = :inet.peername(socket) + + Logger.debug(""" + Accepted new incoming connection. + Local: #{inspect(sock_ip)}:#{inspect(sock_port)} + Remote: #{inspect(peer_ip)}:#{inspect(peer_port)} + Listen socket: #{inspect(listen_socket)} + Socket: #{inspect(socket)} + """) + + acceptor_loop(listen_socket, pid) + + {:error, :closed} -> + Logger.debug(""" + TCP listen socket closed. + Local: #{inspect(sock_ip)}:#{inspect(sock_port)} + Listen socket: #{inspect(listen_socket)} + """) + + :ok + + # TODO: should we keep accepting in this case? + {:error, reason} -> + Logger.debug(""" + TCP listen socket accept failed with reason: #{inspect(reason)}. + Local: #{inspect(sock_ip)}:#{inspect(sock_port)} + Listen socket: #{inspect(listen_socket)} + """) + + acceptor_loop(listen_socket, pid) + end + end + + defp do_send(socket, packet, frame?) do + data = + if frame? do + # RFC 4571 + <> + else + packet + end + + :gen_tcp.send(socket, data) + end +end diff --git a/lib/ex_ice/priv/transport/udp.ex b/lib/ex_ice/priv/transport/udp.ex index 5393431..ab376b8 100644 --- a/lib/ex_ice/priv/transport/udp.ex +++ b/lib/ex_ice/priv/transport/udp.ex @@ -3,13 +3,21 @@ defmodule ExICE.Priv.Transport.UDP do @behaviour ExICE.Priv.Transport @impl true - defdelegate open(port, opts), to: :gen_udp + def transport, do: :udp + + @impl true + def socket_configs, do: [%{}] + + @impl true + def setup_socket(ip, port, socket_opts, _tp_opts) do + :gen_udp.open(port, socket_opts ++ [ip: ip]) + end @impl true defdelegate sockname(socket), to: :inet @impl true - defdelegate send(socket, dest, packet), to: :gen_udp + def send(socket, dest, packet, _tp_opts), do: :gen_udp.send(socket, dest, packet) @impl true defdelegate close(socket), to: :gen_udp diff --git a/mix.lock b/mix.lock index d6fbd93..50f4d94 100644 --- a/mix.lock +++ b/mix.lock @@ -1,10 +1,10 @@ %{ "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "credo": {:hex, :credo, "1.7.10", "6e64fe59be8da5e30a1b96273b247b5cf1cc9e336b5fd66302a64b25749ad44d", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "71fbc9a6b8be21d993deca85bf151df023a3097b01e09a2809d460348561d8cd"}, - "dialyxir": {:hex, :dialyxir, "1.4.5", "ca1571ac18e0f88d4ab245f0b60fa31ff1b12cbae2b11bd25d207f865e8ae78a", [:mix], [{:erlex, ">= 0.2.7", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b0fb08bb8107c750db5c0b324fa2df5ceaa0f9307690ee3c1f6ba5b9eb5d35c3"}, + "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"}, "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"}, - "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, + "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"}, "ex_doc": {:hex, :ex_doc, "0.35.1", "de804c590d3df2d9d5b8aec77d758b00c814b356119b3d4455e4b8a8687aecaf", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "2121c6402c8d44b05622677b761371a759143b958c6c19f6558ff64d0aed40df"}, "ex_stun": {:hex, :ex_stun, "0.2.0", "feb1fc7db0356406655b2a617805e6c712b93308c8ea2bf0ba1197b1f0866deb", [:mix], [], "hexpm", "1e01ba8290082ccbf37acaa5190d1f69b51edd6de2026a8d6d51368b29d115d0"}, "ex_turn": {:hex, :ex_turn, "0.2.0", "4e1f9b089e9a5ee44928d12370cc9ea7a89b84b2f6256832de65271212eb80de", [:mix], [{:ex_stun, "~> 0.2.0", [hex: :ex_stun, repo: "hexpm", optional: false]}], "hexpm", "08e884f0af2c4a147e3f8cd4ffe33e3452a256389f0956e55a8c4d75bf0e74cd"},