Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/run_test_case.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ jobs:
fail-fast: false
matrix:
otp:
- ghcr.io/emqx/emqx-builder/6.2-4:1.19.1-28.4.1-9-ubuntu24.04
- erlang:28
- erlang:27
- erlang:26
Expand Down
9 changes: 9 additions & 0 deletions examples/async_recv/async_recv_echo_server.erl
Original file line number Diff line number Diff line change
Expand Up @@ -86,13 +86,22 @@ handle_info({inet_async, _Sock, _Ref, {error, Reason}}, State) ->
io:format("Shutdown for ~p~n", [Reason]),
shutdown(Reason, State);

%% async_send/2 replies on OTP 25 and older
handle_info({inet_reply, _Sock ,ok}, State) ->
{noreply, State};

handle_info({inet_reply, _Sock, {error, Reason}}, State) ->
io:format("Shutdown for ~p~n", [Reason]),
shutdown(Reason, State);

%% async_send/2 replies on OTP 26 and later
handle_info({inet_reply, _Sock, ok, send_async}, State) ->
{noreply, State};

handle_info({inet_reply, _Sock, {error, Reason}, send_async}, State) ->
io:format("Shutdown for ~p~n", [Reason]),
shutdown(Reason, State);

handle_info(_Info, State) ->
{noreply, State}.

Expand Down
4 changes: 4 additions & 0 deletions examples/dtls/dtls_echo_server.erl
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,12 @@ run_loop(Peername, Transport, Socket) ->
io:format("~s - enter the passive mode.~n", [esockd:format(Peername)]),
Transport:setopts(Socket, [{active, 100}]),
run_loop(Peername, Transport, Socket);
%% async_send/2 reply on OTP 25 and older
{inet_reply, _RawSocket, ok} ->
run_loop(Peername, Transport, Socket);
%% async_send/2 reply on OTP 26 and later
{inet_reply, _RawSocket, ok, send_async} ->
run_loop(Peername, Transport, Socket);
Comment thread
zmstone marked this conversation as resolved.
Resolved
{ssl_closed, _RawSocket} ->
ok;
{ssl_error, _RawSocket, Reason} ->
Expand Down
76 changes: 63 additions & 13 deletions src/esockd_transport.erl
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,18 @@

-export_type([socket/0, listen_socket/0]).

%% Since OTP 26, prim_inet:send/3 prefixes the data with a monitor
%% reference, and the inet driver replies with {inet_reply, Sock, Status,
%% Ref}. A plain erlang:port_command/3 no longer works on a TCP port, so
%% async_send/3 on OTP 26 and later sends with gen_tcp:send/2 or
%% prim_inet:send_async/3, and the reply carries a fourth element.
-if(?OTP_RELEASE >= 26).
Comment thread
zmstone marked this conversation as resolved.
Resolved
-define(ASYNC_SEND_REPLY(Sock, Status), {inet_reply, Sock, Status, send_async}).
-define(SEND_ASYNC_MODULE_KEY, {?MODULE, send_async_module}).
-else.
-define(ASYNC_SEND_REPLY(Sock, Status), {inet_reply, Sock, Status}).
-endif.

-type(ssl_socket() :: #ssl_socket{}).
-type(proxy_socket() :: #proxy_socket{}).
-type(listen_socket() :: inet:socket()).
Expand Down Expand Up @@ -150,21 +162,33 @@ send(#ssl_socket{ssl = SslSock}, Data) ->
send(#proxy_socket{socket = Sock}, Data) ->
send(Sock, Data).

%% @doc Port command to write data.
%% @doc Send data and return without waiting for the result.
%%
%% The result arrives later as a message to the caller, one message for
%% each `ok' return, in send order. On OTP 26 and later the message is
%% `{inet_reply, Sock, ok | {error, Reason}, send_async}'. On older OTP
%% releases it is `{inet_reply, Sock, ok | {error, Reason}}'.
%%
%% For a TCP port, the send is asynchronous when the running OTP exports
%% `prim_inet:send_async/3' (EMQX OTP builds). With the `nosuspend' option
%% and a busy port, the function then returns `{error, busy}' and queues
%% nothing: the caller can send the data again, for example with
%% `send/2'. When `prim_inet:send_async/3' is not exported (OTP 26 and
%% later), the function sends with `gen_tcp:send/2' and sends the reply
%% message to the caller itself. TLS sockets always send synchronously.
-spec(async_send(socket(), iodata()) -> ok | {error, Reason} when
Reason :: close | timeout | inet:posix()).
Reason :: busy | closed | timeout | inet:posix()).
async_send(Sock, Data) ->
async_send(Sock, Data, []).

-spec(async_send(socket(), iodata(), [force|nosuspend]) -> ok | {error, Reason} when
Reason :: close | timeout | inet:posix()).
Reason :: busy | closed | timeout | inet:posix()).
async_send(Sock, Data, Options) when is_port(Sock) ->
do_port_command(Sock, Data, Options);
async_send(Sock = #ssl_socket{ssl = SslSock}, Data, _Options) ->
case ssl:send(SslSock, Data) of
ok ->
%% TODO: get rid of this message send
self() ! {inet_reply, Sock, ok},
self() ! ?ASYNC_SEND_REPLY(Sock, ok),
ok;
Error ->
Error
Expand All @@ -173,14 +197,40 @@ async_send(#proxy_socket{socket = Sock}, Data, _Options) ->
async_send(Sock, Data).

-if(?OTP_RELEASE >= 26).
do_port_command(Sock, Data, _Options) ->
case gen_tcp:send(Sock, Data) of
ok ->
%% TODO: get rid of this message send
self() ! {inet_reply, Sock, ok},
ok;
{error, Reason} ->
{error, Reason}
do_port_command(Sock, Data, Options) ->
case send_async_module() of
undefined ->
case gen_tcp:send(Sock, Data) of
ok ->
self() ! ?ASYNC_SEND_REPLY(Sock, ok),
ok;
{error, Reason} ->
{error, Reason}
end;
Mod ->
%% The driver replies with `{inet_reply, Sock, Status, send_async}'.
Mod:send_async(Sock, Data, Options)
end.

%% Return `prim_inet' when it exports `send_async/3', else `undefined'.
%% The result is computed once and kept in a persistent term. The module
%% name is a runtime value, so that xref and dialyzer accept the call on
%% an OTP without the function.
send_async_module() ->
case persistent_term:get(?SEND_ASYNC_MODULE_KEY, unknown) of
unknown ->
Mod = probe_send_async_module(prim_inet),
persistent_term:put(?SEND_ASYNC_MODULE_KEY, Mod),
Mod;
Mod ->
Mod
end.

probe_send_async_module(Mod) ->
_ = code:ensure_loaded(Mod),
case erlang:function_exported(Mod, send_async, 3) of
true -> Mod;
false -> undefined
end.
-else.
do_port_command(Sock, Data, Options) ->
Expand Down
6 changes: 6 additions & 0 deletions test/async_echo_server.erl
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,15 @@ loop(Transport, Sock) ->
loop(Transport, Sock);
{inet_async, _Sock, Ref, {error, Reason}} ->
exit(Reason);
%% async_send/2 replies on OTP 25 and older
{inet_reply, _Sock ,ok} ->
loop(Transport, Sock);
{inet_reply, _Sock, {error, Reason}} ->
exit(Reason);
%% async_send/2 replies on OTP 26 and later
{inet_reply, _Sock, ok, send_async} ->
loop(Transport, Sock);
{inet_reply, _Sock, {error, Reason}, send_async} ->
exit(Reason)
end.

4 changes: 4 additions & 0 deletions test/dtls_echo_server.erl
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,11 @@ run_loop(Parent, Transport, Socket) ->
{ssl, _RawSocket, Packet} ->
Transport:async_send(Socket, Packet),
run_loop(Parent, Transport, Socket);
%% async_send/2 reply on OTP 25 and older
{inet_reply, _Socket, ok} ->
run_loop(Parent, Transport, Socket);
%% async_send/2 reply on OTP 26 and later
{inet_reply, _Socket, ok, send_async} ->
run_loop(Parent, Transport, Socket)
end.

69 changes: 69 additions & 0 deletions test/esockd_transport_SUITE.erl
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,75 @@ t_async_send(_) ->
ok = esockd_transport:close(Sock),
ok = esockd:close(echo, 3000).

-if(?OTP_RELEASE >= 26).
t_async_send_reply(_) ->
{S, C} = tcp_pair(),
ok = esockd_transport:async_send(S, <<"Hello">>),
receive {inet_reply, S, ok, send_async} -> ok
after 1000 -> ct:fail(no_inet_reply)
end,
{ok, <<"Hello">>} = gen_tcp:recv(C, 5, 1000),
ok = gen_tcp:close(S),
ok = gen_tcp:close(C).

%% The fallback sends with gen_tcp:send/2 and sends the reply itself.
t_async_send_fallback(_) ->
persistent_term:put({esockd_transport, send_async_module}, undefined),
try
{S, C} = tcp_pair(),
ok = esockd_transport:async_send(S, <<"Hello">>, [nosuspend]),
receive {inet_reply, S, ok, send_async} -> ok
after 0 -> ct:fail(no_inet_reply)
end,
{ok, <<"Hello">>} = gen_tcp:recv(C, 5, 1000),
ok = gen_tcp:close(S),
{error, closed} = esockd_transport:async_send(S, <<"Hello">>),
ok = gen_tcp:close(C)
after
persistent_term:erase({esockd_transport, send_async_module})
end.

%% With prim_inet:send_async/3, a busy port returns {error, busy} with
%% nosuspend. The replies of the queued sends arrive after the peer
%% reads the data.
t_async_send_prim_inet_send_async(_) ->
case erlang:function_exported(prim_inet, send_async, 3) of
false ->
{skip, no_prim_inet_send_async};
true ->
persistent_term:erase({esockd_transport, send_async_module}),
{S, C} = tcp_pair(),
ok = inet:setopts(S, [{high_watermark, 4096}, {low_watermark, 1024},
{sndbuf, 4096}]),
ok = inet:setopts(C, [{recbuf, 4096}]),
Chunk = binary:copy(<<"x">>, 16384),
NSent = fill_until_busy(S, Chunk, 0),
prim_inet = persistent_term:get({esockd_transport, send_async_module}),
{ok, _} = gen_tcp:recv(C, NSent * byte_size(Chunk), 10000),
[receive {inet_reply, S, ok, send_async} -> ok
after 5000 -> ct:fail(no_inet_reply)
end || _ <- lists:seq(1, NSent)],
ok = gen_tcp:close(S),
ok = gen_tcp:close(C)
end.

fill_until_busy(_S, _Chunk, 1000) ->
ct:fail(port_never_busy);
fill_until_busy(S, Chunk, N) ->
case esockd_transport:async_send(S, Chunk, [nosuspend]) of
ok -> fill_until_busy(S, Chunk, N + 1);
{error, busy} -> N
end.

tcp_pair() ->
{ok, L} = gen_tcp:listen(0, [binary, {active, false}]),
{ok, Port} = inet:port(L),
{ok, C} = gen_tcp:connect({127,0,0,1}, Port, [binary, {active, false}]),
{ok, S} = gen_tcp:accept(L, 5000),
ok = gen_tcp:close(L),
{S, C}.
-endif.

t_async_recv(_) ->
{ok, _} = esockd:open(echo, 3000, [{tcp_options, ?TCP_OPTS}], {async_echo_server, start_link, []}),
{ok, Sock} = gen_tcp:connect({127,0,0,1}, 3000, [binary, {active, false}]),
Expand Down
Loading