Memory spike when calling ssl:send() with a big binary

Questions / Help
1

Hi, I need to send an HTTP POST request with a big binary blob (up to 10MB). I’m doing this from Elixir via an HTTP client library, but at the end of the day the code calls ssl:send/2. This is causing a memory usage spike which forces me to turn down the concurrency since I’m running on limited memory. I traced how my HTTP client uses the ssl module and wrote it down in code (see below). The interesting thing is that if I manually chunk the binary, the memory spike goes away. Please excuse the fact that it’s in Elixir, but I’m sure you’ll get the gist of it:

defmodule SslTest do
  def run() do
    host = "localhost"
    port = 4040
    method = "POST"
    path = "/test/text"
    content_length = 10 * 1024 * 1024
    ssl_opts = [verify: :verify_none]

    body = String.duplicate("-", content_length)
    chunks = [body]
    # chunks = chunk_binary(body, 1024 * 10)

    IO.puts("waiting 3 seconds...")
    Process.sleep(3000)

    headers = [
      {"host", "#{host}:#{port}"},
      {"content-type", "text/plain"},
      {"content-length", content_length}
    ]

    {:ok, socket} =
      :ssl.connect(
        String.to_charlist(host),
        port,
        ssl_opts(host, ssl_opts)
      )

    request = [
      "#{method} #{path} HTTP/1.1\r\n",
      Enum.map(headers, fn {name, value} -> "#{name}: #{value}\r\n" end),
      "\r\n"
    ]

    :ok = :ssl.send(socket, request)

    for chunk <- chunks do
      :ok = :ssl.send(socket, chunk)
    end

    Process.sleep(500)
    {:ok, response} = :ssl.recv(socket, 0, 0)
    :ok = :ssl.close(socket)

    IO.puts(response)
  end

  def ssl_opts(hostname, opts) do
    [
      server_name_indication: String.to_charlist(hostname),
      versions: [:"tlsv1.3", :"tlsv1.2"],
      depth: 4,
      secure_renegotiate: true,
      reuse_sessions: true,
      packet: :raw,
      mode: :binary,
      active: false,
      ciphers: [
        %{cipher: :aes_256_gcm, key_exchange: :any, mac: :aead, prf: :sha384},
        %{cipher: :aes_128_gcm, key_exchange: :any, mac: :aead, prf: :sha256},
        %{cipher: :chacha20_poly1305, key_exchange: :any, mac: :aead, prf: :sha256},
        %{cipher: :aes_128_ccm, key_exchange: :any, mac: :aead, prf: :sha256},
        %{cipher: :aes_128_ccm_8, key_exchange: :any, mac: :aead, prf: :sha256},
        %{
          cipher: :aes_256_gcm,
          key_exchange: :ecdhe_ecdsa,
          mac: :aead,
          prf: :sha384
        },
        %{cipher: :aes_256_gcm, key_exchange: :ecdhe_rsa, mac: :aead, prf: :sha384},
        %{
          cipher: :aes_256_ccm,
          key_exchange: :ecdhe_ecdsa,
          mac: :aead,
          prf: :default_prf
        },
        %{
          cipher: :aes_256_ccm_8,
          key_exchange: :ecdhe_ecdsa,
          mac: :aead,
          prf: :default_prf
        },
        %{
          cipher: :chacha20_poly1305,
          key_exchange: :ecdhe_ecdsa,
          mac: :aead,
          prf: :sha256
        },
        %{
          cipher: :chacha20_poly1305,
          key_exchange: :ecdhe_rsa,
          mac: :aead,
          prf: :sha256
        },
        %{
          cipher: :aes_128_gcm,
          key_exchange: :ecdhe_ecdsa,
          mac: :aead,
          prf: :sha256
        },
        %{cipher: :aes_128_gcm, key_exchange: :ecdhe_rsa, mac: :aead, prf: :sha256},
        %{
          cipher: :aes_128_ccm,
          key_exchange: :ecdhe_ecdsa,
          mac: :aead,
          prf: :default_prf
        },
        %{
          cipher: :aes_128_ccm_8,
          key_exchange: :ecdhe_ecdsa,
          mac: :aead,
          prf: :default_prf
        },
        %{cipher: :aes_256_gcm, key_exchange: :dhe_rsa, mac: :aead, prf: :sha384},
        %{cipher: :aes_256_gcm, key_exchange: :dhe_dss, mac: :aead, prf: :sha384},
        %{
          cipher: :aes_256_cbc,
          key_exchange: :dhe_rsa,
          mac: :sha256,
          prf: :default_prf
        },
        %{
          cipher: :aes_256_cbc,
          key_exchange: :dhe_dss,
          mac: :sha256,
          prf: :default_prf
        },
        %{cipher: :aes_128_gcm, key_exchange: :dhe_rsa, mac: :aead, prf: :sha256},
        %{cipher: :aes_128_gcm, key_exchange: :dhe_dss, mac: :aead, prf: :sha256},
        %{
          cipher: :chacha20_poly1305,
          key_exchange: :dhe_rsa,
          mac: :aead,
          prf: :sha256
        },
        %{
          cipher: :aes_128_cbc,
          key_exchange: :dhe_rsa,
          mac: :sha256,
          prf: :default_prf
        },
        %{
          cipher: :aes_128_cbc,
          key_exchange: :dhe_dss,
          mac: :sha256,
          prf: :default_prf
        },
        %{
          cipher: :aes_256_cbc,
          key_exchange: :ecdhe_ecdsa,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :aes_256_cbc,
          key_exchange: :ecdhe_rsa,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :aes_256_cbc,
          key_exchange: :ecdh_ecdsa,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :aes_256_cbc,
          key_exchange: :ecdh_rsa,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :aes_128_cbc,
          key_exchange: :ecdhe_ecdsa,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :aes_128_cbc,
          key_exchange: :ecdhe_rsa,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :aes_128_cbc,
          key_exchange: :ecdh_ecdsa,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :aes_128_cbc,
          key_exchange: :ecdh_rsa,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :aes_256_cbc,
          key_exchange: :dhe_rsa,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :aes_256_cbc,
          key_exchange: :dhe_dss,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :aes_128_cbc,
          key_exchange: :dhe_rsa,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :aes_128_cbc,
          key_exchange: :dhe_dss,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :aes_128_cbc,
          key_exchange: :rsa_psk,
          mac: :sha256,
          prf: :default_prf
        },
        %{
          cipher: :aes_256_cbc,
          key_exchange: :rsa_psk,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :aes_128_cbc,
          key_exchange: :rsa_psk,
          mac: :sha,
          prf: :default_prf
        },
        %{cipher: :rc4_128, key_exchange: :rsa_psk, mac: :sha, prf: :default_prf},
        %{
          cipher: :aes_256_cbc,
          key_exchange: :srp_rsa,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :aes_256_cbc,
          key_exchange: :srp_dss,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :aes_128_cbc,
          key_exchange: :srp_rsa,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :aes_128_cbc,
          key_exchange: :srp_dss,
          mac: :sha,
          prf: :default_prf
        },
        %{cipher: :aes_256_cbc, key_exchange: :rsa, mac: :sha256, prf: :default_prf},
        %{cipher: :aes_128_cbc, key_exchange: :rsa, mac: :sha256, prf: :default_prf},
        %{cipher: :aes_256_cbc, key_exchange: :rsa, mac: :sha, prf: :default_prf},
        %{cipher: :aes_128_cbc, key_exchange: :rsa, mac: :sha, prf: :default_prf},
        %{cipher: :"3des_ede_cbc", key_exchange: :rsa, mac: :sha, prf: :default_prf},
        %{
          cipher: :"3des_ede_cbc",
          key_exchange: :ecdhe_ecdsa,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :"3des_ede_cbc",
          key_exchange: :ecdhe_rsa,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :"3des_ede_cbc",
          key_exchange: :dhe_rsa,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :"3des_ede_cbc",
          key_exchange: :dhe_dss,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :"3des_ede_cbc",
          key_exchange: :ecdh_ecdsa,
          mac: :sha,
          prf: :default_prf
        },
        %{
          cipher: :"3des_ede_cbc",
          key_exchange: :ecdh_rsa,
          mac: :sha,
          prf: :default_prf
        },
        %{cipher: :des_cbc, key_exchange: :dhe_rsa, mac: :sha, prf: :default_prf},
        %{cipher: :des_cbc, key_exchange: :rsa, mac: :sha, prf: :default_prf},
        %{
          cipher: :rc4_128,
          key_exchange: :ecdhe_ecdsa,
          mac: :sha,
          prf: :default_prf
        },
        %{cipher: :rc4_128, key_exchange: :ecdhe_rsa, mac: :sha, prf: :default_prf},
        %{cipher: :rc4_128, key_exchange: :ecdh_ecdsa, mac: :sha, prf: :default_prf},
        %{cipher: :rc4_128, key_exchange: :ecdh_rsa, mac: :sha, prf: :default_prf},
        %{cipher: :rc4_128, key_exchange: :rsa, mac: :sha, prf: :default_prf},
        %{cipher: :rc4_128, key_exchange: :rsa, mac: :md5, prf: :default_prf}
      ]
    ]
    |> Keyword.merge(opts)
  end

  def chunk_binary(str, size \\ 102_400, acc \\ []) do
    case String.split_at(str, size) do
      {slice, ""} -> Enum.reverse([slice | acc])
      {slice, rest} -> chunk_binary(rest, size, [slice | acc])
    end
  end
end

(I wanted to upload a screenshot from Observer here, but I’m not yet allowed to post images, sorry!)

The data that I’m sending over the wire is actually JSON as iodata, but one of the values is this big binary blob encoded with base64.

I don’t have much experience with working with sockets directly, nor with the ssl module, but my assumption was that this would just work without me needing to chunk it manually. What am I missing here?

EDIT: This is a cross-post from the Elixir Forum. Here’s the original thread.

1 Like
2

First, I don’t have any experience with the ssl module

I guess memory come from

Are you testing it in a new Elixir node every times?

1 Like
3

I’m observing the same behaviour regardless if I start a new VM each time or do it repeatedly from the same one.

1 Like
4

When you’re manually chunking you’re not creating another whole 10MB message when sending, so you should expect to see a smoother memory usage profile.

If you’re sending the entire payload, you’re sending a copy of the payload to the ssl server for sending. Hence what I’m thinking is the memory spike you’re seeing.

That sound about right? If so, the latter case is expected behavior and not an anomaly.

As far as I understand you may be able to prevent this by using a single binary for the payload, and not an iolist, since you’d not be sending a full copy of the data as the message to the ssl server, but a reference. See Refc Binaries

Hopefully this helps.

2 Likes
5

from ssl

ssl code

send(#sslsocket{pid = [Pid]}, Data) when is_pid(Pid) →
ssl_gen_statem:send(Pid, Data);
send(#sslsocket{pid = [, Pid]}, Data) when is_pid(Pid) →
tls_sender:send_data(Pid, erlang:iolist_to_iovec(Data));
send(#sslsocket{pid = {, #config{transport_info={_, udp, _, _}}}}, ) →
{error,enotconn}; %% Emulate connection behaviour
send(#sslsocket{pid = {dtls,}}, _) →
{error,enotconn}; %% Emulate connection behaviour
send(#sslsocket{pid = {ListenSocket, #config{transport_info = Info}}}, Data) →
Transport = element(1, Info),
tls_socket:send(Transport, ListenSocket, Data). %% {error,enotconn}

seem already call erlang:iolist_to_iovec(Data) to make binary ref @LeonardB

I can’t try out your memory spike without ssl, in my memory spike is come from making chunks.
BTW, split chunks using less memory but cost more time(obvious?)

testing code

-module(test).

-compile([export_all, nowarn_export_all]).

run() →
spawn(fun() →
Memory1 = erlang:memory(total),
% String.duplicate(“-”, content_length)
{CostTime1, Data} = timer:tc(fun() → lists:duplicate(10 * 1024 * 1024, $_) end),
Memory2 = erlang:memory(total),
%% tls_sender:send_data(Pid, erlang:iolist_to_iovec(Data))
{CostTime2, _Data1} = timer:tc(fun() → erlang:iolist_to_iovec(Data) end),
Memory3 = erlang:memory(total),
io:format(“make data memory add: ~.3fMB, use ~.1f ms~niolist_to_iovec memory add: ~.3fMB, use ~.1f ms~n”,
[(Memory2 - Memory1) / 1024 / 1024, CostTime1 / 1000, (Memory3 - Memory2) / 1024 / 1024, CostTime2 / 1000])
end).

run_split() →
spawn(fun() →
Memory1 = erlang:memory(total),
% chunks = chunk_binary(body, 1024 * 10)
{CostTime1, DataL} = timer:tc(fun() → [lists:duplicate(10 * 1024, $_) || _ ← lists:seq(1, 1024)] end),
Memory2 = erlang:memory(total),
%% tls_sender:send_data(Pid, erlang:iolist_to_iovec(Data))
{CostTime2, _DataL1} = timer:tc(fun() →
lists:foreach(fun(Data) →
erlang:iolist_to_iovec(Data)
end, DataL)
end),
Memory3 = erlang:memory(total),
io:format(“make data memory add: ~.3fMB, use ~.1f ms~niolist_to_iovec memory add: ~.3fMB, use ~.1f ms~nn”,
[(Memory2 - Memory1) / 1024 / 1024, CostTime1 / 1000, (Memory3 - Memory2) / 1024 / 1024, CostTime2 / 1000])
end).

testing output

4> test:run().
<0.93.0>
make data memory add: 169.191MB, use 449.7 ms
iolist_to_iovec memory add: 71.169MB, use 57.7 ms
5> test:run_split().
<0.95.0>
make data memory add: 169.184MB, use 488.6 ms
iolist_to_iovec memory add: 34.022MB, use 610.8 ms

but, maybe it’s helpless :smiling_face_with_tear:

1 Like