Connection pool: implement logging and telemetry events

This commit is contained in:
rinpatch 2020-07-15 15:26:25 +03:00
parent 0ffde499b8
commit 7738fbbaf5
3 changed files with 100 additions and 7 deletions

View File

@ -39,6 +39,7 @@ def start(_type, _args) do
# every time the application is restarted, so we disable module # every time the application is restarted, so we disable module
# conflicts at runtime # conflicts at runtime
Code.compiler_options(ignore_module_conflict: true) Code.compiler_options(ignore_module_conflict: true)
Pleroma.Telemetry.Logger.attach()
Config.Holder.save_default() Config.Holder.save_default()
Pleroma.HTML.compile_scrubbers() Pleroma.HTML.compile_scrubbers()
Config.DeprecationWarnings.warn() Config.DeprecationWarnings.warn()

View File

@ -18,8 +18,12 @@ def start_worker(opts) do
case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do
{:error, :max_children} -> {:error, :max_children} ->
case free_pool() do case free_pool() do
:ok -> start_worker(opts) :ok ->
:error -> {:error, :pool_full} start_worker(opts)
:error ->
:telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts})
{:error, :pool_full}
end end
res -> res ->
@ -44,6 +48,14 @@ defp free_pool do
|> round |> round
|> max(1) |> max(1)
:telemetry.execute([:pleroma, :connection_pool, :reclaim, :start], %{}, %{
max_connections: max_connections,
reclaim_max: reclaim_max
})
# :ets.fun2ms(
# fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] ->
# {worker_pid, crf, last_reference} end)
unused_conns = unused_conns =
Registry.select( Registry.select(
@registry, @registry,
@ -55,17 +67,35 @@ defp free_pool do
case unused_conns do case unused_conns do
[] -> [] ->
:telemetry.execute(
[:pleroma, :connection_pool, :reclaim, :stop],
%{reclaimed_count: 0},
%{
max_connections: max_connections
}
)
exit(:no_unused_conns) exit(:no_unused_conns)
unused_conns -> unused_conns ->
unused_conns reclaimed =
|> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} -> unused_conns
crf1 <= crf2 and last_reference1 <= last_reference2 |> Enum.sort(fn {_pid1, crf1, last_reference1},
end) {_pid2, crf2, last_reference2} ->
|> Enum.take(reclaim_max) crf1 <= crf2 and last_reference1 <= last_reference2
end)
|> Enum.take(reclaim_max)
reclaimed
|> Enum.each(fn {pid, _, _} -> |> Enum.each(fn {pid, _, _} ->
DynamicSupervisor.terminate_child(__MODULE__, pid) DynamicSupervisor.terminate_child(__MODULE__, pid)
end) end)
:telemetry.execute(
[:pleroma, :connection_pool, :reclaim, :stop],
%{reclaimed_count: Enum.count(reclaimed)},
%{max_connections: max_connections}
)
end end
end) end)

View File

@ -0,0 +1,62 @@
defmodule Pleroma.Telemetry.Logger do
@moduledoc "Transforms Pleroma telemetry events to logs"
require Logger
@events [
[:pleroma, :connection_pool, :reclaim, :start],
[:pleroma, :connection_pool, :reclaim, :stop],
[:pleroma, :connection_pool, :provision_failure]
]
def attach do
:telemetry.attach_many("pleroma-logger", @events, &handle_event/4, [])
end
# Passing anonymous functions instead of strings to logger is intentional,
# that way strings won't be concatenated if the message is going to be thrown
# out anyway due to higher log level configured
def handle_event(
[:pleroma, :connection_pool, :reclaim, :start],
_,
%{max_connections: max_connections, reclaim_max: reclaim_max},
_
) do
Logger.debug(fn ->
"Connection pool is exhausted (reached #{max_connections} connections). Starting idle connection cleanup to reclaim as much as #{
reclaim_max
} connections"
end)
end
def handle_event(
[:pleroma, :connection_pool, :reclaim, :stop],
%{reclaimed_count: 0},
_,
_
) do
Logger.error(fn ->
"Connection pool failed to reclaim any connections due to all of them being in use. It will have to drop requests for opening connections to new hosts"
end)
end
def handle_event(
[:pleroma, :connection_pool, :reclaim, :stop],
%{reclaimed_count: reclaimed_count},
_,
_
) do
Logger.debug(fn -> "Connection pool cleaned up #{reclaimed_count} idle connections" end)
end
def handle_event(
[:pleroma, :connection_pool, :provision_failure],
%{opts: [key | _]},
_,
_
) do
Logger.error(fn ->
"Connection pool had to refuse opening a connection to #{key} due to connection limit exhaustion"
end)
end
end