From 75290cbfbd0f93cf3ecec5f44c4624b8c8601c51 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 26 Sep 2019 18:49:57 +0700 Subject: [PATCH] Add Pleroma.JobQueueMonitor --- lib/pleroma/application.ex | 1 + lib/pleroma/healthcheck.ex | 8 ++ lib/pleroma/job_queue_monitor.ex | 115 +++++++++++++++++++++++++++ lib/pleroma/workers/worker_helper.ex | 1 + test/healthcheck_test.exs | 9 ++- 5 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/job_queue_monitor.ex diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 7aec2c545..3e21d4403 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -42,6 +42,7 @@ def start(_type, _args) do hackney_pool_children() ++ [ Pleroma.Stats, + Pleroma.JobQueueMonitor, {Oban, Pleroma.Config.get(Oban)} ] ++ task_children(@env) ++ diff --git a/lib/pleroma/healthcheck.ex b/lib/pleroma/healthcheck.ex index 977b78c26..fc2129815 100644 --- a/lib/pleroma/healthcheck.ex +++ b/lib/pleroma/healthcheck.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Healthcheck do active: 0, idle: 0, memory_used: 0, + job_queue_stats: nil, healthy: true @type t :: %__MODULE__{ @@ -21,6 +22,7 @@ defmodule Pleroma.Healthcheck do active: non_neg_integer(), idle: non_neg_integer(), memory_used: number(), + job_queue_stats: map(), healthy: boolean() } @@ -30,6 +32,7 @@ def system_info do memory_used: Float.round(:erlang.memory(:total) / 1024 / 1024, 2) } |> assign_db_info() + |> assign_job_queue_stats() |> check_health() end @@ -55,6 +58,11 @@ defp assign_db_info(healthcheck) do Map.merge(healthcheck, db_info) end + defp assign_job_queue_stats(healthcheck) do + stats = Pleroma.JobQueueMonitor.stats() + Map.put(healthcheck, :job_queue_stats, stats) + end + @spec check_health(Healthcheck.t()) :: Healthcheck.t() def check_health(%{pool_size: pool_size, active: active} = check) when active >= pool_size do diff --git a/lib/pleroma/job_queue_monitor.ex b/lib/pleroma/job_queue_monitor.ex new file mode 100644 index 000000000..685ba2ead --- /dev/null +++ b/lib/pleroma/job_queue_monitor.ex @@ -0,0 +1,115 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.JobQueueMonitor do + use GenServer + + @initial_state %{workers: %{}, queues: %{}, processed_jobs: 0, enqueued: 0} + @queue %{processed_jobs: 0, success: 0, failure: 0, enqueued: 0} + @operation %{processed_jobs: 0, success: 0, failure: 0, enqueued: 0} + + def start_link(_) do + GenServer.start_link(__MODULE__, @initial_state, name: __MODULE__) + end + + @impl true + def init(state) do + :telemetry.attach("oban-monitor-failure", [:oban, :failure], &handle_event/4, nil) + :telemetry.attach("oban-monitor-success", [:oban, :success], &handle_event/4, nil) + + {:ok, state} + end + + def stats do + GenServer.call(__MODULE__, :stats) + end + + def enqueue({:ok, job}) do + meta = Map.take(job, [:args, :queue, :worker]) + GenServer.cast(__MODULE__, {:process_enqueue, meta}) + + {:ok, job} + end + + def enqueue(result), do: result + + def handle_event([:oban, status], %{duration: duration}, meta, _) do + GenServer.cast(__MODULE__, {:process_event, status, duration, meta}) + end + + @impl true + def handle_call(:stats, _from, state) do + {:reply, state, state} + end + + def handle_cast({:process_enqueue, meta}, state) do + state = + state + |> Map.update!(:workers, fn workers -> + workers + |> Map.put_new(meta.worker, %{}) + |> Map.update!(meta.worker, &update_worker(&1, :enqueue, meta)) + end) + |> Map.update!(:queues, fn workers -> + workers + |> Map.put_new(meta.queue, @queue) + |> Map.update!(meta.queue, fn queue -> Map.update!(queue, :enqueued, &(&1 + 1)) end) + end) + |> Map.update!(:enqueued, &(&1 + 1)) + + {:noreply, state} + end + + @impl true + def handle_cast({:process_event, status, duration, meta}, state) do + state = + state + |> Map.update!(:workers, fn workers -> + workers + |> Map.put_new(meta.worker, %{}) + |> Map.update!(meta.worker, &update_worker(&1, status, meta, duration)) + end) + |> Map.update!(:queues, fn workers -> + workers + |> Map.put_new(meta.queue, @queue) + |> Map.update!(meta.queue, &update_queue(&1, status, meta, duration)) + end) + |> Map.update!(:processed_jobs, &(&1 + 1)) + |> decr_enqueued() + + {:noreply, state} + end + + defp update_worker(worker, status, meta, duration \\ 0) do + worker + |> Map.put_new(meta.args["op"], @operation) + |> Map.update!(meta.args["op"], &update_op(&1, status, meta, duration)) + end + + defp update_op(op, :enqueue, _meta, _duration) do + op + |> Map.update!(:enqueued, &(&1 + 1)) + end + + defp update_op(op, status, _meta, _duration) do + op + |> Map.update!(:processed_jobs, &(&1 + 1)) + |> Map.update!(status, &(&1 + 1)) + |> decr_enqueued() + end + + defp update_queue(queue, status, _meta, _duration) do + queue + |> Map.update!(:processed_jobs, &(&1 + 1)) + |> Map.update!(status, &(&1 + 1)) + |> decr_enqueued() + end + + defp decr_enqueued(map) do + Map.update!(map, :enqueued, fn + 0 -> 0 + enqueued -> enqueued - 1 + end) + end +end diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex index 358efa14a..a43ce8bc0 100644 --- a/lib/pleroma/workers/worker_helper.ex +++ b/lib/pleroma/workers/worker_helper.ex @@ -40,6 +40,7 @@ def enqueue(op, params, worker_args \\ []) do unquote(caller_module) |> apply(:new, [params, worker_args]) |> Pleroma.Repo.insert() + |> Pleroma.JobQueueMonitor.enqueue() end end end diff --git a/test/healthcheck_test.exs b/test/healthcheck_test.exs index 6bb8d5b7f..66d5026ff 100644 --- a/test/healthcheck_test.exs +++ b/test/healthcheck_test.exs @@ -9,7 +9,14 @@ defmodule Pleroma.HealthcheckTest do test "system_info/0" do result = Healthcheck.system_info() |> Map.from_struct() - assert Map.keys(result) == [:active, :healthy, :idle, :memory_used, :pool_size] + assert Map.keys(result) == [ + :active, + :healthy, + :idle, + :job_queue_stats, + :memory_used, + :pool_size + ] end describe "check_health/1" do