From 793fc77b160ae2bcaded23d22d7606c2ab499f0a Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 23 Jan 2021 00:37:49 +0400 Subject: [PATCH 1/5] Add active user count --- lib/pleroma/user.ex | 15 ++++++++++ .../web/mastodon_api/views/instance_view.ex | 1 + lib/pleroma/web/plugs/user_tracking_plug.ex | 30 +++++++++++++++++++ lib/pleroma/web/router.ex | 1 + ...0122151424_add_last_active_at_to_users.exs | 11 +++++++ .../controllers/status_controller_test.exs | 1 + .../conversation_controller_test.exs | 2 +- 7 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/web/plugs/user_tracking_plug.ex create mode 100644 priv/repo/migrations/20210122151424_add_last_active_at_to_users.exs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index e422b59f1..1dde65335 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -146,6 +146,7 @@ defmodule Pleroma.User do field(:inbox, :string) field(:shared_inbox, :string) field(:accepts_chat_messages, :boolean, default: nil) + field(:last_active_at, :naive_datetime) embeds_one( :notification_settings, @@ -2444,4 +2445,18 @@ def sanitize_html(%User{} = user, filter) do def get_host(%User{ap_id: ap_id} = _user) do URI.parse(ap_id).host end + + def update_last_active_at(user) do + user + |> cast(%{last_active_at: NaiveDateTime.utc_now()}, [:last_active_at]) + |> update_and_set_cache() + end + + def active_user_count(weeks \\ 4) do + active_after = Timex.shift(NaiveDateTime.utc_now(), weeks: -weeks) + + __MODULE__ + |> where([u], u.last_active_at >= ^active_after) + |> Repo.aggregate(:count) + end end diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 1edbdbe11..73205fb6d 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -45,6 +45,7 @@ def render("show.json", _) do fields_limits: fields_limits(), post_formats: Config.get([:instance, :allowed_post_formats]) }, + stats: %{mau: Pleroma.User.active_user_count()}, vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) } } diff --git a/lib/pleroma/web/plugs/user_tracking_plug.ex b/lib/pleroma/web/plugs/user_tracking_plug.ex new file mode 100644 index 000000000..c9a988f00 --- /dev/null +++ b/lib/pleroma/web/plugs/user_tracking_plug.ex @@ -0,0 +1,30 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.UserTrackingPlug do + alias Pleroma.User + + import Plug.Conn, only: [assign: 3] + + @update_interval :timer.hours(24) + + def init(opts), do: opts + + def call(%{assigns: %{user: %User{id: id} = user}} = conn, _) when not is_nil(id) do + with true <- needs_update?(user), + {:ok, user} <- User.update_last_active_at(user) do + assign(conn, :user, user) + else + _ -> conn + end + end + + def call(conn, _), do: conn + + defp needs_update?(%User{last_active_at: nil}), do: true + + defp needs_update?(%User{last_active_at: last_active_at}) do + NaiveDateTime.diff(NaiveDateTime.utc_now(), last_active_at, :millisecond) >= @update_interval + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a9e332fa1..7521f5dc3 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -56,6 +56,7 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Web.Plugs.UserEnabledPlug) plug(Pleroma.Web.Plugs.SetUserSessionIdPlug) plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug) + plug(Pleroma.Web.Plugs.UserTrackingPlug) end pipeline :base_api do diff --git a/priv/repo/migrations/20210122151424_add_last_active_at_to_users.exs b/priv/repo/migrations/20210122151424_add_last_active_at_to_users.exs new file mode 100644 index 000000000..9671e495b --- /dev/null +++ b/priv/repo/migrations/20210122151424_add_last_active_at_to_users.exs @@ -0,0 +1,11 @@ +defmodule Pleroma.Repo.Migrations.AddLastActiveAtToUsers do + use Ecto.Migration + + def change do + alter table(:users) do + add(:last_active_at, :naive_datetime) + end + + create_if_not_exists(index(:users, [:last_active_at])) + end +end diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index a647cd57f..3c73eb514 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -263,6 +263,7 @@ test "posting a fake status", %{conn: conn} do fake_conn = conn + |> assign(:user, refresh_record(conn.assigns.user)) |> put_req_header("content-type", "application/json") |> post("/api/v1/statuses", %{ "status" => diff --git a/test/pleroma/web/pleroma_api/controllers/conversation_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/conversation_controller_test.exs index 98a23aaaa..54f2c5a58 100644 --- a/test/pleroma/web/pleroma_api/controllers/conversation_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/conversation_controller_test.exs @@ -104,7 +104,7 @@ test "PATCH /api/v1/pleroma/conversations/:id" do [participation] = Participation.for_user(user) participation = Repo.preload(participation, :recipients) - assert user in participation.recipients + assert refresh_record(user) in participation.recipients assert other_user in participation.recipients end From bddb01bdedf93b3aabed6a90a3e4f9eac7ef33f0 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 27 Jan 2021 18:18:46 +0400 Subject: [PATCH 2/5] Add tests --- test/pleroma/user_test.exs | 38 ++++++++++++ .../controllers/instance_controller_test.exs | 1 + .../web/plugs/user_tracking_plug_test.exs | 58 +++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 test/pleroma/web/plugs/user_tracking_plug_test.exs diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index b4df22c2c..1fab519f0 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -2248,4 +2248,42 @@ test "get_host/1" do user = insert(:user, ap_id: "https://lain.com/users/lain", nickname: "lain") assert User.get_host(user) == "lain.com" end + + test "update_last_active_at/1" do + user = insert(:user) + assert is_nil(user.last_active_at) + + test_started_at = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second) + + assert {:ok, user} = User.update_last_active_at(user) + + assert user.last_active_at >= test_started_at + assert user.last_active_at <= NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) + + last_active_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(-:timer.hours(24), :millisecond) + |> NaiveDateTime.truncate(:second) + + assert {:ok, user} = + user + |> cast(%{last_active_at: last_active_at}, [:last_active_at]) + |> User.update_and_set_cache() + + assert user.last_active_at == last_active_at + assert {:ok, user} = User.update_last_active_at(user) + assert user.last_active_at >= test_started_at + assert user.last_active_at <= NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) + end + + test "active_user_count/1" do + insert(:user) + insert(:user, %{last_active_at: Timex.shift(NaiveDateTime.utc_now(), weeks: -5)}) + insert(:user, %{last_active_at: Timex.shift(NaiveDateTime.utc_now(), weeks: -3)}) + insert(:user, %{last_active_at: NaiveDateTime.utc_now()}) + + assert User.active_user_count() == 2 + assert User.active_user_count(6) == 3 + assert User.active_user_count(1) == 1 + end end diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs index 0d4eebb73..b99856659 100644 --- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs @@ -47,6 +47,7 @@ test "get instance information", %{conn: conn} do assert result["pleroma"]["metadata"]["federation"] assert result["pleroma"]["metadata"]["fields_limits"] assert result["pleroma"]["vapid_public_key"] + assert result["pleroma"]["stats"]["mau"] == 0 assert email == from_config_email assert thumbnail == from_config_thumbnail diff --git a/test/pleroma/web/plugs/user_tracking_plug_test.exs b/test/pleroma/web/plugs/user_tracking_plug_test.exs new file mode 100644 index 000000000..8e9d59b99 --- /dev/null +++ b/test/pleroma/web/plugs/user_tracking_plug_test.exs @@ -0,0 +1,58 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.UserTrackingPlugTest do + use Pleroma.Web.ConnCase, async: true + + import Pleroma.Factory + + alias Pleroma.Web.Plugs.UserTrackingPlug + + test "updates last_active_at for a new user", %{conn: conn} do + user = insert(:user) + + assert is_nil(user.last_active_at) + + test_started_at = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second) + + %{assigns: %{user: user}} = + conn + |> assign(:user, user) + |> UserTrackingPlug.call(%{}) + + assert user.last_active_at >= test_started_at + assert user.last_active_at <= NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) + end + + test "doesn't update last_active_at if it was updated recently", %{conn: conn} do + last_active_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(-:timer.hours(1), :millisecond) + |> NaiveDateTime.truncate(:second) + + user = insert(:user, %{last_active_at: last_active_at}) + + %{assigns: %{user: user}} = + conn + |> assign(:user, user) + |> UserTrackingPlug.call(%{}) + + assert user.last_active_at == last_active_at + end + + test "skips updating last_active_at if user ID is nil", %{conn: conn} do + %{assigns: %{user: user}} = + conn + |> assign(:user, %Pleroma.User{}) + |> UserTrackingPlug.call(%{}) + + assert is_nil(user.last_active_at) + end + + test "does nothing if user is not present", %{conn: conn} do + %{assigns: assigns} = UserTrackingPlug.call(conn, %{}) + + refute Map.has_key?(assigns, :user) + end +end From 13a2ae8ce0f6eb642ef9d6b8fb2cc08e712385c5 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 27 Jan 2021 18:23:46 +0400 Subject: [PATCH 3/5] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4f3867a2..905e35f26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: An endpoint to manage frontends. - Streaming API: Add follow relationships updates. - WebPush: Introduce `pleroma:chat_mention` and `pleroma:emoji_reaction` notification types +- Mastodon API: Add monthly active users to `/api/v1/instance` (`pleroma.stats.mau`) ### Fixed From a51d903e0c8c87247df6b3cdecc476269edf58ce Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 28 Jan 2021 22:23:10 +0400 Subject: [PATCH 4/5] Make sure active_user_count/1 counts only local users --- lib/pleroma/user.ex | 3 ++- test/pleroma/user_test.exs | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 1dde65335..06cdb42af 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2446,7 +2446,7 @@ def get_host(%User{ap_id: ap_id} = _user) do URI.parse(ap_id).host end - def update_last_active_at(user) do + def update_last_active_at(%__MODULE__{local: true} = user) do user |> cast(%{last_active_at: NaiveDateTime.utc_now()}, [:last_active_at]) |> update_and_set_cache() @@ -2457,6 +2457,7 @@ def active_user_count(weeks \\ 4) do __MODULE__ |> where([u], u.last_active_at >= ^active_after) + |> where([u], u.local == true) |> Repo.aggregate(:count) end end diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 1fab519f0..ae6fc4c0d 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -2278,6 +2278,7 @@ test "update_last_active_at/1" do test "active_user_count/1" do insert(:user) + insert(:user, %{local: false}) insert(:user, %{last_active_at: Timex.shift(NaiveDateTime.utc_now(), weeks: -5)}) insert(:user, %{last_active_at: Timex.shift(NaiveDateTime.utc_now(), weeks: -3)}) insert(:user, %{last_active_at: NaiveDateTime.utc_now()}) From 13d79c281fd09d3f9dad802e6c11722bc75ed746 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 28 Jan 2021 14:42:20 -0600 Subject: [PATCH 5/5] Make attachment cleanup jobs a noop if the setting is disabled. --- lib/pleroma/workers/attachments_cleanup_worker.ex | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index a2373ebb9..f5090dae7 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -17,12 +17,14 @@ def perform(%Job{ "object" => %{"data" => %{"attachment" => [_ | _] = attachments, "actor" => actor}} } }) do - attachments - |> Enum.flat_map(fn item -> Enum.map(item["url"], & &1["href"]) end) - |> fetch_objects - |> prepare_objects(actor, Enum.map(attachments, & &1["name"])) - |> filter_objects - |> do_clean + if Pleroma.Config.get([:instance, :cleanup_attachments], false) do + attachments + |> Enum.flat_map(fn item -> Enum.map(item["url"], & &1["href"]) end) + |> fetch_objects + |> prepare_objects(actor, Enum.map(attachments, & &1["name"])) + |> filter_objects + |> do_clean + end {:ok, :success} end