Merge branch 'feature/admin-api-status-count-per-instance' into 'develop'

admin api status count per instance

See merge request pleroma/pleroma!2500
This commit is contained in:
lain 2020-06-24 10:42:07 +00:00
commit 88ccade1cb
10 changed files with 324 additions and 54 deletions

View File

@ -15,9 +15,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
<details> <details>
<summary>API Changes</summary> <summary>API Changes</summary>
- **Breaking:** Emoji API: changed methods and renamed routes. - **Breaking:** Emoji API: changed methods and renamed routes.
</details> </details>
<details>
<summary>Admin API Changes</summary>
- Status visibility stats: now can return stats per instance.
- Mix task to refresh counter cache (`mix pleroma.refresh_counter_cache`)
</details>
### Removed ### Removed
- **Breaking:** removed `with_move` parameter from notifications timeline. - **Breaking:** removed `with_move` parameter from notifications timeline.
@ -98,6 +107,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
2. Run database migrations (inside Pleroma directory): 2. Run database migrations (inside Pleroma directory):
- OTP: `./bin/pleroma_ctl migrate` - OTP: `./bin/pleroma_ctl migrate`
- From Source: `mix ecto.migrate` - From Source: `mix ecto.migrate`
3. Reset status visibility counters (inside Pleroma directory):
- OTP: `./bin/pleroma_ctl refresh_counter_cache`
- From Source: `mix pleroma.refresh_counter_cache`
## [2.0.2] - 2020-04-08 ## [2.0.2] - 2020-04-08

View File

@ -1118,6 +1118,10 @@ Loads json generated from `config/descriptions.exs`.
### Stats ### Stats
- Query Params:
- *optional* `instance`: **string** instance hostname (without protocol) to get stats for
- Example: `https://mypleroma.org/api/pleroma/admin/stats?instance=lain.com`
- Response: - Response:
```json ```json

View File

@ -17,30 +17,53 @@ defmodule Mix.Tasks.Pleroma.RefreshCounterCache do
def run([]) do def run([]) do
Mix.Pleroma.start_pleroma() Mix.Pleroma.start_pleroma()
["public", "unlisted", "private", "direct"] instances =
|> Enum.each(fn visibility -> Activity
count = status_visibility_count_query(visibility) |> distinct([a], true)
name = "status_visibility_#{visibility}" |> select([a], fragment("split_part(?, '/', 3)", a.actor))
CounterCache.set(name, count) |> Repo.all()
Mix.Pleroma.shell_info("Set #{name} to #{count}")
instances
|> Enum.with_index(1)
|> Enum.each(fn {instance, i} ->
counters = instance_counters(instance)
CounterCache.set(instance, counters)
Mix.Pleroma.shell_info(
"[#{i}/#{length(instances)}] Setting #{instance} counters: #{inspect(counters)}"
)
end) end)
Mix.Pleroma.shell_info("Done") Mix.Pleroma.shell_info("Done")
end end
defp status_visibility_count_query(visibility) do defp instance_counters(instance) do
counters = %{"public" => 0, "unlisted" => 0, "private" => 0, "direct" => 0}
Activity Activity
|> where( |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
|> where([a], fragment("split_part(?, '/', 3) = ?", a.actor, ^instance))
|> select(
[a], [a],
fragment( {fragment(
"activity_visibility(?, ?, ?) = ?", "activity_visibility(?, ?, ?)",
a.actor, a.actor,
a.recipients, a.recipients,
a.data, a.data
^visibility ), count(a.id)}
)
|> group_by(
[a],
fragment(
"activity_visibility(?, ?, ?)",
a.actor,
a.recipients,
a.data
) )
) )
|> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) |> Repo.all(timeout: :timer.minutes(30))
|> Repo.aggregate(:count, :id, timeout: :timer.minutes(30)) |> Enum.reduce(counters, fn {visibility, count}, acc ->
Map.put(acc, visibility, count)
end)
end end
end end

View File

@ -10,32 +10,70 @@ defmodule Pleroma.CounterCache do
import Ecto.Query import Ecto.Query
schema "counter_cache" do schema "counter_cache" do
field(:name, :string) field(:instance, :string)
field(:count, :integer) field(:public, :integer)
field(:unlisted, :integer)
field(:private, :integer)
field(:direct, :integer)
end end
def changeset(struct, params) do def changeset(struct, params) do
struct struct
|> cast(params, [:name, :count]) |> cast(params, [:instance, :public, :unlisted, :private, :direct])
|> validate_required([:name]) |> validate_required([:instance])
|> unique_constraint(:name) |> unique_constraint(:instance)
end end
def get_as_map(names) when is_list(names) do def get_by_instance(instance) do
CounterCache CounterCache
|> where([cc], cc.name in ^names) |> select([c], %{
|> Repo.all() "public" => c.public,
|> Enum.group_by(& &1.name, & &1.count) "unlisted" => c.unlisted,
|> Map.new(fn {k, v} -> {k, hd(v)} end) "private" => c.private,
"direct" => c.direct
})
|> where([c], c.instance == ^instance)
|> Repo.one()
|> case do
nil -> %{"public" => 0, "unlisted" => 0, "private" => 0, "direct" => 0}
val -> val
end
end end
def set(name, count) do def get_sum do
CounterCache
|> select([c], %{
"public" => type(sum(c.public), :integer),
"unlisted" => type(sum(c.unlisted), :integer),
"private" => type(sum(c.private), :integer),
"direct" => type(sum(c.direct), :integer)
})
|> Repo.one()
end
def set(instance, values) do
params =
Enum.reduce(
["public", "private", "unlisted", "direct"],
%{"instance" => instance},
fn param, acc ->
Map.put_new(acc, param, Map.get(values, param, 0))
end
)
%CounterCache{} %CounterCache{}
|> changeset(%{"name" => name, "count" => count}) |> changeset(params)
|> Repo.insert( |> Repo.insert(
on_conflict: [set: [count: count]], on_conflict: [
set: [
public: params["public"],
private: params["private"],
unlisted: params["unlisted"],
direct: params["direct"]
]
],
returning: true, returning: true,
conflict_target: :name conflict_target: :instance
) )
end end
end end

View File

@ -97,20 +97,11 @@ def calculate_stat_data do
} }
end end
def get_status_visibility_count do def get_status_visibility_count(instance \\ nil) do
counter_cache = if is_nil(instance) do
CounterCache.get_as_map([ CounterCache.get_sum()
"status_visibility_public", else
"status_visibility_private", CounterCache.get_by_instance(instance)
"status_visibility_unlisted", end
"status_visibility_direct"
])
%{
public: counter_cache["status_visibility_public"] || 0,
unlisted: counter_cache["status_visibility_unlisted"] || 0,
private: counter_cache["status_visibility_private"] || 0,
direct: counter_cache["status_visibility_direct"] || 0
}
end end
end end

View File

@ -643,10 +643,10 @@ def resend_confirmation_email(%{assigns: %{user: admin}} = conn, %{"nicknames" =
json(conn, "") json(conn, "")
end end
def stats(conn, _) do def stats(conn, params) do
count = Stats.get_status_visibility_count() counters = Stats.get_status_visibility_count(params["instance"])
json(conn, %{"status_visibility" => count}) json(conn, %{"status_visibility" => counters})
end end
defp page_params(params) do defp page_params(params) do

View File

@ -0,0 +1,143 @@
defmodule Pleroma.Repo.Migrations.UpdateCounterCacheTable do
use Ecto.Migration
@function_name "update_status_visibility_counter_cache"
@trigger_name "status_visibility_counter_cache_trigger"
def up do
execute("drop trigger if exists #{@trigger_name} on activities")
execute("drop function if exists #{@function_name}()")
drop_if_exists(unique_index(:counter_cache, [:name]))
drop_if_exists(table(:counter_cache))
create_if_not_exists table(:counter_cache) do
add(:instance, :string, null: false)
add(:direct, :bigint, null: false, default: 0)
add(:private, :bigint, null: false, default: 0)
add(:unlisted, :bigint, null: false, default: 0)
add(:public, :bigint, null: false, default: 0)
end
create_if_not_exists(unique_index(:counter_cache, [:instance]))
"""
CREATE OR REPLACE FUNCTION #{@function_name}()
RETURNS TRIGGER AS
$$
DECLARE
hostname character varying(255);
visibility_new character varying(64);
visibility_old character varying(64);
actor character varying(255);
BEGIN
IF TG_OP = 'DELETE' THEN
actor := OLD.actor;
ELSE
actor := NEW.actor;
END IF;
hostname := split_part(actor, '/', 3);
IF TG_OP = 'INSERT' THEN
visibility_new := activity_visibility(NEW.actor, NEW.recipients, NEW.data);
IF NEW.data->>'type' = 'Create'
AND visibility_new IN ('public', 'unlisted', 'private', 'direct') THEN
EXECUTE format('INSERT INTO "counter_cache" ("instance", %1$I) VALUES ($1, 1)
ON CONFLICT ("instance") DO
UPDATE SET %1$I = "counter_cache".%1$I + 1', visibility_new)
USING hostname;
END IF;
RETURN NEW;
ELSIF TG_OP = 'UPDATE' THEN
visibility_new := activity_visibility(NEW.actor, NEW.recipients, NEW.data);
visibility_old := activity_visibility(OLD.actor, OLD.recipients, OLD.data);
IF (NEW.data->>'type' = 'Create')
AND (OLD.data->>'type' = 'Create')
AND visibility_new != visibility_old
AND visibility_new IN ('public', 'unlisted', 'private', 'direct') THEN
EXECUTE format('UPDATE "counter_cache" SET
%1$I = greatest("counter_cache".%1$I - 1, 0),
%2$I = "counter_cache".%2$I + 1
WHERE "instance" = $1', visibility_old, visibility_new)
USING hostname;
END IF;
RETURN NEW;
ELSIF TG_OP = 'DELETE' THEN
IF OLD.data->>'type' = 'Create' THEN
visibility_old := activity_visibility(OLD.actor, OLD.recipients, OLD.data);
EXECUTE format('UPDATE "counter_cache" SET
%1$I = greatest("counter_cache".%1$I - 1, 0)
WHERE "instance" = $1', visibility_old)
USING hostname;
END IF;
RETURN OLD;
END IF;
END;
$$
LANGUAGE 'plpgsql';
"""
|> execute()
execute("DROP TRIGGER IF EXISTS #{@trigger_name} ON activities")
"""
CREATE TRIGGER #{@trigger_name}
BEFORE
INSERT
OR UPDATE of recipients, data
OR DELETE
ON activities
FOR EACH ROW
EXECUTE PROCEDURE #{@function_name}();
"""
|> execute()
end
def down do
execute("DROP TRIGGER IF EXISTS #{@trigger_name} ON activities")
execute("DROP FUNCTION IF EXISTS #{@function_name}()")
drop_if_exists(unique_index(:counter_cache, [:instance]))
drop_if_exists(table(:counter_cache))
create_if_not_exists table(:counter_cache) do
add(:name, :string, null: false)
add(:count, :bigint, null: false, default: 0)
end
create_if_not_exists(unique_index(:counter_cache, [:name]))
"""
CREATE OR REPLACE FUNCTION #{@function_name}()
RETURNS TRIGGER AS
$$
DECLARE
BEGIN
IF TG_OP = 'INSERT' THEN
IF NEW.data->>'type' = 'Create' THEN
EXECUTE 'INSERT INTO counter_cache (name, count) VALUES (''status_visibility_' || activity_visibility(NEW.actor, NEW.recipients, NEW.data) || ''', 1) ON CONFLICT (name) DO UPDATE SET count = counter_cache.count + 1';
END IF;
RETURN NEW;
ELSIF TG_OP = 'UPDATE' THEN
IF (NEW.data->>'type' = 'Create') and (OLD.data->>'type' = 'Create') and activity_visibility(NEW.actor, NEW.recipients, NEW.data) != activity_visibility(OLD.actor, OLD.recipients, OLD.data) THEN
EXECUTE 'INSERT INTO counter_cache (name, count) VALUES (''status_visibility_' || activity_visibility(NEW.actor, NEW.recipients, NEW.data) || ''', 1) ON CONFLICT (name) DO UPDATE SET count = counter_cache.count + 1';
EXECUTE 'update counter_cache SET count = counter_cache.count - 1 where count > 0 and name = ''status_visibility_' || activity_visibility(OLD.actor, OLD.recipients, OLD.data) || ''';';
END IF;
RETURN NEW;
ELSIF TG_OP = 'DELETE' THEN
IF OLD.data->>'type' = 'Create' THEN
EXECUTE 'update counter_cache SET count = counter_cache.count - 1 where count > 0 and name = ''status_visibility_' || activity_visibility(OLD.actor, OLD.recipients, OLD.data) || ''';';
END IF;
RETURN OLD;
END IF;
END;
$$
LANGUAGE 'plpgsql';
"""
|> execute()
"""
CREATE TRIGGER #{@trigger_name} BEFORE INSERT OR UPDATE of recipients, data OR DELETE ON activities
FOR EACH ROW
EXECUTE PROCEDURE #{@function_name}();
"""
|> execute()
end
end

View File

@ -17,10 +17,11 @@ test "it ignores internal users" do
end end
end end
describe "status visibility count" do describe "status visibility sum count" do
test "on new status" do test "on new status" do
instance2 = "instance2.tld"
user = insert(:user) user = insert(:user)
other_user = insert(:user) other_user = insert(:user, %{ap_id: "https://#{instance2}/@actor"})
CommonAPI.post(user, %{visibility: "public", status: "hey"}) CommonAPI.post(user, %{visibility: "public", status: "hey"})
@ -45,24 +46,24 @@ test "on new status" do
}) })
end) end)
assert %{direct: 3, private: 4, public: 1, unlisted: 2} = assert %{"direct" => 3, "private" => 4, "public" => 1, "unlisted" => 2} =
Pleroma.Stats.get_status_visibility_count() Pleroma.Stats.get_status_visibility_count()
end end
test "on status delete" do test "on status delete" do
user = insert(:user) user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"}) {:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"})
assert %{public: 1} = Pleroma.Stats.get_status_visibility_count() assert %{"public" => 1} = Pleroma.Stats.get_status_visibility_count()
CommonAPI.delete(activity.id, user) CommonAPI.delete(activity.id, user)
assert %{public: 0} = Pleroma.Stats.get_status_visibility_count() assert %{"public" => 0} = Pleroma.Stats.get_status_visibility_count()
end end
test "on status visibility update" do test "on status visibility update" do
user = insert(:user) user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"}) {:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"})
assert %{public: 1, private: 0} = Pleroma.Stats.get_status_visibility_count() assert %{"public" => 1, "private" => 0} = Pleroma.Stats.get_status_visibility_count()
{:ok, _} = CommonAPI.update_activity_scope(activity.id, %{visibility: "private"}) {:ok, _} = CommonAPI.update_activity_scope(activity.id, %{visibility: "private"})
assert %{public: 0, private: 1} = Pleroma.Stats.get_status_visibility_count() assert %{"public" => 0, "private" => 1} = Pleroma.Stats.get_status_visibility_count()
end end
test "doesn't count unrelated activities" do test "doesn't count unrelated activities" do
@ -73,8 +74,46 @@ test "doesn't count unrelated activities" do
CommonAPI.favorite(other_user, activity.id) CommonAPI.favorite(other_user, activity.id)
CommonAPI.repeat(activity.id, other_user) CommonAPI.repeat(activity.id, other_user)
assert %{direct: 0, private: 0, public: 1, unlisted: 0} = assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 0} =
Pleroma.Stats.get_status_visibility_count() Pleroma.Stats.get_status_visibility_count()
end end
end end
describe "status visibility by instance count" do
test "single instance" do
local_instance = Pleroma.Web.Endpoint.url() |> String.split("//") |> Enum.at(1)
instance2 = "instance2.tld"
user1 = insert(:user)
user2 = insert(:user, %{ap_id: "https://#{instance2}/@actor"})
CommonAPI.post(user1, %{visibility: "public", status: "hey"})
Enum.each(1..5, fn _ ->
CommonAPI.post(user1, %{
visibility: "unlisted",
status: "hey"
})
end)
Enum.each(1..10, fn _ ->
CommonAPI.post(user1, %{
visibility: "direct",
status: "hey @#{user2.nickname}"
})
end)
Enum.each(1..20, fn _ ->
CommonAPI.post(user2, %{
visibility: "private",
status: "hey"
})
end)
assert %{"direct" => 10, "private" => 0, "public" => 1, "unlisted" => 5} =
Pleroma.Stats.get_status_visibility_count(local_instance)
assert %{"direct" => 0, "private" => 20, "public" => 0, "unlisted" => 0} =
Pleroma.Stats.get_status_visibility_count(instance2)
end
end
end end

View File

@ -37,7 +37,7 @@ test "counts statuses" do
assert capture_io(fn -> Mix.Tasks.Pleroma.RefreshCounterCache.run([]) end) =~ "Done\n" assert capture_io(fn -> Mix.Tasks.Pleroma.RefreshCounterCache.run([]) end) =~ "Done\n"
assert %{direct: 3, private: 4, public: 1, unlisted: 2} = assert %{"direct" => 3, "private" => 4, "public" => 1, "unlisted" => 2} =
Pleroma.Stats.get_status_visibility_count() Pleroma.Stats.get_status_visibility_count()
end end
end end

View File

@ -1732,6 +1732,26 @@ test "status visibility count", %{conn: conn} do
assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 2} = assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 2} =
response["status_visibility"] response["status_visibility"]
end end
test "by instance", %{conn: conn} do
admin = insert(:user, is_admin: true)
user1 = insert(:user)
instance2 = "instance2.tld"
user2 = insert(:user, %{ap_id: "https://#{instance2}/@actor"})
CommonAPI.post(user1, %{visibility: "public", status: "hey"})
CommonAPI.post(user2, %{visibility: "unlisted", status: "hey"})
CommonAPI.post(user2, %{visibility: "private", status: "hey"})
response =
conn
|> assign(:user, admin)
|> get("/api/pleroma/admin/stats", instance: instance2)
|> json_response(200)
assert %{"direct" => 0, "private" => 1, "public" => 0, "unlisted" => 1} =
response["status_visibility"]
end
end end
end end