Mastodon API: Preloading and normalization optimizations

- Try to normalize the activity instead of object wherever possible
- Put the `user` key on non-home timelines as well so bookmarks and
thread mutes are preloaded there as well
- Skip trying to get the user when rendering mentions if the id ==
as:Public or user's follower collection
- Preload the object when getting replied to activities and do not crash
if it's not present

This almost solves the problem of Pleroma hammering the db with a lot
of queries when rendering timelines, the things left are
1. When rendering mentions and the user is not in cache, save it for
later and request all uncached users in one go
2. Somehow get rid of needing to get the latest follow activity to
detect the value of `requested` in a relationship. (create a database
view for user relationship and cache it maybe?)
This commit is contained in:
rinpatch 2019-08-13 20:34:34 +03:00
parent 984d7be1a4
commit c1b6952d2a
3 changed files with 46 additions and 12 deletions

View File

@ -224,6 +224,29 @@ def get_create_by_object_ap_id(ap_id) when is_binary(ap_id) do
def get_create_by_object_ap_id(_), do: nil def get_create_by_object_ap_id(_), do: nil
def create_by_object_ap_id_with_object(ap_ids) when is_list(ap_ids) do
from(
activity in Activity,
where:
fragment(
"coalesce((?)->'object'->>'id', (?)->>'object') = ANY(?)",
activity.data,
activity.data,
^ap_ids
),
where: fragment("(?)->>'type' = 'Create'", activity.data),
inner_join: o in Object,
on:
fragment(
"(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
o.data,
activity.data,
activity.data
),
preload: [object: o]
)
end
def create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do def create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
from( from(
activity in Activity, activity in Activity,
@ -263,8 +286,8 @@ defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}
defp get_in_reply_to_activity_from_object(_), do: nil defp get_in_reply_to_activity_from_object(_), do: nil
def get_in_reply_to_activity(%Activity{data: %{"object" => object}}) do def get_in_reply_to_activity(%Activity{} = activity) do
get_in_reply_to_activity_from_object(Object.normalize(object)) get_in_reply_to_activity_from_object(Object.normalize(activity))
end end
def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"]) def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"])

View File

@ -435,6 +435,7 @@ def public_timeline(%{assigns: %{user: user}} = conn, params) do
|> Map.put("local_only", local_only) |> Map.put("local_only", local_only)
|> Map.put("blocking_user", user) |> Map.put("blocking_user", user)
|> Map.put("muting_user", user) |> Map.put("muting_user", user)
|> Map.put("user", user)
|> ActivityPub.fetch_public_activities() |> ActivityPub.fetch_public_activities()
|> Enum.reverse() |> Enum.reverse()
@ -885,8 +886,8 @@ def get_mascot(%{assigns: %{user: user}} = conn, _params) do
end end
def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
with %Activity{data: %{"object" => object}} <- Activity.get_by_id(id), with %Activity{} = activity <- Activity.get_by_id_with_object(id),
%Object{data: %{"likes" => likes}} <- Object.normalize(object) do %Object{data: %{"likes" => likes}} <- Object.normalize(activity) do
q = from(u in User, where: u.ap_id in ^likes) q = from(u in User, where: u.ap_id in ^likes)
users = users =
@ -902,8 +903,8 @@ def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
end end
def reblogged_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do def reblogged_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
with %Activity{data: %{"object" => object}} <- Activity.get_by_id(id), with %Activity{} = activity <- Activity.get_by_id_with_object(id),
%Object{data: %{"announcements" => announces}} <- Object.normalize(object) do %Object{data: %{"announcements" => announces}} <- Object.normalize(activity) do
q = from(u in User, where: u.ap_id in ^announces) q = from(u in User, where: u.ap_id in ^announces)
users = users =
@ -944,6 +945,7 @@ def hashtag_timeline(%{assigns: %{user: user}} = conn, params) do
|> Map.put("local_only", local_only) |> Map.put("local_only", local_only)
|> Map.put("blocking_user", user) |> Map.put("blocking_user", user)
|> Map.put("muting_user", user) |> Map.put("muting_user", user)
|> Map.put("user", user)
|> Map.put("tag", tags) |> Map.put("tag", tags)
|> Map.put("tag_all", tag_all) |> Map.put("tag_all", tag_all)
|> Map.put("tag_reject", tag_reject) |> Map.put("tag_reject", tag_reject)
@ -1350,6 +1352,7 @@ def list_timeline(%{assigns: %{user: user}} = conn, %{"list_id" => id} = params)
params params
|> Map.put("type", "Create") |> Map.put("type", "Create")
|> Map.put("blocking_user", user) |> Map.put("blocking_user", user)
|> Map.put("user", user)
|> Map.put("muting_user", user) |> Map.put("muting_user", user)
# we must filter the following list for the user to avoid leaking statuses the user # we must filter the following list for the user to avoid leaking statuses the user

View File

@ -5,6 +5,8 @@
defmodule Pleroma.Web.MastodonAPI.StatusView do defmodule Pleroma.Web.MastodonAPI.StatusView do
use Pleroma.Web, :view use Pleroma.Web, :view
require Pleroma.Constants
alias Pleroma.Activity alias Pleroma.Activity
alias Pleroma.HTML alias Pleroma.HTML
alias Pleroma.Object alias Pleroma.Object
@ -24,19 +26,19 @@ defp get_replied_to_activities([]), do: %{}
defp get_replied_to_activities(activities) do defp get_replied_to_activities(activities) do
activities activities
|> Enum.map(fn |> Enum.map(fn
%{data: %{"type" => "Create", "object" => object}} -> %{data: %{"type" => "Create"}} = activity ->
object = Object.normalize(object) object = Object.normalize(activity)
object.data["inReplyTo"] != "" && object.data["inReplyTo"] object && object.data["inReplyTo"] != "" && object.data["inReplyTo"]
_ -> _ ->
nil nil
end) end)
|> Enum.filter(& &1) |> Enum.filter(& &1)
|> Activity.create_by_object_ap_id() |> Activity.create_by_object_ap_id_with_object()
|> Repo.all() |> Repo.all()
|> Enum.reduce(%{}, fn activity, acc -> |> Enum.reduce(%{}, fn activity, acc ->
object = Object.normalize(activity) object = Object.normalize(activity)
Map.put(acc, object.data["id"], activity) if object, do: Map.put(acc, object.data["id"], activity), else: acc
end) end)
end end
@ -88,6 +90,7 @@ def render(
reblogged_activity = reblogged_activity =
Activity.create_by_object_ap_id(activity_object.data["id"]) Activity.create_by_object_ap_id(activity_object.data["id"])
|> Activity.with_preloaded_bookmark(opts[:for]) |> Activity.with_preloaded_bookmark(opts[:for])
|> Activity.with_set_thread_muted_field(opts[:for])
|> Repo.one() |> Repo.one()
reblogged = render("status.json", Map.put(opts, :activity, reblogged_activity)) reblogged = render("status.json", Map.put(opts, :activity, reblogged_activity))
@ -142,6 +145,7 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity
object = Object.normalize(activity) object = Object.normalize(activity)
user = get_user(activity.data["actor"]) user = get_user(activity.data["actor"])
user_follower_address = user.follower_address
like_count = object.data["like_count"] || 0 like_count = object.data["like_count"] || 0
announcement_count = object.data["announcement_count"] || 0 announcement_count = object.data["announcement_count"] || 0
@ -157,7 +161,11 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity
mentions = mentions =
(object.data["to"] ++ tag_mentions) (object.data["to"] ++ tag_mentions)
|> Enum.uniq() |> Enum.uniq()
|> Enum.map(fn ap_id -> User.get_cached_by_ap_id(ap_id) end) |> Enum.map(fn
Pleroma.Constants.as_public() -> nil
^user_follower_address -> nil
ap_id -> User.get_cached_by_ap_id(ap_id)
end)
|> Enum.filter(& &1) |> Enum.filter(& &1)
|> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end) |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end)