From b3b7ab5d9a0528bdfa385bd61d7f97ae4c7de374 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 2 Apr 2018 16:27:36 +0200 Subject: [PATCH] Load all users at once in timelines. --- lib/pleroma/web/activity_pub/utils.ex | 7 +- .../mastodon_api/mastodon_api_controller.ex | 10 +- .../web/twitter_api/views/activity_view.ex | 94 ++++++++++++++----- test/web/activity_pub/activity_pub_test.exs | 2 +- .../twitter_api/views/activity_view_test.exs | 11 ++- 5 files changed, 94 insertions(+), 30 deletions(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index ac3a57e03..7a0762e9f 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -50,10 +50,13 @@ def create_context(context) do changeset = Object.context_mapping(context) case Repo.insert(changeset) do - {:ok, object} -> object + {:ok, object} -> + object + # This should be solved by an upsert, but it seems ecto # has problems accessing the constraint inside the jsonb. - {:error, _} -> Object.get_cached_by_ap_id(context) + {:error, _} -> + Object.get_cached_by_ap_id(context) end end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index dc92e30c5..64928fda5 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -513,10 +513,12 @@ def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do ) statuses = Repo.all(q) ++ fetched - tags = String.split(query) - |> Enum.uniq() - |> Enum.filter(fn tag -> String.starts_with?(tag, "#") end) - |> Enum.map(fn tag -> String.slice(tag, 1..-1) end) + + tags = + String.split(query) + |> Enum.uniq() + |> Enum.filter(fn tag -> String.starts_with?(tag, "#") end) + |> Enum.map(fn tag -> String.slice(tag, 1..-1) end) res = %{ "accounts" => AccountView.render("accounts.json", users: accounts, for: user, as: :user), diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index 760840239..4e4433aed 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -8,46 +8,99 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do alias Pleroma.Web.TwitterAPI.Representers.ObjectRepresenter alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.User alias Pleroma.Repo alias Pleroma.Formatter import Ecto.Query defp query_context_ids([]), do: [] + defp query_context_ids(contexts) do - query = from o in Object, - where: fragment("(?)->>'id' = ANY(?)", o.data, ^contexts) + query = from(o in Object, where: fragment("(?)->>'id' = ANY(?)", o.data, ^contexts)) + + Repo.all(query) + end + + defp query_users([]), do: [] + + defp query_users(user_ids) do + query = from(user in User, where: user.ap_id in ^user_ids) Repo.all(query) end defp collect_context_ids(activities) do - contexts = activities - |> Enum.reject(&(&1.data["context_id"])) - |> Enum.map(fn(%{data: data}) -> - data["context"] + contexts = + activities + |> Enum.reject(& &1.data["context_id"]) + |> Enum.map(fn %{data: data} -> + data["context"] + end) + |> Enum.filter(& &1) + |> query_context_ids() + |> Enum.reduce(%{}, fn %{data: %{"id" => ap_id}, id: id}, acc -> + Map.put(acc, ap_id, id) + end) + end + + defp collect_users(activities) do + activities + |> Enum.map(fn activity -> + case activity.data do + data = %{"type" => "Follow"} -> + [data["actor"], data["object"]] + + data -> + [data["actor"]] + end ++ activity.recipients end) - |> Enum.filter(&(&1)) - |> query_context_ids() - |> Enum.reduce(%{}, fn(%{data: %{"id" => ap_id}, id: id}, acc) -> - Map.put(acc, ap_id, id) + |> List.flatten() + |> Enum.uniq() + |> query_users() + |> Enum.reduce(%{}, fn user, acc -> + Map.put(acc, user.ap_id, user) end) end - defp get_context_id(%{data: %{"context_id" => context_id}}, _) when not is_nil(context_id), do: context_id + defp get_context_id(%{data: %{"context_id" => context_id}}, _) when not is_nil(context_id), + do: context_id + defp get_context_id(%{data: %{"context" => nil}}, _), do: nil + defp get_context_id(%{data: %{"context" => context}}, options) do cond do id = options[:context_ids][context] -> id true -> TwitterAPI.context_to_conversation_id(context) end end + defp get_context_id(_, _), do: nil + defp get_user(ap_id, opts) do + cond do + user = opts[:users][ap_id] -> + user + + String.ends_with?(ap_id, "/followers") -> + nil + + ap_id == "https://www.w3.org/ns/activitystreams#Public" -> + nil + + true -> + User.get_cached_by_ap_id(ap_id) + end + end + def render("index.json", opts) do context_ids = collect_context_ids(opts.activities) - opts = opts - |> Map.put(:context_ids, context_ids) + users = collect_users(opts.activities) + + opts = + opts + |> Map.put(:context_ids, context_ids) + |> Map.put(:users, users) render_many( opts.activities, @@ -58,7 +111,7 @@ def render("index.json", opts) do end def render("activity.json", %{activity: %{data: %{"type" => "Delete"}} = activity} = opts) do - user = User.get_cached_by_ap_id(activity.data["actor"]) + user = get_user(activity.data["actor"], opts) created_at = activity.data["published"] |> Utils.date_to_asctime() %{ @@ -78,11 +131,11 @@ def render("activity.json", %{activity: %{data: %{"type" => "Delete"}} = activit end def render("activity.json", %{activity: %{data: %{"type" => "Follow"}} = activity} = opts) do - user = User.get_cached_by_ap_id(activity.data["actor"]) + user = get_user(activity.data["actor"], opts) created_at = activity.data["published"] || DateTime.to_iso8601(activity.inserted_at) created_at = created_at |> Utils.date_to_asctime() - followed = User.get_cached_by_ap_id(activity.data["object"]) + followed = get_user(activity.data["object"], opts) text = "#{user.nickname} started following #{followed.nickname}" %{ @@ -101,7 +154,7 @@ def render("activity.json", %{activity: %{data: %{"type" => "Follow"}} = activit end def render("activity.json", %{activity: %{data: %{"type" => "Announce"}} = activity} = opts) do - user = User.get_by_ap_id(activity.data["actor"]) + user = get_user(activity.data["actor"], opts) created_at = activity.data["published"] |> Utils.date_to_asctime() announced_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"]) @@ -126,7 +179,7 @@ def render("activity.json", %{activity: %{data: %{"type" => "Announce"}} = activ end def render("activity.json", %{activity: %{data: %{"type" => "Like"}} = activity} = opts) do - user = User.get_cached_by_ap_id(activity.data["actor"]) + user = get_user(activity.data["actor"], opts) liked_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"]) created_at = @@ -154,8 +207,7 @@ def render( "activity.json", %{activity: %{data: %{"type" => "Create", "object" => object}} = activity} = opts ) do - actor = get_in(activity.data, ["actor"]) - user = User.get_cached_by_ap_id(actor) + user = get_user(activity.data["actor"], opts) created_at = object["published"] |> Utils.date_to_asctime() like_count = object["like_count"] || 0 @@ -165,7 +217,7 @@ def render( attentions = activity.recipients - |> Enum.map(fn ap_id -> User.get_cached_by_ap_id(ap_id) end) + |> Enum.map(fn ap_id -> get_user(ap_id, opts) end) |> Enum.filter(& &1) |> Enum.map(fn user -> UserView.render("show.json", %{user: user, for: opts[:for]}) end) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index c6434f789..657d75a55 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -56,7 +56,7 @@ test "adds a context when none is there" do "id" => "some_id", "object" => %{ "id" => "object_id" - } + } } {:ok, %Activity{} = activity} = ActivityPub.insert(data) diff --git a/test/web/twitter_api/views/activity_view_test.exs b/test/web/twitter_api/views/activity_view_test.exs index 46ffdef44..7f2017d3c 100644 --- a/test/web/twitter_api/views/activity_view_test.exs +++ b/test/web/twitter_api/views/activity_view_test.exs @@ -64,7 +64,12 @@ test "a list of activities" do { TwitterAPI, [], - [context_to_conversation_id: fn(_) -> false end] + [context_to_conversation_id: fn _ -> false end] + }, + { + User, + [:passthrough], + [get_cached_by_ap_id: fn _ -> nil end] } ] @@ -73,7 +78,9 @@ test "a list of activities" do assert result["statusnet_conversation_id"] == convo_id assert result["user"] - refute called TwitterAPI.context_to_conversation_id(:_) + refute called(TwitterAPI.context_to_conversation_id(:_)) + refute called(User.get_cached_by_ap_id(user.ap_id)) + refute called(User.get_cached_by_ap_id(other_user.ap_id)) end end