From e5c6bf3673a8361d1417eba1ccc44edec7658ac4 Mon Sep 17 00:00:00 2001 From: shadowfacts Date: Sat, 7 Sep 2019 19:50:45 +0000 Subject: [PATCH 1/7] Mastodon API: URI encode hashtag name in generated URLs Otherwise hashtags with word characters other than those allowed in URLs (e.g. Japanese characters) produce hashtag URLs that are invalid. --- lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index e71083b91..708b8c2fd 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -499,7 +499,7 @@ def build_tags(object_tags) when is_list(object_tags) do object_tags = for tag when is_binary(tag) <- object_tags, do: tag Enum.reduce(object_tags, [], fn tag, tags -> - tags ++ [%{name: tag, url: "/tag/#{tag}"}] + tags ++ [%{name: tag, url: "/tag/#{URI.encode(tag)}"}] end) end From b40b10b53d00d13f24b5667acc02b1642abc6ec4 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 3 Sep 2019 16:23:03 +0700 Subject: [PATCH 2/7] Add an endpoint to get multiple statuses by IDs --- CHANGELOG.md | 1 + docs/api/differences_in_mastoapi_responses.md | 12 ++++++++++++ lib/pleroma/activity.ex | 7 +++++++ .../controllers/mastodon_api_controller.ex | 14 ++++++++++++++ lib/pleroma/web/router.ex | 1 + test/activity_test.exs | 8 ++++++++ .../mastodon_api/mastodon_api_controller_test.exs | 10 ++++++++++ 7 files changed, 53 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3fa9dfb6..f7f1aee0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Federation: Remove `likes` from objects. - Admin API: Added moderation log - Web response cache (currently, enabled for ActivityPub) +- Mastodon API: Added an endpoint to get multiple statuses by IDs (`GET /api/v1/statuses/?ids[]=1&ids[]=2`) ### Changed - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 02f90f3e8..a21eebc96 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -91,6 +91,18 @@ Additional parameters can be added to the JSON body/Form data: - `expires_in`: The number of seconds the posted activity should expire in. When a posted activity expires it will be deleted from the server, and a delete request for it will be federated. This needs to be longer than an hour. - `in_reply_to_conversation_id`: Will reply to a given conversation, addressing only the people who are part of the recipient set of that conversation. Sets the visibility to `direct`. +## GET `/api/v1/statuses` + +An endpoint to get multiple statuses by IDs. + +Required parameters: + +- `ids`: array of activity ids + +Usage example: `GET /api/v1/statuses/?ids[]=1&ids[]=2`. + +Returns: array of Status. + ## PATCH `/api/v1/update_credentials` Additional parameters can be added to the JSON body/Form data: diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 6a51d4cf3..44f1e3011 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -173,6 +173,13 @@ def get_by_id_with_object(id) do |> Repo.one() end + def all_by_ids_with_object(ids) do + Activity + |> where([a], a.id in ^ids) + |> with_preloaded_object() + |> Repo.all() + end + def by_object_ap_id(ap_id) do from( activity in Activity, diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 8dfad7a54..c54462bb3 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -427,6 +427,20 @@ def dm_timeline(%{assigns: %{user: user}} = conn, params) do |> render("index.json", %{activities: activities, for: user, as: :activity}) end + def get_statuses(%{assigns: %{user: user}} = conn, %{"ids" => ids}) do + limit = 100 + + activities = + ids + |> Enum.take(limit) + |> Activity.all_by_ids_with_object() + |> Enum.filter(&Visibility.visible_for_user?(&1, user)) + + conn + |> put_view(StatusView) + |> render("index.json", activities: activities, for: user, as: :activity) + end + def get_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{} = activity <- Activity.get_by_id_with_object(id), true <- Visibility.visible_for_user?(activity, user) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index cfb973f53..7cd59acb2 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -443,6 +443,7 @@ defmodule Pleroma.Web.Router do get("/timelines/tag/:tag", MastodonAPIController, :hashtag_timeline) get("/timelines/list/:list_id", MastodonAPIController, :list_timeline) + get("/statuses", MastodonAPIController, :get_statuses) get("/statuses/:id", MastodonAPIController, :get_status) get("/statuses/:id/context", MastodonAPIController, :get_context) diff --git a/test/activity_test.exs b/test/activity_test.exs index 785c4b3cf..49654bd67 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -173,4 +173,12 @@ test "add an activity with an expiration" do |> where([a], a.activity_id == ^activity.id) |> Repo.one!() end + + test "all_by_ids_with_object/1" do + %{id: id1} = insert(:note_activity) + %{id: id2} = insert(:note_activity) + + assert [%{id: ^id1, object: %Object{}}, %{id: ^id2, object: %Object{}}] = + Activity.all_by_ids_with_object([id1, id2]) + end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index e18f8f0d1..f4902d043 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -744,6 +744,16 @@ test "get a status", %{conn: conn} do assert id == to_string(activity.id) end + test "get statuses by IDs", %{conn: conn} do + %{id: id1} = insert(:note_activity) + %{id: id2} = insert(:note_activity) + + query_string = "ids[]=#{id1}&ids[]=#{id2}" + conn = get(conn, "/api/v1/statuses/?#{query_string}") + + assert [%{"id" => ^id1}, %{"id" => ^id2}] = json_response(conn, :ok) + end + describe "deleting a status" do test "when you created it", %{conn: conn} do activity = insert(:note_activity) From 30f0cec49a39a2f17cd7fae89830bf0d7922a738 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 3 Sep 2019 16:58:33 +0700 Subject: [PATCH 3/7] Add note about limit to the doc --- docs/api/differences_in_mastoapi_responses.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index a21eebc96..9b32baf3a 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -103,6 +103,8 @@ Usage example: `GET /api/v1/statuses/?ids[]=1&ids[]=2`. Returns: array of Status. +The maximum number of statuses is limited to 100 per request. + ## PATCH `/api/v1/update_credentials` Additional parameters can be added to the JSON body/Form data: From 3d12e05f43763d3ebc693e69f3f945044939bcc2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 11 Sep 2019 01:08:37 +0700 Subject: [PATCH 4/7] Fix `Activity.all_by_ids_with_object/1` test --- test/activity_test.exs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/activity_test.exs b/test/activity_test.exs index 49654bd67..4152aaa7e 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -178,7 +178,11 @@ test "all_by_ids_with_object/1" do %{id: id1} = insert(:note_activity) %{id: id2} = insert(:note_activity) - assert [%{id: ^id1, object: %Object{}}, %{id: ^id2, object: %Object{}}] = - Activity.all_by_ids_with_object([id1, id2]) + activities = + [id1, id2] + |> Activity.all_by_ids_with_object() + |> Enum.sort(&(&1.id < &2.id)) + + assert [%{id: ^id1, object: %Object{}}, %{id: ^id2, object: %Object{}}] = activities end end From 43f02dfe38547e07fb189aab78539af9e02302b3 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 10 Sep 2019 22:01:45 +0300 Subject: [PATCH 5/7] Revert "Parallelize template rendering" This reverts commit 1ad71592adb47762287aec8c36d0fca565c38362. Since it had no limit on the number on concurrent processes it OOM killed instances while rendering hellthreads. When I tried introducing a concurrency limit with Task.async_stream/manual folds it lead to about 3 times worse performance on threads larger than 1000 activities (we are talking 30s vs 1.2 minutes), I think this is not worth the about 1.5 times performance increase on smaller threads when using it. --- lib/mix/tasks/pleroma/benchmark.ex | 38 +++++-------------- .../web/mastodon_api/views/status_view.ex | 4 +- lib/pleroma/web/web.ex | 18 +-------- 3 files changed, 12 insertions(+), 48 deletions(-) diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index 4cc634727..a45940bf3 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -37,37 +37,17 @@ def run(["render_timeline", nickname]) do |> Map.put("blocking_user", user) |> Map.put("muting_user", user) |> Map.put("user", user) - |> Map.put("limit", 80) |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() |> Enum.reverse() - inputs = %{ - "One activity" => Enum.take_random(activities, 1), - "Ten activities" => Enum.take_random(activities, 10), - "Twenty activities" => Enum.take_random(activities, 20), - "Forty activities" => Enum.take_random(activities, 40), - "Eighty activities" => Enum.take_random(activities, 80) - } - - Benchee.run( - %{ - "Parallel rendering" => fn activities -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: activities, - for: user, - as: :activity - }) - end, - "Standart rendering" => fn activities -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: activities, - for: user, - as: :activity, - parallel: false - }) - end - }, - inputs: inputs - ) + Benchee.run(%{ + "render_timeline" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: activities, + for: user, + as: :activity + }) + end + }) end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index e71083b91..b6a3431f9 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -73,14 +73,12 @@ defp reblogged?(activity, user) do def render("index.json", opts) do replied_to_activities = get_replied_to_activities(opts.activities) - parallel = unless is_nil(opts[:parallel]), do: opts[:parallel], else: true opts.activities |> safe_render_many( StatusView, "status.json", - Map.put(opts, :replied_to_activities, replied_to_activities), - parallel + Map.put(opts, :replied_to_activities, replied_to_activities) ) end diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex index bfb6c7287..687346554 100644 --- a/lib/pleroma/web/web.ex +++ b/lib/pleroma/web/web.ex @@ -66,23 +66,9 @@ def safe_render(view, template, assigns \\ %{}) do end @doc """ - Same as `render_many/4` but wrapped in rescue block and parallelized (unless disabled by passing false as a fifth argument). + Same as `render_many/4` but wrapped in rescue block. """ - def safe_render_many(collection, view, template, assigns \\ %{}, parallel \\ true) - - def safe_render_many(collection, view, template, assigns, true) do - Enum.map(collection, fn resource -> - Task.async(fn -> - as = Map.get(assigns, :as) || view.__resource__ - assigns = Map.put(assigns, as, resource) - safe_render(view, template, assigns) - end) - end) - |> Enum.map(&Task.await(&1, :infinity)) - |> Enum.filter(& &1) - end - - def safe_render_many(collection, view, template, assigns, false) do + def safe_render_many(collection, view, template, assigns \\ %{}) do Enum.map(collection, fn resource -> as = Map.get(assigns, :as) || view.__resource__ assigns = Map.put(assigns, as, resource) From 5a76d5d2391f0fa6b4c58bf190b0cdbfff96014f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 10 Sep 2019 23:08:15 +0300 Subject: [PATCH 6/7] Add extended benchmark --- lib/mix/tasks/pleroma/benchmark.ex | 40 ++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index a45940bf3..84dccf7f3 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -27,7 +27,7 @@ def run(["tag"]) do }) end - def run(["render_timeline", nickname]) do + def run(["render_timeline", nickname | _] = args) do start_pleroma() user = Pleroma.User.get_by_nickname(nickname) @@ -37,17 +37,41 @@ def run(["render_timeline", nickname]) do |> Map.put("blocking_user", user) |> Map.put("muting_user", user) |> Map.put("user", user) + |> Map.put("limit", 4096) |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() |> Enum.reverse() - Benchee.run(%{ - "render_timeline" => fn -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: activities, - for: user, - as: :activity + inputs = %{ + "1 activity" => Enum.take_random(activities, 1), + "10 activities" => Enum.take_random(activities, 10), + "20 activities" => Enum.take_random(activities, 20), + "40 activities" => Enum.take_random(activities, 40), + "80 activities" => Enum.take_random(activities, 80) + } + + inputs = + if Enum.at(args, 2) == "extended" do + Map.merge(inputs, %{ + "200 activities" => Enum.take_random(activities, 200), + "500 activities" => Enum.take_random(activities, 500), + "2000 activities" => Enum.take_random(activities, 2000), + "4096 activities" => Enum.take_random(activities, 4096) }) + else + inputs end - }) + + Benchee.run( + %{ + "Standart rendering" => fn activities -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: activities, + for: user, + as: :activity + }) + end + }, + inputs: inputs + ) end end From 56828abf6de81a52079b26cf899b91fdd822f2ce Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 11 Sep 2019 23:04:01 +0300 Subject: [PATCH 7/7] Use Jason for rendering responses Although Jason readme says Phoenix 1.4+ already does it by default, [it actually does it only for new projects](https://github.com/phoenixframework/phoenix/blob/3bfb9f6e900c9a2e31cb95736e2cb5bdad329b61/lib/phoenix.ex#L58-L59) --- config/config.exs | 2 ++ lib/{ => pleroma}/healthcheck.ex | 1 + 2 files changed, 3 insertions(+) rename lib/{ => pleroma}/healthcheck.ex (98%) diff --git a/config/config.exs b/config/config.exs index 5206fe375..0e91df886 100644 --- a/config/config.exs +++ b/config/config.exs @@ -373,6 +373,8 @@ config :phoenix, :format_encoders, json: Jason +config :phoenix, :json_library, Jason + config :pleroma, :gopher, enabled: false, ip: {0, 0, 0, 0}, diff --git a/lib/healthcheck.ex b/lib/pleroma/healthcheck.ex similarity index 98% rename from lib/healthcheck.ex rename to lib/pleroma/healthcheck.ex index f97d14432..977b78c26 100644 --- a/lib/healthcheck.ex +++ b/lib/pleroma/healthcheck.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Healthcheck do alias Pleroma.Healthcheck alias Pleroma.Repo + @derive Jason.Encoder defstruct pool_size: 0, active: 0, idle: 0,