From 063e6b9841ec72c7e89339c54581d199fa31e675 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 9 Jun 2020 10:53:40 +0200 Subject: [PATCH 1/8] StatusController: Correctly paginate favorites. Favorites were paginating wrongly, because the pagination headers where using the id of the id of the `Create` activity, while the ordering was by the id of the `Like` activity. This isn't easy to notice in most cases, as they usually have a similar order because people tend to favorite posts as they come in. This commit adds a way to give different pagination ids to the pagination helper, so we can paginate correctly in cases like this. --- lib/pleroma/activity.ex | 4 ++ lib/pleroma/web/activity_pub/activity_pub.ex | 5 +- .../api_spec/operations/status_operation.ex | 3 +- lib/pleroma/web/controller_helper.ex | 56 +++++++++++-------- .../controllers/status_controller_test.exs | 43 ++++++++++++-- 5 files changed, 79 insertions(+), 32 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 6213d0eb7..f800447fd 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -41,6 +41,10 @@ defmodule Pleroma.Activity do field(:recipients, {:array, :string}, default: []) field(:thread_muted?, :boolean, virtual: true) + # A field that can be used if you need to join some kind of other + # id to order / paginate this field by + field(:pagination_id, :string, virtual: true) + # This is a fake relation, # do not use outside of with_preloaded_user_actor/with_joined_user_actor has_one(:user_actor, User, on_delete: :nothing, foreign_key: :id) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index eb73c95fe..cc883ccce 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1138,12 +1138,11 @@ def fetch_favourites(user, params \\ %{}, pagination \\ :keyset) do |> Activity.Queries.by_type("Like") |> Activity.with_joined_object() |> Object.with_joined_activity() - |> select([_like, object, activity], %{activity | object: object}) + |> select([like, object, activity], %{activity | object: object, pagination_id: like.id}) |> order_by([like, _, _], desc_nulls_last: like.id) |> Pagination.fetch_paginated( Map.merge(params, %{skip_order: true}), - pagination, - :object_activity + pagination ) end diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index ca9db01e5..0b7fad793 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -333,7 +333,8 @@ def favourites_operation do %Operation{ tags: ["Statuses"], summary: "Favourited statuses", - description: "Statuses the user has favourited", + description: + "Statuses the user has favourited. Please note that you have to use the link headers to paginate this. You can not build the query parameters yourself.", operationId: "StatusController.favourites", parameters: pagination_params(), security: [%{"oAuth" => ["read:favourites"]}], diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index 5d67d75b5..5e33e0810 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -57,35 +57,45 @@ def add_link_headers(conn, activities, extra_params) do end end + defp build_pagination_fields(conn, min_id, max_id, extra_params) do + params = + conn.params + |> Map.drop(Map.keys(conn.path_params)) + |> Map.merge(extra_params) + |> Map.drop(Pagination.page_keys() -- ["limit", "order"]) + + fields = %{ + "next" => current_url(conn, Map.put(params, :max_id, max_id)), + "prev" => current_url(conn, Map.put(params, :min_id, min_id)) + } + + # Generating an `id` without already present pagination keys would + # need a query-restriction with an `q.id >= ^id` or `q.id <= ^id` + # instead of the `q.id > ^min_id` and `q.id < ^max_id`. + # This is because we only have ids present inside of the page, while + # `min_id`, `since_id` and `max_id` requires to know one outside of it. + if Map.take(conn.params, Pagination.page_keys() -- ["limit", "order"]) != [] do + Map.put(fields, "id", current_url(conn, conn.params)) + else + fields + end + end + def get_pagination_fields(conn, activities, extra_params \\ %{}) do case List.last(activities) do - %{id: max_id} -> - params = - conn.params - |> Map.drop(Map.keys(conn.path_params)) - |> Map.merge(extra_params) - |> Map.drop(Pagination.page_keys() -- ["limit", "order"]) - - min_id = + %{pagination_id: max_id} when not is_nil(max_id) -> + %{pagination_id: min_id} = activities |> List.first() - |> Map.get(:id) - fields = %{ - "next" => current_url(conn, Map.put(params, :max_id, max_id)), - "prev" => current_url(conn, Map.put(params, :min_id, min_id)) - } + build_pagination_fields(conn, min_id, max_id, extra_params) - # Generating an `id` without already present pagination keys would - # need a query-restriction with an `q.id >= ^id` or `q.id <= ^id` - # instead of the `q.id > ^min_id` and `q.id < ^max_id`. - # This is because we only have ids present inside of the page, while - # `min_id`, `since_id` and `max_id` requires to know one outside of it. - if Map.take(conn.params, Pagination.page_keys() -- ["limit", "order"]) != [] do - Map.put(fields, "id", current_url(conn, conn.params)) - else - fields - end + %{id: max_id} -> + %{id: min_id} = + activities + |> List.first() + + build_pagination_fields(conn, min_id, max_id, extra_params) _ -> %{} diff --git a/test/web/mastodon_api/controllers/status_controller_test.exs b/test/web/mastodon_api/controllers/status_controller_test.exs index 700c82e4f..648e6f2ce 100644 --- a/test/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/web/mastodon_api/controllers/status_controller_test.exs @@ -1541,14 +1541,49 @@ test "context" do } = response end + test "favorites paginate correctly" do + %{user: user, conn: conn} = oauth_access(["read:favourites"]) + other_user = insert(:user) + {:ok, first_post} = CommonAPI.post(other_user, %{status: "bla"}) + {:ok, second_post} = CommonAPI.post(other_user, %{status: "bla"}) + {:ok, third_post} = CommonAPI.post(other_user, %{status: "bla"}) + + {:ok, _first_favorite} = CommonAPI.favorite(user, third_post.id) + {:ok, _second_favorite} = CommonAPI.favorite(user, first_post.id) + {:ok, third_favorite} = CommonAPI.favorite(user, second_post.id) + + result = + conn + |> get("/api/v1/favourites?limit=1") + + assert [%{"id" => post_id}] = json_response_and_validate_schema(result, 200) + assert post_id == second_post.id + + # Using the header for pagination works correctly + [next, _] = get_resp_header(result, "link") |> hd() |> String.split(", ") + [_, max_id] = Regex.run(~r/max_id=(.*)>;/, next) + + assert max_id == third_favorite.id + + result = + conn + |> get("/api/v1/favourites?max_id=#{max_id}") + + assert [%{"id" => first_post_id}, %{"id" => third_post_id}] = + json_response_and_validate_schema(result, 200) + + assert first_post_id == first_post.id + assert third_post_id == third_post.id + end + test "returns the favorites of a user" do %{user: user, conn: conn} = oauth_access(["read:favourites"]) other_user = insert(:user) {:ok, _} = CommonAPI.post(other_user, %{status: "bla"}) - {:ok, activity} = CommonAPI.post(other_user, %{status: "traps are happy"}) + {:ok, activity} = CommonAPI.post(other_user, %{status: "trees are happy"}) - {:ok, _} = CommonAPI.favorite(user, activity.id) + {:ok, last_like} = CommonAPI.favorite(user, activity.id) first_conn = get(conn, "/api/v1/favourites") @@ -1566,9 +1601,7 @@ test "returns the favorites of a user" do {:ok, _} = CommonAPI.favorite(user, second_activity.id) - last_like = status["id"] - - second_conn = get(conn, "/api/v1/favourites?since_id=#{last_like}") + second_conn = get(conn, "/api/v1/favourites?since_id=#{last_like.id}") assert [second_status] = json_response_and_validate_schema(second_conn, 200) assert second_status["id"] == to_string(second_activity.id) From c4f267b3bef90dcac21b7db2a91f86d3ba5dc7c2 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 10 Jun 2020 08:02:26 +0000 Subject: [PATCH 2/8] Apply suggestion to lib/pleroma/web/controller_helper.ex --- lib/pleroma/web/controller_helper.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index 5e33e0810..6cb19d539 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -57,6 +57,7 @@ def add_link_headers(conn, activities, extra_params) do end end + @id_keys Pagination.page_keys() -- ["limit", "order"] defp build_pagination_fields(conn, min_id, max_id, extra_params) do params = conn.params From be7c322865b2b7aa1c8c25147cc598b6362ab187 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 10 Jun 2020 08:02:35 +0000 Subject: [PATCH 3/8] Apply suggestion to lib/pleroma/web/controller_helper.ex --- lib/pleroma/web/controller_helper.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index 6cb19d539..b7971e940 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -63,7 +63,7 @@ defp build_pagination_fields(conn, min_id, max_id, extra_params) do conn.params |> Map.drop(Map.keys(conn.path_params)) |> Map.merge(extra_params) - |> Map.drop(Pagination.page_keys() -- ["limit", "order"]) + |> Map.drop(@id_keys) fields = %{ "next" => current_url(conn, Map.put(params, :max_id, max_id)), From b4c50be9df701dc9faf0a25f776f631d2175c99f Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 10 Jun 2020 08:12:29 +0000 Subject: [PATCH 4/8] Apply suggestion to lib/pleroma/web/controller_helper.ex --- lib/pleroma/web/controller_helper.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index b7971e940..ab6e6c61a 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -75,7 +75,7 @@ defp build_pagination_fields(conn, min_id, max_id, extra_params) do # instead of the `q.id > ^min_id` and `q.id < ^max_id`. # This is because we only have ids present inside of the page, while # `min_id`, `since_id` and `max_id` requires to know one outside of it. - if Map.take(conn.params, Pagination.page_keys() -- ["limit", "order"]) != [] do + if Map.take(conn.params, @id_keys) != %{} do Map.put(fields, "id", current_url(conn, conn.params)) else fields From 86fec45f40dfa45cc89eddc6dcc7799e89d6f461 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 10 Jun 2020 11:09:45 +0200 Subject: [PATCH 5/8] ControllerHelper: Fix wrong comparison. --- lib/pleroma/web/controller_helper.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index ab6e6c61a..88f2cc6f1 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -75,7 +75,7 @@ defp build_pagination_fields(conn, min_id, max_id, extra_params) do # instead of the `q.id > ^min_id` and `q.id < ^max_id`. # This is because we only have ids present inside of the page, while # `min_id`, `since_id` and `max_id` requires to know one outside of it. - if Map.take(conn.params, @id_keys) != %{} do + if Map.take(conn.params, @id_keys) != [] do Map.put(fields, "id", current_url(conn, conn.params)) else fields From 5e44e9d69871f2e5805a8dddcfce43ae713eb52d Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 10 Jun 2020 18:56:46 +0000 Subject: [PATCH 6/8] Apply suggestion to lib/pleroma/web/controller_helper.ex --- lib/pleroma/web/controller_helper.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index 88f2cc6f1..a5eb3e9e0 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -76,7 +76,7 @@ defp build_pagination_fields(conn, min_id, max_id, extra_params) do # This is because we only have ids present inside of the page, while # `min_id`, `since_id` and `max_id` requires to know one outside of it. if Map.take(conn.params, @id_keys) != [] do - Map.put(fields, "id", current_url(conn, conn.params)) + Map.put(fields, "id", current_url(conn)) else fields end From 4b865bba107b0db1de886cefd14227454cbece1e Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 13 Jun 2020 10:37:15 +0000 Subject: [PATCH 7/8] Apply suggestion to lib/pleroma/web/controller_helper.ex --- lib/pleroma/web/controller_helper.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index a5eb3e9e0..d5e9c33f5 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -75,7 +75,7 @@ defp build_pagination_fields(conn, min_id, max_id, extra_params) do # instead of the `q.id > ^min_id` and `q.id < ^max_id`. # This is because we only have ids present inside of the page, while # `min_id`, `since_id` and `max_id` requires to know one outside of it. - if Map.take(conn.params, @id_keys) != [] do + if Map.take(conn.params, @id_keys) != %{} do Map.put(fields, "id", current_url(conn)) else fields From 1d625c29a09cf7c0fb415d5606a91315902efaad Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 13 Jun 2020 13:12:43 +0200 Subject: [PATCH 8/8] ControllerHelper: Always return id field. --- lib/pleroma/web/controller_helper.ex | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index d5e9c33f5..69946fb81 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -65,21 +65,11 @@ defp build_pagination_fields(conn, min_id, max_id, extra_params) do |> Map.merge(extra_params) |> Map.drop(@id_keys) - fields = %{ + %{ "next" => current_url(conn, Map.put(params, :max_id, max_id)), - "prev" => current_url(conn, Map.put(params, :min_id, min_id)) + "prev" => current_url(conn, Map.put(params, :min_id, min_id)), + "id" => current_url(conn) } - - # Generating an `id` without already present pagination keys would - # need a query-restriction with an `q.id >= ^id` or `q.id <= ^id` - # instead of the `q.id > ^min_id` and `q.id < ^max_id`. - # This is because we only have ids present inside of the page, while - # `min_id`, `since_id` and `max_id` requires to know one outside of it. - if Map.take(conn.params, @id_keys) != %{} do - Map.put(fields, "id", current_url(conn)) - else - fields - end end def get_pagination_fields(conn, activities, extra_params \\ %{}) do