From af4cf35e2096a6d1660271f6935b6b9ce77c6757 Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Sat, 10 Aug 2019 18:47:40 +0000 Subject: [PATCH] Strip internal fields including likes from incoming and outgoing activities --- CHANGELOG.md | 2 ++ lib/mix/tasks/pleroma/database.ex | 36 +++++++++++++++++++ .../web/activity_pub/transmogrifier.ex | 34 ++---------------- test/tasks/database_test.exs | 36 +++++++++++++++++++ test/web/activity_pub/transmogrifier_test.exs | 30 +++++++++++----- 5 files changed, 98 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31caef499..759779034 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: Endpoint for fetching latest user's statuses - Pleroma API: Add `/api/v1/pleroma/accounts/confirmation_resend?email=` for resending account confirmation. - Relays: Added a task to list relay subscriptions. +- Mix Tasks: `mix pleroma.database fix_likes_collections` +- Federation: Remove `likes` from objects. ### Changed - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 8547a329a..bcc2052d6 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -36,6 +36,10 @@ defmodule Mix.Tasks.Pleroma.Database do ## Remove duplicated items from following and update followers count for all users mix pleroma.database update_users_following_followers_counts + + ## Fix the pre-existing "likes" collections for all objects + + mix pleroma.database fix_likes_collections """ def run(["remove_embedded_objects" | args]) do {options, [], []} = @@ -125,4 +129,36 @@ def run(["prune_objects" | args]) do ) end end + + def run(["fix_likes_collections"]) do + import Ecto.Query + + start_pleroma() + + from(object in Object, + where: fragment("(?)->>'likes' is not null", object.data), + select: %{id: object.id, likes: fragment("(?)->>'likes'", object.data)} + ) + |> Pleroma.RepoStreamer.chunk_stream(100) + |> Stream.each(fn objects -> + ids = + objects + |> Enum.filter(fn object -> object.likes |> Jason.decode!() |> is_map() end) + |> Enum.map(& &1.id) + + Object + |> where([object], object.id in ^ids) + |> update([object], + set: [ + data: + fragment( + "jsonb_set(?, '{likes}', '[]'::jsonb, true)", + object.data + ) + ] + ) + |> Repo.update_all([], timeout: :infinity) + end) + |> Stream.run() + end end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 5403b71d8..b7bc48f0a 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -26,6 +26,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do """ def fix_object(object, options \\ []) do object + |> strip_internal_fields |> fix_actor |> fix_url |> fix_attachments @@ -34,7 +35,6 @@ def fix_object(object, options \\ []) do |> fix_emoji |> fix_tag |> fix_content_map - |> fix_likes |> fix_addressing |> fix_summary |> fix_type(options) @@ -151,20 +151,6 @@ def fix_actor(%{"attributedTo" => actor} = object) do |> Map.put("actor", Containment.get_actor(%{"actor" => actor})) end - # Check for standardisation - # This is what Peertube does - # curl -H 'Accept: application/activity+json' $likes | jq .totalItems - # Prismo returns only an integer (count) as "likes" - def fix_likes(%{"likes" => likes} = object) when not is_map(likes) do - object - |> Map.put("likes", []) - |> Map.put("like_count", 0) - end - - def fix_likes(object) do - object - end - def fix_in_reply_to(object, options \\ []) def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options) @@ -784,7 +770,6 @@ def prepare_object(object) do |> add_mention_tags |> add_emoji_tags |> add_attributed_to - |> add_likes |> prepare_attachments |> set_conversation |> set_reply_to_uri @@ -971,22 +956,6 @@ def add_attributed_to(object) do |> Map.put("attributedTo", attributed_to) end - def add_likes(%{"id" => id, "like_count" => likes} = object) do - likes = %{ - "id" => "#{id}/likes", - "first" => "#{id}/likes?page=1", - "type" => "OrderedCollection", - "totalItems" => likes - } - - object - |> Map.put("likes", likes) - end - - def add_likes(object) do - object - end - def prepare_attachments(object) do attachments = (object["attachment"] || []) @@ -1002,6 +971,7 @@ def prepare_attachments(object) do defp strip_internal_fields(object) do object |> Map.drop([ + "likes", "like_count", "announcements", "announcement_count", diff --git a/test/tasks/database_test.exs b/test/tasks/database_test.exs index 579130b05..a8f25f500 100644 --- a/test/tasks/database_test.exs +++ b/test/tasks/database_test.exs @@ -3,8 +3,11 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.DatabaseTest do + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Web.CommonAPI + use Pleroma.DataCase import Pleroma.Factory @@ -46,4 +49,37 @@ test "following and followers count are updated" do assert user.info.follower_count == 0 end end + + describe "running fix_likes_collections" do + test "it turns OrderedCollection likes into empty arrays" do + [user, user2] = insert_pair(:user) + + {:ok, %{id: id, object: object}} = CommonAPI.post(user, %{"status" => "test"}) + {:ok, %{object: object2}} = CommonAPI.post(user, %{"status" => "test test"}) + + CommonAPI.favorite(id, user2) + + likes = %{ + "first" => + "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1", + "id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes", + "totalItems" => 3, + "type" => "OrderedCollection" + } + + new_data = Map.put(object2.data, "likes", likes) + + object2 + |> Ecto.Changeset.change(%{data: new_data}) + |> Repo.update() + + assert length(Object.get_by_id(object.id).data["likes"]) == 1 + assert is_map(Object.get_by_id(object2.id).data["likes"]) + + assert :ok == Mix.Tasks.Pleroma.Database.run(["fix_likes_collections"]) + + assert length(Object.get_by_id(object.id).data["likes"]) == 1 + assert Enum.empty?(Object.get_by_id(object2.id).data["likes"]) + end + end end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index e7498e005..060b91e29 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -450,6 +450,27 @@ test "it ensures that address fields become lists" do assert !is_nil(data["cc"]) end + test "it strips internal likes" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + + likes = %{ + "first" => + "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1", + "id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes", + "totalItems" => 3, + "type" => "OrderedCollection" + } + + object = Map.put(data["object"], "likes", likes) + data = Map.put(data, "object", object) + + {:ok, %Activity{object: object}} = Transmogrifier.handle_incoming(data) + + refute Map.has_key?(object.data, "likes") + end + test "it works for incoming update activities" do data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() @@ -1061,14 +1082,7 @@ test "it strips internal fields of article" do assert is_nil(modified["object"]["announcements"]) assert is_nil(modified["object"]["announcement_count"]) assert is_nil(modified["object"]["context_id"]) - end - - test "it adds like collection to object" do - activity = insert(:note_activity) - {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data) - - assert modified["object"]["likes"]["type"] == "OrderedCollection" - assert modified["object"]["likes"]["totalItems"] == 0 + assert is_nil(modified["object"]["likes"]) end test "the directMessage flag is present" do