diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex
index 37d455cfc..e76eb0087 100644
--- a/lib/pleroma/conversation.ex
+++ b/lib/pleroma/conversation.ex
@@ -63,7 +63,7 @@ def create_or_bump_for(activity, opts \\ []) do
          ap_id when is_binary(ap_id) and byte_size(ap_id) > 0 <- object.data["context"] do
       {:ok, conversation} = create_for_ap_id(ap_id)
 
-      users = User.get_users_from_set(activity.recipients, false)
+      users = User.get_users_from_set(activity.recipients, local_only: false)
 
       participations =
         Enum.map(users, fn user ->
diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index fb16ec896..7eca55ac9 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -361,7 +361,8 @@ def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, lo
       when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact"] do
     potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity)
 
-    potential_receivers = User.get_users_from_set(potential_receiver_ap_ids, local_only)
+    potential_receivers =
+      User.get_users_from_set(potential_receiver_ap_ids, local_only: local_only)
 
     notification_enabled_ap_ids =
       potential_receiver_ap_ids
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 842b28c06..42c4c4e3e 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1208,8 +1208,9 @@ def increment_unread_conversation_count(conversation, %User{local: true} = user)
 
   def increment_unread_conversation_count(_, user), do: {:ok, user}
 
-  @spec get_users_from_set([String.t()], boolean()) :: [User.t()]
-  def get_users_from_set(ap_ids, local_only \\ true) do
+  @spec get_users_from_set([String.t()], keyword()) :: [User.t()]
+  def get_users_from_set(ap_ids, opts \\ []) do
+    local_only = Keyword.get(opts, :local_only, true)
     criteria = %{ap_id: ap_ids, deactivated: false}
     criteria = if local_only, do: Map.put(criteria, :local, true), else: criteria
 
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index d594c64f4..8443c284c 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -1045,10 +1045,14 @@ def add_hashtags(object) do
     Map.put(object, "tag", tags)
   end
 
+  # TODO These should be added on our side on insertion, it doesn't make much
+  # sense to regenerate these all the time
   def add_mention_tags(object) do
-    {enabled_receivers, disabled_receivers} = Utils.get_notified_from_object(object)
-    potential_receivers = enabled_receivers ++ disabled_receivers
-    mentions = Enum.map(potential_receivers, &build_mention_tag/1)
+    to = object["to"] || []
+    cc = object["cc"] || []
+    mentioned = User.get_users_from_set(to ++ cc, local_only: false)
+
+    mentions = Enum.map(mentioned, &build_mention_tag/1)
 
     tags = object["tag"] || []
     Map.put(object, "tag", tags ++ mentions)
diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs
index 356004d48..94d8552e8 100644
--- a/test/web/activity_pub/transmogrifier_test.exs
+++ b/test/web/activity_pub/transmogrifier_test.exs
@@ -1094,23 +1094,28 @@ test "it turns mentions into tags" do
       {:ok, activity} =
         CommonAPI.post(user, %{status: "hey, @#{other_user.nickname}, how are ya? #2hu"})
 
-      {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
-      object = modified["object"]
+      with_mock Pleroma.Notification,
+        get_notified_from_activity: fn _, _ -> [] end do
+        {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
 
-      expected_mention = %{
-        "href" => other_user.ap_id,
-        "name" => "@#{other_user.nickname}",
-        "type" => "Mention"
-      }
+        object = modified["object"]
 
-      expected_tag = %{
-        "href" => Pleroma.Web.Endpoint.url() <> "/tags/2hu",
-        "type" => "Hashtag",
-        "name" => "#2hu"
-      }
+        expected_mention = %{
+          "href" => other_user.ap_id,
+          "name" => "@#{other_user.nickname}",
+          "type" => "Mention"
+        }
 
-      assert Enum.member?(object["tag"], expected_tag)
-      assert Enum.member?(object["tag"], expected_mention)
+        expected_tag = %{
+          "href" => Pleroma.Web.Endpoint.url() <> "/tags/2hu",
+          "type" => "Hashtag",
+          "name" => "#2hu"
+        }
+
+        refute called(Pleroma.Notification.get_notified_from_activity(:_, :_))
+        assert Enum.member?(object["tag"], expected_tag)
+        assert Enum.member?(object["tag"], expected_mention)
+      end
     end
 
     test "it adds the sensitive property" do