diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index ffaac767e..bb193475a 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -109,6 +109,10 @@ defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do
     {:ok, object}
   end
 
+  defp intersection(list1, list2) do
+    list1 -- list1 -- list2
+  end
+
   defp check_followers_only(%{host: actor_host} = _actor_info, object) do
     followers_only =
       Config.get([:mrf_simple, :followers_only])
@@ -125,8 +129,8 @@ defp check_followers_only(%{host: actor_host} = _actor_info, object) do
         cc = FollowingRelationship.followers_ap_ids(user, fixed_cc)
 
         object
-        |> Map.put("to", [user.follower_address] ++ to)
-        |> Map.put("cc", cc)
+        |> Map.put("to", intersection([user.follower_address | to], fixed_to))
+        |> Map.put("cc", intersection([user.follower_address | cc], fixed_cc))
       else
         _ -> object
       end
diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs
index 9a1a7bdc8..d7dde62c4 100644
--- a/test/web/activity_pub/mrf/simple_policy_test.exs
+++ b/test/web/activity_pub/mrf/simple_policy_test.exs
@@ -308,7 +308,7 @@ test "has a matching host" do
       Config.put([:mrf_simple, :followers_only], [actor_domain])
 
       assert {:ok, new_activity} = SimplePolicy.filter(activity)
-      assert actor.follower_address in new_activity["to"]
+      assert actor.follower_address in new_activity["cc"]
       assert following_user.ap_id in new_activity["to"]
       refute "https://www.w3.org/ns/activitystreams#Public" in new_activity["to"]
       refute "https://www.w3.org/ns/activitystreams#Public" in new_activity["cc"]