diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex
index e1f88e6cc..f2392ce79 100644
--- a/lib/pleroma/web/activity_pub/builder.ex
+++ b/lib/pleroma/web/activity_pub/builder.ex
@@ -14,19 +14,28 @@ defmodule Pleroma.Web.ActivityPub.Builder do
 
   require Pleroma.Constants
 
-  @spec accept(User.t(), Activity.t()) :: {:ok, map(), keyword()}
-  def accept(actor, accepted_activity) do
+  def accept_or_reject(actor, activity, type) do
     data = %{
       "id" => Utils.generate_activity_id(),
       "actor" => actor.ap_id,
-      "type" => "Accept",
-      "object" => accepted_activity.data["id"],
-      "to" => [accepted_activity.actor]
+      "type" => type,
+      "object" => activity.data["id"],
+      "to" => [activity.actor]
     }
 
     {:ok, data, []}
   end
 
+  @spec reject(User.t(), Activity.t()) :: {:ok, map(), keyword()}
+  def reject(actor, rejected_activity) do
+    accept_or_reject(actor, rejected_activity, "Reject")
+  end
+
+  @spec accept(User.t(), Activity.t()) :: {:ok, map(), keyword()}
+  def accept(actor, accepted_activity) do
+    accept_or_reject(actor, accepted_activity, "Accept")
+  end
+
   @spec follow(User.t(), User.t()) :: {:ok, map(), keyword()}
   def follow(follower, followed) do
     data = %{
diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index d9dd2bc30..3f1dffe2b 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -13,7 +13,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object
   alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator
@@ -31,10 +31,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()}
   def validate(object, meta)
 
-  def validate(%{"type" => "Accept"} = object, meta) do
+  def validate(%{"type" => type} = object, meta)
+      when type in ~w[Accept Reject] do
     with {:ok, object} <-
            object
-           |> AcceptValidator.cast_and_validate()
+           |> AcceptRejectValidator.cast_and_validate()
            |> Ecto.Changeset.apply_action(:insert) do
       object = stringify_keys(object)
       {:ok, object, meta}
diff --git a/lib/pleroma/web/activity_pub/object_validators/accept_validator.ex b/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex
similarity index 82%
rename from lib/pleroma/web/activity_pub/object_validators/accept_validator.ex
rename to lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex
index fd75f4b6e..179beda58 100644
--- a/lib/pleroma/web/activity_pub/object_validators/accept_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex
@@ -2,7 +2,7 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.AcceptValidator do
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator do
   use Ecto.Schema
 
   alias Pleroma.Activity
@@ -30,10 +30,10 @@ def cast_data(data) do
   def validate_data(cng) do
     cng
     |> validate_required([:id, :type, :actor, :to, :cc, :object])
-    |> validate_inclusion(:type, ["Accept"])
+    |> validate_inclusion(:type, ["Accept", "Reject"])
     |> validate_actor_presence()
     |> validate_object_presence(allowed_types: ["Follow"])
-    |> validate_accept_rights()
+    |> validate_accept_reject_rights()
   end
 
   def cast_and_validate(data) do
@@ -42,7 +42,7 @@ def cast_and_validate(data) do
     |> validate_data
   end
 
-  def validate_accept_rights(cng) do
+  def validate_accept_reject_rights(cng) do
     with object_id when is_binary(object_id) <- get_field(cng, :object),
          %Activity{data: %{"object" => followed_actor}} <- Activity.get_by_ap_id(object_id),
          true <- followed_actor == get_field(cng, :actor) do
@@ -50,7 +50,7 @@ def validate_accept_rights(cng) do
     else
       _e ->
         cng
-        |> add_error(:actor, "can't accept the given activity")
+        |> add_error(:actor, "can't accept or reject the given activity")
     end
   end
 end
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index e1fa75e1c..a4ad12d53 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -52,6 +52,30 @@ def handle(
     {:ok, object, meta}
   end
 
+  # Task this handles
+  # - Rejects all existing follow activities for this person
+  # - Updates the follow state
+  def handle(
+        %{
+          data: %{
+            "actor" => actor,
+            "type" => "Reject",
+            "object" => follow_activity_id
+          }
+        } = object,
+        meta
+      ) do
+    with %Activity{actor: follower_id} = follow_activity <-
+           Activity.get_by_ap_id(follow_activity_id),
+         %User{} = followed <- User.get_cached_by_ap_id(actor),
+         %User{} = follower <- User.get_cached_by_ap_id(follower_id),
+         {:ok, _follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "reject") do
+      FollowingRelationship.update(follower, followed, :follow_reject)
+    end
+
+    {:ok, object, meta}
+  end
+
   # Tasks this handle
   # - Follows if possible
   # - Sends a notification
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 24da1ef9c..544f3f3b6 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -9,7 +9,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   alias Pleroma.Activity
   alias Pleroma.EarmarkRenderer
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
-  alias Pleroma.FollowingRelationship
   alias Pleroma.Maps
   alias Pleroma.Object
   alias Pleroma.Object.Containment
@@ -390,16 +389,6 @@ defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = objec
 
   defp fix_content(object), do: object
 
-  defp get_follow_activity(follow_object, _followed) do
-    with object_id when not is_nil(object_id) <- Utils.get_ap_id(follow_object),
-         {_, %Activity{} = activity} <- {:activity, Activity.get_by_ap_id(object_id)} do
-      {:ok, activity}
-    else
-      _ ->
-        {:error, nil}
-    end
-  end
-
   # Reduce the object list to find the reported user.
   defp get_reported(objects) do
     Enum.reduce_while(objects, nil, fn ap_id, _ ->
@@ -534,31 +523,6 @@ def handle_incoming(
     end
   end
 
-  def handle_incoming(
-        %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => id} = data,
-        _options
-      ) do
-    with actor <- Containment.get_actor(data),
-         {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor),
-         {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
-         {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "reject"),
-         %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
-         {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_reject),
-         {:ok, activity} <-
-           ActivityPub.reject(%{
-             to: follow_activity.data["to"],
-             type: "Reject",
-             actor: followed,
-             object: follow_activity.data["id"],
-             local: false,
-             activity_id: id
-           }) do
-      {:ok, activity}
-    else
-      _e -> :error
-    end
-  end
-
   @misskey_reactions %{
     "like" => "👍",
     "love" => "❤️",
@@ -613,7 +577,7 @@ def handle_incoming(
         %{"type" => type} = data,
         _options
       )
-      when type in ~w{Update Block Follow Accept} do
+      when type in ~w{Update Block Follow Accept Reject} do
     with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
          {:ok, activity, _} <-
            Pipeline.common_pipeline(data, local: false) do
diff --git a/test/web/activity_pub/object_validators/reject_validation_test.exs b/test/web/activity_pub/object_validators/reject_validation_test.exs
new file mode 100644
index 000000000..370bb6e5c
--- /dev/null
+++ b/test/web/activity_pub/object_validators/reject_validation_test.exs
@@ -0,0 +1,56 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.RejectValidationTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Web.ActivityPub.Builder
+  alias Pleroma.Web.ActivityPub.ObjectValidator
+  alias Pleroma.Web.ActivityPub.Pipeline
+
+  import Pleroma.Factory
+
+  setup do
+    follower = insert(:user)
+    followed = insert(:user, local: false)
+
+    {:ok, follow_data, _} = Builder.follow(follower, followed)
+    {:ok, follow_activity, _} = Pipeline.common_pipeline(follow_data, local: true)
+
+    {:ok, reject_data, _} = Builder.reject(followed, follow_activity)
+
+    %{reject_data: reject_data, followed: followed}
+  end
+
+  test "it validates a basic 'reject'", %{reject_data: reject_data} do
+    assert {:ok, _, _} = ObjectValidator.validate(reject_data, [])
+  end
+
+  test "it fails when the actor doesn't exist", %{reject_data: reject_data} do
+    reject_data =
+      reject_data
+      |> Map.put("actor", "https://gensokyo.2hu/users/raymoo")
+
+    assert {:error, _} = ObjectValidator.validate(reject_data, [])
+  end
+
+  test "it fails when the rejected activity doesn't exist", %{reject_data: reject_data} do
+    reject_data =
+      reject_data
+      |> Map.put("object", "https://gensokyo.2hu/users/raymoo/follows/1")
+
+    assert {:error, _} = ObjectValidator.validate(reject_data, [])
+  end
+
+  test "for an rejected follow, it only validates if the actor of the reject is the followed actor",
+       %{reject_data: reject_data} do
+    stranger = insert(:user)
+
+    reject_data =
+      reject_data
+      |> Map.put("actor", stranger.ap_id)
+
+    assert {:error, _} = ObjectValidator.validate(reject_data, [])
+  end
+end
diff --git a/test/web/activity_pub/transmogrifier/reject_handling_test.exs b/test/web/activity_pub/transmogrifier/reject_handling_test.exs
index 5e5248641..7592fbe1c 100644
--- a/test/web/activity_pub/transmogrifier/reject_handling_test.exs
+++ b/test/web/activity_pub/transmogrifier/reject_handling_test.exs
@@ -24,7 +24,7 @@ test "it fails for incoming rejects which cannot be correlated" do
     accept_data =
       Map.put(accept_data, "object", Map.put(accept_data["object"], "actor", follower.ap_id))
 
-    :error = Transmogrifier.handle_incoming(accept_data)
+    {:error, _} = Transmogrifier.handle_incoming(accept_data)
 
     follower = User.get_cached_by_id(follower.id)