diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index c252e8bff..2e9b01205 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -406,6 +406,8 @@ def follow(%User{} = follower, %User{info: info} = followed) do
 
         {1, [follower]} = Repo.update_all(q, [])
 
+        follower = maybe_update_following_count(follower)
+
         {:ok, _} = update_follower_count(followed)
 
         set_cache(follower)
@@ -425,6 +427,8 @@ def unfollow(%User{} = follower, %User{} = followed) do
 
       {1, [follower]} = Repo.update_all(q, [])
 
+      follower = maybe_update_following_count(follower)
+
       {:ok, followed} = update_follower_count(followed)
 
       set_cache(follower)
@@ -698,32 +702,75 @@ def update_note_count(%User{} = user) do
     |> update_and_set_cache()
   end
 
-  def update_follower_count(%User{} = user) do
-    follower_count_query =
-      User.Query.build(%{followers: user, deactivated: false})
-      |> select([u], %{count: count(u.id)})
+  def maybe_fetch_follow_information(user) do
+    with {:ok, user} <- fetch_follow_information(user) do
+      user
+    else
+      e ->
+        Logger.error(
+          "Follower/Following counter update for #{user.ap_id} failed.\n" <> inspect(e)
+        )
 
-    User
-    |> where(id: ^user.id)
-    |> join(:inner, [u], s in subquery(follower_count_query))
-    |> update([u, s],
-      set: [
-        info:
-          fragment(
-            "jsonb_set(?, '{follower_count}', ?::varchar::jsonb, true)",
-            u.info,
-            s.count
-          )
-      ]
-    )
-    |> select([u], u)
-    |> Repo.update_all([])
-    |> case do
-      {1, [user]} -> set_cache(user)
-      _ -> {:error, user}
+        user
     end
   end
 
+  def fetch_follow_information(user) do
+    with {:ok, info} <- ActivityPub.fetch_follow_information_for_user(user) do
+      info_cng = User.Info.follow_information_update(user.info, info)
+
+      changeset =
+        user
+        |> change()
+        |> put_embed(:info, info_cng)
+
+      update_and_set_cache(changeset)
+    else
+      {:error, _} = e -> e
+      e -> {:error, e}
+    end
+  end
+
+  def update_follower_count(%User{} = user) do
+    unless user.local == false and Pleroma.Config.get([:instance, :external_user_synchronization]) do
+      follower_count_query =
+        User.Query.build(%{followers: user, deactivated: false})
+        |> select([u], %{count: count(u.id)})
+
+      User
+      |> where(id: ^user.id)
+      |> join(:inner, [u], s in subquery(follower_count_query))
+      |> update([u, s],
+        set: [
+          info:
+            fragment(
+              "jsonb_set(?, '{follower_count}', ?::varchar::jsonb, true)",
+              u.info,
+              s.count
+            )
+        ]
+      )
+      |> select([u], u)
+      |> Repo.update_all([])
+      |> case do
+        {1, [user]} -> set_cache(user)
+        _ -> {:error, user}
+      end
+    else
+      {:ok, maybe_fetch_follow_information(user)}
+    end
+  end
+
+  def maybe_update_following_count(%User{local: false} = user) do
+    if Pleroma.Config.get([:instance, :external_user_synchronization]) do
+      {:ok, maybe_fetch_follow_information(user)}
+    else
+      user
+    end
+  end
+
+  def maybe_update_following_count(user), do: user
+
   def remove_duplicated_following(%User{following: following} = user) do
     uniq_following = Enum.uniq(following)
 
diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex
index 67e8801ea..4cc3f2f2c 100644
--- a/lib/pleroma/user/info.ex
+++ b/lib/pleroma/user/info.ex
@@ -330,4 +330,14 @@ def remove_reblog_mute(info, ap_id) do
 
     cast(info, params, [:muted_reblogs])
   end
+
+  def follow_information_update(info, params) do
+    info
+    |> cast(params, [
+      :hide_followers,
+      :hide_follows,
+      :follower_count,
+      :following_count
+    ])
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index c821ba45f..2dd9dbf7f 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1022,15 +1022,13 @@ def fetch_follow_information_for_user(user) do
            Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address),
          followers_count when is_integer(followers_count) <- followers_data["totalItems"],
          {:ok, hide_followers} <- collection_private(followers_data) do
-      info = %{
-        hide_follows: hide_follows,
-        follower_count: followers_count,
-        following_count: following_count,
-        hide_followers: hide_followers
-      }
-
-      info = Map.merge(user.info, info)
-      {:ok, Map.put(user, :info, info)}
+      {:ok,
+       %{
+         hide_follows: hide_follows,
+         follower_count: followers_count,
+         following_count: following_count,
+         hide_followers: hide_followers
+       }}
     else
       {:error, _} = e ->
         e
@@ -1043,8 +1041,9 @@ def fetch_follow_information_for_user(user) do
   defp maybe_update_follow_information(data) do
     with {:enabled, true} <-
            {:enabled, Pleroma.Config.get([:instance, :external_user_synchronization])},
-         {:ok, data} <- fetch_follow_information_for_user(data) do
-      data
+         {:ok, info} <- fetch_follow_information_for_user(data) do
+      info = Map.merge(data.info, info)
+      Map.put(data, :info, info)
     else
       {:enabled, false} ->
         data
diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
index 448ffbf54..24d8493fe 100644
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@ -1232,9 +1232,9 @@ test "syncronizes following/followers counters" do
           following_address: "http://localhost:4001/users/fuser2/following"
         )
 
-      {:ok, user} = ActivityPub.fetch_follow_information_for_user(user)
-      assert user.info.follower_count == 527
-      assert user.info.following_count == 267
+      {:ok, info} = ActivityPub.fetch_follow_information_for_user(user)
+      assert info.follower_count == 527
+      assert info.following_count == 267
     end
 
     test "detects hidden followers" do
@@ -1265,9 +1265,9 @@ test "detects hidden followers" do
           following_address: "http://localhost:4001/users/masto_closed/following"
         )
 
-      {:ok, user} = ActivityPub.fetch_follow_information_for_user(user)
-      assert user.info.hide_followers == true
-      assert user.info.hide_follows == false
+      {:ok, info} = ActivityPub.fetch_follow_information_for_user(user)
+      assert info.hide_followers == true
+      assert info.hide_follows == false
     end
 
     test "detects hidden follows" do
@@ -1298,9 +1298,9 @@ test "detects hidden follows" do
           following_address: "http://localhost:4001/users/masto_closed/following"
         )
 
-      {:ok, user} = ActivityPub.fetch_follow_information_for_user(user)
-      assert user.info.hide_followers == false
-      assert user.info.hide_follows == true
+      {:ok, info} = ActivityPub.fetch_follow_information_for_user(user)
+      assert info.hide_followers == false
+      assert info.hide_follows == true
     end
   end
 end