diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 64b6c0aa1..b3d0712d4 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -440,11 +440,13 @@ def update_note_count(%User{} = user) do
 
     note_count = Repo.one(note_count_query)
 
-    new_info = Map.put(user.info, "note_count", note_count)
+    info_cng = User.Info.set_note_count(user.info, note_count)
 
-    cs = info_changeset(user, %{info: new_info})
+    cng =
+      change(user)
+      |> put_embed(:info, info_cng)
 
-    update_and_set_cache(cs)
+    update_and_set_cache(cng)
   end
 
   def update_follower_count(%User{} = user) do
diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex
index cae5562f0..619e58b16 100644
--- a/lib/pleroma/user/info.ex
+++ b/lib/pleroma/user/info.ex
@@ -27,6 +27,10 @@ def set_activation_status(info, deactivated) do
   end
 
   def add_to_note_count(info, number) do
+    set_note_count(info, info.note_count + number)
+  end
+
+  def set_note_count(info, number) do
     params = %{note_count: Enum.max([0, number])}
 
     info
diff --git a/test/user_test.exs b/test/user_test.exs
index fdc908dcd..3d330b963 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -311,11 +311,11 @@ test "it sets the info->note_count property" do
 
       user = User.get_by_ap_id(note.data["actor"])
 
-      assert user.info["note_count"] == nil
+      assert user.info.note_count == 0
 
       {:ok, user} = User.update_note_count(user)
 
-      assert user.info["note_count"] == 1
+      assert user.info.note_count == 1
     end
 
     test "it increases the info->note_count property" do