Merge branch 'develop' into admin-be

CHANGELOG.md

@@ -94,6 +94,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Captcha: Support native provider
 - Captcha: Enable by default
 - Mastodon API: Add support for `account_id` param to filter notifications by the account
+- Mastodon API: Add `emoji_reactions` property to Statuses
 </details>
 
 ### Fixed

config/config.exs

@@ -500,7 +500,8 @@
     mailer: 10,
     transmogrifier: 20,
     scheduled_activities: 10,
-    background: 5
+    background: 5,
+    attachments_cleanup: 5
   ]
 
 config :pleroma, :workers,

docs/API/differences_in_mastoapi_responses.md

@@ -29,6 +29,7 @@ Has these additional fields under the `pleroma` object:
 - `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`
 - `expires_at`: a datetime (iso8601) that states when the post will expire (be deleted automatically), or empty if the post won't expire
 - `thread_muted`: true if the thread the post belongs to is muted
+- `emoji_reactions`: An object with all the emoji reactions with count. Contains no information about the reacting users, for that use the `emoji_reactions_by` endpoint.
 
 ## Attachments
 

lib/pleroma/activity.ex

@@ -312,9 +312,7 @@ def restrict_deactivated_users(query) do
       from(u in User.Query.build(deactivated: true), select: u.ap_id)
       |> Repo.all()
 
-    from(activity in query,
+    Activity.Queries.exclude_authors(query, deactivated_users)
-      where: activity.actor not in ^deactivated_users
-    )
   end
 
   defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search

lib/pleroma/activity/queries.ex

@@ -12,6 +12,7 @@ defmodule Pleroma.Activity.Queries do
   @type query :: Ecto.Queryable.t() | Activity.t()
 
   alias Pleroma.Activity
+  alias Pleroma.User
 
   @spec by_ap_id(query, String.t()) :: query
   def by_ap_id(query \\ Activity, ap_id) do
@@ -29,6 +30,11 @@ def by_actor(query \\ Activity, actor) do
     )
   end
 
+  @spec by_author(query, String.t()) :: query
+  def by_author(query \\ Activity, %User{ap_id: ap_id}) do
+    from(a in query, where: a.actor == ^ap_id)
+  end
+
   @spec by_object_id(query, String.t() | [String.t()]) :: query
   def by_object_id(query \\ Activity, object_id)
 
@@ -72,4 +78,8 @@ def exclude_type(query \\ Activity, activity_type) do
       where: fragment("(?)->>'type' != ?", activity.data, ^activity_type)
     )
   end
+
+  def exclude_authors(query \\ Activity, actors) do
+    from(activity in query, where: activity.actor not in ^actors)
+  end
 end

lib/pleroma/activity/search.ex

@@ -26,18 +26,23 @@ def search(user, search_query, options \\ []) do
     |> query_with(index_type, search_query)
     |> maybe_restrict_local(user)
     |> maybe_restrict_author(author)
+    |> maybe_restrict_blocked(user)
     |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => limit}, :offset)
     |> maybe_fetch(user, search_query)
   end
 
   def maybe_restrict_author(query, %User{} = author) do
-    from([a, o] in query,
+    Activity.Queries.by_author(query, author)
-      where: a.actor == ^author.ap_id
-    )
   end
 
   def maybe_restrict_author(query, _), do: query
 
+  def maybe_restrict_blocked(query, %User{} = user) do
+    Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user))
+  end
+
+  def maybe_restrict_blocked(query, _), do: query
+
   defp restrict_public(q) do
     from([a, o] in q,
       where: fragment("?->>'type' = 'Create'", a.data),

lib/pleroma/object.ex

@@ -19,6 +19,8 @@ defmodule Pleroma.Object do
 
   @type t() :: %__MODULE__{}
 
+  @derive {Jason.Encoder, only: [:data]}
+
   schema "objects" do
     field(:data, :map)
 
@@ -180,85 +182,17 @@ def swap_object_with_tombstone(object) do
 
   def delete(%Object{data: %{"id" => id}} = object) do
     with {:ok, _obj} = swap_object_with_tombstone(object),
-         :ok <- delete_attachments(object),
          deleted_activity = Activity.delete_all_by_object_ap_id(id),
          {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
-         {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
+         {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path),
+         {:ok, _} <-
+           Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{
+             "object" => object
+           }) do
       {:ok, object, deleted_activity}
     end
   end
 
-  defp delete_attachments(%{data: %{"attachment" => [_ | _] = attachments, "actor" => actor}}) do
-    hrefs =
-      Enum.flat_map(attachments, fn attachment ->
-        Enum.map(attachment["url"], & &1["href"])
-      end)
-
-    names = Enum.map(attachments, & &1["name"])
-
-    uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
-
-    # find all objects for copies of the attachments, name and actor doesn't matter here
-    delete_ids =
-      from(o in Object,
-        where:
-          fragment(
-            "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href'))::jsonb \\?| (?)",
-            o.data,
-            ^hrefs
-          )
-      )
-      |> Repo.all()
-      # we should delete 1 object for any given attachment, but don't delete files if
-      # there are more than 1 object for it
-      |> Enum.reduce(%{}, fn %{
-                               id: id,
-                               data: %{
-                                 "url" => [%{"href" => href}],
-                                 "actor" => obj_actor,
-                                 "name" => name
-                               }
-                             },
-                             acc ->
-        Map.update(acc, href, %{id: id, count: 1}, fn val ->
-          case obj_actor == actor and name in names do
-            true ->
-              # set id of the actor's object that will be deleted
-              %{val | id: id, count: val.count + 1}
-
-            false ->
-              # another actor's object, just increase count to not delete file
-              %{val | count: val.count + 1}
-          end
-        end)
-      end)
-      |> Enum.map(fn {href, %{id: id, count: count}} ->
-        # only delete files that have single instance
-        with 1 <- count do
-          prefix =
-            case Pleroma.Config.get([Pleroma.Upload, :base_url]) do
-              nil -> "media"
-              _ -> ""
-            end
-
-          base_url = Pleroma.Config.get([__MODULE__, :base_url], Pleroma.Web.base_url())
-
-          file_path = String.trim_leading(href, "#{base_url}/#{prefix}")
-
-          uploader.delete_file(file_path)
-        end
-
-        id
-      end)
-
-    from(o in Object, where: o.id in ^delete_ids)
-    |> Repo.delete_all()
-
-    :ok
-  end
-
-  defp delete_attachments(%{data: _data}), do: :ok
-
   def prune(%Object{data: %{"id" => id}} = object) do
     with {:ok, object} <- Repo.delete(object),
          {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),

lib/pleroma/web/mastodon_api/controllers/search_controller.ex

@@ -43,7 +43,7 @@ defp do_search(version, %{assigns: %{user: user}} = conn, %{"q" => query} = para
     result =
       default_values
       |> Enum.map(fn {resource, default_value} ->
-        if params["type"] == nil or params["type"] == resource do
+        if params["type"] in [nil, resource] do
           {resource, fn -> resource_search(version, resource, query, options) end}
         else
           {resource, fn -> default_value end}

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -253,6 +253,16 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
           nil
       end
 
+    emoji_reactions =
+      with %{data: %{"reactions" => emoji_reactions}} <- object do
+        Enum.map(emoji_reactions, fn {emoji, users} ->
+          {emoji, length(users)}
+        end)
+        |> Enum.into(%{})
+      else
+        _ -> %{}
+      end
+
     %{
       id: to_string(activity.id),
       uri: object.data["id"],
@@ -293,7 +303,8 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
         spoiler_text: %{"text/plain" => summary_plaintext},
         expires_at: expires_at,
         direct_conversation_id: direct_conversation_id,
-        thread_muted: thread_muted?
+        thread_muted: thread_muted?,
+        emoji_reactions: emoji_reactions
       }
     }
   end

lib/pleroma/workers/attachments_cleanup_worker.ex

@@ -0,0 +1,88 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Workers.AttachmentsCleanupWorker do
+  import Ecto.Query
+
+  alias Pleroma.Object
+  alias Pleroma.Repo
+
+  use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup"
+
+  @impl Oban.Worker
+  def perform(
+        %{"object" => %{"data" => %{"attachment" => [_ | _] = attachments, "actor" => actor}}},
+        _job
+      ) do
+    hrefs =
+      Enum.flat_map(attachments, fn attachment ->
+        Enum.map(attachment["url"], & &1["href"])
+      end)
+
+    names = Enum.map(attachments, & &1["name"])
+
+    uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
+
+    # find all objects for copies of the attachments, name and actor doesn't matter here
+    delete_ids =
+      from(o in Object,
+        where:
+          fragment(
+            "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)",
+            o.data,
+            o.data,
+            ^hrefs
+          )
+      )
+      # The query above can be time consumptive on large instances until we
+      # refactor how uploads are stored
+      |> Repo.all(timout: :infinity)
+      # we should delete 1 object for any given attachment, but don't delete
+      # files if there are more than 1 object for it
+      |> Enum.reduce(%{}, fn %{
+                               id: id,
+                               data: %{
+                                 "url" => [%{"href" => href}],
+                                 "actor" => obj_actor,
+                                 "name" => name
+                               }
+                             },
+                             acc ->
+        Map.update(acc, href, %{id: id, count: 1}, fn val ->
+          case obj_actor == actor and name in names do
+            true ->
+              # set id of the actor's object that will be deleted
+              %{val | id: id, count: val.count + 1}
+
+            false ->
+              # another actor's object, just increase count to not delete file
+              %{val | count: val.count + 1}
+          end
+        end)
+      end)
+      |> Enum.map(fn {href, %{id: id, count: count}} ->
+        # only delete files that have single instance
+        with 1 <- count do
+          prefix =
+            case Pleroma.Config.get([Pleroma.Upload, :base_url]) do
+              nil -> "media"
+              _ -> ""
+            end
+
+          base_url = Pleroma.Config.get([__MODULE__, :base_url], Pleroma.Web.base_url())
+
+          file_path = String.trim_leading(href, "#{base_url}/#{prefix}")
+
+          uploader.delete_file(file_path)
+        end
+
+        id
+      end)
+
+    from(o in Object, where: o.id in ^delete_ids)
+    |> Repo.delete_all()
+  end
+
+  def perform(%{"object" => _object}, _job), do: :ok
+end

priv/static/adminfe/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge,chrome=1"><meta name=renderer content=webkit><meta name=viewport content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><title>Admin FE</title><link rel="shortcut icon" href=favicon.ico><link href=chunk-elementUI.a842fb0a.css rel=stylesheet><link href=chunk-libs.57fe98a3.css rel=stylesheet><link href=app.fdd73ce4.css rel=stylesheet></head><body><div id=app></div><script type=text/javascript src=static/js/runtime.d6d1aaab.js></script><script type=text/javascript src=static/js/chunk-elementUI.fa319e7b.js></script><script type=text/javascript src=static/js/chunk-libs.35c18287.js></script><script type=text/javascript src=static/js/app.19b7049e.js></script></body></html>
+<!DOCTYPE html><html><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge,chrome=1"><meta name=renderer content=webkit><meta name=viewport content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><title>Admin FE</title><link rel="shortcut icon" href=favicon.ico><link href=chunk-elementUI.1abbc9b8.css rel=stylesheet><link href=chunk-libs.57fe98a3.css rel=stylesheet><link href=app.fdd73ce4.css rel=stylesheet></head><body><div id=app></div><script type=text/javascript src=static/js/runtime.cab03b3e.js></script><script type=text/javascript src=static/js/chunk-elementUI.2de79b84.js></script><script type=text/javascript src=static/js/chunk-libs.680db3fc.js></script><script type=text/javascript src=static/js/app.3da0f475.js></script></body></html>

test/object_test.exs

@@ -4,12 +4,14 @@
 
 defmodule Pleroma.ObjectTest do
   use Pleroma.DataCase
+  use Oban.Testing, repo: Pleroma.Repo
   import ExUnit.CaptureLog
   import Pleroma.Factory
   import Tesla.Mock
   alias Pleroma.Activity
   alias Pleroma.Object
   alias Pleroma.Repo
+  alias Pleroma.Tests.ObanHelpers
   alias Pleroma.Web.CommonAPI
 
   setup do
@@ -99,6 +101,8 @@ test "in subdirectories" do
 
       Object.delete(note)
 
+      ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))
+
       assert Object.get_by_id(attachment.id) == nil
 
       assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
@@ -133,10 +137,46 @@ test "with dedupe enabled" do
 
       Object.delete(note)
 
+      ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))
+
       assert Object.get_by_id(attachment.id) == nil
       assert {:ok, files} = File.ls(uploads_dir)
       refute filename in files
     end
+
+    test "with objects that have legacy data.url attribute" do
+      Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+
+      file = %Plug.Upload{
+        content_type: "image/jpg",
+        path: Path.absname("test/fixtures/image.jpg"),
+        filename: "an_image.jpg"
+      }
+
+      user = insert(:user)
+
+      {:ok, %Object{} = attachment} =
+        Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)
+
+      {:ok, %Object{}} = Object.create(%{url: "https://google.com", actor: user.ap_id})
+
+      %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
+        note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
+
+      uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
+
+      path = href |> Path.dirname() |> Path.basename()
+
+      assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
+
+      Object.delete(note)
+
+      ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))
+
+      assert Object.get_by_id(attachment.id) == nil
+
+      assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
+    end
   end
 
   describe "normalizer" do

test/web/mastodon_api/controllers/search_controller_test.exs

@@ -53,7 +53,8 @@ test "search", %{conn: conn} do
       {:ok, _} = CommonAPI.post(user_two, %{"status" => "This isn't"})
 
       results =
-        get(conn, "/api/v2/search", %{"q" => "2hu #private"})
+        conn
+        |> get("/api/v2/search", %{"q" => "2hu #private"})
         |> json_response(200)
 
       [account | _] = results["accounts"]
@@ -73,6 +74,30 @@ test "search", %{conn: conn} do
       [status] = results["statuses"]
       assert status["id"] == to_string(activity.id)
     end
+
+    test "excludes a blocked users from search results", %{conn: conn} do
+      user = insert(:user)
+      user_smith = insert(:user, %{nickname: "Agent", name: "I love 2hu"})
+      user_neo = insert(:user, %{nickname: "Agent Neo", name: "Agent"})
+
+      {:ok, act1} = CommonAPI.post(user, %{"status" => "This is about 2hu private 天子"})
+      {:ok, act2} = CommonAPI.post(user_smith, %{"status" => "Agent Smith"})
+      {:ok, act3} = CommonAPI.post(user_neo, %{"status" => "Agent Smith"})
+      Pleroma.User.block(user, user_smith)
+
+      results =
+        conn
+        |> assign(:user, user)
+        |> assign(:token, insert(:oauth_token, user: user, scopes: ["read"]))
+        |> get("/api/v2/search", %{"q" => "Agent"})
+        |> json_response(200)
+
+      status_ids = Enum.map(results["statuses"], fn g -> g["id"] end)
+
+      assert act3.id in status_ids
+      refute act2.id in status_ids
+      refute act1.id in status_ids
+    end
   end
 
   describe ".account_search" do
@@ -146,11 +171,10 @@ test "search", %{conn: conn} do
 
       {:ok, _} = CommonAPI.post(user_two, %{"status" => "This isn't"})
 
-      conn =
+      results =
         conn
         |> get("/api/v1/search", %{"q" => "2hu"})
-
+        |> json_response(200)
-      assert results = json_response(conn, 200)
 
       [account | _] = results["accounts"]
       assert account["id"] == to_string(user_three.id)
@@ -168,11 +192,10 @@ test "search fetches remote statuses and prefers them over other results", %{con
             "status" => "check out https://shitposter.club/notice/2827873"
           })
 
-        conn =
+        results =
           conn
           |> get("/api/v1/search", %{"q" => "https://shitposter.club/notice/2827873"})
-
+          |> json_response(200)
-        assert results = json_response(conn, 200)
 
         [status, %{"id" => ^activity_id}] = results["statuses"]
 
@@ -189,11 +212,10 @@ test "search doesn't show statuses that it shouldn't", %{conn: conn} do
         })
 
       capture_log(fn ->
-        conn =
+        results =
           conn
           |> get("/api/v1/search", %{"q" => Object.normalize(activity).data["id"]})
-
+          |> json_response(200)
-        assert results = json_response(conn, 200)
 
         [] = results["statuses"]
       end)
@@ -202,23 +224,23 @@ test "search doesn't show statuses that it shouldn't", %{conn: conn} do
     test "search fetches remote accounts", %{conn: conn} do
       user = insert(:user)
 
-      conn =
+      results =
         conn
         |> assign(:user, user)
         |> assign(:token, insert(:oauth_token, user: user, scopes: ["read"]))
         |> get("/api/v1/search", %{"q" => "mike@osada.macgirvin.com", "resolve" => "true"})
+        |> json_response(200)
 
-      assert results = json_response(conn, 200)
       [account] = results["accounts"]
       assert account["acct"] == "mike@osada.macgirvin.com"
     end
 
     test "search doesn't fetch remote accounts if resolve is false", %{conn: conn} do
-      conn =
+      results =
         conn
         |> get("/api/v1/search", %{"q" => "mike@osada.macgirvin.com", "resolve" => "false"})
+        |> json_response(200)
 
-      assert results = json_response(conn, 200)
       assert [] == results["accounts"]
     end
 

test/web/mastodon_api/views/status_view_test.exs

@@ -24,6 +24,22 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
     :ok
   end
 
+  test "has an emoji reaction list" do
+    user = insert(:user)
+    other_user = insert(:user)
+    third_user = insert(:user)
+    {:ok, activity} = CommonAPI.post(user, %{"status" => "dae cofe??"})
+
+    {:ok, _, _} = CommonAPI.react_with_emoji(activity.id, user, "☕")
+    {:ok, _, _} = CommonAPI.react_with_emoji(activity.id, other_user, "☕")
+    {:ok, _, _} = CommonAPI.react_with_emoji(activity.id, third_user, "🍵")
+    activity = Repo.get(Activity, activity.id)
+    status = StatusView.render("show.json", activity: activity)
+
+    assert status[:pleroma][:emoji_reactions]["🍵"] == 1
+    assert status[:pleroma][:emoji_reactions]["☕"] == 2
+  end
+
   test "loads and returns the direct conversation id when given the `with_direct_conversation_id` option" do
     user = insert(:user)
 
@@ -172,7 +188,8 @@ test "a note activity" do
         spoiler_text: %{"text/plain" => HTML.strip_tags(object_data["summary"])},
         expires_at: nil,
         direct_conversation_id: nil,
-        thread_muted: false
+        thread_muted: false,
+        emoji_reactions: %{}
       }
     }