From 5ae56aafb2edc737f7e9fb36e00377815f028ce6 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Sun, 6 Sep 2020 21:42:51 +0300 Subject: [PATCH 1/4] added import mutes --- docs/API/pleroma_api.md | 17 ++ lib/pleroma/user.ex | 51 ----- lib/pleroma/user/import.ex | 91 ++++++++ .../controllers/user_import_controller.ex | 57 +++++ lib/pleroma/web/router.ex | 7 +- .../controllers/util_controller.ex | 35 --- lib/pleroma/workers/background_worker.ex | 11 +- test/user/import_test.exs | 78 +++++++ test/user_test.exs | 34 --- .../user_import_controller_test.exs | 205 ++++++++++++++++++ test/web/twitter_api/util_controller_test.exs | 164 -------------- 11 files changed, 461 insertions(+), 289 deletions(-) create mode 100644 lib/pleroma/user/import.ex create mode 100644 lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex create mode 100644 test/user/import_test.exs create mode 100644 test/web/pleroma_api/controllers/user_import_controller_test.exs diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index 4e97d26c0..22f3ad7d6 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -44,6 +44,23 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi * Response: HTTP 200 on success, 500 on error * Note: Users that can't be followed are silently skipped. +## `/api/pleroma/blocks_import` +### Imports your blocks. +* Method: `POST` +* Authentication: required +* Params: + * `list`: STRING or FILE containing a whitespace-separated list of accounts to follow +* Response: HTTP 200 on success, 500 on error + +## `/api/pleroma/mutes_import` +### Imports your mutes. +* Method: `POST` +* Authentication: required +* Params: + * `list`: STRING or FILE containing a whitespace-separated list of accounts to follow +* Response: HTTP 200 on success, 500 on error + + ## `/api/pleroma/captcha` ### Get a new captcha * Method: `GET` diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 94c96de8d..be2ef0d1b 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1686,42 +1686,6 @@ def perform(:delete, %User{} = user) do def perform(:deactivate_async, user, status), do: deactivate(user, status) - @spec perform(atom(), User.t(), list()) :: list() | {:error, any()} - def perform(:blocks_import, %User{} = blocker, blocked_identifiers) - when is_list(blocked_identifiers) do - Enum.map( - blocked_identifiers, - fn blocked_identifier -> - with {:ok, %User{} = blocked} <- get_or_fetch(blocked_identifier), - {:ok, _block} <- CommonAPI.block(blocker, blocked) do - blocked - else - err -> - Logger.debug("blocks_import failed for #{blocked_identifier} with: #{inspect(err)}") - err - end - end - ) - end - - def perform(:follow_import, %User{} = follower, followed_identifiers) - when is_list(followed_identifiers) do - Enum.map( - followed_identifiers, - fn followed_identifier -> - with {:ok, %User{} = followed} <- get_or_fetch(followed_identifier), - {:ok, follower} <- maybe_direct_follow(follower, followed), - {:ok, _, _, _} <- CommonAPI.follow(follower, followed) do - followed - else - err -> - Logger.debug("follow_import failed for #{followed_identifier} with: #{inspect(err)}") - err - end - end - ) - end - @spec external_users_query() :: Ecto.Query.t() def external_users_query do User.Query.build(%{ @@ -1750,21 +1714,6 @@ def external_users(opts \\ []) do Repo.all(query) end - def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do - BackgroundWorker.enqueue("blocks_import", %{ - "blocker_id" => blocker.id, - "blocked_identifiers" => blocked_identifiers - }) - end - - def follow_import(%User{} = follower, followed_identifiers) - when is_list(followed_identifiers) do - BackgroundWorker.enqueue("follow_import", %{ - "follower_id" => follower.id, - "followed_identifiers" => followed_identifiers - }) - end - def delete_notifications_from_user_activities(%User{ap_id: ap_id}) do Notification |> join(:inner, [n], activity in assoc(n, :activity)) diff --git a/lib/pleroma/user/import.ex b/lib/pleroma/user/import.ex new file mode 100644 index 000000000..de27bdc4c --- /dev/null +++ b/lib/pleroma/user/import.ex @@ -0,0 +1,91 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.User.Import do + use Ecto.Schema + + alias Pleroma.User + alias Pleroma.Web.CommonAPI + alias Pleroma.Workers.BackgroundWorker + + require Logger + + @spec perform(atom(), User.t(), list()) :: :ok | list() | {:error, any()} + def perform(:mutes_import, %User{} = user, [_ | _] = identifiers) do + Enum.map( + identifiers, + fn identifier -> + with {:ok, %User{} = muted_user} <- User.get_or_fetch(identifier), + {:ok, _} <- User.mute(user, muted_user) do + muted_user + else + error -> handle_error(:mutes_import, identifier, error) + end + end + ) + end + + def perform(:blocks_import, %User{} = blocker, [_ | _] = identifiers) do + Enum.map( + identifiers, + fn identifier -> + with {:ok, %User{} = blocked} <- User.get_or_fetch(identifier), + {:ok, _block} <- CommonAPI.block(blocker, blocked) do + blocked + else + error -> handle_error(:blocks_import, identifier, error) + end + end + ) + end + + def perform(:follow_import, %User{} = follower, [_ | _] = identifiers) do + Enum.map( + identifiers, + fn identifier -> + with {:ok, %User{} = followed} <- User.get_or_fetch(identifier), + {:ok, follower} <- User.maybe_direct_follow(follower, followed), + {:ok, _, _, _} <- CommonAPI.follow(follower, followed) do + followed + else + error -> handle_error(:follow_import, identifier, error) + end + end + ) + end + + def perform(_, _, _), do: :ok + + defp handle_error(op, user_id, error) do + Logger.debug("#{op} failed for #{user_id} with: #{inspect(error)}") + error + end + + def blocks_import(%User{} = blocker, [_ | _] = identifiers) do + BackgroundWorker.enqueue( + "blocks_import", + %{ + "blocker_id" => blocker.id, + "blocked_identifiers" => identifiers + } + ) + end + + def follow_import(%User{} = follower, [_ | _] = identifiers) do + BackgroundWorker.enqueue( + "follow_import", + %{ + "follower_id" => follower.id, + "followed_identifiers" => identifiers + } + ) + end + + def mutes_import(%User{} = user, [_ | _] = identifiers) do + BackgroundWorker.enqueue( + "mutes_import", + %{"user_id" => user.id, "identifiers" => identifiers} + ) + end +end diff --git a/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex b/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex new file mode 100644 index 000000000..df6a0f131 --- /dev/null +++ b/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex @@ -0,0 +1,57 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.UserImportController do + use Pleroma.Web, :controller + + require Logger + + alias Pleroma.Plugs.OAuthScopesPlug + alias Pleroma.User + + plug(OAuthScopesPlug, %{scopes: ["follow", "write:follows"]} when action == :follow) + plug(OAuthScopesPlug, %{scopes: ["follow", "write:blocks"]} when action == :blocks) + plug(OAuthScopesPlug, %{scopes: ["follow", "write:mutes"]} when action == :mutes) + + def follow(conn, %{"list" => %Plug.Upload{path: path}}) do + follow(conn, %{"list" => File.read!(path)}) + end + + def follow(%{assigns: %{user: follower}} = conn, %{"list" => list}) do + identifiers = + list + |> String.split("\n") + |> Enum.map(&(&1 |> String.split(",") |> List.first())) + |> List.delete("Account address") + |> Enum.map(&(&1 |> String.trim() |> String.trim_leading("@"))) + |> Enum.reject(&(&1 == "")) + + User.Import.follow_import(follower, identifiers) + json(conn, "job started") + end + + def blocks(conn, %{"list" => %Plug.Upload{path: path}}) do + blocks(conn, %{"list" => File.read!(path)}) + end + + def blocks(%{assigns: %{user: blocker}} = conn, %{"list" => list}) do + User.Import.blocks_import(blocker, prepare_user_identifiers(list)) + json(conn, "job started") + end + + def mutes(conn, %{"list" => %Plug.Upload{path: path}}) do + mutes(conn, %{"list" => File.read!(path)}) + end + + def mutes(%{assigns: %{user: user}} = conn, %{"list" => list}) do + User.Import.mutes_import(user, prepare_user_identifiers(list)) + json(conn, "job started") + end + + defp prepare_user_identifiers(list) do + list + |> String.split() + |> Enum.map(&String.trim_leading(&1, "@")) + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index c6433cc53..f69b1545f 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -260,14 +260,15 @@ defmodule Pleroma.Web.Router do post("/delete_account", UtilController, :delete_account) put("/notification_settings", UtilController, :update_notificaton_settings) post("/disable_account", UtilController, :disable_account) - - post("/blocks_import", UtilController, :blocks_import) - post("/follow_import", UtilController, :follow_import) end scope "/api/pleroma", Pleroma.Web.PleromaAPI do pipe_through(:authenticated_api) + post("/mutes_import", UserImportController, :mutes) + post("/blocks_import", UserImportController, :blocks) + post("/follow_import", UserImportController, :follow) + get("/accounts/mfa", TwoFactorAuthenticationController, :settings) get("/accounts/mfa/backup_codes", TwoFactorAuthenticationController, :backup_codes) get("/accounts/mfa/setup/:method", TwoFactorAuthenticationController, :setup) diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index f02c4075c..70b0fbd54 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -18,14 +18,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do plug(Pleroma.Web.FederatingPlug when action == :remote_subscribe) - plug( - OAuthScopesPlug, - %{scopes: ["follow", "write:follows"]} - when action == :follow_import - ) - - plug(OAuthScopesPlug, %{scopes: ["follow", "write:blocks"]} when action == :blocks_import) - plug( OAuthScopesPlug, %{scopes: ["write:accounts"]} @@ -104,33 +96,6 @@ def update_notificaton_settings(%{assigns: %{user: user}} = conn, params) do end end - def follow_import(conn, %{"list" => %Plug.Upload{} = listfile}) do - follow_import(conn, %{"list" => File.read!(listfile.path)}) - end - - def follow_import(%{assigns: %{user: follower}} = conn, %{"list" => list}) do - followed_identifiers = - list - |> String.split("\n") - |> Enum.map(&(&1 |> String.split(",") |> List.first())) - |> List.delete("Account address") - |> Enum.map(&(&1 |> String.trim() |> String.trim_leading("@"))) - |> Enum.reject(&(&1 == "")) - - User.follow_import(follower, followed_identifiers) - json(conn, "job started") - end - - def blocks_import(conn, %{"list" => %Plug.Upload{} = listfile}) do - blocks_import(conn, %{"list" => File.read!(listfile.path)}) - end - - def blocks_import(%{assigns: %{user: blocker}} = conn, %{"list" => list}) do - blocked_identifiers = list |> String.split() |> Enum.map(&String.trim_leading(&1, "@")) - User.blocks_import(blocker, blocked_identifiers) - json(conn, "job started") - end - def change_password(%{assigns: %{user: user}} = conn, params) do case CommonAPI.Utils.confirm_current_password(user, params["password"]) do {:ok, user} -> diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index cec5a7462..f9c767ee0 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -34,7 +34,7 @@ def perform(%Job{ } }) do blocker = User.get_cached_by_id(blocker_id) - {:ok, User.perform(:blocks_import, blocker, blocked_identifiers)} + {:ok, User.Import.perform(:blocks_import, blocker, blocked_identifiers)} end def perform(%Job{ @@ -45,7 +45,14 @@ def perform(%Job{ } }) do follower = User.get_cached_by_id(follower_id) - {:ok, User.perform(:follow_import, follower, followed_identifiers)} + {:ok, User.Import.perform(:follow_import, follower, followed_identifiers)} + end + + def perform(%Job{ + args: %{"op" => "mutes_import", "user_id" => user_id, "identifiers" => identifiers} + }) do + user = User.get_cached_by_id(user_id) + {:ok, User.Import.perform(:mutes_import, user, identifiers)} end def perform(%Job{args: %{"op" => "media_proxy_preload", "message" => message}}) do diff --git a/test/user/import_test.exs b/test/user/import_test.exs new file mode 100644 index 000000000..476b22678 --- /dev/null +++ b/test/user/import_test.exs @@ -0,0 +1,78 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.User.ImportTest do + + alias Pleroma.Repo + alias Pleroma.Tests.ObanHelpers + alias Pleroma.User + + use Pleroma.DataCase + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + describe "follow_import" do + test "it imports user followings from list" do + [user1, user2, user3] = insert_list(3, :user) + + identifiers = [ + user2.ap_id, + user3.nickname + ] + + {:ok, job} = User.Import.follow_import(user1, identifiers) + + assert {:ok, result} = ObanHelpers.perform(job) + assert is_list(result) + assert result == [user2, user3] + assert User.following?(user1, user2) + assert User.following?(user1, user3) + end + end + + + describe "blocks_import" do + test "it imports user blocks from list" do + [user1, user2, user3] = insert_list(3, :user) + + identifiers = [ + user2.ap_id, + user3.nickname + ] + + {:ok, job} = User.Import.blocks_import(user1, identifiers) + + assert {:ok, result} = ObanHelpers.perform(job) + assert is_list(result) + assert result == [user2, user3] + assert User.blocks?(user1, user2) + assert User.blocks?(user1, user3) + end + end + + describe "mutes_import" do + test "it imports user mutes from list" do + [user1, user2, user3] = insert_list(3, :user) + + identifiers = [ + user2.ap_id, + user3.nickname + ] + + {:ok, job} = User.Import.mutes_import(user1, identifiers) + + assert {:ok, result} = ObanHelpers.perform(job) + assert is_list(result) + assert result == [user2, user3] + assert User.mutes?(user1, user2) + assert User.mutes?(user1, user3) + end + end +end diff --git a/test/user_test.exs b/test/user_test.exs index 50f72549e..13ac633b8 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -932,23 +932,6 @@ test "it sets the follower_count property" do end end - describe "follow_import" do - test "it imports user followings from list" do - [user1, user2, user3] = insert_list(3, :user) - - identifiers = [ - user2.ap_id, - user3.nickname - ] - - {:ok, job} = User.follow_import(user1, identifiers) - - assert {:ok, result} = ObanHelpers.perform(job) - assert is_list(result) - assert result == [user2, user3] - end - end - describe "mutes" do test "it mutes people" do user = insert(:user) @@ -1155,23 +1138,6 @@ test "follows take precedence over domain blocks" do end end - describe "blocks_import" do - test "it imports user blocks from list" do - [user1, user2, user3] = insert_list(3, :user) - - identifiers = [ - user2.ap_id, - user3.nickname - ] - - {:ok, job} = User.blocks_import(user1, identifiers) - - assert {:ok, result} = ObanHelpers.perform(job) - assert is_list(result) - assert result == [user2, user3] - end - end - describe "get_recipients_from_activity" do test "works for announces" do actor = insert(:user) diff --git a/test/web/pleroma_api/controllers/user_import_controller_test.exs b/test/web/pleroma_api/controllers/user_import_controller_test.exs new file mode 100644 index 000000000..d1a8a46fc --- /dev/null +++ b/test/web/pleroma_api/controllers/user_import_controller_test.exs @@ -0,0 +1,205 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.UserImportControllerTest do + use Pleroma.Web.ConnCase + use Oban.Testing, repo: Pleroma.Repo + + alias Pleroma.Config + alias Pleroma.Tests.ObanHelpers + + import Pleroma.Factory + import Mock + + setup do + Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + describe "POST /api/pleroma/follow_import" do + setup do: oauth_access(["follow"]) + + test "it returns HTTP 200", %{conn: conn} do + user2 = insert(:user) + + assert "job started" == conn + |> post("/api/pleroma/follow_import", %{"list" => "#{user2.ap_id}"}) + |> json_response(:ok) + end + + test "it imports follow lists from file", %{conn: conn} do + user2 = insert(:user) + + with_mocks([ + {File, [], + read!: fn "follow_list.txt" -> + "Account address,Show boosts\n#{user2.ap_id},true" + end} + ]) do + assert "job started" == conn + |> post("/api/pleroma/follow_import", %{"list" => %Plug.Upload{path: "follow_list.txt"}}) + |> json_response(:ok) + + assert [{:ok, job_result}] = ObanHelpers.perform_all() + assert job_result == [user2] + end + end + + test "it imports new-style mastodon follow lists", %{conn: conn} do + user2 = insert(:user) + + response = conn + |> post("/api/pleroma/follow_import", %{ + "list" => "Account address,Show boosts\n#{user2.ap_id},true"} + ) + |> json_response(:ok) + + assert response == "job started" + end + + test "requires 'follow' or 'write:follows' permissions" do + token1 = insert(:oauth_token, scopes: ["read", "write"]) + token2 = insert(:oauth_token, scopes: ["follow"]) + token3 = insert(:oauth_token, scopes: ["something"]) + another_user = insert(:user) + + for token <- [token1, token2, token3] do + conn = + build_conn() + |> put_req_header("authorization", "Bearer #{token.token}") + |> post("/api/pleroma/follow_import", %{"list" => "#{another_user.ap_id}"}) + + if token == token3 do + assert %{"error" => "Insufficient permissions: follow | write:follows."} == + json_response(conn, 403) + else + assert json_response(conn, 200) + end + end + end + + test "it imports follows with different nickname variations", %{conn: conn} do + users = [user2, user3, user4, user5, user6] = insert_list(5, :user) + + identifiers = + [ + user2.ap_id, + user3.nickname, + " ", + "@" <> user4.nickname, + user5.nickname <> "@localhost", + "@" <> user6.nickname <> "@localhost" + ] + |> Enum.join("\n") + + assert "job started" == conn + |> post("/api/pleroma/follow_import", %{"list" => identifiers}) + |> json_response(:ok) + + assert [{:ok, job_result}] = ObanHelpers.perform_all() + assert job_result == users + end + end + + describe "POST /api/pleroma/blocks_import" do + # Note: "follow" or "write:blocks" permission is required + setup do: oauth_access(["write:blocks"]) + + test "it returns HTTP 200", %{conn: conn} do + user2 = insert(:user) + + assert "job started" == conn + |> post("/api/pleroma/blocks_import", %{"list" => "#{user2.ap_id}"}) + |> json_response(:ok) + end + + test "it imports blocks users from file", %{conn: conn} do + users = [user2, user3] = insert_list(2, :user) + + with_mocks([ + {File, [], read!: fn "blocks_list.txt" -> "#{user2.ap_id} #{user3.ap_id}" end} + ]) do + + assert "job started" == conn + |> post("/api/pleroma/blocks_import", %{"list" => %Plug.Upload{path: "blocks_list.txt"}}) + |> json_response(:ok) + + assert [{:ok, job_result}] = ObanHelpers.perform_all() + assert job_result == users + end + end + + test "it imports blocks with different nickname variations", %{conn: conn} do + users = [user2, user3, user4, user5, user6] = insert_list(5, :user) + + identifiers = + [ + user2.ap_id, + user3.nickname, + "@" <> user4.nickname, + user5.nickname <> "@localhost", + "@" <> user6.nickname <> "@localhost" + ] + |> Enum.join(" ") + + assert "job started" == conn + |> post("/api/pleroma/blocks_import", %{"list" => identifiers}) + |> json_response(:ok) + + assert [{:ok, job_result}] = ObanHelpers.perform_all() + assert job_result == users + end + end + + describe "POST /api/pleroma/mutes_import" do + # Note: "follow" or "write:mutes" permission is required + setup do: oauth_access(["write:mutes"]) + + test "it returns HTTP 200", %{user: user, conn: conn} do + user2 = insert(:user) + + assert "job started" == conn + |> post("/api/pleroma/mutes_import", %{"list" => "#{user2.ap_id}"}) + |> json_response(:ok) + + assert [{:ok, job_result}] = ObanHelpers.perform_all() + assert job_result == [user2] + assert Pleroma.User.mutes?(user, user2) + end + + + test "it imports mutes users from file", %{user: user, conn: conn} do + users = [user2, user3] = insert_list(2, :user) + + with_mocks([ + {File, [], read!: fn "mutes_list.txt" -> "#{user2.ap_id} #{user3.ap_id}" end} + ]) do + assert "job started" == conn + |> post("/api/pleroma/mutes_import", %{"list" => %Plug.Upload{path: "mutes_list.txt"}}) + |> json_response(:ok) + + assert [{:ok, job_result}] = ObanHelpers.perform_all() + assert job_result == users + assert Enum.all?(users, &Pleroma.User.mutes?(user, &1)) + end + end + + test "it imports mutes with different nickname variations", %{user: user, conn: conn} do + users = [user2, user3, user4, user5, user6] = insert_list(5, :user) + + identifiers = [ + user2.ap_id, user3.nickname, "@" <> user4.nickname, + user5.nickname <> "@localhost", "@" <> user6.nickname <> "@localhost" + ] + |> Enum.join(" ") + + assert "job started" == conn + |> post("/api/pleroma/mutes_import", %{"list" => identifiers}) + |> json_response(:ok) + assert [{:ok, job_result}] = ObanHelpers.perform_all() + assert job_result == users + assert Enum.all?(users, &Pleroma.User.mutes?(user, &1)) + end + end +end diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index d164127ee..60f2fb052 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -21,170 +21,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do setup do: clear_config([:instance]) setup do: clear_config([:frontend_configurations, :pleroma_fe]) - describe "POST /api/pleroma/follow_import" do - setup do: oauth_access(["follow"]) - - test "it returns HTTP 200", %{conn: conn} do - user2 = insert(:user) - - response = - conn - |> post("/api/pleroma/follow_import", %{"list" => "#{user2.ap_id}"}) - |> json_response(:ok) - - assert response == "job started" - end - - test "it imports follow lists from file", %{user: user1, conn: conn} do - user2 = insert(:user) - - with_mocks([ - {File, [], - read!: fn "follow_list.txt" -> - "Account address,Show boosts\n#{user2.ap_id},true" - end} - ]) do - response = - conn - |> post("/api/pleroma/follow_import", %{"list" => %Plug.Upload{path: "follow_list.txt"}}) - |> json_response(:ok) - - assert response == "job started" - - assert ObanHelpers.member?( - %{ - "op" => "follow_import", - "follower_id" => user1.id, - "followed_identifiers" => [user2.ap_id] - }, - all_enqueued(worker: Pleroma.Workers.BackgroundWorker) - ) - end - end - - test "it imports new-style mastodon follow lists", %{conn: conn} do - user2 = insert(:user) - - response = - conn - |> post("/api/pleroma/follow_import", %{ - "list" => "Account address,Show boosts\n#{user2.ap_id},true" - }) - |> json_response(:ok) - - assert response == "job started" - end - - test "requires 'follow' or 'write:follows' permissions" do - token1 = insert(:oauth_token, scopes: ["read", "write"]) - token2 = insert(:oauth_token, scopes: ["follow"]) - token3 = insert(:oauth_token, scopes: ["something"]) - another_user = insert(:user) - - for token <- [token1, token2, token3] do - conn = - build_conn() - |> put_req_header("authorization", "Bearer #{token.token}") - |> post("/api/pleroma/follow_import", %{"list" => "#{another_user.ap_id}"}) - - if token == token3 do - assert %{"error" => "Insufficient permissions: follow | write:follows."} == - json_response(conn, 403) - else - assert json_response(conn, 200) - end - end - end - - test "it imports follows with different nickname variations", %{conn: conn} do - [user2, user3, user4, user5, user6] = insert_list(5, :user) - - identifiers = - [ - user2.ap_id, - user3.nickname, - " ", - "@" <> user4.nickname, - user5.nickname <> "@localhost", - "@" <> user6.nickname <> "@localhost" - ] - |> Enum.join("\n") - - response = - conn - |> post("/api/pleroma/follow_import", %{"list" => identifiers}) - |> json_response(:ok) - - assert response == "job started" - assert [{:ok, job_result}] = ObanHelpers.perform_all() - assert job_result == [user2, user3, user4, user5, user6] - end - end - - describe "POST /api/pleroma/blocks_import" do - # Note: "follow" or "write:blocks" permission is required - setup do: oauth_access(["write:blocks"]) - - test "it returns HTTP 200", %{conn: conn} do - user2 = insert(:user) - - response = - conn - |> post("/api/pleroma/blocks_import", %{"list" => "#{user2.ap_id}"}) - |> json_response(:ok) - - assert response == "job started" - end - - test "it imports blocks users from file", %{user: user1, conn: conn} do - user2 = insert(:user) - user3 = insert(:user) - - with_mocks([ - {File, [], read!: fn "blocks_list.txt" -> "#{user2.ap_id} #{user3.ap_id}" end} - ]) do - response = - conn - |> post("/api/pleroma/blocks_import", %{"list" => %Plug.Upload{path: "blocks_list.txt"}}) - |> json_response(:ok) - - assert response == "job started" - - assert ObanHelpers.member?( - %{ - "op" => "blocks_import", - "blocker_id" => user1.id, - "blocked_identifiers" => [user2.ap_id, user3.ap_id] - }, - all_enqueued(worker: Pleroma.Workers.BackgroundWorker) - ) - end - end - - test "it imports blocks with different nickname variations", %{conn: conn} do - [user2, user3, user4, user5, user6] = insert_list(5, :user) - - identifiers = - [ - user2.ap_id, - user3.nickname, - "@" <> user4.nickname, - user5.nickname <> "@localhost", - "@" <> user6.nickname <> "@localhost" - ] - |> Enum.join(" ") - - response = - conn - |> post("/api/pleroma/blocks_import", %{"list" => identifiers}) - |> json_response(:ok) - - assert response == "job started" - assert [{:ok, job_result}] = ObanHelpers.perform_all() - assert job_result == [user2, user3, user4, user5, user6] - end - end - describe "PUT /api/pleroma/notification_settings" do setup do: oauth_access(["write:accounts"]) From 917d325972e3aeb367583c61aaa109d62fcba837 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 7 Sep 2020 07:17:30 +0300 Subject: [PATCH 2/4] added api spec --- docs/API/pleroma_api.md | 4 +- .../operations/user_import_operation.ex | 80 +++++++++++++ .../controllers/user_import_controller.ex | 22 ++-- test/user/import_test.exs | 2 - .../user_import_controller_test.exs | 108 +++++++++++------- 5 files changed, 164 insertions(+), 52 deletions(-) create mode 100644 lib/pleroma/web/api_spec/operations/user_import_operation.ex diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index 22f3ad7d6..567ad5732 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -49,7 +49,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi * Method: `POST` * Authentication: required * Params: - * `list`: STRING or FILE containing a whitespace-separated list of accounts to follow + * `list`: STRING or FILE containing a whitespace-separated list of accounts to block * Response: HTTP 200 on success, 500 on error ## `/api/pleroma/mutes_import` @@ -57,7 +57,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi * Method: `POST` * Authentication: required * Params: - * `list`: STRING or FILE containing a whitespace-separated list of accounts to follow + * `list`: STRING or FILE containing a whitespace-separated list of accounts to mute * Response: HTTP 200 on success, 500 on error diff --git a/lib/pleroma/web/api_spec/operations/user_import_operation.ex b/lib/pleroma/web/api_spec/operations/user_import_operation.ex new file mode 100644 index 000000000..a50314fb7 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/user_import_operation.ex @@ -0,0 +1,80 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.UserImportOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + + import Pleroma.Web.ApiSpec.Helpers + + @spec open_api_operation(atom) :: Operation.t() + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def follow_operation do + %Operation{ + tags: ["follow_import"], + summary: "Imports your follows.", + operationId: "UserImportController.follow", + requestBody: request_body("Parameters", import_request(), required: true), + responses: %{ + 200 => ok_response(), + 500 => Operation.response("Error", "application/json", ApiError) + }, + security: [%{"oAuth" => ["write:follow"]}] + } + end + + def blocks_operation do + %Operation{ + tags: ["blocks_import"], + summary: "Imports your blocks.", + operationId: "UserImportController.blocks", + requestBody: request_body("Parameters", import_request(), required: true), + responses: %{ + 200 => ok_response(), + 500 => Operation.response("Error", "application/json", ApiError) + }, + security: [%{"oAuth" => ["write:blocks"]}] + } + end + + def mutes_operation do + %Operation{ + tags: ["mutes_import"], + summary: "Imports your mutes.", + operationId: "UserImportController.mutes", + requestBody: request_body("Parameters", import_request(), required: true), + responses: %{ + 200 => ok_response(), + 500 => Operation.response("Error", "application/json", ApiError) + }, + security: [%{"oAuth" => ["write:mutes"]}] + } + end + + defp import_request do + %Schema{ + type: :object, + required: [:list], + properties: %{ + list: %Schema{ + description: + "STRING or FILE containing a whitespace-separated list of accounts to import.", + anyOf: [ + %Schema{type: :string, format: :binary}, + %Schema{type: :string} + ] + } + } + } + end + + defp ok_response do + Operation.response("Ok", "application/json", %Schema{type: :string, example: "ok"}) + end +end diff --git a/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex b/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex index df6a0f131..f10c45750 100644 --- a/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex @@ -9,16 +9,20 @@ defmodule Pleroma.Web.PleromaAPI.UserImportController do alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.User + alias Pleroma.Web.ApiSpec plug(OAuthScopesPlug, %{scopes: ["follow", "write:follows"]} when action == :follow) plug(OAuthScopesPlug, %{scopes: ["follow", "write:blocks"]} when action == :blocks) plug(OAuthScopesPlug, %{scopes: ["follow", "write:mutes"]} when action == :mutes) - def follow(conn, %{"list" => %Plug.Upload{path: path}}) do - follow(conn, %{"list" => File.read!(path)}) + plug(OpenApiSpex.Plug.CastAndValidate) + defdelegate open_api_operation(action), to: ApiSpec.UserImportOperation + + def follow(%{body_params: %{list: %Plug.Upload{path: path}}} = conn, _) do + follow(%Plug.Conn{conn | body_params: %{list: File.read!(path)}}, %{}) end - def follow(%{assigns: %{user: follower}} = conn, %{"list" => list}) do + def follow(%{assigns: %{user: follower}, body_params: %{list: list}} = conn, _) do identifiers = list |> String.split("\n") @@ -31,20 +35,20 @@ def follow(%{assigns: %{user: follower}} = conn, %{"list" => list}) do json(conn, "job started") end - def blocks(conn, %{"list" => %Plug.Upload{path: path}}) do - blocks(conn, %{"list" => File.read!(path)}) + def blocks(%{body_params: %{list: %Plug.Upload{path: path}}} = conn, _) do + blocks(%Plug.Conn{conn | body_params: %{list: File.read!(path)}}, %{}) end - def blocks(%{assigns: %{user: blocker}} = conn, %{"list" => list}) do + def blocks(%{assigns: %{user: blocker}, body_params: %{list: list}} = conn, _) do User.Import.blocks_import(blocker, prepare_user_identifiers(list)) json(conn, "job started") end - def mutes(conn, %{"list" => %Plug.Upload{path: path}}) do - mutes(conn, %{"list" => File.read!(path)}) + def mutes(%{body_params: %{list: %Plug.Upload{path: path}}} = conn, _) do + mutes(%Plug.Conn{conn | body_params: %{list: File.read!(path)}}, %{}) end - def mutes(%{assigns: %{user: user}} = conn, %{"list" => list}) do + def mutes(%{assigns: %{user: user}, body_params: %{list: list}} = conn, _) do User.Import.mutes_import(user, prepare_user_identifiers(list)) json(conn, "job started") end diff --git a/test/user/import_test.exs b/test/user/import_test.exs index 476b22678..e404deeb5 100644 --- a/test/user/import_test.exs +++ b/test/user/import_test.exs @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.User.ImportTest do - alias Pleroma.Repo alias Pleroma.Tests.ObanHelpers alias Pleroma.User @@ -37,7 +36,6 @@ test "it imports user followings from list" do end end - describe "blocks_import" do test "it imports user blocks from list" do [user1, user2, user3] = insert_list(3, :user) diff --git a/test/web/pleroma_api/controllers/user_import_controller_test.exs b/test/web/pleroma_api/controllers/user_import_controller_test.exs index d1a8a46fc..433c97e81 100644 --- a/test/web/pleroma_api/controllers/user_import_controller_test.exs +++ b/test/web/pleroma_api/controllers/user_import_controller_test.exs @@ -23,9 +23,11 @@ defmodule Pleroma.Web.PleromaAPI.UserImportControllerTest do test "it returns HTTP 200", %{conn: conn} do user2 = insert(:user) - assert "job started" == conn - |> post("/api/pleroma/follow_import", %{"list" => "#{user2.ap_id}"}) - |> json_response(:ok) + assert "job started" == + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/follow_import", %{"list" => "#{user2.ap_id}"}) + |> json_response_and_validate_schema(200) end test "it imports follow lists from file", %{conn: conn} do @@ -37,9 +39,13 @@ test "it imports follow lists from file", %{conn: conn} do "Account address,Show boosts\n#{user2.ap_id},true" end} ]) do - assert "job started" == conn - |> post("/api/pleroma/follow_import", %{"list" => %Plug.Upload{path: "follow_list.txt"}}) - |> json_response(:ok) + assert "job started" == + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/follow_import", %{ + "list" => %Plug.Upload{path: "follow_list.txt"} + }) + |> json_response_and_validate_schema(200) assert [{:ok, job_result}] = ObanHelpers.perform_all() assert job_result == [user2] @@ -49,11 +55,13 @@ test "it imports follow lists from file", %{conn: conn} do test "it imports new-style mastodon follow lists", %{conn: conn} do user2 = insert(:user) - response = conn - |> post("/api/pleroma/follow_import", %{ - "list" => "Account address,Show boosts\n#{user2.ap_id},true"} - ) - |> json_response(:ok) + response = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/follow_import", %{ + "list" => "Account address,Show boosts\n#{user2.ap_id},true" + }) + |> json_response_and_validate_schema(200) assert response == "job started" end @@ -68,6 +76,7 @@ test "requires 'follow' or 'write:follows' permissions" do conn = build_conn() |> put_req_header("authorization", "Bearer #{token.token}") + |> put_req_header("content-type", "application/json") |> post("/api/pleroma/follow_import", %{"list" => "#{another_user.ap_id}"}) if token == token3 do @@ -93,9 +102,11 @@ test "it imports follows with different nickname variations", %{conn: conn} do ] |> Enum.join("\n") - assert "job started" == conn - |> post("/api/pleroma/follow_import", %{"list" => identifiers}) - |> json_response(:ok) + assert "job started" == + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/follow_import", %{"list" => identifiers}) + |> json_response_and_validate_schema(200) assert [{:ok, job_result}] = ObanHelpers.perform_all() assert job_result == users @@ -109,9 +120,11 @@ test "it imports follows with different nickname variations", %{conn: conn} do test "it returns HTTP 200", %{conn: conn} do user2 = insert(:user) - assert "job started" == conn - |> post("/api/pleroma/blocks_import", %{"list" => "#{user2.ap_id}"}) - |> json_response(:ok) + assert "job started" == + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/blocks_import", %{"list" => "#{user2.ap_id}"}) + |> json_response_and_validate_schema(200) end test "it imports blocks users from file", %{conn: conn} do @@ -120,10 +133,13 @@ test "it imports blocks users from file", %{conn: conn} do with_mocks([ {File, [], read!: fn "blocks_list.txt" -> "#{user2.ap_id} #{user3.ap_id}" end} ]) do - - assert "job started" == conn - |> post("/api/pleroma/blocks_import", %{"list" => %Plug.Upload{path: "blocks_list.txt"}}) - |> json_response(:ok) + assert "job started" == + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/blocks_import", %{ + "list" => %Plug.Upload{path: "blocks_list.txt"} + }) + |> json_response_and_validate_schema(200) assert [{:ok, job_result}] = ObanHelpers.perform_all() assert job_result == users @@ -143,9 +159,11 @@ test "it imports blocks with different nickname variations", %{conn: conn} do ] |> Enum.join(" ") - assert "job started" == conn - |> post("/api/pleroma/blocks_import", %{"list" => identifiers}) - |> json_response(:ok) + assert "job started" == + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/blocks_import", %{"list" => identifiers}) + |> json_response_and_validate_schema(200) assert [{:ok, job_result}] = ObanHelpers.perform_all() assert job_result == users @@ -159,25 +177,30 @@ test "it imports blocks with different nickname variations", %{conn: conn} do test "it returns HTTP 200", %{user: user, conn: conn} do user2 = insert(:user) - assert "job started" == conn - |> post("/api/pleroma/mutes_import", %{"list" => "#{user2.ap_id}"}) - |> json_response(:ok) + assert "job started" == + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/mutes_import", %{"list" => "#{user2.ap_id}"}) + |> json_response_and_validate_schema(200) assert [{:ok, job_result}] = ObanHelpers.perform_all() assert job_result == [user2] assert Pleroma.User.mutes?(user, user2) end - test "it imports mutes users from file", %{user: user, conn: conn} do users = [user2, user3] = insert_list(2, :user) with_mocks([ {File, [], read!: fn "mutes_list.txt" -> "#{user2.ap_id} #{user3.ap_id}" end} ]) do - assert "job started" == conn - |> post("/api/pleroma/mutes_import", %{"list" => %Plug.Upload{path: "mutes_list.txt"}}) - |> json_response(:ok) + assert "job started" == + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/mutes_import", %{ + "list" => %Plug.Upload{path: "mutes_list.txt"} + }) + |> json_response_and_validate_schema(200) assert [{:ok, job_result}] = ObanHelpers.perform_all() assert job_result == users @@ -188,15 +211,22 @@ test "it imports mutes users from file", %{user: user, conn: conn} do test "it imports mutes with different nickname variations", %{user: user, conn: conn} do users = [user2, user3, user4, user5, user6] = insert_list(5, :user) - identifiers = [ - user2.ap_id, user3.nickname, "@" <> user4.nickname, - user5.nickname <> "@localhost", "@" <> user6.nickname <> "@localhost" - ] - |> Enum.join(" ") + identifiers = + [ + user2.ap_id, + user3.nickname, + "@" <> user4.nickname, + user5.nickname <> "@localhost", + "@" <> user6.nickname <> "@localhost" + ] + |> Enum.join(" ") + + assert "job started" == + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/mutes_import", %{"list" => identifiers}) + |> json_response_and_validate_schema(200) - assert "job started" == conn - |> post("/api/pleroma/mutes_import", %{"list" => identifiers}) - |> json_response(:ok) assert [{:ok, job_result}] = ObanHelpers.perform_all() assert job_result == users assert Enum.all?(users, &Pleroma.User.mutes?(user, &1)) From 5a8ea0a5b07c22d567a60af36345483fe880b638 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 17 Sep 2020 09:13:43 +0300 Subject: [PATCH 3/4] small refactoring --- lib/pleroma/user/import.ex | 10 ++------ lib/pleroma/workers/background_worker.ex | 29 +++--------------------- 2 files changed, 5 insertions(+), 34 deletions(-) diff --git a/lib/pleroma/user/import.ex b/lib/pleroma/user/import.ex index de27bdc4c..e458021c8 100644 --- a/lib/pleroma/user/import.ex +++ b/lib/pleroma/user/import.ex @@ -65,20 +65,14 @@ defp handle_error(op, user_id, error) do def blocks_import(%User{} = blocker, [_ | _] = identifiers) do BackgroundWorker.enqueue( "blocks_import", - %{ - "blocker_id" => blocker.id, - "blocked_identifiers" => identifiers - } + %{"user_id" => blocker.id, "identifiers" => identifiers} ) end def follow_import(%User{} = follower, [_ | _] = identifiers) do BackgroundWorker.enqueue( "follow_import", - %{ - "follower_id" => follower.id, - "followed_identifiers" => identifiers - } + %{"user_id" => follower.id, "identifiers" => identifiers} ) end diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index f9c767ee0..55b5a13d9 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -26,33 +26,10 @@ def perform(%Job{args: %{"op" => "force_password_reset", "user_id" => user_id}}) User.perform(:force_password_reset, user) end - def perform(%Job{ - args: %{ - "op" => "blocks_import", - "blocker_id" => blocker_id, - "blocked_identifiers" => blocked_identifiers - } - }) do - blocker = User.get_cached_by_id(blocker_id) - {:ok, User.Import.perform(:blocks_import, blocker, blocked_identifiers)} - end - - def perform(%Job{ - args: %{ - "op" => "follow_import", - "follower_id" => follower_id, - "followed_identifiers" => followed_identifiers - } - }) do - follower = User.get_cached_by_id(follower_id) - {:ok, User.Import.perform(:follow_import, follower, followed_identifiers)} - end - - def perform(%Job{ - args: %{"op" => "mutes_import", "user_id" => user_id, "identifiers" => identifiers} - }) do + def perform(%Job{args: %{"op" => op, "user_id" => user_id, "identifiers" => identifiers}}) + when op in ["blocks_import", "follow_import", "mutes_import"] do user = User.get_cached_by_id(user_id) - {:ok, User.Import.perform(:mutes_import, user, identifiers)} + {:ok, User.Import.perform(String.to_atom(op), user, identifiers)} end def perform(%Job{args: %{"op" => "media_proxy_preload", "message" => message}}) do From 2159daa9af8967fe4b6d31e19c7048979b4cb165 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 18 Sep 2020 07:09:53 +0300 Subject: [PATCH 4/4] update changelog --- CHANGELOG.md | 4 ++++ docs/API/pleroma_api.md | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1266d2dfe..5abafdf69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - The `discoverable` field in the `User` struct will now add a NOINDEX metatag to profile pages when false. - Users with the `discoverable` field set to false will not show up in searches. - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option). + ### Removed - **Breaking:** `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab` (moved to a simpler implementation). @@ -20,6 +21,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Removed `:managed_config` option. In practice, it was accidentally removed with 2.0.0 release when frontends were switched to a new configuration mechanism, however it was not officially removed until now. +### Added +- Pleroma API: Importing the mutes users from CSV files. + ## [2.1.2] - 2020-09-17 ### Security diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index 567ad5732..94b6a4fda 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -60,7 +60,6 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi * `list`: STRING or FILE containing a whitespace-separated list of accounts to mute * Response: HTTP 200 on success, 500 on error - ## `/api/pleroma/captcha` ### Get a new captcha * Method: `GET`