From b429a49504b1df6fa085cccbb3e461cd378b15c4 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 23 Apr 2020 23:44:03 +0200 Subject: [PATCH 01/56] mix.exs: Fix for MacOS --- mix.exs | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/mix.exs b/mix.exs index b76aef180..021217400 100644 --- a/mix.exs +++ b/mix.exs @@ -220,32 +220,39 @@ defp aliases do defp version(version) do identifier_filter = ~r/[^0-9a-z\-]+/i - # Pre-release version, denoted from patch version with a hyphen - {tag, tag_err} = - System.cmd("git", ["describe", "--tags", "--abbrev=0"], stderr_to_stdout: true) - - {describe, describe_err} = System.cmd("git", ["describe", "--tags", "--abbrev=8"]) - {commit_hash, commit_hash_err} = System.cmd("git", ["rev-parse", "--short", "HEAD"]) + {_gitpath, git_present} = System.cmd("sh", ["-c", "command -v git"]) git_pre_release = - cond do - tag_err == 0 and describe_err == 0 -> - describe - |> String.trim() - |> String.replace(String.trim(tag), "") - |> String.trim_leading("-") - |> String.trim() + if git_present do + {tag, tag_err} = + System.cmd("git", ["describe", "--tags", "--abbrev=0"], stderr_to_stdout: true) - commit_hash_err == 0 -> - "0-g" <> String.trim(commit_hash) + {describe, describe_err} = System.cmd("git", ["describe", "--tags", "--abbrev=8"]) + {commit_hash, commit_hash_err} = System.cmd("git", ["rev-parse", "--short", "HEAD"]) - true -> - "" + # Pre-release version, denoted from patch version with a hyphen + cond do + tag_err == 0 and describe_err == 0 -> + describe + |> String.trim() + |> String.replace(String.trim(tag), "") + |> String.trim_leading("-") + |> String.trim() + + commit_hash_err == 0 -> + "0-g" <> String.trim(commit_hash) + + true -> + "" + end + else + "" end # Branch name as pre-release version component, denoted with a dot branch_name = - with {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]), + with true <- git_present, + {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]), branch_name <- String.trim(branch_name), branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name, true <- From c63d6ba0b2686db847f70cf251f92bfed57c4e5f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 23 Apr 2020 23:44:30 +0200 Subject: [PATCH 02/56] mix.exs: branch_name fallbacks to "" --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 021217400..15a65c0fb 100644 --- a/mix.exs +++ b/mix.exs @@ -266,7 +266,7 @@ defp version(version) do branch_name else - _ -> "stable" + _ -> "" end build_name = From 053c46153076f351c5273c2d6b1fb0843e7b6a6d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 24 Apr 2020 00:26:24 +0200 Subject: [PATCH 03/56] mix.exs: proper check on 0, remove else in git_pre_release --- mix.exs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/mix.exs b/mix.exs index 15a65c0fb..ebb8bdb08 100644 --- a/mix.exs +++ b/mix.exs @@ -220,10 +220,10 @@ defp aliases do defp version(version) do identifier_filter = ~r/[^0-9a-z\-]+/i - {_gitpath, git_present} = System.cmd("sh", ["-c", "command -v git"]) + {_cmdgit, cmdgit_err} = System.cmd("sh", ["-c", "command -v git"]) git_pre_release = - if git_present do + if cmdgit_err == 0 do {tag, tag_err} = System.cmd("git", ["describe", "--tags", "--abbrev=0"], stderr_to_stdout: true) @@ -243,15 +243,13 @@ defp version(version) do "0-g" <> String.trim(commit_hash) true -> - "" + nil end - else - "" end # Branch name as pre-release version component, denoted with a dot branch_name = - with true <- git_present, + with 0 <- cmdgit_err, {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]), branch_name <- String.trim(branch_name), branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name, From e7bc2f980cce170731960e024614c497b821fe90 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 7 May 2020 13:44:38 +0300 Subject: [PATCH 04/56] account visibility --- lib/pleroma/user.ex | 52 ++++++++++------ .../api_spec/operations/account_operation.ex | 8 ++- .../controllers/account_controller.ex | 21 +++++-- .../web/mastodon_api/views/account_view.ex | 2 +- test/user_test.exs | 10 ++-- .../controllers/account_controller_test.exs | 59 ++++++++++++++----- 6 files changed, 105 insertions(+), 47 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index cba391072..7a2558c29 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -262,37 +262,51 @@ def account_status(%User{deactivated: true}), do: :deactivated def account_status(%User{password_reset_pending: true}), do: :password_reset_pending def account_status(%User{confirmation_pending: true}) do - case Config.get([:instance, :account_activation_required]) do - true -> :confirmation_pending - _ -> :active + if Config.get([:instance, :account_activation_required]) do + :confirmation_pending + else + :active end end def account_status(%User{}), do: :active - @spec visible_for?(User.t(), User.t() | nil) :: boolean() - def visible_for?(user, for_user \\ nil) + @spec visible_for(User.t(), User.t() | nil) :: + boolean() + | :invisible + | :restricted_unauthenticated + | :deactivated + | :confirmation_pending + def visible_for(user, for_user \\ nil) - def visible_for?(%User{invisible: true}, _), do: false + def visible_for(%User{invisible: true}, _), do: :invisible - def visible_for?(%User{id: user_id}, %User{id: user_id}), do: true + def visible_for(%User{id: user_id}, %User{id: user_id}), do: true - def visible_for?(%User{local: local} = user, nil) do - cfg_key = - if local, - do: :local, - else: :remote - - if Config.get([:restrict_unauthenticated, :profiles, cfg_key]), - do: false, - else: account_status(user) == :active + def visible_for(%User{} = user, nil) do + if restrict_unauthenticated?(user) do + :restrict_unauthenticated + else + visible_account_status(user) + end end - def visible_for?(%User{} = user, for_user) do - account_status(user) == :active || superuser?(for_user) + def visible_for(%User{} = user, for_user) do + superuser?(for_user) || visible_account_status(user) end - def visible_for?(_, _), do: false + def visible_for(_, _), do: false + + defp restrict_unauthenticated?(%User{local: local}) do + config_key = if local, do: :local, else: :remote + + Config.get([:restrict_unauthenticated, :profiles, config_key], false) + end + + defp visible_account_status(user) do + status = account_status(user) + status in [:active, :password_reset_pending] || status + end @spec superuser?(User.t()) :: boolean() def superuser?(%User{local: true, is_admin: true}), do: true diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 934f6038e..43168acf7 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -102,7 +102,9 @@ def show_operation do parameters: [%Reference{"$ref": "#/components/parameters/accountIdOrNickname"}], responses: %{ 200 => Operation.response("Account", "application/json", Account), - 404 => Operation.response("Error", "application/json", ApiError) + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError), + 410 => Operation.response("Error", "application/json", ApiError) } } end @@ -142,7 +144,9 @@ def statuses_operation do ] ++ pagination_params(), responses: %{ 200 => Operation.response("Statuses", "application/json", array_of_statuses()), - 404 => Operation.response("Error", "application/json", ApiError) + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError), + 410 => Operation.response("Error", "application/json", ApiError) } } end diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index ef41f9e96..ffa82731f 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -221,17 +221,17 @@ def relationships(%{assigns: %{user: _user}} = conn, _), do: json(conn, []) @doc "GET /api/v1/accounts/:id" def show(%{assigns: %{user: for_user}} = conn, %{id: nickname_or_id}) do with %User{} = user <- User.get_cached_by_nickname_or_id(nickname_or_id, for: for_user), - true <- User.visible_for?(user, for_user) do + true <- User.visible_for(user, for_user) do render(conn, "show.json", user: user, for: for_user) else - _e -> render_error(conn, :not_found, "Can't find user") + error -> user_visibility_error(conn, error) end end @doc "GET /api/v1/accounts/:id/statuses" def statuses(%{assigns: %{user: reading_user}} = conn, params) do with %User{} = user <- User.get_cached_by_nickname_or_id(params.id, for: reading_user), - true <- User.visible_for?(user, reading_user) do + true <- User.visible_for(user, reading_user) do params = params |> Map.delete(:tagged) @@ -250,7 +250,20 @@ def statuses(%{assigns: %{user: reading_user}} = conn, params) do as: :activity ) else - _e -> render_error(conn, :not_found, "Can't find user") + error -> user_visibility_error(conn, error) + end + end + + defp user_visibility_error(conn, error) do + case error do + :deactivated -> + render_error(conn, :gone, "") + + :restrict_unauthenticated -> + render_error(conn, :unauthorized, "This API requires an authenticated user") + + _ -> + render_error(conn, :not_found, "Can't find user") end end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 45fffaad2..8e723d013 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -35,7 +35,7 @@ def render("index.json", %{users: users} = opts) do end def render("show.json", %{user: user} = opts) do - if User.visible_for?(user, opts[:for]) do + if User.visible_for(user, opts[:for]) == true do do_render("show.json", opts) else %{} diff --git a/test/user_test.exs b/test/user_test.exs index 6b9df60a4..3bfcfd10c 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1289,11 +1289,11 @@ test "returns false for a non-invisible user" do end end - describe "visible_for?/2" do + describe "visible_for/2" do test "returns true when the account is itself" do user = insert(:user, local: true) - assert User.visible_for?(user, user) + assert User.visible_for(user, user) end test "returns false when the account is unauthenticated and auth is required" do @@ -1302,14 +1302,14 @@ test "returns false when the account is unauthenticated and auth is required" do user = insert(:user, local: true, confirmation_pending: true) other_user = insert(:user, local: true) - refute User.visible_for?(user, other_user) + refute User.visible_for(user, other_user) == true end test "returns true when the account is unauthenticated and auth is not required" do user = insert(:user, local: true, confirmation_pending: true) other_user = insert(:user, local: true) - assert User.visible_for?(user, other_user) + assert User.visible_for(user, other_user) end test "returns true when the account is unauthenticated and being viewed by a privileged account (auth required)" do @@ -1318,7 +1318,7 @@ test "returns true when the account is unauthenticated and being viewed by a pri user = insert(:user, local: true, confirmation_pending: true) other_user = insert(:user, local: true, is_admin: true) - assert User.visible_for?(user, other_user) + assert User.visible_for(user, other_user) end end diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs index 280bd6aca..7dfea2f9e 100644 --- a/test/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/web/mastodon_api/controllers/account_controller_test.exs @@ -127,6 +127,15 @@ test "returns 404 for internal.fetch actor", %{conn: conn} do |> get("/api/v1/accounts/internal.fetch") |> json_response_and_validate_schema(404) end + + test "returns 401 for deactivated user", %{conn: conn} do + user = insert(:user, deactivated: true) + + assert %{} = + conn + |> get("/api/v1/accounts/#{user.id}") + |> json_response_and_validate_schema(:gone) + end end defp local_and_remote_users do @@ -143,15 +152,15 @@ defp local_and_remote_users do setup do: clear_config([:restrict_unauthenticated, :profiles, :remote], true) test "if user is unauthenticated", %{conn: conn, local: local, remote: remote} do - assert %{"error" => "Can't find user"} == + assert %{"error" => "This API requires an authenticated user"} == conn |> get("/api/v1/accounts/#{local.id}") - |> json_response_and_validate_schema(:not_found) + |> json_response_and_validate_schema(:unauthorized) - assert %{"error" => "Can't find user"} == + assert %{"error" => "This API requires an authenticated user"} == conn |> get("/api/v1/accounts/#{remote.id}") - |> json_response_and_validate_schema(:not_found) + |> json_response_and_validate_schema(:unauthorized) end test "if user is authenticated", %{local: local, remote: remote} do @@ -173,8 +182,8 @@ test "if user is authenticated", %{local: local, remote: remote} do test "if user is unauthenticated", %{conn: conn, local: local, remote: remote} do res_conn = get(conn, "/api/v1/accounts/#{local.id}") - assert json_response_and_validate_schema(res_conn, :not_found) == %{ - "error" => "Can't find user" + assert json_response_and_validate_schema(res_conn, :unauthorized) == %{ + "error" => "This API requires an authenticated user" } res_conn = get(conn, "/api/v1/accounts/#{remote.id}") @@ -203,8 +212,8 @@ test "if user is unauthenticated", %{conn: conn, local: local, remote: remote} d res_conn = get(conn, "/api/v1/accounts/#{remote.id}") - assert json_response_and_validate_schema(res_conn, :not_found) == %{ - "error" => "Can't find user" + assert json_response_and_validate_schema(res_conn, :unauthorized) == %{ + "error" => "This API requires an authenticated user" } end @@ -249,6 +258,24 @@ test "works with announces that are just addressed to public", %{conn: conn} do assert id == announce.id end + test "deactivated user", %{conn: conn} do + user = insert(:user, deactivated: true) + + assert %{} == + conn + |> get("/api/v1/accounts/#{user.id}/statuses") + |> json_response_and_validate_schema(:gone) + end + + test "returns 404 when user is invisible", %{conn: conn} do + user = insert(:user, %{invisible: true}) + + assert %{"error" => "Can't find user"} = + conn + |> get("/api/v1/accounts/#{user.id}") + |> json_response_and_validate_schema(404) + end + test "respects blocks", %{user: user_one, conn: conn} do user_two = insert(:user) user_three = insert(:user) @@ -422,15 +449,15 @@ defp local_and_remote_activities(%{local: local, remote: remote}) do setup do: clear_config([:restrict_unauthenticated, :profiles, :remote], true) test "if user is unauthenticated", %{conn: conn, local: local, remote: remote} do - assert %{"error" => "Can't find user"} == + assert %{"error" => "This API requires an authenticated user"} == conn |> get("/api/v1/accounts/#{local.id}/statuses") - |> json_response_and_validate_schema(:not_found) + |> json_response_and_validate_schema(:unauthorized) - assert %{"error" => "Can't find user"} == + assert %{"error" => "This API requires an authenticated user"} == conn |> get("/api/v1/accounts/#{remote.id}/statuses") - |> json_response_and_validate_schema(:not_found) + |> json_response_and_validate_schema(:unauthorized) end test "if user is authenticated", %{local: local, remote: remote} do @@ -451,10 +478,10 @@ test "if user is authenticated", %{local: local, remote: remote} do setup do: clear_config([:restrict_unauthenticated, :profiles, :local], true) test "if user is unauthenticated", %{conn: conn, local: local, remote: remote} do - assert %{"error" => "Can't find user"} == + assert %{"error" => "This API requires an authenticated user"} == conn |> get("/api/v1/accounts/#{local.id}/statuses") - |> json_response_and_validate_schema(:not_found) + |> json_response_and_validate_schema(:unauthorized) res_conn = get(conn, "/api/v1/accounts/#{remote.id}/statuses") assert length(json_response_and_validate_schema(res_conn, 200)) == 1 @@ -481,10 +508,10 @@ test "if user is unauthenticated", %{conn: conn, local: local, remote: remote} d res_conn = get(conn, "/api/v1/accounts/#{local.id}/statuses") assert length(json_response_and_validate_schema(res_conn, 200)) == 1 - assert %{"error" => "Can't find user"} == + assert %{"error" => "This API requires an authenticated user"} == conn |> get("/api/v1/accounts/#{remote.id}/statuses") - |> json_response_and_validate_schema(:not_found) + |> json_response_and_validate_schema(:unauthorized) end test "if user is authenticated", %{local: local, remote: remote} do From b1aa402229b6422a5ab1aa7102c7a104e218d0e3 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 13 May 2020 11:11:10 +0300 Subject: [PATCH 05/56] removing 410 status --- lib/pleroma/web/api_spec/operations/account_operation.ex | 6 ++---- .../web/mastodon_api/controllers/account_controller.ex | 3 --- .../mastodon_api/controllers/account_controller_test.exs | 8 ++++---- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 43168acf7..74b395dfe 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -103,8 +103,7 @@ def show_operation do responses: %{ 200 => Operation.response("Account", "application/json", Account), 401 => Operation.response("Error", "application/json", ApiError), - 404 => Operation.response("Error", "application/json", ApiError), - 410 => Operation.response("Error", "application/json", ApiError) + 404 => Operation.response("Error", "application/json", ApiError) } } end @@ -145,8 +144,7 @@ def statuses_operation do responses: %{ 200 => Operation.response("Statuses", "application/json", array_of_statuses()), 401 => Operation.response("Error", "application/json", ApiError), - 404 => Operation.response("Error", "application/json", ApiError), - 410 => Operation.response("Error", "application/json", ApiError) + 404 => Operation.response("Error", "application/json", ApiError) } } end diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index ffa82731f..1edc0d96a 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -256,9 +256,6 @@ def statuses(%{assigns: %{user: reading_user}} = conn, params) do defp user_visibility_error(conn, error) do case error do - :deactivated -> - render_error(conn, :gone, "") - :restrict_unauthenticated -> render_error(conn, :unauthorized, "This API requires an authenticated user") diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs index 7dfea2f9e..8700ab2f5 100644 --- a/test/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/web/mastodon_api/controllers/account_controller_test.exs @@ -131,10 +131,10 @@ test "returns 404 for internal.fetch actor", %{conn: conn} do test "returns 401 for deactivated user", %{conn: conn} do user = insert(:user, deactivated: true) - assert %{} = + assert %{"error" => "Can't find user"} = conn |> get("/api/v1/accounts/#{user.id}") - |> json_response_and_validate_schema(:gone) + |> json_response_and_validate_schema(:not_found) end end @@ -261,10 +261,10 @@ test "works with announces that are just addressed to public", %{conn: conn} do test "deactivated user", %{conn: conn} do user = insert(:user, deactivated: true) - assert %{} == + assert %{"error" => "Can't find user"} == conn |> get("/api/v1/accounts/#{user.id}/statuses") - |> json_response_and_validate_schema(:gone) + |> json_response_and_validate_schema(:not_found) end test "returns 404 when user is invisible", %{conn: conn} do From 1671864d886bf63d11bbf3d7303719e8744bfc32 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 15 May 2020 20:29:09 +0300 Subject: [PATCH 06/56] return :visible instead of boolean --- lib/pleroma/user.ex | 19 ++++++++++++++----- .../controllers/account_controller.ex | 4 ++-- .../web/mastodon_api/views/account_view.ex | 2 +- test/user_test.exs | 8 ++++---- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 7a2558c29..5052f7b97 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -272,7 +272,7 @@ def account_status(%User{confirmation_pending: true}) do def account_status(%User{}), do: :active @spec visible_for(User.t(), User.t() | nil) :: - boolean() + :visible | :invisible | :restricted_unauthenticated | :deactivated @@ -281,7 +281,7 @@ def visible_for(user, for_user \\ nil) def visible_for(%User{invisible: true}, _), do: :invisible - def visible_for(%User{id: user_id}, %User{id: user_id}), do: true + def visible_for(%User{id: user_id}, %User{id: user_id}), do: :visible def visible_for(%User{} = user, nil) do if restrict_unauthenticated?(user) do @@ -292,10 +292,14 @@ def visible_for(%User{} = user, nil) do end def visible_for(%User{} = user, for_user) do - superuser?(for_user) || visible_account_status(user) + if superuser?(for_user) do + :visible + else + visible_account_status(user) + end end - def visible_for(_, _), do: false + def visible_for(_, _), do: :invisible defp restrict_unauthenticated?(%User{local: local}) do config_key = if local, do: :local, else: :remote @@ -305,7 +309,12 @@ defp restrict_unauthenticated?(%User{local: local}) do defp visible_account_status(user) do status = account_status(user) - status in [:active, :password_reset_pending] || status + + if status in [:active, :password_reset_pending] do + :visible + else + status + end end @spec superuser?(User.t()) :: boolean() diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 1edc0d96a..8727faab7 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -221,7 +221,7 @@ def relationships(%{assigns: %{user: _user}} = conn, _), do: json(conn, []) @doc "GET /api/v1/accounts/:id" def show(%{assigns: %{user: for_user}} = conn, %{id: nickname_or_id}) do with %User{} = user <- User.get_cached_by_nickname_or_id(nickname_or_id, for: for_user), - true <- User.visible_for(user, for_user) do + :visible <- User.visible_for(user, for_user) do render(conn, "show.json", user: user, for: for_user) else error -> user_visibility_error(conn, error) @@ -231,7 +231,7 @@ def show(%{assigns: %{user: for_user}} = conn, %{id: nickname_or_id}) do @doc "GET /api/v1/accounts/:id/statuses" def statuses(%{assigns: %{user: reading_user}} = conn, params) do with %User{} = user <- User.get_cached_by_nickname_or_id(params.id, for: reading_user), - true <- User.visible_for(user, reading_user) do + :visible <- User.visible_for(user, reading_user) do params = params |> Map.delete(:tagged) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 8e723d013..4a1508b22 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -35,7 +35,7 @@ def render("index.json", %{users: users} = opts) do end def render("show.json", %{user: user} = opts) do - if User.visible_for(user, opts[:for]) == true do + if User.visible_for(user, opts[:for]) == :visible do do_render("show.json", opts) else %{} diff --git a/test/user_test.exs b/test/user_test.exs index 3bfcfd10c..6865bd9be 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1293,7 +1293,7 @@ test "returns false for a non-invisible user" do test "returns true when the account is itself" do user = insert(:user, local: true) - assert User.visible_for(user, user) + assert User.visible_for(user, user) == :visible end test "returns false when the account is unauthenticated and auth is required" do @@ -1302,14 +1302,14 @@ test "returns false when the account is unauthenticated and auth is required" do user = insert(:user, local: true, confirmation_pending: true) other_user = insert(:user, local: true) - refute User.visible_for(user, other_user) == true + refute User.visible_for(user, other_user) == :visible end test "returns true when the account is unauthenticated and auth is not required" do user = insert(:user, local: true, confirmation_pending: true) other_user = insert(:user, local: true) - assert User.visible_for(user, other_user) + assert User.visible_for(user, other_user) == :visible end test "returns true when the account is unauthenticated and being viewed by a privileged account (auth required)" do @@ -1318,7 +1318,7 @@ test "returns true when the account is unauthenticated and being viewed by a pri user = insert(:user, local: true, confirmation_pending: true) other_user = insert(:user, local: true, is_admin: true) - assert User.visible_for(user, other_user) + assert User.visible_for(user, other_user) == :visible end end From 0321a3e07814c3f225f19e0372b69a7813cef15e Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 18 May 2020 10:34:34 +0300 Subject: [PATCH 07/56] test naming fix --- test/web/mastodon_api/controllers/account_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs index 8700ab2f5..3008970af 100644 --- a/test/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/web/mastodon_api/controllers/account_controller_test.exs @@ -128,7 +128,7 @@ test "returns 404 for internal.fetch actor", %{conn: conn} do |> json_response_and_validate_schema(404) end - test "returns 401 for deactivated user", %{conn: conn} do + test "returns 404 for deactivated user", %{conn: conn} do user = insert(:user, deactivated: true) assert %{"error" => "Can't find user"} = From f9dcf15ecb684b4b802d731a216448c76913d462 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 12 Jun 2020 14:49:54 +0300 Subject: [PATCH 08/56] added admin api for MediaProxy cache invalidation --- .../media_proxy_cache_controller.ex | 38 ++++++ .../admin_api/views/media_proxy_cache_view.ex | 11 ++ .../admin/media_proxy_cache_operation.ex | 109 ++++++++++++++++++ lib/pleroma/web/router.ex | 4 + .../media_proxy_cache_controller_test.exs | 66 +++++++++++ 5 files changed, 228 insertions(+) create mode 100644 lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex create mode 100644 lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex create mode 100644 lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex create mode 100644 test/web/admin_api/controllers/media_proxy_cache_controller_test.exs diff --git a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex new file mode 100644 index 000000000..7b28f7c72 --- /dev/null +++ b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex @@ -0,0 +1,38 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do + use Pleroma.Web, :controller + + alias Pleroma.Plugs.OAuthScopesPlug + alias Pleroma.Web.ApiSpec.Admin, as: Spec + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug( + OAuthScopesPlug, + %{scopes: ["read:media_proxy_caches"], admin: true} when action in [:index] + ) + + plug( + OAuthScopesPlug, + %{scopes: ["write:media_proxy_caches"], admin: true} when action in [:purge, :delete] + ) + + action_fallback(Pleroma.Web.AdminAPI.FallbackController) + + defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation + + def index(%{assigns: %{user: _}} = conn, _) do + render(conn, "index.json", urls: []) + end + + def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do + render(conn, "index.json", urls: urls) + end + + def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: _ban}} = conn, _) do + render(conn, "index.json", urls: urls) + end +end diff --git a/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex b/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex new file mode 100644 index 000000000..c97400beb --- /dev/null +++ b/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex @@ -0,0 +1,11 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do + use Pleroma.Web, :view + + def render("index.json", %{urls: urls}) do + %{urls: urls} + end +end diff --git a/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex new file mode 100644 index 000000000..0358cfbad --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex @@ -0,0 +1,109 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Admin", "MediaProxyCache"], + summary: "Fetch a paginated list of all banned MediaProxy URLs in Cachex", + operationId: "AdminAPI.MediaProxyCacheController.index", + security: [%{"oAuth" => ["read:media_proxy_caches"]}], + parameters: [ + Operation.parameter( + :page, + :query, + %Schema{type: :integer, default: 1}, + "Page" + ), + Operation.parameter( + :page_size, + :query, + %Schema{type: :integer, default: 50}, + "Number of statuses to return" + ) + ], + responses: %{ + 200 => success_response() + } + } + end + + def delete_operation do + %Operation{ + tags: ["Admin", "MediaProxyCache"], + summary: "Remove a banned MediaProxy URL from Cachex", + operationId: "AdminAPI.MediaProxyCacheController.delete", + security: [%{"oAuth" => ["write:media_proxy_caches"]}], + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + required: [:urls], + properties: %{ + urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}} + } + }, + required: true + ), + responses: %{ + 200 => success_response(), + 400 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def purge_operation do + %Operation{ + tags: ["Admin", "MediaProxyCache"], + summary: "Purge and optionally ban a MediaProxy URL", + operationId: "AdminAPI.MediaProxyCacheController.purge", + security: [%{"oAuth" => ["write:media_proxy_caches"]}], + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + required: [:urls], + properties: %{ + urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}}, + ban: %Schema{type: :boolean, default: true} + } + }, + required: true + ), + responses: %{ + 200 => success_response(), + 400 => Operation.response("Error", "application/json", ApiError) + } + } + end + + defp success_response do + Operation.response("Array of banned MediaProxy URLs in Cachex", "application/json", %Schema{ + type: :object, + properties: %{ + urls: %Schema{ + type: :array, + items: %Schema{ + type: :string, + format: :uri, + description: "MediaProxy URLs" + } + } + } + }) + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 57570b672..eda74a171 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -209,6 +209,10 @@ defmodule Pleroma.Web.Router do post("/oauth_app", OAuthAppController, :create) patch("/oauth_app/:id", OAuthAppController, :update) delete("/oauth_app/:id", OAuthAppController, :delete) + + get("/media_proxy_caches", MediaProxyCacheController, :index) + post("/media_proxy_caches/delete", MediaProxyCacheController, :delete) + post("/media_proxy_caches/purge", MediaProxyCacheController, :purge) end scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs new file mode 100644 index 000000000..1b1d6bc36 --- /dev/null +++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs @@ -0,0 +1,66 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do + use Pleroma.Web.ConnCase + + import Pleroma.Factory + + setup do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, token) + + {:ok, %{admin: admin, token: token, conn: conn}} + end + + describe "GET /api/pleroma/admin/media_proxy_caches" do + test "shows banned MediaProxy URLs", %{conn: conn} do + response = + conn + |> get("/api/pleroma/admin/media_proxy_caches") + |> json_response_and_validate_schema(200) + + assert response["urls"] == [] + end + end + + describe "DELETE /api/pleroma/admin/media_proxy_caches/delete" do + test "deleted MediaProxy URLs from banned", %{conn: conn} do + response = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/media_proxy_caches/delete", %{ + urls: ["http://example.com/media/a688346.jpg", "http://example.com/media/fb1f4d.jpg"] + }) + |> json_response_and_validate_schema(200) + + assert response["urls"] == [ + "http://example.com/media/a688346.jpg", + "http://example.com/media/fb1f4d.jpg" + ] + end + end + + describe "PURGE /api/pleroma/admin/media_proxy_caches/purge" do + test "perform invalidates cache of MediaProxy", %{conn: conn} do + response = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/media_proxy_caches/purge", %{ + urls: ["http://example.com/media/a688346.jpg", "http://example.com/media/fb1f4d.jpg"] + }) + |> json_response_and_validate_schema(200) + + assert response["urls"] == [ + "http://example.com/media/a688346.jpg", + "http://example.com/media/fb1f4d.jpg" + ] + end + end +end From 2e8a236cef28c0b754aecb04a5c60c3b7655c5a6 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Sun, 14 Jun 2020 21:02:57 +0300 Subject: [PATCH 09/56] fix invalidates media url's --- config/config.exs | 7 + config/description.exs | 64 +++++++++ docs/configuration/cheatsheet.md | 6 +- installation/nginx-cache-purge.sh.example | 4 +- lib/pleroma/application.ex | 3 +- lib/pleroma/plugs/uploaded_media.ex | 16 ++- lib/pleroma/web/media_proxy/invalidation.ex | 29 ++-- .../web/media_proxy/invalidations/http.ex | 8 +- .../web/media_proxy/invalidations/script.ex | 36 ++--- lib/pleroma/web/media_proxy/media_proxy.ex | 35 ++++- .../web/media_proxy/media_proxy_controller.ex | 3 +- .../workers/attachments_cleanup_worker.ex | 133 ++++++++++-------- test/web/media_proxy/invalidation_test.exs | 65 +++++++++ .../media_proxy/invalidations/http_test.exs | 13 +- .../media_proxy/invalidations/script_test.exs | 21 ++- .../media_proxy_controller_test.exs | 17 +++ 16 files changed, 346 insertions(+), 114 deletions(-) create mode 100644 test/web/media_proxy/invalidation_test.exs diff --git a/config/config.exs b/config/config.exs index 9508ae077..e299fb8dd 100644 --- a/config/config.exs +++ b/config/config.exs @@ -406,6 +406,13 @@ ], whitelist: [] +config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http, + method: :purge, + headers: [], + options: [] + +config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, script_path: nil + config :pleroma, :chat, enabled: true config :phoenix, :format_encoders, json: Jason diff --git a/config/description.exs b/config/description.exs index 807c945e0..857293794 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1637,6 +1637,31 @@ "The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.", suggestions: ["https://example.com"] }, + %{ + key: :invalidation, + type: :keyword, + descpiption: "", + suggestions: [ + enabled: true, + provider: Pleroma.Web.MediaProxy.Invalidation.Script + ], + children: [ + %{ + key: :enabled, + type: :boolean, + description: "Enables invalidate media cache" + }, + %{ + key: :provider, + type: :module, + description: "Module which will be used to cache purge.", + suggestions: [ + Pleroma.Web.MediaProxy.Invalidation.Script, + Pleroma.Web.MediaProxy.Invalidation.Http + ] + } + ] + }, %{ key: :proxy_opts, type: :keyword, @@ -1709,6 +1734,45 @@ } ] }, + %{ + group: :pleroma, + key: Pleroma.Web.MediaProxy.Invalidation.Http, + type: :group, + description: "HTTP invalidate settings", + children: [ + %{ + key: :method, + type: :atom, + description: "HTTP method of request. Default: :purge" + }, + %{ + key: :headers, + type: {:list, :tuple}, + description: "HTTP headers of request.", + suggestions: [{"x-refresh", 1}] + }, + %{ + key: :options, + type: :keyword, + description: "Request options.", + suggestions: [params: %{ts: "xxx"}] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.Web.MediaProxy.Invalidation.Script, + type: :group, + description: "Script invalidate settings", + children: [ + %{ + key: :script_path, + type: :string, + description: "Path to shell script. Which will run purge cache.", + suggestions: ["./installation/nginx-cache-purge.sh.example"] + } + ] + }, %{ group: :pleroma, key: :gopher, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 505acb293..20bd0ed85 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -262,7 +262,7 @@ This section describe PWA manifest instance-specific values. Currently this opti #### Pleroma.Web.MediaProxy.Invalidation.Script -This strategy allow perform external bash script to purge cache. +This strategy allow perform external shell script to purge cache. Urls of attachments pass to script as arguments. * `script_path`: path to external script. @@ -278,8 +278,8 @@ config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, This strategy allow perform custom http request to purge cache. * `method`: http method. default is `purge` -* `headers`: http headers. default is empty -* `options`: request options. default is empty +* `headers`: http headers. +* `options`: request options. Example: ```elixir diff --git a/installation/nginx-cache-purge.sh.example b/installation/nginx-cache-purge.sh.example index b2915321c..5f6cbb128 100755 --- a/installation/nginx-cache-purge.sh.example +++ b/installation/nginx-cache-purge.sh.example @@ -13,7 +13,7 @@ CACHE_DIRECTORY="/tmp/pleroma-media-cache" ## $3 - (optional) the number of parallel processes to run for grep. get_cache_files() { local max_parallel=${3-16} - find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E Rl "^KEY:.*$1" | sort -u + find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E -Rl "^KEY:.*$1" | sort -u } ## Removes an item from the given cache zone. @@ -37,4 +37,4 @@ purge() { } -purge $1 +purge $@ diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 9d3d92b38..adebebc7a 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -148,7 +148,8 @@ defp cachex_children do build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500), build_cachex("web_resp", limit: 2500), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), - build_cachex("failed_proxy_url", limit: 2500) + build_cachex("failed_proxy_url", limit: 2500), + build_cachex("deleted_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) ] end diff --git a/lib/pleroma/plugs/uploaded_media.ex b/lib/pleroma/plugs/uploaded_media.ex index 94147e0c4..2f3fde002 100644 --- a/lib/pleroma/plugs/uploaded_media.ex +++ b/lib/pleroma/plugs/uploaded_media.ex @@ -10,6 +10,8 @@ defmodule Pleroma.Plugs.UploadedMedia do import Pleroma.Web.Gettext require Logger + alias Pleroma.Web.MediaProxy + @behaviour Plug # no slashes @path "media" @@ -35,8 +37,7 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do %{query_params: %{"name" => name}} = conn -> name = String.replace(name, "\"", "\\\"") - conn - |> put_resp_header("content-disposition", "filename=\"#{name}\"") + put_resp_header(conn, "content-disposition", "filename=\"#{name}\"") conn -> conn @@ -47,7 +48,8 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do with uploader <- Keyword.fetch!(config, :uploader), proxy_remote = Keyword.get(config, :proxy_remote, false), - {:ok, get_method} <- uploader.get_file(file) do + {:ok, get_method} <- uploader.get_file(file), + false <- media_is_deleted(conn, get_method) do get_media(conn, get_method, proxy_remote, opts) else _ -> @@ -59,6 +61,14 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do def call(conn, _opts), do: conn + defp media_is_deleted(%{request_path: path} = _conn, {:static_dir, _}) do + MediaProxy.in_deleted_urls(Pleroma.Web.base_url() <> path) + end + + defp media_is_deleted(_, {:url, url}), do: MediaProxy.in_deleted_urls(url) + + defp media_is_deleted(_, _), do: false + defp get_media(conn, {:static_dir, directory}, _, opts) do static_opts = Map.get(opts, :static_plug_opts) diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index c037ff13e..83ff8589c 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -5,22 +5,33 @@ defmodule Pleroma.Web.MediaProxy.Invalidation do @moduledoc false - @callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()} + @callback purge(list(String.t()), Keyword.t()) :: {:ok, list(String.t())} | {:error, String.t()} alias Pleroma.Config + alias Pleroma.Web.MediaProxy - @spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()} + @spec enabled?() :: boolean() + def enabled?, do: Config.get([:media_proxy, :invalidation, :enabled]) + + @spec purge(list(String.t()) | String.t()) :: {:ok, list(String.t())} | {:error, String.t()} def purge(urls) do - [:media_proxy, :invalidation, :enabled] - |> Config.get() - |> do_purge(urls) + prepared_urls = prepare_urls(urls) + + if enabled?() do + do_purge(prepared_urls) + else + {:ok, prepared_urls} + end end - defp do_purge(true, urls) do + defp do_purge(urls) do provider = Config.get([:media_proxy, :invalidation, :provider]) - options = Config.get(provider) - provider.purge(urls, options) + provider.purge(urls, Config.get(provider)) end - defp do_purge(_, _), do: :ok + def prepare_urls(urls) do + urls + |> List.wrap() + |> Enum.map(&MediaProxy.url(&1)) + end end diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex index 07248df6e..3694b56e8 100644 --- a/lib/pleroma/web/media_proxy/invalidations/http.ex +++ b/lib/pleroma/web/media_proxy/invalidations/http.ex @@ -10,9 +10,9 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do @impl Pleroma.Web.MediaProxy.Invalidation def purge(urls, opts) do - method = Map.get(opts, :method, :purge) - headers = Map.get(opts, :headers, []) - options = Map.get(opts, :options, []) + method = Keyword.get(opts, :method, :purge) + headers = Keyword.get(opts, :headers, []) + options = Keyword.get(opts, :options, []) Logger.debug("Running cache purge: #{inspect(urls)}") @@ -22,7 +22,7 @@ def purge(urls, opts) do end end) - {:ok, "success"} + {:ok, urls} end defp do_purge(method, url, headers, options) do diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index 6be782132..d41d647bb 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -10,32 +10,34 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do require Logger @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, %{script_path: script_path} = _options) do + def purge(urls, opts) do args = urls |> List.wrap() |> Enum.uniq() |> Enum.join(" ") - path = Path.expand(script_path) - - Logger.debug("Running cache purge: #{inspect(urls)}, #{path}") - - case do_purge(path, [args]) do - {result, exit_status} when exit_status > 0 -> - Logger.error("Error while cache purge: #{inspect(result)}") - {:error, inspect(result)} - - _ -> - {:ok, "success"} - end + opts + |> Keyword.get(:script_path, nil) + |> do_purge([args]) + |> handle_result(urls) end - def purge(_, _), do: {:error, "not found script path"} - - defp do_purge(path, args) do + defp do_purge(script_path, args) when is_binary(script_path) do + path = Path.expand(script_path) + Logger.debug("Running cache purge: #{inspect(args)}, #{inspect(path)}") System.cmd(path, args) rescue - error -> {inspect(error), 1} + error -> error + end + + defp do_purge(_, _), do: {:error, "not found script path"} + + defp handle_result({_result, 0}, urls), do: {:ok, urls} + defp handle_result({:error, error}, urls), do: handle_result(error, urls) + + defp handle_result(error, _) do + Logger.error("Error while cache purge: #{inspect(error)}") + {:error, inspect(error)} end end diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index b2b524524..59ca217ab 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -6,20 +6,53 @@ defmodule Pleroma.Web.MediaProxy do alias Pleroma.Config alias Pleroma.Upload alias Pleroma.Web + alias Pleroma.Web.MediaProxy.Invalidation @base64_opts [padding: false] + @spec in_deleted_urls(String.t()) :: boolean() + def in_deleted_urls(url), do: elem(Cachex.exists?(:deleted_urls_cache, url(url)), 1) + + def remove_from_deleted_urls(urls) when is_list(urls) do + Cachex.execute!(:deleted_urls_cache, fn cache -> + Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1)) + end) + end + + def remove_from_deleted_urls(url) when is_binary(url) do + Cachex.del(:deleted_urls_cache, url(url)) + end + + def put_in_deleted_urls(urls) when is_list(urls) do + Cachex.execute!(:deleted_urls_cache, fn cache -> + Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true)) + end) + end + + def put_in_deleted_urls(url) when is_binary(url) do + Cachex.put(:deleted_urls_cache, url(url), true) + end + def url(url) when is_nil(url) or url == "", do: nil def url("/" <> _ = url), do: url def url(url) do - if disabled?() or local?(url) or whitelisted?(url) do + if disabled?() or not is_url_proxiable?(url) do url else encode_url(url) end end + @spec is_url_proxiable?(String.t()) :: boolean() + def is_url_proxiable?(url) do + if local?(url) or whitelisted?(url) do + false + else + true + end + end + defp disabled?, do: !Config.get([:media_proxy, :enabled], false) defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url()) diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index 4657a4383..ff0158d83 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -14,10 +14,11 @@ def remote(conn, %{"sig" => sig64, "url" => url64} = params) do with config <- Pleroma.Config.get([:media_proxy], []), true <- Keyword.get(config, :enabled, false), {:ok, url} <- MediaProxy.decode_url(sig64, url64), + {_, false} <- {:in_deleted_urls, MediaProxy.in_deleted_urls(url)}, :ok <- filename_matches(params, conn.request_path, url) do ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts)) else - false -> + error when error in [false, {:in_deleted_urls, true}] -> send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404)) {:error, :invalid_signature} -> diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 49352db2a..4ad19c0fc 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -23,8 +23,25 @@ def perform( Enum.map(attachment["url"], & &1["href"]) end) - names = Enum.map(attachments, & &1["name"]) + # find all objects for copies of the attachments, name and actor doesn't matter here + hrefs + |> fetch_objects + |> prepare_objects(actor, Enum.map(attachments, & &1["name"])) + |> Enum.reduce({[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} -> + with 1 <- count do + {ids ++ [id], hrefs ++ [href]} + else + _ -> {ids ++ [id], hrefs} + end + end) + |> do_clean + {:ok, :success} + end + + def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip} + + defp do_clean({object_ids, attachment_urls}) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) prefix = @@ -39,68 +56,60 @@ def perform( "/" ) - # find all objects for copies of the attachments, name and actor doesn't matter here - object_ids_and_hrefs = - from(o in Object, - where: - fragment( - "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)", - o.data, - o.data, - ^hrefs - ) - ) - # The query above can be time consumptive on large instances until we - # refactor how uploads are stored - |> Repo.all(timeout: :infinity) - # we should delete 1 object for any given attachment, but don't delete - # files if there are more than 1 object for it - |> Enum.reduce(%{}, fn %{ - id: id, - data: %{ - "url" => [%{"href" => href}], - "actor" => obj_actor, - "name" => name - } - }, - acc -> - Map.update(acc, href, %{id: id, count: 1}, fn val -> - case obj_actor == actor and name in names do - true -> - # set id of the actor's object that will be deleted - %{val | id: id, count: val.count + 1} + Enum.each(attachment_urls, fn href -> + href + |> String.trim_leading("#{base_url}/#{prefix}") + |> uploader.delete_file() + end) - false -> - # another actor's object, just increase count to not delete file - %{val | count: val.count + 1} - end - end) - end) - |> Enum.map(fn {href, %{id: id, count: count}} -> - # only delete files that have single instance - with 1 <- count do - href - |> String.trim_leading("#{base_url}/#{prefix}") - |> uploader.delete_file() - - {id, href} - else - _ -> {id, nil} - end - end) - - object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end) - - from(o in Object, where: o.id in ^object_ids) - |> Repo.delete_all() - - object_ids_and_hrefs - |> Enum.filter(fn {_, href} -> not is_nil(href) end) - |> Enum.map(&elem(&1, 1)) - |> Pleroma.Web.MediaProxy.Invalidation.purge() - - {:ok, :success} + delete_objects(object_ids) end - def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip} + defp delete_objects([_ | _] = object_ids) do + Repo.delete_all(from(o in Object, where: o.id in ^object_ids)) + end + + defp delete_objects(_), do: :ok + + # we should delete 1 object for any given attachment, but don't delete + # files if there are more than 1 object for it + def prepare_objects(objects, actor, names) do + objects + |> Enum.reduce(%{}, fn %{ + id: id, + data: %{ + "url" => [%{"href" => href}], + "actor" => obj_actor, + "name" => name + } + }, + acc -> + Map.update(acc, href, %{id: id, count: 1}, fn val -> + case obj_actor == actor and name in names do + true -> + # set id of the actor's object that will be deleted + %{val | id: id, count: val.count + 1} + + false -> + # another actor's object, just increase count to not delete file + %{val | count: val.count + 1} + end + end) + end) + end + + def fetch_objects(hrefs) do + from(o in Object, + where: + fragment( + "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)", + o.data, + o.data, + ^hrefs + ) + ) + # The query above can be time consumptive on large instances until we + # refactor how uploads are stored + |> Repo.all(timeout: :infinity) + end end diff --git a/test/web/media_proxy/invalidation_test.exs b/test/web/media_proxy/invalidation_test.exs new file mode 100644 index 000000000..3a9fa8c88 --- /dev/null +++ b/test/web/media_proxy/invalidation_test.exs @@ -0,0 +1,65 @@ +defmodule Pleroma.Web.MediaProxy.InvalidationTest do + use ExUnit.Case + use Pleroma.Tests.Helpers + + alias Pleroma.Config + alias Pleroma.Web.MediaProxy.Invalidation + + import ExUnit.CaptureLog + import Mock + import Tesla.Mock + + setup do: clear_config([:media_proxy]) + + setup do + on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + :ok + end + + describe "Invalidation.Http" do + test "perform request to clear cache" do + Config.put([:media_proxy, :enabled], false) + Config.put([:media_proxy, :invalidation, :enabled], true) + Config.put([:media_proxy, :invalidation, :provider], Invalidation.Http) + + Config.put([Invalidation.Http], method: :purge, headers: [{"x-refresh", 1}]) + image_url = "http://example.com/media/example.jpg" + Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url) + + mock(fn + %{ + method: :purge, + url: "http://example.com/media/example.jpg", + headers: [{"x-refresh", 1}] + } -> + %Tesla.Env{status: 200} + end) + + assert capture_log(fn -> + assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + assert Invalidation.purge([image_url]) == {:ok, [image_url]} + assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + end) =~ "Running cache purge: [\"#{image_url}\"]" + end + end + + describe "Invalidation.Script" do + test "run script to clear cache" do + Config.put([:media_proxy, :enabled], false) + Config.put([:media_proxy, :invalidation, :enabled], true) + Config.put([:media_proxy, :invalidation, :provider], Invalidation.Script) + Config.put([Invalidation.Script], script_path: "purge-nginx") + + image_url = "http://example.com/media/example.jpg" + Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url) + + with_mocks [{System, [], [cmd: fn _, _ -> {"ok", 0} end]}] do + assert capture_log(fn -> + assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + assert Invalidation.purge([image_url]) == {:ok, [image_url]} + assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + end) =~ "Running cache purge: [\"#{image_url}\"]" + end + end + end +end diff --git a/test/web/media_proxy/invalidations/http_test.exs b/test/web/media_proxy/invalidations/http_test.exs index 8a3b4141c..09e7ca0fb 100644 --- a/test/web/media_proxy/invalidations/http_test.exs +++ b/test/web/media_proxy/invalidations/http_test.exs @@ -5,6 +5,11 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do import ExUnit.CaptureLog import Tesla.Mock + setup do + on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + :ok + end + test "logs hasn't error message when request is valid" do mock(fn %{method: :purge, url: "http://example.com/media/example.jpg"} -> @@ -14,8 +19,8 @@ test "logs hasn't error message when request is valid" do refute capture_log(fn -> assert Invalidation.Http.purge( ["http://example.com/media/example.jpg"], - %{} - ) == {:ok, "success"} + [] + ) == {:ok, ["http://example.com/media/example.jpg"]} end) =~ "Error while cache purge" end @@ -28,8 +33,8 @@ test "it write error message in logs when request invalid" do assert capture_log(fn -> assert Invalidation.Http.purge( ["http://example.com/media/example1.jpg"], - %{} - ) == {:ok, "success"} + [] + ) == {:ok, ["http://example.com/media/example1.jpg"]} end) =~ "Error while cache purge: url - http://example.com/media/example1.jpg" end end diff --git a/test/web/media_proxy/invalidations/script_test.exs b/test/web/media_proxy/invalidations/script_test.exs index 1358963ab..c69cec07a 100644 --- a/test/web/media_proxy/invalidations/script_test.exs +++ b/test/web/media_proxy/invalidations/script_test.exs @@ -4,17 +4,24 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.ScriptTest do import ExUnit.CaptureLog + setup do + on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + :ok + end + test "it logger error when script not found" do assert capture_log(fn -> assert Invalidation.Script.purge( ["http://example.com/media/example.jpg"], - %{script_path: "./example"} - ) == {:error, "\"%ErlangError{original: :enoent}\""} - end) =~ "Error while cache purge: \"%ErlangError{original: :enoent}\"" + script_path: "./example" + ) == {:error, "%ErlangError{original: :enoent}"} + end) =~ "Error while cache purge: %ErlangError{original: :enoent}" - assert Invalidation.Script.purge( - ["http://example.com/media/example.jpg"], - %{} - ) == {:error, "not found script path"} + capture_log(fn -> + assert Invalidation.Script.purge( + ["http://example.com/media/example.jpg"], + [] + ) == {:error, "\"not found script path\""} + end) end end diff --git a/test/web/media_proxy/media_proxy_controller_test.exs b/test/web/media_proxy/media_proxy_controller_test.exs index da79d38a5..2b6b25221 100644 --- a/test/web/media_proxy/media_proxy_controller_test.exs +++ b/test/web/media_proxy/media_proxy_controller_test.exs @@ -10,6 +10,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do setup do: clear_config(:media_proxy) setup do: clear_config([Pleroma.Web.Endpoint, :secret_key_base]) + setup do + on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + :ok + end + test "it returns 404 when MediaProxy disabled", %{conn: conn} do Config.put([:media_proxy, :enabled], false) @@ -66,4 +71,16 @@ test "it performs ReverseProxy.call when signature valid", %{conn: conn} do assert %Plug.Conn{status: :success} = get(conn, url) end end + + test "it returns 404 when url contains in deleted_urls cache", %{conn: conn} do + Config.put([:media_proxy, :enabled], true) + Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000") + url = Pleroma.Web.MediaProxy.encode_url("https://google.fn/test.png") + Pleroma.Web.MediaProxy.put_in_deleted_urls("https://google.fn/test.png") + + with_mock Pleroma.ReverseProxy, + call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do + assert %Plug.Conn{status: 404, resp_body: "Not Found"} = get(conn, url) + end + end end From 62b8c31b7a84dadb2a46861fe0f2dd1dbf9d40f0 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 15 Jun 2020 14:55:00 +0300 Subject: [PATCH 10/56] added tests --- .../media_proxy_cache_controller.ex | 31 ++++- .../web/media_proxy/invalidations/script.ex | 2 +- .../media_proxy_cache_controller_test.exs | 116 +++++++++++++++--- 3 files changed, 127 insertions(+), 22 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex index 7b28f7c72..e3fa0ac28 100644 --- a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.Web.ApiSpec.Admin, as: Spec + alias Pleroma.Web.MediaProxy plug(Pleroma.Web.ApiSpec.CastAndValidate) @@ -24,15 +25,39 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation - def index(%{assigns: %{user: _}} = conn, _) do - render(conn, "index.json", urls: []) + def index(%{assigns: %{user: _}} = conn, params) do + cursor = + :deleted_urls_cache + |> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}]) + |> :qlc.cursor() + + urls = + case params.page do + 1 -> + :qlc.next_answers(cursor, params.page_size) + + _ -> + :qlc.next_answers(cursor, (params.page - 1) * params.page_size) + :qlc.next_answers(cursor, params.page_size) + end + + :qlc.delete_cursor(cursor) + + render(conn, "index.json", urls: urls) end def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do + MediaProxy.remove_from_deleted_urls(urls) render(conn, "index.json", urls: urls) end - def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: _ban}} = conn, _) do + def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do + MediaProxy.Invalidation.purge(urls) + + if ban do + MediaProxy.put_in_deleted_urls(urls) + end + render(conn, "index.json", urls: urls) end end diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index d41d647bb..0217b119d 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do require Logger @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, opts) do + def purge(urls, opts \\ %{}) do args = urls |> List.wrap() diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs index 1b1d6bc36..76a96f46f 100644 --- a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs +++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs @@ -6,6 +6,16 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory + import Mock + + alias Pleroma.Web.MediaProxy + + setup do: clear_config([:media_proxy]) + + setup do + on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + :ok + end setup do admin = insert(:user, is_admin: true) @@ -16,51 +26,121 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do |> assign(:user, admin) |> assign(:token, token) + Config.put([:media_proxy, :enabled], true) + Config.put([:media_proxy, :invalidation, :enabled], true) + Config.put([:media_proxy, :invalidation, :provider], MediaProxy.Invalidation.Script) + {:ok, %{admin: admin, token: token, conn: conn}} end describe "GET /api/pleroma/admin/media_proxy_caches" do test "shows banned MediaProxy URLs", %{conn: conn} do + MediaProxy.put_in_deleted_urls([ + "http://localhost:4001/media/a688346.jpg", + "http://localhost:4001/media/fb1f4d.jpg" + ]) + + MediaProxy.put_in_deleted_urls("http://localhost:4001/media/gb1f44.jpg") + MediaProxy.put_in_deleted_urls("http://localhost:4001/media/tb13f47.jpg") + MediaProxy.put_in_deleted_urls("http://localhost:4001/media/wb1f46.jpg") + response = conn - |> get("/api/pleroma/admin/media_proxy_caches") + |> get("/api/pleroma/admin/media_proxy_caches?page_size=2") |> json_response_and_validate_schema(200) - assert response["urls"] == [] + assert response["urls"] == [ + "http://localhost:4001/media/fb1f4d.jpg", + "http://localhost:4001/media/a688346.jpg" + ] + + response = + conn + |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=2") + |> json_response_and_validate_schema(200) + + assert response["urls"] == [ + "http://localhost:4001/media/gb1f44.jpg", + "http://localhost:4001/media/tb13f47.jpg" + ] + + response = + conn + |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3") + |> json_response_and_validate_schema(200) + + assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"] end end describe "DELETE /api/pleroma/admin/media_proxy_caches/delete" do test "deleted MediaProxy URLs from banned", %{conn: conn} do + MediaProxy.put_in_deleted_urls([ + "http://localhost:4001/media/a688346.jpg", + "http://localhost:4001/media/fb1f4d.jpg" + ]) + response = conn |> put_req_header("content-type", "application/json") |> post("/api/pleroma/admin/media_proxy_caches/delete", %{ - urls: ["http://example.com/media/a688346.jpg", "http://example.com/media/fb1f4d.jpg"] + urls: ["http://localhost:4001/media/a688346.jpg"] }) |> json_response_and_validate_schema(200) - assert response["urls"] == [ - "http://example.com/media/a688346.jpg", - "http://example.com/media/fb1f4d.jpg" - ] + assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"] + refute MediaProxy.in_deleted_urls("http://localhost:4001/media/a688346.jpg") + assert MediaProxy.in_deleted_urls("http://localhost:4001/media/fb1f4d.jpg") end end describe "PURGE /api/pleroma/admin/media_proxy_caches/purge" do test "perform invalidates cache of MediaProxy", %{conn: conn} do - response = - conn - |> put_req_header("content-type", "application/json") - |> post("/api/pleroma/admin/media_proxy_caches/purge", %{ - urls: ["http://example.com/media/a688346.jpg", "http://example.com/media/fb1f4d.jpg"] - }) - |> json_response_and_validate_schema(200) + urls = [ + "http://example.com/media/a688346.jpg", + "http://example.com/media/fb1f4d.jpg" + ] - assert response["urls"] == [ - "http://example.com/media/a688346.jpg", - "http://example.com/media/fb1f4d.jpg" - ] + with_mocks [ + {MediaProxy.Invalidation.Script, [], + [ + purge: fn _, _ -> {"ok", 0} end + ]} + ] do + response = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false}) + |> json_response_and_validate_schema(200) + + assert response["urls"] == urls + + refute MediaProxy.in_deleted_urls("http://example.com/media/a688346.jpg") + refute MediaProxy.in_deleted_urls("http://example.com/media/fb1f4d.jpg") + end + end + + test "perform invalidates cache of MediaProxy and adds url to banned", %{conn: conn} do + urls = [ + "http://example.com/media/a688346.jpg", + "http://example.com/media/fb1f4d.jpg" + ] + + with_mocks [{MediaProxy.Invalidation.Script, [], [purge: fn _, _ -> {"ok", 0} end]}] do + response = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/media_proxy_caches/purge", %{ + urls: urls, + ban: true + }) + |> json_response_and_validate_schema(200) + + assert response["urls"] == urls + + assert MediaProxy.in_deleted_urls("http://example.com/media/a688346.jpg") + assert MediaProxy.in_deleted_urls("http://example.com/media/fb1f4d.jpg") + end end end end From efdfc85c2d8e5118c1aa18e4f04026ec90cd11d2 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 15 Jun 2020 15:24:00 +0300 Subject: [PATCH 11/56] update docs --- docs/API/admin_api.md | 64 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 92816baf9..6659b605d 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -1224,4 +1224,66 @@ Loads json generated from `config/descriptions.exs`. - Response: - On success: `204`, empty response - On failure: - - 400 Bad Request `"Invalid parameters"` when `status` is missing \ No newline at end of file + - 400 Bad Request `"Invalid parameters"` when `status` is missing + +## `GET /api/pleroma/admin/media_proxy_caches` + +### Get a list of all banned MediaProxy URLs in Cachex + +- Authentication: required +- Params: +- *optional* `page`: **integer** page number +- *optional* `page_size`: **integer** number of log entries per page (default is `50`) + +- Response: + +``` json +{ + "urls": [ + "http://example.com/media/a688346.jpg", + "http://example.com/media/fb1f4d.jpg" + ] +} + +``` + +## `POST /api/pleroma/admin/media_proxy_caches/delete` + +### Remove a banned MediaProxy URL from Cachex + +- Authentication: required +- Params: + - `urls` + +- Response: + +``` json +{ + "urls": [ + "http://example.com/media/a688346.jpg", + "http://example.com/media/fb1f4d.jpg" + ] +} + +``` + +## `POST /api/pleroma/admin/media_proxy_caches/purge` + +### Purge a MediaProxy URL + +- Authentication: required +- Params: + - `urls` + - `ban` + +- Response: + +``` json +{ + "urls": [ + "http://example.com/media/a688346.jpg", + "http://example.com/media/fb1f4d.jpg" + ] +} + +``` From e1ee8bc1da17a356c88b535db7a9228fccc5251f Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 15 Jun 2020 14:29:34 +0200 Subject: [PATCH 12/56] User: update_follower_count refactor. --- lib/pleroma/user.ex | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 52ac9052b..39a9e13e8 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -747,7 +747,6 @@ def follow(%User{} = follower, %User{} = followed, state \\ :follow_accept) do follower |> update_following_count() - |> set_cache() end end @@ -776,7 +775,6 @@ defp do_unfollow(%User{} = follower, %User{} = followed) do {:ok, follower} = follower |> update_following_count() - |> set_cache() {:ok, follower, followed} @@ -1128,35 +1126,25 @@ defp follow_information_changeset(user, params) do ]) end + @spec update_follower_count(User.t()) :: {:ok, User.t()} def update_follower_count(%User{} = user) do if user.local or !Pleroma.Config.get([:instance, :external_user_synchronization]) do - follower_count_query = - User.Query.build(%{followers: user, deactivated: false}) - |> select([u], %{count: count(u.id)}) + follower_count = FollowingRelationship.follower_count(user) - User - |> where(id: ^user.id) - |> join(:inner, [u], s in subquery(follower_count_query)) - |> update([u, s], - set: [follower_count: s.count] - ) - |> select([u], u) - |> Repo.update_all([]) - |> case do - {1, [user]} -> set_cache(user) - _ -> {:error, user} - end + user + |> follow_information_changeset(%{follower_count: follower_count}) + |> update_and_set_cache else {:ok, maybe_fetch_follow_information(user)} end end - @spec update_following_count(User.t()) :: User.t() + @spec update_following_count(User.t()) :: {:ok, User.t()} def update_following_count(%User{local: false} = user) do if Pleroma.Config.get([:instance, :external_user_synchronization]) do - maybe_fetch_follow_information(user) + {:ok, maybe_fetch_follow_information(user)} else - user + {:ok, user} end end @@ -1165,7 +1153,7 @@ def update_following_count(%User{local: true} = user) do user |> follow_information_changeset(%{following_count: following_count}) - |> Repo.update!() + |> update_and_set_cache() end def set_unread_conversation_count(%User{local: true} = user) do From b02311079961c5193af1c144516a3caeee72b582 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 15 Jun 2020 20:47:02 +0300 Subject: [PATCH 13/56] fixed a visibility of functions --- .../workers/attachments_cleanup_worker.ex | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 4ad19c0fc..8deeabda0 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -18,22 +18,11 @@ def perform( }, _job ) do - hrefs = - Enum.flat_map(attachments, fn attachment -> - Enum.map(attachment["url"], & &1["href"]) - end) - - # find all objects for copies of the attachments, name and actor doesn't matter here - hrefs + attachments + |> Enum.flat_map(fn item -> Enum.map(item["url"], & &1["href"]) end) |> fetch_objects |> prepare_objects(actor, Enum.map(attachments, & &1["name"])) - |> Enum.reduce({[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} -> - with 1 <- count do - {ids ++ [id], hrefs ++ [href]} - else - _ -> {ids ++ [id], hrefs} - end - end) + |> filter_objects |> do_clean {:ok, :success} @@ -73,7 +62,17 @@ defp delete_objects(_), do: :ok # we should delete 1 object for any given attachment, but don't delete # files if there are more than 1 object for it - def prepare_objects(objects, actor, names) do + defp filter_objects(objects) do + Enum.reduce(objects, {[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} -> + with 1 <- count do + {ids ++ [id], hrefs ++ [href]} + else + _ -> {ids ++ [id], hrefs} + end + end) + end + + defp prepare_objects(objects, actor, names) do objects |> Enum.reduce(%{}, fn %{ id: id, @@ -98,7 +97,7 @@ def prepare_objects(objects, actor, names) do end) end - def fetch_objects(hrefs) do + defp fetch_objects(hrefs) do from(o in Object, where: fragment( From 1eb6cedaadee4e1ab3e0885b4e03a8dd17ba08ea Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 16 Jun 2020 13:08:27 +0200 Subject: [PATCH 14/56] ActivityPub: When restricting to media posts, only show 'Creates'. --- lib/pleroma/web/activity_pub/activity_pub.ex | 3 ++- .../controllers/account_controller_test.exs | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index c9dc6135c..efb8b81db 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -833,7 +833,8 @@ defp restrict_media(_query, %{only_media: _val, skip_preload: true}) do defp restrict_media(query, %{only_media: true}) do from( - [_activity, object] in query, + [activity, object] in query, + where: fragment("(?)->>'type' = ?", activity.data, "Create"), where: fragment("not (?)->'attachment' = (?)", object.data, ^[]) ) end diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs index 1ce97378d..2343a9d2d 100644 --- a/test/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/web/mastodon_api/controllers/account_controller_test.exs @@ -350,9 +350,10 @@ test "unimplemented pinned statuses feature", %{conn: conn} do assert json_response_and_validate_schema(conn, 200) == [] end - test "gets an users media", %{conn: conn} do + test "gets an users media, excludes reblogs", %{conn: conn} do note = insert(:note_activity) user = User.get_cached_by_ap_id(note.data["actor"]) + other_user = insert(:user) file = %Plug.Upload{ content_type: "image/jpg", @@ -364,6 +365,13 @@ test "gets an users media", %{conn: conn} do {:ok, %{id: image_post_id}} = CommonAPI.post(user, %{status: "cofe", media_ids: [media_id]}) + {:ok, %{id: media_id}} = ActivityPub.upload(file, actor: other_user.ap_id) + + {:ok, %{id: other_image_post_id}} = + CommonAPI.post(other_user, %{status: "cofe2", media_ids: [media_id]}) + + {:ok, _announce} = CommonAPI.repeat(other_image_post_id, user) + conn = get(conn, "/api/v1/accounts/#{user.id}/statuses?only_media=true") assert [%{"id" => ^image_post_id}] = json_response_and_validate_schema(conn, 200) From 4733f6a3371504ebb3eeb447d7c20d56c10b43bf Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 16 Jun 2020 13:09:28 +0200 Subject: [PATCH 15/56] Changelog: Add info about `only_media` changes. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2629bf84..eee442817 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Changed +- Using the `only_media` filter on timelines will now exclude reblog media - MFR policy to set global expiration for all local Create activities - OGP rich media parser merged with TwitterCard
From 90613348ed8078e1906f7ffd18eebfa1a3b7f25a Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:56:13 +0000 Subject: [PATCH 16/56] Apply suggestion to docs/API/admin_api.md --- docs/API/admin_api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 6659b605d..8a3d60187 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -1253,7 +1253,7 @@ Loads json generated from `config/descriptions.exs`. - Authentication: required - Params: - - `urls` + - `urls` (array) - Response: From abfb1c756b62c24589d2881d77bf5974a80809d3 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:56:17 +0000 Subject: [PATCH 17/56] Apply suggestion to docs/API/admin_api.md --- docs/API/admin_api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 8a3d60187..c7f56cf5f 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -1273,8 +1273,8 @@ Loads json generated from `config/descriptions.exs`. - Authentication: required - Params: - - `urls` - - `ban` + - `urls` (array) + - `ban` (boolean) - Response: From 74fd761637f737822d01aed945b6e0c75ced7008 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:56:30 +0000 Subject: [PATCH 18/56] Apply suggestion to lib/pleroma/web/media_proxy/invalidation.ex --- lib/pleroma/web/media_proxy/invalidation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index 83ff8589c..6da7eb720 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -32,6 +32,6 @@ defp do_purge(urls) do def prepare_urls(urls) do urls |> List.wrap() - |> Enum.map(&MediaProxy.url(&1)) + |> Enum.map(&MediaProxy.url/1) end end From 1b45bc7b2ac53e56a4868da7e2b5b198d16306ab Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:58:08 +0000 Subject: [PATCH 19/56] Apply suggestion to test/web/admin_api/controllers/media_proxy_cache_controller_test.exs --- .../admin_api/controllers/media_proxy_cache_controller_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs index 76a96f46f..ddaf39f14 100644 --- a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs +++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs @@ -14,7 +14,6 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do setup do on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) - :ok end setup do From 793a53f1ec18a42f15f58494e40ed3c37d35f95c Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:58:16 +0000 Subject: [PATCH 20/56] Apply suggestion to test/web/admin_api/controllers/media_proxy_cache_controller_test.exs --- .../admin_api/controllers/media_proxy_cache_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs index ddaf39f14..81e20d001 100644 --- a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs +++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs @@ -72,7 +72,7 @@ test "shows banned MediaProxy URLs", %{conn: conn} do end end - describe "DELETE /api/pleroma/admin/media_proxy_caches/delete" do + describe "POST /api/pleroma/admin/media_proxy_caches/delete" do test "deleted MediaProxy URLs from banned", %{conn: conn} do MediaProxy.put_in_deleted_urls([ "http://localhost:4001/media/a688346.jpg", From 6d33a3a51bb8ff0afdf7f4f9880f8f5c5f2dfebc Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:58:28 +0000 Subject: [PATCH 21/56] Apply suggestion to test/web/admin_api/controllers/media_proxy_cache_controller_test.exs --- .../admin_api/controllers/media_proxy_cache_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs index 81e20d001..42a3c0dd8 100644 --- a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs +++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs @@ -93,7 +93,7 @@ test "deleted MediaProxy URLs from banned", %{conn: conn} do end end - describe "PURGE /api/pleroma/admin/media_proxy_caches/purge" do + describe "POST /api/pleroma/admin/media_proxy_caches/purge" do test "perform invalidates cache of MediaProxy", %{conn: conn} do urls = [ "http://example.com/media/a688346.jpg", From 11b22a42293ec2ac0e66897bf4b29b5363913c19 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:58:33 +0000 Subject: [PATCH 22/56] Apply suggestion to test/web/media_proxy/invalidations/http_test.exs --- test/web/media_proxy/invalidations/http_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/web/media_proxy/invalidations/http_test.exs b/test/web/media_proxy/invalidations/http_test.exs index 09e7ca0fb..9d181dd8b 100644 --- a/test/web/media_proxy/invalidations/http_test.exs +++ b/test/web/media_proxy/invalidations/http_test.exs @@ -7,7 +7,6 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do setup do on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) - :ok end test "logs hasn't error message when request is valid" do From 2991aae4c4e4ae430539c1e6fac53fb8a0c991e9 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:58:38 +0000 Subject: [PATCH 23/56] Apply suggestion to test/web/media_proxy/invalidations/script_test.exs --- test/web/media_proxy/invalidations/script_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/web/media_proxy/invalidations/script_test.exs b/test/web/media_proxy/invalidations/script_test.exs index c69cec07a..8e155b705 100644 --- a/test/web/media_proxy/invalidations/script_test.exs +++ b/test/web/media_proxy/invalidations/script_test.exs @@ -6,7 +6,6 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.ScriptTest do setup do on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) - :ok end test "it logger error when script not found" do From 078d687e6ed66f921d7f54114f2dc6bf4abbf237 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:58:50 +0000 Subject: [PATCH 24/56] Apply suggestion to test/web/media_proxy/media_proxy_controller_test.exs --- test/web/media_proxy/media_proxy_controller_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/web/media_proxy/media_proxy_controller_test.exs b/test/web/media_proxy/media_proxy_controller_test.exs index 2b6b25221..72da98a6a 100644 --- a/test/web/media_proxy/media_proxy_controller_test.exs +++ b/test/web/media_proxy/media_proxy_controller_test.exs @@ -12,7 +12,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do setup do on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) - :ok end test "it returns 404 when MediaProxy disabled", %{conn: conn} do From 44ce97a9c9e90d5906386d1b51dea144cd258c32 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 13:12:32 +0000 Subject: [PATCH 25/56] Apply suggestion to lib/pleroma/web/media_proxy/invalidations/script.ex --- lib/pleroma/web/media_proxy/invalidations/script.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index 0217b119d..b0f44e8e2 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do require Logger @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, opts \\ %{}) do + def purge(urls, opts \\ []) do args = urls |> List.wrap() From 9a371bf5f6245b0f372bec7af15e2f4fc41d4ab7 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 13:12:38 +0000 Subject: [PATCH 26/56] Apply suggestion to lib/pleroma/web/media_proxy/invalidations/script.ex --- lib/pleroma/web/media_proxy/invalidations/script.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index b0f44e8e2..d32ffc50b 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -18,7 +18,7 @@ def purge(urls, opts \\ []) do |> Enum.join(" ") opts - |> Keyword.get(:script_path, nil) + |> Keyword.get(:script_path) |> do_purge([args]) |> handle_result(urls) end From 96493da7bdab4ff4a51cbebf18df4127ddc47990 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 13:14:01 +0000 Subject: [PATCH 27/56] Apply suggestion to test/web/media_proxy/invalidation_test.exs --- test/web/media_proxy/invalidation_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/web/media_proxy/invalidation_test.exs b/test/web/media_proxy/invalidation_test.exs index 3a9fa8c88..bf9af251c 100644 --- a/test/web/media_proxy/invalidation_test.exs +++ b/test/web/media_proxy/invalidation_test.exs @@ -13,7 +13,6 @@ defmodule Pleroma.Web.MediaProxy.InvalidationTest do setup do on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) - :ok end describe "Invalidation.Http" do From 71a5d9bffb33d9424ea28900ea006678617e0096 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 17 Jun 2020 12:54:02 -0500 Subject: [PATCH 28/56] Empty list as default --- lib/pleroma/web/media_proxy/invalidations/http.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex index 3694b56e8..bb81d8888 100644 --- a/lib/pleroma/web/media_proxy/invalidations/http.ex +++ b/lib/pleroma/web/media_proxy/invalidations/http.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do require Logger @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, opts) do + def purge(urls, opts \\ []) do method = Keyword.get(opts, :method, :purge) headers = Keyword.get(opts, :headers, []) options = Keyword.get(opts, :options, []) From c08c9db0c137d36896910194a6dc50a391a8fee2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 17 Jun 2020 13:02:01 -0500 Subject: [PATCH 29/56] Remove misleading is_ prefix from boolean function --- lib/pleroma/web/media_proxy/media_proxy.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index 59ca217ab..3dccd6b7f 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -37,15 +37,15 @@ def url(url) when is_nil(url) or url == "", do: nil def url("/" <> _ = url), do: url def url(url) do - if disabled?() or not is_url_proxiable?(url) do + if disabled?() or not url_proxiable?(url) do url else encode_url(url) end end - @spec is_url_proxiable?(String.t()) :: boolean() - def is_url_proxiable?(url) do + @spec url_proxiable?(String.t()) :: boolean() + def url_proxiable?(url) do if local?(url) or whitelisted?(url) do false else From 2731ea1334c2c91315465659a0874829cb9e1e11 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 17 Jun 2020 13:13:55 -0500 Subject: [PATCH 30/56] Change references from "deleted_urls" to "banned_urls" as nothing is handled via media deletions anymore; all actions are manual operations by an admin to ban the url --- lib/pleroma/application.ex | 2 +- lib/pleroma/plugs/uploaded_media.ex | 10 ++++---- .../media_proxy_cache_controller.ex | 6 ++--- lib/pleroma/web/media_proxy/media_proxy.ex | 20 ++++++++-------- .../web/media_proxy/media_proxy_controller.ex | 4 ++-- .../media_proxy_cache_controller_test.exs | 24 +++++++++---------- test/web/media_proxy/invalidation_test.exs | 14 +++++------ .../media_proxy_controller_test.exs | 6 ++--- 8 files changed, 43 insertions(+), 43 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index adebebc7a..4a21bf138 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -149,7 +149,7 @@ defp cachex_children do build_cachex("web_resp", limit: 2500), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("failed_proxy_url", limit: 2500), - build_cachex("deleted_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) + build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) ] end diff --git a/lib/pleroma/plugs/uploaded_media.ex b/lib/pleroma/plugs/uploaded_media.ex index 2f3fde002..40984cfc0 100644 --- a/lib/pleroma/plugs/uploaded_media.ex +++ b/lib/pleroma/plugs/uploaded_media.ex @@ -49,7 +49,7 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do with uploader <- Keyword.fetch!(config, :uploader), proxy_remote = Keyword.get(config, :proxy_remote, false), {:ok, get_method} <- uploader.get_file(file), - false <- media_is_deleted(conn, get_method) do + false <- media_is_banned(conn, get_method) do get_media(conn, get_method, proxy_remote, opts) else _ -> @@ -61,13 +61,13 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do def call(conn, _opts), do: conn - defp media_is_deleted(%{request_path: path} = _conn, {:static_dir, _}) do - MediaProxy.in_deleted_urls(Pleroma.Web.base_url() <> path) + defp media_is_banned(%{request_path: path} = _conn, {:static_dir, _}) do + MediaProxy.in_banned_urls(Pleroma.Web.base_url() <> path) end - defp media_is_deleted(_, {:url, url}), do: MediaProxy.in_deleted_urls(url) + defp media_is_banned(_, {:url, url}), do: MediaProxy.in_banned_urls(url) - defp media_is_deleted(_, _), do: false + defp media_is_banned(_, _), do: false defp get_media(conn, {:static_dir, directory}, _, opts) do static_opts = diff --git a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex index e3fa0ac28..e2759d59f 100644 --- a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do def index(%{assigns: %{user: _}} = conn, params) do cursor = - :deleted_urls_cache + :banned_urls_cache |> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}]) |> :qlc.cursor() @@ -47,7 +47,7 @@ def index(%{assigns: %{user: _}} = conn, params) do end def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do - MediaProxy.remove_from_deleted_urls(urls) + MediaProxy.remove_from_banned_urls(urls) render(conn, "index.json", urls: urls) end @@ -55,7 +55,7 @@ def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _ MediaProxy.Invalidation.purge(urls) if ban do - MediaProxy.put_in_deleted_urls(urls) + MediaProxy.put_in_banned_urls(urls) end render(conn, "index.json", urls: urls) diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index 3dccd6b7f..077fabe47 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -10,27 +10,27 @@ defmodule Pleroma.Web.MediaProxy do @base64_opts [padding: false] - @spec in_deleted_urls(String.t()) :: boolean() - def in_deleted_urls(url), do: elem(Cachex.exists?(:deleted_urls_cache, url(url)), 1) + @spec in_banned_urls(String.t()) :: boolean() + def in_banned_urls(url), do: elem(Cachex.exists?(:banned_urls_cache, url(url)), 1) - def remove_from_deleted_urls(urls) when is_list(urls) do - Cachex.execute!(:deleted_urls_cache, fn cache -> + def remove_from_banned_urls(urls) when is_list(urls) do + Cachex.execute!(:banned_urls_cache, fn cache -> Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1)) end) end - def remove_from_deleted_urls(url) when is_binary(url) do - Cachex.del(:deleted_urls_cache, url(url)) + def remove_from_banned_urls(url) when is_binary(url) do + Cachex.del(:banned_urls_cache, url(url)) end - def put_in_deleted_urls(urls) when is_list(urls) do - Cachex.execute!(:deleted_urls_cache, fn cache -> + def put_in_banned_urls(urls) when is_list(urls) do + Cachex.execute!(:banned_urls_cache, fn cache -> Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true)) end) end - def put_in_deleted_urls(url) when is_binary(url) do - Cachex.put(:deleted_urls_cache, url(url), true) + def put_in_banned_urls(url) when is_binary(url) do + Cachex.put(:banned_urls_cache, url(url), true) end def url(url) when is_nil(url) or url == "", do: nil diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index ff0158d83..9a64b0ef3 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -14,11 +14,11 @@ def remote(conn, %{"sig" => sig64, "url" => url64} = params) do with config <- Pleroma.Config.get([:media_proxy], []), true <- Keyword.get(config, :enabled, false), {:ok, url} <- MediaProxy.decode_url(sig64, url64), - {_, false} <- {:in_deleted_urls, MediaProxy.in_deleted_urls(url)}, + {_, false} <- {:in_banned_urls, MediaProxy.in_banned_urls(url)}, :ok <- filename_matches(params, conn.request_path, url) do ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts)) else - error when error in [false, {:in_deleted_urls, true}] -> + error when error in [false, {:in_banned_urls, true}] -> send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404)) {:error, :invalid_signature} -> diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs index 42a3c0dd8..5ab6cb78a 100644 --- a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs +++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs @@ -13,7 +13,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do setup do: clear_config([:media_proxy]) setup do - on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + on_exit(fn -> Cachex.clear(:banned_urls_cache) end) end setup do @@ -34,14 +34,14 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do describe "GET /api/pleroma/admin/media_proxy_caches" do test "shows banned MediaProxy URLs", %{conn: conn} do - MediaProxy.put_in_deleted_urls([ + MediaProxy.put_in_banned_urls([ "http://localhost:4001/media/a688346.jpg", "http://localhost:4001/media/fb1f4d.jpg" ]) - MediaProxy.put_in_deleted_urls("http://localhost:4001/media/gb1f44.jpg") - MediaProxy.put_in_deleted_urls("http://localhost:4001/media/tb13f47.jpg") - MediaProxy.put_in_deleted_urls("http://localhost:4001/media/wb1f46.jpg") + MediaProxy.put_in_banned_urls("http://localhost:4001/media/gb1f44.jpg") + MediaProxy.put_in_banned_urls("http://localhost:4001/media/tb13f47.jpg") + MediaProxy.put_in_banned_urls("http://localhost:4001/media/wb1f46.jpg") response = conn @@ -74,7 +74,7 @@ test "shows banned MediaProxy URLs", %{conn: conn} do describe "POST /api/pleroma/admin/media_proxy_caches/delete" do test "deleted MediaProxy URLs from banned", %{conn: conn} do - MediaProxy.put_in_deleted_urls([ + MediaProxy.put_in_banned_urls([ "http://localhost:4001/media/a688346.jpg", "http://localhost:4001/media/fb1f4d.jpg" ]) @@ -88,8 +88,8 @@ test "deleted MediaProxy URLs from banned", %{conn: conn} do |> json_response_and_validate_schema(200) assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"] - refute MediaProxy.in_deleted_urls("http://localhost:4001/media/a688346.jpg") - assert MediaProxy.in_deleted_urls("http://localhost:4001/media/fb1f4d.jpg") + refute MediaProxy.in_banned_urls("http://localhost:4001/media/a688346.jpg") + assert MediaProxy.in_banned_urls("http://localhost:4001/media/fb1f4d.jpg") end end @@ -114,8 +114,8 @@ test "perform invalidates cache of MediaProxy", %{conn: conn} do assert response["urls"] == urls - refute MediaProxy.in_deleted_urls("http://example.com/media/a688346.jpg") - refute MediaProxy.in_deleted_urls("http://example.com/media/fb1f4d.jpg") + refute MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg") + refute MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg") end end @@ -137,8 +137,8 @@ test "perform invalidates cache of MediaProxy and adds url to banned", %{conn: c assert response["urls"] == urls - assert MediaProxy.in_deleted_urls("http://example.com/media/a688346.jpg") - assert MediaProxy.in_deleted_urls("http://example.com/media/fb1f4d.jpg") + assert MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg") + assert MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg") end end end diff --git a/test/web/media_proxy/invalidation_test.exs b/test/web/media_proxy/invalidation_test.exs index bf9af251c..926ae74ca 100644 --- a/test/web/media_proxy/invalidation_test.exs +++ b/test/web/media_proxy/invalidation_test.exs @@ -12,7 +12,7 @@ defmodule Pleroma.Web.MediaProxy.InvalidationTest do setup do: clear_config([:media_proxy]) setup do - on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + on_exit(fn -> Cachex.clear(:banned_urls_cache) end) end describe "Invalidation.Http" do @@ -23,7 +23,7 @@ test "perform request to clear cache" do Config.put([Invalidation.Http], method: :purge, headers: [{"x-refresh", 1}]) image_url = "http://example.com/media/example.jpg" - Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url) + Pleroma.Web.MediaProxy.put_in_banned_urls(image_url) mock(fn %{ @@ -35,9 +35,9 @@ test "perform request to clear cache" do end) assert capture_log(fn -> - assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + assert Pleroma.Web.MediaProxy.in_banned_urls(image_url) assert Invalidation.purge([image_url]) == {:ok, [image_url]} - assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + assert Pleroma.Web.MediaProxy.in_banned_urls(image_url) end) =~ "Running cache purge: [\"#{image_url}\"]" end end @@ -50,13 +50,13 @@ test "run script to clear cache" do Config.put([Invalidation.Script], script_path: "purge-nginx") image_url = "http://example.com/media/example.jpg" - Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url) + Pleroma.Web.MediaProxy.put_in_banned_urls(image_url) with_mocks [{System, [], [cmd: fn _, _ -> {"ok", 0} end]}] do assert capture_log(fn -> - assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + assert Pleroma.Web.MediaProxy.in_banned_urls(image_url) assert Invalidation.purge([image_url]) == {:ok, [image_url]} - assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + assert Pleroma.Web.MediaProxy.in_banned_urls(image_url) end) =~ "Running cache purge: [\"#{image_url}\"]" end end diff --git a/test/web/media_proxy/media_proxy_controller_test.exs b/test/web/media_proxy/media_proxy_controller_test.exs index 72da98a6a..d61cef83b 100644 --- a/test/web/media_proxy/media_proxy_controller_test.exs +++ b/test/web/media_proxy/media_proxy_controller_test.exs @@ -11,7 +11,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do setup do: clear_config([Pleroma.Web.Endpoint, :secret_key_base]) setup do - on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + on_exit(fn -> Cachex.clear(:banned_urls_cache) end) end test "it returns 404 when MediaProxy disabled", %{conn: conn} do @@ -71,11 +71,11 @@ test "it performs ReverseProxy.call when signature valid", %{conn: conn} do end end - test "it returns 404 when url contains in deleted_urls cache", %{conn: conn} do + test "it returns 404 when url contains in banned_urls cache", %{conn: conn} do Config.put([:media_proxy, :enabled], true) Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000") url = Pleroma.Web.MediaProxy.encode_url("https://google.fn/test.png") - Pleroma.Web.MediaProxy.put_in_deleted_urls("https://google.fn/test.png") + Pleroma.Web.MediaProxy.put_in_banned_urls("https://google.fn/test.png") with_mock Pleroma.ReverseProxy, call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do From 4044f24e2e4935757e038e7f06373ed1c9172560 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 18 Jun 2020 05:02:33 +0300 Subject: [PATCH 31/56] fix test --- lib/pleroma/web/media_proxy/invalidation.ex | 3 ++- test/web/media_proxy/invalidations/http_test.exs | 2 +- test/web/media_proxy/invalidations/script_test.exs | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index 6da7eb720..5808861e6 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -26,7 +26,8 @@ def purge(urls) do defp do_purge(urls) do provider = Config.get([:media_proxy, :invalidation, :provider]) - provider.purge(urls, Config.get(provider)) + options = Config.get(provider) + provider.purge(urls, options) end def prepare_urls(urls) do diff --git a/test/web/media_proxy/invalidations/http_test.exs b/test/web/media_proxy/invalidations/http_test.exs index 9d181dd8b..a1bef5237 100644 --- a/test/web/media_proxy/invalidations/http_test.exs +++ b/test/web/media_proxy/invalidations/http_test.exs @@ -6,7 +6,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do import Tesla.Mock setup do - on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + on_exit(fn -> Cachex.clear(:banned_urls_cache) end) end test "logs hasn't error message when request is valid" do diff --git a/test/web/media_proxy/invalidations/script_test.exs b/test/web/media_proxy/invalidations/script_test.exs index 8e155b705..51833ab18 100644 --- a/test/web/media_proxy/invalidations/script_test.exs +++ b/test/web/media_proxy/invalidations/script_test.exs @@ -5,7 +5,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.ScriptTest do import ExUnit.CaptureLog setup do - on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + on_exit(fn -> Cachex.clear(:banned_urls_cache) end) end test "it logger error when script not found" do From 3becdafd335f95d9320d287ecf9a55ea1b1765cd Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 18 Jun 2020 14:32:21 +0300 Subject: [PATCH 32/56] emoji packs pagination --- lib/pleroma/emoji/pack.ex | 19 ++++++++++++---- .../pleroma_emoji_pack_operation.ex | 14 ++++++++++++ .../controllers/emoji_pack_controller.ex | 4 ++-- .../emoji_pack_controller_test.exs | 22 +++++++++++++++++++ 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 14a5185be..5660c4c9d 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Emoji.Pack do alias Pleroma.Emoji - @spec create(String.t()) :: :ok | {:error, File.posix()} | {:error, :empty_values} + @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values} def create(name) do with :ok <- validate_not_empty([name]), dir <- Path.join(emoji_path(), name), @@ -120,8 +120,8 @@ def list_remote(url) do end end - @spec list_local() :: {:ok, map()} - def list_local do + @spec list_local(keyword()) :: {:ok, map()} + def list_local(opts) do with {:ok, results} <- list_packs_dir() do packs = results @@ -132,6 +132,17 @@ def list_local do end end) |> Enum.reject(&is_nil/1) + + packs = + case opts[:page] do + 1 -> + Enum.take(packs, opts[:page_size]) + + _ -> + packs + |> Enum.take(opts[:page] * opts[:page_size]) + |> Enum.take(-opts[:page_size]) + end |> Map.new(fn pack -> {pack.name, validate_pack(pack)} end) {:ok, packs} @@ -146,7 +157,7 @@ def get_archive(name) do end end - @spec download(String.t(), String.t(), String.t()) :: :ok | {:error, atom()} + @spec download(String.t(), String.t(), String.t()) :: {:ok, t()} | {:error, atom()} def download(name, url, as) do uri = url |> String.trim() |> URI.parse() diff --git a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex index 567688ff5..0d842382b 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex @@ -33,6 +33,20 @@ def index_operation do tags: ["Emoji Packs"], summary: "Lists local custom emoji packs", operationId: "PleromaAPI.EmojiPackController.index", + parameters: [ + Operation.parameter( + :page, + :query, + %Schema{type: :integer, default: 1}, + "Page" + ), + Operation.parameter( + :page_size, + :query, + %Schema{type: :integer, default: 50}, + "Number of statuses to return" + ) + ], responses: %{ 200 => emoji_packs_response() } diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex index d1efdeb5d..5654b3fbe 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex @@ -37,13 +37,13 @@ def remote(conn, %{url: url}) do end end - def index(conn, _params) do + def index(conn, params) do emoji_path = [:instance, :static_dir] |> Pleroma.Config.get!() |> Path.join("emoji") - with {:ok, packs} <- Pack.list_local() do + with {:ok, packs} <- Pack.list_local(page: params.page, page_size: params.page_size) do json(conn, packs) else {:error, :create_dir, e} -> diff --git a/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs b/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs index ee3d281a0..aafca6359 100644 --- a/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs +++ b/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs @@ -39,6 +39,28 @@ test "GET /api/pleroma/emoji/packs", %{conn: conn} do non_shared = resp["test_pack_nonshared"] assert non_shared["pack"]["share-files"] == false assert non_shared["pack"]["can-download"] == false + + resp = + conn + |> get("/api/pleroma/emoji/packs?page_size=1") + |> json_response_and_validate_schema(200) + + [pack1] = Map.keys(resp) + + resp = + conn + |> get("/api/pleroma/emoji/packs?page_size=1&page=2") + |> json_response_and_validate_schema(200) + + [pack2] = Map.keys(resp) + + resp = + conn + |> get("/api/pleroma/emoji/packs?page_size=1&page=3") + |> json_response_and_validate_schema(200) + + [pack3] = Map.keys(resp) + assert [pack1, pack2, pack3] |> Enum.uniq() |> length() == 3 end describe "GET /api/pleroma/emoji/packs/remote" do From 4975ed86bcca330373a68c9e6c6798a6b2167b14 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 18 Jun 2020 18:50:03 +0300 Subject: [PATCH 33/56] emoji pagination for pack show action --- docs/API/pleroma_api.md | 12 ++- lib/pleroma/emoji/pack.ex | 37 +++++---- .../pleroma_emoji_pack_operation.ex | 16 +++- .../controllers/emoji_pack_controller.ex | 4 +- .../emoji/test_pack/blank2.png | Bin 0 -> 95 bytes .../instance_static/emoji/test_pack/pack.json | 3 +- .../emoji/test_pack_nonshared/nonshared.zip | Bin 256 -> 548 bytes .../emoji/test_pack_nonshared/pack.json | 2 +- .../emoji_pack_controller_test.exs | 72 +++++++++++++----- 9 files changed, 104 insertions(+), 42 deletions(-) create mode 100644 test/instance_static/emoji/test_pack/blank2.png diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index 70d4755b7..d8d3ba85f 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -450,17 +450,25 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa * Response: JSON, list with updated files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message. ## `GET /api/pleroma/emoji/packs` + ### Lists local custom emoji packs + * Method `GET` * Authentication: not required -* Params: None +* Params: + * `page`: page number for packs (default 1) + * `page_size`: page size for packs (default 50) * Response: JSON, "ok" and 200 status and the JSON hashmap of pack name to pack contents ## `GET /api/pleroma/emoji/packs/:name` + ### Get pack.json for the pack + * Method `GET` * Authentication: not required -* Params: None +* Params: + * `page`: page number for files (default 1) + * `page_size`: page size for files (default 50) * Response: JSON, pack json with `files` and `pack` keys with 200 status or 404 if the pack does not exist ## `GET /api/pleroma/emoji/packs/:name/archive` diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 5660c4c9d..c033572c1 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -26,10 +26,27 @@ def create(name) do end end - @spec show(String.t()) :: {:ok, t()} | {:error, atom()} - def show(name) do + defp paginate(entities, 1, page_size), do: Enum.take(entities, page_size) + + defp paginate(entities, page, page_size) do + entities + |> Enum.take(page * page_size) + |> Enum.take(-page_size) + end + + @spec show(keyword()) :: {:ok, t()} | {:error, atom()} + def show(opts) do + name = opts[:name] + with :ok <- validate_not_empty([name]), {:ok, pack} <- load_pack(name) do + shortcodes = + pack.files + |> Map.keys() + |> paginate(opts[:page], opts[:page_size]) + + pack = Map.put(pack, :files, Map.take(pack.files, shortcodes)) + {:ok, validate_pack(pack)} end end @@ -132,17 +149,7 @@ def list_local(opts) do end end) |> Enum.reject(&is_nil/1) - - packs = - case opts[:page] do - 1 -> - Enum.take(packs, opts[:page_size]) - - _ -> - packs - |> Enum.take(opts[:page] * opts[:page_size]) - |> Enum.take(-opts[:page_size]) - end + |> paginate(opts[:page], opts[:page_size]) |> Map.new(fn pack -> {pack.name, validate_pack(pack)} end) {:ok, packs} @@ -307,7 +314,9 @@ defp downloadable?(pack) do # Otherwise, they'd have to download it from external-src pack.pack["share-files"] && Enum.all?(pack.files, fn {_, file} -> - File.exists?(Path.join(pack.path, file)) + pack.path + |> Path.join(file) + |> File.exists?() end) end diff --git a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex index 0d842382b..e8abe654d 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex @@ -58,7 +58,21 @@ def show_operation do tags: ["Emoji Packs"], summary: "Show emoji pack", operationId: "PleromaAPI.EmojiPackController.show", - parameters: [name_param()], + parameters: [ + name_param(), + Operation.parameter( + :page, + :query, + %Schema{type: :integer, default: 1}, + "Page" + ), + Operation.parameter( + :page_size, + :query, + %Schema{type: :integer, default: 50}, + "Number of statuses to return" + ) + ], responses: %{ 200 => Operation.response("Emoji Pack", "application/json", emoji_pack()), 400 => Operation.response("Bad Request", "application/json", ApiError), diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex index 5654b3fbe..078fb88dd 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex @@ -60,10 +60,10 @@ def index(conn, params) do end end - def show(conn, %{name: name}) do + def show(conn, %{name: name, page: page, page_size: page_size}) do name = String.trim(name) - with {:ok, pack} <- Pack.show(name) do + with {:ok, pack} <- Pack.show(name: name, page: page, page_size: page_size) do json(conn, pack) else {:error, :not_found} -> diff --git a/test/instance_static/emoji/test_pack/blank2.png b/test/instance_static/emoji/test_pack/blank2.png new file mode 100644 index 0000000000000000000000000000000000000000..8f50fa02340e7e09e562f86e00b6e4bd6ad1d565 GIT binary patch literal 95 zcmeAS@N?(olHy`uVBq!ia0vp^4Is=2Bp6=1#-sr$rjj7PU&+=QbNLmuU`r{nJ)1voZ(QBh}(T^38Ut+NecTD*xELo zOxJC&;q(_jK7s4l6V_uwYr2J9s%A0q?zqF3WnfTXVqj2rerA=xlFOH`o==?{>?F(( z;LXkvAKoY$0ki|;r~sVK<$^ia2*th8K(|H>qqNMi7hW;12M{ gZ7e7tU|>n(dYG|91xtW8D;r2J6A*%Q#xsI=0P0(p8vpv$50|Stl=o%x-$Rx*%%Mgjlb&OIvtPtHOIvE%Oyjj_RHZd>)p+AtG I4dO5W0E3SliU0rr diff --git a/test/instance_static/emoji/test_pack_nonshared/pack.json b/test/instance_static/emoji/test_pack_nonshared/pack.json index 93d643a5f..09f6274d1 100644 --- a/test/instance_static/emoji/test_pack_nonshared/pack.json +++ b/test/instance_static/emoji/test_pack_nonshared/pack.json @@ -4,7 +4,7 @@ "homepage": "https://pleroma.social", "description": "Test description", "fallback-src": "https://nonshared-pack", - "fallback-src-sha256": "74409E2674DAA06C072729C6C8426C4CB3B7E0B85ED77792DB7A436E11D76DAF", + "fallback-src-sha256": "1967BB4E42BCC34BCC12D57BE7811D3B7BE52F965BCE45C87BD377B9499CE11D", "share-files": false }, "files": { diff --git a/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs b/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs index aafca6359..f6239cae5 100644 --- a/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs +++ b/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs @@ -31,7 +31,7 @@ test "GET /api/pleroma/emoji/packs", %{conn: conn} do resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200) shared = resp["test_pack"] - assert shared["files"] == %{"blank" => "blank.png"} + assert shared["files"] == %{"blank" => "blank.png", "blank2" => "blank2.png"} assert Map.has_key?(shared["pack"], "download-sha256") assert shared["pack"]["can-download"] assert shared["pack"]["share-files"] @@ -354,7 +354,7 @@ test "for a pack with a fallback source", ctx do Map.put( new_data, "fallback-src-sha256", - "74409E2674DAA06C072729C6C8426C4CB3B7E0B85ED77792DB7A436E11D76DAF" + "1967BB4E42BCC34BCC12D57BE7811D3B7BE52F965BCE45C87BD377B9499CE11D" ) assert ctx[:admin_conn] @@ -420,7 +420,7 @@ test "don't rewrite old emoji", %{admin_conn: admin_conn} do assert admin_conn |> put_req_header("content-type", "multipart/form-data") |> post("/api/pleroma/emoji/packs/test_pack/files", %{ - shortcode: "blank2", + shortcode: "blank3", filename: "dir/blank.png", file: %Plug.Upload{ filename: "blank.png", @@ -429,7 +429,8 @@ test "don't rewrite old emoji", %{admin_conn: admin_conn} do }) |> json_response_and_validate_schema(200) == %{ "blank" => "blank.png", - "blank2" => "dir/blank.png" + "blank2" => "blank2.png", + "blank3" => "dir/blank.png" } assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png") @@ -453,7 +454,7 @@ test "rewrite old emoji with force option", %{admin_conn: admin_conn} do assert admin_conn |> put_req_header("content-type", "multipart/form-data") |> post("/api/pleroma/emoji/packs/test_pack/files", %{ - shortcode: "blank2", + shortcode: "blank3", filename: "dir/blank.png", file: %Plug.Upload{ filename: "blank.png", @@ -462,7 +463,8 @@ test "rewrite old emoji with force option", %{admin_conn: admin_conn} do }) |> json_response_and_validate_schema(200) == %{ "blank" => "blank.png", - "blank2" => "dir/blank.png" + "blank2" => "blank2.png", + "blank3" => "dir/blank.png" } assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png") @@ -470,14 +472,15 @@ test "rewrite old emoji with force option", %{admin_conn: admin_conn} do assert admin_conn |> put_req_header("content-type", "multipart/form-data") |> patch("/api/pleroma/emoji/packs/test_pack/files", %{ - shortcode: "blank2", - new_shortcode: "blank3", + shortcode: "blank3", + new_shortcode: "blank4", new_filename: "dir_2/blank_3.png", force: true }) |> json_response_and_validate_schema(200) == %{ "blank" => "blank.png", - "blank3" => "dir_2/blank_3.png" + "blank2" => "blank2.png", + "blank4" => "dir_2/blank_3.png" } assert File.exists?("#{@emoji_path}/test_pack/dir_2/blank_3.png") @@ -503,7 +506,7 @@ test "add file with not loaded pack", %{admin_conn: admin_conn} do assert admin_conn |> put_req_header("content-type", "multipart/form-data") |> post("/api/pleroma/emoji/packs/not_loaded/files", %{ - shortcode: "blank2", + shortcode: "blank3", filename: "dir/blank.png", file: %Plug.Upload{ filename: "blank.png", @@ -557,7 +560,8 @@ test "new with shortcode as file with update", %{admin_conn: admin_conn} do }) |> json_response_and_validate_schema(200) == %{ "blank" => "blank.png", - "blank4" => "dir/blank.png" + "blank4" => "dir/blank.png", + "blank2" => "blank2.png" } assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png") @@ -571,7 +575,8 @@ test "new with shortcode as file with update", %{admin_conn: admin_conn} do }) |> json_response_and_validate_schema(200) == %{ "blank3" => "dir_2/blank_3.png", - "blank" => "blank.png" + "blank" => "blank.png", + "blank2" => "blank2.png" } refute File.exists?("#{@emoji_path}/test_pack/dir/") @@ -579,7 +584,10 @@ test "new with shortcode as file with update", %{admin_conn: admin_conn} do assert admin_conn |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank3") - |> json_response_and_validate_schema(200) == %{"blank" => "blank.png"} + |> json_response_and_validate_schema(200) == %{ + "blank" => "blank.png", + "blank2" => "blank2.png" + } refute File.exists?("#{@emoji_path}/test_pack/dir_2/") @@ -603,7 +611,8 @@ test "new with shortcode from url", %{admin_conn: admin_conn} do }) |> json_response_and_validate_schema(200) == %{ "blank_url" => "blank_url.png", - "blank" => "blank.png" + "blank" => "blank.png", + "blank2" => "blank2.png" } assert File.exists?("#{@emoji_path}/test_pack/blank_url.png") @@ -624,15 +633,16 @@ test "new without shortcode", %{admin_conn: admin_conn} do }) |> json_response_and_validate_schema(200) == %{ "shortcode" => "shortcode.png", - "blank" => "blank.png" + "blank" => "blank.png", + "blank2" => "blank2.png" } end test "remove non existing shortcode in pack.json", %{admin_conn: admin_conn} do assert admin_conn - |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank2") + |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank3") |> json_response_and_validate_schema(:bad_request) == %{ - "error" => "Emoji \"blank2\" does not exist" + "error" => "Emoji \"blank3\" does not exist" } end @@ -640,12 +650,12 @@ test "update non existing emoji", %{admin_conn: admin_conn} do assert admin_conn |> put_req_header("content-type", "multipart/form-data") |> patch("/api/pleroma/emoji/packs/test_pack/files", %{ - shortcode: "blank2", - new_shortcode: "blank3", + shortcode: "blank3", + new_shortcode: "blank4", new_filename: "dir_2/blank_3.png" }) |> json_response_and_validate_schema(:bad_request) == %{ - "error" => "Emoji \"blank2\" does not exist" + "error" => "Emoji \"blank3\" does not exist" } end @@ -768,7 +778,7 @@ test "filesystem import", %{admin_conn: admin_conn, conn: conn} do describe "GET /api/pleroma/emoji/packs/:name" do test "shows pack.json", %{conn: conn} do assert %{ - "files" => %{"blank" => "blank.png"}, + "files" => files, "pack" => %{ "can-download" => true, "description" => "Test description", @@ -781,6 +791,26 @@ test "shows pack.json", %{conn: conn} do conn |> get("/api/pleroma/emoji/packs/test_pack") |> json_response_and_validate_schema(200) + + assert files == %{"blank" => "blank.png", "blank2" => "blank2.png"} + + assert %{ + "files" => files + } = + conn + |> get("/api/pleroma/emoji/packs/test_pack?page_size=1") + |> json_response_and_validate_schema(200) + + assert files |> Map.keys() |> length() == 1 + + assert %{ + "files" => files + } = + conn + |> get("/api/pleroma/emoji/packs/test_pack?page_size=1&page=2") + |> json_response_and_validate_schema(200) + + assert files |> Map.keys() |> length() == 1 end test "non existing pack", %{conn: conn} do From 3e3f9253e6db17b691c7393ad7a5f89df84348ea Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 19 Jun 2020 10:17:24 +0300 Subject: [PATCH 34/56] adding overall count for packs and files --- docs/API/pleroma_api.md | 22 ++++++++++++-- lib/pleroma/emoji/pack.ex | 20 +++++++++---- .../controllers/emoji_pack_controller.ex | 4 +-- .../emoji_pack_controller_test.exs | 30 ++++++++++++------- 4 files changed, 56 insertions(+), 20 deletions(-) diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index d8d3ba85f..e5bc29eb2 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -458,7 +458,17 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa * Params: * `page`: page number for packs (default 1) * `page_size`: page size for packs (default 50) -* Response: JSON, "ok" and 200 status and the JSON hashmap of pack name to pack contents +* Response: `packs` key with JSON hashmap of pack name to pack contents and `count` key for count of packs. + +```json +{ + "packs": { + "pack_name": {...}, // pack contents + ... + }, + "count": 0 // packs count +} +``` ## `GET /api/pleroma/emoji/packs/:name` @@ -469,7 +479,15 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa * Params: * `page`: page number for files (default 1) * `page_size`: page size for files (default 50) -* Response: JSON, pack json with `files` and `pack` keys with 200 status or 404 if the pack does not exist +* Response: JSON, pack json with `files`, `files_count` and `pack` keys with 200 status or 404 if the pack does not exist. + +```json +{ + "files": {...}, + "files_count": 0, // emoji count in pack + "pack": {...} +} +``` ## `GET /api/pleroma/emoji/packs/:name/archive` ### Requests a local pack archive from the instance diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index c033572c1..2dca21c93 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -1,6 +1,7 @@ defmodule Pleroma.Emoji.Pack do - @derive {Jason.Encoder, only: [:files, :pack]} + @derive {Jason.Encoder, only: [:files, :pack, :files_count]} defstruct files: %{}, + files_count: 0, pack_file: nil, path: nil, pack: %{}, @@ -8,6 +9,7 @@ defmodule Pleroma.Emoji.Pack do @type t() :: %__MODULE__{ files: %{String.t() => Path.t()}, + files_count: non_neg_integer(), pack_file: Path.t(), path: Path.t(), pack: map(), @@ -137,10 +139,10 @@ def list_remote(url) do end end - @spec list_local(keyword()) :: {:ok, map()} + @spec list_local(keyword()) :: {:ok, map(), non_neg_integer()} def list_local(opts) do with {:ok, results} <- list_packs_dir() do - packs = + all_packs = results |> Enum.map(fn name -> case load_pack(name) do @@ -149,10 +151,13 @@ def list_local(opts) do end end) |> Enum.reject(&is_nil/1) + + packs = + all_packs |> paginate(opts[:page], opts[:page_size]) |> Map.new(fn pack -> {pack.name, validate_pack(pack)} end) - {:ok, packs} + {:ok, packs, length(all_packs)} end end @@ -215,7 +220,12 @@ def load_pack(name) do |> Map.put(:path, Path.dirname(pack_file)) |> Map.put(:name, name) - {:ok, pack} + files_count = + pack.files + |> Map.keys() + |> length() + + {:ok, Map.put(pack, :files_count, files_count)} else {:error, :not_found} end diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex index 078fb88dd..33ecd1f70 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex @@ -43,8 +43,8 @@ def index(conn, params) do |> Pleroma.Config.get!() |> Path.join("emoji") - with {:ok, packs} <- Pack.list_local(page: params.page, page_size: params.page_size) do - json(conn, packs) + with {:ok, packs, count} <- Pack.list_local(page: params.page, page_size: params.page_size) do + json(conn, %{packs: packs, count: count}) else {:error, :create_dir, e} -> conn diff --git a/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs b/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs index f6239cae5..91312c832 100644 --- a/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs +++ b/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs @@ -30,13 +30,14 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerTest do test "GET /api/pleroma/emoji/packs", %{conn: conn} do resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200) - shared = resp["test_pack"] + assert resp["count"] == 3 + shared = resp["packs"]["test_pack"] assert shared["files"] == %{"blank" => "blank.png", "blank2" => "blank2.png"} assert Map.has_key?(shared["pack"], "download-sha256") assert shared["pack"]["can-download"] assert shared["pack"]["share-files"] - non_shared = resp["test_pack_nonshared"] + non_shared = resp["packs"]["test_pack_nonshared"] assert non_shared["pack"]["share-files"] == false assert non_shared["pack"]["can-download"] == false @@ -45,21 +46,24 @@ test "GET /api/pleroma/emoji/packs", %{conn: conn} do |> get("/api/pleroma/emoji/packs?page_size=1") |> json_response_and_validate_schema(200) - [pack1] = Map.keys(resp) + assert resp["count"] == 3 + [pack1] = Map.keys(resp["packs"]) resp = conn |> get("/api/pleroma/emoji/packs?page_size=1&page=2") |> json_response_and_validate_schema(200) - [pack2] = Map.keys(resp) + assert resp["count"] == 3 + [pack2] = Map.keys(resp["packs"]) resp = conn |> get("/api/pleroma/emoji/packs?page_size=1&page=3") |> json_response_and_validate_schema(200) - [pack3] = Map.keys(resp) + assert resp["count"] == 3 + [pack3] = Map.keys(resp["packs"]) assert [pack1, pack2, pack3] |> Enum.uniq() |> length() == 3 end @@ -683,7 +687,8 @@ test "creating and deleting a pack", %{admin_conn: admin_conn} do assert Jason.decode!(File.read!("#{@emoji_path}/test_created/pack.json")) == %{ "pack" => %{}, - "files" => %{} + "files" => %{}, + "files_count" => 0 } assert admin_conn @@ -741,14 +746,14 @@ test "filesystem import", %{admin_conn: admin_conn, conn: conn} do resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200) - refute Map.has_key?(resp, "test_pack_for_import") + refute Map.has_key?(resp["packs"], "test_pack_for_import") assert admin_conn |> get("/api/pleroma/emoji/packs/import") |> json_response_and_validate_schema(200) == ["test_pack_for_import"] resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200) - assert resp["test_pack_for_import"]["files"] == %{"blank" => "blank.png"} + assert resp["packs"]["test_pack_for_import"]["files"] == %{"blank" => "blank.png"} File.rm!("#{@emoji_path}/test_pack_for_import/pack.json") refute File.exists?("#{@emoji_path}/test_pack_for_import/pack.json") @@ -768,7 +773,7 @@ test "filesystem import", %{admin_conn: admin_conn, conn: conn} do resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200) - assert resp["test_pack_for_import"]["files"] == %{ + assert resp["packs"]["test_pack_for_import"]["files"] == %{ "blank" => "blank.png", "blank2" => "blank.png", "foo" => "blank.png" @@ -779,6 +784,7 @@ test "filesystem import", %{admin_conn: admin_conn, conn: conn} do test "shows pack.json", %{conn: conn} do assert %{ "files" => files, + "files_count" => 2, "pack" => %{ "can-download" => true, "description" => "Test description", @@ -795,7 +801,8 @@ test "shows pack.json", %{conn: conn} do assert files == %{"blank" => "blank.png", "blank2" => "blank2.png"} assert %{ - "files" => files + "files" => files, + "files_count" => 2 } = conn |> get("/api/pleroma/emoji/packs/test_pack?page_size=1") @@ -804,7 +811,8 @@ test "shows pack.json", %{conn: conn} do assert files |> Map.keys() |> length() == 1 assert %{ - "files" => files + "files" => files, + "files_count" => 2 } = conn |> get("/api/pleroma/emoji/packs/test_pack?page_size=1&page=2") From 0c739b423aad4cc6baa3a59308200ca5a5060716 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 19 Jun 2020 12:31:55 +0300 Subject: [PATCH 35/56] changelog entry --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ee13904f..ee657de13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking:** removed `with_move` parameter from notifications timeline. ### Added + - Chats: Added support for federated chats. For details, see the docs. - ActivityPub: Added support for existing AP ids for instances migrated from Mastodon. - Instance: Add `background_image` to configuration and `/api/v1/instance` @@ -34,6 +35,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Notifications: Added `follow_request` notification type. - Added `:reject_deletes` group to SimplePolicy - MRF (`EmojiStealPolicy`): New MRF Policy which allows to automatically download emojis from remote instances +- Support pagination in emoji packs API (for packs and for files in pack) +
API Changes - Mastodon API: Extended `/api/v1/instance`. From 02ca8a363f738ece7b605940690f6a538f6c2fa8 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 19 Jun 2020 14:46:38 +0300 Subject: [PATCH 36/56] default page size for files --- docs/API/pleroma_api.md | 2 +- .../web/api_spec/operations/pleroma_emoji_pack_operation.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index e5bc29eb2..b7eee5192 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -478,7 +478,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa * Authentication: not required * Params: * `page`: page number for files (default 1) - * `page_size`: page size for files (default 50) + * `page_size`: page size for files (default 30) * Response: JSON, pack json with `files`, `files_count` and `pack` keys with 200 status or 404 if the pack does not exist. ```json diff --git a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex index e8abe654d..da7cc5154 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex @@ -69,7 +69,7 @@ def show_operation do Operation.parameter( :page_size, :query, - %Schema{type: :integer, default: 50}, + %Schema{type: :integer, default: 30}, "Number of statuses to return" ) ], From 5237a2df9f123f661de30a53193b7d9fec69ecae Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 19 Jun 2020 16:14:06 +0300 Subject: [PATCH 37/56] [#1873] Fixes missing :offset pagination param support. Added pagination support for hashtags search. --- lib/pleroma/pagination.ex | 6 ++++ lib/pleroma/web/api_spec/helpers.ex | 6 ++++ .../controllers/search_controller.ex | 31 ++++++++++++------- .../controllers/search_controller_test.exs | 16 ++++++++++ .../controllers/status_controller_test.exs | 2 +- 5 files changed, 48 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/pagination.ex b/lib/pleroma/pagination.ex index 1b99e44f9..9a3795769 100644 --- a/lib/pleroma/pagination.ex +++ b/lib/pleroma/pagination.ex @@ -64,6 +64,12 @@ def fetch_paginated(query, params, :offset, table_binding) do @spec paginate(Ecto.Query.t(), map(), type(), atom() | nil) :: [Ecto.Schema.t()] def paginate(query, options, method \\ :keyset, table_binding \\ nil) + def paginate(list, options, _method, _table_binding) when is_list(list) do + offset = options[:offset] || 0 + limit = options[:limit] || 0 + Enum.slice(list, offset, limit) + end + def paginate(query, options, :keyset, table_binding) do query |> restrict(:min_id, options, table_binding) diff --git a/lib/pleroma/web/api_spec/helpers.ex b/lib/pleroma/web/api_spec/helpers.ex index a9cfe0fed..a258e8421 100644 --- a/lib/pleroma/web/api_spec/helpers.ex +++ b/lib/pleroma/web/api_spec/helpers.ex @@ -39,6 +39,12 @@ def pagination_params do :string, "Return the newest items newer than this ID" ), + Operation.parameter( + :offset, + :query, + %Schema{type: :integer, default: 0}, + "Return items past this number of items" + ), Operation.parameter( :limit, :query, diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 3be0ca095..e50980122 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -107,21 +107,21 @@ defp resource_search(_, "statuses", query, options) do ) end - defp resource_search(:v2, "hashtags", query, _options) do + defp resource_search(:v2, "hashtags", query, options) do tags_path = Web.base_url() <> "/tag/" query - |> prepare_tags() + |> prepare_tags(options) |> Enum.map(fn tag -> %{name: tag, url: tags_path <> tag} end) end - defp resource_search(:v1, "hashtags", query, _options) do - prepare_tags(query) + defp resource_search(:v1, "hashtags", query, options) do + prepare_tags(query, options) end - defp prepare_tags(query, add_joined_tag \\ true) do + defp prepare_tags(query, options) do tags = query |> preprocess_uri_query() @@ -139,13 +139,20 @@ defp prepare_tags(query, add_joined_tag \\ true) do tags = Enum.map(tags, fn tag -> String.trim_leading(tag, "#") end) - if Enum.empty?(explicit_tags) && add_joined_tag do - tags - |> Kernel.++([joined_tag(tags)]) - |> Enum.uniq_by(&String.downcase/1) - else - tags - end + tags = + if Enum.empty?(explicit_tags) && !options[:skip_joined_tag] do + add_joined_tag(tags) + else + tags + end + + Pleroma.Pagination.paginate(tags, options) + end + + defp add_joined_tag(tags) do + tags + |> Kernel.++([joined_tag(tags)]) + |> Enum.uniq_by(&String.downcase/1) end # If `query` is a URI, returns last component of its path, otherwise returns `query` diff --git a/test/web/mastodon_api/controllers/search_controller_test.exs b/test/web/mastodon_api/controllers/search_controller_test.exs index c605957b1..826f37fbc 100644 --- a/test/web/mastodon_api/controllers/search_controller_test.exs +++ b/test/web/mastodon_api/controllers/search_controller_test.exs @@ -151,6 +151,22 @@ test "constructs hashtags from search query", %{conn: conn} do ] end + test "supports pagination of hashtags search results", %{conn: conn} do + results = + conn + |> get( + "/api/v2/search?#{ + URI.encode_query(%{q: "#some #text #with #hashtags", limit: 2, offset: 1}) + }" + ) + |> json_response_and_validate_schema(200) + + assert results["hashtags"] == [ + %{"name" => "text", "url" => "#{Web.base_url()}/tag/text"}, + %{"name" => "with", "url" => "#{Web.base_url()}/tag/with"} + ] + end + test "excludes a blocked users from search results", %{conn: conn} do user = insert(:user) user_smith = insert(:user, %{nickname: "Agent", name: "I love 2hu"}) diff --git a/test/web/mastodon_api/controllers/status_controller_test.exs b/test/web/mastodon_api/controllers/status_controller_test.exs index 648e6f2ce..a98e939e8 100644 --- a/test/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/web/mastodon_api/controllers/status_controller_test.exs @@ -1561,7 +1561,7 @@ test "favorites paginate correctly" do # Using the header for pagination works correctly [next, _] = get_resp_header(result, "link") |> hd() |> String.split(", ") - [_, max_id] = Regex.run(~r/max_id=(.*)>;/, next) + [_, max_id] = Regex.run(~r/max_id=([^&]+)/, next) assert max_id == third_favorite.id From d54b0432eae74a830a0294cf48f23933a16382aa Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 19 Jun 2020 15:49:34 +0200 Subject: [PATCH 38/56] README: Add some troubleshooting info for compilation issues. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 7fc1fd381..6ca3118fb 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,16 @@ Currently Pleroma is not packaged by any OS/Distros, but if you want to package ### Docker While we don’t provide docker files, other people have written very good ones. Take a look at or . +### Compilation Troubleshooting +If you ever encounter compilation issues during the updating of Pleroma, you can try these commands and see if they fix things: + +- `mix deps.clean --all` +- `mix local.rebar` +- `mix local.hex` +- `rm -r _build` + +If you are not developing Pleroma, it is better to use the OTP release, which comes with everything precompiled. + ## Documentation - Latest Released revision: - Latest Git revision: From b63646169dbed68814bdb867c4b8b3c88a3d2360 Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Fri, 19 Jun 2020 21:18:07 +0200 Subject: [PATCH 39/56] Add support for bot field in update_credentials --- CHANGELOG.md | 1 + lib/pleroma/user.ex | 1 + .../controllers/account_controller.ex | 3 + .../update_credentials_test.exs | 67 +++++++++++++++++++ 4 files changed, 72 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ee13904f..8a8798e8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Support for `include_types` in `/api/v1/notifications`. - Mastodon API: Added `/api/v1/notifications/:id/dismiss` endpoint. - Mastodon API: Add support for filtering replies in public and home timelines +- Mastodon API: Support for `bot` field in `/api/v1/accounts/update_credentials` - Admin API: endpoints for create/update/delete OAuth Apps. - Admin API: endpoint for status view. - OTP: Add command to reload emoji packs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f0ccc7c79..ae4f96aac 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -465,6 +465,7 @@ def update_changeset(struct, params \\ %{}) do |> validate_format(:nickname, local_nickname_regex()) |> validate_length(:bio, max: bio_limit) |> validate_length(:name, min: 1, max: name_limit) + |> validate_inclusion(:actor_type, ["Person", "Service"]) |> put_fields() |> put_emoji() |> put_change_if_present(:bio, &{:ok, parse_bio(&1, struct)}) diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index c38c2b895..adbbac624 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -177,6 +177,9 @@ def update_credentials(%{assigns: %{user: user}, body_params: params} = conn, _p |> Maps.put_if_present(:pleroma_settings_store, params[:pleroma_settings_store]) |> Maps.put_if_present(:default_scope, params[:default_scope]) |> Maps.put_if_present(:default_scope, params["source"]["privacy"]) + |> Maps.put_if_present(:actor_type, params[:bot], fn bot -> + if bot, do: {:ok, "Service"}, else: {:ok, "Person"} + end) |> Maps.put_if_present(:actor_type, params[:actor_type]) changeset = User.update_changeset(user, user_params) diff --git a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs index 76e6d603a..f67d294ba 100644 --- a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs @@ -400,4 +400,71 @@ test "update fields when invalid request", %{conn: conn} do |> json_response_and_validate_schema(403) end end + + describe "Mark account as bot" do + setup do: oauth_access(["write:accounts"]) + setup :request_content_type + + test "changing actor_type to Service makes account a bot", %{conn: conn} do + account = + conn + |> patch("/api/v1/accounts/update_credentials", %{actor_type: "Service"}) + |> json_response_and_validate_schema(200) + + assert account["bot"] + assert account["source"]["pleroma"]["actor_type"] == "Service" + end + + test "changing actor_type to Person makes account a human", %{conn: conn} do + account = + conn + |> patch("/api/v1/accounts/update_credentials", %{actor_type: "Person"}) + |> json_response_and_validate_schema(200) + + refute account["bot"] + assert account["source"]["pleroma"]["actor_type"] == "Person" + end + + test "changing actor_type to Application causes error", %{conn: conn} do + response = + conn + |> patch("/api/v1/accounts/update_credentials", %{actor_type: "Application"}) + |> json_response_and_validate_schema(403) + + assert %{"error" => "Invalid request"} == response + end + + test "changing bot field to true changes actor_type to Service", %{conn: conn} do + account = + conn + |> patch("/api/v1/accounts/update_credentials", %{bot: "true"}) + |> json_response_and_validate_schema(200) + + assert account["bot"] + assert account["source"]["pleroma"]["actor_type"] == "Service" + end + + test "changing bot field to false changes actor_type to Person", %{conn: conn} do + account = + conn + |> patch("/api/v1/accounts/update_credentials", %{bot: "false"}) + |> json_response_and_validate_schema(200) + + refute account["bot"] + assert account["source"]["pleroma"]["actor_type"] == "Person" + end + + test "actor_type field has a higher priority than bot", %{conn: conn} do + account = + conn + |> patch("/api/v1/accounts/update_credentials", %{ + actor_type: "Person", + bot: "true" + }) + |> json_response_and_validate_schema(200) + + refute account["bot"] + assert account["source"]["pleroma"]["actor_type"] == "Person" + end + end end From ac0344dd24d520ab61e835b9caea97529f4c1dad Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Fri, 19 Jun 2020 21:19:00 +0200 Subject: [PATCH 40/56] Only accounts with Service actor_type are considered as bots --- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 68beb69b8..6c40b8ccd 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -179,7 +179,7 @@ defp do_render("show.json", %{user: user} = opts) do 0 end - bot = user.actor_type in ["Application", "Service"] + bot = user.actor_type == "Service" emojis = Enum.map(user.emoji, fn {shortcode, raw_url} -> From 3d4cfc9c5f3969e08c32781385c86f310eba70a2 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 18 Jun 2020 19:32:03 +0200 Subject: [PATCH 41/56] Stop filling conversation field on incoming objects (legacy, unused) conversation field is still set for outgoing federation for compatibility. --- .../web/activity_pub/object_validators/note_validator.ex | 1 - lib/pleroma/web/activity_pub/transmogrifier.ex | 6 +++--- test/web/activity_pub/transmogrifier_test.exs | 3 --- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex index a10728ac6..56b93dde8 100644 --- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex @@ -41,7 +41,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do field(:announcements, {:array, :string}, default: []) # see if needed - field(:conversation, :string) field(:context_id, :string) end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 851f474b8..1c60ef8f5 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -172,8 +172,8 @@ def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options) object |> Map.put("inReplyTo", replied_object.data["id"]) |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id) - |> Map.put("conversation", replied_object.data["context"] || object["conversation"]) |> Map.put("context", replied_object.data["context"] || object["conversation"]) + |> Map.drop(["conversation"]) else e -> Logger.error("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") @@ -207,7 +207,7 @@ def fix_context(object) do object |> Map.put("context", context) - |> Map.put("conversation", context) + |> Map.drop(["conversation"]) end def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachment) do @@ -458,7 +458,7 @@ def handle_incoming( to: data["to"], object: object, actor: user, - context: object["conversation"], + context: object["context"], local: false, published: data["published"], additional: diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 94d8552e8..47d6e843a 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -1571,9 +1571,6 @@ test "returns modified object when allowed incoming reply", %{data: data} do assert modified_object["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873" - assert modified_object["conversation"] == - "tag:shitposter.club,2017-05-05:objectType=thread:nonce=3c16e9c2681f6d26" - assert modified_object["context"] == "tag:shitposter.club,2017-05-05:objectType=thread:nonce=3c16e9c2681f6d26" end From 1a704e1f1e0acb73cbfb49acc4f614dd01799c46 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 20 Jun 2020 10:56:28 +0300 Subject: [PATCH 42/56] fix for packs pagination --- lib/pleroma/emoji/pack.ex | 6 +++--- .../emoji_pack_controller_test.exs | 20 ++++++++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 2dca21c93..787ff8141 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -32,8 +32,8 @@ defp paginate(entities, 1, page_size), do: Enum.take(entities, page_size) defp paginate(entities, page, page_size) do entities - |> Enum.take(page * page_size) - |> Enum.take(-page_size) + |> Enum.chunk_every(page_size) + |> Enum.at(page - 1) end @spec show(keyword()) :: {:ok, t()} | {:error, atom()} @@ -470,7 +470,7 @@ defp list_packs_dir do # with the API so it should be sufficient with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_path)}, {:ls, {:ok, results}} <- {:ls, File.ls(emoji_path)} do - {:ok, results} + {:ok, Enum.sort(results)} else {:create_dir, {:error, e}} -> {:error, :create_dir, e} {:ls, {:error, e}} -> {:error, :ls, e} diff --git a/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs b/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs index 91312c832..df58a5eb6 100644 --- a/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs +++ b/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs @@ -31,6 +31,11 @@ test "GET /api/pleroma/emoji/packs", %{conn: conn} do resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200) assert resp["count"] == 3 + + assert resp["packs"] + |> Map.keys() + |> length() == 3 + shared = resp["packs"]["test_pack"] assert shared["files"] == %{"blank" => "blank.png", "blank2" => "blank2.png"} assert Map.has_key?(shared["pack"], "download-sha256") @@ -47,7 +52,12 @@ test "GET /api/pleroma/emoji/packs", %{conn: conn} do |> json_response_and_validate_schema(200) assert resp["count"] == 3 - [pack1] = Map.keys(resp["packs"]) + + packs = Map.keys(resp["packs"]) + + assert length(packs) == 1 + + [pack1] = packs resp = conn @@ -55,7 +65,9 @@ test "GET /api/pleroma/emoji/packs", %{conn: conn} do |> json_response_and_validate_schema(200) assert resp["count"] == 3 - [pack2] = Map.keys(resp["packs"]) + packs = Map.keys(resp["packs"]) + assert length(packs) == 1 + [pack2] = packs resp = conn @@ -63,7 +75,9 @@ test "GET /api/pleroma/emoji/packs", %{conn: conn} do |> json_response_and_validate_schema(200) assert resp["count"] == 3 - [pack3] = Map.keys(resp["packs"]) + packs = Map.keys(resp["packs"]) + assert length(packs) == 1 + [pack3] = packs assert [pack1, pack2, pack3] |> Enum.uniq() |> length() == 3 end From 15ba5392584a2d4e8129a99e825f5025e57e6ebd Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 20 Jun 2020 11:39:06 +0200 Subject: [PATCH 43/56] =?UTF-8?q?cheatsheet.md:=20no=5Fattachment=5Flinks?= =?UTF-8?q?=20=E2=86=92=20attachment=5Flinks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/configuration/cheatsheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 6ebdab546..7e5f1cd29 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -60,7 +60,7 @@ To add configuration to your config file, you can copy it from the base config. older software for theses nicknames. * `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature. * `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow. -* `no_attachment_links`: Set to true to disable automatically adding attachment link text to statuses. +* `attachment_links`: Set to true to enable automatically adding attachment link text to statuses. * `welcome_message`: A message that will be send to a newly registered users as a direct message. * `welcome_user_nickname`: The nickname of the local user that sends the welcome message. * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`). From 0e789bc55fed24fd913d6bf1a5c6be135320b0c9 Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 20 Jun 2020 09:39:50 +0000 Subject: [PATCH 44/56] Apply suggestion to lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex --- .../web/api_spec/operations/pleroma_emoji_pack_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex index da7cc5154..caa849721 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex @@ -44,7 +44,7 @@ def index_operation do :page_size, :query, %Schema{type: :integer, default: 50}, - "Number of statuses to return" + "Number of emoji packs to return" ) ], responses: %{ From c5863438ba9079a01a832fe48e203907fe5b37cd Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 20 Jun 2020 13:53:57 +0300 Subject: [PATCH 45/56] proper error codes for error in adminFE --- docs/API/admin_api.md | 54 ++++++++++--------- .../controllers/admin_api_controller.ex | 29 +++++----- .../controllers/fallback_controller.ex | 6 +++ .../controllers/admin_api_controller_test.exs | 4 +- 4 files changed, 50 insertions(+), 43 deletions(-) diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index c7f56cf5f..b6fb43dcb 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -488,35 +488,39 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret ### Change the user's email, password, display and settings-related fields -- Params: - - `email` - - `password` - - `name` - - `bio` - - `avatar` - - `locked` - - `no_rich_text` - - `default_scope` - - `banner` - - `hide_follows` - - `hide_followers` - - `hide_followers_count` - - `hide_follows_count` - - `hide_favorites` - - `allow_following_move` - - `background` - - `show_role` - - `skip_thread_containment` - - `fields` - - `discoverable` - - `actor_type` +* Params: + * `email` + * `password` + * `name` + * `bio` + * `avatar` + * `locked` + * `no_rich_text` + * `default_scope` + * `banner` + * `hide_follows` + * `hide_followers` + * `hide_followers_count` + * `hide_follows_count` + * `hide_favorites` + * `allow_following_move` + * `background` + * `show_role` + * `skip_thread_containment` + * `fields` + * `discoverable` + * `actor_type` -- Response: +* Responses: + +Status: 200 ```json {"status": "success"} ``` +Status: 400 + ```json {"errors": {"actor_type": "is invalid"}, @@ -525,8 +529,10 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret } ``` +Status: 404 + ```json -{"error": "Unable to update user."} +{"error": "Not found"} ``` ## `GET /api/pleroma/admin/reports` diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index 5cbf0dd4f..db2413dfe 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -111,8 +111,7 @@ def user_delete(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) action: "delete" }) - conn - |> json(nicknames) + json(conn, nicknames) end def user_follow(%{assigns: %{user: admin}} = conn, %{ @@ -131,8 +130,7 @@ def user_follow(%{assigns: %{user: admin}} = conn, %{ }) end - conn - |> json("ok") + json(conn, "ok") end def user_unfollow(%{assigns: %{user: admin}} = conn, %{ @@ -151,8 +149,7 @@ def user_unfollow(%{assigns: %{user: admin}} = conn, %{ }) end - conn - |> json("ok") + json(conn, "ok") end def users_create(%{assigns: %{user: admin}} = conn, %{"users" => users}) do @@ -191,8 +188,7 @@ def users_create(%{assigns: %{user: admin}} = conn, %{"users" => users}) do action: "create" }) - conn - |> json(res) + json(conn, res) {:error, id, changeset, _} -> res = @@ -363,8 +359,8 @@ defp maybe_parse_filters(filters) do filters |> String.split(",") |> Enum.filter(&Enum.member?(@filters, &1)) - |> Enum.map(&String.to_atom(&1)) - |> Enum.into(%{}, &{&1, true}) + |> Enum.map(&String.to_atom/1) + |> Map.new(&{&1, true}) end def right_add_multiple(%{assigns: %{user: admin}} = conn, %{ @@ -568,10 +564,10 @@ def update_user_credentials( {:error, changeset} -> errors = Map.new(changeset.errors, fn {key, {error, _}} -> {key, error} end) - json(conn, %{errors: errors}) + {:errors, errors} _ -> - json(conn, %{error: "Unable to update user."}) + {:error, :not_found} end end @@ -616,7 +612,7 @@ defp configurable_from_database do def reload_emoji(conn, _params) do Pleroma.Emoji.reload() - conn |> json("ok") + json(conn, "ok") end def confirm_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do @@ -630,7 +626,7 @@ def confirm_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames} action: "confirm_email" }) - conn |> json("") + json(conn, "") end def resend_confirmation_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do @@ -644,14 +640,13 @@ def resend_confirmation_email(%{assigns: %{user: admin}} = conn, %{"nicknames" = action: "resend_confirmation_email" }) - conn |> json("") + json(conn, "") end def stats(conn, _) do count = Stats.get_status_visibility_count() - conn - |> json(%{"status_visibility" => count}) + json(conn, %{"status_visibility" => count}) end defp page_params(params) do diff --git a/lib/pleroma/web/admin_api/controllers/fallback_controller.ex b/lib/pleroma/web/admin_api/controllers/fallback_controller.ex index 82965936d..34d90db07 100644 --- a/lib/pleroma/web/admin_api/controllers/fallback_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/fallback_controller.ex @@ -17,6 +17,12 @@ def call(conn, {:error, reason}) do |> json(%{error: reason}) end + def call(conn, {:errors, errors}) do + conn + |> put_status(:bad_request) + |> json(%{errors: errors}) + end + def call(conn, {:param_cast, _}) do conn |> put_status(:bad_request) diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index e3d3ccb8d..3a3eb822d 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -1599,14 +1599,14 @@ test "changes actor type from permitted list", %{conn: conn, user: user} do assert patch(conn, "/api/pleroma/admin/users/#{user.nickname}/credentials", %{ "actor_type" => "Application" }) - |> json_response(200) == %{"errors" => %{"actor_type" => "is invalid"}} + |> json_response(400) == %{"errors" => %{"actor_type" => "is invalid"}} end test "update non existing user", %{conn: conn} do assert patch(conn, "/api/pleroma/admin/users/non-existing/credentials", %{ "password" => "new_password" }) - |> json_response(200) == %{"error" => "Unable to update user."} + |> json_response(404) == %{"error" => "Not found"} end end From b5f13af7ba66924f6aed448bd519f6becc269922 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 20 Jun 2020 10:59:08 +0000 Subject: [PATCH 46/56] Apply suggestion to lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex --- .../web/api_spec/operations/pleroma_emoji_pack_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex index caa849721..b2b4f8713 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex @@ -70,7 +70,7 @@ def show_operation do :page_size, :query, %Schema{type: :integer, default: 30}, - "Number of statuses to return" + "Number of emoji to return" ) ], responses: %{ From 35e9282ffdafd8a04d1c09ec5eff3f176bb389de Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 22 Jun 2020 10:35:11 +0200 Subject: [PATCH 47/56] HellthreadPolicy: Restrict to Notes and Articles. --- .../web/activity_pub/mrf/hellthread_policy.ex | 7 +++++-- .../mrf/hellthread_policy_test.exs | 18 +++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 1764bc789..f6b2c4415 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -13,8 +13,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do defp delist_message(message, threshold) when threshold > 0 do follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address + to = message["to"] || [] + cc = message["cc"] || [] - follower_collection? = Enum.member?(message["to"] ++ message["cc"], follower_collection) + follower_collection? = Enum.member?(to ++ cc, follower_collection) message = case get_recipient_count(message) do @@ -71,7 +73,8 @@ defp get_recipient_count(message) do end @impl true - def filter(%{"type" => "Create"} = message) do + def filter(%{"type" => "Create", "object" => %{"type" => object_type}} = message) + when object_type in ~w{Note Article} do reject_threshold = Pleroma.Config.get( [:mrf_hellthread, :reject_threshold], diff --git a/test/web/activity_pub/mrf/hellthread_policy_test.exs b/test/web/activity_pub/mrf/hellthread_policy_test.exs index 95ef0b168..6e9daa7f9 100644 --- a/test/web/activity_pub/mrf/hellthread_policy_test.exs +++ b/test/web/activity_pub/mrf/hellthread_policy_test.exs @@ -8,6 +8,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicyTest do import Pleroma.Web.ActivityPub.MRF.HellthreadPolicy + alias Pleroma.Web.CommonAPI + setup do user = insert(:user) @@ -20,7 +22,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicyTest do "https://instance.tld/users/user1", "https://instance.tld/users/user2", "https://instance.tld/users/user3" - ] + ], + "object" => %{ + "type" => "Note" + } } [user: user, message: message] @@ -28,6 +33,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicyTest do setup do: clear_config(:mrf_hellthread) + test "doesn't die on chat messages" do + Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 2, reject_threshold: 0}) + + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = CommonAPI.post_chat_message(user, other_user, "moin") + + assert {:ok, _} = filter(activity.data) + end + describe "reject" do test "rejects the message if the recipient count is above reject_threshold", %{ message: message From 9f7ee5dfa283f8db9a5fcb006630674263425ac8 Mon Sep 17 00:00:00 2001 From: Ilja Date: Mon, 22 Jun 2020 11:41:22 +0200 Subject: [PATCH 48/56] Add include for the "Further reading" section * I added an include and use this include for the installation guides that already had this section * I added the "Further reading" section as well as te "Questions" section to the English guides that didn't have it yet * I added a first point "How Federation Works/Why is my Federated Timeline empty?" to link to lains blogpost about this because we still get this question a lot in the #pleroma support channel * I reordered the list a bit --- docs/installation/alpine_linux_en.md | 5 +---- docs/installation/arch_linux_en.md | 5 +---- docs/installation/debian_based_en.md | 5 +---- docs/installation/debian_based_jp.md | 5 +---- docs/installation/further_reading.include | 5 +++++ docs/installation/gentoo_en.md | 5 +---- docs/installation/netbsd_en.md | 8 ++++++++ docs/installation/openbsd_en.md | 8 ++++++++ docs/installation/otp_en.md | 5 +---- 9 files changed, 27 insertions(+), 24 deletions(-) create mode 100644 docs/installation/further_reading.include diff --git a/docs/installation/alpine_linux_en.md b/docs/installation/alpine_linux_en.md index 2a9b8f6ff..c726d559f 100644 --- a/docs/installation/alpine_linux_en.md +++ b/docs/installation/alpine_linux_en.md @@ -225,10 +225,7 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new + +#### Further reading + +{! backend/installation/further_reading.include !} + +## Questions + +Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**. diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index e8c5d844c..5dbe24f75 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -242,3 +242,11 @@ If your instance is up and running, you can create your first user with administ ``` LC_ALL=en_US.UTF-8 MIX_ENV=prod mix pleroma.user new --admin ``` + +#### Further reading + +{! backend/installation/further_reading.include !} + +## Questions + +Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**. diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index 86135cd20..e4f822d1c 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -270,10 +270,7 @@ This will create an account withe the username of 'joeuser' with the email addre ## Further reading -* [Backup your instance](../administration/backup.md) -* [Hardening your instance](../configuration/hardening.md) -* [How to activate mediaproxy](../configuration/howto_mediaproxy.md) -* [Updating your instance](../administration/updating.md) +{! backend/installation/further_reading.include !} ## Questions From 4f5af68b3e96c5b5b62185f86af39fc2f8955e10 Mon Sep 17 00:00:00 2001 From: Ben Is Date: Fri, 19 Jun 2020 14:33:58 +0000 Subject: [PATCH 49/56] Added translation using Weblate (Italian) --- priv/gettext/it/LC_MESSAGES/errors.po | 578 ++++++++++++++++++++++++++ 1 file changed, 578 insertions(+) create mode 100644 priv/gettext/it/LC_MESSAGES/errors.po diff --git a/priv/gettext/it/LC_MESSAGES/errors.po b/priv/gettext/it/LC_MESSAGES/errors.po new file mode 100644 index 000000000..18ec03c83 --- /dev/null +++ b/priv/gettext/it/LC_MESSAGES/errors.po @@ -0,0 +1,578 @@ +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2020-06-19 14:33+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: Automatically generated\n" +"Language-Team: none\n" +"Language: it\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"X-Generator: Translate Toolkit 2.5.1\n" + +## This file is a PO Template file. +## +## `msgid`s here are often extracted from source code. +## Add new translations manually only if they're dynamic +## translations that can't be statically extracted. +## +## Run `mix gettext.extract` to bring this file up to +## date. Leave `msgstr`s empty as changing them here as no +## effect: edit them in PO (`.po`) files instead. +## From Ecto.Changeset.cast/4 +msgid "can't be blank" +msgstr "" + +## From Ecto.Changeset.unique_constraint/3 +msgid "has already been taken" +msgstr "" + +## From Ecto.Changeset.put_change/3 +msgid "is invalid" +msgstr "" + +## From Ecto.Changeset.validate_format/3 +msgid "has invalid format" +msgstr "" + +## From Ecto.Changeset.validate_subset/3 +msgid "has an invalid entry" +msgstr "" + +## From Ecto.Changeset.validate_exclusion/3 +msgid "is reserved" +msgstr "" + +## From Ecto.Changeset.validate_confirmation/3 +msgid "does not match confirmation" +msgstr "" + +## From Ecto.Changeset.no_assoc_constraint/3 +msgid "is still associated with this entry" +msgstr "" + +msgid "are still associated with this entry" +msgstr "" + +## From Ecto.Changeset.validate_length/3 +msgid "should be %{count} character(s)" +msgid_plural "should be %{count} character(s)" +msgstr[0] "" +msgstr[1] "" + +msgid "should have %{count} item(s)" +msgid_plural "should have %{count} item(s)" +msgstr[0] "" +msgstr[1] "" + +msgid "should be at least %{count} character(s)" +msgid_plural "should be at least %{count} character(s)" +msgstr[0] "" +msgstr[1] "" + +msgid "should have at least %{count} item(s)" +msgid_plural "should have at least %{count} item(s)" +msgstr[0] "" +msgstr[1] "" + +msgid "should be at most %{count} character(s)" +msgid_plural "should be at most %{count} character(s)" +msgstr[0] "" +msgstr[1] "" + +msgid "should have at most %{count} item(s)" +msgid_plural "should have at most %{count} item(s)" +msgstr[0] "" +msgstr[1] "" + +## From Ecto.Changeset.validate_number/3 +msgid "must be less than %{number}" +msgstr "" + +msgid "must be greater than %{number}" +msgstr "" + +msgid "must be less than or equal to %{number}" +msgstr "" + +msgid "must be greater than or equal to %{number}" +msgstr "" + +msgid "must be equal to %{number}" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:421 +#, elixir-format +msgid "Account not found" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:249 +#, elixir-format +msgid "Already voted" +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:360 +#, elixir-format +msgid "Bad request" +msgstr "" + +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:425 +#, elixir-format +msgid "Can't delete object" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/status_controller.ex:196 +#, elixir-format +msgid "Can't delete this post" +msgstr "" + +#: lib/pleroma/web/controller_helper.ex:95 +#: lib/pleroma/web/controller_helper.ex:101 +#, elixir-format +msgid "Can't display this activity" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:227 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:254 +#, elixir-format +msgid "Can't find user" +msgstr "" + +#: lib/pleroma/web/pleroma_api/controllers/account_controller.ex:114 +#, elixir-format +msgid "Can't get favorites" +msgstr "" + +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:437 +#, elixir-format +msgid "Can't like object" +msgstr "" + +#: lib/pleroma/web/common_api/utils.ex:556 +#, elixir-format +msgid "Cannot post an empty status without attachments" +msgstr "" + +#: lib/pleroma/web/common_api/utils.ex:504 +#, elixir-format +msgid "Comment must be up to %{max_size} characters" +msgstr "" + +#: lib/pleroma/config/config_db.ex:222 +#, elixir-format +msgid "Config with params %{params} not found" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:95 +#, elixir-format +msgid "Could not delete" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:141 +#, elixir-format +msgid "Could not favorite" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:370 +#, elixir-format +msgid "Could not pin" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:112 +#, elixir-format +msgid "Could not repeat" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:188 +#, elixir-format +msgid "Could not unfavorite" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:380 +#, elixir-format +msgid "Could not unpin" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:126 +#, elixir-format +msgid "Could not unrepeat" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:428 +#: lib/pleroma/web/common_api/common_api.ex:437 +#, elixir-format +msgid "Could not update state" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:202 +#, elixir-format +msgid "Error." +msgstr "" + +#: lib/pleroma/web/twitter_api/twitter_api.ex:106 +#, elixir-format +msgid "Invalid CAPTCHA" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:117 +#: lib/pleroma/web/oauth/oauth_controller.ex:569 +#, elixir-format +msgid "Invalid credentials" +msgstr "" + +#: lib/pleroma/plugs/ensure_authenticated_plug.ex:38 +#, elixir-format +msgid "Invalid credentials." +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:265 +#, elixir-format +msgid "Invalid indices" +msgstr "" + +#: lib/pleroma/web/admin_api/admin_api_controller.ex:1147 +#, elixir-format +msgid "Invalid parameters" +msgstr "" + +#: lib/pleroma/web/common_api/utils.ex:411 +#, elixir-format +msgid "Invalid password." +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:187 +#, elixir-format +msgid "Invalid request" +msgstr "" + +#: lib/pleroma/web/twitter_api/twitter_api.ex:109 +#, elixir-format +msgid "Kocaptcha service unavailable" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:113 +#, elixir-format +msgid "Missing parameters" +msgstr "" + +#: lib/pleroma/web/common_api/utils.ex:540 +#, elixir-format +msgid "No such conversation" +msgstr "" + +#: lib/pleroma/web/admin_api/admin_api_controller.ex:439 +#: lib/pleroma/web/admin_api/admin_api_controller.ex:465 lib/pleroma/web/admin_api/admin_api_controller.ex:507 +#, elixir-format +msgid "No such permission_group" +msgstr "" + +#: lib/pleroma/plugs/uploaded_media.ex:74 +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:485 lib/pleroma/web/admin_api/admin_api_controller.ex:1135 +#: lib/pleroma/web/feed/user_controller.ex:73 lib/pleroma/web/ostatus/ostatus_controller.ex:143 +#, elixir-format +msgid "Not found" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:241 +#, elixir-format +msgid "Poll's author can't vote" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:20 +#: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:37 lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:49 +#: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:50 lib/pleroma/web/mastodon_api/controllers/status_controller.ex:290 +#: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:71 +#, elixir-format +msgid "Record not found" +msgstr "" + +#: lib/pleroma/web/admin_api/admin_api_controller.ex:1153 +#: lib/pleroma/web/feed/user_controller.ex:79 lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:32 +#: lib/pleroma/web/ostatus/ostatus_controller.ex:149 +#, elixir-format +msgid "Something went wrong" +msgstr "" + +#: lib/pleroma/web/common_api/activity_draft.ex:107 +#, elixir-format +msgid "The message visibility must be direct" +msgstr "" + +#: lib/pleroma/web/common_api/utils.ex:566 +#, elixir-format +msgid "The status is over the character limit" +msgstr "" + +#: lib/pleroma/plugs/ensure_public_or_authenticated_plug.ex:31 +#, elixir-format +msgid "This resource requires authentication." +msgstr "" + +#: lib/pleroma/plugs/rate_limiter/rate_limiter.ex:206 +#, elixir-format +msgid "Throttled" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:266 +#, elixir-format +msgid "Too many choices" +msgstr "" + +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:442 +#, elixir-format +msgid "Unhandled activity type" +msgstr "" + +#: lib/pleroma/web/admin_api/admin_api_controller.ex:536 +#, elixir-format +msgid "You can't revoke your own admin status." +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:218 +#: lib/pleroma/web/oauth/oauth_controller.ex:309 +#, elixir-format +msgid "Your account is currently disabled" +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:180 +#: lib/pleroma/web/oauth/oauth_controller.ex:332 +#, elixir-format +msgid "Your login is missing a confirmed e-mail address" +msgstr "" + +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:389 +#, elixir-format +msgid "can't read inbox of %{nickname} as %{as_nickname}" +msgstr "" + +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:472 +#, elixir-format +msgid "can't update outbox of %{nickname} as %{as_nickname}" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:388 +#, elixir-format +msgid "conversation is already muted" +msgstr "" + +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:316 +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:491 +#, elixir-format +msgid "error" +msgstr "" + +#: lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex:29 +#, elixir-format +msgid "mascots can only be images" +msgstr "" + +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:60 +#, elixir-format +msgid "not found" +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:395 +#, elixir-format +msgid "Bad OAuth request." +msgstr "" + +#: lib/pleroma/web/twitter_api/twitter_api.ex:115 +#, elixir-format +msgid "CAPTCHA already used" +msgstr "" + +#: lib/pleroma/web/twitter_api/twitter_api.ex:112 +#, elixir-format +msgid "CAPTCHA expired" +msgstr "" + +#: lib/pleroma/plugs/uploaded_media.ex:55 +#, elixir-format +msgid "Failed" +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:411 +#, elixir-format +msgid "Failed to authenticate: %{message}." +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:442 +#, elixir-format +msgid "Failed to set up user account." +msgstr "" + +#: lib/pleroma/plugs/oauth_scopes_plug.ex:38 +#, elixir-format +msgid "Insufficient permissions: %{permissions}." +msgstr "" + +#: lib/pleroma/plugs/uploaded_media.ex:94 +#, elixir-format +msgid "Internal Error" +msgstr "" + +#: lib/pleroma/web/oauth/fallback_controller.ex:22 +#: lib/pleroma/web/oauth/fallback_controller.ex:29 +#, elixir-format +msgid "Invalid Username/Password" +msgstr "" + +#: lib/pleroma/web/twitter_api/twitter_api.ex:118 +#, elixir-format +msgid "Invalid answer data" +msgstr "" + +#: lib/pleroma/web/nodeinfo/nodeinfo_controller.ex:128 +#, elixir-format +msgid "Nodeinfo schema version not handled" +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:169 +#, elixir-format +msgid "This action is outside the authorized scopes" +msgstr "" + +#: lib/pleroma/web/oauth/fallback_controller.ex:14 +#, elixir-format +msgid "Unknown error, please check the details and try again." +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:116 +#: lib/pleroma/web/oauth/oauth_controller.ex:155 +#, elixir-format +msgid "Unlisted redirect_uri." +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:391 +#, elixir-format +msgid "Unsupported OAuth provider: %{provider}." +msgstr "" + +#: lib/pleroma/uploaders/uploader.ex:72 +#, elixir-format +msgid "Uploader callback timeout" +msgstr "" + +#: lib/pleroma/web/uploader_controller.ex:23 +#, elixir-format +msgid "bad request" +msgstr "" + +#: lib/pleroma/web/twitter_api/twitter_api.ex:103 +#, elixir-format +msgid "CAPTCHA Error" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:200 +#, elixir-format +msgid "Could not add reaction emoji" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:211 +#, elixir-format +msgid "Could not remove reaction emoji" +msgstr "" + +#: lib/pleroma/web/twitter_api/twitter_api.ex:129 +#, elixir-format +msgid "Invalid CAPTCHA (Missing parameter: %{name})" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/list_controller.ex:92 +#, elixir-format +msgid "List not found" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:124 +#, elixir-format +msgid "Missing parameter: %{name}" +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:207 +#: lib/pleroma/web/oauth/oauth_controller.ex:322 +#, elixir-format +msgid "Password reset is required" +msgstr "" + +#: lib/pleroma/tests/auth_test_controller.ex:9 +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:6 lib/pleroma/web/admin_api/admin_api_controller.ex:6 +#: lib/pleroma/web/controller_helper.ex:6 lib/pleroma/web/fallback_redirect_controller.ex:6 +#: lib/pleroma/web/feed/tag_controller.ex:6 lib/pleroma/web/feed/user_controller.ex:6 +#: lib/pleroma/web/mailer/subscription_controller.ex:2 lib/pleroma/web/masto_fe_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/app_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/auth_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/conversation_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/custom_emoji_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/filter_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/follow_request_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/instance_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/list_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/marker_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex:14 lib/pleroma/web/mastodon_api/controllers/media_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/notification_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/report_controller.ex:8 lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/search_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/status_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:7 lib/pleroma/web/mastodon_api/controllers/suggestion_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:6 lib/pleroma/web/media_proxy/media_proxy_controller.ex:6 +#: lib/pleroma/web/mongooseim/mongoose_im_controller.ex:6 lib/pleroma/web/nodeinfo/nodeinfo_controller.ex:6 +#: lib/pleroma/web/oauth/fallback_controller.ex:6 lib/pleroma/web/oauth/mfa_controller.ex:10 +#: lib/pleroma/web/oauth/oauth_controller.ex:6 lib/pleroma/web/ostatus/ostatus_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/account_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:2 +#: lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/scrobble_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/two_factor_authentication_controller.ex:7 lib/pleroma/web/static_fe/static_fe_controller.ex:6 +#: lib/pleroma/web/twitter_api/controllers/password_controller.ex:10 lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex:6 +#: lib/pleroma/web/twitter_api/controllers/util_controller.ex:6 lib/pleroma/web/twitter_api/twitter_api_controller.ex:6 +#: lib/pleroma/web/uploader_controller.ex:6 lib/pleroma/web/web_finger/web_finger_controller.ex:6 +#, elixir-format +msgid "Security violation: OAuth scopes check was neither handled nor explicitly skipped." +msgstr "" + +#: lib/pleroma/plugs/ensure_authenticated_plug.ex:28 +#, elixir-format +msgid "Two-factor authentication enabled, you must use a access token." +msgstr "" + +#: lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:210 +#, elixir-format +msgid "Unexpected error occurred while adding file to pack." +msgstr "" + +#: lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:138 +#, elixir-format +msgid "Unexpected error occurred while creating pack." +msgstr "" + +#: lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:278 +#, elixir-format +msgid "Unexpected error occurred while removing file from pack." +msgstr "" + +#: lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:250 +#, elixir-format +msgid "Unexpected error occurred while updating file in pack." +msgstr "" + +#: lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:179 +#, elixir-format +msgid "Unexpected error occurred while updating pack metadata." +msgstr "" + +#: lib/pleroma/plugs/user_is_admin_plug.ex:40 +#, elixir-format +msgid "User is not an admin or OAuth admin scope is not granted." +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:61 +#, elixir-format +msgid "Web push subscription is disabled on this Pleroma instance" +msgstr "" + +#: lib/pleroma/web/admin_api/admin_api_controller.ex:502 +#, elixir-format +msgid "You can't revoke your own admin/moderator status." +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:105 +#, elixir-format +msgid "authorization required for timeline view" +msgstr "" From 68c812eb2ecbfb1d582925c15d90bd1bd4e62b4b Mon Sep 17 00:00:00 2001 From: Ben Is Date: Fri, 19 Jun 2020 14:35:01 +0000 Subject: [PATCH 50/56] Translated using Weblate (Italian) Currently translated at 0.9% (1 of 106 strings) Translation: Pleroma/Pleroma backend Translate-URL: https://translate.pleroma.social/projects/pleroma/pleroma/it/ --- priv/gettext/it/LC_MESSAGES/errors.po | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/priv/gettext/it/LC_MESSAGES/errors.po b/priv/gettext/it/LC_MESSAGES/errors.po index 18ec03c83..726be628b 100644 --- a/priv/gettext/it/LC_MESSAGES/errors.po +++ b/priv/gettext/it/LC_MESSAGES/errors.po @@ -3,14 +3,16 @@ msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2020-06-19 14:33+0000\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: Automatically generated\n" -"Language-Team: none\n" +"PO-Revision-Date: 2020-06-19 20:38+0000\n" +"Last-Translator: Ben Is \n" +"Language-Team: Italian \n" "Language: it\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"X-Generator: Translate Toolkit 2.5.1\n" +"Plural-Forms: nplurals=2; plural=n != 1;\n" +"X-Generator: Weblate 4.0.4\n" ## This file is a PO Template file. ## @@ -23,7 +25,7 @@ msgstr "" ## effect: edit them in PO (`.po`) files instead. ## From Ecto.Changeset.cast/4 msgid "can't be blank" -msgstr "" +msgstr "non può essere nullo" ## From Ecto.Changeset.unique_constraint/3 msgid "has already been taken" From 7e6f43c0d7c625a03ee0216c2d9474253ef87b5a Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 22 Jun 2020 19:03:04 +0400 Subject: [PATCH 51/56] Add `is_muted` to notifications --- .../mastodon_api/views/notification_view.ex | 8 ++--- .../views/notification_view_test.exs | 36 +++++++++++++++---- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 3865be280..c97e6d32f 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -84,12 +84,7 @@ def render( # Note: :relationships contain user mutes (needed for :muted flag in :status) status_render_opts = %{relationships: opts[:relationships]} - - account = - AccountView.render( - "show.json", - %{user: actor, for: reading_user} - ) + account = AccountView.render("show.json", %{user: actor, for: reading_user}) response = %{ id: to_string(notification.id), @@ -97,6 +92,7 @@ def render( created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at), account: account, pleroma: %{ + is_muted: User.mutes?(reading_user, actor), is_seen: notification.seen } } diff --git a/test/web/mastodon_api/views/notification_view_test.exs b/test/web/mastodon_api/views/notification_view_test.exs index 9c399b2df..8e0e58538 100644 --- a/test/web/mastodon_api/views/notification_view_test.exs +++ b/test/web/mastodon_api/views/notification_view_test.exs @@ -49,7 +49,7 @@ test "ChatMessage notification" do expected = %{ id: to_string(notification.id), - pleroma: %{is_seen: false}, + pleroma: %{is_seen: false, is_muted: false}, type: "pleroma:chat_mention", account: AccountView.render("show.json", %{user: user, for: recipient}), chat_message: MessageReferenceView.render("show.json", %{chat_message_reference: cm_ref}), @@ -68,7 +68,7 @@ test "Mention notification" do expected = %{ id: to_string(notification.id), - pleroma: %{is_seen: false}, + pleroma: %{is_seen: false, is_muted: false}, type: "mention", account: AccountView.render("show.json", %{ @@ -92,7 +92,7 @@ test "Favourite notification" do expected = %{ id: to_string(notification.id), - pleroma: %{is_seen: false}, + pleroma: %{is_seen: false, is_muted: false}, type: "favourite", account: AccountView.render("show.json", %{user: another_user, for: user}), status: StatusView.render("show.json", %{activity: create_activity, for: user}), @@ -112,7 +112,7 @@ test "Reblog notification" do expected = %{ id: to_string(notification.id), - pleroma: %{is_seen: false}, + pleroma: %{is_seen: false, is_muted: false}, type: "reblog", account: AccountView.render("show.json", %{user: another_user, for: user}), status: StatusView.render("show.json", %{activity: reblog_activity, for: user}), @@ -130,7 +130,7 @@ test "Follow notification" do expected = %{ id: to_string(notification.id), - pleroma: %{is_seen: false}, + pleroma: %{is_seen: false, is_muted: false}, type: "follow", account: AccountView.render("show.json", %{user: follower, for: followed}), created_at: Utils.to_masto_date(notification.inserted_at) @@ -171,7 +171,7 @@ test "Move notification" do expected = %{ id: to_string(notification.id), - pleroma: %{is_seen: false}, + pleroma: %{is_seen: false, is_muted: false}, type: "move", account: AccountView.render("show.json", %{user: old_user, for: follower}), target: AccountView.render("show.json", %{user: new_user, for: follower}), @@ -196,7 +196,7 @@ test "EmojiReact notification" do expected = %{ id: to_string(notification.id), - pleroma: %{is_seen: false}, + pleroma: %{is_seen: false, is_muted: false}, type: "pleroma:emoji_reaction", emoji: "☕", account: AccountView.render("show.json", %{user: other_user, for: user}), @@ -206,4 +206,26 @@ test "EmojiReact notification" do test_notifications_rendering([notification], user, [expected]) end + + test "muted notification" do + user = insert(:user) + another_user = insert(:user) + + {:ok, _} = Pleroma.UserRelationship.create_mute(user, another_user) + {:ok, create_activity} = CommonAPI.post(user, %{status: "hey"}) + {:ok, favorite_activity} = CommonAPI.favorite(another_user, create_activity.id) + {:ok, [notification]} = Notification.create_notifications(favorite_activity) + create_activity = Activity.get_by_id(create_activity.id) + + expected = %{ + id: to_string(notification.id), + pleroma: %{is_seen: false, is_muted: true}, + type: "favourite", + account: AccountView.render("show.json", %{user: another_user, for: user}), + status: StatusView.render("show.json", %{activity: create_activity, for: user}), + created_at: Utils.to_masto_date(notification.inserted_at) + } + + test_notifications_rendering([notification], user, [expected]) + end end From b3a549e916c2a721da16f60e7665b6eb64d756dd Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 22 Jun 2020 19:18:33 +0400 Subject: [PATCH 52/56] Update NotificationOperation spec --- .../web/api_spec/operations/notification_operation.ex | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index 41328b5f2..f09be64cb 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -163,6 +163,13 @@ def notification do description: "Status that was the object of the notification, e.g. in mentions, reblogs, favourites, or polls.", nullable: true + }, + pleroma: %Schema{ + type: :object, + properties: %{ + is_seen: %Schema{type: :boolean}, + is_muted: %Schema{type: :boolean} + } } }, example: %{ @@ -170,7 +177,8 @@ def notification do "type" => "mention", "created_at" => "2019-11-23T07:49:02.064Z", "account" => Account.schema().example, - "status" => Status.schema().example + "status" => Status.schema().example, + "pleroma" => %{"is_seen" => false, "is_muted" => false} } } end From 8f6ba4b22f48dcd0256d6a9cf7259aa475895b84 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 22 Jun 2020 23:45:29 +0200 Subject: [PATCH 53/56] Add warning against parsing/reusing MastoFE settings blob --- lib/pleroma/web/masto_fe_controller.ex | 2 +- lib/pleroma/web/router.ex | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/masto_fe_controller.ex b/lib/pleroma/web/masto_fe_controller.ex index d0d8bc8eb..43ec70021 100644 --- a/lib/pleroma/web/masto_fe_controller.ex +++ b/lib/pleroma/web/masto_fe_controller.ex @@ -49,7 +49,7 @@ def manifest(conn, _params) do |> render("manifest.json") end - @doc "PUT /api/web/settings" + @doc "PUT /api/web/settings: Backend-obscure settings blob for MastoFE, don't parse/reuse elsewhere" def put_settings(%{assigns: %{user: user}} = conn, %{"data" => settings} = _params) do with {:ok, _} <- User.mastodon_settings_update(user, settings) do json(conn, %{}) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index eda74a171..419aa55e4 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -467,6 +467,7 @@ defmodule Pleroma.Web.Router do scope "/api/web", Pleroma.Web do pipe_through(:authenticated_api) + # Backend-obscure settings blob for MastoFE, don't parse/reuse elsewhere put("/settings", MastoFEController, :put_settings) end From bf8310f3802c46e6305fcb3832bca297582990d9 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jun 2020 17:35:02 -0500 Subject: [PATCH 54/56] Add missing default config value for :instance, instance_thumbnail Follows up on b7fc61e17b --- config/config.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/config/config.exs b/config/config.exs index 4bf31f3fc..e0888fa9a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -186,6 +186,7 @@ notify_email: "noreply@example.com", description: "Pleroma: An efficient and flexible fediverse server", background_image: "/images/city.jpg", + instance_thumbnail: "/instance/thumbnail.jpeg", limit: 5_000, chat_limit: 5_000, remote_limit: 100_000, From df5e048cbb7d349b34203ccba49a8f646e4d93a3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jun 2020 17:39:02 -0500 Subject: [PATCH 55/56] Do not need a function to provide fallback value with default defined in config.exs --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index c498fe632..c6b54e570 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -23,7 +23,7 @@ def render("show.json", _) do streaming_api: Pleroma.Web.Endpoint.websocket_url() }, stats: Pleroma.Stats.get_stats(), - thumbnail: instance_thumbnail(), + thumbnail: Keyword.get(instance, :instance_thumbnail), languages: ["en"], registrations: Keyword.get(instance, :registrations_open), # Extra (not present in Mastodon): @@ -88,9 +88,4 @@ def federation do end |> Map.put(:enabled, Config.get([:instance, :federating])) end - - defp instance_thumbnail do - Pleroma.Config.get([:instance, :instance_thumbnail]) || - "#{Pleroma.Web.base_url()}/instance/thumbnail.jpeg" - end end From c116b6d6d6e4b12d9d751481926183f19cdb5248 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 23 Jun 2020 04:42:44 +0200 Subject: [PATCH 56/56] ActivityPubController: Update upload_media @doc Small cherry-pick from https://git.pleroma.social/pleroma/pleroma/-/merge_requests/1810 --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index f0b5c6e93..220c4fe52 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -514,7 +514,6 @@ defp ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do {new_user, for_user} end - # TODO: Add support for "object" field @doc """ Endpoint based on @@ -525,6 +524,8 @@ defp ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do Response: - HTTP Code: 201 Created - HTTP Body: ActivityPub object to be inserted into another's `attachment` field + + Note: Will not point to a URL with a `Location` header because no standalone Activity has been created. """ def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do with {:ok, object} <-