From 875fbaae35fe73f01d7fd55c255043dda929e365 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 25 Jan 2021 15:34:59 +0300 Subject: [PATCH 1/2] support for expires_in/expires_at in filters --- CHANGELOG.md | 1 + benchmarks/load_testing/fetcher.ex | 5 +- config/config.exs | 1 + lib/pleroma/filter.ex | 126 ++++- .../api_spec/operations/filter_operation.ex | 33 +- .../controllers/filter_controller.ex | 61 +-- lib/pleroma/workers/purge_expired_filter.ex | 43 ++ test/pleroma/filter_test.exs | 181 ++++--- .../controllers/filter_controller_test.exs | 459 ++++++++++++++---- .../workers/purge_expired_filter_test.exs | 30 ++ test/support/factory.ex | 3 +- 11 files changed, 709 insertions(+), 234 deletions(-) create mode 100644 lib/pleroma/workers/purge_expired_filter.ex create mode 100644 test/pleroma/workers/purge_expired_filter_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ea242e11..226fa2bbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Current user is now included in conversation if it's the only participant. - Mastodon API: Fixed last_status.account being not filled with account data. - Mastodon API: Fixed own_votes being not returned with poll data. + - Mastodon API: Support for expires_in/expires_at in the Filters. ## Unreleased (Patch) diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index dfbd916be..607b7d4cb 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -33,10 +33,11 @@ defp fetch_user(user) do end defp create_filter(user) do - Pleroma.Filter.create(%Pleroma.Filter{ + Pleroma.Filter.create(%{ user_id: user.id, phrase: "must be filtered", - hide: true + hide: true, + context: ["home"] }) end diff --git a/config/config.exs b/config/config.exs index 5eca250bb..715524e84 100644 --- a/config/config.exs +++ b/config/config.exs @@ -543,6 +543,7 @@ queues: [ activity_expiration: 10, token_expiration: 5, + filter_expiration: 1, backup: 1, federator_incoming: 50, federator_outgoing: 50, diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index fc531f7fc..82b9caf9b 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -11,6 +11,9 @@ defmodule Pleroma.Filter do alias Pleroma.Repo alias Pleroma.User + @type t() :: %__MODULE__{} + @type format() :: :postgres | :re + schema "filters" do belongs_to(:user, User, type: FlakeId.Ecto.CompatType) field(:filter_id, :integer) @@ -18,15 +21,16 @@ defmodule Pleroma.Filter do field(:whole_word, :boolean, default: true) field(:phrase, :string) field(:context, {:array, :string}) - field(:expires_at, :utc_datetime) + field(:expires_at, :naive_datetime) timestamps() end + @spec get(integer() | String.t(), User.t()) :: t() | nil def get(id, %{id: user_id} = _user) do query = from( - f in Pleroma.Filter, + f in __MODULE__, where: f.filter_id == ^id, where: f.user_id == ^user_id ) @@ -34,14 +38,17 @@ def get(id, %{id: user_id} = _user) do Repo.one(query) end + @spec get_active(Ecto.Query.t() | module()) :: Ecto.Query.t() def get_active(query) do from(f in query, where: is_nil(f.expires_at) or f.expires_at > ^NaiveDateTime.utc_now()) end + @spec get_irreversible(Ecto.Query.t()) :: Ecto.Query.t() def get_irreversible(query) do from(f in query, where: f.hide) end + @spec get_filters(Ecto.Query.t() | module(), User.t()) :: [t()] def get_filters(query \\ __MODULE__, %User{id: user_id}) do query = from( @@ -53,7 +60,32 @@ def get_filters(query \\ __MODULE__, %User{id: user_id}) do Repo.all(query) end - def create(%Pleroma.Filter{user_id: user_id, filter_id: nil} = filter) do + @spec create(map()) :: {:ok, t()} | {:error, Ecto.Changeset.t()} + def create(attrs \\ %{}) do + Repo.transaction(fn -> create_with_expiration(attrs) end) + end + + defp create_with_expiration(attrs) do + with {:ok, filter} <- do_create(attrs), + {:ok, _} <- maybe_add_expiration_job(filter) do + filter + else + {:error, error} -> Repo.rollback(error) + end + end + + defp do_create(attrs) do + %__MODULE__{} + |> cast(attrs, [:phrase, :context, :hide, :expires_at, :whole_word, :user_id, :filter_id]) + |> maybe_add_filter_id() + |> validate_required([:phrase, :context, :user_id, :filter_id]) + |> maybe_add_expires_at(attrs) + |> Repo.insert() + end + + defp maybe_add_filter_id(%{changes: %{filter_id: _}} = changeset), do: changeset + + defp maybe_add_filter_id(%{changes: %{user_id: user_id}} = changeset) do # If filter_id wasn't given, use the max filter_id for this user plus 1. # XXX This could result in a race condition if a user tries to add two # different filters for their account from two different clients at the @@ -61,7 +93,7 @@ def create(%Pleroma.Filter{user_id: user_id, filter_id: nil} = filter) do max_id_query = from( - f in Pleroma.Filter, + f in __MODULE__, where: f.user_id == ^user_id, select: max(f.filter_id) ) @@ -76,34 +108,92 @@ def create(%Pleroma.Filter{user_id: user_id, filter_id: nil} = filter) do max_id + 1 end - filter - |> Map.put(:filter_id, filter_id) - |> Repo.insert() + change(changeset, filter_id: filter_id) end - def create(%Pleroma.Filter{} = filter) do - Repo.insert(filter) + # don't override expires_at, if passed expires_at and expires_in + defp maybe_add_expires_at(%{changes: %{expires_at: %NaiveDateTime{} = _}} = changeset, _) do + changeset end - def delete(%Pleroma.Filter{id: filter_key} = filter) when is_number(filter_key) do - Repo.delete(filter) + defp maybe_add_expires_at(changeset, %{expires_in: expires_in}) + when is_integer(expires_in) and expires_in > 0 do + expires_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(expires_in) + |> NaiveDateTime.truncate(:second) + + change(changeset, expires_at: expires_at) end - def delete(%Pleroma.Filter{id: filter_key} = filter) when is_nil(filter_key) do - %Pleroma.Filter{id: id} = get(filter.filter_id, %{id: filter.user_id}) - - filter - |> Map.put(:id, id) - |> Repo.delete() + defp maybe_add_expires_at(changeset, %{expires_in: nil}) do + change(changeset, expires_at: nil) end - def update(%Pleroma.Filter{} = filter, params) do + defp maybe_add_expires_at(changeset, _), do: changeset + + defp maybe_add_expiration_job(%{expires_at: %NaiveDateTime{} = expires_at} = filter) do + Pleroma.Workers.PurgeExpiredFilter.enqueue(%{ + filter_id: filter.id, + expires_at: DateTime.from_naive!(expires_at, "Etc/UTC") + }) + end + + defp maybe_add_expiration_job(_), do: {:ok, nil} + + @spec delete(t()) :: {:ok, t()} | {:error, Ecto.Changeset.t()} + def delete(%__MODULE__{} = filter) do + Repo.transaction(fn -> delete_with_expiration(filter) end) + end + + defp delete_with_expiration(filter) do + with {:ok, _} <- maybe_delete_old_expiration_job(filter, nil), + {:ok, filter} <- Repo.delete(filter) do + filter + else + {:error, error} -> Repo.rollback(error) + end + end + + @spec update(t(), map()) :: {:ok, t()} | {:error, Ecto.Changeset.t()} + def update(%__MODULE__{} = filter, params) do + Repo.transaction(fn -> update_with_expiration(filter, params) end) + end + + defp update_with_expiration(filter, params) do + with {:ok, updated} <- do_update(filter, params), + {:ok, _} <- maybe_delete_old_expiration_job(filter, updated), + {:ok, _} <- + maybe_add_expiration_job(updated) do + updated + else + {:error, error} -> Repo.rollback(error) + end + end + + defp do_update(filter, params) do filter |> cast(params, [:phrase, :context, :hide, :expires_at, :whole_word]) |> validate_required([:phrase, :context]) + |> maybe_add_expires_at(params) |> Repo.update() end + defp maybe_delete_old_expiration_job(%{expires_at: nil}, _), do: {:ok, nil} + + defp maybe_delete_old_expiration_job(%{expires_at: expires_at}, %{expires_at: expires_at}) do + {:ok, nil} + end + + defp maybe_delete_old_expiration_job(%{id: id}, _) do + with %Oban.Job{} = job <- Pleroma.Workers.PurgeExpiredFilter.get_expiration(id) do + Repo.delete(job) + else + nil -> {:ok, nil} + end + end + + @spec compose_regex(User.t() | [t()], format()) :: String.t() | Regex.t() | nil def compose_regex(user_or_filters, format \\ :postgres) def compose_regex(%User{} = user, format) do diff --git a/lib/pleroma/web/api_spec/operations/filter_operation.ex b/lib/pleroma/web/api_spec/operations/filter_operation.ex index c5b0c035b..9374a7868 100644 --- a/lib/pleroma/web/api_spec/operations/filter_operation.ex +++ b/lib/pleroma/web/api_spec/operations/filter_operation.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ApiSpec.FilterOperation do alias OpenApiSpex.Operation alias OpenApiSpex.Schema alias Pleroma.Web.ApiSpec.Helpers + alias Pleroma.Web.ApiSpec.Schemas.ApiError alias Pleroma.Web.ApiSpec.Schemas.BooleanLike def open_api_operation(action) do @@ -20,7 +21,8 @@ def index_operation do operationId: "FilterController.index", security: [%{"oAuth" => ["read:filters"]}], responses: %{ - 200 => Operation.response("Filters", "application/json", array_of_filters()) + 200 => Operation.response("Filters", "application/json", array_of_filters()), + 403 => Operation.response("Error", "application/json", ApiError) } } end @@ -32,7 +34,10 @@ def create_operation do operationId: "FilterController.create", requestBody: Helpers.request_body("Parameters", create_request(), required: true), security: [%{"oAuth" => ["write:filters"]}], - responses: %{200 => Operation.response("Filter", "application/json", filter())} + responses: %{ + 200 => Operation.response("Filter", "application/json", filter()), + 403 => Operation.response("Error", "application/json", ApiError) + } } end @@ -44,7 +49,9 @@ def show_operation do operationId: "FilterController.show", security: [%{"oAuth" => ["read:filters"]}], responses: %{ - 200 => Operation.response("Filter", "application/json", filter()) + 200 => Operation.response("Filter", "application/json", filter()), + 403 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) } } end @@ -58,7 +65,8 @@ def update_operation do requestBody: Helpers.request_body("Parameters", update_request(), required: true), security: [%{"oAuth" => ["write:filters"]}], responses: %{ - 200 => Operation.response("Filter", "application/json", filter()) + 200 => Operation.response("Filter", "application/json", filter()), + 403 => Operation.response("Error", "application/json", ApiError) } } end @@ -75,7 +83,8 @@ def delete_operation do Operation.response("Filter", "application/json", %Schema{ type: :object, description: "Empty object" - }) + }), + 403 => Operation.response("Error", "application/json", ApiError) } } end @@ -210,15 +219,13 @@ defp update_request do nullable: true, description: "Consider word boundaries?", default: true + }, + expires_in: %Schema{ + nullable: true, + type: :integer, + description: + "Number of seconds from now the filter should expire. Otherwise, null for a filter that doesn't expire." } - # TODO: probably should implement filter expiration - # expires_in: %Schema{ - # type: :string, - # format: :"date-time", - # description: - # "ISO 8601 Datetime for when the filter expires. Otherwise, - # null for a filter that doesn't expire." - # } }, required: [:phrase, :context], example: %{ diff --git a/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex b/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex index c8b4a3095..9b1ae809d 100644 --- a/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex @@ -20,6 +20,8 @@ defmodule Pleroma.Web.MastodonAPI.FilterController do defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.FilterOperation + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + @doc "GET /api/v1/filters" def index(%{assigns: %{user: user}} = conn, _) do filters = Filter.get_filters(user) @@ -29,25 +31,23 @@ def index(%{assigns: %{user: user}} = conn, _) do @doc "POST /api/v1/filters" def create(%{assigns: %{user: user}, body_params: params} = conn, _) do - query = %Filter{ - user_id: user.id, - phrase: params.phrase, - context: params.context, - hide: params.irreversible, - whole_word: params.whole_word - # TODO: support `expires_in` parameter (as in Mastodon API) - } - - {:ok, response} = Filter.create(query) - - render(conn, "show.json", filter: response) + with {:ok, response} <- + params + |> Map.put(:user_id, user.id) + |> Map.put(:hide, params[:irreversible]) + |> Map.delete(:irreversible) + |> Filter.create() do + render(conn, "show.json", filter: response) + end end @doc "GET /api/v1/filters/:id" def show(%{assigns: %{user: user}} = conn, %{id: filter_id}) do - filter = Filter.get(filter_id, user) - - render(conn, "show.json", filter: filter) + with %Filter{} = filter <- Filter.get(filter_id, user) do + render(conn, "show.json", filter: filter) + else + nil -> {:error, :not_found} + end end @doc "PUT /api/v1/filters/:id" @@ -56,28 +56,31 @@ def update( %{id: filter_id} ) do params = - params - |> Map.delete(:irreversible) - |> Map.put(:hide, params[:irreversible]) - |> Enum.reject(fn {_key, value} -> is_nil(value) end) - |> Map.new() - - # TODO: support `expires_in` parameter (as in Mastodon API) + if is_boolean(params[:irreversible]) do + params + |> Map.put(:hide, params[:irreversible]) + |> Map.delete(:irreversible) + else + params + end with %Filter{} = filter <- Filter.get(filter_id, user), {:ok, %Filter{} = filter} <- Filter.update(filter, params) do render(conn, "show.json", filter: filter) + else + nil -> {:error, :not_found} + error -> error end end @doc "DELETE /api/v1/filters/:id" def delete(%{assigns: %{user: user}} = conn, %{id: filter_id}) do - query = %Filter{ - user_id: user.id, - filter_id: filter_id - } - - {:ok, _} = Filter.delete(query) - json(conn, %{}) + with %Filter{} = filter <- Filter.get(filter_id, user), + {:ok, _} <- Filter.delete(filter) do + json(conn, %{}) + else + nil -> {:error, :not_found} + error -> error + end end end diff --git a/lib/pleroma/workers/purge_expired_filter.ex b/lib/pleroma/workers/purge_expired_filter.ex new file mode 100644 index 000000000..9c3db8af2 --- /dev/null +++ b/lib/pleroma/workers/purge_expired_filter.ex @@ -0,0 +1,43 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.PurgeExpiredFilter do + @moduledoc """ + Worker which purges expired filters + """ + + use Oban.Worker, queue: :filter_expiration, max_attempts: 1, unique: [fields: [:args]] + + import Ecto.Query + + alias Oban.Job + alias Pleroma.Repo + + @spec enqueue(%{filter_id: integer(), expires_at: DateTime.t()}) :: + {:ok, Job.t()} | {:error, Ecto.Changeset.t()} + def enqueue(args) do + {scheduled_at, args} = Map.pop(args, :expires_at) + + args + |> new(scheduled_at: scheduled_at) + |> Oban.insert() + end + + @impl true + def perform(%Job{args: %{"filter_id" => id}}) do + Pleroma.Filter + |> Repo.get(id) + |> Repo.delete() + end + + @spec get_expiration(pos_integer()) :: Job.t() | nil + def get_expiration(id) do + from(j in Job, + where: j.state == "scheduled", + where: j.queue == "filter_expiration", + where: fragment("?->'filter_id' = ?", j.args, ^id) + ) + |> Repo.one() + end +end diff --git a/test/pleroma/filter_test.exs b/test/pleroma/filter_test.exs index a9e256e8c..19ad6b8c0 100644 --- a/test/pleroma/filter_test.exs +++ b/test/pleroma/filter_test.exs @@ -7,81 +7,120 @@ defmodule Pleroma.FilterTest do import Pleroma.Factory + alias Oban.Job alias Pleroma.Filter - alias Pleroma.Repo + + setup do + [user: insert(:user)] + end describe "creating filters" do - test "creating one filter" do - user = insert(:user) + test "creation validation error", %{user: user} do + attrs = %{ + user_id: user.id, + expires_in: 60 + } - query = %Filter{ + {:error, _} = Filter.create(attrs) + + assert Repo.all(Job) == [] + end + + test "use passed expires_at instead expires_in", %{user: user} do + now = NaiveDateTime.utc_now() + + attrs = %{ + user_id: user.id, + expires_at: now, + phrase: "knights", + context: ["home"], + expires_in: 600 + } + + {:ok, %Filter{} = filter} = Filter.create(attrs) + + result = Filter.get(filter.filter_id, user) + assert result.expires_at == NaiveDateTime.truncate(now, :second) + + [job] = Repo.all(Job) + + assert DateTime.truncate(job.scheduled_at, :second) == + now |> NaiveDateTime.truncate(:second) |> DateTime.from_naive!("Etc/UTC") + end + + test "creating one filter", %{user: user} do + attrs = %{ user_id: user.id, filter_id: 42, phrase: "knights", context: ["home"] } - {:ok, %Filter{} = filter} = Filter.create(query) + {:ok, %Filter{} = filter} = Filter.create(attrs) result = Filter.get(filter.filter_id, user) - assert query.phrase == result.phrase + assert attrs.phrase == result.phrase end - test "creating one filter without a pre-defined filter_id" do - user = insert(:user) + test "creating with expired_at", %{user: user} do + attrs = %{ + user_id: user.id, + filter_id: 42, + phrase: "knights", + context: ["home"], + expires_in: 60 + } - query = %Filter{ + {:ok, %Filter{} = filter} = Filter.create(attrs) + result = Filter.get(filter.filter_id, user) + assert attrs.phrase == result.phrase + + assert [_] = Repo.all(Job) + end + + test "creating one filter without a pre-defined filter_id", %{user: user} do + attrs = %{ user_id: user.id, phrase: "knights", context: ["home"] } - {:ok, %Filter{} = filter} = Filter.create(query) + {:ok, %Filter{} = filter} = Filter.create(attrs) # Should start at 1 assert filter.filter_id == 1 end - test "creating additional filters uses previous highest filter_id + 1" do - user = insert(:user) + test "creating additional filters uses previous highest filter_id + 1", %{user: user} do + filter1 = insert(:filter, user: user) - query_one = %Filter{ - user_id: user.id, - filter_id: 42, - phrase: "knights", - context: ["home"] - } - - {:ok, %Filter{} = filter_one} = Filter.create(query_one) - - query_two = %Filter{ + attrs = %{ user_id: user.id, # No filter_id phrase: "who", context: ["home"] } - {:ok, %Filter{} = filter_two} = Filter.create(query_two) - assert filter_two.filter_id == filter_one.filter_id + 1 + {:ok, %Filter{} = filter2} = Filter.create(attrs) + assert filter2.filter_id == filter1.filter_id + 1 end - test "filter_id is unique per user" do - user_one = insert(:user) + test "filter_id is unique per user", %{user: user_one} do user_two = insert(:user) - query_one = %Filter{ + attrs1 = %{ user_id: user_one.id, phrase: "knights", context: ["home"] } - {:ok, %Filter{} = filter_one} = Filter.create(query_one) + {:ok, %Filter{} = filter_one} = Filter.create(attrs1) - query_two = %Filter{ + attrs2 = %{ user_id: user_two.id, phrase: "who", context: ["home"] } - {:ok, %Filter{} = filter_two} = Filter.create(query_two) + {:ok, %Filter{} = filter_two} = Filter.create(attrs2) assert filter_one.filter_id == 1 assert filter_two.filter_id == 1 @@ -94,65 +133,61 @@ test "filter_id is unique per user" do end end - test "deleting a filter" do - user = insert(:user) + test "deleting a filter", %{user: user} do + filter = insert(:filter, user: user) - query = %Filter{ - user_id: user.id, - filter_id: 0, - phrase: "knights", - context: ["home"] - } - - {:ok, _filter} = Filter.create(query) - {:ok, filter} = Filter.delete(query) - assert is_nil(Repo.get(Filter, filter.filter_id)) + assert Repo.get(Filter, filter.id) + {:ok, filter} = Filter.delete(filter) + refute Repo.get(Filter, filter.id) end - test "getting all filters by an user" do - user = insert(:user) - - query_one = %Filter{ + test "deleting a filter with expires_at is removing Oban job too", %{user: user} do + attrs = %{ user_id: user.id, - filter_id: 1, - phrase: "knights", - context: ["home"] + phrase: "cofe", + context: ["home"], + expires_in: 600 } - query_two = %Filter{ - user_id: user.id, - filter_id: 2, - phrase: "who", - context: ["home"] - } + {:ok, filter} = Filter.create(attrs) + assert %Job{id: job_id} = Pleroma.Workers.PurgeExpiredFilter.get_expiration(filter.id) + {:ok, _} = Filter.delete(filter) - {:ok, filter_one} = Filter.create(query_one) - {:ok, filter_two} = Filter.create(query_two) - filters = Filter.get_filters(user) - assert filter_one in filters - assert filter_two in filters + assert Repo.get(Job, job_id) == nil end - test "updating a filter" do - user = insert(:user) + test "getting all filters by an user", %{user: user} do + filter1 = insert(:filter, user: user) + filter2 = insert(:filter, user: user) - query_one = %Filter{ - user_id: user.id, - filter_id: 1, - phrase: "knights", - context: ["home"] - } + filter_ids = user |> Filter.get_filters() |> collect_ids() + + assert filter1.id in filter_ids + assert filter2.id in filter_ids + end + + test "updating a filter", %{user: user} do + filter = insert(:filter, user: user) changes = %{ phrase: "who", context: ["home", "timeline"] } - {:ok, filter_one} = Filter.create(query_one) - {:ok, filter_two} = Filter.update(filter_one, changes) + {:ok, updated_filter} = Filter.update(filter, changes) - assert filter_one != filter_two - assert filter_two.phrase == changes.phrase - assert filter_two.context == changes.context + assert filter != updated_filter + assert updated_filter.phrase == changes.phrase + assert updated_filter.context == changes.context + end + + test "updating with error", %{user: user} do + filter = insert(:filter, user: user) + + changes = %{ + phrase: nil + } + + {:error, _} = Filter.update(filter, changes) end end diff --git a/test/pleroma/web/mastodon_api/controllers/filter_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/filter_controller_test.exs index dc6739178..98ab9e717 100644 --- a/test/pleroma/web/mastodon_api/controllers/filter_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/filter_controller_test.exs @@ -4,149 +4,412 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do use Pleroma.Web.ConnCase, async: true + use Oban.Testing, repo: Pleroma.Repo - alias Pleroma.Web.MastodonAPI.FilterView + import Pleroma.Factory - test "creating a filter" do - %{conn: conn} = oauth_access(["write:filters"]) + alias Pleroma.Filter + alias Pleroma.Repo + alias Pleroma.Workers.PurgeExpiredFilter - filter = %Pleroma.Filter{ - phrase: "knights", - context: ["home"] - } - - conn = + test "non authenticated creation request", %{conn: conn} do + response = conn |> put_req_header("content-type", "application/json") - |> post("/api/v1/filters", %{"phrase" => filter.phrase, context: filter.context}) + |> post("/api/v1/filters", %{"phrase" => "knights", context: ["home"]}) + |> json_response(403) - assert response = json_response_and_validate_schema(conn, 200) - assert response["phrase"] == filter.phrase - assert response["context"] == filter.context - assert response["irreversible"] == false - assert response["id"] != nil - assert response["id"] != "" + assert response["error"] == "Invalid credentials." + end + + describe "creating" do + setup do: oauth_access(["write:filters"]) + + test "a common filter", %{conn: conn, user: user} do + params = %{ + phrase: "knights", + context: ["home"], + irreversible: true + } + + response = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/filters", params) + |> json_response_and_validate_schema(200) + + assert response["phrase"] == params.phrase + assert response["context"] == params.context + assert response["irreversible"] == true + assert response["id"] != nil + assert response["id"] != "" + assert response["expires_at"] == nil + + filter = Filter.get(response["id"], user) + assert filter.hide == true + end + + test "a filter with expires_in", %{conn: conn, user: user} do + in_seconds = 600 + + response = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/filters", %{ + "phrase" => "knights", + context: ["home"], + expires_in: in_seconds + }) + |> json_response_and_validate_schema(200) + + assert response["irreversible"] == false + + expires_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(in_seconds) + |> Pleroma.Web.CommonAPI.Utils.to_masto_date() + + assert response["expires_at"] == expires_at + + filter = Filter.get(response["id"], user) + + id = filter.id + + assert_enqueued( + worker: PurgeExpiredFilter, + args: %{filter_id: filter.id} + ) + + assert {:ok, %{id: ^id}} = + perform_job(PurgeExpiredFilter, %{ + filter_id: filter.id + }) + + assert Repo.aggregate(Filter, :count, :id) == 0 + end end test "fetching a list of filters" do %{user: user, conn: conn} = oauth_access(["read:filters"]) - query_one = %Pleroma.Filter{ - user_id: user.id, - filter_id: 1, - phrase: "knights", - context: ["home"] - } + %{filter_id: id1} = insert(:filter, user: user) + %{filter_id: id2} = insert(:filter, user: user) - query_two = %Pleroma.Filter{ - user_id: user.id, - filter_id: 2, - phrase: "who", - context: ["home"] - } + id1 = to_string(id1) + id2 = to_string(id2) - {:ok, filter_one} = Pleroma.Filter.create(query_one) - {:ok, filter_two} = Pleroma.Filter.create(query_two) + assert [%{"id" => ^id2}, %{"id" => ^id1}] = + conn + |> get("/api/v1/filters") + |> json_response_and_validate_schema(200) + end + + test "fetching a list of filters without token", %{conn: conn} do + insert(:filter) response = conn |> get("/api/v1/filters") - |> json_response_and_validate_schema(200) + |> json_response(403) - assert response == - render_json( - FilterView, - "index.json", - filters: [filter_two, filter_one] - ) + assert response["error"] == "Invalid credentials." end test "get a filter" do %{user: user, conn: conn} = oauth_access(["read:filters"]) # check whole_word false - query = %Pleroma.Filter{ - user_id: user.id, - filter_id: 2, - phrase: "knight", - context: ["home"], - whole_word: false - } + filter = insert(:filter, user: user, whole_word: false) - {:ok, filter} = Pleroma.Filter.create(query) + resp1 = + conn |> get("/api/v1/filters/#{filter.filter_id}") |> json_response_and_validate_schema(200) - conn = get(conn, "/api/v1/filters/#{filter.filter_id}") - - assert response = json_response_and_validate_schema(conn, 200) - assert response["whole_word"] == false + assert resp1["whole_word"] == false # check whole_word true - %{user: user, conn: conn} = oauth_access(["read:filters"]) + filter = insert(:filter, user: user, whole_word: true) - query = %Pleroma.Filter{ - user_id: user.id, - filter_id: 3, - phrase: "knight", - context: ["home"], - whole_word: true - } + resp2 = + conn |> get("/api/v1/filters/#{filter.filter_id}") |> json_response_and_validate_schema(200) - {:ok, filter} = Pleroma.Filter.create(query) - - conn = get(conn, "/api/v1/filters/#{filter.filter_id}") - - assert response = json_response_and_validate_schema(conn, 200) - assert response["whole_word"] == true + assert resp2["whole_word"] == true end - test "update a filter" do - %{user: user, conn: conn} = oauth_access(["write:filters"]) + test "get a filter not_found error" do + filter = insert(:filter) + %{conn: conn} = oauth_access(["read:filters"]) - query = %Pleroma.Filter{ - user_id: user.id, - filter_id: 2, - phrase: "knight", - context: ["home"], - hide: true, - whole_word: true - } + response = + conn |> get("/api/v1/filters/#{filter.filter_id}") |> json_response_and_validate_schema(404) - {:ok, _filter} = Pleroma.Filter.create(query) + assert response["error"] == "Record not found" + end - new = %Pleroma.Filter{ - phrase: "nii", - context: ["home"] - } + describe "updating a filter" do + setup do: oauth_access(["write:filters"]) - conn = + test "common" do + %{conn: conn, user: user} = oauth_access(["write:filters"]) + + filter = + insert(:filter, + user: user, + hide: true, + whole_word: true + ) + + params = %{ + phrase: "nii", + context: ["public"], + irreversible: false + } + + response = + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/filters/#{filter.filter_id}", params) + |> json_response_and_validate_schema(200) + + assert response["phrase"] == params.phrase + assert response["context"] == params.context + assert response["irreversible"] == false + assert response["whole_word"] == true + end + + test "with adding expires_at", %{conn: conn, user: user} do + filter = insert(:filter, user: user) + in_seconds = 600 + + response = + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/filters/#{filter.filter_id}", %{ + phrase: "nii", + context: ["public"], + expires_in: in_seconds, + irreversible: true + }) + |> json_response_and_validate_schema(200) + + assert response["irreversible"] == true + + assert response["expires_at"] == + NaiveDateTime.utc_now() + |> NaiveDateTime.add(in_seconds) + |> Pleroma.Web.CommonAPI.Utils.to_masto_date() + + filter = Filter.get(response["id"], user) + + id = filter.id + + assert_enqueued( + worker: PurgeExpiredFilter, + args: %{filter_id: id} + ) + + assert {:ok, %{id: ^id}} = + perform_job(PurgeExpiredFilter, %{ + filter_id: id + }) + + assert Repo.aggregate(Filter, :count, :id) == 0 + end + + test "with removing expires_at", %{conn: conn, user: user} do + response = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/filters", %{ + phrase: "cofe", + context: ["home"], + expires_in: 600 + }) + |> json_response_and_validate_schema(200) + + filter = Filter.get(response["id"], user) + + assert_enqueued( + worker: PurgeExpiredFilter, + args: %{filter_id: filter.id} + ) + + response = + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/filters/#{filter.filter_id}", %{ + phrase: "nii", + context: ["public"], + expires_in: nil, + whole_word: true + }) + |> json_response_and_validate_schema(200) + + refute_enqueued( + worker: PurgeExpiredFilter, + args: %{filter_id: filter.id} + ) + + assert response["irreversible"] == false + assert response["whole_word"] == true + assert response["expires_at"] == nil + end + + test "expires_at is the same in create and update so job is in db", %{conn: conn, user: user} do + resp1 = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/filters", %{ + phrase: "cofe", + context: ["home"], + expires_in: 600 + }) + |> json_response_and_validate_schema(200) + + filter = Filter.get(resp1["id"], user) + + assert_enqueued( + worker: PurgeExpiredFilter, + args: %{filter_id: filter.id} + ) + + job = PurgeExpiredFilter.get_expiration(filter.id) + + resp2 = + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/filters/#{filter.filter_id}", %{ + phrase: "nii", + context: ["public"] + }) + |> json_response_and_validate_schema(200) + + updated_filter = Filter.get(resp2["id"], user) + + assert_enqueued( + worker: PurgeExpiredFilter, + args: %{filter_id: updated_filter.id} + ) + + after_update = PurgeExpiredFilter.get_expiration(updated_filter.id) + + assert resp1["expires_at"] == resp2["expires_at"] + + assert job.scheduled_at == after_update.scheduled_at + end + + test "updating expires_at updates oban job too", %{conn: conn, user: user} do + resp1 = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/filters", %{ + phrase: "cofe", + context: ["home"], + expires_in: 600 + }) + |> json_response_and_validate_schema(200) + + filter = Filter.get(resp1["id"], user) + + assert_enqueued( + worker: PurgeExpiredFilter, + args: %{filter_id: filter.id} + ) + + job = PurgeExpiredFilter.get_expiration(filter.id) + + resp2 = + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/filters/#{filter.filter_id}", %{ + phrase: "nii", + context: ["public"], + expires_in: 300 + }) + |> json_response_and_validate_schema(200) + + updated_filter = Filter.get(resp2["id"], user) + + assert_enqueued( + worker: PurgeExpiredFilter, + args: %{filter_id: updated_filter.id} + ) + + after_update = PurgeExpiredFilter.get_expiration(updated_filter.id) + + refute resp1["expires_at"] == resp2["expires_at"] + + refute job.scheduled_at == after_update.scheduled_at + end + end + + test "update filter without token", %{conn: conn} do + filter = insert(:filter) + + response = conn |> put_req_header("content-type", "application/json") - |> put("/api/v1/filters/#{query.filter_id}", %{ - phrase: new.phrase, - context: new.context + |> put("/api/v1/filters/#{filter.filter_id}", %{ + phrase: "nii", + context: ["public"] }) + |> json_response(403) - assert response = json_response_and_validate_schema(conn, 200) - assert response["phrase"] == new.phrase - assert response["context"] == new.context - assert response["irreversible"] == true - assert response["whole_word"] == true + assert response["error"] == "Invalid credentials." end - test "delete a filter" do - %{user: user, conn: conn} = oauth_access(["write:filters"]) + describe "delete a filter" do + setup do: oauth_access(["write:filters"]) - query = %Pleroma.Filter{ - user_id: user.id, - filter_id: 2, - phrase: "knight", - context: ["home"] - } + test "common", %{conn: conn, user: user} do + filter = insert(:filter, user: user) - {:ok, filter} = Pleroma.Filter.create(query) + assert conn + |> delete("/api/v1/filters/#{filter.filter_id}") + |> json_response_and_validate_schema(200) == %{} - conn = delete(conn, "/api/v1/filters/#{filter.filter_id}") + assert Repo.all(Filter) == [] + end - assert json_response_and_validate_schema(conn, 200) == %{} + test "with expires_at", %{conn: conn, user: user} do + response = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/filters", %{ + phrase: "cofe", + context: ["home"], + expires_in: 600 + }) + |> json_response_and_validate_schema(200) + + filter = Filter.get(response["id"], user) + + assert_enqueued( + worker: PurgeExpiredFilter, + args: %{filter_id: filter.id} + ) + + assert conn + |> delete("/api/v1/filters/#{filter.filter_id}") + |> json_response_and_validate_schema(200) == %{} + + refute_enqueued( + worker: PurgeExpiredFilter, + args: %{filter_id: filter.id} + ) + + assert Repo.all(Filter) == [] + assert Repo.all(Oban.Job) == [] + end + end + + test "delete a filter without token", %{conn: conn} do + filter = insert(:filter) + + response = + conn + |> delete("/api/v1/filters/#{filter.filter_id}") + |> json_response(403) + + assert response["error"] == "Invalid credentials." end end diff --git a/test/pleroma/workers/purge_expired_filter_test.exs b/test/pleroma/workers/purge_expired_filter_test.exs new file mode 100644 index 000000000..d10586be9 --- /dev/null +++ b/test/pleroma/workers/purge_expired_filter_test.exs @@ -0,0 +1,30 @@ +defmodule Pleroma.Workers.PurgeExpiredFilterTest do + use Pleroma.DataCase, async: true + use Oban.Testing, repo: Repo + + import Pleroma.Factory + + test "purges expired filter" do + %{id: user_id} = insert(:user) + + {:ok, %{id: id}} = + Pleroma.Filter.create(%{ + user_id: user_id, + phrase: "cofe", + context: ["home"], + expires_in: 600 + }) + + assert_enqueued( + worker: Pleroma.Workers.PurgeExpiredFilter, + args: %{filter_id: id} + ) + + assert {:ok, %{id: ^id}} = + perform_job(Pleroma.Workers.PurgeExpiredFilter, %{ + filter_id: id + }) + + assert Repo.aggregate(Pleroma.Filter, :count, :id) == 0 + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index bf9592064..284d573f9 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -455,7 +455,8 @@ def filter_factory do %Pleroma.Filter{ user: build(:user), filter_id: sequence(:filter_id, & &1), - phrase: "cofe" + phrase: "cofe", + context: ["home"] } end end From 39335d42513e47289fc825d04680531b84862686 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 28 Jan 2021 16:57:03 +0300 Subject: [PATCH 2/2] fix for unique oban worker option --- lib/pleroma/workers/purge_expired_filter.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/workers/purge_expired_filter.ex b/lib/pleroma/workers/purge_expired_filter.ex index 9c3db8af2..4740d52e9 100644 --- a/lib/pleroma/workers/purge_expired_filter.ex +++ b/lib/pleroma/workers/purge_expired_filter.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do Worker which purges expired filters """ - use Oban.Worker, queue: :filter_expiration, max_attempts: 1, unique: [fields: [:args]] + use Oban.Worker, queue: :filter_expiration, max_attempts: 1, unique: [period: :infinity] import Ecto.Query