From f161a92cb1abd981e37367fcd5d315ac14510d12 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 23 Jan 2019 18:37:25 +0300 Subject: [PATCH 01/12] [#534] Initial implementation of unreachable federation targets retirement. --- lib/pleroma/instances.ex | 12 +++ lib/pleroma/instances/instance.ex | 77 +++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 39 +++++++--- lib/pleroma/web/salmon/salmon.ex | 13 +++- lib/pleroma/web/websub/websub.ex | 10 ++- .../20190123125546_create_instances.exs | 16 ++++ 6 files changed, 153 insertions(+), 14 deletions(-) create mode 100644 lib/pleroma/instances.ex create mode 100644 lib/pleroma/instances/instance.ex create mode 100644 priv/repo/migrations/20190123125546_create_instances.exs diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex new file mode 100644 index 000000000..25b739520 --- /dev/null +++ b/lib/pleroma/instances.ex @@ -0,0 +1,12 @@ +defmodule Pleroma.Instances do + @moduledoc "Instances context." + + @adapter Pleroma.Instances.Instance + + defdelegate reachable?(url), to: @adapter + defdelegate set_reachable(url), to: @adapter + defdelegate set_unreachable(url, unreachable_since \\ nil), to: @adapter + + def reachability_time_threshold, + do: NaiveDateTime.add(NaiveDateTime.utc_now(), -30 * 24 * 3600, :second) +end diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex new file mode 100644 index 000000000..4507ef6d5 --- /dev/null +++ b/lib/pleroma/instances/instance.ex @@ -0,0 +1,77 @@ +defmodule Pleroma.Instances.Instance do + @moduledoc "Instance." + + alias Pleroma.Instances + alias Pleroma.Instances.Instance + + use Ecto.Schema + + import Ecto.{Query, Changeset} + + alias Pleroma.Repo + + schema "instances" do + field(:host, :string) + field(:unreachable_since, :naive_datetime) + field(:reachability_checked_at, :naive_datetime) + + timestamps() + end + + def update_changeset(struct, params \\ %{}) do + struct + |> cast(params, [:host, :unreachable_since, :reachability_checked_at]) + |> unique_constraint(:host) + end + + def reachable?(url) do + !Repo.one( + from(i in Instance, + where: + i.host == ^host(url) and i.unreachable_since <= ^Instances.reachability_time_threshold(), + select: true + ) + ) + end + + def set_reachable(url) do + Repo.update_all( + from(i in Instance, where: i.host == ^host(url)), + set: [ + unreachable_since: nil, + reachability_checked_at: DateTime.utc_now() + ] + ) + end + + def set_unreachable(url, unreachable_since \\ nil) do + unreachable_since = unreachable_since || DateTime.utc_now() + host = host(url) + existing_record = Repo.get_by(Instance, %{host: host}) + + changes = %{ + unreachable_since: unreachable_since, + reachability_checked_at: NaiveDateTime.utc_now() + } + + if existing_record do + update_changes = + if existing_record.unreachable_since && + NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt, + do: Map.delete(changes, :unreachable_since), + else: changes + + {:ok, _instance} = Repo.update(update_changeset(existing_record, update_changes)) + else + {:ok, _instance} = Repo.insert(update_changeset(%Instance{}, Map.put(changes, :host, host))) + end + end + + defp host(url_or_host) do + if url_or_host =~ ~r/^http/i do + URI.parse(url_or_host).host + else + url_or_host + end + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 82fffd324..b14c91c18 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.ActivityPub do - alias Pleroma.{Activity, Repo, Object, Upload, User, Notification} + alias Pleroma.{Activity, Repo, Object, Upload, User, Notification, Instances} alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF} alias Pleroma.Web.WebFinger alias Pleroma.Web.Federator @@ -721,7 +721,15 @@ def publish(actor, activity) do end) end - def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do + def publish_one(%{inbox: inbox} = activity) do + if Instances.reachable?(inbox) do + do_publish_one(activity) + else + {:error, :noop} + end + end + + defp do_publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do Logger.info("Federating #{id} to #{inbox}") host = URI.parse(inbox).host @@ -734,15 +742,24 @@ def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do digest: digest }) - @httpoison.post( - inbox, - json, - [ - {"Content-Type", "application/activity+json"}, - {"signature", signature}, - {"digest", digest} - ] - ) + with {:ok, _} <- + result = + @httpoison.post( + inbox, + json, + [ + {"Content-Type", "application/activity+json"}, + {"signature", signature}, + {"digest", digest} + ] + ) do + Instances.set_reachable(inbox) + result + else + e -> + Instances.set_unreachable(inbox) + e + end end # TODO: diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index e41657da1..0a0b91433 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.Salmon do @httpoison Application.get_env(:pleroma, :httpoison) use Bitwise + alias Pleroma.Instances alias Pleroma.Web.XML alias Pleroma.Web.OStatus.ActivityRepresenter alias Pleroma.User @@ -167,15 +168,23 @@ defp send_to_user(%{info: %{salmon: salmon}}, feed, poster), do: send_to_user(salmon, feed, poster) defp send_to_user(url, feed, poster) when is_binary(url) do - with {:ok, %{status: code}} <- + with {:reachable, true} <- {:reachable, Instances.reachable?(url)}, + {:ok, %{status: code}} <- poster.( url, feed, [{"Content-Type", "application/magic-envelope+xml"}] ) do + Instances.set_reachable(url) Logger.debug(fn -> "Pushed to #{url}, code #{code}" end) else - e -> Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end) + {:reachable, false} -> + Logger.debug(fn -> "Pushing Salmon to #{url} skipped as marked unreachable)" end) + :noop + + e -> + Instances.set_unreachable(url) + Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end) end end diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 7ca62c83b..a6bbaef37 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.Websub do alias Ecto.Changeset alias Pleroma.Repo + alias Pleroma.Instances alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription} alias Pleroma.Web.OStatus.FeedRepresenter alias Pleroma.Web.{XML, Endpoint, OStatus} @@ -267,7 +268,8 @@ def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret}) d signature = sign(secret || "", xml) Logger.info(fn -> "Pushing #{topic} to #{callback}" end) - with {:ok, %{status: code}} <- + with {:reachable, true} <- {:reachable, Instances.reachable?(callback)}, + {:ok, %{status: code}} <- @httpoison.post( callback, xml, @@ -276,10 +278,16 @@ def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret}) d {"X-Hub-Signature", "sha1=#{signature}"} ] ) do + Instances.set_reachable(callback) Logger.info(fn -> "Pushed to #{callback}, code #{code}" end) {:ok, code} else + {:reachable, false} -> + Logger.debug(fn -> "Pushing to #{callback} skipped as marked unreachable)" end) + {:error, :noop} + e -> + Instances.set_unreachable(callback) Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(e)}" end) {:error, e} end diff --git a/priv/repo/migrations/20190123125546_create_instances.exs b/priv/repo/migrations/20190123125546_create_instances.exs new file mode 100644 index 000000000..4f5915fba --- /dev/null +++ b/priv/repo/migrations/20190123125546_create_instances.exs @@ -0,0 +1,16 @@ +defmodule Pleroma.Repo.Migrations.CreateInstances do + use Ecto.Migration + + def change do + create table(:instances) do + add :host, :string + add :unreachable_since, :naive_datetime + add :reachability_checked_at, :naive_datetime + + timestamps() + end + + create unique_index(:instances, [:host]) + create index(:instances, [:unreachable_since]) + end +end From 20b54366ee916677b3865acf36baeeb062dd550b Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 24 Jan 2019 11:54:52 +0300 Subject: [PATCH 02/12] [#534] Federation publish requests status control (enforced 2xx response code check). --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/salmon/salmon.ex | 2 +- lib/pleroma/web/websub/websub.ex | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index b14c91c18..10155ff5a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -742,7 +742,7 @@ defp do_publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do digest: digest }) - with {:ok, _} <- + with {:ok, %{status: code}} when code in 200..299 <- result = @httpoison.post( inbox, diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 0a0b91433..0423ccee0 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -169,7 +169,7 @@ defp send_to_user(%{info: %{salmon: salmon}}, feed, poster), defp send_to_user(url, feed, poster) when is_binary(url) do with {:reachable, true} <- {:reachable, Instances.reachable?(url)}, - {:ok, %{status: code}} <- + {:ok, %{status: code}} when code in 200..299 <- poster.( url, feed, diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index a6bbaef37..9ceb5fbf7 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -269,7 +269,7 @@ def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret}) d Logger.info(fn -> "Pushing #{topic} to #{callback}" end) with {:reachable, true} <- {:reachable, Instances.reachable?(callback)}, - {:ok, %{status: code}} <- + {:ok, %{status: code}} when code in 200..299 <- @httpoison.post( callback, xml, From 8654a591f08c7d8d5d61f075906f0c6907e877bb Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 24 Jan 2019 17:37:23 +0300 Subject: [PATCH 03/12] [#534] Updating external instances reachability on incoming federation. --- lib/pleroma/instances/instance.ex | 14 +++++++++++--- lib/pleroma/reverse_proxy.ex | 3 ++- lib/pleroma/web/activity_pub/activity_pub.ex | 3 ++- .../web/activity_pub/activity_pub_controller.ex | 7 +++++++ lib/pleroma/web/controller_helper.ex | 5 +++++ lib/pleroma/web/ostatus/ostatus_controller.ex | 7 +++++++ lib/pleroma/web/salmon/salmon.ex | 5 ++++- lib/pleroma/web/websub/websub.ex | 3 ++- lib/pleroma/web/websub/websub_controller.ex | 9 +++++++++ 9 files changed, 49 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 4507ef6d5..fe52331a3 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -24,7 +24,7 @@ def update_changeset(struct, params \\ %{}) do |> unique_constraint(:host) end - def reachable?(url) do + def reachable?(url) when is_binary(url) do !Repo.one( from(i in Instance, where: @@ -34,7 +34,9 @@ def reachable?(url) do ) end - def set_reachable(url) do + def reachable?(_), do: true + + def set_reachable(url) when is_binary(url) do Repo.update_all( from(i in Instance, where: i.host == ^host(url)), set: [ @@ -44,7 +46,11 @@ def set_reachable(url) do ) end - def set_unreachable(url, unreachable_since \\ nil) do + def set_reachable(_), do: {0, :noop} + + def set_unreachable(url, unreachable_since \\ nil) + + def set_unreachable(url, unreachable_since) when is_binary(url) do unreachable_since = unreachable_since || DateTime.utc_now() host = host(url) existing_record = Repo.get_by(Instance, %{host: host}) @@ -67,6 +73,8 @@ def set_unreachable(url, unreachable_since \\ nil) do end end + def set_unreachable(_, _), do: {0, :noop} + defp host(url_or_host) do if url_or_host =~ ~r/^http/i do URI.parse(url_or_host).host diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index a25b5ea4e..d8b17212b 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -3,7 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy do - @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since if-unmodified-since if-none-match if-range range) + @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since if-unmodified-since) ++ + ~w(if-none-match if-range range referer) @resp_cache_headers ~w(etag date last-modified cache-control) @keep_resp_headers @resp_cache_headers ++ ~w(content-type content-disposition content-encoding content-range accept-ranges vary) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 10155ff5a..44c295d65 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -750,7 +750,8 @@ defp do_publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do [ {"Content-Type", "application/activity+json"}, {"signature", signature}, - {"digest", digest} + {"digest", digest}, + {"referer", Pleroma.Web.Endpoint.url()} ] ) do Instances.set_reachable(inbox) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 7eed0a600..dc353dff0 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do use Pleroma.Web, :controller + alias Pleroma.{Activity, User, Object} alias Pleroma.Web.ActivityPub.{ObjectView, UserView} alias Pleroma.Web.ActivityPub.ActivityPub @@ -18,6 +19,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay]) plug(:relay_active? when action in [:relay]) + plug(:set_requester_reachable when action in [:inbox]) def relay_active?(conn, _) do if Keyword.get(Application.get_env(:pleroma, :instance), :allow_relay) do @@ -289,4 +291,9 @@ def errors(conn, _e) do |> put_status(500) |> json("error") end + + defp set_requester_reachable(conn, _) do + Pleroma.Web.ControllerHelper.set_requester_reachable(conn) + conn + end end diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index 14e3d19fd..13cf1877f 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -10,4 +10,9 @@ def json_response(conn, status, json) do |> put_status(status) |> json(json) end + + def set_requester_reachable(conn) do + with [referer] <- get_req_header(conn, "referer"), + do: Pleroma.Instances.set_reachable(referer) + end end diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index ce022bcc1..a89f16b94 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -15,6 +15,8 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Pleroma.Web.ActivityPub.ActivityPub plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming]) + plug(:set_requester_reachable when action in [:salmon_incoming]) + action_fallback(:errors) def feed_redirect(conn, %{"nickname" => nickname}) do @@ -201,4 +203,9 @@ def errors(conn, _) do |> put_status(500) |> text("Something went wrong") end + + defp set_requester_reachable(conn, _) do + Pleroma.Web.ControllerHelper.set_requester_reachable(conn) + conn + end end diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 0423ccee0..e4d2d9517 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -173,7 +173,10 @@ defp send_to_user(url, feed, poster) when is_binary(url) do poster.( url, feed, - [{"Content-Type", "application/magic-envelope+xml"}] + [ + {"Content-Type", "application/magic-envelope+xml"}, + {"referer", Pleroma.Web.Endpoint.url()} + ] ) do Instances.set_reachable(url) Logger.debug(fn -> "Pushed to #{url}, code #{code}" end) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 9ceb5fbf7..ac8903913 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -275,7 +275,8 @@ def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret}) d xml, [ {"Content-Type", "application/atom+xml"}, - {"X-Hub-Signature", "sha1=#{signature}"} + {"X-Hub-Signature", "sha1=#{signature}"}, + {"referer", Pleroma.Web.Endpoint.url()} ] ) do Instances.set_reachable(callback) diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex index e58f144e5..02fe075d7 100644 --- a/lib/pleroma/web/websub/websub_controller.ex +++ b/lib/pleroma/web/websub/websub_controller.ex @@ -4,9 +4,11 @@ defmodule Pleroma.Web.Websub.WebsubController do use Pleroma.Web, :controller + alias Pleroma.{Repo, User} alias Pleroma.Web.{Websub, Federator} alias Pleroma.Web.Websub.WebsubClientSubscription + require Logger plug( @@ -18,6 +20,8 @@ defmodule Pleroma.Web.Websub.WebsubController do ] ) + plug(:set_requester_reachable when action in [:websub_incoming]) + def websub_subscription_request(conn, %{"nickname" => nickname} = params) do user = User.get_cached_by_nickname(nickname) @@ -92,4 +96,9 @@ def websub_incoming(conn, %{"id" => id}) do |> send_resp(500, "Error") end end + + defp set_requester_reachable(conn, _) do + Pleroma.Web.ControllerHelper.set_requester_reachable(conn) + conn + end end From 3e9399ec0b498c0c9783ccb0fea9f682c8b9d0ca Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 24 Jan 2019 19:15:23 +0300 Subject: [PATCH 04/12] [#534] Optimized bulk publish ops to filter on reachability early. `Instance` refactoring. --- lib/pleroma/instances.ex | 9 ++++ lib/pleroma/instances/instance.ex | 57 +++++++++++++------- lib/pleroma/web/activity_pub/activity_pub.ex | 5 +- lib/pleroma/web/salmon/salmon.ex | 8 ++- lib/pleroma/web/websub/websub.ex | 18 ++++--- 5 files changed, 69 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex index 25b739520..6d445d6b3 100644 --- a/lib/pleroma/instances.ex +++ b/lib/pleroma/instances.ex @@ -3,10 +3,19 @@ defmodule Pleroma.Instances do @adapter Pleroma.Instances.Instance + defdelegate filter_reachable(urls), to: @adapter defdelegate reachable?(url), to: @adapter defdelegate set_reachable(url), to: @adapter defdelegate set_unreachable(url, unreachable_since \\ nil), to: @adapter def reachability_time_threshold, do: NaiveDateTime.add(NaiveDateTime.utc_now(), -30 * 24 * 3600, :second) + + def host(url_or_host) when is_binary(url_or_host) do + if url_or_host =~ ~r/^http/i do + URI.parse(url_or_host).host + else + url_or_host + end + end end diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index fe52331a3..a17c8dab1 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -18,12 +18,35 @@ defmodule Pleroma.Instances.Instance do timestamps() end - def update_changeset(struct, params \\ %{}) do + defdelegate host(url), to: Instances + + def changeset(struct, params \\ %{}) do struct |> cast(params, [:host, :unreachable_since, :reachability_checked_at]) + |> validate_required([:host]) |> unique_constraint(:host) end + def filter_reachable([]), do: [] + + def filter_reachable(urls) when is_list(urls) do + hosts = + urls + |> Enum.map(&(&1 && host(&1))) + |> Enum.filter(&(to_string(&1) != "")) + + unreachable_hosts = + Repo.all( + from(i in Instance, + where: + i.host in ^hosts and i.unreachable_since <= ^Instances.reachability_time_threshold(), + select: i.host + ) + ) + + Enum.filter(urls, &(&1 && host(&1) not in unreachable_hosts)) + end + def reachable?(url) when is_binary(url) do !Repo.one( from(i in Instance, @@ -37,13 +60,13 @@ def reachable?(url) when is_binary(url) do def reachable?(_), do: true def set_reachable(url) when is_binary(url) do - Repo.update_all( - from(i in Instance, where: i.host == ^host(url)), - set: [ - unreachable_since: nil, - reachability_checked_at: DateTime.utc_now() - ] - ) + with host <- host(url), + %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do + {:ok, _instance} = + existing_record + |> changeset(%{unreachable_since: nil, reachability_checked_at: DateTime.utc_now()}) + |> Repo.update() + end end def set_reachable(_), do: {0, :noop} @@ -67,19 +90,17 @@ def set_unreachable(url, unreachable_since) when is_binary(url) do do: Map.delete(changes, :unreachable_since), else: changes - {:ok, _instance} = Repo.update(update_changeset(existing_record, update_changes)) + {:ok, _instance} = + existing_record + |> changeset(update_changes) + |> Repo.update() else - {:ok, _instance} = Repo.insert(update_changeset(%Instance{}, Map.put(changes, :host, host))) + {:ok, _instance} = + %Instance{} + |> changeset(Map.put(changes, :host, host)) + |> Repo.insert() end end def set_unreachable(_, _), do: {0, :noop} - - defp host(url_or_host) do - if url_or_host =~ ~r/^http/i do - URI.parse(url_or_host).host - else - url_or_host - end - end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 44c295d65..4b34334a0 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -689,7 +689,7 @@ def should_federate?(inbox, public) do end def publish(actor, activity) do - followers = + remote_followers = if actor.follower_address in activity.recipients do {:ok, followers} = User.get_followers(actor) followers |> Enum.filter(&(!&1.local)) @@ -700,13 +700,14 @@ def publish(actor, activity) do public = is_public?(activity) remote_inboxes = - (Pleroma.Web.Salmon.remote_users(activity) ++ followers) + (Pleroma.Web.Salmon.remote_users(activity) ++ remote_followers) |> Enum.filter(fn user -> User.ap_enabled?(user) end) |> Enum.map(fn %{info: %{source_data: data}} -> (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"] end) |> Enum.uniq() |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) + |> Instances.filter_reachable() {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) json = Jason.encode!(data) diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index e4d2d9517..848131d52 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -221,7 +221,13 @@ def publish(%{info: %{keys: keys}} = user, %{data: %{"type" => type}} = activity {:ok, private, _} = keys_from_pem(keys) {:ok, feed} = encode(private, feed) - remote_users(activity) + remote_users = remote_users(activity) + + salmon_urls = Enum.map(remote_users, & &1.info.salmon) + reachable_salmon_urls = Instances.filter_reachable(salmon_urls) + + remote_users + |> Enum.filter(&(&1.info.salmon in reachable_salmon_urls)) |> Enum.each(fn remote_user -> Task.start(fn -> Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index ac8903913..bb4442591 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -54,7 +54,12 @@ def verify(subscription, getter \\ &@httpoison.get/3) do ] def publish(topic, user, %{data: %{"type" => type}} = activity) when type in @supported_activities do - # TODO: Only send to still valid subscriptions. + response = + user + |> FeedRepresenter.to_simple_form([activity], [user]) + |> :xmerl.export_simple(:xmerl_xml) + |> to_string + query = from( sub in WebsubServerSubscription, @@ -64,13 +69,12 @@ def publish(topic, user, %{data: %{"type" => type}} = activity) subscriptions = Repo.all(query) - Enum.each(subscriptions, fn sub -> - response = - user - |> FeedRepresenter.to_simple_form([activity], [user]) - |> :xmerl.export_simple(:xmerl_xml) - |> to_string + callbacks = Enum.map(subscriptions, & &1.callback) + reachable_callbacks = Instances.filter_reachable(callbacks) + subscriptions + |> Enum.filter(&(&1.callback in reachable_callbacks)) + |> Enum.each(fn sub -> data = %{ xml: response, topic: topic, From 656ed7c84a5d8e423999457f66d8259ec8aa9a44 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 25 Jan 2019 15:10:21 +0300 Subject: [PATCH 05/12] [#534] Configurable outgoing federation reachability timeout. --- config/config.exs | 1 + docs/config.md | 1 + lib/pleroma/instances.ex | 16 ++++++++++++++-- lib/pleroma/instances/instance.ex | 6 ++++-- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index d30b0aad0..7a1a875c9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -125,6 +125,7 @@ banner_upload_limit: 4_000_000, registrations_open: true, federating: true, + federation_reachability_timeout_days: 90, allow_relay: true, rewrite_policy: Pleroma.Web.ActivityPub.MRF.NoOpPolicy, public: true, diff --git a/docs/config.md b/docs/config.md index 6bf7b9ea7..3f4588299 100644 --- a/docs/config.md +++ b/docs/config.md @@ -72,6 +72,7 @@ config :pleroma, Pleroma.Mailer, * `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`). * `account_activation_required`: Require users to confirm their emails before signing in. * `federating`: Enable federation with other instances +* `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it. * `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance * `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default: * `Pleroma.Web.ActivityPub.MRF.NoOpPolicy`: Doesn’t modify activities (default) diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex index 6d445d6b3..0b08f0eb8 100644 --- a/lib/pleroma/instances.ex +++ b/lib/pleroma/instances.ex @@ -8,8 +8,20 @@ defmodule Pleroma.Instances do defdelegate set_reachable(url), to: @adapter defdelegate set_unreachable(url, unreachable_since \\ nil), to: @adapter - def reachability_time_threshold, - do: NaiveDateTime.add(NaiveDateTime.utc_now(), -30 * 24 * 3600, :second) + def reachability_datetime_threshold do + federation_reachability_timeout_days = + Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 90 + + if federation_reachability_timeout_days > 0 do + NaiveDateTime.add( + NaiveDateTime.utc_now(), + -federation_reachability_timeout_days * 24 * 3600, + :second + ) + else + ~N[0000-01-01 00:00:00] + end + end def host(url_or_host) when is_binary(url_or_host) do if url_or_host =~ ~r/^http/i do diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index a17c8dab1..60e8d0e21 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -39,7 +39,8 @@ def filter_reachable(urls) when is_list(urls) do Repo.all( from(i in Instance, where: - i.host in ^hosts and i.unreachable_since <= ^Instances.reachability_time_threshold(), + i.host in ^hosts and + i.unreachable_since <= ^Instances.reachability_datetime_threshold(), select: i.host ) ) @@ -51,7 +52,8 @@ def reachable?(url) when is_binary(url) do !Repo.one( from(i in Instance, where: - i.host == ^host(url) and i.unreachable_since <= ^Instances.reachability_time_threshold(), + i.host == ^host(url) and + i.unreachable_since <= ^Instances.reachability_datetime_threshold(), select: true ) ) From 465adedb7cc457303278444d0f56960f87fde1e9 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 25 Jan 2019 18:29:43 +0300 Subject: [PATCH 06/12] [#534] Websub fix: made SQL use UTC time zone when comparing with `valid_until` (instead of postgresql-server default time zone). --- lib/pleroma/web/websub/websub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index bb4442591..cbb7a5ac7 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -64,7 +64,7 @@ def publish(topic, user, %{data: %{"type" => type}} = activity) from( sub in WebsubServerSubscription, where: sub.topic == ^topic and sub.state == "active", - where: fragment("? > NOW()", sub.valid_until) + where: fragment("? > (NOW() at time zone 'UTC')", sub.valid_until) ) subscriptions = Repo.all(query) From 060d280e64c201d3f8bec5615cc3b02cd460d3e1 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 25 Jan 2019 20:38:13 +0300 Subject: [PATCH 07/12] [#534] Made Salmon.send_to_user calls be handled through Federator.enqueue. --- lib/pleroma/web/federator/federator.ex | 6 +++++- lib/pleroma/web/salmon/salmon.ex | 14 +++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index f3a0e18b8..46f7a4973 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.Federator do use GenServer alias Pleroma.User alias Pleroma.Activity - alias Pleroma.Web.{WebFinger, Websub} + alias Pleroma.Web.{WebFinger, Websub, Salmon} alias Pleroma.Web.Federator.RetryQueue alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Relay @@ -124,6 +124,10 @@ def handle(:incoming_ap_doc, params) do end end + def handle(:publish_single_salmon, {user_or_url, feed, poster}) do + Salmon.send_to_user(user_or_url, feed, poster) + end + def handle(:publish_single_ap, params) do case ActivityPub.publish_one(params) do {:ok, _} -> diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 848131d52..17ca7a6e8 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -164,10 +164,10 @@ def remote_users(%{data: %{"to" => to} = data}) do # push an activity to remote accounts # - defp send_to_user(%{info: %{salmon: salmon}}, feed, poster), + def send_to_user(%{info: %{salmon: salmon}}, feed, poster), do: send_to_user(salmon, feed, poster) - defp send_to_user(url, feed, poster) when is_binary(url) do + def send_to_user(url, feed, poster) when is_binary(url) do with {:reachable, true} <- {:reachable, Instances.reachable?(url)}, {:ok, %{status: code}} when code in 200..299 <- poster.( @@ -180,6 +180,7 @@ defp send_to_user(url, feed, poster) when is_binary(url) do ) do Instances.set_reachable(url) Logger.debug(fn -> "Pushed to #{url}, code #{code}" end) + :ok else {:reachable, false} -> Logger.debug(fn -> "Pushing Salmon to #{url} skipped as marked unreachable)" end) @@ -188,10 +189,11 @@ defp send_to_user(url, feed, poster) when is_binary(url) do e -> Instances.set_unreachable(url) Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end) + :error end end - defp send_to_user(_, _, _), do: nil + def send_to_user(_, _, _), do: :noop @supported_activities [ "Create", @@ -229,10 +231,8 @@ def publish(%{info: %{keys: keys}} = user, %{data: %{"type" => type}} = activity remote_users |> Enum.filter(&(&1.info.salmon in reachable_salmon_urls)) |> Enum.each(fn remote_user -> - Task.start(fn -> - Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end) - send_to_user(remote_user, feed, poster) - end) + Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end) + Pleroma.Web.Federator.enqueue(:publish_single_salmon, {remote_user, feed, poster}) end) end end From d99650270b980c006690a7051a2d5cffe07779f1 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 25 Jan 2019 20:38:54 +0300 Subject: [PATCH 08/12] [#534] Federation reachability filtering tests. --- test/support/factory.ex | 2 +- .../activity_pub_controller_test.exs | 44 +++++++- test/web/federator_test.exs | 101 +++++++++++++++++- test/web/ostatus/ostatus_controller_test.exs | 90 ++++++++++------ test/web/websub/websub_controller_test.exs | 70 +++++++----- 5 files changed, 243 insertions(+), 64 deletions(-) diff --git a/test/support/factory.ex b/test/support/factory.ex index 4ac77981a..964b2b61c 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -193,7 +193,7 @@ def follow_activity_factory do def websub_subscription_factory do %Pleroma.Web.Websub.WebsubServerSubscription{ topic: "http://example.org", - callback: "http://example/org/callback", + callback: "http://example.org/callback", secret: "here's a secret", valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), 100), state: "requested" diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 52e67f046..1b704330f 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -6,8 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory alias Pleroma.Web.ActivityPub.{UserView, ObjectView} - alias Pleroma.{Object, Repo, User} - alias Pleroma.Activity + alias Pleroma.{Object, Repo, Activity, User, Instances} setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -144,6 +143,24 @@ test "it inserts an incoming activity into the database", %{conn: conn} do :timer.sleep(500) assert Activity.get_by_ap_id(data["id"]) end + + test "it clears `unreachable` federation status of the sender", %{conn: conn} do + sender_url = "https://pleroma.soykaf.com" + Instances.set_unreachable(sender_url, Instances.reachability_datetime_threshold()) + refute Instances.reachable?(sender_url) + + data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() + + conn = + conn + |> assign(:valid_signature, true) + |> put_req_header("content-type", "application/activity+json") + |> put_req_header("referer", sender_url) + |> post("/inbox", data) + + assert "ok" == json_response(conn, 200) + assert Instances.reachable?(sender_url) + end end describe "/users/:nickname/inbox" do @@ -191,6 +208,29 @@ test "it returns a note activity in a collection", %{conn: conn} do assert response(conn, 200) =~ note_activity.data["object"]["content"] end + + test "it clears `unreachable` federation status of the sender", %{conn: conn} do + sender_host = "pleroma.soykaf.com" + Instances.set_unreachable(sender_host, Instances.reachability_datetime_threshold()) + refute Instances.reachable?(sender_host) + + user = insert(:user) + + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + |> Map.put("bcc", [user.ap_id]) + + conn = + conn + |> assign(:valid_signature, true) + |> put_req_header("content-type", "application/activity+json") + |> put_req_header("referer", "https://#{sender_host}") + |> post("/users/#{user.nickname}/inbox", data) + + assert "ok" == json_response(conn, 200) + assert Instances.reachable?(sender_host) + end end describe "/users/:nickname/outbox" do diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index a49265c0c..f4234aea8 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -3,8 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.FederatorTest do - alias Pleroma.Web.Federator - alias Pleroma.Web.CommonAPI + alias Pleroma.Web.{CommonAPI, Federator} + alias Pleroma.Instances use Pleroma.DataCase import Pleroma.Factory import Mock @@ -71,6 +71,103 @@ test "with relays deactivated, it does not publish to the relay", %{ end end + describe "Targets reachability filtering in `publish`" do + test_with_mock "it federates only to reachable instances via AP", + Federator, + [:passthrough], + [] do + user = insert(:user) + + {inbox1, inbox2} = + {"https://domain.com/users/nick1/inbox", "https://domain2.com/users/nick2/inbox"} + + insert(:user, %{ + local: false, + nickname: "nick1@domain.com", + ap_id: "https://domain.com/users/nick1", + info: %{ap_enabled: true, source_data: %{"inbox" => inbox1}} + }) + + insert(:user, %{ + local: false, + nickname: "nick2@domain2.com", + ap_id: "https://domain2.com/users/nick2", + info: %{ap_enabled: true, source_data: %{"inbox" => inbox2}} + }) + + Instances.set_unreachable( + URI.parse(inbox2).host, + Instances.reachability_datetime_threshold() + ) + + {:ok, _activity} = + CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"}) + + assert called(Federator.enqueue(:publish_single_ap, %{inbox: inbox1})) + refute called(Federator.enqueue(:publish_single_ap, %{inbox: inbox2})) + end + + test_with_mock "it federates only to reachable instances via Websub", + Federator, + [:passthrough], + [] do + user = insert(:user) + websub_topic = Pleroma.Web.OStatus.feed_path(user) + + sub1 = + insert(:websub_subscription, %{ + topic: websub_topic, + state: "active", + callback: "http://pleroma.soykaf.com/cb" + }) + + sub2 = + insert(:websub_subscription, %{ + topic: websub_topic, + state: "active", + callback: "https://pleroma2.soykaf.com/cb" + }) + + Instances.set_unreachable(sub1.callback, Instances.reachability_datetime_threshold()) + + {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI"}) + + assert called(Federator.enqueue(:publish_single_websub, %{callback: sub2.callback})) + refute called(Federator.enqueue(:publish_single_websub, %{callback: sub1.callback})) + end + + test_with_mock "it federates only to reachable instances via Salmon", + Federator, + [:passthrough], + [] do + user = insert(:user) + + remote_user1 = + insert(:user, %{ + local: false, + nickname: "nick1@domain.com", + ap_id: "https://domain.com/users/nick1", + info: %{salmon: "https://domain.com/salmon"} + }) + + remote_user2 = + insert(:user, %{ + local: false, + nickname: "nick2@domain2.com", + ap_id: "https://domain2.com/users/nick2", + info: %{salmon: "https://domain2.com/salmon"} + }) + + Instances.set_unreachable("domain.com", Instances.reachability_datetime_threshold()) + + {:ok, _activity} = + CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"}) + + assert called(Federator.enqueue(:publish_single_salmon, {remote_user2, :_, :_})) + refute called(Federator.enqueue(:publish_single_websub, {remote_user1, :_, :_})) + end + end + describe "Receive an activity" do test "successfully processes incoming AP docs with correct origin" do params = %{ diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index 995cc00d6..ca447aa5d 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -5,7 +5,7 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory - alias Pleroma.{User, Repo, Object} + alias Pleroma.{User, Repo, Object, Instances} alias Pleroma.Web.CommonAPI alias Pleroma.Web.OStatus.ActivityRepresenter @@ -14,49 +14,69 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do :ok end - test "decodes a salmon", %{conn: conn} do - user = insert(:user) - salmon = File.read!("test/fixtures/salmon.xml") + describe "salmon_incoming" do + test "decodes a salmon", %{conn: conn} do + user = insert(:user) + salmon = File.read!("test/fixtures/salmon.xml") - conn = - conn - |> put_req_header("content-type", "application/atom+xml") - |> post("/users/#{user.nickname}/salmon", salmon) + conn = + conn + |> put_req_header("content-type", "application/atom+xml") + |> post("/users/#{user.nickname}/salmon", salmon) - assert response(conn, 200) - end + assert response(conn, 200) + end - test "decodes a salmon with a changed magic key", %{conn: conn} do - user = insert(:user) - salmon = File.read!("test/fixtures/salmon.xml") + test "decodes a salmon with a changed magic key", %{conn: conn} do + user = insert(:user) + salmon = File.read!("test/fixtures/salmon.xml") - conn = - conn - |> put_req_header("content-type", "application/atom+xml") - |> post("/users/#{user.nickname}/salmon", salmon) + conn = + conn + |> put_req_header("content-type", "application/atom+xml") + |> post("/users/#{user.nickname}/salmon", salmon) - assert response(conn, 200) + assert response(conn, 200) - # Set a wrong magic-key for a user so it has to refetch - salmon_user = User.get_by_ap_id("http://gs.example.org:4040/index.php/user/1") - # Wrong key - info_cng = - User.Info.remote_user_creation(salmon_user.info, %{ - magic_key: - "RSA.pu0s-halox4tu7wmES1FVSx6u-4wc0YrUFXcqWXZG4-27UmbCOpMQftRCldNRfyA-qLbz-eqiwrong1EwUvjsD4cYbAHNGHwTvDOyx5AKthQUP44ykPv7kjKGh3DWKySJvcs9tlUG87hlo7AvnMo9pwRS_Zz2CacQ-MKaXyDepk=.AQAB" - }) + # Set a wrong magic-key for a user so it has to refetch + salmon_user = User.get_by_ap_id("http://gs.example.org:4040/index.php/user/1") + # Wrong key + info_cng = + User.Info.remote_user_creation(salmon_user.info, %{ + magic_key: + "RSA.pu0s-halox4tu7wmES1FVSx6u-4wc0YrUFXcqWXZG4-27UmbCOpMQftRCldNRfyA-qLbz-eqiwrong1EwUvjsD4cYbAHNGHwTvDOyx5AKthQUP44ykPv7kjKGh3DWKySJvcs9tlUG87hlo7AvnMo9pwRS_Zz2CacQ-MKaXyDepk=.AQAB" + }) - salmon_user - |> Ecto.Changeset.change() - |> Ecto.Changeset.put_embed(:info, info_cng) - |> Repo.update() + salmon_user + |> Ecto.Changeset.change() + |> Ecto.Changeset.put_embed(:info, info_cng) + |> Repo.update() - conn = - build_conn() - |> put_req_header("content-type", "application/atom+xml") - |> post("/users/#{user.nickname}/salmon", salmon) + conn = + build_conn() + |> put_req_header("content-type", "application/atom+xml") + |> post("/users/#{user.nickname}/salmon", salmon) - assert response(conn, 200) + assert response(conn, 200) + end + + test "it clears `unreachable` federation status of the sender", %{conn: conn} do + sender_url = "https://pleroma.soykaf.com" + Instances.set_unreachable(sender_url, Instances.reachability_datetime_threshold()) + refute Instances.reachable?(sender_url) + + user = insert(:user) + salmon = File.read!("test/fixtures/salmon.xml") + + conn = + conn + |> put_req_header("content-type", "application/atom+xml") + |> put_req_header("referer", sender_url) + |> post("/users/#{user.nickname}/salmon", salmon) + + assert response(conn, 200) + assert Instances.reachable?(sender_url) + end end test "gets a feed", %{conn: conn} do diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs index 9cbcda063..c445ed676 100644 --- a/test/web/websub/websub_controller_test.exs +++ b/test/web/websub/websub_controller_test.exs @@ -6,7 +6,7 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory alias Pleroma.Web.Websub.WebsubClientSubscription - alias Pleroma.{Repo, Activity} + alias Pleroma.{Repo, Activity, Instances} alias Pleroma.Web.Websub test "websub subscription request", %{conn: conn} do @@ -50,35 +50,57 @@ test "websub subscription confirmation", %{conn: conn} do assert_in_delta NaiveDateTime.diff(websub.valid_until, NaiveDateTime.utc_now()), 100, 5 end - test "handles incoming feed updates", %{conn: conn} do - websub = insert(:websub_client_subscription) - doc = "some stuff" - signature = Websub.sign(websub.secret, doc) + describe "websub_incoming" do + test "handles incoming feed updates", %{conn: conn} do + websub = insert(:websub_client_subscription) + doc = "some stuff" + signature = Websub.sign(websub.secret, doc) - conn = - conn - |> put_req_header("x-hub-signature", "sha1=" <> signature) - |> put_req_header("content-type", "application/atom+xml") - |> post("/push/subscriptions/#{websub.id}", doc) + conn = + conn + |> put_req_header("x-hub-signature", "sha1=" <> signature) + |> put_req_header("content-type", "application/atom+xml") + |> post("/push/subscriptions/#{websub.id}", doc) - assert response(conn, 200) == "OK" + assert response(conn, 200) == "OK" - assert length(Repo.all(Activity)) == 1 - end + assert length(Repo.all(Activity)) == 1 + end - test "rejects incoming feed updates with the wrong signature", %{conn: conn} do - websub = insert(:websub_client_subscription) - doc = "some stuff" - signature = Websub.sign("wrong secret", doc) + test "rejects incoming feed updates with the wrong signature", %{conn: conn} do + websub = insert(:websub_client_subscription) + doc = "some stuff" + signature = Websub.sign("wrong secret", doc) - conn = - conn - |> put_req_header("x-hub-signature", "sha1=" <> signature) - |> put_req_header("content-type", "application/atom+xml") - |> post("/push/subscriptions/#{websub.id}", doc) + conn = + conn + |> put_req_header("x-hub-signature", "sha1=" <> signature) + |> put_req_header("content-type", "application/atom+xml") + |> post("/push/subscriptions/#{websub.id}", doc) - assert response(conn, 500) == "Error" + assert response(conn, 500) == "Error" - assert length(Repo.all(Activity)) == 0 + assert length(Repo.all(Activity)) == 0 + end + + test "it clears `unreachable` federation status of the sender", %{conn: conn} do + sender_url = "https://pleroma.soykaf.com" + Instances.set_unreachable(sender_url, Instances.reachability_datetime_threshold()) + refute Instances.reachable?(sender_url) + + websub = insert(:websub_client_subscription) + doc = "some stuff" + signature = Websub.sign(websub.secret, doc) + + conn = + conn + |> put_req_header("x-hub-signature", "sha1=" <> signature) + |> put_req_header("content-type", "application/atom+xml") + |> put_req_header("referer", sender_url) + |> post("/push/subscriptions/#{websub.id}", doc) + + assert response(conn, 200) == "OK" + assert Instances.reachable?(sender_url) + end end end From 9560abea102b8cd4927c9350bbd0a1a2f1800ea6 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 28 Jan 2019 11:03:52 +0300 Subject: [PATCH 09/12] [#534] Refactoring / tweaks per MR review. --- lib/pleroma/instances/instance.ex | 34 ++++++++----------- .../plugs/set_requester_reachable_plug.ex | 16 +++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 10 +----- .../activity_pub/activity_pub_controller.ex | 7 +--- lib/pleroma/web/controller_helper.ex | 5 --- lib/pleroma/web/ostatus/ostatus_controller.ex | 7 +--- lib/pleroma/web/salmon/salmon.ex | 3 +- lib/pleroma/web/websub/websub.ex | 3 +- lib/pleroma/web/websub/websub_controller.ex | 7 +--- .../20190123125546_create_instances.exs | 1 - 10 files changed, 36 insertions(+), 57 deletions(-) create mode 100644 lib/pleroma/plugs/set_requester_reachable_plug.ex diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 60e8d0e21..e3af4a8a7 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -13,7 +13,6 @@ defmodule Pleroma.Instances.Instance do schema "instances" do field(:host, :string) field(:unreachable_since, :naive_datetime) - field(:reachability_checked_at, :naive_datetime) timestamps() end @@ -22,7 +21,7 @@ defmodule Pleroma.Instances.Instance do def changeset(struct, params \\ %{}) do struct - |> cast(params, [:host, :unreachable_since, :reachability_checked_at]) + |> cast(params, [:host, :unreachable_since]) |> validate_required([:host]) |> unique_constraint(:host) end @@ -66,7 +65,7 @@ def set_reachable(url) when is_binary(url) do %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do {:ok, _instance} = existing_record - |> changeset(%{unreachable_since: nil, reachability_checked_at: DateTime.utc_now()}) + |> changeset(%{unreachable_since: nil}) |> Repo.update() end end @@ -80,27 +79,22 @@ def set_unreachable(url, unreachable_since) when is_binary(url) do host = host(url) existing_record = Repo.get_by(Instance, %{host: host}) - changes = %{ - unreachable_since: unreachable_since, - reachability_checked_at: NaiveDateTime.utc_now() - } + changes = %{unreachable_since: unreachable_since} - if existing_record do - update_changes = - if existing_record.unreachable_since && - NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt, - do: Map.delete(changes, :unreachable_since), - else: changes - - {:ok, _instance} = - existing_record - |> changeset(update_changes) - |> Repo.update() - else - {:ok, _instance} = + cond do + is_nil(existing_record) -> %Instance{} |> changeset(Map.put(changes, :host, host)) |> Repo.insert() + + existing_record.unreachable_since && + NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt -> + {:noop, existing_record} + + true -> + existing_record + |> changeset(changes) + |> Repo.update() end end diff --git a/lib/pleroma/plugs/set_requester_reachable_plug.ex b/lib/pleroma/plugs/set_requester_reachable_plug.ex new file mode 100644 index 000000000..88551be70 --- /dev/null +++ b/lib/pleroma/plugs/set_requester_reachable_plug.ex @@ -0,0 +1,16 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.SetRequesterReachablePlug do + import Plug.Conn + + def init(_), do: [] + + def call(%Plug.Conn{} = conn, _) do + with [referer] <- get_req_header(conn, "referer"), + do: Pleroma.Instances.set_reachable(referer) + + conn + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 4b34334a0..4232d6c0a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -722,15 +722,7 @@ def publish(actor, activity) do end) end - def publish_one(%{inbox: inbox} = activity) do - if Instances.reachable?(inbox) do - do_publish_one(activity) - else - {:error, :noop} - end - end - - defp do_publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do + def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do Logger.info("Federating #{id} to #{inbox}") host = URI.parse(inbox).host diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index dc353dff0..fadb038a2 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -18,8 +18,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do action_fallback(:errors) plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay]) + plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:inbox]) plug(:relay_active? when action in [:relay]) - plug(:set_requester_reachable when action in [:inbox]) def relay_active?(conn, _) do if Keyword.get(Application.get_env(:pleroma, :instance), :allow_relay) do @@ -291,9 +291,4 @@ def errors(conn, _e) do |> put_status(500) |> json("error") end - - defp set_requester_reachable(conn, _) do - Pleroma.Web.ControllerHelper.set_requester_reachable(conn) - conn - end end diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index 13cf1877f..14e3d19fd 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -10,9 +10,4 @@ def json_response(conn, status, json) do |> put_status(status) |> json(json) end - - def set_requester_reachable(conn) do - with [referer] <- get_req_header(conn, "referer"), - do: Pleroma.Instances.set_reachable(referer) - end end diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index a89f16b94..e483447ed 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -15,7 +15,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Pleroma.Web.ActivityPub.ActivityPub plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming]) - plug(:set_requester_reachable when action in [:salmon_incoming]) + plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:salmon_incoming]) action_fallback(:errors) @@ -203,9 +203,4 @@ def errors(conn, _) do |> put_status(500) |> text("Something went wrong") end - - defp set_requester_reachable(conn, _) do - Pleroma.Web.ControllerHelper.set_requester_reachable(conn) - conn - end end diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 17ca7a6e8..80023127c 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -168,8 +168,7 @@ def send_to_user(%{info: %{salmon: salmon}}, feed, poster), do: send_to_user(salmon, feed, poster) def send_to_user(url, feed, poster) when is_binary(url) do - with {:reachable, true} <- {:reachable, Instances.reachable?(url)}, - {:ok, %{status: code}} when code in 200..299 <- + with {:ok, %{status: code}} when code in 200..299 <- poster.( url, feed, diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index cbb7a5ac7..64eba7221 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -272,8 +272,7 @@ def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret}) d signature = sign(secret || "", xml) Logger.info(fn -> "Pushing #{topic} to #{callback}" end) - with {:reachable, true} <- {:reachable, Instances.reachable?(callback)}, - {:ok, %{status: code}} when code in 200..299 <- + with {:ok, %{status: code}} when code in 200..299 <- @httpoison.post( callback, xml, diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex index 02fe075d7..9da7e70a1 100644 --- a/lib/pleroma/web/websub/websub_controller.ex +++ b/lib/pleroma/web/websub/websub_controller.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Web.Websub.WebsubController do ] ) - plug(:set_requester_reachable when action in [:websub_incoming]) + plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:websub_incoming]) def websub_subscription_request(conn, %{"nickname" => nickname} = params) do user = User.get_cached_by_nickname(nickname) @@ -96,9 +96,4 @@ def websub_incoming(conn, %{"id" => id}) do |> send_resp(500, "Error") end end - - defp set_requester_reachable(conn, _) do - Pleroma.Web.ControllerHelper.set_requester_reachable(conn) - conn - end end diff --git a/priv/repo/migrations/20190123125546_create_instances.exs b/priv/repo/migrations/20190123125546_create_instances.exs index 4f5915fba..b527ad7ec 100644 --- a/priv/repo/migrations/20190123125546_create_instances.exs +++ b/priv/repo/migrations/20190123125546_create_instances.exs @@ -5,7 +5,6 @@ def change do create table(:instances) do add :host, :string add :unreachable_since, :naive_datetime - add :reachability_checked_at, :naive_datetime timestamps() end From 1d2f41642cfec5710055bcf8409778bb362beecb Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 28 Jan 2019 15:25:06 +0300 Subject: [PATCH 10/12] [#534] Various tweaks. Tests for Instances and Instance. --- config/config.exs | 2 +- lib/pleroma/instances.ex | 13 ++- lib/pleroma/instances/instance.ex | 28 ++--- lib/pleroma/web/activity_pub/activity_pub.ex | 4 +- lib/pleroma/web/salmon/salmon.ex | 4 - lib/pleroma/web/websub/websub.ex | 10 +- test/support/factory.ex | 7 ++ test/support/http_request_mock.ex | 28 +++++ .../activity_pub_controller_test.exs | 4 +- test/web/activity_pub/activity_pub_test.exs | 43 ++++++- test/web/federator_test.exs | 4 +- test/web/instances/instance_test.exs | 107 ++++++++++++++++++ test/web/instances/instances_test.exs | 94 +++++++++++++++ test/web/ostatus/ostatus_controller_test.exs | 2 +- test/web/websub/websub_controller_test.exs | 2 +- 15 files changed, 312 insertions(+), 40 deletions(-) create mode 100644 test/web/instances/instance_test.exs create mode 100644 test/web/instances/instances_test.exs diff --git a/config/config.exs b/config/config.exs index 7a1a875c9..c0a936b17 100644 --- a/config/config.exs +++ b/config/config.exs @@ -125,7 +125,7 @@ banner_upload_limit: 4_000_000, registrations_open: true, federating: true, - federation_reachability_timeout_days: 90, + federation_reachability_timeout_days: 7, allow_relay: true, rewrite_policy: Pleroma.Web.ActivityPub.MRF.NoOpPolicy, public: true, diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex index 0b08f0eb8..5e107f4c9 100644 --- a/lib/pleroma/instances.ex +++ b/lib/pleroma/instances.ex @@ -3,14 +3,17 @@ defmodule Pleroma.Instances do @adapter Pleroma.Instances.Instance - defdelegate filter_reachable(urls), to: @adapter - defdelegate reachable?(url), to: @adapter - defdelegate set_reachable(url), to: @adapter - defdelegate set_unreachable(url, unreachable_since \\ nil), to: @adapter + defdelegate filter_reachable(urls_or_hosts), to: @adapter + defdelegate reachable?(url_or_host), to: @adapter + defdelegate set_reachable(url_or_host), to: @adapter + defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: @adapter + + def set_consistently_unreachable(url_or_host), + do: set_unreachable(url_or_host, reachability_datetime_threshold()) def reachability_datetime_threshold do federation_reachability_timeout_days = - Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 90 + Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 0 if federation_reachability_timeout_days > 0 do NaiveDateTime.add( diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index e3af4a8a7..a87590d8b 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Instances.Instance do timestamps() end - defdelegate host(url), to: Instances + defdelegate host(url_or_host), to: Instances def changeset(struct, params \\ %{}) do struct @@ -28,9 +28,9 @@ def changeset(struct, params \\ %{}) do def filter_reachable([]), do: [] - def filter_reachable(urls) when is_list(urls) do + def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do hosts = - urls + urls_or_hosts |> Enum.map(&(&1 && host(&1))) |> Enum.filter(&(to_string(&1) != "")) @@ -44,14 +44,14 @@ def filter_reachable(urls) when is_list(urls) do ) ) - Enum.filter(urls, &(&1 && host(&1) not in unreachable_hosts)) + Enum.filter(urls_or_hosts, &(&1 && host(&1) not in unreachable_hosts)) end - def reachable?(url) when is_binary(url) do + def reachable?(url_or_host) when is_binary(url_or_host) do !Repo.one( from(i in Instance, where: - i.host == ^host(url) and + i.host == ^host(url_or_host) and i.unreachable_since <= ^Instances.reachability_datetime_threshold(), select: true ) @@ -60,8 +60,8 @@ def reachable?(url) when is_binary(url) do def reachable?(_), do: true - def set_reachable(url) when is_binary(url) do - with host <- host(url), + def set_reachable(url_or_host) when is_binary(url_or_host) do + with host <- host(url_or_host), %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do {:ok, _instance} = existing_record @@ -70,13 +70,13 @@ def set_reachable(url) when is_binary(url) do end end - def set_reachable(_), do: {0, :noop} + def set_reachable(_), do: {:error, nil} - def set_unreachable(url, unreachable_since \\ nil) + def set_unreachable(url_or_host, unreachable_since \\ nil) - def set_unreachable(url, unreachable_since) when is_binary(url) do + def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do unreachable_since = unreachable_since || DateTime.utc_now() - host = host(url) + host = host(url_or_host) existing_record = Repo.get_by(Instance, %{host: host}) changes = %{unreachable_since: unreachable_since} @@ -89,7 +89,7 @@ def set_unreachable(url, unreachable_since) when is_binary(url) do existing_record.unreachable_since && NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt -> - {:noop, existing_record} + {:ok, existing_record} true -> existing_record @@ -98,5 +98,5 @@ def set_unreachable(url, unreachable_since) when is_binary(url) do end end - def set_unreachable(_, _), do: {0, :noop} + def set_unreachable(_, _), do: {:error, nil} end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 4232d6c0a..6cad02da6 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -750,9 +750,9 @@ def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do Instances.set_reachable(inbox) result else - e -> + {_post_result, response} -> Instances.set_unreachable(inbox) - e + {:error, response} end end diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 80023127c..e96455423 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -181,10 +181,6 @@ def send_to_user(url, feed, poster) when is_binary(url) do Logger.debug(fn -> "Pushed to #{url}, code #{code}" end) :ok else - {:reachable, false} -> - Logger.debug(fn -> "Pushing Salmon to #{url} skipped as marked unreachable)" end) - :noop - e -> Instances.set_unreachable(url) Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 64eba7221..abe148270 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -286,14 +286,10 @@ def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret}) d Logger.info(fn -> "Pushed to #{callback}, code #{code}" end) {:ok, code} else - {:reachable, false} -> - Logger.debug(fn -> "Pushing to #{callback} skipped as marked unreachable)" end) - {:error, :noop} - - e -> + {_post_result, response} -> Instances.set_unreachable(callback) - Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(e)}" end) - {:error, e} + Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(response)}" end) + {:error, response} end end end diff --git a/test/support/factory.ex b/test/support/factory.ex index 964b2b61c..0c21093ce 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -220,4 +220,11 @@ def oauth_app_factory do client_secret: "aaa;/&bbb" } end + + def instance_factory do + %Pleroma.Instances.Instance{ + host: "domain.com", + unreachable_since: nil + } + end end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index e4279e14d..3d6efd52c 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -653,6 +653,14 @@ def get("https://social.heldscal.la/user/23211", _, _, Accept: "application/acti {:ok, Tesla.Mock.json(%{"id" => "https://social.heldscal.la/user/23211"}, status: 200)} end + def get("http://404.site" <> _, _, _, _) do + {:ok, + %Tesla.Env{ + status: 404, + body: "" + }} + end + def get(url, query, body, headers) do {:error, "Not implemented the mock response for get #{inspect(url)}, #{query}, #{inspect(body)}, #{ @@ -673,6 +681,26 @@ def post("http://example.org/needs_refresh", _, _, _) do }} end + def post("http://200.site" <> _, _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: "" + }} + end + + def post("http://connrefused.site" <> _, _, _, _) do + {:error, :connrefused} + end + + def post("http://404.site" <> _, _, _, _) do + {:ok, + %Tesla.Env{ + status: 404, + body: "" + }} + end + def post(url, _query, _body, _headers) do {:error, "Not implemented the mock response for post #{inspect(url)}"} end diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 1b704330f..eca5c134d 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -146,7 +146,7 @@ test "it inserts an incoming activity into the database", %{conn: conn} do test "it clears `unreachable` federation status of the sender", %{conn: conn} do sender_url = "https://pleroma.soykaf.com" - Instances.set_unreachable(sender_url, Instances.reachability_datetime_threshold()) + Instances.set_consistently_unreachable(sender_url) refute Instances.reachable?(sender_url) data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() @@ -211,7 +211,7 @@ test "it returns a note activity in a collection", %{conn: conn} do test "it clears `unreachable` federation status of the sender", %{conn: conn} do sender_host = "pleroma.soykaf.com" - Instances.set_unreachable(sender_host, Instances.reachability_datetime_threshold()) + Instances.set_consistently_unreachable(sender_host) refute Instances.reachable?(sender_host) user = insert(:user) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 18f094379..d517c7aa7 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -7,11 +7,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI - alias Pleroma.{Activity, Object, User} + alias Pleroma.{Activity, Object, User, Instances} alias Pleroma.Builders.ActivityBuilder import Pleroma.Factory import Tesla.Mock + import Mock setup do mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -659,6 +660,46 @@ test "returned pinned statuses" do assert 3 = length(activities) end + describe "publish_one/1" do + test_with_mock "it calls `Instances.set_unreachable` on target inbox on non-2xx HTTP response code", + Instances, + [:passthrough], + [] do + actor = insert(:user) + inbox = "http://404.site/users/nick1/inbox" + + assert {:error, _} = + ActivityPub.publish_one(%{inbox: inbox, json: "{}", actor: actor, id: 1}) + + assert called(Instances.set_unreachable(inbox)) + end + + test_with_mock "it calls `Instances.set_unreachable` on target inbox on request error of any kind", + Instances, + [:passthrough], + [] do + actor = insert(:user) + inbox = "http://connrefused.site/users/nick1/inbox" + + assert {:error, _} = + ActivityPub.publish_one(%{inbox: inbox, json: "{}", actor: actor, id: 1}) + + assert called(Instances.set_unreachable(inbox)) + end + + test_with_mock "it does NOT call `Instances.set_unreachable` if target is reachable", + Instances, + [:passthrough], + [] do + actor = insert(:user) + inbox = "http://200.site/users/nick1/inbox" + + assert {:ok, _} = ActivityPub.publish_one(%{inbox: inbox, json: "{}", actor: actor, id: 1}) + + refute called(Instances.set_unreachable(inbox)) + end + end + def data_uri do File.read!("test/fixtures/avatar_data_uri") end diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index f4234aea8..c6d10ef78 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -128,7 +128,7 @@ test "with relays deactivated, it does not publish to the relay", %{ callback: "https://pleroma2.soykaf.com/cb" }) - Instances.set_unreachable(sub1.callback, Instances.reachability_datetime_threshold()) + Instances.set_consistently_unreachable(sub1.callback) {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI"}) @@ -158,7 +158,7 @@ test "with relays deactivated, it does not publish to the relay", %{ info: %{salmon: "https://domain2.com/salmon"} }) - Instances.set_unreachable("domain.com", Instances.reachability_datetime_threshold()) + Instances.set_consistently_unreachable("domain.com") {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"}) diff --git a/test/web/instances/instance_test.exs b/test/web/instances/instance_test.exs new file mode 100644 index 000000000..a158c0a42 --- /dev/null +++ b/test/web/instances/instance_test.exs @@ -0,0 +1,107 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Instances.InstanceTest do + alias Pleroma.Repo + alias Pleroma.Instances.Instance + + use Pleroma.DataCase + + import Pleroma.Factory + + setup_all do + config_path = [:instance, :federation_reachability_timeout_days] + initial_setting = Pleroma.Config.get(config_path) + + Pleroma.Config.put(config_path, 1) + on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) + + :ok + end + + describe "set_reachable/1" do + test "clears `unreachable_since` of existing matching Instance record having non-nil `unreachable_since`" do + instance = insert(:instance, unreachable_since: NaiveDateTime.utc_now()) + + assert {:ok, instance} = Instance.set_reachable(instance.host) + refute instance.unreachable_since + end + + test "keeps nil `unreachable_since` of existing matching Instance record having nil `unreachable_since`" do + instance = insert(:instance, unreachable_since: nil) + + assert {:ok, instance} = Instance.set_reachable(instance.host) + refute instance.unreachable_since + end + + test "does NOT create an Instance record in case of no existing matching record" do + host = "domain.org" + assert nil == Instance.set_reachable(host) + + assert [] = Repo.all(Ecto.Query.from(i in Instance)) + assert Instance.reachable?(host) + end + end + + describe "set_unreachable/1" do + test "creates new record having `unreachable_since` to current time if record does not exist" do + assert {:ok, instance} = Instance.set_unreachable("https://domain.com/path") + + instance = Repo.get(Instance, instance.id) + assert instance.unreachable_since + assert "domain.com" == instance.host + end + + test "sets `unreachable_since` of existing record having nil `unreachable_since`" do + instance = insert(:instance, unreachable_since: nil) + refute instance.unreachable_since + + assert {:ok, _} = Instance.set_unreachable(instance.host) + + instance = Repo.get(Instance, instance.id) + assert instance.unreachable_since + end + + test "does NOT modify `unreachable_since` value of existing record in case it's present" do + instance = + insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10)) + + assert instance.unreachable_since + initial_value = instance.unreachable_since + + assert {:ok, _} = Instance.set_unreachable(instance.host) + + instance = Repo.get(Instance, instance.id) + assert initial_value == instance.unreachable_since + end + end + + describe "set_unreachable/2" do + test "sets `unreachable_since` value of existing record in case it's newer than supplied value" do + instance = + insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10)) + + assert instance.unreachable_since + + past_value = NaiveDateTime.add(NaiveDateTime.utc_now(), -100) + assert {:ok, _} = Instance.set_unreachable(instance.host, past_value) + + instance = Repo.get(Instance, instance.id) + assert past_value == instance.unreachable_since + end + + test "does NOT modify `unreachable_since` value of existing record in case it's equal to or older than supplied value" do + instance = + insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10)) + + assert instance.unreachable_since + initial_value = instance.unreachable_since + + assert {:ok, _} = Instance.set_unreachable(instance.host, NaiveDateTime.utc_now()) + + instance = Repo.get(Instance, instance.id) + assert initial_value == instance.unreachable_since + end + end +end diff --git a/test/web/instances/instances_test.exs b/test/web/instances/instances_test.exs new file mode 100644 index 000000000..a2fdf1019 --- /dev/null +++ b/test/web/instances/instances_test.exs @@ -0,0 +1,94 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.InstancesTest do + alias Pleroma.Instances + + use Pleroma.DataCase + + setup_all do + config_path = [:instance, :federation_reachability_timeout_days] + initial_setting = Pleroma.Config.get(config_path) + + Pleroma.Config.put(config_path, 1) + on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) + + :ok + end + + describe "reachable?/1" do + test "returns `true` for host / url with unknown reachability status" do + assert Instances.reachable?("unknown.site") + assert Instances.reachable?("http://unknown.site") + end + + test "returns `false` for host / url marked unreachable for at least `reachability_datetime_threshold()`" do + host = "consistently-unreachable.name" + Instances.set_consistently_unreachable(host) + + refute Instances.reachable?(host) + refute Instances.reachable?("http://#{host}/path") + end + + test "returns `true` for host / url marked unreachable for less than `reachability_datetime_threshold()`" do + url = "http://eventually-unreachable.name/path" + + Instances.set_unreachable(url) + + assert Instances.reachable?(url) + assert Instances.reachable?(URI.parse(url).host) + end + end + + describe "filter_reachable/1" do + test "keeps only reachable elements of supplied list" do + host = "consistently-unreachable.name" + url1 = "http://eventually-unreachable.com/path" + url2 = "http://domain.com/path" + + Instances.set_consistently_unreachable(host) + Instances.set_unreachable(url1) + + assert [url1, url2] == Instances.filter_reachable([host, url1, url2]) + end + end + + describe "set_reachable/1" do + test "sets unreachable url or host reachable" do + host = "domain.com" + Instances.set_consistently_unreachable(host) + refute Instances.reachable?(host) + + Instances.set_reachable(host) + assert Instances.reachable?(host) + end + + test "keeps reachable url or host reachable" do + url = "https://site.name?q=" + assert Instances.reachable?(url) + + Instances.set_reachable(url) + assert Instances.reachable?(url) + end + end + + describe "set_consistently_unreachable/1" do + test "sets reachable url or host unreachable" do + url = "http://domain.com?q=" + assert Instances.reachable?(url) + + Instances.set_consistently_unreachable(url) + refute Instances.reachable?(url) + end + + test "keeps unreachable url or host unreachable" do + host = "site.name" + Instances.set_consistently_unreachable(host) + refute Instances.reachable?(host) + + Instances.set_consistently_unreachable(host) + refute Instances.reachable?(host) + end + end +end diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index ca447aa5d..ad9bc418a 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -62,7 +62,7 @@ test "decodes a salmon with a changed magic key", %{conn: conn} do test "it clears `unreachable` federation status of the sender", %{conn: conn} do sender_url = "https://pleroma.soykaf.com" - Instances.set_unreachable(sender_url, Instances.reachability_datetime_threshold()) + Instances.set_consistently_unreachable(sender_url) refute Instances.reachable?(sender_url) user = insert(:user) diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs index c445ed676..cb19d6fe6 100644 --- a/test/web/websub/websub_controller_test.exs +++ b/test/web/websub/websub_controller_test.exs @@ -85,7 +85,7 @@ test "rejects incoming feed updates with the wrong signature", %{conn: conn} do test "it clears `unreachable` federation status of the sender", %{conn: conn} do sender_url = "https://pleroma.soykaf.com" - Instances.set_unreachable(sender_url, Instances.reachability_datetime_threshold()) + Instances.set_consistently_unreachable(sender_url) refute Instances.reachable?(sender_url) websub = insert(:websub_client_subscription) From 92753b0cd9cfcdc5edb64a5e55ad27f73079f9e0 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 29 Jan 2019 13:12:28 +0300 Subject: [PATCH 11/12] [#534] Made federation push sender be determined basing on content instead of `referer` header. Updated tests. --- .../plugs/set_requester_reachable_plug.ex | 16 -------------- lib/pleroma/reverse_proxy.ex | 3 +-- lib/pleroma/web/activity_pub/activity_pub.ex | 3 +-- .../activity_pub/activity_pub_controller.ex | 11 +++++++++- lib/pleroma/web/ostatus/ostatus.ex | 3 +++ lib/pleroma/web/ostatus/ostatus_controller.ex | 1 - lib/pleroma/web/salmon/salmon.ex | 5 +---- lib/pleroma/web/websub/websub.ex | 3 +-- lib/pleroma/web/websub/websub_controller.ex | 2 -- .../activity_pub_controller_test.exs | 16 ++++++-------- test/web/instances/instances_test.exs | 18 +++++++++++++++ .../delete_handling_test.exs | 7 ++++++ test/web/ostatus/ostatus_controller_test.exs | 20 +---------------- test/web/ostatus/ostatus_test.exs | 18 ++++++++++++++- test/web/websub/websub_controller_test.exs | 22 +------------------ 15 files changed, 68 insertions(+), 80 deletions(-) delete mode 100644 lib/pleroma/plugs/set_requester_reachable_plug.ex diff --git a/lib/pleroma/plugs/set_requester_reachable_plug.ex b/lib/pleroma/plugs/set_requester_reachable_plug.ex deleted file mode 100644 index 88551be70..000000000 --- a/lib/pleroma/plugs/set_requester_reachable_plug.ex +++ /dev/null @@ -1,16 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Plugs.SetRequesterReachablePlug do - import Plug.Conn - - def init(_), do: [] - - def call(%Plug.Conn{} = conn, _) do - with [referer] <- get_req_header(conn, "referer"), - do: Pleroma.Instances.set_reachable(referer) - - conn - end -end diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index d8b17212b..a25b5ea4e 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -3,8 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy do - @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since if-unmodified-since) ++ - ~w(if-none-match if-range range referer) + @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since if-unmodified-since if-none-match if-range range) @resp_cache_headers ~w(etag date last-modified cache-control) @keep_resp_headers @resp_cache_headers ++ ~w(content-type content-disposition content-encoding content-range accept-ranges vary) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 22c7824fa..4016808e8 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -784,8 +784,7 @@ def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do [ {"Content-Type", "application/activity+json"}, {"signature", signature}, - {"digest", digest}, - {"referer", Pleroma.Web.Endpoint.url()} + {"digest", digest} ] ) do Instances.set_reachable(inbox) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index fadb038a2..4dea6ab83 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -18,7 +18,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do action_fallback(:errors) plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay]) - plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:inbox]) + plug(:set_requester_reachable when action in [:inbox]) plug(:relay_active? when action in [:relay]) def relay_active?(conn, _) do @@ -291,4 +291,13 @@ def errors(conn, _e) do |> put_status(500) |> json("error") end + + defp set_requester_reachable(%Plug.Conn{} = conn, _) do + with actor <- conn.params["actor"], + true <- is_binary(actor) do + Pleroma.Instances.set_reachable(actor) + end + + conn + end end diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index a3155b79d..a20ca17bb 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -48,6 +48,9 @@ def remote_follow_path do def handle_incoming(xml_string) do with doc when doc != :error <- parse_document(xml_string) do + with {:ok, actor_user} <- find_make_or_update_user(doc), + do: Pleroma.Instances.set_reachable(actor_user.ap_id) + entries = :xmerl_xpath.string('//entry', doc) activities = diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 9392a97f0..302ff38a4 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -14,7 +14,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Pleroma.Web.ActivityPub.ActivityPub plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming]) - plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:salmon_incoming]) action_fallback(:errors) diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index e96455423..07ca42a5f 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -172,10 +172,7 @@ def send_to_user(url, feed, poster) when is_binary(url) do poster.( url, feed, - [ - {"Content-Type", "application/magic-envelope+xml"}, - {"referer", Pleroma.Web.Endpoint.url()} - ] + [{"Content-Type", "application/magic-envelope+xml"}] ) do Instances.set_reachable(url) Logger.debug(fn -> "Pushed to #{url}, code #{code}" end) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index abe148270..8f7d53b03 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -278,8 +278,7 @@ def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret}) d xml, [ {"Content-Type", "application/atom+xml"}, - {"X-Hub-Signature", "sha1=#{signature}"}, - {"referer", Pleroma.Web.Endpoint.url()} + {"X-Hub-Signature", "sha1=#{signature}"} ] ) do Instances.set_reachable(callback) diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex index 9da7e70a1..a92dfe87b 100644 --- a/lib/pleroma/web/websub/websub_controller.ex +++ b/lib/pleroma/web/websub/websub_controller.ex @@ -20,8 +20,6 @@ defmodule Pleroma.Web.Websub.WebsubController do ] ) - plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:websub_incoming]) - def websub_subscription_request(conn, %{"nickname" => nickname} = params) do user = User.get_cached_by_nickname(nickname) diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index eca5c134d..d3dd160dd 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -145,17 +145,16 @@ test "it inserts an incoming activity into the database", %{conn: conn} do end test "it clears `unreachable` federation status of the sender", %{conn: conn} do - sender_url = "https://pleroma.soykaf.com" + data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() + + sender_url = data["actor"] Instances.set_consistently_unreachable(sender_url) refute Instances.reachable?(sender_url) - data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() - conn = conn |> assign(:valid_signature, true) |> put_req_header("content-type", "application/activity+json") - |> put_req_header("referer", sender_url) |> post("/inbox", data) assert "ok" == json_response(conn, 200) @@ -210,10 +209,6 @@ test "it returns a note activity in a collection", %{conn: conn} do end test "it clears `unreachable` federation status of the sender", %{conn: conn} do - sender_host = "pleroma.soykaf.com" - Instances.set_consistently_unreachable(sender_host) - refute Instances.reachable?(sender_host) - user = insert(:user) data = @@ -221,11 +216,14 @@ test "it clears `unreachable` federation status of the sender", %{conn: conn} do |> Poison.decode!() |> Map.put("bcc", [user.ap_id]) + sender_host = URI.parse(data["actor"]).host + Instances.set_consistently_unreachable(sender_host) + refute Instances.reachable?(sender_host) + conn = conn |> assign(:valid_signature, true) |> put_req_header("content-type", "application/activity+json") - |> put_req_header("referer", "https://#{sender_host}") |> post("/users/#{user.nickname}/inbox", data) assert "ok" == json_response(conn, 200) diff --git a/test/web/instances/instances_test.exs b/test/web/instances/instances_test.exs index a2fdf1019..adb8560a7 100644 --- a/test/web/instances/instances_test.exs +++ b/test/web/instances/instances_test.exs @@ -39,6 +39,11 @@ test "returns `true` for host / url marked unreachable for less than `reachabili assert Instances.reachable?(url) assert Instances.reachable?(URI.parse(url).host) end + + test "returns true on non-binary input" do + assert Instances.reachable?(nil) + assert Instances.reachable?(1) + end end describe "filter_reachable/1" do @@ -71,6 +76,19 @@ test "keeps reachable url or host reachable" do Instances.set_reachable(url) assert Instances.reachable?(url) end + + test "returns error status on non-binary input" do + assert {:error, _} = Instances.set_reachable(nil) + assert {:error, _} = Instances.set_reachable(1) + end + end + + # Note: implementation-specific (e.g. Instance) details of set_unreachable/1 should be tested in implementation-specific tests + describe "set_unreachable/1" do + test "returns error status on non-binary input" do + assert {:error, _} = Instances.set_unreachable(nil) + assert {:error, _} = Instances.set_unreachable(1) + end end describe "set_consistently_unreachable/1" do diff --git a/test/web/ostatus/incoming_documents/delete_handling_test.exs b/test/web/ostatus/incoming_documents/delete_handling_test.exs index c8fbff6cc..d97cd79f4 100644 --- a/test/web/ostatus/incoming_documents/delete_handling_test.exs +++ b/test/web/ostatus/incoming_documents/delete_handling_test.exs @@ -2,9 +2,16 @@ defmodule Pleroma.Web.OStatus.DeleteHandlingTest do use Pleroma.DataCase import Pleroma.Factory + import Tesla.Mock + alias Pleroma.{Repo, Activity, Object} alias Pleroma.Web.OStatus + setup do + mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + describe "deletions" do test "it removes the mentioned activity" do note = insert(:note_activity) diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index cba12b3f7..3145ca9a1 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -5,7 +5,7 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory - alias Pleroma.{User, Repo, Object, Instances} + alias Pleroma.{User, Repo, Object} alias Pleroma.Web.CommonAPI alias Pleroma.Web.OStatus.ActivityRepresenter @@ -59,24 +59,6 @@ test "decodes a salmon with a changed magic key", %{conn: conn} do assert response(conn, 200) end - - test "it clears `unreachable` federation status of the sender", %{conn: conn} do - sender_url = "https://pleroma.soykaf.com" - Instances.set_consistently_unreachable(sender_url) - refute Instances.reachable?(sender_url) - - user = insert(:user) - salmon = File.read!("test/fixtures/salmon.xml") - - conn = - conn - |> put_req_header("content-type", "application/atom+xml") - |> put_req_header("referer", sender_url) - |> post("/users/#{user.nickname}/salmon", salmon) - - assert response(conn, 200) - assert Instances.reachable?(sender_url) - end end test "gets a feed", %{conn: conn} do diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 403cc7095..0c63dd84d 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -6,7 +6,7 @@ defmodule Pleroma.Web.OStatusTest do use Pleroma.DataCase alias Pleroma.Web.OStatus alias Pleroma.Web.XML - alias Pleroma.{Object, Repo, User, Activity} + alias Pleroma.{Object, Repo, User, Activity, Instances} import Pleroma.Factory import ExUnit.CaptureLog @@ -311,6 +311,22 @@ test "handle incoming unfollows with existing follow" do refute User.following?(follower, followed) end + test "it clears `unreachable` federation status of the sender" do + incoming_reaction_xml = File.read!("test/fixtures/share-gs.xml") + doc = XML.parse_document(incoming_reaction_xml) + actor_uri = XML.string_from_xpath("//author/uri[1]", doc) + reacted_to_author_uri = XML.string_from_xpath("//author/uri[2]", doc) + + Instances.set_consistently_unreachable(actor_uri) + Instances.set_consistently_unreachable(reacted_to_author_uri) + refute Instances.reachable?(actor_uri) + refute Instances.reachable?(reacted_to_author_uri) + + {:ok, _} = OStatus.handle_incoming(incoming_reaction_xml) + assert Instances.reachable?(actor_uri) + refute Instances.reachable?(reacted_to_author_uri) + end + describe "new remote user creation" do test "returns local users" do local_user = insert(:user) diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs index cb19d6fe6..6492df2a0 100644 --- a/test/web/websub/websub_controller_test.exs +++ b/test/web/websub/websub_controller_test.exs @@ -6,7 +6,7 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory alias Pleroma.Web.Websub.WebsubClientSubscription - alias Pleroma.{Repo, Activity, Instances} + alias Pleroma.{Repo, Activity} alias Pleroma.Web.Websub test "websub subscription request", %{conn: conn} do @@ -82,25 +82,5 @@ test "rejects incoming feed updates with the wrong signature", %{conn: conn} do assert length(Repo.all(Activity)) == 0 end - - test "it clears `unreachable` federation status of the sender", %{conn: conn} do - sender_url = "https://pleroma.soykaf.com" - Instances.set_consistently_unreachable(sender_url) - refute Instances.reachable?(sender_url) - - websub = insert(:websub_client_subscription) - doc = "some stuff" - signature = Websub.sign(websub.secret, doc) - - conn = - conn - |> put_req_header("x-hub-signature", "sha1=" <> signature) - |> put_req_header("content-type", "application/atom+xml") - |> put_req_header("referer", sender_url) - |> post("/push/subscriptions/#{websub.id}", doc) - - assert response(conn, 200) == "OK" - assert Instances.reachable?(sender_url) - end end end From a7c55c61bb581ddbd8159261e167b08486547b20 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 31 Jan 2019 23:23:53 +0100 Subject: [PATCH 12/12] docs/Pleroma-API.md: Add missing token field documentation [ci skip] --- docs/Pleroma-API.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/Pleroma-API.md b/docs/Pleroma-API.md index 3ebea565c..e1448d3f0 100644 --- a/docs/Pleroma-API.md +++ b/docs/Pleroma-API.md @@ -52,6 +52,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi * `confirm` * `captcha_solution`: optional, contains provider-specific captcha solution, * `captcha_token`: optional, contains provider-specific captcha token + * `token`: invite token required when the registerations aren't public. * Response: JSON. Returns a user object on success, otherwise returns `{"error": "error_msg"}` * Example response: ```