From 77e39e6aae0852df13b9576a081f30da13d7ec5b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 23 Dec 2020 15:06:20 -0600 Subject: [PATCH 1/8] Create dir for EmojiStealPolicy automatically --- lib/pleroma/application.ex | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index bd568d858..a1655721a 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -114,6 +114,9 @@ def start(_type, _args) do set_postgres_server_version() + # Requires Config.TransferTask so ConfigDB values are loaded + steal_emoji_policy_setup() + result end @@ -300,4 +303,20 @@ def limiters_setup do [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.MediaProxy] |> Enum.each(&ConcurrentLimiter.new(&1, 1, 0)) end + + @spec steal_emoji_policy_setup() :: :ok + def steal_emoji_policy_setup() do + with true <- + Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy in Config.get!([:mrf, :policies]) do + path = + [:instance, :static_dir] + |> Config.get!() + |> Path.join("emoji/steal") + + if !File.exists?(path), do: File.mkdir_p!(path) + else + _ -> + :ok + end + end end From 72aeb2e73b77611504e1cd524a9cb47faef6816f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 23 Dec 2020 15:16:58 -0600 Subject: [PATCH 2/8] Mark private --- lib/pleroma/application.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index a1655721a..e3edd05ca 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -305,7 +305,7 @@ def limiters_setup do end @spec steal_emoji_policy_setup() :: :ok - def steal_emoji_policy_setup() do + defp steal_emoji_policy_setup() do with true <- Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy in Config.get!([:mrf, :policies]) do path = From e02889edb29ca5631c346b59c299907e9f7b0161 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 23 Dec 2020 15:19:08 -0600 Subject: [PATCH 3/8] Add MRFs to the list of things that may need a soft reboot --- lib/pleroma/config/transfer_task.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index a0d7b7d71..8a6d66891 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -28,7 +28,8 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, Pleroma.Captcha, [:seconds_valid]}, {:pleroma, Pleroma.Upload, [:proxy_remote]}, {:pleroma, :instance, [:upload_limit]}, - {:pleroma, :gopher, [:enabled]} + {:pleroma, :gopher, [:enabled]}, + {:pleroma, :mrf, [:policies]} ] def start_link(restart_pleroma? \\ true) do From aafd7b44ceaf50788f12061ee88071a288155b95 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 24 Dec 2020 20:27:28 +0300 Subject: [PATCH 4/8] check dir existence in policy --- lib/pleroma/application.ex | 19 ----------- .../activity_pub/mrf/steal_emoji_policy.ex | 34 ++++++++++++------- 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index e3edd05ca..bd568d858 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -114,9 +114,6 @@ def start(_type, _args) do set_postgres_server_version() - # Requires Config.TransferTask so ConfigDB values are loaded - steal_emoji_policy_setup() - result end @@ -303,20 +300,4 @@ def limiters_setup do [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.MediaProxy] |> Enum.each(&ConcurrentLimiter.new(&1, 1, 0)) end - - @spec steal_emoji_policy_setup() :: :ok - defp steal_emoji_policy_setup() do - with true <- - Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy in Config.get!([:mrf, :policies]) do - path = - [:instance, :static_dir] - |> Config.get!() - |> Path.join("emoji/steal") - - if !File.exists?(path), do: File.mkdir_p!(path) - else - _ -> - :ok - end - end end diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index 2858af9eb..eabee6542 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -14,18 +14,12 @@ defp remote_host?(host), do: host != Config.get([Pleroma.Web.Endpoint, :url, :ho defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], []) - defp steal_emoji({shortcode, url}) do + defp steal_emoji({shortcode, url}, emoji_dir_path) do url = Pleroma.Web.MediaProxy.url(url) {:ok, response} = Pleroma.HTTP.get(url) size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000) if byte_size(response.body) <= size_limit do - emoji_dir_path = - Config.get( - [:mrf_steal_emoji, :path], - Path.join(Config.get([:instance, :static_dir]), "emoji/stolen") - ) - extension = url |> URI.parse() @@ -35,11 +29,9 @@ defp steal_emoji({shortcode, url}) do file_path = Path.join([emoji_dir_path, shortcode <> (extension || ".png")]) - try do - :ok = File.write(file_path, response.body) - + with :ok <- File.write(file_path, response.body) do shortcode - rescue + else e -> Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}") nil @@ -66,6 +58,16 @@ def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = messa if remote_host?(host) and accept_host?(host) do installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) + emoji_dir_path = + Config.get( + [:mrf_steal_emoji, :path], + Path.join(Config.get([:instance, :static_dir]), "emoji/stolen") + ) + + if not Config.get([:mrf_steal_emoji, :dir_exists?], false) do + create_dir(emoji_dir_path) + end + new_emojis = foreign_emojis |> Enum.filter(fn {shortcode, _url} -> shortcode not in installed_emoji end) @@ -76,7 +78,7 @@ def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = messa !reject_emoji? end) - |> Enum.map(&steal_emoji(&1)) + |> Enum.map(&steal_emoji(&1, emoji_dir_path)) |> Enum.filter(& &1) if !Enum.empty?(new_emojis) do @@ -94,4 +96,12 @@ def filter(message), do: {:ok, message} def describe do {:ok, %{}} end + + defp create_dir(path) do + if not File.exists?(path) do + File.mkdir_p!(path) + end + + Config.put([:mrf_steal_emoji, :dir_exists?], true) + end end From 7bfb041658708f002e5da8f7b911b6011169dfef Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 25 Dec 2020 11:30:36 +0300 Subject: [PATCH 5/8] insreasing test coverage for StealEmojiPolicy --- .../activity_pub/mrf/steal_emoji_policy.ex | 64 +++++------ .../mrf/steal_emoji_policy_test.exs | 105 ++++++++++++------ 2 files changed, 104 insertions(+), 65 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index eabee6542..0311ca433 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -10,52 +10,53 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do @moduledoc "Detect new emojis by their shortcode and steals them" @behaviour Pleroma.Web.ActivityPub.MRF - defp remote_host?(host), do: host != Config.get([Pleroma.Web.Endpoint, :url, :host]) - defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], []) defp steal_emoji({shortcode, url}, emoji_dir_path) do url = Pleroma.Web.MediaProxy.url(url) - {:ok, response} = Pleroma.HTTP.get(url) - size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000) - if byte_size(response.body) <= size_limit do - extension = - url - |> URI.parse() - |> Map.get(:path) - |> Path.basename() - |> Path.extname() + with {:ok, %{status: status} = response} when status in 200..299 <- Pleroma.HTTP.get(url) do + size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000) - file_path = Path.join([emoji_dir_path, shortcode <> (extension || ".png")]) + if byte_size(response.body) <= size_limit do + extension = + url + |> URI.parse() + |> Map.get(:path) + |> Path.basename() + |> Path.extname() - with :ok <- File.write(file_path, response.body) do - shortcode + file_path = Path.join(emoji_dir_path, shortcode <> (extension || ".png")) + + case File.write(file_path, response.body) do + :ok -> + shortcode + + e -> + Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}") + nil + end else - e -> - Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}") - nil + Logger.debug( + "MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{ + size_limit + } B)" + ) + + nil end else - Logger.debug( - "MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{ - size_limit - } B)" - ) - - nil + e -> + Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}") + nil end - rescue - e -> - Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}") - nil end @impl true def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = message) do host = URI.parse(actor).host - if remote_host?(host) and accept_host?(host) do + if host != Pleroma.Web.Endpoint.host() and accept_host?(host) do installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) emoji_dir_path = @@ -70,10 +71,11 @@ def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = messa new_emojis = foreign_emojis - |> Enum.filter(fn {shortcode, _url} -> shortcode not in installed_emoji end) + |> Enum.reject(fn {shortcode, _url} -> shortcode in installed_emoji end) |> Enum.filter(fn {shortcode, _url} -> reject_emoji? = - Config.get([:mrf_steal_emoji, :rejected_shortcodes], []) + [:mrf_steal_emoji, :rejected_shortcodes] + |> Config.get([]) |> Enum.find(false, fn regex -> String.match?(shortcode, regex) end) !reject_emoji? diff --git a/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs b/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs index 3f8222736..7665d00d0 100644 --- a/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do use Pleroma.DataCase alias Pleroma.Config + alias Pleroma.Emoji alias Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy setup_all do @@ -14,55 +15,91 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do end setup do - emoji_path = Path.join(Config.get([:instance, :static_dir]), "emoji/stolen") - File.rm_rf!(emoji_path) - File.mkdir!(emoji_path) + emoji_path = [:instance, :static_dir] |> Config.get() |> Path.join("emoji/stolen") - Pleroma.Emoji.reload() + Emoji.reload() + + message = %{ + "type" => "Create", + "object" => %{ + "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}], + "actor" => "https://example.org/users/admin" + } + } on_exit(fn -> File.rm_rf!(emoji_path) end) - :ok + [message: message, path: emoji_path] end - test "does nothing by default" do - installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) - refute "firedfox" in installed_emoji + test "does nothing by default", %{message: message} do + refute "firedfox" in installed() - message = %{ - "type" => "Create", - "object" => %{ - "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}], - "actor" => "https://example.org/users/admin" - } - } + assert {:ok, _message} = StealEmojiPolicy.filter(message) - assert {:ok, message} == StealEmojiPolicy.filter(message) - - installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) - refute "firedfox" in installed_emoji + refute "firedfox" in installed() end - test "Steals emoji on unknown shortcode from allowed remote host" do - installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) - refute "firedfox" in installed_emoji + test "Steals emoji on unknown shortcode from allowed remote host", %{ + message: message, + path: path + } do + refute "firedfox" in installed() + refute File.exists?(path) - message = %{ - "type" => "Create", - "object" => %{ - "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}], - "actor" => "https://example.org/users/admin" - } - } + clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468) - clear_config([:mrf_steal_emoji, :hosts], ["example.org"]) - clear_config([:mrf_steal_emoji, :size_limit], 284_468) + assert {:ok, _message} = StealEmojiPolicy.filter(message) - assert {:ok, message} == StealEmojiPolicy.filter(message) + assert "firedfox" in installed() + assert File.exists?(path) - installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) - assert "firedfox" in installed_emoji + assert path + |> Path.join("firedfox.png") + |> File.exists?() end + + test "reject shortcode", %{message: message} do + refute "firedfox" in installed() + + clear_config(:mrf_steal_emoji, + hosts: ["example.org"], + size_limit: 284_468, + rejected_shortcodes: [~r/firedfox/] + ) + + assert {:ok, _message} = StealEmojiPolicy.filter(message) + + refute "firedfox" in installed() + end + + test "reject if size is above the limit", %{message: message} do + refute "firedfox" in installed() + + clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 50_000) + + assert {:ok, _message} = StealEmojiPolicy.filter(message) + + refute "firedfox" in installed() + end + + test "reject if host returns error", %{message: message} do + refute "firedfox" in installed() + + Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox.png"} -> + {:ok, %Tesla.Env{status: 404, body: "Not found"}} + end) + + clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468) + + ExUnit.CaptureLog.capture_log(fn -> + assert {:ok, _message} = StealEmojiPolicy.filter(message) + end) =~ "MRF.StealEmojiPolicy: Failed to fetch https://example.org/emoji/firedfox.png" + + refute "firedfox" in installed() + end + + defp installed, do: Emoji.get_all() |> Enum.map(fn {k, _} -> k end) end From dad76703aaf750e3811b0c963a92e39aa06b9c76 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 25 Dec 2020 11:34:09 +0300 Subject: [PATCH 6/8] not needed --- lib/pleroma/config/transfer_task.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 8a6d66891..a0d7b7d71 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -28,8 +28,7 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, Pleroma.Captcha, [:seconds_valid]}, {:pleroma, Pleroma.Upload, [:proxy_remote]}, {:pleroma, :instance, [:upload_limit]}, - {:pleroma, :gopher, [:enabled]}, - {:pleroma, :mrf, [:policies]} + {:pleroma, :gopher, [:enabled]} ] def start_link(restart_pleroma? \\ true) do From 546da68a1186ba4f8f901f9da1f1f6065cd9846a Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 25 Dec 2020 11:53:01 +0300 Subject: [PATCH 7/8] changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1604ab3a..716fbb75d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Fix ability to update Pleroma Chat push notifications with PUT /api/v1/push/subscription and alert type pleroma:chat_mention - Emoji Reaction activity filtering from blocked and muted accounts. +- StealEmojiPolicy creates dir for emojis, if it doesn't exist. ## [2.2.1] - 2020-12-22 From 2e859794ee25bdf22216587f8c8260e47b2a038f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sun, 27 Dec 2020 21:58:15 +0300 Subject: [PATCH 8/8] non condition dir creation --- .../web/activity_pub/mrf/steal_emoji_policy.ex | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index 0311ca433..788f21261 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -65,9 +65,7 @@ def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = messa Path.join(Config.get([:instance, :static_dir]), "emoji/stolen") ) - if not Config.get([:mrf_steal_emoji, :dir_exists?], false) do - create_dir(emoji_dir_path) - end + File.mkdir_p(emoji_dir_path) new_emojis = foreign_emojis @@ -98,12 +96,4 @@ def filter(message), do: {:ok, message} def describe do {:ok, %{}} end - - defp create_dir(path) do - if not File.exists?(path) do - File.mkdir_p!(path) - end - - Config.put([:mrf_steal_emoji, :dir_exists?], true) - end end