From 034986e1fd389066c1ca4445af0cf8410da746e7 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 27 Jun 2019 03:06:58 +0000 Subject: [PATCH 1/3] MRF: add mediaproxy warming policy --- .../mrf/mediaproxy_warming_policy.ex | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex diff --git a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex new file mode 100644 index 000000000..01d21a299 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex @@ -0,0 +1,56 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do + @moduledoc "Preloads any attachments in the MediaProxy cache by prefetching them" + @behaviour Pleroma.Web.ActivityPub.MRF + + alias Pleroma.HTTP + alias Pleroma.Web.MediaProxy + + require Logger + + @hackney_options [ + pool: :media, + recv_timeout: 10_000 + ] + + def perform(:prefetch, url) do + Logger.info("Prefetching #{inspect(url)}") + + url + |> MediaProxy.url() + |> HTTP.get([], adapter: @hackney_options) + end + + def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do + Enum.each(attachments, fn + %{"url" => url} when is_list(url) -> + url + |> Enum.each(fn + %{"href" => href} -> + PleromaJobQueue.enqueue(:background, __MODULE__, [:prefetch, href]) + + x -> + Logger.debug("Unhandled attachment URL object #{inspect(x)}") + end) + + x -> + Logger.debug("Unhandled attachment #{inspect(x)}") + end) + end + + @impl true + def filter( + %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message + ) + when is_list(attachments) and length(attachments) > 0 do + PleromaJobQueue.enqueue(:background, __MODULE__, [:preload, message]) + + {:ok, message} + end + + @impl true + def filter(message), do: {:ok, message} +end From 074ffee0b2eeae5b3911f47f4ce17b3bbf74152d Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 27 Jun 2019 21:34:27 +0000 Subject: [PATCH 2/3] docs: add documentation for MediaProxyWarmingPolicy --- CHANGELOG.md | 4 ++++ docs/config.md | 1 + 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a65988991..a6ec8674d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [Unreleased] +### Added +- MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) + ## [1.0.0] - 2019-06-29 ### Security - Mastodon API: Fix display names not being sanitized diff --git a/docs/config.md b/docs/config.md index 7a53364ef..7d5be3980 100644 --- a/docs/config.md +++ b/docs/config.md @@ -98,6 +98,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section) * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links. + * `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`: Crawls attachments using their MediaProxy URLs so that the MediaProxy cache is primed. * `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. * `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send. * `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json`` From c22f0bfb8056889ea3180e0adbf46458eabf63a7 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 28 Jun 2019 23:41:42 +0000 Subject: [PATCH 3/3] test: add tests for mediaproxy warming policy --- .../mrf/mediaproxy_warming_policy_test.exs | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs diff --git a/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs b/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs new file mode 100644 index 000000000..372e789be --- /dev/null +++ b/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do + use Pleroma.DataCase + + alias Pleroma.HTTP + alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy + + import Mock + + @message %{ + "type" => "Create", + "object" => %{ + "type" => "Note", + "content" => "content", + "attachment" => [ + %{"url" => [%{"href" => "http://example.com/image.jpg"}]} + ] + } + } + + test "it prefetches media proxy URIs" do + with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do + MediaProxyWarmingPolicy.filter(@message) + assert called(HTTP.get(:_, :_, :_)) + end + end + + test "it does nothing when no attachments are present" do + object = + @message["object"] + |> Map.delete("attachment") + + message = + @message + |> Map.put("object", object) + + with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do + MediaProxyWarmingPolicy.filter(message) + refute called(HTTP.get(:_, :_, :_)) + end + end +end