From f170d471307ba0082b98351190b3d6b808bdfe1a Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 5 Sep 2020 20:19:09 +0300 Subject: [PATCH] [#2497] Adjusted media proxy preview invalidation. Allowed client-side caching for media preview. Adjusted prewarmer to fetch only proxiable URIs. Removed :preview pool in favor of existing :media one. Misc. refactoring. --- config/config.exs | 5 ---- lib/pleroma/helpers/media_helper.ex | 4 +-- lib/pleroma/reverse_proxy/reverse_proxy.ex | 1 + .../mrf/media_proxy_warming_policy.ex | 27 +++++++++++-------- lib/pleroma/web/media_proxy/invalidation.ex | 4 ++- lib/pleroma/web/media_proxy/media_proxy.ex | 20 +++++++------- .../web/media_proxy/media_proxy_controller.ex | 5 ++-- 7 files changed, 34 insertions(+), 32 deletions(-) diff --git a/config/config.exs b/config/config.exs index b92d3ccbb..e5b7e18df 100644 --- a/config/config.exs +++ b/config/config.exs @@ -754,11 +754,6 @@ timeout: 10_000 ], media: [ - size: 50, - max_waiting: 10, - timeout: 10_000 - ], - preview: [ size: 50, max_waiting: 20, timeout: 15_000 diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex index bb93d4915..a1205e10d 100644 --- a/lib/pleroma/helpers/media_helper.ex +++ b/lib/pleroma/helpers/media_helper.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Helpers.MediaHelper do def image_resize(url, options) do with executable when is_binary(executable) <- System.find_executable("convert"), {:ok, args} <- prepare_image_resize_args(options), - {:ok, env} <- HTTP.get(url, [], adapter: [pool: :preview]), + {:ok, env} <- HTTP.get(url, [], adapter: [pool: :media]), {:ok, fifo_path} <- mkfifo() do args = List.flatten([fifo_path, args]) run_fifo(fifo_path, env, executable, args) @@ -62,7 +62,7 @@ defp prepare_image_resize_args(_), do: {:error, :missing_options} def video_framegrab(url) do with executable when is_binary(executable) <- System.find_executable("ffmpeg"), - {:ok, env} <- HTTP.get(url, [], adapter: [pool: :preview]), + {:ok, env} <- HTTP.get(url, [], adapter: [pool: :media]), {:ok, fifo_path} <- mkfifo(), args = [ "-y", diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 35637e934..8ae1157df 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -18,6 +18,7 @@ defmodule Pleroma.ReverseProxy do @methods ~w(GET HEAD) def max_read_duration_default, do: @max_read_duration + def default_cache_control_header, do: @default_cache_control_header @moduledoc """ A reverse proxy. diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index 5d8bb72aa..1050b74ba 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -12,23 +12,28 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger - @options [ + @adapter_options [ pool: :media ] def perform(:prefetch, url) do - Logger.debug("Prefetching #{inspect(url)}") + # Fetching only proxiable resources + if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do + # If preview proxy is enabled, it'll also hit media proxy (so we're caching both requests) + prefetch_url = MediaProxy.preview_url(url) - opts = - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do - Keyword.put(@options, :recv_timeout, 10_000) - else - @options - end + Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}") - url - |> MediaProxy.preview_url() - |> HTTP.get([], adapter: opts) + HTTP.get(prefetch_url, [], adapter: adapter_options()) + end + end + + defp adapter_options do + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do + Keyword.put(@adapter_options, :recv_timeout, 10_000) + else + @adapter_options + end end def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index 5808861e6..4f4340478 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -33,6 +33,8 @@ defp do_purge(urls) do def prepare_urls(urls) do urls |> List.wrap() - |> Enum.map(&MediaProxy.url/1) + |> Enum.map(fn url -> [MediaProxy.url(url), MediaProxy.preview_url(url)] end) + |> List.flatten() + |> Enum.uniq() end end diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index 80017cde1..ba553998b 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -41,20 +41,16 @@ def url(url) when is_nil(url) or url == "", do: nil def url("/" <> _ = url), do: url def url(url) do - if not enabled?() or not url_proxiable?(url) do - url - else + if enabled?() and url_proxiable?(url) do encode_url(url) + else + url end end @spec url_proxiable?(String.t()) :: boolean() def url_proxiable?(url) do - if local?(url) or whitelisted?(url) do - false - else - true - end + not local?(url) and not whitelisted?(url) end def preview_url(url, preview_params \\ []) do @@ -69,7 +65,7 @@ def enabled?, do: Config.get([:media_proxy, :enabled], false) # Note: media proxy must be enabled for media preview proxy in order to load all # non-local non-whitelisted URLs through it and be sure that body size constraint is preserved. - def preview_enabled?, do: enabled?() and Config.get([:media_preview_proxy, :enabled], false) + def preview_enabled?, do: enabled?() and !!Config.get([:media_preview_proxy, :enabled]) def local?(url), do: String.starts_with?(url, Pleroma.Web.base_url()) @@ -138,9 +134,13 @@ def filename(url_or_path) do if path = URI.parse(url_or_path).path, do: Path.basename(path) end + def base_url do + Config.get([:media_proxy, :base_url], Web.base_url()) + end + defp proxy_url(path, sig_base64, url_base64, filename) do [ - Config.get([:media_proxy, :base_url], Web.base_url()), + base_url(), path, sig_base64, url_base64, diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index 469fbae59..89f4a23bd 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -51,7 +51,7 @@ defp handle_preview(conn, url) do media_proxy_url = MediaProxy.url(url) with {:ok, %{status: status} = head_response} when status in 200..299 <- - Pleroma.HTTP.request("head", media_proxy_url, [], [], adapter: [pool: :preview]) do + Pleroma.HTTP.request("head", media_proxy_url, [], [], adapter: [pool: :media]) do content_type = Tesla.get_header(head_response, "content-type") handle_preview(content_type, conn, media_proxy_url) else @@ -154,8 +154,7 @@ defp put_preview_response_headers( conn |> put_resp_header("content-type", content_type) |> put_resp_header("content-disposition", "inline; filename=\"#{filename}\"") - # TODO: enable caching - |> put_resp_header("cache-control", "max-age=0, private, must-revalidate") + |> put_resp_header("cache-control", ReverseProxy.default_cache_control_header()) end defp thumbnail_max_dimensions(params) do