[#2497] Adjusted media proxy preview invalidation. Allowed client-side caching for media preview. Adjusted prewarmer to fetch only proxiable URIs.

Removed :preview pool in favor of existing :media one. Misc. refactoring.
This commit is contained in:
Ivan Tashkinov 2020-09-05 20:19:09 +03:00
parent c3b02341bf
commit f170d47130
7 changed files with 34 additions and 32 deletions

View File

@ -754,11 +754,6 @@
timeout: 10_000
],
media: [
size: 50,
max_waiting: 10,
timeout: 10_000
],
preview: [
size: 50,
max_waiting: 20,
timeout: 15_000

View File

@ -14,7 +14,7 @@ defmodule Pleroma.Helpers.MediaHelper do
def image_resize(url, options) do
with executable when is_binary(executable) <- System.find_executable("convert"),
{:ok, args} <- prepare_image_resize_args(options),
{:ok, env} <- HTTP.get(url, [], adapter: [pool: :preview]),
{:ok, env} <- HTTP.get(url, [], adapter: [pool: :media]),
{:ok, fifo_path} <- mkfifo() do
args = List.flatten([fifo_path, args])
run_fifo(fifo_path, env, executable, args)
@ -62,7 +62,7 @@ defp prepare_image_resize_args(_), do: {:error, :missing_options}
def video_framegrab(url) do
with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
{:ok, env} <- HTTP.get(url, [], adapter: [pool: :preview]),
{:ok, env} <- HTTP.get(url, [], adapter: [pool: :media]),
{:ok, fifo_path} <- mkfifo(),
args = [
"-y",

View File

@ -18,6 +18,7 @@ defmodule Pleroma.ReverseProxy do
@methods ~w(GET HEAD)
def max_read_duration_default, do: @max_read_duration
def default_cache_control_header, do: @default_cache_control_header
@moduledoc """
A reverse proxy.

View File

@ -12,23 +12,28 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
require Logger
@options [
@adapter_options [
pool: :media
]
def perform(:prefetch, url) do
Logger.debug("Prefetching #{inspect(url)}")
# Fetching only proxiable resources
if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do
# If preview proxy is enabled, it'll also hit media proxy (so we're caching both requests)
prefetch_url = MediaProxy.preview_url(url)
opts =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.put(@options, :recv_timeout, 10_000)
else
@options
end
Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
url
|> MediaProxy.preview_url()
|> HTTP.get([], adapter: opts)
HTTP.get(prefetch_url, [], adapter: adapter_options())
end
end
defp adapter_options do
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.put(@adapter_options, :recv_timeout, 10_000)
else
@adapter_options
end
end
def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do

View File

@ -33,6 +33,8 @@ defp do_purge(urls) do
def prepare_urls(urls) do
urls
|> List.wrap()
|> Enum.map(&MediaProxy.url/1)
|> Enum.map(fn url -> [MediaProxy.url(url), MediaProxy.preview_url(url)] end)
|> List.flatten()
|> Enum.uniq()
end
end

View File

@ -41,20 +41,16 @@ def url(url) when is_nil(url) or url == "", do: nil
def url("/" <> _ = url), do: url
def url(url) do
if not enabled?() or not url_proxiable?(url) do
url
else
if enabled?() and url_proxiable?(url) do
encode_url(url)
else
url
end
end
@spec url_proxiable?(String.t()) :: boolean()
def url_proxiable?(url) do
if local?(url) or whitelisted?(url) do
false
else
true
end
not local?(url) and not whitelisted?(url)
end
def preview_url(url, preview_params \\ []) do
@ -69,7 +65,7 @@ def enabled?, do: Config.get([:media_proxy, :enabled], false)
# Note: media proxy must be enabled for media preview proxy in order to load all
# non-local non-whitelisted URLs through it and be sure that body size constraint is preserved.
def preview_enabled?, do: enabled?() and Config.get([:media_preview_proxy, :enabled], false)
def preview_enabled?, do: enabled?() and !!Config.get([:media_preview_proxy, :enabled])
def local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
@ -138,9 +134,13 @@ def filename(url_or_path) do
if path = URI.parse(url_or_path).path, do: Path.basename(path)
end
def base_url do
Config.get([:media_proxy, :base_url], Web.base_url())
end
defp proxy_url(path, sig_base64, url_base64, filename) do
[
Config.get([:media_proxy, :base_url], Web.base_url()),
base_url(),
path,
sig_base64,
url_base64,

View File

@ -51,7 +51,7 @@ defp handle_preview(conn, url) do
media_proxy_url = MediaProxy.url(url)
with {:ok, %{status: status} = head_response} when status in 200..299 <-
Pleroma.HTTP.request("head", media_proxy_url, [], [], adapter: [pool: :preview]) do
Pleroma.HTTP.request("head", media_proxy_url, [], [], adapter: [pool: :media]) do
content_type = Tesla.get_header(head_response, "content-type")
handle_preview(content_type, conn, media_proxy_url)
else
@ -154,8 +154,7 @@ defp put_preview_response_headers(
conn
|> put_resp_header("content-type", content_type)
|> put_resp_header("content-disposition", "inline; filename=\"#{filename}\"")
# TODO: enable caching
|> put_resp_header("cache-control", "max-age=0, private, must-revalidate")
|> put_resp_header("cache-control", ReverseProxy.default_cache_control_header())
end
defp thumbnail_max_dimensions(params) do