Merge branch 'issue/1855' into 'develop'

#1855 MediaProxy cache invalidation via Admin API

See merge request pleroma/pleroma!2648
This commit is contained in:
feld 2020-06-19 20:38:22 +00:00
commit f928267773
22 changed files with 740 additions and 118 deletions

View File

@ -407,6 +407,13 @@
], ],
whitelist: [] whitelist: []
config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http,
method: :purge,
headers: [],
options: []
config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, script_path: nil
config :pleroma, :chat, enabled: true config :pleroma, :chat, enabled: true
config :phoenix, :format_encoders, json: Jason config :phoenix, :format_encoders, json: Jason

View File

@ -1650,6 +1650,31 @@
"The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.", "The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.",
suggestions: ["https://example.com"] suggestions: ["https://example.com"]
}, },
%{
key: :invalidation,
type: :keyword,
descpiption: "",
suggestions: [
enabled: true,
provider: Pleroma.Web.MediaProxy.Invalidation.Script
],
children: [
%{
key: :enabled,
type: :boolean,
description: "Enables invalidate media cache"
},
%{
key: :provider,
type: :module,
description: "Module which will be used to cache purge.",
suggestions: [
Pleroma.Web.MediaProxy.Invalidation.Script,
Pleroma.Web.MediaProxy.Invalidation.Http
]
}
]
},
%{ %{
key: :proxy_opts, key: :proxy_opts,
type: :keyword, type: :keyword,
@ -1722,6 +1747,45 @@
} }
] ]
}, },
%{
group: :pleroma,
key: Pleroma.Web.MediaProxy.Invalidation.Http,
type: :group,
description: "HTTP invalidate settings",
children: [
%{
key: :method,
type: :atom,
description: "HTTP method of request. Default: :purge"
},
%{
key: :headers,
type: {:list, :tuple},
description: "HTTP headers of request.",
suggestions: [{"x-refresh", 1}]
},
%{
key: :options,
type: :keyword,
description: "Request options.",
suggestions: [params: %{ts: "xxx"}]
}
]
},
%{
group: :pleroma,
key: Pleroma.Web.MediaProxy.Invalidation.Script,
type: :group,
description: "Script invalidate settings",
children: [
%{
key: :script_path,
type: :string,
description: "Path to shell script. Which will run purge cache.",
suggestions: ["./installation/nginx-cache-purge.sh.example"]
}
]
},
%{ %{
group: :pleroma, group: :pleroma,
key: :gopher, key: :gopher,

View File

@ -1224,4 +1224,66 @@ Loads json generated from `config/descriptions.exs`.
- Response: - Response:
- On success: `204`, empty response - On success: `204`, empty response
- On failure: - On failure:
- 400 Bad Request `"Invalid parameters"` when `status` is missing - 400 Bad Request `"Invalid parameters"` when `status` is missing
## `GET /api/pleroma/admin/media_proxy_caches`
### Get a list of all banned MediaProxy URLs in Cachex
- Authentication: required
- Params:
- *optional* `page`: **integer** page number
- *optional* `page_size`: **integer** number of log entries per page (default is `50`)
- Response:
``` json
{
"urls": [
"http://example.com/media/a688346.jpg",
"http://example.com/media/fb1f4d.jpg"
]
}
```
## `POST /api/pleroma/admin/media_proxy_caches/delete`
### Remove a banned MediaProxy URL from Cachex
- Authentication: required
- Params:
- `urls` (array)
- Response:
``` json
{
"urls": [
"http://example.com/media/a688346.jpg",
"http://example.com/media/fb1f4d.jpg"
]
}
```
## `POST /api/pleroma/admin/media_proxy_caches/purge`
### Purge a MediaProxy URL
- Authentication: required
- Params:
- `urls` (array)
- `ban` (boolean)
- Response:
``` json
{
"urls": [
"http://example.com/media/a688346.jpg",
"http://example.com/media/fb1f4d.jpg"
]
}
```

View File

@ -268,7 +268,7 @@ This section describe PWA manifest instance-specific values. Currently this opti
#### Pleroma.Web.MediaProxy.Invalidation.Script #### Pleroma.Web.MediaProxy.Invalidation.Script
This strategy allow perform external bash script to purge cache. This strategy allow perform external shell script to purge cache.
Urls of attachments pass to script as arguments. Urls of attachments pass to script as arguments.
* `script_path`: path to external script. * `script_path`: path to external script.
@ -284,8 +284,8 @@ config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script,
This strategy allow perform custom http request to purge cache. This strategy allow perform custom http request to purge cache.
* `method`: http method. default is `purge` * `method`: http method. default is `purge`
* `headers`: http headers. default is empty * `headers`: http headers.
* `options`: request options. default is empty * `options`: request options.
Example: Example:
```elixir ```elixir

View File

@ -13,7 +13,7 @@ CACHE_DIRECTORY="/tmp/pleroma-media-cache"
## $3 - (optional) the number of parallel processes to run for grep. ## $3 - (optional) the number of parallel processes to run for grep.
get_cache_files() { get_cache_files() {
local max_parallel=${3-16} local max_parallel=${3-16}
find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E Rl "^KEY:.*$1" | sort -u find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E -Rl "^KEY:.*$1" | sort -u
} }
## Removes an item from the given cache zone. ## Removes an item from the given cache zone.
@ -37,4 +37,4 @@ purge() {
} }
purge $1 purge $@

View File

@ -148,7 +148,8 @@ defp cachex_children do
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500), build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
build_cachex("web_resp", limit: 2500), build_cachex("web_resp", limit: 2500),
build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
build_cachex("failed_proxy_url", limit: 2500) build_cachex("failed_proxy_url", limit: 2500),
build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000)
] ]
end end

View File

@ -10,6 +10,8 @@ defmodule Pleroma.Plugs.UploadedMedia do
import Pleroma.Web.Gettext import Pleroma.Web.Gettext
require Logger require Logger
alias Pleroma.Web.MediaProxy
@behaviour Plug @behaviour Plug
# no slashes # no slashes
@path "media" @path "media"
@ -35,8 +37,7 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do
%{query_params: %{"name" => name}} = conn -> %{query_params: %{"name" => name}} = conn ->
name = String.replace(name, "\"", "\\\"") name = String.replace(name, "\"", "\\\"")
conn put_resp_header(conn, "content-disposition", "filename=\"#{name}\"")
|> put_resp_header("content-disposition", "filename=\"#{name}\"")
conn -> conn ->
conn conn
@ -47,7 +48,8 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do
with uploader <- Keyword.fetch!(config, :uploader), with uploader <- Keyword.fetch!(config, :uploader),
proxy_remote = Keyword.get(config, :proxy_remote, false), proxy_remote = Keyword.get(config, :proxy_remote, false),
{:ok, get_method} <- uploader.get_file(file) do {:ok, get_method} <- uploader.get_file(file),
false <- media_is_banned(conn, get_method) do
get_media(conn, get_method, proxy_remote, opts) get_media(conn, get_method, proxy_remote, opts)
else else
_ -> _ ->
@ -59,6 +61,14 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do
def call(conn, _opts), do: conn def call(conn, _opts), do: conn
defp media_is_banned(%{request_path: path} = _conn, {:static_dir, _}) do
MediaProxy.in_banned_urls(Pleroma.Web.base_url() <> path)
end
defp media_is_banned(_, {:url, url}), do: MediaProxy.in_banned_urls(url)
defp media_is_banned(_, _), do: false
defp get_media(conn, {:static_dir, directory}, _, opts) do defp get_media(conn, {:static_dir, directory}, _, opts) do
static_opts = static_opts =
Map.get(opts, :static_plug_opts) Map.get(opts, :static_plug_opts)

View File

@ -0,0 +1,63 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
use Pleroma.Web, :controller
alias Pleroma.Plugs.OAuthScopesPlug
alias Pleroma.Web.ApiSpec.Admin, as: Spec
alias Pleroma.Web.MediaProxy
plug(Pleroma.Web.ApiSpec.CastAndValidate)
plug(
OAuthScopesPlug,
%{scopes: ["read:media_proxy_caches"], admin: true} when action in [:index]
)
plug(
OAuthScopesPlug,
%{scopes: ["write:media_proxy_caches"], admin: true} when action in [:purge, :delete]
)
action_fallback(Pleroma.Web.AdminAPI.FallbackController)
defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation
def index(%{assigns: %{user: _}} = conn, params) do
cursor =
:banned_urls_cache
|> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}])
|> :qlc.cursor()
urls =
case params.page do
1 ->
:qlc.next_answers(cursor, params.page_size)
_ ->
:qlc.next_answers(cursor, (params.page - 1) * params.page_size)
:qlc.next_answers(cursor, params.page_size)
end
:qlc.delete_cursor(cursor)
render(conn, "index.json", urls: urls)
end
def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do
MediaProxy.remove_from_banned_urls(urls)
render(conn, "index.json", urls: urls)
end
def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do
MediaProxy.Invalidation.purge(urls)
if ban do
MediaProxy.put_in_banned_urls(urls)
end
render(conn, "index.json", urls: urls)
end
end

View File

@ -0,0 +1,11 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do
use Pleroma.Web, :view
def render("index.json", %{urls: urls}) do
%{urls: urls}
end
end

View File

@ -0,0 +1,109 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do
alias OpenApiSpex.Operation
alias OpenApiSpex.Schema
alias Pleroma.Web.ApiSpec.Schemas.ApiError
import Pleroma.Web.ApiSpec.Helpers
def open_api_operation(action) do
operation = String.to_existing_atom("#{action}_operation")
apply(__MODULE__, operation, [])
end
def index_operation do
%Operation{
tags: ["Admin", "MediaProxyCache"],
summary: "Fetch a paginated list of all banned MediaProxy URLs in Cachex",
operationId: "AdminAPI.MediaProxyCacheController.index",
security: [%{"oAuth" => ["read:media_proxy_caches"]}],
parameters: [
Operation.parameter(
:page,
:query,
%Schema{type: :integer, default: 1},
"Page"
),
Operation.parameter(
:page_size,
:query,
%Schema{type: :integer, default: 50},
"Number of statuses to return"
)
],
responses: %{
200 => success_response()
}
}
end
def delete_operation do
%Operation{
tags: ["Admin", "MediaProxyCache"],
summary: "Remove a banned MediaProxy URL from Cachex",
operationId: "AdminAPI.MediaProxyCacheController.delete",
security: [%{"oAuth" => ["write:media_proxy_caches"]}],
requestBody:
request_body(
"Parameters",
%Schema{
type: :object,
required: [:urls],
properties: %{
urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}}
}
},
required: true
),
responses: %{
200 => success_response(),
400 => Operation.response("Error", "application/json", ApiError)
}
}
end
def purge_operation do
%Operation{
tags: ["Admin", "MediaProxyCache"],
summary: "Purge and optionally ban a MediaProxy URL",
operationId: "AdminAPI.MediaProxyCacheController.purge",
security: [%{"oAuth" => ["write:media_proxy_caches"]}],
requestBody:
request_body(
"Parameters",
%Schema{
type: :object,
required: [:urls],
properties: %{
urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}},
ban: %Schema{type: :boolean, default: true}
}
},
required: true
),
responses: %{
200 => success_response(),
400 => Operation.response("Error", "application/json", ApiError)
}
}
end
defp success_response do
Operation.response("Array of banned MediaProxy URLs in Cachex", "application/json", %Schema{
type: :object,
properties: %{
urls: %Schema{
type: :array,
items: %Schema{
type: :string,
format: :uri,
description: "MediaProxy URLs"
}
}
}
})
end
end

View File

@ -5,22 +5,34 @@
defmodule Pleroma.Web.MediaProxy.Invalidation do defmodule Pleroma.Web.MediaProxy.Invalidation do
@moduledoc false @moduledoc false
@callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()} @callback purge(list(String.t()), Keyword.t()) :: {:ok, list(String.t())} | {:error, String.t()}
alias Pleroma.Config alias Pleroma.Config
alias Pleroma.Web.MediaProxy
@spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()} @spec enabled?() :: boolean()
def enabled?, do: Config.get([:media_proxy, :invalidation, :enabled])
@spec purge(list(String.t()) | String.t()) :: {:ok, list(String.t())} | {:error, String.t()}
def purge(urls) do def purge(urls) do
[:media_proxy, :invalidation, :enabled] prepared_urls = prepare_urls(urls)
|> Config.get()
|> do_purge(urls) if enabled?() do
do_purge(prepared_urls)
else
{:ok, prepared_urls}
end
end end
defp do_purge(true, urls) do defp do_purge(urls) do
provider = Config.get([:media_proxy, :invalidation, :provider]) provider = Config.get([:media_proxy, :invalidation, :provider])
options = Config.get(provider) options = Config.get(provider)
provider.purge(urls, options) provider.purge(urls, options)
end end
defp do_purge(_, _), do: :ok def prepare_urls(urls) do
urls
|> List.wrap()
|> Enum.map(&MediaProxy.url/1)
end
end end

View File

@ -9,10 +9,10 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do
require Logger require Logger
@impl Pleroma.Web.MediaProxy.Invalidation @impl Pleroma.Web.MediaProxy.Invalidation
def purge(urls, opts) do def purge(urls, opts \\ []) do
method = Map.get(opts, :method, :purge) method = Keyword.get(opts, :method, :purge)
headers = Map.get(opts, :headers, []) headers = Keyword.get(opts, :headers, [])
options = Map.get(opts, :options, []) options = Keyword.get(opts, :options, [])
Logger.debug("Running cache purge: #{inspect(urls)}") Logger.debug("Running cache purge: #{inspect(urls)}")
@ -22,7 +22,7 @@ def purge(urls, opts) do
end end
end) end)
{:ok, "success"} {:ok, urls}
end end
defp do_purge(method, url, headers, options) do defp do_purge(method, url, headers, options) do

View File

@ -10,32 +10,34 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do
require Logger require Logger
@impl Pleroma.Web.MediaProxy.Invalidation @impl Pleroma.Web.MediaProxy.Invalidation
def purge(urls, %{script_path: script_path} = _options) do def purge(urls, opts \\ []) do
args = args =
urls urls
|> List.wrap() |> List.wrap()
|> Enum.uniq() |> Enum.uniq()
|> Enum.join(" ") |> Enum.join(" ")
path = Path.expand(script_path) opts
|> Keyword.get(:script_path)
Logger.debug("Running cache purge: #{inspect(urls)}, #{path}") |> do_purge([args])
|> handle_result(urls)
case do_purge(path, [args]) do
{result, exit_status} when exit_status > 0 ->
Logger.error("Error while cache purge: #{inspect(result)}")
{:error, inspect(result)}
_ ->
{:ok, "success"}
end
end end
def purge(_, _), do: {:error, "not found script path"} defp do_purge(script_path, args) when is_binary(script_path) do
path = Path.expand(script_path)
defp do_purge(path, args) do Logger.debug("Running cache purge: #{inspect(args)}, #{inspect(path)}")
System.cmd(path, args) System.cmd(path, args)
rescue rescue
error -> {inspect(error), 1} error -> error
end
defp do_purge(_, _), do: {:error, "not found script path"}
defp handle_result({_result, 0}, urls), do: {:ok, urls}
defp handle_result({:error, error}, urls), do: handle_result(error, urls)
defp handle_result(error, _) do
Logger.error("Error while cache purge: #{inspect(error)}")
{:error, inspect(error)}
end end
end end

View File

@ -6,20 +6,53 @@ defmodule Pleroma.Web.MediaProxy do
alias Pleroma.Config alias Pleroma.Config
alias Pleroma.Upload alias Pleroma.Upload
alias Pleroma.Web alias Pleroma.Web
alias Pleroma.Web.MediaProxy.Invalidation
@base64_opts [padding: false] @base64_opts [padding: false]
@spec in_banned_urls(String.t()) :: boolean()
def in_banned_urls(url), do: elem(Cachex.exists?(:banned_urls_cache, url(url)), 1)
def remove_from_banned_urls(urls) when is_list(urls) do
Cachex.execute!(:banned_urls_cache, fn cache ->
Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1))
end)
end
def remove_from_banned_urls(url) when is_binary(url) do
Cachex.del(:banned_urls_cache, url(url))
end
def put_in_banned_urls(urls) when is_list(urls) do
Cachex.execute!(:banned_urls_cache, fn cache ->
Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true))
end)
end
def put_in_banned_urls(url) when is_binary(url) do
Cachex.put(:banned_urls_cache, url(url), true)
end
def url(url) when is_nil(url) or url == "", do: nil def url(url) when is_nil(url) or url == "", do: nil
def url("/" <> _ = url), do: url def url("/" <> _ = url), do: url
def url(url) do def url(url) do
if disabled?() or local?(url) or whitelisted?(url) do if disabled?() or not url_proxiable?(url) do
url url
else else
encode_url(url) encode_url(url)
end end
end end
@spec url_proxiable?(String.t()) :: boolean()
def url_proxiable?(url) do
if local?(url) or whitelisted?(url) do
false
else
true
end
end
defp disabled?, do: !Config.get([:media_proxy, :enabled], false) defp disabled?, do: !Config.get([:media_proxy, :enabled], false)
defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url()) defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())

View File

@ -14,10 +14,11 @@ def remote(conn, %{"sig" => sig64, "url" => url64} = params) do
with config <- Pleroma.Config.get([:media_proxy], []), with config <- Pleroma.Config.get([:media_proxy], []),
true <- Keyword.get(config, :enabled, false), true <- Keyword.get(config, :enabled, false),
{:ok, url} <- MediaProxy.decode_url(sig64, url64), {:ok, url} <- MediaProxy.decode_url(sig64, url64),
{_, false} <- {:in_banned_urls, MediaProxy.in_banned_urls(url)},
:ok <- filename_matches(params, conn.request_path, url) do :ok <- filename_matches(params, conn.request_path, url) do
ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts)) ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
else else
false -> error when error in [false, {:in_banned_urls, true}] ->
send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404)) send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
{:error, :invalid_signature} -> {:error, :invalid_signature} ->

View File

@ -209,6 +209,10 @@ defmodule Pleroma.Web.Router do
post("/oauth_app", OAuthAppController, :create) post("/oauth_app", OAuthAppController, :create)
patch("/oauth_app/:id", OAuthAppController, :update) patch("/oauth_app/:id", OAuthAppController, :update)
delete("/oauth_app/:id", OAuthAppController, :delete) delete("/oauth_app/:id", OAuthAppController, :delete)
get("/media_proxy_caches", MediaProxyCacheController, :index)
post("/media_proxy_caches/delete", MediaProxyCacheController, :delete)
post("/media_proxy_caches/purge", MediaProxyCacheController, :purge)
end end
scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do

View File

@ -18,13 +18,19 @@ def perform(
}, },
_job _job
) do ) do
hrefs = attachments
Enum.flat_map(attachments, fn attachment -> |> Enum.flat_map(fn item -> Enum.map(item["url"], & &1["href"]) end)
Enum.map(attachment["url"], & &1["href"]) |> fetch_objects
end) |> prepare_objects(actor, Enum.map(attachments, & &1["name"]))
|> filter_objects
|> do_clean
names = Enum.map(attachments, & &1["name"]) {:ok, :success}
end
def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}
defp do_clean({object_ids, attachment_urls}) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
prefix = prefix =
@ -39,68 +45,70 @@ def perform(
"/" "/"
) )
# find all objects for copies of the attachments, name and actor doesn't matter here Enum.each(attachment_urls, fn href ->
object_ids_and_hrefs = href
from(o in Object, |> String.trim_leading("#{base_url}/#{prefix}")
where: |> uploader.delete_file()
fragment( end)
"to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)",
o.data,
o.data,
^hrefs
)
)
# The query above can be time consumptive on large instances until we
# refactor how uploads are stored
|> Repo.all(timeout: :infinity)
# we should delete 1 object for any given attachment, but don't delete
# files if there are more than 1 object for it
|> Enum.reduce(%{}, fn %{
id: id,
data: %{
"url" => [%{"href" => href}],
"actor" => obj_actor,
"name" => name
}
},
acc ->
Map.update(acc, href, %{id: id, count: 1}, fn val ->
case obj_actor == actor and name in names do
true ->
# set id of the actor's object that will be deleted
%{val | id: id, count: val.count + 1}
false -> delete_objects(object_ids)
# another actor's object, just increase count to not delete file
%{val | count: val.count + 1}
end
end)
end)
|> Enum.map(fn {href, %{id: id, count: count}} ->
# only delete files that have single instance
with 1 <- count do
href
|> String.trim_leading("#{base_url}/#{prefix}")
|> uploader.delete_file()
{id, href}
else
_ -> {id, nil}
end
end)
object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end)
from(o in Object, where: o.id in ^object_ids)
|> Repo.delete_all()
object_ids_and_hrefs
|> Enum.filter(fn {_, href} -> not is_nil(href) end)
|> Enum.map(&elem(&1, 1))
|> Pleroma.Web.MediaProxy.Invalidation.purge()
{:ok, :success}
end end
def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip} defp delete_objects([_ | _] = object_ids) do
Repo.delete_all(from(o in Object, where: o.id in ^object_ids))
end
defp delete_objects(_), do: :ok
# we should delete 1 object for any given attachment, but don't delete
# files if there are more than 1 object for it
defp filter_objects(objects) do
Enum.reduce(objects, {[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} ->
with 1 <- count do
{ids ++ [id], hrefs ++ [href]}
else
_ -> {ids ++ [id], hrefs}
end
end)
end
defp prepare_objects(objects, actor, names) do
objects
|> Enum.reduce(%{}, fn %{
id: id,
data: %{
"url" => [%{"href" => href}],
"actor" => obj_actor,
"name" => name
}
},
acc ->
Map.update(acc, href, %{id: id, count: 1}, fn val ->
case obj_actor == actor and name in names do
true ->
# set id of the actor's object that will be deleted
%{val | id: id, count: val.count + 1}
false ->
# another actor's object, just increase count to not delete file
%{val | count: val.count + 1}
end
end)
end)
end
defp fetch_objects(hrefs) do
from(o in Object,
where:
fragment(
"to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)",
o.data,
o.data,
^hrefs
)
)
# The query above can be time consumptive on large instances until we
# refactor how uploads are stored
|> Repo.all(timeout: :infinity)
end
end end

View File

@ -0,0 +1,145 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
use Pleroma.Web.ConnCase
import Pleroma.Factory
import Mock
alias Pleroma.Web.MediaProxy
setup do: clear_config([:media_proxy])
setup do
on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
end
setup do
admin = insert(:user, is_admin: true)
token = insert(:oauth_admin_token, user: admin)
conn =
build_conn()
|> assign(:user, admin)
|> assign(:token, token)
Config.put([:media_proxy, :enabled], true)
Config.put([:media_proxy, :invalidation, :enabled], true)
Config.put([:media_proxy, :invalidation, :provider], MediaProxy.Invalidation.Script)
{:ok, %{admin: admin, token: token, conn: conn}}
end
describe "GET /api/pleroma/admin/media_proxy_caches" do
test "shows banned MediaProxy URLs", %{conn: conn} do
MediaProxy.put_in_banned_urls([
"http://localhost:4001/media/a688346.jpg",
"http://localhost:4001/media/fb1f4d.jpg"
])
MediaProxy.put_in_banned_urls("http://localhost:4001/media/gb1f44.jpg")
MediaProxy.put_in_banned_urls("http://localhost:4001/media/tb13f47.jpg")
MediaProxy.put_in_banned_urls("http://localhost:4001/media/wb1f46.jpg")
response =
conn
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2")
|> json_response_and_validate_schema(200)
assert response["urls"] == [
"http://localhost:4001/media/fb1f4d.jpg",
"http://localhost:4001/media/a688346.jpg"
]
response =
conn
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=2")
|> json_response_and_validate_schema(200)
assert response["urls"] == [
"http://localhost:4001/media/gb1f44.jpg",
"http://localhost:4001/media/tb13f47.jpg"
]
response =
conn
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3")
|> json_response_and_validate_schema(200)
assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"]
end
end
describe "POST /api/pleroma/admin/media_proxy_caches/delete" do
test "deleted MediaProxy URLs from banned", %{conn: conn} do
MediaProxy.put_in_banned_urls([
"http://localhost:4001/media/a688346.jpg",
"http://localhost:4001/media/fb1f4d.jpg"
])
response =
conn
|> put_req_header("content-type", "application/json")
|> post("/api/pleroma/admin/media_proxy_caches/delete", %{
urls: ["http://localhost:4001/media/a688346.jpg"]
})
|> json_response_and_validate_schema(200)
assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"]
refute MediaProxy.in_banned_urls("http://localhost:4001/media/a688346.jpg")
assert MediaProxy.in_banned_urls("http://localhost:4001/media/fb1f4d.jpg")
end
end
describe "POST /api/pleroma/admin/media_proxy_caches/purge" do
test "perform invalidates cache of MediaProxy", %{conn: conn} do
urls = [
"http://example.com/media/a688346.jpg",
"http://example.com/media/fb1f4d.jpg"
]
with_mocks [
{MediaProxy.Invalidation.Script, [],
[
purge: fn _, _ -> {"ok", 0} end
]}
] do
response =
conn
|> put_req_header("content-type", "application/json")
|> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false})
|> json_response_and_validate_schema(200)
assert response["urls"] == urls
refute MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
refute MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
end
end
test "perform invalidates cache of MediaProxy and adds url to banned", %{conn: conn} do
urls = [
"http://example.com/media/a688346.jpg",
"http://example.com/media/fb1f4d.jpg"
]
with_mocks [{MediaProxy.Invalidation.Script, [], [purge: fn _, _ -> {"ok", 0} end]}] do
response =
conn
|> put_req_header("content-type", "application/json")
|> post("/api/pleroma/admin/media_proxy_caches/purge", %{
urls: urls,
ban: true
})
|> json_response_and_validate_schema(200)
assert response["urls"] == urls
assert MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
assert MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
end
end
end
end

View File

@ -0,0 +1,64 @@
defmodule Pleroma.Web.MediaProxy.InvalidationTest do
use ExUnit.Case
use Pleroma.Tests.Helpers
alias Pleroma.Config
alias Pleroma.Web.MediaProxy.Invalidation
import ExUnit.CaptureLog
import Mock
import Tesla.Mock
setup do: clear_config([:media_proxy])
setup do
on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
end
describe "Invalidation.Http" do
test "perform request to clear cache" do
Config.put([:media_proxy, :enabled], false)
Config.put([:media_proxy, :invalidation, :enabled], true)
Config.put([:media_proxy, :invalidation, :provider], Invalidation.Http)
Config.put([Invalidation.Http], method: :purge, headers: [{"x-refresh", 1}])
image_url = "http://example.com/media/example.jpg"
Pleroma.Web.MediaProxy.put_in_banned_urls(image_url)
mock(fn
%{
method: :purge,
url: "http://example.com/media/example.jpg",
headers: [{"x-refresh", 1}]
} ->
%Tesla.Env{status: 200}
end)
assert capture_log(fn ->
assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
assert Invalidation.purge([image_url]) == {:ok, [image_url]}
assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
end) =~ "Running cache purge: [\"#{image_url}\"]"
end
end
describe "Invalidation.Script" do
test "run script to clear cache" do
Config.put([:media_proxy, :enabled], false)
Config.put([:media_proxy, :invalidation, :enabled], true)
Config.put([:media_proxy, :invalidation, :provider], Invalidation.Script)
Config.put([Invalidation.Script], script_path: "purge-nginx")
image_url = "http://example.com/media/example.jpg"
Pleroma.Web.MediaProxy.put_in_banned_urls(image_url)
with_mocks [{System, [], [cmd: fn _, _ -> {"ok", 0} end]}] do
assert capture_log(fn ->
assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
assert Invalidation.purge([image_url]) == {:ok, [image_url]}
assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
end) =~ "Running cache purge: [\"#{image_url}\"]"
end
end
end
end

View File

@ -5,6 +5,10 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do
import ExUnit.CaptureLog import ExUnit.CaptureLog
import Tesla.Mock import Tesla.Mock
setup do
on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
end
test "logs hasn't error message when request is valid" do test "logs hasn't error message when request is valid" do
mock(fn mock(fn
%{method: :purge, url: "http://example.com/media/example.jpg"} -> %{method: :purge, url: "http://example.com/media/example.jpg"} ->
@ -14,8 +18,8 @@ test "logs hasn't error message when request is valid" do
refute capture_log(fn -> refute capture_log(fn ->
assert Invalidation.Http.purge( assert Invalidation.Http.purge(
["http://example.com/media/example.jpg"], ["http://example.com/media/example.jpg"],
%{} []
) == {:ok, "success"} ) == {:ok, ["http://example.com/media/example.jpg"]}
end) =~ "Error while cache purge" end) =~ "Error while cache purge"
end end
@ -28,8 +32,8 @@ test "it write error message in logs when request invalid" do
assert capture_log(fn -> assert capture_log(fn ->
assert Invalidation.Http.purge( assert Invalidation.Http.purge(
["http://example.com/media/example1.jpg"], ["http://example.com/media/example1.jpg"],
%{} []
) == {:ok, "success"} ) == {:ok, ["http://example.com/media/example1.jpg"]}
end) =~ "Error while cache purge: url - http://example.com/media/example1.jpg" end) =~ "Error while cache purge: url - http://example.com/media/example1.jpg"
end end
end end

View File

@ -4,17 +4,23 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.ScriptTest do
import ExUnit.CaptureLog import ExUnit.CaptureLog
setup do
on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
end
test "it logger error when script not found" do test "it logger error when script not found" do
assert capture_log(fn -> assert capture_log(fn ->
assert Invalidation.Script.purge( assert Invalidation.Script.purge(
["http://example.com/media/example.jpg"], ["http://example.com/media/example.jpg"],
%{script_path: "./example"} script_path: "./example"
) == {:error, "\"%ErlangError{original: :enoent}\""} ) == {:error, "%ErlangError{original: :enoent}"}
end) =~ "Error while cache purge: \"%ErlangError{original: :enoent}\"" end) =~ "Error while cache purge: %ErlangError{original: :enoent}"
assert Invalidation.Script.purge( capture_log(fn ->
["http://example.com/media/example.jpg"], assert Invalidation.Script.purge(
%{} ["http://example.com/media/example.jpg"],
) == {:error, "not found script path"} []
) == {:error, "\"not found script path\""}
end)
end end
end end

View File

@ -10,6 +10,10 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do
setup do: clear_config(:media_proxy) setup do: clear_config(:media_proxy)
setup do: clear_config([Pleroma.Web.Endpoint, :secret_key_base]) setup do: clear_config([Pleroma.Web.Endpoint, :secret_key_base])
setup do
on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
end
test "it returns 404 when MediaProxy disabled", %{conn: conn} do test "it returns 404 when MediaProxy disabled", %{conn: conn} do
Config.put([:media_proxy, :enabled], false) Config.put([:media_proxy, :enabled], false)
@ -66,4 +70,16 @@ test "it performs ReverseProxy.call when signature valid", %{conn: conn} do
assert %Plug.Conn{status: :success} = get(conn, url) assert %Plug.Conn{status: :success} = get(conn, url)
end end
end end
test "it returns 404 when url contains in banned_urls cache", %{conn: conn} do
Config.put([:media_proxy, :enabled], true)
Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
url = Pleroma.Web.MediaProxy.encode_url("https://google.fn/test.png")
Pleroma.Web.MediaProxy.put_in_banned_urls("https://google.fn/test.png")
with_mock Pleroma.ReverseProxy,
call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do
assert %Plug.Conn{status: 404, resp_body: "Not Found"} = get(conn, url)
end
end
end end