From cb40602a167f4637dc6df6633ec2dfe33f774177 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 15 May 2020 21:34:46 +0300 Subject: [PATCH 1/6] added media proxy invalidation --- config/config.exs | 7 +++ lib/pleroma/object.ex | 34 ++++++++----- lib/pleroma/web/media_proxy/invalidation.ex | 19 +++++++ .../web/media_proxy/invalidations/nginx.ex | 12 +++++ .../web/media_proxy/invalidations/script.ex | 10 ++++ .../workers/attachments_cleanup_worker.ex | 49 ++++++++++++------- 6 files changed, 101 insertions(+), 30 deletions(-) create mode 100644 lib/pleroma/web/media_proxy/invalidation.ex create mode 100644 lib/pleroma/web/media_proxy/invalidations/nginx.ex create mode 100644 lib/pleroma/web/media_proxy/invalidations/script.ex diff --git a/config/config.exs b/config/config.exs index e703c1632..5394c7c7a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -378,6 +378,13 @@ config :pleroma, :media_proxy, enabled: false, + invalidation: [ + enabled: false, + provider: Pleroma.Web.MediaProxy.Invalidation.Script, + options: %{ + script_path: "" + } + ], proxy_opts: [ redirect_on_failure: false, max_body_length: 25 * 1_048_576, diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index e678fd415..66b233498 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -9,11 +9,13 @@ defmodule Pleroma.Object do import Ecto.Changeset alias Pleroma.Activity + alias Pleroma.Config alias Pleroma.Object alias Pleroma.Object.Fetcher alias Pleroma.ObjectTombstone alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Workers.AttachmentsCleanupWorker require Logger @@ -183,27 +185,37 @@ def swap_object_with_tombstone(object) do def delete(%Object{data: %{"id" => id}} = object) do with {:ok, _obj} = swap_object_with_tombstone(object), deleted_activity = Activity.delete_all_by_object_ap_id(id), - {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), - {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do - with true <- Pleroma.Config.get([:instance, :cleanup_attachments]) do - {:ok, _} = - Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{ - "object" => object - }) - end + {:ok, _} <- invalid_object_cache(object) do + cleanup_attachments( + Config.get([:instance, :cleanup_attachments]), + %{"object" => object} + ) {:ok, object, deleted_activity} end end - def prune(%Object{data: %{"id" => id}} = object) do + @spec cleanup_attachments(boolean(), %{required(:object) => map()}) :: + {:ok, Oban.Job.t() | nil} + def cleanup_attachments(true, %{"object" => _} = params) do + AttachmentsCleanupWorker.enqueue("cleanup_attachments", params) + end + + def cleanup_attachments(_, _), do: {:ok, nil} + + def prune(%Object{data: %{"id" => _id}} = object) do with {:ok, object} <- Repo.delete(object), - {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), - {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do + {:ok, _} <- invalid_object_cache(object) do {:ok, object} end end + def invalid_object_cache(%Object{data: %{"id" => id}}) do + with {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do + Cachex.del(:web_resp_cache, URI.parse(id).path) + end + end + def set_cache(%Object{data: %{"id" => ap_id}} = object) do Cachex.put(:object_cache, "object:#{ap_id}", object) {:ok, object} diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex new file mode 100644 index 000000000..dd9a53a27 --- /dev/null +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -0,0 +1,19 @@ +defmodule Pleroma.Web.MediaProxy.Invalidation do + @callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()} + + alias Pleroma.Config + + def purge(urls) do + [:media_proxy, :invalidation, :enabled] + |> Config.get() + |> do_purge(urls) + end + + defp do_purge(true, urls) do + config = Config.get([:media_proxy, :invalidation]) + config[:provider].purge(urls, config[:options]) + :ok + end + + defp do_purge(_, _), do: :ok +end diff --git a/lib/pleroma/web/media_proxy/invalidations/nginx.ex b/lib/pleroma/web/media_proxy/invalidations/nginx.ex new file mode 100644 index 000000000..5bfdd505c --- /dev/null +++ b/lib/pleroma/web/media_proxy/invalidations/nginx.ex @@ -0,0 +1,12 @@ +defmodule Pleroma.Web.MediaProxy.Invalidation.Nginx do + @behaviour Pleroma.Web.MediaProxy.Invalidation + + @impl Pleroma.Web.MediaProxy.Invalidation + def purge(urls, _opts) do + Enum.each(urls, fn url -> + Pleroma.HTTP.request(:purge, url, "", [], []) + end) + + {:ok, "success"} + end +end diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex new file mode 100644 index 000000000..f458845a0 --- /dev/null +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -0,0 +1,10 @@ +defmodule Pleroma.Web.MediaProxy.Invalidation.Script do + @behaviour Pleroma.Web.MediaProxy.Invalidation + + @impl Pleroma.Web.MediaProxy.Invalidation + def purge(urls, %{script_path: script_path} = options) do + script_args = List.wrap(Map.get(options, :script_args, [])) + System.cmd(Path.expand(script_path), [urls] ++ script_args) + {:ok, "success"} + end +end diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 3c5820a86..49352db2a 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -27,8 +27,20 @@ def perform( uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + prefix = + case Pleroma.Config.get([Pleroma.Upload, :base_url]) do + nil -> "media" + _ -> "" + end + + base_url = + String.trim_trailing( + Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()), + "/" + ) + # find all objects for copies of the attachments, name and actor doesn't matter here - delete_ids = + object_ids_and_hrefs = from(o in Object, where: fragment( @@ -67,29 +79,28 @@ def perform( |> Enum.map(fn {href, %{id: id, count: count}} -> # only delete files that have single instance with 1 <- count do - prefix = - case Pleroma.Config.get([Pleroma.Upload, :base_url]) do - nil -> "media" - _ -> "" - end + href + |> String.trim_leading("#{base_url}/#{prefix}") + |> uploader.delete_file() - base_url = - String.trim_trailing( - Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()), - "/" - ) - - file_path = String.trim_leading(href, "#{base_url}/#{prefix}") - - uploader.delete_file(file_path) + {id, href} + else + _ -> {id, nil} end - - id end) - from(o in Object, where: o.id in ^delete_ids) + object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end) + + from(o in Object, where: o.id in ^object_ids) |> Repo.delete_all() + + object_ids_and_hrefs + |> Enum.filter(fn {_, href} -> not is_nil(href) end) + |> Enum.map(&elem(&1, 1)) + |> Pleroma.Web.MediaProxy.Invalidation.purge() + + {:ok, :success} end - def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: :ok + def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip} end From 3f8d68bdf3224cd6023b3d7f8e64221222872820 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Sat, 16 May 2020 15:16:33 +0300 Subject: [PATCH 2/6] added example cache purge script --- config/config.exs | 2 +- installation/nginx-cache-purge.example | 39 +++++++++++++++++++ .../web/media_proxy/invalidations/http.ex | 16 ++++++++ .../web/media_proxy/invalidations/nginx.ex | 12 ------ .../web/media_proxy/invalidations/script.ex | 11 ++++-- 5 files changed, 64 insertions(+), 16 deletions(-) create mode 100755 installation/nginx-cache-purge.example create mode 100644 lib/pleroma/web/media_proxy/invalidations/http.ex delete mode 100644 lib/pleroma/web/media_proxy/invalidations/nginx.ex diff --git a/config/config.exs b/config/config.exs index 5394c7c7a..882d25069 100644 --- a/config/config.exs +++ b/config/config.exs @@ -382,7 +382,7 @@ enabled: false, provider: Pleroma.Web.MediaProxy.Invalidation.Script, options: %{ - script_path: "" + script_path: "./installation/nginx-cache-purge.example" } ], proxy_opts: [ diff --git a/installation/nginx-cache-purge.example b/installation/nginx-cache-purge.example new file mode 100755 index 000000000..12dfa733c --- /dev/null +++ b/installation/nginx-cache-purge.example @@ -0,0 +1,39 @@ +#!/bin/bash + +# A simple Bash script to delete an media from the Nginx cache. + +SCRIPTNAME=${0##*/} + +# NGINX cache directory +CACHE_DIRECTORY="/tmp/pleroma-media-cache" + +function get_cache_files() { + local max_parallel=${3-16} + find $2 -maxdepth 1 -type d | xargs -P $max_parallel -n 1 grep -ERl "^KEY:.*$1" | sort -u +} + +function purge_item() { + local cache_files + cache_files=$(get_cache_files "$1" "$2") + + if [ -n "$cache_files" ]; then + for i in $cache_files; do + [ -f $i ] || continue + echo "Deleting $i from $2." + rm $i + done + else + echo "$1 is not cached." + fi +} + +function purge() { + for url in "$@" + do + echo "$SCRIPTNAME delete $url from cache ($CACHE_DIRECTORY)" + purge_item $url $CACHE_DIRECTORY + done + +} + +purge $1 diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex new file mode 100644 index 000000000..40c624efc --- /dev/null +++ b/lib/pleroma/web/media_proxy/invalidations/http.ex @@ -0,0 +1,16 @@ +defmodule Pleroma.Web.MediaProxy.Invalidation.Http do + @behaviour Pleroma.Web.MediaProxy.Invalidation + + @impl Pleroma.Web.MediaProxy.Invalidation + def purge(urls, opts) do + method = Map.get(opts, :http_method, :purge) + headers = Map.get(opts, :http_headers, []) + options = Map.get(opts, :http_options, []) + + Enum.each(urls, fn url -> + Pleroma.HTTP.request(method, url, "", headers, options) + end) + + {:ok, "success"} + end +end diff --git a/lib/pleroma/web/media_proxy/invalidations/nginx.ex b/lib/pleroma/web/media_proxy/invalidations/nginx.ex deleted file mode 100644 index 5bfdd505c..000000000 --- a/lib/pleroma/web/media_proxy/invalidations/nginx.ex +++ /dev/null @@ -1,12 +0,0 @@ -defmodule Pleroma.Web.MediaProxy.Invalidation.Nginx do - @behaviour Pleroma.Web.MediaProxy.Invalidation - - @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, _opts) do - Enum.each(urls, fn url -> - Pleroma.HTTP.request(:purge, url, "", [], []) - end) - - {:ok, "success"} - end -end diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index f458845a0..94c79511a 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -2,9 +2,14 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do @behaviour Pleroma.Web.MediaProxy.Invalidation @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, %{script_path: script_path} = options) do - script_args = List.wrap(Map.get(options, :script_args, [])) - System.cmd(Path.expand(script_path), [urls] ++ script_args) + def purge(urls, %{script_path: script_path} = _options) do + args = + urls + |> List.wrap() + |> Enum.uniq() + |> Enum.join(" ") + + System.cmd(Path.expand(script_path), [args]) {:ok, "success"} end end From c33a4315fb09e67d0ed5f644877054a3fb7b1fe1 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 18 May 2020 06:48:19 +0300 Subject: [PATCH 3/6] updated docs --- config/config.exs | 5 +---- docs/configuration/cheatsheet.md | 20 +++++++++++++++++++ lib/pleroma/web/media_proxy/invalidation.ex | 5 +++-- .../web/media_proxy/invalidations/http.ex | 6 +++--- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/config/config.exs b/config/config.exs index 882d25069..25cf2a9b9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -380,10 +380,7 @@ enabled: false, invalidation: [ enabled: false, - provider: Pleroma.Web.MediaProxy.Invalidation.Script, - options: %{ - script_path: "./installation/nginx-cache-purge.example" - } + provider: Pleroma.Web.MediaProxy.Invalidation.Script ], proxy_opts: [ redirect_on_failure: false, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 1078c4e87..aaea3f46c 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -249,6 +249,26 @@ This section describe PWA manifest instance-specific values. Currently this opti * `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts. * `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`. * `whitelist`: List of domains to bypass the mediaproxy +* `invalidation`: options for remove media from cache after delete object: + * `enabled`: Enables purge cache + * `provider`: Which one of the [purge cache strategy](#purge-cache-strategy) to use. + +### Purge cache strategy + +#### Pleroma.Web.MediaProxy.Invalidation.Script + +This strategy allow perform external bash script to purge cache. +Urls of attachments pass to script as arguments. + +* `script_path`: path to external script. + +#### Pleroma.Web.MediaProxy.Invalidation.Http + +This strategy allow perform custom http request to purge cache. + +* `method`: http method. default is `purge` +* `headers`: http headers. default is empty +* `options`: request options. default is empty ## Link previews diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index dd9a53a27..371aa8ae0 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -10,8 +10,9 @@ def purge(urls) do end defp do_purge(true, urls) do - config = Config.get([:media_proxy, :invalidation]) - config[:provider].purge(urls, config[:options]) + provider = Config.get([:media_proxy, :invalidation, :provider]) + options = Config.get(provider) + provider.purge(urls, options) :ok end diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex index 40c624efc..66fafa7ba 100644 --- a/lib/pleroma/web/media_proxy/invalidations/http.ex +++ b/lib/pleroma/web/media_proxy/invalidations/http.ex @@ -3,9 +3,9 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do @impl Pleroma.Web.MediaProxy.Invalidation def purge(urls, opts) do - method = Map.get(opts, :http_method, :purge) - headers = Map.get(opts, :http_headers, []) - options = Map.get(opts, :http_options, []) + method = Map.get(opts, :method, :purge) + headers = Map.get(opts, :headers, []) + options = Map.get(opts, :options, []) Enum.each(urls, fn url -> Pleroma.HTTP.request(method, url, "", headers, options) From 5f0a3ac74d51333a778e6be26876fe26b0ff625b Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 18 May 2020 09:22:26 +0300 Subject: [PATCH 4/6] added tests --- docs/configuration/cheatsheet.md | 14 ++++++++ lib/pleroma/web/media_proxy/invalidation.ex | 8 ++++- .../web/media_proxy/invalidations/http.ex | 26 +++++++++++++- .../web/media_proxy/invalidations/script.ex | 30 ++++++++++++++-- .../media_proxy/invalidations/http_test.exs | 35 +++++++++++++++++++ .../media_proxy/invalidations/script_test.exs | 20 +++++++++++ 6 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 test/web/media_proxy/invalidations/http_test.exs create mode 100644 test/web/media_proxy/invalidations/script_test.exs diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index aaea3f46c..ddea6a4fb 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -262,6 +262,12 @@ Urls of attachments pass to script as arguments. * `script_path`: path to external script. +Example: +```elixir +config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, + script_path: "./installation/nginx-cache-purge.example" +``` + #### Pleroma.Web.MediaProxy.Invalidation.Http This strategy allow perform custom http request to purge cache. @@ -270,6 +276,14 @@ This strategy allow perform custom http request to purge cache. * `headers`: http headers. default is empty * `options`: request options. default is empty +Example: +```elixir +config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http, + method: :purge, + headers: [], + options: [] +``` + ## Link previews ### Pleroma.Web.Metadata (provider) diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index 371aa8ae0..c037ff13e 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -1,8 +1,15 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.MediaProxy.Invalidation do + @moduledoc false + @callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()} alias Pleroma.Config + @spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()} def purge(urls) do [:media_proxy, :invalidation, :enabled] |> Config.get() @@ -13,7 +20,6 @@ defp do_purge(true, urls) do provider = Config.get([:media_proxy, :invalidation, :provider]) options = Config.get(provider) provider.purge(urls, options) - :ok end defp do_purge(_, _), do: :ok diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex index 66fafa7ba..07248df6e 100644 --- a/lib/pleroma/web/media_proxy/invalidations/http.ex +++ b/lib/pleroma/web/media_proxy/invalidations/http.ex @@ -1,16 +1,40 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.MediaProxy.Invalidation.Http do + @moduledoc false @behaviour Pleroma.Web.MediaProxy.Invalidation + require Logger + @impl Pleroma.Web.MediaProxy.Invalidation def purge(urls, opts) do method = Map.get(opts, :method, :purge) headers = Map.get(opts, :headers, []) options = Map.get(opts, :options, []) + Logger.debug("Running cache purge: #{inspect(urls)}") + Enum.each(urls, fn url -> - Pleroma.HTTP.request(method, url, "", headers, options) + with {:error, error} <- do_purge(method, url, headers, options) do + Logger.error("Error while cache purge: url - #{url}, error: #{inspect(error)}") + end end) {:ok, "success"} end + + defp do_purge(method, url, headers, options) do + case Pleroma.HTTP.request(method, url, "", headers, options) do + {:ok, %{status: status} = env} when 400 <= status and status < 500 -> + {:error, env} + + {:error, error} = error -> + error + + _ -> + {:ok, "success"} + end + end end diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index 94c79511a..6be782132 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -1,6 +1,14 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.MediaProxy.Invalidation.Script do + @moduledoc false + @behaviour Pleroma.Web.MediaProxy.Invalidation + require Logger + @impl Pleroma.Web.MediaProxy.Invalidation def purge(urls, %{script_path: script_path} = _options) do args = @@ -9,7 +17,25 @@ def purge(urls, %{script_path: script_path} = _options) do |> Enum.uniq() |> Enum.join(" ") - System.cmd(Path.expand(script_path), [args]) - {:ok, "success"} + path = Path.expand(script_path) + + Logger.debug("Running cache purge: #{inspect(urls)}, #{path}") + + case do_purge(path, [args]) do + {result, exit_status} when exit_status > 0 -> + Logger.error("Error while cache purge: #{inspect(result)}") + {:error, inspect(result)} + + _ -> + {:ok, "success"} + end + end + + def purge(_, _), do: {:error, "not found script path"} + + defp do_purge(path, args) do + System.cmd(path, args) + rescue + error -> {inspect(error), 1} end end diff --git a/test/web/media_proxy/invalidations/http_test.exs b/test/web/media_proxy/invalidations/http_test.exs new file mode 100644 index 000000000..8a3b4141c --- /dev/null +++ b/test/web/media_proxy/invalidations/http_test.exs @@ -0,0 +1,35 @@ +defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do + use ExUnit.Case + alias Pleroma.Web.MediaProxy.Invalidation + + import ExUnit.CaptureLog + import Tesla.Mock + + test "logs hasn't error message when request is valid" do + mock(fn + %{method: :purge, url: "http://example.com/media/example.jpg"} -> + %Tesla.Env{status: 200} + end) + + refute capture_log(fn -> + assert Invalidation.Http.purge( + ["http://example.com/media/example.jpg"], + %{} + ) == {:ok, "success"} + end) =~ "Error while cache purge" + end + + test "it write error message in logs when request invalid" do + mock(fn + %{method: :purge, url: "http://example.com/media/example1.jpg"} -> + %Tesla.Env{status: 404} + end) + + assert capture_log(fn -> + assert Invalidation.Http.purge( + ["http://example.com/media/example1.jpg"], + %{} + ) == {:ok, "success"} + end) =~ "Error while cache purge: url - http://example.com/media/example1.jpg" + end +end diff --git a/test/web/media_proxy/invalidations/script_test.exs b/test/web/media_proxy/invalidations/script_test.exs new file mode 100644 index 000000000..1358963ab --- /dev/null +++ b/test/web/media_proxy/invalidations/script_test.exs @@ -0,0 +1,20 @@ +defmodule Pleroma.Web.MediaProxy.Invalidation.ScriptTest do + use ExUnit.Case + alias Pleroma.Web.MediaProxy.Invalidation + + import ExUnit.CaptureLog + + test "it logger error when script not found" do + assert capture_log(fn -> + assert Invalidation.Script.purge( + ["http://example.com/media/example.jpg"], + %{script_path: "./example"} + ) == {:error, "\"%ErlangError{original: :enoent}\""} + end) =~ "Error while cache purge: \"%ErlangError{original: :enoent}\"" + + assert Invalidation.Script.purge( + ["http://example.com/media/example.jpg"], + %{} + ) == {:error, "not found script path"} + end +end From b5b9d161cddd1b6650cde00cf0f3cbf56ab7a4a3 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 20 May 2020 06:56:04 +0300 Subject: [PATCH 5/6] update purge script --- installation/nginx-cache-purge.example | 39 ---------------------- installation/nginx-cache-purge.sh.example | 40 +++++++++++++++++++++++ 2 files changed, 40 insertions(+), 39 deletions(-) delete mode 100755 installation/nginx-cache-purge.example create mode 100755 installation/nginx-cache-purge.sh.example diff --git a/installation/nginx-cache-purge.example b/installation/nginx-cache-purge.example deleted file mode 100755 index 12dfa733c..000000000 --- a/installation/nginx-cache-purge.example +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# A simple Bash script to delete an media from the Nginx cache. - -SCRIPTNAME=${0##*/} - -# NGINX cache directory -CACHE_DIRECTORY="/tmp/pleroma-media-cache" - -function get_cache_files() { - local max_parallel=${3-16} - find $2 -maxdepth 1 -type d | xargs -P $max_parallel -n 1 grep -ERl "^KEY:.*$1" | sort -u -} - -function purge_item() { - local cache_files - cache_files=$(get_cache_files "$1" "$2") - - if [ -n "$cache_files" ]; then - for i in $cache_files; do - [ -f $i ] || continue - echo "Deleting $i from $2." - rm $i - done - else - echo "$1 is not cached." - fi -} - -function purge() { - for url in "$@" - do - echo "$SCRIPTNAME delete $url from cache ($CACHE_DIRECTORY)" - purge_item $url $CACHE_DIRECTORY - done - -} - -purge $1 diff --git a/installation/nginx-cache-purge.sh.example b/installation/nginx-cache-purge.sh.example new file mode 100755 index 000000000..aaa195324 --- /dev/null +++ b/installation/nginx-cache-purge.sh.example @@ -0,0 +1,40 @@ +#!/bin/sh + +# A simple shell script to delete a media from the Nginx cache. + +SCRIPTNAME=${0##*/} + +# NGINX cache directory +CACHE_DIRECTORY="/tmp/pleroma-media-cache" + +## Return the files where the items are cached. +## $1 - the filename, can be a pattern . +## $2 - the cache directory. +## $3 - (optional) the number of parallel processes to run for grep. +get_cache_files() { + local max_parallel=${3-16} + find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -ERl "^KEY:.*$1" | sort -u +} + +## Removes an item from the given cache zone. +## $1 - the filename, can be a pattern . +## $2 - the cache directory. +purge_item() { + for f in $(get_cache_files $1 $2); do + echo "found file: $f" + [ -f $f ] || continue + echo "Deleting $f from $2." + rm $f + done +} # purge_item + +purge() { + for url in "$@" + do + echo "$SCRIPTNAME delete \`$url\` from cache ($CACHE_DIRECTORY)" + purge_item $url $CACHE_DIRECTORY + done + +} + +purge $1 From 376147fb828a75b5000262a376cee173bfc98551 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 20 May 2020 04:12:21 +0000 Subject: [PATCH 6/6] Apply suggestion to installation/nginx-cache-purge.sh.example --- installation/nginx-cache-purge.sh.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installation/nginx-cache-purge.sh.example b/installation/nginx-cache-purge.sh.example index aaa195324..b2915321c 100755 --- a/installation/nginx-cache-purge.sh.example +++ b/installation/nginx-cache-purge.sh.example @@ -13,7 +13,7 @@ CACHE_DIRECTORY="/tmp/pleroma-media-cache" ## $3 - (optional) the number of parallel processes to run for grep. get_cache_files() { local max_parallel=${3-16} - find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -ERl "^KEY:.*$1" | sort -u + find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E Rl "^KEY:.*$1" | sort -u } ## Removes an item from the given cache zone.