Merge branch 'feature/make_attachments_cleanup_optional' into 'develop'

Make attachments cleanup optional

See merge request pleroma/pleroma!2170
This commit is contained in:
lain 2020-02-04 15:29:26 +00:00
commit a56db78935
6 changed files with 64 additions and 8 deletions

View File

@ -12,7 +12,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Changed ### Changed
- **Breaking:** Pleroma won't start if it detects unapplied migrations - **Breaking:** Pleroma won't start if it detects unapplied migrations
- **Breaking:** attachments are removed along with statuses. Does not affect duplicate files and attachments without status.
- **Breaking:** Elixir >=1.8 is now required (was >= 1.7) - **Breaking:** Elixir >=1.8 is now required (was >= 1.7)
- **Breaking:** `Pleroma.Plugs.RemoteIp` and `:rate_limiter` enabled by default. Please ensure your reverse proxy forwards the real IP! - **Breaking:** `Pleroma.Plugs.RemoteIp` and `:rate_limiter` enabled by default. Please ensure your reverse proxy forwards the real IP!
- **Breaking:** attachment links (`config :pleroma, :instance, no_attachment_links` and `config :pleroma, Pleroma.Upload, link_name`) disabled by default - **Breaking:** attachment links (`config :pleroma, :instance, no_attachment_links` and `config :pleroma, Pleroma.Upload, link_name`) disabled by default
@ -54,6 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Added ### Added
- `:chat_limit` option to limit chat characters. - `:chat_limit` option to limit chat characters.
- `cleanup_attachments` option to remove attachments along with statuses. Does not affect duplicate files and attachments without status. Enabling this will increase load to database when deleting statuses on larger instances.
- Refreshing poll results for remote polls - Refreshing poll results for remote polls
- Authentication: Added rate limit for password-authorized actions / login existence checks - Authentication: Added rate limit for password-authorized actions / login existence checks
- Static Frontend: Add the ability to render user profiles and notices server-side without requiring JS app. - Static Frontend: Add the ability to render user profiles and notices server-side without requiring JS app.

View File

@ -271,7 +271,8 @@
account_field_name_length: 512, account_field_name_length: 512,
account_field_value_length: 2048, account_field_value_length: 2048,
external_user_synchronization: true, external_user_synchronization: true,
extended_nickname_format: true extended_nickname_format: true,
cleanup_attachments: false
config :pleroma, :feed, config :pleroma, :feed,
post_title: %{ post_title: %{

View File

@ -764,6 +764,15 @@
"Set to `true` to use extended local nicknames format (allows underscores/dashes)." <> "Set to `true` to use extended local nicknames format (allows underscores/dashes)." <>
" This will break federation with older software for theses nicknames." " This will break federation with older software for theses nicknames."
}, },
%{
key: :cleanup_attachments,
type: :boolean,
description: """
"Set to `true` to remove associated attachments when status is removed.
This will not affect duplicates and attachments without status.
Enabling this will increase load to database when deleting statuses on larger instances.
"""
},
%{ %{
key: :max_pinned_statuses, key: :max_pinned_statuses,
type: :integer, type: :integer,

View File

@ -69,6 +69,7 @@ You shouldn't edit the base config directly to avoid breakages and merge conflic
* `account_field_name_length`: An account field name maximum length (default: `512`). * `account_field_name_length`: An account field name maximum length (default: `512`).
* `account_field_value_length`: An account field value maximum length (default: `2048`). * `account_field_value_length`: An account field value maximum length (default: `2048`).
* `external_user_synchronization`: Enabling following/followers counters synchronization for external users. * `external_user_synchronization`: Enabling following/followers counters synchronization for external users.
* `cleanup_attachments`: Remove attachments along with statuses. Does not affect duplicate files and attachments without status. Enabling this will increase load to database when deleting statuses on larger instances.
## Federation ## Federation
### MRF policies ### MRF policies

View File

@ -184,11 +184,14 @@ def delete(%Object{data: %{"id" => id}} = object) do
with {:ok, _obj} = swap_object_with_tombstone(object), with {:ok, _obj} = swap_object_with_tombstone(object),
deleted_activity = Activity.delete_all_by_object_ap_id(id), deleted_activity = Activity.delete_all_by_object_ap_id(id),
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
{:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path), {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
{:ok, _} <- with true <- Pleroma.Config.get([:instance, :cleanup_attachments]) do
{:ok, _} =
Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{ Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{
"object" => object "object" => object
}) do })
end
{:ok, object, deleted_activity} {:ok, object, deleted_activity}
end end
end end

View File

@ -76,8 +76,9 @@ test "ensures cache is cleared for the object" do
describe "delete attachments" do describe "delete attachments" do
clear_config([Pleroma.Upload]) clear_config([Pleroma.Upload])
test "in subdirectories" do test "Disabled via config" do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
Pleroma.Config.put([:instance, :cleanup_attachments], false)
file = %Plug.Upload{ file = %Plug.Upload{
content_type: "image/jpg", content_type: "image/jpg",
@ -103,6 +104,41 @@ test "in subdirectories" do
ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker)) ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))
assert Object.get_by_id(note.id).data["deleted"]
refute Object.get_by_id(attachment.id) == nil
assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
end
test "in subdirectories" do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
Pleroma.Config.put([:instance, :cleanup_attachments], true)
file = %Plug.Upload{
content_type: "image/jpg",
path: Path.absname("test/fixtures/image.jpg"),
filename: "an_image.jpg"
}
user = insert(:user)
{:ok, %Object{} = attachment} =
Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)
%{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
path = href |> Path.dirname() |> Path.basename()
assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
Object.delete(note)
ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))
assert Object.get_by_id(note.id).data["deleted"]
assert Object.get_by_id(attachment.id) == nil assert Object.get_by_id(attachment.id) == nil
assert {:ok, []} == File.ls("#{uploads_dir}/#{path}") assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
@ -111,6 +147,7 @@ test "in subdirectories" do
test "with dedupe enabled" do test "with dedupe enabled" do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
Pleroma.Config.put([Pleroma.Upload, :filters], [Pleroma.Upload.Filter.Dedupe]) Pleroma.Config.put([Pleroma.Upload, :filters], [Pleroma.Upload.Filter.Dedupe])
Pleroma.Config.put([:instance, :cleanup_attachments], true)
uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads]) uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
@ -139,6 +176,7 @@ test "with dedupe enabled" do
ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker)) ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))
assert Object.get_by_id(note.id).data["deleted"]
assert Object.get_by_id(attachment.id) == nil assert Object.get_by_id(attachment.id) == nil
assert {:ok, files} = File.ls(uploads_dir) assert {:ok, files} = File.ls(uploads_dir)
refute filename in files refute filename in files
@ -146,6 +184,7 @@ test "with dedupe enabled" do
test "with objects that have legacy data.url attribute" do test "with objects that have legacy data.url attribute" do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
Pleroma.Config.put([:instance, :cleanup_attachments], true)
file = %Plug.Upload{ file = %Plug.Upload{
content_type: "image/jpg", content_type: "image/jpg",
@ -173,6 +212,7 @@ test "with objects that have legacy data.url attribute" do
ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker)) ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))
assert Object.get_by_id(note.id).data["deleted"]
assert Object.get_by_id(attachment.id) == nil assert Object.get_by_id(attachment.id) == nil
assert {:ok, []} == File.ls("#{uploads_dir}/#{path}") assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
@ -181,6 +221,7 @@ test "with objects that have legacy data.url attribute" do
test "With custom base_url" do test "With custom base_url" do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
Pleroma.Config.put([Pleroma.Upload, :base_url], "https://sub.domain.tld/dir/") Pleroma.Config.put([Pleroma.Upload, :base_url], "https://sub.domain.tld/dir/")
Pleroma.Config.put([:instance, :cleanup_attachments], true)
file = %Plug.Upload{ file = %Plug.Upload{
content_type: "image/jpg", content_type: "image/jpg",
@ -206,6 +247,7 @@ test "With custom base_url" do
ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker)) ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))
assert Object.get_by_id(note.id).data["deleted"]
assert Object.get_by_id(attachment.id) == nil assert Object.get_by_id(attachment.id) == nil
assert {:ok, []} == File.ls("#{uploads_dir}/#{path}") assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")