Add backups deletion

This commit is contained in:
Egor Kislitsyn 2020-09-04 21:48:52 +04:00
parent 3ad7492f9d
commit 739cb1463b
No known key found for this signature in database
GPG Key ID: 1B49CB15B71E7805
4 changed files with 91 additions and 10 deletions

View File

@ -15,6 +15,7 @@ defmodule Pleroma.Backup do
alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.UserView alias Pleroma.Web.ActivityPub.UserView
alias Pleroma.Workers.BackupWorker
schema "backups" do schema "backups" do
field(:content_type, :string) field(:content_type, :string)
@ -30,7 +31,7 @@ defmodule Pleroma.Backup do
def create(user) do def create(user) do
with :ok <- validate_limit(user), with :ok <- validate_limit(user),
{:ok, backup} <- user |> new() |> Repo.insert() do {:ok, backup} <- user |> new() |> Repo.insert() do
Pleroma.Workers.BackupWorker.enqueue("process", %{"backup_id" => backup.id}) BackupWorker.process(backup)
end end
end end
@ -46,6 +47,14 @@ def new(user) do
} }
end end
def delete(backup) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
with :ok <- uploader.delete_file("backups/" <> backup.file_name) do
Repo.delete(backup)
end
end
defp validate_limit(user) do defp validate_limit(user) do
case get_last(user.id) do case get_last(user.id) do
%__MODULE__{inserted_at: inserted_at} -> %__MODULE__{inserted_at: inserted_at} ->
@ -75,7 +84,8 @@ def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
__MODULE__ __MODULE__
|> where(user_id: ^user_id) |> where(user_id: ^user_id)
|> where([b], b.id != ^latest_id) |> where([b], b.id != ^latest_id)
|> Repo.delete_all() |> Repo.all()
|> Enum.each(&BackupWorker.delete/1)
end end
def get(id), do: Repo.get(__MODULE__, id) def get(id), do: Repo.get(__MODULE__, id)

View File

@ -3,15 +3,46 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.BackupWorker do defmodule Pleroma.Workers.BackupWorker do
use Oban.Worker, queue: :backup, max_attempts: 1
alias Oban.Job
alias Pleroma.Backup alias Pleroma.Backup
use Pleroma.Workers.WorkerHelper, queue: "backup" def process(backup) do
%{"op" => "process", "backup_id" => backup.id}
|> new()
|> Oban.insert()
end
def schedule_deletion(backup) do
days = Pleroma.Config.get([Pleroma.Backup, :purge_after_days])
time = 60 * 60 * 24 * days
scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time)
%{"op" => "delete", "backup_id" => backup.id}
|> new(scheduled_at: scheduled_at)
|> Oban.insert()
end
def delete(backup) do
%{"op" => "delete", "backup_id" => backup.id}
|> new()
|> Oban.insert()
end
@impl Oban.Worker
def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do
with {:ok, %Backup{} = backup} <- with {:ok, %Backup{} = backup} <-
backup_id |> Backup.get() |> Backup.process() do backup_id |> Backup.get() |> Backup.process(),
{:ok, _job} <- schedule_deletion(backup),
:ok <- Backup.remove_outdated(backup) do
{:ok, backup} {:ok, backup}
end end
end end
def perform(%Job{args: %{"op" => "delete", "backup_id" => backup_id}}) do
case Backup.get(backup_id) do
%Backup{} = backup -> Backup.delete(backup)
nil -> :ok
end
end
end end

View File

@ -13,8 +13,12 @@ defmodule Pleroma.BackupTest do
alias Pleroma.Bookmark alias Pleroma.Bookmark
alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI
alias Pleroma.Workers.BackupWorker alias Pleroma.Workers.BackupWorker
alias Pleroma.Tests.ObanHelpers
setup do: clear_config([Pleroma.Upload, :uploader]) setup do
clear_config([Pleroma.Upload, :uploader])
clear_config([Pleroma.Backup, :limit_days])
end
test "it creates a backup record and an Oban job" do test "it creates a backup record and an Oban job" do
%{id: user_id} = user = insert(:user) %{id: user_id} = user = insert(:user)
@ -38,10 +42,34 @@ test "it return an error if the export limit is over" do
test "it process a backup record" do test "it process a backup record" do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
%{id: user_id} = user = insert(:user) %{id: user_id} = user = insert(:user)
assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}} = job} = Backup.create(user)
assert {:ok, backup} = BackupWorker.perform(job) assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id} = args}} = Backup.create(user)
assert {:ok, backup} = perform_job(BackupWorker, args)
assert backup.file_size > 0 assert backup.file_size > 0
assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
delete_job_args = %{"op" => "delete", "backup_id" => backup_id}
assert_enqueued(worker: BackupWorker, args: delete_job_args)
assert {:ok, backup} = perform_job(BackupWorker, delete_job_args)
refute Backup.get(backup_id)
end
test "it removes outdated backups after creating a fresh one" do
Pleroma.Config.put([Pleroma.Backup, :limit_days], -1)
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
user = insert(:user)
assert {:ok, job1} = Backup.create(user)
assert {:ok, %Backup{id: backup1_id}} = ObanHelpers.perform(job1)
assert {:ok, job2} = Backup.create(user)
assert Pleroma.Repo.aggregate(Backup, :count) == 2
assert {:ok, backup2} = ObanHelpers.perform(job2)
ObanHelpers.perform_all()
assert [^backup2] = Pleroma.Repo.all(Backup)
end end
test "it creates a zip archive with user data" do test "it creates a zip archive with user data" do
@ -145,7 +173,7 @@ test "it creates a zip archive with user data" do
File.rm!(path) File.rm!(path)
end end
describe "it uploads a backup archive" do describe "it uploads and deletes a backup archive" do
setup do setup do
clear_config(Pleroma.Uploaders.S3, clear_config(Pleroma.Uploaders.S3,
bucket: "test_bucket", bucket: "test_bucket",
@ -173,8 +201,16 @@ test "it creates a zip archive with user data" do
test "S3", %{path: path, backup: backup} do test "S3", %{path: path, backup: backup} do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3) Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
with_mock ExAws, request: fn _ -> {:ok, :ok} end do with_mock ExAws,
request: fn
%{http_method: :put} -> {:ok, :ok}
%{http_method: :delete} -> {:ok, %{status_code: 204}}
end do
assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
assert {:ok, _backup} = Backup.delete(backup)
end
with_mock ExAws, request: fn %{http_method: :delete} -> {:ok, %{status_code: 204}} end do
end end
end end
@ -182,6 +218,7 @@ test "Local", %{path: path, backup: backup} do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
assert {:ok, _backup} = Backup.delete(backup)
end end
end end
end end

View File

@ -7,6 +7,8 @@ defmodule Pleroma.Tests.ObanHelpers do
Oban test helpers. Oban test helpers.
""" """
require Ecto.Query
alias Pleroma.Repo alias Pleroma.Repo
def wipe_all do def wipe_all do
@ -15,6 +17,7 @@ def wipe_all do
def perform_all do def perform_all do
Oban.Job Oban.Job
|> Ecto.Query.where(state: "available")
|> Repo.all() |> Repo.all()
|> perform() |> perform()
end end