example of flexible storage backends

This commit is contained in:
Thurloat 2018-08-27 22:20:54 -03:00
parent 49b165ddc6
commit 709816a0f8
5 changed files with 77 additions and 67 deletions

View File

@ -11,9 +11,13 @@
config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes
config :pleroma, Pleroma.Upload, config :pleroma, Pleroma.Upload,
uploads: "uploads", uploader: Pleroma.Uploaders.Local
strip_exif: false, strip_exif: false
use_s3: false,
config :pleroma, Pleroma.Uploaders.Local,
uploads: "uploads"
config :pleroma, Pleroma.Uploaders.S3,
s3_bucket: nil s3_bucket: nil
config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"] config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"]

View File

@ -4,31 +4,15 @@ defmodule Pleroma.Upload do
def store(%Plug.Upload{} = file, should_dedupe) do def store(%Plug.Upload{} = file, should_dedupe) do
settings = Application.get_env(:pleroma, Pleroma.Upload) settings = Application.get_env(:pleroma, Pleroma.Upload)
use_s3 = Keyword.fetch!(settings, :use_s3) storage_backend = Keyword.fetch!(settings, :storage_backend)
content_type = get_content_type(file.path) content_type = get_content_type(file.path)
uuid = get_uuid(file, should_dedupe) uuid = get_uuid(file, should_dedupe)
name = get_name(file, uuid, content_type, should_dedupe) name = get_name(file, uuid, content_type, should_dedupe)
upload_folder = get_upload_path(uuid, should_dedupe)
url_path = get_url(name, uuid, should_dedupe)
strip_exif_data(content_type, file.path) strip_exif_data(content_type, file.path)
File.mkdir_p!(upload_folder) url_path = storage_backend.put_file(name, uuid, content_type)
result_file = Path.join(upload_folder, name)
if File.exists?(result_file) do
File.rm!(file.path)
else
File.cp!(file.path, result_file)
end
url_path =
if use_s3 do
put_s3_file(name, uuid, result_file, content_type)
else
url_path
end
%{ %{
"type" => "Document", "type" => "Document",
@ -115,11 +99,6 @@ def strip_exif_data(content_type, file) do
end end
end end
def upload_path do
settings = Application.get_env(:pleroma, Pleroma.Upload)
Keyword.fetch!(settings, :uploads)
end
defp create_name(uuid, ext, type) do defp create_name(uuid, ext, type) do
case type do case type do
"application/octet-stream" -> "application/octet-stream" ->
@ -163,26 +142,6 @@ defp get_name(file, uuid, type, should_dedupe) do
end end
end end
defp get_upload_path(uuid, should_dedupe) do
if should_dedupe do
upload_path()
else
Path.join(upload_path(), uuid)
end
end
defp get_url(name, uuid, should_dedupe) do
if should_dedupe do
url_for(:cow_uri.urlencode(name))
else
url_for(Path.join(uuid, :cow_uri.urlencode(name)))
end
end
defp url_for(file) do
"#{Web.base_url()}/media/#{file}"
end
def get_content_type(file) do def get_content_type(file) do
match = match =
File.open(file, [:read], fn f -> File.open(file, [:read], fn f ->
@ -224,25 +183,4 @@ def get_content_type(file) do
_e -> "application/octet-stream" _e -> "application/octet-stream"
end end
end end
defp put_s3_file(name, uuid, path, content_type) do
settings = Application.get_env(:pleroma, Pleroma.Upload)
bucket = Keyword.fetch!(settings, :bucket)
public_endpoint = Keyword.fetch!(settings, :public_endpoint)
{:ok, file_data} = File.read(path)
File.rm!(path)
s3_name = "#{uuid}/#{name}"
{:ok, result} =
ExAws.S3.put_object(bucket, s3_name, file_data, [
{:acl, :public_read},
{:content_type, content_type}
])
|> ExAws.request()
"#{public_endpoint}/#{bucket}/#{s3_name}"
end
end end

View File

@ -0,0 +1,44 @@
defmodule Pleroma.Uploaders.Local do
def put_file(name, uuid, file, content_type) do
upload_path = get_upload_path(uuid, should_dedupe)
url_path = get_url(name, uuid, should_dedupe)
File.mkdir_p!(upload_folder)
result_file = Path.join(upload_folder, name)
if File.exists?(result_file) do
File.rm!(file.path)
else
File.cp!(file.path, result_file)
end
url_path
end
def upload_path do
settings = Application.get_env(:pleroma, Pleroma.Uploaders.Local)
Keyword.fetch!(settings, :uploads)
end
defp get_upload_path(uuid, should_dedupe) do
if should_dedupe do
upload_path()
else
Path.join(upload_path(), uuid)
end
end
defp get_url(name, uuid, should_dedupe) do
if should_dedupe do
url_for(:cow_uri.urlencode(name))
else
url_for(Path.join(uuid, :cow_uri.urlencode(name)))
end
end
defp url_for(file) do
"#{Web.base_url()}/media/#{file}"
end
end

View File

@ -0,0 +1,24 @@
defmodule Pleroma.Uploaders.S3 do
def put_file(name, uuid, path, content_type) do
settings = Application.get_env(:pleroma, Pleroma.Uploaders.S3)
bucket = Keyword.fetch!(settings, :bucket)
public_endpoint = Keyword.fetch!(settings, :public_endpoint)
{:ok, file_data} = File.read(path)
File.rm!(path)
s3_name = "#{uuid}/#{name}"
{:ok, result} =
ExAws.S3.put_object(bucket, s3_name, file_data, [
{:acl, :public_read},
{:content_type, content_type}
])
|> ExAws.request()
"#{public_endpoint}/#{bucket}/#{s3_name}"
end
end

View File