Merge branch 'feature/uploader_s3' into 'develop'

Uploader.S3 added support stream uploads

See merge request pleroma/pleroma!1545
This commit is contained in:
rinpatch 2019-08-10 11:28:00 +00:00
commit 337edb3e50
4 changed files with 100 additions and 6 deletions

View File

@ -6,10 +6,12 @@ defmodule Pleroma.Uploaders.S3 do
@behaviour Pleroma.Uploaders.Uploader @behaviour Pleroma.Uploaders.Uploader
require Logger require Logger
alias Pleroma.Config
# The file name is re-encoded with S3's constraints here to comply with previous # The file name is re-encoded with S3's constraints here to comply with previous
# links with less strict filenames # links with less strict filenames
def get_file(file) do def get_file(file) do
config = Pleroma.Config.get([__MODULE__]) config = Config.get([__MODULE__])
bucket = Keyword.fetch!(config, :bucket) bucket = Keyword.fetch!(config, :bucket)
bucket_with_namespace = bucket_with_namespace =
@ -34,15 +36,15 @@ def get_file(file) do
end end
def put_file(%Pleroma.Upload{} = upload) do def put_file(%Pleroma.Upload{} = upload) do
config = Pleroma.Config.get([__MODULE__]) config = Config.get([__MODULE__])
bucket = Keyword.get(config, :bucket) bucket = Keyword.get(config, :bucket)
{:ok, file_data} = File.read(upload.tempfile)
s3_name = strict_encode(upload.path) s3_name = strict_encode(upload.path)
op = op =
ExAws.S3.put_object(bucket, s3_name, file_data, [ upload.tempfile
|> ExAws.S3.Upload.stream_file()
|> ExAws.S3.upload(bucket, s3_name, [
{:acl, :public_read}, {:acl, :public_read},
{:content_type, upload.content_type} {:content_type, upload.content_type}
]) ])

View File

@ -114,8 +114,9 @@ defp deps do
{:tesla, "~> 1.2"}, {:tesla, "~> 1.2"},
{:jason, "~> 1.0"}, {:jason, "~> 1.0"},
{:mogrify, "~> 0.6.1"}, {:mogrify, "~> 0.6.1"},
{:ex_aws, "~> 2.0"}, {:ex_aws, "~> 2.1"},
{:ex_aws_s3, "~> 2.0"}, {:ex_aws_s3, "~> 2.0"},
{:sweet_xml, "~> 0.6.6"},
{:earmark, "~> 1.3"}, {:earmark, "~> 1.3"},
{:bbcode, "~> 0.1.1"}, {:bbcode, "~> 0.1.1"},
{:ex_machina, "~> 2.3", only: :test}, {:ex_machina, "~> 2.3", only: :test},

View File

@ -80,6 +80,7 @@
"ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"},
"recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]},
"ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
"sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"},
"swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"},
"syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]},
"telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"},

View File

@ -0,0 +1,90 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Uploaders.S3Test do
use Pleroma.DataCase
alias Pleroma.Config
alias Pleroma.Uploaders.S3
import Mock
import ExUnit.CaptureLog
setup do
config = Config.get([Pleroma.Uploaders.S3])
Config.put([Pleroma.Uploaders.S3],
bucket: "test_bucket",
public_endpoint: "https://s3.amazonaws.com"
)
on_exit(fn ->
Config.put([Pleroma.Uploaders.S3], config)
end)
:ok
end
describe "get_file/1" do
test "it returns path to local folder for files" do
assert S3.get_file("test_image.jpg") == {
:ok,
{:url, "https://s3.amazonaws.com/test_bucket/test_image.jpg"}
}
end
test "it returns path without bucket when truncated_namespace set to ''" do
Config.put([Pleroma.Uploaders.S3],
bucket: "test_bucket",
public_endpoint: "https://s3.amazonaws.com",
truncated_namespace: ""
)
assert S3.get_file("test_image.jpg") == {
:ok,
{:url, "https://s3.amazonaws.com/test_image.jpg"}
}
end
test "it returns path with bucket namespace when namespace is set" do
Config.put([Pleroma.Uploaders.S3],
bucket: "test_bucket",
public_endpoint: "https://s3.amazonaws.com",
bucket_namespace: "family"
)
assert S3.get_file("test_image.jpg") == {
:ok,
{:url, "https://s3.amazonaws.com/family:test_bucket/test_image.jpg"}
}
end
end
describe "put_file/1" do
setup do
file_upload = %Pleroma.Upload{
name: "image-tet.jpg",
content_type: "image/jpg",
path: "test_folder/image-tet.jpg",
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
[file_upload: file_upload]
end
test "save file", %{file_upload: file_upload} do
with_mock ExAws, request: fn _ -> {:ok, :ok} end do
assert S3.put_file(file_upload) == {:ok, {:file, "test_folder/image-tet.jpg"}}
end
end
test "returns error", %{file_upload: file_upload} do
with_mock ExAws, request: fn _ -> {:error, "S3 Upload failed"} end do
assert capture_log(fn ->
assert S3.put_file(file_upload) == {:error, "S3 Upload failed"}
end) =~ "Elixir.Pleroma.Uploaders.S3: {:error, \"S3 Upload failed\"}"
end
end
end
end