Pleroma.MIME: use gen_magic

This commit is contained in:
href 2020-05-14 21:36:31 +02:00
parent 1e49bfa9ac
commit ec1452fd1c
13 changed files with 52 additions and 64 deletions

View File

@ -24,6 +24,8 @@ stages:
before_script: before_script:
- mix local.hex --force - mix local.hex --force
- mix local.rebar --force - mix local.rebar --force
- apt-get -qq update
- apt-get install -y libmagic-dev
build: build:
stage: build stage: build

View File

@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Changed ### Changed
- MFR policy to set global expiration for all local Create activities - MFR policy to set global expiration for all local Create activities
- OGP rich media parser merged with TwitterCard - OGP rich media parser merged with TwitterCard
- **Breaking** Requires `libmagic` (or `file`) to guess file types.
<details> <details>
<summary>API Changes</summary> <summary>API Changes</summary>
- **Breaking:** Emoji API: changed methods and renamed routes. - **Breaking:** Emoji API: changed methods and renamed routes.

View File

@ -678,6 +678,8 @@
timeout: 300_000 timeout: 300_000
] ]
config :pleroma, :gen_magic_pool, size: 2
config :pleroma, :restrict_unauthenticated, config :pleroma, :restrict_unauthenticated,
timelines: %{local: false, federated: false}, timelines: %{local: false, federated: false},
profiles: %{local: false, remote: false}, profiles: %{local: false, remote: false},

View File

@ -3325,5 +3325,19 @@
suggestions: [false] suggestions: [false]
} }
] ]
},
%{
group: :pleroma,
key: :gen_magic_pool,
type: :group,
description: "GenMagic/libmagic configuration",
children: [
%{
key: :size,
type: :integer,
description: "Number of gen_magic workers to start.",
suggestions: [2]
}
]
} }
] ]

View File

@ -13,6 +13,7 @@ It assumes that you have administrative rights, either as root or a user with [s
* `erlang-parsetools` * `erlang-parsetools`
* `erlang-xmerl` * `erlang-xmerl`
* `git` * `git`
* `file-dev`
* Development Tools * Development Tools
#### Optional packages used in this guide #### Optional packages used in this guide

View File

@ -9,6 +9,7 @@ This guide will assume that you have administrative rights, either as root or a
* `elixir` * `elixir`
* `git` * `git`
* `base-devel` * `base-devel`
* `file`
#### Optional packages used in this guide #### Optional packages used in this guide

View File

@ -10,6 +10,7 @@ This guide will assume you are on Debian Stretch. This guide should also work wi
* `elixir` (1.8+, Follow the guide to install from the Erlang Solutions repo or use [asdf](https://github.com/asdf-vm/asdf) as the pleroma user) * `elixir` (1.8+, Follow the guide to install from the Erlang Solutions repo or use [asdf](https://github.com/asdf-vm/asdf) as the pleroma user)
* `erlang-dev` * `erlang-dev`
* `erlang-nox` * `erlang-nox`
* `libmagic-dev`
* `git` * `git`
* `build-essential` * `build-essential`

View File

@ -28,6 +28,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
* `dev-db/postgresql` * `dev-db/postgresql`
* `dev-lang/elixir` * `dev-lang/elixir`
* `dev-vcs/git` * `dev-vcs/git`
* `sys-apps/file`
#### Optional ebuilds used in this guide #### Optional ebuilds used in this guide

View File

@ -27,15 +27,16 @@ Other than things bundled in the OTP release Pleroma depends on:
* PostgreSQL (also utilizes extensions in postgresql-contrib) * PostgreSQL (also utilizes extensions in postgresql-contrib)
* nginx (could be swapped with another reverse proxy but this guide covers only it) * nginx (could be swapped with another reverse proxy but this guide covers only it)
* certbot (for Let's Encrypt certificates, could be swapped with another ACME client, but this guide covers only it) * certbot (for Let's Encrypt certificates, could be swapped with another ACME client, but this guide covers only it)
* libmagic/file
```sh tab="Alpine" ```sh tab="Alpine"
echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories
apk update apk update
apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot libmagic
``` ```
```sh tab="Debian/Ubuntu" ```sh tab="Debian/Ubuntu"
apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot libmagic
``` ```
## Setup ## Setup

View File

@ -80,6 +80,7 @@ def start(_type, _args) do
[ [
Pleroma.Stats, Pleroma.Stats,
Pleroma.JobQueueMonitor, Pleroma.JobQueueMonitor,
Pleroma.MIME,
{Oban, Config.get(Oban)} {Oban, Config.get(Oban)}
] ++ ] ++
task_children(@env) ++ task_children(@env) ++

View File

@ -6,8 +6,21 @@ defmodule Pleroma.MIME do
@moduledoc """ @moduledoc """
Returns the mime-type of a binary and optionally a normalized file-name. Returns the mime-type of a binary and optionally a normalized file-name.
""" """
@default "application/octet-stream"
@read_bytes 35 @read_bytes 35
@pool __MODULE__.GenMagicPool
def child_spec(_) do
pool_size = Pleroma.Config.get!([:gen_magic_pool, :size])
name = @pool
%{
id: __MODULE__,
start: {GenMagic.Pool, :start_link, [[name: name, pool_size: pool_size]]},
type: :worker,
restart: :permanent,
shutdown: 500
}
end
@spec file_mime_type(String.t(), String.t()) :: @spec file_mime_type(String.t(), String.t()) ::
{:ok, content_type :: String.t(), filename :: String.t()} | {:error, any()} | :error {:ok, content_type :: String.t(), filename :: String.t()} | {:error, any()} | :error
@ -20,9 +33,10 @@ def file_mime_type(path, filename) do
@spec file_mime_type(String.t()) :: {:ok, String.t()} | {:error, any()} | :error @spec file_mime_type(String.t()) :: {:ok, String.t()} | {:error, any()} | :error
def file_mime_type(filename) do def file_mime_type(filename) do
File.open(filename, [:read], fn f -> case GenMagic.Pool.perform(@pool, filename) do
check_mime_type(IO.binread(f, @read_bytes)) {:ok, %GenMagic.Result{mime_type: content_type}} -> {:ok, content_type}
end) error -> error
end
end end
def bin_mime_type(binary, filename) do def bin_mime_type(binary, filename) do
@ -34,13 +48,14 @@ def bin_mime_type(binary, filename) do
@spec bin_mime_type(binary()) :: {:ok, String.t()} | :error @spec bin_mime_type(binary()) :: {:ok, String.t()} | :error
def bin_mime_type(<<head::binary-size(@read_bytes), _::binary>>) do def bin_mime_type(<<head::binary-size(@read_bytes), _::binary>>) do
{:ok, check_mime_type(head)} case GenMagic.Pool.perform(@pool, {:bytes, head}) do
{:ok, %GenMagic.Result{mime_type: content_type}} -> {:ok, content_type}
error -> error
end
end end
def bin_mime_type(_), do: :error def bin_mime_type(_), do: :error
def mime_type(<<_::binary>>), do: {:ok, @default}
defp fix_extension(filename, content_type) do defp fix_extension(filename, content_type) do
parts = String.split(filename, ".") parts = String.split(filename, ".")
@ -62,59 +77,4 @@ defp fix_extension(filename, content_type) do
Enum.join([new_filename, String.split(content_type, "/") |> List.last()], ".") Enum.join([new_filename, String.split(content_type, "/") |> List.last()], ".")
end end
end end
defp check_mime_type(<<0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, _::binary>>) do
"image/png"
end
defp check_mime_type(<<0x47, 0x49, 0x46, 0x38, _, 0x61, _::binary>>) do
"image/gif"
end
defp check_mime_type(<<0xFF, 0xD8, 0xFF, _::binary>>) do
"image/jpeg"
end
defp check_mime_type(<<0x1A, 0x45, 0xDF, 0xA3, _::binary>>) do
"video/webm"
end
defp check_mime_type(<<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70, _::binary>>) do
"video/mp4"
end
defp check_mime_type(<<0x49, 0x44, 0x33, _::binary>>) do
"audio/mpeg"
end
defp check_mime_type(<<255, 251, _, 68, 0, 0, 0, 0, _::binary>>) do
"audio/mpeg"
end
defp check_mime_type(
<<0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00, _::size(160), 0x80, 0x74, 0x68, 0x65,
0x6F, 0x72, 0x61, _::binary>>
) do
"video/ogg"
end
defp check_mime_type(<<0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00, _::binary>>) do
"audio/ogg"
end
defp check_mime_type(<<"RIFF", _::binary-size(4), "WAVE", _::binary>>) do
"audio/wav"
end
defp check_mime_type(<<"RIFF", _::binary-size(4), "WEBP", _::binary>>) do
"image/webp"
end
defp check_mime_type(<<"RIFF", _::binary-size(4), "AVI.", _::binary>>) do
"video/avi"
end
defp check_mime_type(_) do
@default
end
end end

View File

@ -197,6 +197,7 @@ defp deps do
ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"}, ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"},
{:mox, "~> 0.5", only: :test}, {:mox, "~> 0.5", only: :test},
{:restarter, path: "./restarter"}, {:restarter, path: "./restarter"},
{:gen_magic, git: "https://github.com/hrefhref/gen_magic", branch: "develop"},
{:open_api_spex, {:open_api_spex,
git: "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git", git: "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git",
ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"} ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"}

View File

@ -45,6 +45,7 @@
"fast_sanitize": {:hex, :fast_sanitize, "0.1.7", "2a7cd8734c88a2de6de55022104f8a3b87f1fdbe8bbf131d9049764b53d50d0d", [:mix], [{:fast_html, "~> 1.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f39fe8ea08fbac17487c30bf09b7d9f3e12472e51fb07a88ffeb8fd17da8ab67"}, "fast_sanitize": {:hex, :fast_sanitize, "0.1.7", "2a7cd8734c88a2de6de55022104f8a3b87f1fdbe8bbf131d9049764b53d50d0d", [:mix], [{:fast_html, "~> 1.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f39fe8ea08fbac17487c30bf09b7d9f3e12472e51fb07a88ffeb8fd17da8ab67"},
"flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "31fc8090fde1acd267c07c36ea7365b8604055f897d3a53dd967658c691bd827"}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "31fc8090fde1acd267c07c36ea7365b8604055f897d3a53dd967658c691bd827"},
"floki": {:hex, :floki, "0.25.0", "b1c9ddf5f32a3a90b43b76f3386ca054325dc2478af020e87b5111c19f2284ac", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "631f4e627c46d5ecd347df5a2accdaf0621c77c3693c5b75a8ad58e84c61f242"}, "floki": {:hex, :floki, "0.25.0", "b1c9ddf5f32a3a90b43b76f3386ca054325dc2478af020e87b5111c19f2284ac", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "631f4e627c46d5ecd347df5a2accdaf0621c77c3693c5b75a8ad58e84c61f242"},
"gen_magic": {:git, "https://github.com/hrefhref/gen_magic", "eafdc2ea156433ccd87d1d99eaf1be758064a1db", [branch: "develop"]},
"gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm", "29bd14a88030980849c7ed2447b8db6d6c9278a28b11a44cafe41b791205440f"}, "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm", "29bd14a88030980849c7ed2447b8db6d6c9278a28b11a44cafe41b791205440f"},
"gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm"}, "gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm"},
"gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"}, "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"},
@ -74,6 +75,7 @@
"mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm", "052346cf322311c49a0f22789f3698eea030eec09b8c47367f0686ef2634ae14"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm", "052346cf322311c49a0f22789f3698eea030eec09b8c47367f0686ef2634ae14"},
"myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]}, "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]},
"nimble_parsec": {:hex, :nimble_parsec, "0.5.3", "def21c10a9ed70ce22754fdeea0810dafd53c2db3219a0cd54cf5526377af1c6", [:mix], [], "hexpm", "589b5af56f4afca65217a1f3eb3fee7e79b09c40c742fddc1c312b3ac0b3399f"}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.3", "def21c10a9ed70ce22754fdeea0810dafd53c2db3219a0cd54cf5526377af1c6", [:mix], [], "hexpm", "589b5af56f4afca65217a1f3eb3fee7e79b09c40c742fddc1c312b3ac0b3399f"},
"nimble_pool": {:hex, :nimble_pool, "0.1.0", "ffa9d5be27eee2b00b0c634eb649aa27f97b39186fec3c493716c2a33e784ec6", [:mix], [], "hexpm", "343a1eaa620ddcf3430a83f39f2af499fe2370390d4f785cd475b4df5acaf3f9"},
"nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]},
"oban": {:hex, :oban, "1.2.0", "7cca94d341be43d220571e28f69131c4afc21095b25257397f50973d3fc59b07", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ba5f8b3f7d76967b3e23cf8014f6a13e4ccb33431e4808f036709a7f822362ee"}, "oban": {:hex, :oban, "1.2.0", "7cca94d341be43d220571e28f69131c4afc21095b25257397f50973d3fc59b07", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ba5f8b3f7d76967b3e23cf8014f6a13e4ccb33431e4808f036709a7f822362ee"},
"open_api_spex": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git", "f296ac0924ba3cf79c7a588c4c252889df4c2edd", [ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"]}, "open_api_spex": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git", "f296ac0924ba3cf79c7a588c4c252889df4c2edd", [ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"]},