Runtime configured emojis

The changes are a bit heavy since the emojis were loaded into module
attributes from filesystem.

This introduces a GenServer using an ETS table to cache in memory the
emojis, and allows a runtime-reload with `Pleroma.Emoji.reload()`.
This commit is contained in:
href 2018-11-05 13:24:00 +01:00
parent bd97b3614f
commit 763fc7b44f
No known key found for this signature in database
GPG Key ID: EE8296C1A152C325
5 changed files with 206 additions and 128 deletions

View File

@ -13,8 +13,7 @@ def start(_type, _args) do
worker(Pleroma.Config, [Application.get_all_env(:pleroma)]), worker(Pleroma.Config, [Application.get_all_env(:pleroma)]),
# Start the Ecto repository # Start the Ecto repository
supervisor(Pleroma.Repo, []), supervisor(Pleroma.Repo, []),
# Start the endpoint when the application starts worker(Pleroma.Emoji, []),
supervisor(Pleroma.Web.Endpoint, []),
# Start your own worker by calling: Pleroma.Worker.start_link(arg1, arg2, arg3) # Start your own worker by calling: Pleroma.Worker.start_link(arg1, arg2, arg3)
# worker(Pleroma.Worker, [arg1, arg2, arg3]), # worker(Pleroma.Worker, [arg1, arg2, arg3]),
worker( worker(
@ -57,8 +56,10 @@ def start(_type, _args) do
id: :cachex_idem id: :cachex_idem
), ),
worker(Pleroma.Web.Federator, []), worker(Pleroma.Web.Federator, []),
worker(Pleroma.Gopher.Server, []), worker(Pleroma.Stats, []),
worker(Pleroma.Stats, []) # Start the endpoint when the application starts
supervisor(Pleroma.Web.Endpoint, []),
worker(Pleroma.Gopher.Server, [])
] ++ ] ++
if Mix.env() == :test, if Mix.env() == :test,
do: [], do: [],

193
lib/pleroma/emoji.ex Normal file
View File

@ -0,0 +1,193 @@
defmodule Pleroma.Emoji do
@moduledoc """
The emojis are loaded from:
* the built-in Finmojis (if enabled in configuration),
* the files: `config/emoji.txt` and `config/custom_emoji.txt`
* glob paths
This GenServer stores in an ETS table the list of the loaded emojis, and also allows to reload the list at runtime.
"""
use GenServer
@ets __MODULE__.Ets
@ets_options [:set, :protected, :named_table, {:read_concurrency, true}]
@doc false
def start_link() do
GenServer.start_link(__MODULE__, [], name: __MODULE__)
end
@doc "Reloads the emojis from disk."
@spec reload() :: :ok
def reload() do
GenServer.call(__MODULE__, :reload)
end
@doc "Returns the path of the emoji `name`."
@spec get(String.t()) :: String.t() | nil
def get(name) do
case :ets.lookup(@ets, name) do
[{_, path}] -> path
_ -> nil
end
end
@doc "Returns all the emojos!!"
@spec get_all() :: [{String.t(), String.t()}, ...]
def get_all() do
:ets.tab2list(@ets)
end
@doc false
def init(_) do
@ets = :ets.new(@ets, @ets_options)
{:ok, nil, {:continue, :reload}}
end
@doc false
def handle_continue(:reload, state) do
load()
{:noreply, state}
end
@doc false
def handle_call(:reload, _from, state) do
load()
{:reply, :ok, state}
end
@doc false
def terminate(_, _) do
:ok
end
@doc false
def code_change(_old_vsn, state, _extra) do
load()
{:ok, state}
end
defp load() do
emojis =
(load_finmoji(Keyword.get(Application.get_env(:pleroma, :instance), :finmoji_enabled)) ++
load_from_file("config/emoji.txt") ++
load_from_file("config/custom_emoji.txt") ++
load_from_globs(
Keyword.get(Application.get_env(:pleroma, :emoji, []), :shortcode_globs, [])
))
|> Enum.reject(fn value -> value == nil end)
true = :ets.insert(@ets, emojis)
:ok
end
@finmoji [
"a_trusted_friend",
"alandislands",
"association",
"auroraborealis",
"baby_in_a_box",
"bear",
"black_gold",
"christmasparty",
"crosscountryskiing",
"cupofcoffee",
"education",
"fashionista_finns",
"finnishlove",
"flag",
"forest",
"four_seasons_of_bbq",
"girlpower",
"handshake",
"happiness",
"headbanger",
"icebreaker",
"iceman",
"joulutorttu",
"kaamos",
"kalsarikannit_f",
"kalsarikannit_m",
"karjalanpiirakka",
"kicksled",
"kokko",
"lavatanssit",
"losthopes_f",
"losthopes_m",
"mattinykanen",
"meanwhileinfinland",
"moominmamma",
"nordicfamily",
"out_of_office",
"peacemaker",
"perkele",
"pesapallo",
"polarbear",
"pusa_hispida_saimensis",
"reindeer",
"sami",
"sauna_f",
"sauna_m",
"sauna_whisk",
"sisu",
"stuck",
"suomimainittu",
"superfood",
"swan",
"the_cap",
"the_conductor",
"the_king",
"the_voice",
"theoriginalsanta",
"tomoffinland",
"torillatavataan",
"unbreakable",
"waiting",
"white_nights",
"woollysocks"
]
defp load_finmoji(true) do
Enum.map(@finmoji, fn finmoji ->
{finmoji, "/finmoji/128px/#{finmoji}-128.png"}
end)
end
defp load_finmoji(_), do: :ok
defp load_from_file(file) do
if File.exists?(file) do
load_from_file_stream(File.stream!(file))
else
[]
end
end
defp load_from_file_stream(stream) do
stream
|> Stream.map(&String.strip/1)
|> Stream.map(fn line ->
case String.split(line, ~r/,\s*/) do
[name, file] -> {name, file}
_ -> nil
end
end)
|> Enum.to_list()
end
defp load_from_globs(globs) do
static_path = Path.join(:code.priv_dir(:pleroma), "static")
paths =
Enum.map(globs, fn glob ->
Path.join(static_path, glob)
|> Path.wildcard()
end)
|> Enum.concat()
Enum.map(paths, fn path ->
shortcode = Path.basename(path, Path.extname(path))
external_path = Path.join("/", Path.relative_to(path, static_path))
{shortcode, external_path}
end)
end
end

View File

@ -2,6 +2,7 @@ defmodule Pleroma.Formatter do
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.MediaProxy alias Pleroma.Web.MediaProxy
alias Pleroma.HTML alias Pleroma.HTML
alias Pleroma.Emoji
@tag_regex ~r/\#\w+/u @tag_regex ~r/\#\w+/u
def parse_tags(text, data \\ %{}) do def parse_tags(text, data \\ %{}) do
@ -28,125 +29,12 @@ def parse_mentions(text) do
|> Enum.filter(fn {_match, user} -> user end) |> Enum.filter(fn {_match, user} -> user end)
end end
@finmoji [
"a_trusted_friend",
"alandislands",
"association",
"auroraborealis",
"baby_in_a_box",
"bear",
"black_gold",
"christmasparty",
"crosscountryskiing",
"cupofcoffee",
"education",
"fashionista_finns",
"finnishlove",
"flag",
"forest",
"four_seasons_of_bbq",
"girlpower",
"handshake",
"happiness",
"headbanger",
"icebreaker",
"iceman",
"joulutorttu",
"kaamos",
"kalsarikannit_f",
"kalsarikannit_m",
"karjalanpiirakka",
"kicksled",
"kokko",
"lavatanssit",
"losthopes_f",
"losthopes_m",
"mattinykanen",
"meanwhileinfinland",
"moominmamma",
"nordicfamily",
"out_of_office",
"peacemaker",
"perkele",
"pesapallo",
"polarbear",
"pusa_hispida_saimensis",
"reindeer",
"sami",
"sauna_f",
"sauna_m",
"sauna_whisk",
"sisu",
"stuck",
"suomimainittu",
"superfood",
"swan",
"the_cap",
"the_conductor",
"the_king",
"the_voice",
"theoriginalsanta",
"tomoffinland",
"torillatavataan",
"unbreakable",
"waiting",
"white_nights",
"woollysocks"
]
@instance Application.get_env(:pleroma, :instance) @instance Application.get_env(:pleroma, :instance)
@finmoji_with_filenames (if Keyword.get(@instance, :finmoji_enabled) do def emojify(text) do
Enum.map(@finmoji, fn finmoji -> emojify(text, Emoji.get_all())
{finmoji, "/finmoji/128px/#{finmoji}-128.png"} end
end)
else
[]
end)
@emoji_from_file (with {:ok, default} <- File.read("config/emoji.txt") do
custom =
with {:ok, custom} <- File.read("config/custom_emoji.txt") do
custom
else
_e -> ""
end
(default <> "\n" <> custom)
|> String.trim()
|> String.split(~r/\n+/)
|> Enum.map(fn line ->
[name, file] = String.split(line, ~r/,\s*/)
{name, file}
end)
else
_ -> []
end)
@emoji_from_globs (
static_path = Path.join(:code.priv_dir(:pleroma), "static")
globs =
Application.get_env(:pleroma, :emoji, [])
|> Keyword.get(:shortcode_globs, [])
paths =
Enum.map(globs, fn glob ->
Path.join(static_path, glob)
|> Path.wildcard()
end)
|> Enum.concat()
Enum.map(paths, fn path ->
shortcode = Path.basename(path, Path.extname(path))
external_path = Path.join("/", Path.relative_to(path, static_path))
{shortcode, external_path}
end)
)
@emoji @finmoji_with_filenames ++ @emoji_from_globs ++ @emoji_from_file
def emojify(text, emoji \\ @emoji)
def emojify(text, nil), do: text def emojify(text, nil), do: text
def emojify(text, emoji) do def emojify(text, emoji) do
@ -166,15 +54,11 @@ def emojify(text, emoji) do
end end
def get_emoji(text) when is_binary(text) do def get_emoji(text) when is_binary(text) do
Enum.filter(@emoji, fn {emoji, _} -> String.contains?(text, ":#{emoji}:") end) Enum.filter(Emoji.get_all(), fn {emoji, _} -> String.contains?(text, ":#{emoji}:") end)
end end
def get_emoji(_), do: [] def get_emoji(_), do: []
def get_custom_emoji() do
@emoji
end
@link_regex ~r/[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+/ui @link_regex ~r/[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+/ui
@uri_schemes Application.get_env(:pleroma, :uri_schemes, []) @uri_schemes Application.get_env(:pleroma, :uri_schemes, [])

View File

@ -158,7 +158,7 @@ def peers(conn, _params) do
end end
defp mastodonized_emoji do defp mastodonized_emoji do
Pleroma.Formatter.get_custom_emoji() Pleroma.Emoji.get_all()
|> Enum.map(fn {shortcode, relative_url} -> |> Enum.map(fn {shortcode, relative_url} ->
url = to_string(URI.merge(Web.base_url(), relative_url)) url = to_string(URI.merge(Web.base_url(), relative_url))

View File

@ -6,7 +6,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
alias Pleroma.Web.WebFinger alias Pleroma.Web.WebFinger
alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI
alias Comeonin.Pbkdf2 alias Comeonin.Pbkdf2
alias Pleroma.Formatter alias Pleroma.{Formatter, Emoji}
alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.{Repo, PasswordResetToken, User} alias Pleroma.{Repo, PasswordResetToken, User}
@ -212,7 +212,7 @@ def version(conn, _params) do
end end
def emoji(conn, _params) do def emoji(conn, _params) do
json(conn, Enum.into(Formatter.get_custom_emoji(), %{})) json(conn, Enum.into(Emoji.get_all(), %{}))
end end
def follow_import(conn, %{"list" => %Plug.Upload{} = listfile}) do def follow_import(conn, %{"list" => %Plug.Upload{} = listfile}) do