Runtime configured emojis

The changes are a bit heavy since the emojis were loaded into module
attributes from filesystem.

This introduces a GenServer using an ETS table to cache in memory the
emojis, and allows a runtime-reload with `Pleroma.Emoji.reload()`.

lib/pleroma/application.ex

@@ -13,8 +13,7 @@ def start(_type, _args) do
         worker(Pleroma.Config, [Application.get_all_env(:pleroma)]),
         # Start the Ecto repository
         supervisor(Pleroma.Repo, []),
-        # Start the endpoint when the application starts
+        worker(Pleroma.Emoji, []),
-        supervisor(Pleroma.Web.Endpoint, []),
         # Start your own worker by calling: Pleroma.Worker.start_link(arg1, arg2, arg3)
         # worker(Pleroma.Worker, [arg1, arg2, arg3]),
         worker(
@@ -57,8 +56,10 @@ def start(_type, _args) do
           id: :cachex_idem
         ),
         worker(Pleroma.Web.Federator, []),
-        worker(Pleroma.Gopher.Server, []),
+        worker(Pleroma.Stats, []),
-        worker(Pleroma.Stats, [])
+        # Start the endpoint when the application starts
+        supervisor(Pleroma.Web.Endpoint, []),
+        worker(Pleroma.Gopher.Server, [])
       ] ++
         if Mix.env() == :test,
           do: [],

lib/pleroma/emoji.ex

@@ -0,0 +1,193 @@
+defmodule Pleroma.Emoji do
+  @moduledoc """
+  The emojis are loaded from:
+
+    * the built-in Finmojis (if enabled in configuration),
+    * the files: `config/emoji.txt` and `config/custom_emoji.txt`
+    * glob paths
+
+  This GenServer stores in an ETS table the list of the loaded emojis, and also allows to reload the list at runtime.
+  """
+  use GenServer
+  @ets __MODULE__.Ets
+  @ets_options [:set, :protected, :named_table, {:read_concurrency, true}]
+
+  @doc false
+  def start_link() do
+    GenServer.start_link(__MODULE__, [], name: __MODULE__)
+  end
+
+  @doc "Reloads the emojis from disk."
+  @spec reload() :: :ok
+  def reload() do
+    GenServer.call(__MODULE__, :reload)
+  end
+
+  @doc "Returns the path of the emoji `name`."
+  @spec get(String.t()) :: String.t() | nil
+  def get(name) do
+    case :ets.lookup(@ets, name) do
+      [{_, path}] -> path
+      _ -> nil
+    end
+  end
+
+  @doc "Returns all the emojos!!"
+  @spec get_all() :: [{String.t(), String.t()}, ...]
+  def get_all() do
+    :ets.tab2list(@ets)
+  end
+
+  @doc false
+  def init(_) do
+    @ets = :ets.new(@ets, @ets_options)
+    {:ok, nil, {:continue, :reload}}
+  end
+
+  @doc false
+  def handle_continue(:reload, state) do
+    load()
+    {:noreply, state}
+  end
+
+  @doc false
+  def handle_call(:reload, _from, state) do
+    load()
+    {:reply, :ok, state}
+  end
+
+  @doc false
+  def terminate(_, _) do
+    :ok
+  end
+
+  @doc false
+  def code_change(_old_vsn, state, _extra) do
+    load()
+    {:ok, state}
+  end
+
+  defp load() do
+    emojis =
+      (load_finmoji(Keyword.get(Application.get_env(:pleroma, :instance), :finmoji_enabled)) ++
+         load_from_file("config/emoji.txt") ++
+         load_from_file("config/custom_emoji.txt") ++
+         load_from_globs(
+           Keyword.get(Application.get_env(:pleroma, :emoji, []), :shortcode_globs, [])
+         ))
+      |> Enum.reject(fn value -> value == nil end)
+
+    true = :ets.insert(@ets, emojis)
+    :ok
+  end
+
+  @finmoji [
+    "a_trusted_friend",
+    "alandislands",
+    "association",
+    "auroraborealis",
+    "baby_in_a_box",
+    "bear",
+    "black_gold",
+    "christmasparty",
+    "crosscountryskiing",
+    "cupofcoffee",
+    "education",
+    "fashionista_finns",
+    "finnishlove",
+    "flag",
+    "forest",
+    "four_seasons_of_bbq",
+    "girlpower",
+    "handshake",
+    "happiness",
+    "headbanger",
+    "icebreaker",
+    "iceman",
+    "joulutorttu",
+    "kaamos",
+    "kalsarikannit_f",
+    "kalsarikannit_m",
+    "karjalanpiirakka",
+    "kicksled",
+    "kokko",
+    "lavatanssit",
+    "losthopes_f",
+    "losthopes_m",
+    "mattinykanen",
+    "meanwhileinfinland",
+    "moominmamma",
+    "nordicfamily",
+    "out_of_office",
+    "peacemaker",
+    "perkele",
+    "pesapallo",
+    "polarbear",
+    "pusa_hispida_saimensis",
+    "reindeer",
+    "sami",
+    "sauna_f",
+    "sauna_m",
+    "sauna_whisk",
+    "sisu",
+    "stuck",
+    "suomimainittu",
+    "superfood",
+    "swan",
+    "the_cap",
+    "the_conductor",
+    "the_king",
+    "the_voice",
+    "theoriginalsanta",
+    "tomoffinland",
+    "torillatavataan",
+    "unbreakable",
+    "waiting",
+    "white_nights",
+    "woollysocks"
+  ]
+  defp load_finmoji(true) do
+    Enum.map(@finmoji, fn finmoji ->
+      {finmoji, "/finmoji/128px/#{finmoji}-128.png"}
+    end)
+  end
+
+  defp load_finmoji(_), do: :ok
+
+  defp load_from_file(file) do
+    if File.exists?(file) do
+      load_from_file_stream(File.stream!(file))
+    else
+      []
+    end
+  end
+
+  defp load_from_file_stream(stream) do
+    stream
+    |> Stream.map(&String.strip/1)
+    |> Stream.map(fn line ->
+      case String.split(line, ~r/,\s*/) do
+        [name, file] -> {name, file}
+        _ -> nil
+      end
+    end)
+    |> Enum.to_list()
+  end
+
+  defp load_from_globs(globs) do
+    static_path = Path.join(:code.priv_dir(:pleroma), "static")
+
+    paths =
+      Enum.map(globs, fn glob ->
+        Path.join(static_path, glob)
+        |> Path.wildcard()
+      end)
+      |> Enum.concat()
+
+    Enum.map(paths, fn path ->
+      shortcode = Path.basename(path, Path.extname(path))
+      external_path = Path.join("/", Path.relative_to(path, static_path))
+      {shortcode, external_path}
+    end)
+  end
+end

lib/pleroma/formatter.ex

@@ -2,6 +2,7 @@ defmodule Pleroma.Formatter do
   alias Pleroma.User
   alias Pleroma.Web.MediaProxy
   alias Pleroma.HTML
+  alias Pleroma.Emoji
 
   @tag_regex ~r/\#\w+/u
   def parse_tags(text, data \\ %{}) do
@@ -28,125 +29,12 @@ def parse_mentions(text) do
     |> Enum.filter(fn {_match, user} -> user end)
   end
 
-  @finmoji [
-    "a_trusted_friend",
-    "alandislands",
-    "association",
-    "auroraborealis",
-    "baby_in_a_box",
-    "bear",
-    "black_gold",
-    "christmasparty",
-    "crosscountryskiing",
-    "cupofcoffee",
-    "education",
-    "fashionista_finns",
-    "finnishlove",
-    "flag",
-    "forest",
-    "four_seasons_of_bbq",
-    "girlpower",
-    "handshake",
-    "happiness",
-    "headbanger",
-    "icebreaker",
-    "iceman",
-    "joulutorttu",
-    "kaamos",
-    "kalsarikannit_f",
-    "kalsarikannit_m",
-    "karjalanpiirakka",
-    "kicksled",
-    "kokko",
-    "lavatanssit",
-    "losthopes_f",
-    "losthopes_m",
-    "mattinykanen",
-    "meanwhileinfinland",
-    "moominmamma",
-    "nordicfamily",
-    "out_of_office",
-    "peacemaker",
-    "perkele",
-    "pesapallo",
-    "polarbear",
-    "pusa_hispida_saimensis",
-    "reindeer",
-    "sami",
-    "sauna_f",
-    "sauna_m",
-    "sauna_whisk",
-    "sisu",
-    "stuck",
-    "suomimainittu",
-    "superfood",
-    "swan",
-    "the_cap",
-    "the_conductor",
-    "the_king",
-    "the_voice",
-    "theoriginalsanta",
-    "tomoffinland",
-    "torillatavataan",
-    "unbreakable",
-    "waiting",
-    "white_nights",
-    "woollysocks"
-  ]
-
   @instance Application.get_env(:pleroma, :instance)
 
-  @finmoji_with_filenames (if Keyword.get(@instance, :finmoji_enabled) do
+  def emojify(text) do
-                             Enum.map(@finmoji, fn finmoji ->
+    emojify(text, Emoji.get_all())
-                               {finmoji, "/finmoji/128px/#{finmoji}-128.png"}
-                             end)
-                           else
-                             []
-                           end)
-
-  @emoji_from_file (with {:ok, default} <- File.read("config/emoji.txt") do
-                      custom =
-                        with {:ok, custom} <- File.read("config/custom_emoji.txt") do
-                          custom
-                        else
-                          _e -> ""
   end
 
-                      (default <> "\n" <> custom)
-                      |> String.trim()
-                      |> String.split(~r/\n+/)
-                      |> Enum.map(fn line ->
-                        [name, file] = String.split(line, ~r/,\s*/)
-                        {name, file}
-                      end)
-                    else
-                      _ -> []
-                    end)
-
-  @emoji_from_globs (
-                      static_path = Path.join(:code.priv_dir(:pleroma), "static")
-
-                      globs =
-                        Application.get_env(:pleroma, :emoji, [])
-                        |> Keyword.get(:shortcode_globs, [])
-
-                      paths =
-                        Enum.map(globs, fn glob ->
-                          Path.join(static_path, glob)
-                          |> Path.wildcard()
-                        end)
-                        |> Enum.concat()
-
-                      Enum.map(paths, fn path ->
-                        shortcode = Path.basename(path, Path.extname(path))
-                        external_path = Path.join("/", Path.relative_to(path, static_path))
-                        {shortcode, external_path}
-                      end)
-                    )
-
-  @emoji @finmoji_with_filenames ++ @emoji_from_globs ++ @emoji_from_file
-
-  def emojify(text, emoji \\ @emoji)
   def emojify(text, nil), do: text
 
   def emojify(text, emoji) do
@@ -166,15 +54,11 @@ def emojify(text, emoji) do
   end
 
   def get_emoji(text) when is_binary(text) do
-    Enum.filter(@emoji, fn {emoji, _} -> String.contains?(text, ":#{emoji}:") end)
+    Enum.filter(Emoji.get_all(), fn {emoji, _} -> String.contains?(text, ":#{emoji}:") end)
   end
 
   def get_emoji(_), do: []
 
-  def get_custom_emoji() do
-    @emoji
-  end
-
   @link_regex ~r/[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+/ui
 
   @uri_schemes Application.get_env(:pleroma, :uri_schemes, [])

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -158,7 +158,7 @@ def peers(conn, _params) do
   end
 
   defp mastodonized_emoji do
-    Pleroma.Formatter.get_custom_emoji()
+    Pleroma.Emoji.get_all()
     |> Enum.map(fn {shortcode, relative_url} ->
       url = to_string(URI.merge(Web.base_url(), relative_url))
 

lib/pleroma/web/twitter_api/controllers/util_controller.ex

@@ -6,7 +6,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
   alias Pleroma.Web.WebFinger
   alias Pleroma.Web.CommonAPI
   alias Comeonin.Pbkdf2
-  alias Pleroma.Formatter
+  alias Pleroma.{Formatter, Emoji}
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.{Repo, PasswordResetToken, User}
 
@@ -212,7 +212,7 @@ def version(conn, _params) do
   end
 
   def emoji(conn, _params) do
-    json(conn, Enum.into(Formatter.get_custom_emoji(), %{}))
+    json(conn, Enum.into(Emoji.get_all(), %{}))
   end
 
   def follow_import(conn, %{"list" => %Plug.Upload{} = listfile}) do