Merge branch 'feature/1854-emoji-pagination' into 'develop'

Emoji and packs pagination

Closes #1854

See merge request pleroma/pleroma!2658

CHANGELOG.md

@@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - **Breaking:** removed `with_move` parameter from notifications timeline.
 
 ### Added
+
 - Chats: Added support for federated chats. For details, see the docs.
 - ActivityPub: Added support for existing AP ids for instances migrated from Mastodon.
 - Instance: Add `background_image` to configuration and `/api/v1/instance`
@@ -34,6 +35,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Notifications: Added `follow_request` notification type.
 - Added `:reject_deletes` group to SimplePolicy
 - MRF (`EmojiStealPolicy`): New MRF Policy which allows to automatically download emojis from remote instances
+- Support pagination in emoji packs API (for packs and for files in pack)
+
 <details>
   <summary>API Changes</summary>
 - Mastodon API: Extended `/api/v1/instance`.

docs/API/pleroma_api.md

@@ -450,18 +450,44 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Response: JSON, list with updated files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
 
 ## `GET /api/pleroma/emoji/packs`
+
 ### Lists local custom emoji packs
+
 * Method `GET`
 * Authentication: not required
-* Params: None
+* Params:
-* Response: JSON, "ok" and 200 status and the JSON hashmap of pack name to pack contents
+  * `page`: page number for packs (default 1)
+  * `page_size`: page size for packs (default 50)
+* Response: `packs` key with JSON hashmap of pack name to pack contents and `count` key for count of packs.
+
+```json
+{
+  "packs": {
+    "pack_name": {...}, // pack contents
+    ...
+  },
+  "count": 0 // packs count
+}
+```
 
 ## `GET /api/pleroma/emoji/packs/:name`
+
 ### Get pack.json for the pack
+
 * Method `GET`
 * Authentication: not required
-* Params: None
+* Params:
-* Response: JSON, pack json with `files` and `pack` keys with 200 status or 404 if the pack does not exist
+  * `page`: page number for files (default 1)
+  * `page_size`: page size for files (default 30)
+* Response: JSON, pack json with `files`, `files_count` and `pack` keys with 200 status or 404 if the pack does not exist.
+
+```json
+{
+  "files": {...},
+  "files_count": 0, // emoji count in pack
+  "pack": {...}
+}
+```
 
 ## `GET /api/pleroma/emoji/packs/:name/archive`
 ### Requests a local pack archive from the instance

lib/pleroma/emoji/pack.ex

@@ -1,6 +1,7 @@
 defmodule Pleroma.Emoji.Pack do
-  @derive {Jason.Encoder, only: [:files, :pack]}
+  @derive {Jason.Encoder, only: [:files, :pack, :files_count]}
   defstruct files: %{},
+            files_count: 0,
             pack_file: nil,
             path: nil,
             pack: %{},
@@ -8,6 +9,7 @@ defmodule Pleroma.Emoji.Pack do
 
   @type t() :: %__MODULE__{
           files: %{String.t() => Path.t()},
+          files_count: non_neg_integer(),
           pack_file: Path.t(),
           path: Path.t(),
           pack: map(),
@@ -16,7 +18,7 @@ defmodule Pleroma.Emoji.Pack do
 
   alias Pleroma.Emoji
 
-  @spec create(String.t()) :: :ok | {:error, File.posix()} | {:error, :empty_values}
+  @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
   def create(name) do
     with :ok <- validate_not_empty([name]),
          dir <- Path.join(emoji_path(), name),
@@ -26,10 +28,27 @@ def create(name) do
     end
   end
 
-  @spec show(String.t()) :: {:ok, t()} | {:error, atom()}
+  defp paginate(entities, 1, page_size), do: Enum.take(entities, page_size)
-  def show(name) do
+
+  defp paginate(entities, page, page_size) do
+    entities
+    |> Enum.chunk_every(page_size)
+    |> Enum.at(page - 1)
+  end
+
+  @spec show(keyword()) :: {:ok, t()} | {:error, atom()}
+  def show(opts) do
+    name = opts[:name]
+
     with :ok <- validate_not_empty([name]),
          {:ok, pack} <- load_pack(name) do
+      shortcodes =
+        pack.files
+        |> Map.keys()
+        |> paginate(opts[:page], opts[:page_size])
+
+      pack = Map.put(pack, :files, Map.take(pack.files, shortcodes))
+
       {:ok, validate_pack(pack)}
     end
   end
@@ -120,10 +139,10 @@ def list_remote(url) do
     end
   end
 
-  @spec list_local() :: {:ok, map()}
+  @spec list_local(keyword()) :: {:ok, map(), non_neg_integer()}
-  def list_local do
+  def list_local(opts) do
     with {:ok, results} <- list_packs_dir() do
-      packs =
+      all_packs =
         results
         |> Enum.map(fn name ->
           case load_pack(name) do
@@ -132,9 +151,13 @@ def list_local do
           end
         end)
         |> Enum.reject(&is_nil/1)
+
+      packs =
+        all_packs
+        |> paginate(opts[:page], opts[:page_size])
         |> Map.new(fn pack -> {pack.name, validate_pack(pack)} end)
 
-      {:ok, packs}
+      {:ok, packs, length(all_packs)}
     end
   end
 
@@ -146,7 +169,7 @@ def get_archive(name) do
     end
   end
 
-  @spec download(String.t(), String.t(), String.t()) :: :ok | {:error, atom()}
+  @spec download(String.t(), String.t(), String.t()) :: {:ok, t()} | {:error, atom()}
   def download(name, url, as) do
     uri = url |> String.trim() |> URI.parse()
 
@@ -197,7 +220,12 @@ def load_pack(name) do
         |> Map.put(:path, Path.dirname(pack_file))
         |> Map.put(:name, name)
 
-      {:ok, pack}
+      files_count =
+        pack.files
+        |> Map.keys()
+        |> length()
+
+      {:ok, Map.put(pack, :files_count, files_count)}
     else
       {:error, :not_found}
     end
@@ -296,7 +324,9 @@ defp downloadable?(pack) do
     # Otherwise, they'd have to download it from external-src
     pack.pack["share-files"] &&
       Enum.all?(pack.files, fn {_, file} ->
-        File.exists?(Path.join(pack.path, file))
+        pack.path
+        |> Path.join(file)
+        |> File.exists?()
       end)
   end
 
@@ -440,7 +470,7 @@ defp list_packs_dir do
     # with the API so it should be sufficient
     with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_path)},
          {:ls, {:ok, results}} <- {:ls, File.ls(emoji_path)} do
-      {:ok, results}
+      {:ok, Enum.sort(results)}
     else
       {:create_dir, {:error, e}} -> {:error, :create_dir, e}
       {:ls, {:error, e}} -> {:error, :ls, e}

lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex

@@ -33,6 +33,20 @@ def index_operation do
       tags: ["Emoji Packs"],
       summary: "Lists local custom emoji packs",
       operationId: "PleromaAPI.EmojiPackController.index",
+      parameters: [
+        Operation.parameter(
+          :page,
+          :query,
+          %Schema{type: :integer, default: 1},
+          "Page"
+        ),
+        Operation.parameter(
+          :page_size,
+          :query,
+          %Schema{type: :integer, default: 50},
+          "Number of emoji packs to return"
+        )
+      ],
       responses: %{
         200 => emoji_packs_response()
       }
@@ -44,7 +58,21 @@ def show_operation do
       tags: ["Emoji Packs"],
       summary: "Show emoji pack",
       operationId: "PleromaAPI.EmojiPackController.show",
-      parameters: [name_param()],
+      parameters: [
+        name_param(),
+        Operation.parameter(
+          :page,
+          :query,
+          %Schema{type: :integer, default: 1},
+          "Page"
+        ),
+        Operation.parameter(
+          :page_size,
+          :query,
+          %Schema{type: :integer, default: 30},
+          "Number of emoji to return"
+        )
+      ],
       responses: %{
         200 => Operation.response("Emoji Pack", "application/json", emoji_pack()),
         400 => Operation.response("Bad Request", "application/json", ApiError),

lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex

@@ -37,14 +37,14 @@ def remote(conn, %{url: url}) do
     end
   end
 
-  def index(conn, _params) do
+  def index(conn, params) do
     emoji_path =
       [:instance, :static_dir]
       |> Pleroma.Config.get!()
       |> Path.join("emoji")
 
-    with {:ok, packs} <- Pack.list_local() do
+    with {:ok, packs, count} <- Pack.list_local(page: params.page, page_size: params.page_size) do
-      json(conn, packs)
+      json(conn, %{packs: packs, count: count})
     else
       {:error, :create_dir, e} ->
         conn
@@ -60,10 +60,10 @@ def index(conn, _params) do
     end
   end
 
-  def show(conn, %{name: name}) do
+  def show(conn, %{name: name, page: page, page_size: page_size}) do
     name = String.trim(name)
 
-    with {:ok, pack} <- Pack.show(name) do
+    with {:ok, pack} <- Pack.show(name: name, page: page, page_size: page_size) do
       json(conn, pack)
     else
       {:error, :not_found} ->

test/instance_static/emoji/test_pack/pack.json

@@ -1,6 +1,7 @@
 {
     "files": {
-        "blank": "blank.png"
+        "blank": "blank.png",
+        "blank2": "blank2.png"
     },
     "pack": {
         "description": "Test description",

test/instance_static/emoji/test_pack_nonshared/pack.json

@@ -4,7 +4,7 @@
         "homepage": "https://pleroma.social",
         "description": "Test description",
         "fallback-src": "https://nonshared-pack",
-        "fallback-src-sha256": "74409E2674DAA06C072729C6C8426C4CB3B7E0B85ED77792DB7A436E11D76DAF",
+        "fallback-src-sha256": "1967BB4E42BCC34BCC12D57BE7811D3B7BE52F965BCE45C87BD377B9499CE11D",
         "share-files": false
     },
     "files": {

test/web/pleroma_api/controllers/emoji_pack_controller_test.exs

@@ -30,15 +30,55 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerTest do
   test "GET /api/pleroma/emoji/packs", %{conn: conn} do
     resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200)
 
-    shared = resp["test_pack"]
+    assert resp["count"] == 3
-    assert shared["files"] == %{"blank" => "blank.png"}
+
+    assert resp["packs"]
+           |> Map.keys()
+           |> length() == 3
+
+    shared = resp["packs"]["test_pack"]
+    assert shared["files"] == %{"blank" => "blank.png", "blank2" => "blank2.png"}
     assert Map.has_key?(shared["pack"], "download-sha256")
     assert shared["pack"]["can-download"]
     assert shared["pack"]["share-files"]
 
-    non_shared = resp["test_pack_nonshared"]
+    non_shared = resp["packs"]["test_pack_nonshared"]
     assert non_shared["pack"]["share-files"] == false
     assert non_shared["pack"]["can-download"] == false
+
+    resp =
+      conn
+      |> get("/api/pleroma/emoji/packs?page_size=1")
+      |> json_response_and_validate_schema(200)
+
+    assert resp["count"] == 3
+
+    packs = Map.keys(resp["packs"])
+
+    assert length(packs) == 1
+
+    [pack1] = packs
+
+    resp =
+      conn
+      |> get("/api/pleroma/emoji/packs?page_size=1&page=2")
+      |> json_response_and_validate_schema(200)
+
+    assert resp["count"] == 3
+    packs = Map.keys(resp["packs"])
+    assert length(packs) == 1
+    [pack2] = packs
+
+    resp =
+      conn
+      |> get("/api/pleroma/emoji/packs?page_size=1&page=3")
+      |> json_response_and_validate_schema(200)
+
+    assert resp["count"] == 3
+    packs = Map.keys(resp["packs"])
+    assert length(packs) == 1
+    [pack3] = packs
+    assert [pack1, pack2, pack3] |> Enum.uniq() |> length() == 3
   end
 
   describe "GET /api/pleroma/emoji/packs/remote" do
@@ -332,7 +372,7 @@ test "for a pack with a fallback source", ctx do
         Map.put(
           new_data,
           "fallback-src-sha256",
-          "74409E2674DAA06C072729C6C8426C4CB3B7E0B85ED77792DB7A436E11D76DAF"
+          "1967BB4E42BCC34BCC12D57BE7811D3B7BE52F965BCE45C87BD377B9499CE11D"
         )
 
       assert ctx[:admin_conn]
@@ -398,7 +438,7 @@ test "don't rewrite old emoji", %{admin_conn: admin_conn} do
       assert admin_conn
              |> put_req_header("content-type", "multipart/form-data")
              |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank2",
+               shortcode: "blank3",
                filename: "dir/blank.png",
                file: %Plug.Upload{
                  filename: "blank.png",
@@ -407,7 +447,8 @@ test "don't rewrite old emoji", %{admin_conn: admin_conn} do
              })
              |> json_response_and_validate_schema(200) == %{
                "blank" => "blank.png",
-               "blank2" => "dir/blank.png"
+               "blank2" => "blank2.png",
+               "blank3" => "dir/blank.png"
              }
 
       assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
@@ -431,7 +472,7 @@ test "rewrite old emoji with force option", %{admin_conn: admin_conn} do
       assert admin_conn
              |> put_req_header("content-type", "multipart/form-data")
              |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank2",
+               shortcode: "blank3",
                filename: "dir/blank.png",
                file: %Plug.Upload{
                  filename: "blank.png",
@@ -440,7 +481,8 @@ test "rewrite old emoji with force option", %{admin_conn: admin_conn} do
              })
              |> json_response_and_validate_schema(200) == %{
                "blank" => "blank.png",
-               "blank2" => "dir/blank.png"
+               "blank2" => "blank2.png",
+               "blank3" => "dir/blank.png"
              }
 
       assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
@@ -448,14 +490,15 @@ test "rewrite old emoji with force option", %{admin_conn: admin_conn} do
       assert admin_conn
              |> put_req_header("content-type", "multipart/form-data")
              |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank2",
+               shortcode: "blank3",
-               new_shortcode: "blank3",
+               new_shortcode: "blank4",
                new_filename: "dir_2/blank_3.png",
                force: true
              })
              |> json_response_and_validate_schema(200) == %{
                "blank" => "blank.png",
-               "blank3" => "dir_2/blank_3.png"
+               "blank2" => "blank2.png",
+               "blank4" => "dir_2/blank_3.png"
              }
 
       assert File.exists?("#{@emoji_path}/test_pack/dir_2/blank_3.png")
@@ -481,7 +524,7 @@ test "add file with not loaded pack", %{admin_conn: admin_conn} do
       assert admin_conn
              |> put_req_header("content-type", "multipart/form-data")
              |> post("/api/pleroma/emoji/packs/not_loaded/files", %{
-               shortcode: "blank2",
+               shortcode: "blank3",
                filename: "dir/blank.png",
                file: %Plug.Upload{
                  filename: "blank.png",
@@ -535,7 +578,8 @@ test "new with shortcode as file with update", %{admin_conn: admin_conn} do
              })
              |> json_response_and_validate_schema(200) == %{
                "blank" => "blank.png",
-               "blank4" => "dir/blank.png"
+               "blank4" => "dir/blank.png",
+               "blank2" => "blank2.png"
              }
 
       assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
@@ -549,7 +593,8 @@ test "new with shortcode as file with update", %{admin_conn: admin_conn} do
              })
              |> json_response_and_validate_schema(200) == %{
                "blank3" => "dir_2/blank_3.png",
-               "blank" => "blank.png"
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
              }
 
       refute File.exists?("#{@emoji_path}/test_pack/dir/")
@@ -557,7 +602,10 @@ test "new with shortcode as file with update", %{admin_conn: admin_conn} do
 
       assert admin_conn
              |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank3")
-             |> json_response_and_validate_schema(200) == %{"blank" => "blank.png"}
+             |> json_response_and_validate_schema(200) == %{
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
+             }
 
       refute File.exists?("#{@emoji_path}/test_pack/dir_2/")
 
@@ -581,7 +629,8 @@ test "new with shortcode from url", %{admin_conn: admin_conn} do
              })
              |> json_response_and_validate_schema(200) == %{
                "blank_url" => "blank_url.png",
-               "blank" => "blank.png"
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
              }
 
       assert File.exists?("#{@emoji_path}/test_pack/blank_url.png")
@@ -602,15 +651,16 @@ test "new without shortcode", %{admin_conn: admin_conn} do
              })
              |> json_response_and_validate_schema(200) == %{
                "shortcode" => "shortcode.png",
-               "blank" => "blank.png"
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
              }
     end
 
     test "remove non existing shortcode in pack.json", %{admin_conn: admin_conn} do
       assert admin_conn
-             |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank2")
+             |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank3")
              |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "Emoji \"blank2\" does not exist"
+               "error" => "Emoji \"blank3\" does not exist"
              }
     end
 
@@ -618,12 +668,12 @@ test "update non existing emoji", %{admin_conn: admin_conn} do
       assert admin_conn
              |> put_req_header("content-type", "multipart/form-data")
              |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank2",
+               shortcode: "blank3",
-               new_shortcode: "blank3",
+               new_shortcode: "blank4",
                new_filename: "dir_2/blank_3.png"
              })
              |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "Emoji \"blank2\" does not exist"
+               "error" => "Emoji \"blank3\" does not exist"
              }
     end
 
@@ -651,7 +701,8 @@ test "creating and deleting a pack", %{admin_conn: admin_conn} do
 
       assert Jason.decode!(File.read!("#{@emoji_path}/test_created/pack.json")) == %{
                "pack" => %{},
-               "files" => %{}
+               "files" => %{},
+               "files_count" => 0
              }
 
       assert admin_conn
@@ -709,14 +760,14 @@ test "filesystem import", %{admin_conn: admin_conn, conn: conn} do
 
     resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200)
 
-    refute Map.has_key?(resp, "test_pack_for_import")
+    refute Map.has_key?(resp["packs"], "test_pack_for_import")
 
     assert admin_conn
            |> get("/api/pleroma/emoji/packs/import")
            |> json_response_and_validate_schema(200) == ["test_pack_for_import"]
 
     resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200)
-    assert resp["test_pack_for_import"]["files"] == %{"blank" => "blank.png"}
+    assert resp["packs"]["test_pack_for_import"]["files"] == %{"blank" => "blank.png"}
 
     File.rm!("#{@emoji_path}/test_pack_for_import/pack.json")
     refute File.exists?("#{@emoji_path}/test_pack_for_import/pack.json")
@@ -736,7 +787,7 @@ test "filesystem import", %{admin_conn: admin_conn, conn: conn} do
 
     resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200)
 
-    assert resp["test_pack_for_import"]["files"] == %{
+    assert resp["packs"]["test_pack_for_import"]["files"] == %{
              "blank" => "blank.png",
              "blank2" => "blank.png",
              "foo" => "blank.png"
@@ -746,7 +797,8 @@ test "filesystem import", %{admin_conn: admin_conn, conn: conn} do
   describe "GET /api/pleroma/emoji/packs/:name" do
     test "shows pack.json", %{conn: conn} do
       assert %{
-               "files" => %{"blank" => "blank.png"},
+               "files" => files,
+               "files_count" => 2,
                "pack" => %{
                  "can-download" => true,
                  "description" => "Test description",
@@ -759,6 +811,28 @@ test "shows pack.json", %{conn: conn} do
                conn
                |> get("/api/pleroma/emoji/packs/test_pack")
                |> json_response_and_validate_schema(200)
+
+      assert files == %{"blank" => "blank.png", "blank2" => "blank2.png"}
+
+      assert %{
+               "files" => files,
+               "files_count" => 2
+             } =
+               conn
+               |> get("/api/pleroma/emoji/packs/test_pack?page_size=1")
+               |> json_response_and_validate_schema(200)
+
+      assert files |> Map.keys() |> length() == 1
+
+      assert %{
+               "files" => files,
+               "files_count" => 2
+             } =
+               conn
+               |> get("/api/pleroma/emoji/packs/test_pack?page_size=1&page=2")
+               |> json_response_and_validate_schema(200)
+
+      assert files |> Map.keys() |> length() == 1
     end
 
     test "non existing pack", %{conn: conn} do