Merge remote-tracking branch 'upstream/develop' into restrict-origin

.gitignore

@@ -27,8 +27,6 @@ erl_crash.dump
 # variables.
 /config/*.secret.exs
 /config/generated_config.exs
-/config/*.env
-
 
 # Database setup file, some may forget to delete it
 /config/setup_db.psql

.gitlab-ci.yml

@@ -22,6 +22,7 @@ stages:
   - docker
 
 before_script:
+  - apt-get update && apt-get install -y cmake
   - mix local.hex --force
   - mix local.rebar --force
 
@@ -58,25 +59,27 @@ unit-testing:
     alias: postgres
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
-    - apt-get update && apt-get install -y libimage-exiftool-perl
+    - apt-get update && apt-get install -y libimage-exiftool-perl ffmpeg
     - mix deps.get
     - mix ecto.create
     - mix ecto.migrate
     - mix coveralls --preload-modules
 
-federated-testing:
+# Removed to fix CI issue. In this early state it wasn't adding much value anyway.
-  stage: test
+# TODO Fix and reinstate federated testing
-  cache: *testing_cache_policy
+# federated-testing:
-  services:
+#   stage: test
-  - name: minibikini/postgres-with-rum:12
+#   cache: *testing_cache_policy
-    alias: postgres
+#   services:
-    command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+#   - name: minibikini/postgres-with-rum:12
-  script:
+#     alias: postgres
-    - mix deps.get
+#     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
-    - mix ecto.create
+#   script:
-    - mix ecto.migrate
+#     - mix deps.get
-    - epmd -daemon
+#     - mix ecto.create
-    - mix test --trace --only federated
+#     - mix ecto.migrate
+#     - epmd -daemon
+#     - mix test --trace --only federated
 
 unit-testing-rum:
   stage: test
@@ -90,7 +93,7 @@ unit-testing-rum:
     <<: *global_variables
     RUM_ENABLED: "true"
   script:
-    - apt-get update && apt-get install -y libimage-exiftool-perl
+    - apt-get update && apt-get install -y libimage-exiftool-perl ffmpeg
     - mix deps.get
     - mix ecto.create
     - mix ecto.migrate
@@ -193,6 +196,7 @@ amd64:
   variables: &release-variables
     MIX_ENV: prod
   before_script: &before-release
+  - apt-get update && apt-get install -y cmake
   - echo "import Mix.Config" > config/prod.secret.exs
   - mix local.hex --force
   - mix local.rebar --force
@@ -211,7 +215,7 @@ amd64-musl:
   cache: *release-cache
   variables: *release-variables
   before_script: &before-release-musl
-  - apk add git gcc g++ musl-dev make
+  - apk add git gcc g++ musl-dev make cmake
   - echo "import Mix.Config" > config/prod.secret.exs
   - mix local.hex --force
   - mix local.rebar --force
@@ -278,6 +282,8 @@ docker:
     IMAGE_TAG_SLUG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG
     IMAGE_TAG_LATEST: $CI_REGISTRY_IMAGE:latest
     IMAGE_TAG_LATEST_STABLE: $CI_REGISTRY_IMAGE:latest-stable
+    DOCKER_BUILDX_URL: https://github.com/docker/buildx/releases/download/v0.4.1/buildx-v0.4.1.linux-amd64
+    DOCKER_BUILDX_HASH: 71a7d01439aa8c165a25b59c44d3f016fddbd98b
   before_script: &before-docker
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
     - docker pull $IMAGE_TAG_SLUG || true
@@ -285,10 +291,14 @@ docker:
     - export CI_VCS_REF=$CI_COMMIT_SHORT_SHA
   allow_failure: true
   script:
-    - docker build --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t $IMAGE_TAG_SLUG -t $IMAGE_TAG_LATEST .
+    - mkdir -p /root/.docker/cli-plugins
-    - docker push $IMAGE_TAG
+    - wget "${DOCKER_BUILDX_URL}" -O ~/.docker/cli-plugins/docker-buildx
-    - docker push $IMAGE_TAG_SLUG
+    - echo "${DOCKER_BUILDX_HASH}  /root/.docker/cli-plugins/docker-buildx" | sha1sum -c
-    - docker push $IMAGE_TAG_LATEST
+    - chmod +x ~/.docker/cli-plugins/docker-buildx
+    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+    - docker buildx create --name mbuilder --driver docker-container --use
+    - docker buildx inspect --bootstrap
+    - docker buildx build --platform linux/amd64,linux/arm/v7,linux/arm64/v8 --push --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t $IMAGE_TAG_SLUG -t $IMAGE_TAG_LATEST .
   tags:
     - dind
   only:
@@ -303,10 +313,14 @@ docker-stable:
   before_script: *before-docker
   allow_failure: true
   script:
-    - docker build --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t $IMAGE_TAG_SLUG -t $IMAGE_TAG_LATEST_STABLE .
+    - mkdir -p /root/.docker/cli-plugins
-    - docker push $IMAGE_TAG
+    - wget "${DOCKER_BUILDX_URL}" -O ~/.docker/cli-plugins/docker-buildx
-    - docker push $IMAGE_TAG_SLUG
+    - echo "${DOCKER_BUILDX_HASH}  /root/.docker/cli-plugins/docker-buildx" | sha1sum -c
-    - docker push $IMAGE_TAG_LATEST_STABLE
+    - chmod +x ~/.docker/cli-plugins/docker-buildx
+    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+    - docker buildx create --name mbuilder --driver docker-container --use
+    - docker buildx inspect --bootstrap
+    - docker buildx build --platform linux/amd64,linux/arm/v7,linux/arm64/v8 --push --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t $IMAGE_TAG_SLUG -t $IMAGE_TAG_LATEST_STABLE .
   tags:
     - dind
   only:
@@ -321,9 +335,15 @@ docker-release:
   before_script: *before-docker
   allow_failure: true
   script:
-    - docker build --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t       $IMAGE_TAG_SLUG .
+  script:
-    - docker push $IMAGE_TAG
+    - mkdir -p /root/.docker/cli-plugins
-    - docker push $IMAGE_TAG_SLUG
+    - wget "${DOCKER_BUILDX_URL}" -O ~/.docker/cli-plugins/docker-buildx
+    - echo "${DOCKER_BUILDX_HASH}  /root/.docker/cli-plugins/docker-buildx" | sha1sum -c
+    - chmod +x ~/.docker/cli-plugins/docker-buildx
+    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+    - docker buildx create --name mbuilder --driver docker-container --use
+    - docker buildx inspect --bootstrap
+    - docker buildx build --platform linux/amd64,linux/arm/v7,linux/arm64/v8 --push --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t $IMAGE_TAG_SLUG .
   tags:
     - dind
   only:

.gitlab/merge_request_templates/Release.md

@@ -3,3 +3,4 @@
 * [ ]  Compile a changelog
 * [ ]  Create an MR with an announcement to pleroma.social
 * [ ]  Tag the release
+* [ ] Merge `stable` into `develop` (in case the fixes are already in develop, use `git merge -s ours --no-commit` and manually merge the changelogs)

CC-BY-4.0

@@ -0,0 +1,395 @@
+Attribution 4.0 International
+
+=======================================================================
+
+Creative Commons Corporation ("Creative Commons") is not a law firm and
+does not provide legal services or legal advice. Distribution of
+Creative Commons public licenses does not create a lawyer-client or
+other relationship. Creative Commons makes its licenses and related
+information available on an "as-is" basis. Creative Commons gives no
+warranties regarding its licenses, any material licensed under their
+terms and conditions, or any related information. Creative Commons
+disclaims all liability for damages resulting from their use to the
+fullest extent possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and
+conditions that creators and other rights holders may use to share
+original works of authorship and other material subject to copyright
+and certain other rights specified in the public license below. The
+following considerations are for informational purposes only, are not
+exhaustive, and do not form part of our licenses.
+
+     Considerations for licensors: Our public licenses are
+     intended for use by those authorized to give the public
+     permission to use material in ways otherwise restricted by
+     copyright and certain other rights. Our licenses are
+     irrevocable. Licensors should read and understand the terms
+     and conditions of the license they choose before applying it.
+     Licensors should also secure all rights necessary before
+     applying our licenses so that the public can reuse the
+     material as expected. Licensors should clearly mark any
+     material not subject to the license. This includes other CC-
+     licensed material, or material used under an exception or
+     limitation to copyright. More considerations for licensors:
+    wiki.creativecommons.org/Considerations_for_licensors
+
+     Considerations for the public: By using one of our public
+     licenses, a licensor grants the public permission to use the
+     licensed material under specified terms and conditions. If
+     the licensor's permission is not necessary for any reason--for
+     example, because of any applicable exception or limitation to
+     copyright--then that use is not regulated by the license. Our
+     licenses grant only permissions under copyright and certain
+     other rights that a licensor has authority to grant. Use of
+     the licensed material may still be restricted for other
+     reasons, including because others have copyright or other
+     rights in the material. A licensor may make special requests,
+     such as asking that all changes be marked or described.
+     Although not required by our licenses, you are encouraged to
+     respect those requests where reasonable. More considerations
+     for the public:
+    wiki.creativecommons.org/Considerations_for_licensees
+
+=======================================================================
+
+Creative Commons Attribution 4.0 International Public License
+
+By exercising the Licensed Rights (defined below), You accept and agree
+to be bound by the terms and conditions of this Creative Commons
+Attribution 4.0 International Public License ("Public License"). To the
+extent this Public License may be interpreted as a contract, You are
+granted the Licensed Rights in consideration of Your acceptance of
+these terms and conditions, and the Licensor grants You such rights in
+consideration of benefits the Licensor receives from making the
+Licensed Material available under these terms and conditions.
+
+
+Section 1 -- Definitions.
+
+  a. Adapted Material means material subject to Copyright and Similar
+     Rights that is derived from or based upon the Licensed Material
+     and in which the Licensed Material is translated, altered,
+     arranged, transformed, or otherwise modified in a manner requiring
+     permission under the Copyright and Similar Rights held by the
+     Licensor. For purposes of this Public License, where the Licensed
+     Material is a musical work, performance, or sound recording,
+     Adapted Material is always produced where the Licensed Material is
+     synched in timed relation with a moving image.
+
+  b. Adapter's License means the license You apply to Your Copyright
+     and Similar Rights in Your contributions to Adapted Material in
+     accordance with the terms and conditions of this Public License.
+
+  c. Copyright and Similar Rights means copyright and/or similar rights
+     closely related to copyright including, without limitation,
+     performance, broadcast, sound recording, and Sui Generis Database
+     Rights, without regard to how the rights are labeled or
+     categorized. For purposes of this Public License, the rights
+     specified in Section 2(b)(1)-(2) are not Copyright and Similar
+     Rights.
+
+  d. Effective Technological Measures means those measures that, in the
+     absence of proper authority, may not be circumvented under laws
+     fulfilling obligations under Article 11 of the WIPO Copyright
+     Treaty adopted on December 20, 1996, and/or similar international
+     agreements.
+
+  e. Exceptions and Limitations means fair use, fair dealing, and/or
+     any other exception or limitation to Copyright and Similar Rights
+     that applies to Your use of the Licensed Material.
+
+  f. Licensed Material means the artistic or literary work, database,
+     or other material to which the Licensor applied this Public
+     License.
+
+  g. Licensed Rights means the rights granted to You subject to the
+     terms and conditions of this Public License, which are limited to
+     all Copyright and Similar Rights that apply to Your use of the
+     Licensed Material and that the Licensor has authority to license.
+
+  h. Licensor means the individual(s) or entity(ies) granting rights
+     under this Public License.
+
+  i. Share means to provide material to the public by any means or
+     process that requires permission under the Licensed Rights, such
+     as reproduction, public display, public performance, distribution,
+     dissemination, communication, or importation, and to make material
+     available to the public including in ways that members of the
+     public may access the material from a place and at a time
+     individually chosen by them.
+
+  j. Sui Generis Database Rights means rights other than copyright
+     resulting from Directive 96/9/EC of the European Parliament and of
+     the Council of 11 March 1996 on the legal protection of databases,
+     as amended and/or succeeded, as well as other essentially
+     equivalent rights anywhere in the world.
+
+  k. You means the individual or entity exercising the Licensed Rights
+     under this Public License. Your has a corresponding meaning.
+
+
+Section 2 -- Scope.
+
+  a. License grant.
+
+       1. Subject to the terms and conditions of this Public License,
+          the Licensor hereby grants You a worldwide, royalty-free,
+          non-sublicensable, non-exclusive, irrevocable license to
+          exercise the Licensed Rights in the Licensed Material to:
+
+            a. reproduce and Share the Licensed Material, in whole or
+               in part; and
+
+            b. produce, reproduce, and Share Adapted Material.
+
+       2. Exceptions and Limitations. For the avoidance of doubt, where
+          Exceptions and Limitations apply to Your use, this Public
+          License does not apply, and You do not need to comply with
+          its terms and conditions.
+
+       3. Term. The term of this Public License is specified in Section
+          6(a).
+
+       4. Media and formats; technical modifications allowed. The
+          Licensor authorizes You to exercise the Licensed Rights in
+          all media and formats whether now known or hereafter created,
+          and to make technical modifications necessary to do so. The
+          Licensor waives and/or agrees not to assert any right or
+          authority to forbid You from making technical modifications
+          necessary to exercise the Licensed Rights, including
+          technical modifications necessary to circumvent Effective
+          Technological Measures. For purposes of this Public License,
+          simply making modifications authorized by this Section 2(a)
+          (4) never produces Adapted Material.
+
+       5. Downstream recipients.
+
+            a. Offer from the Licensor -- Licensed Material. Every
+               recipient of the Licensed Material automatically
+               receives an offer from the Licensor to exercise the
+               Licensed Rights under the terms and conditions of this
+               Public License.
+
+            b. No downstream restrictions. You may not offer or impose
+               any additional or different terms or conditions on, or
+               apply any Effective Technological Measures to, the
+               Licensed Material if doing so restricts exercise of the
+               Licensed Rights by any recipient of the Licensed
+               Material.
+
+       6. No endorsement. Nothing in this Public License constitutes or
+          may be construed as permission to assert or imply that You
+          are, or that Your use of the Licensed Material is, connected
+          with, or sponsored, endorsed, or granted official status by,
+          the Licensor or others designated to receive attribution as
+          provided in Section 3(a)(1)(A)(i).
+
+  b. Other rights.
+
+       1. Moral rights, such as the right of integrity, are not
+          licensed under this Public License, nor are publicity,
+          privacy, and/or other similar personality rights; however, to
+          the extent possible, the Licensor waives and/or agrees not to
+          assert any such rights held by the Licensor to the limited
+          extent necessary to allow You to exercise the Licensed
+          Rights, but not otherwise.
+
+       2. Patent and trademark rights are not licensed under this
+          Public License.
+
+       3. To the extent possible, the Licensor waives any right to
+          collect royalties from You for the exercise of the Licensed
+          Rights, whether directly or through a collecting society
+          under any voluntary or waivable statutory or compulsory
+          licensing scheme. In all other cases the Licensor expressly
+          reserves any right to collect such royalties.
+
+
+Section 3 -- License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the
+following conditions.
+
+  a. Attribution.
+
+       1. If You Share the Licensed Material (including in modified
+          form), You must:
+
+            a. retain the following if it is supplied by the Licensor
+               with the Licensed Material:
+
+                 i. identification of the creator(s) of the Licensed
+                    Material and any others designated to receive
+                    attribution, in any reasonable manner requested by
+                    the Licensor (including by pseudonym if
+                    designated);
+
+                ii. a copyright notice;
+
+               iii. a notice that refers to this Public License;
+
+                iv. a notice that refers to the disclaimer of
+                    warranties;
+
+                 v. a URI or hyperlink to the Licensed Material to the
+                    extent reasonably practicable;
+
+            b. indicate if You modified the Licensed Material and
+               retain an indication of any previous modifications; and
+
+            c. indicate the Licensed Material is licensed under this
+               Public License, and include the text of, or the URI or
+               hyperlink to, this Public License.
+
+       2. You may satisfy the conditions in Section 3(a)(1) in any
+          reasonable manner based on the medium, means, and context in
+          which You Share the Licensed Material. For example, it may be
+          reasonable to satisfy the conditions by providing a URI or
+          hyperlink to a resource that includes the required
+          information.
+
+       3. If requested by the Licensor, You must remove any of the
+          information required by Section 3(a)(1)(A) to the extent
+          reasonably practicable.
+
+       4. If You Share Adapted Material You produce, the Adapter's
+          License You apply must not prevent recipients of the Adapted
+          Material from complying with this Public License.
+
+
+Section 4 -- Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that
+apply to Your use of the Licensed Material:
+
+  a. for the avoidance of doubt, Section 2(a)(1) grants You the right
+     to extract, reuse, reproduce, and Share all or a substantial
+     portion of the contents of the database;
+
+  b. if You include all or a substantial portion of the database
+     contents in a database in which You have Sui Generis Database
+     Rights, then the database in which You have Sui Generis Database
+     Rights (but not its individual contents) is Adapted Material; and
+
+  c. You must comply with the conditions in Section 3(a) if You Share
+     all or a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not
+replace Your obligations under this Public License where the Licensed
+Rights include other Copyright and Similar Rights.
+
+
+Section 5 -- Disclaimer of Warranties and Limitation of Liability.
+
+  a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
+     EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
+     AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
+     ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
+     IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
+     WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
+     PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
+     ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
+     KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
+     ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
+
+  b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
+     TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
+     NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
+     INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
+     COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
+     USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
+     ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
+     DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
+     IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
+
+  c. The disclaimer of warranties and limitation of liability provided
+     above shall be interpreted in a manner that, to the extent
+     possible, most closely approximates an absolute disclaimer and
+     waiver of all liability.
+
+
+Section 6 -- Term and Termination.
+
+  a. This Public License applies for the term of the Copyright and
+     Similar Rights licensed here. However, if You fail to comply with
+     this Public License, then Your rights under this Public License
+     terminate automatically.
+
+  b. Where Your right to use the Licensed Material has terminated under
+     Section 6(a), it reinstates:
+
+       1. automatically as of the date the violation is cured, provided
+          it is cured within 30 days of Your discovery of the
+          violation; or
+
+       2. upon express reinstatement by the Licensor.
+
+     For the avoidance of doubt, this Section 6(b) does not affect any
+     right the Licensor may have to seek remedies for Your violations
+     of this Public License.
+
+  c. For the avoidance of doubt, the Licensor may also offer the
+     Licensed Material under separate terms or conditions or stop
+     distributing the Licensed Material at any time; however, doing so
+     will not terminate this Public License.
+
+  d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
+     License.
+
+
+Section 7 -- Other Terms and Conditions.
+
+  a. The Licensor shall not be bound by any additional or different
+     terms or conditions communicated by You unless expressly agreed.
+
+  b. Any arrangements, understandings, or agreements regarding the
+     Licensed Material not stated herein are separate from and
+     independent of the terms and conditions of this Public License.
+
+
+Section 8 -- Interpretation.
+
+  a. For the avoidance of doubt, this Public License does not, and
+     shall not be interpreted to, reduce, limit, restrict, or impose
+     conditions on any use of the Licensed Material that could lawfully
+     be made without permission under this Public License.
+
+  b. To the extent possible, if any provision of this Public License is
+     deemed unenforceable, it shall be automatically reformed to the
+     minimum extent necessary to make it enforceable. If the provision
+     cannot be reformed, it shall be severed from this Public License
+     without affecting the enforceability of the remaining terms and
+     conditions.
+
+  c. No term or condition of this Public License will be waived and no
+     failure to comply consented to unless expressly agreed to by the
+     Licensor.
+
+  d. Nothing in this Public License constitutes or may be interpreted
+     as a limitation upon, or waiver of, any privileges and immunities
+     that apply to the Licensor or You, including from the legal
+     processes of any jurisdiction or authority.
+
+
+=======================================================================
+
+Creative Commons is not a party to its public
+licenses. Notwithstanding, Creative Commons may elect to apply one of
+its public licenses to material it publishes and in those instances
+will be considered the “Licensor.” The text of the Creative Commons
+public licenses is dedicated to the public domain under the CC0 Public
+Domain Dedication. Except for the limited purpose of indicating that
+material is shared under a Creative Commons public license or as
+otherwise permitted by the Creative Commons policies published at
+creativecommons.org/policies, Creative Commons does not authorize the
+use of the trademark "Creative Commons" or any other trademark or logo
+of Creative Commons without its prior written consent including,
+without limitation, in connection with any unauthorized modifications
+to any of its public licenses or any other arrangements,
+understandings, or agreements concerning use of licensed material. For
+the avoidance of doubt, this paragraph does not form part of the
+public licenses.
+
+Creative Commons may be contacted at creativecommons.org.

CHANGELOG.md

@@ -3,18 +3,127 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
-## [unreleased]
+## Unreleased
+
+### Added
+- Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
+- Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
+- Mix task option for force-unfollowing relays
 
 ### Changed
+
+- **Breaking:** Pleroma Admin API: emoji packs and files routes changed.
+- **Breaking:** Sensitive/NSFW statuses no longer disable link previews.
+- Search: Users are now findable by their urls.
+- Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
+- Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.
+- The `discoverable` field in the `User` struct will now add a NOINDEX metatag to profile pages when false.
+- Users with the `discoverable` field set to false will not show up in searches.
+- Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
+- Introduced optional dependencies on `ffmpeg`, `ImageMagick`, `exiftool` software packages. Please refer to `docs/installation/optional/media_graphics_packages.md`.
+
+### Added
+- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
+- Pleroma API: Importing the mutes users from CSV files.
+- Experimental websocket-based federation between Pleroma instances.
+
+<details>
+  <summary>API Changes</summary>
+
+- Pleroma API: Importing the mutes users from CSV files.
+- Admin API: Importing emoji from a zip file
+- Pleroma API: Pagination for remote/local packs and emoji.
+
+</details>
+
+### Removed
+
+- **Breaking:** `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab` (moved to a simpler implementation).
+- **Breaking:** `Pleroma.Workers.Cron.ClearOauthTokenWorker` setting from Oban `:crontab` (moved to scheduled jobs).
+- **Breaking:** `Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker` setting from Oban `:crontab` (moved to scheduled jobs).
+- Removed `:managed_config` option. In practice, it was accidentally removed with 2.0.0 release when frontends were
+switched to a new configuration mechanism, however it was not officially removed until now.
+
+### Fixed
+
+- Add documented-but-missing chat pagination.
+- Allow sending out emails again.
+
+## Unreleased (Patch)
+
+### Changed
+- API: Empty parameter values for integer parameters are now ignored in non-strict validaton mode.
+
+## [2.1.2] - 2020-09-17
+
+### Security
+
+- Fix most MRF rules either crashing or not being applied to objects passed into the Common Pipeline (ChatMessage, Question, Answer, Audio, Event).
+
+### Fixed
+
+- Welcome Chat messages preventing user registration with MRF Simple Policy applied to the local instance.
+- Mastodon API: the public timeline returning an error when the `reply_visibility` parameter is set to `self` for an unauthenticated user.
+- Mastodon Streaming API: Handler crashes on authentication failures, resulting in error logs.
+- Mastodon Streaming API: Error logs on client pings.
+- Rich media: Log spam on failures. Now the error is only logged once per attempt.
+
+### Changed
+
+- Rich Media: A HEAD request is now done to the url, to ensure it has the appropriate content type and size before proceeding with a GET.
+
+### Upgrade notes
+
+1. Restart Pleroma
+
+## [2.1.1] - 2020-09-08
+
+### Security
+- Fix possible DoS in Mastodon API user search due to an error in match clauses, leading to an infinite recursion and subsequent OOM with certain inputs.
+- Fix metadata leak for accounts and statuses on private instances.
+- Fix possible DoS in Admin API search using an atom leak vulnerability. Authentication with admin rights was required to exploit.
+
+### Changed
+
+- **Breaking:** The metadata providers RelMe and Feed are no longer configurable. RelMe should always be activated and Feed only provides a <link> header tag for the actual RSS/Atom feed when the instance is public.
+- Improved error message when cmake is not available at build stage.
+
+### Added
+- Rich media failure tracking (along with `:failure_backoff` option).
+
+<details>
+  <summary>Admin API Changes</summary>
+
+- Add `PATCH /api/pleroma/admin/instance_document/:document_name` to modify the Terms of Service and Instance Panel HTML pages via Admin API
+</details>
+
+### Fixed
+- Default HTTP adapter not respecting pool setting, leading to possible OOM.
+- Fixed uploading webp images when the Exiftool Upload Filter is enabled by skipping them
+- Mastodon API: Search parameter `following` now correctly returns the followings rather than the followers
+- Mastodon API: Timelines hanging for (`number of posts with links * rich media timeout`) in the worst case.
+  Reduced to just rich media timeout.
+- Mastodon API: Cards being wrong for preview statuses due to cache key collision.
+- Password resets no longer processed for deactivated accounts.
+- Favicon scraper raising exceptions on URLs longer than 255 characters.
+
+## [2.1.0] - 2020-08-28
+
+### Changed
+
+- **Breaking:** The default descriptions on uploads are now empty. The old behavior (filename as default) can be configured, see the cheat sheet.
+- **Breaking:** Added the ObjectAgePolicy to the default set of MRFs. This will delist and strip the follower collection of any message received that is older than 7 days. This will stop users from seeing very old messages in the timelines. The messages can still be viewed on the user's page and in conversations. They also still trigger notifications.
 - **Breaking:** Elixir >=1.9 is now required (was >= 1.8)
 - **Breaking:** Configuration: `:auto_linker, :opts` moved to `:pleroma, Pleroma.Formatter`. Old config namespace is deprecated.
+- **Breaking:** Configuration: `:instance, welcome_user_nickname` moved to `:welcome, :direct_message, :sender_nickname`, `:instance, :welcome_message` moved to `:welcome, :direct_message, :message`. Old config namespace is deprecated.
+- **Breaking:** LDAP: Fallback to local database authentication has been removed for security reasons and lack of a mechanism to ensure the passwords are synchronized when LDAP passwords are updated.
+- **Breaking** Changed defaults for `:restrict_unauthenticated` so that when `:instance, :public` is set to `false` then all `:restrict_unauthenticated` items be effectively set to `true`. If you'd like to allow unauthenticated access to specific API endpoints on a private instance, please explicitly set `:restrict_unauthenticated` to non-default value in `config/prod.secret.exs`.
 - In Conversations, return only direct messages as `last_status`
 - Using the `only_media` filter on timelines will now exclude reblog media
 - MFR policy to set global expiration for all local Create activities
 - OGP rich media parser merged with TwitterCard
 - Configuration: `:instance, rewrite_policy` moved to `:mrf, policies`, `:instance, :mrf_transparency` moved to `:mrf, :transparency`, `:instance, :mrf_transparency_exclusions` moved to `:mrf, :transparency_exclusions`. Old config namespace is deprecated.
 - Configuration: `:media_proxy, whitelist` format changed to host with scheme (e.g. `http://example.com` instead of `example.com`). Domain format is deprecated.
-- **Breaking:** Configuration: `:instance, welcome_user_nickname` moved to `:welcome, :direct_message, :sender_nickname`, `:instance, :welcome_message` moved to `:welcome, :direct_message, :message`. Old config namespace is deprecated.
 
 <details>
   <summary>API Changes</summary>
@@ -22,33 +131,37 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - **Breaking:** Pleroma API: The routes to update avatar, banner and background have been removed.
 - **Breaking:** Image description length is limited now.
 - **Breaking:** Emoji API: changed methods and renamed routes.
+- **Breaking:** Notification Settings API for suppressing notifications has been simplified down to `block_from_strangers`.
+- **Breaking:** Notification Settings API option for hiding push notification contents has been renamed to `hide_notification_contents`.
 - MastodonAPI: Allow removal of avatar, banner and background.
 - Streaming: Repeats of a user's posts will no longer be pushed to the user's stream.
 - Mastodon API: Added `pleroma.metadata.fields_limits` to /api/v1/instance
 - Mastodon API: On deletion, returns the original post text.
 - Mastodon API: Add `pleroma.unread_count` to the Marker entity.
-- **Breaking:** Notification Settings API for suppressing notifications
-  has been simplified down to `block_from_strangers`.
-- **Breaking:** Notification Settings API option for hiding push notification
-  contents has been renamed to `hide_notification_contents`
 - Mastodon API: Added `pleroma.metadata.post_formats` to /api/v1/instance
 - Mastodon API (legacy): Allow query parameters for `/api/v1/domain_blocks`, e.g. `/api/v1/domain_blocks?domain=badposters.zone`
+- Mastodon API: Make notifications about statuses from muted users and threads read automatically
 - Pleroma API: `/api/pleroma/captcha` responses now include `seconds_valid` with an integer value.
+
 </details>
 
 <details>
   <summary>Admin API Changes</summary>
 
+- **Breaking** Changed relay `/api/pleroma/admin/relay` endpoints response format.
 - Status visibility stats: now can return stats per instance.
-
 - Mix task to refresh counter cache (`mix pleroma.refresh_counter_cache`)
+
 </details>
 
 ### Removed
+
 - **Breaking:** removed `with_move` parameter from notifications timeline.
 
 ### Added
 
+- Frontends: Add mix task to install frontends.
+- Frontends: Add configurable frontends for primary and admin fe.
 - Configuration: Added a blacklist for email servers.
 - Chats: Added `accepts_chat_messages` field to user, exposed in APIs and federation.
 - Chats: Added support for federated chats. For details, see the docs.
@@ -90,6 +203,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 </details>
 
 ### Fixed
+- Fix list pagination and other list issues.
 - Support pagination in conversations API
 - **Breaking**: SimplePolicy `:reject` and `:accept` allow deletions again
 - Fix follower/blocks import when nicknames starts with @
@@ -102,12 +216,82 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fix CSP policy generation to include remote Captcha services
 - Fix edge case where MediaProxy truncates media, usually caused when Caddy is serving content for the other Federated instance.
 - Emoji Packs could not be listed when instance was set to `public: false`
+- Fix whole_word always returning false on filter get requests
+- Migrations not working on OTP releases if the database was connected over ssl
+- Fix relay following
 
-## [Unreleased (patch)]
+## [2.0.7] - 2020-06-13
+
+### Security
+- Fix potential DoSes exploiting atom leaks in rich media parser and the `UserAllowListPolicy` MRF policy
 
 ### Fixed
+- CSP: not allowing images/media from every host when mediaproxy is disabled
+- CSP: not adding mediaproxy base url to image/media hosts
+- StaticFE missing the CSS file
+
+### Upgrade notes
+
+1. Restart Pleroma
+
+## [2.0.6] - 2020-06-09
+
+### Security
+- CSP: harden `image-src` and `media-src` when MediaProxy is used
+
+### Fixed
+- AP C2S: Fix pagination in inbox/outbox
+- Various compilation errors on OTP 23
+- Mastodon API streaming: Repeats from muted threads not being filtered
+
+### Changed
+- Various database performance improvements
+
+### Upgrade notes
+1. Run database migrations (inside Pleroma directory):
+  - OTP: `./bin/pleroma_ctl migrate`
+  - From Source: `mix ecto.migrate`
+2. Restart Pleroma
+
+## [2.0.5] - 2020-05-13
+
+### Security
+- Fix possible private status leaks in Mastodon Streaming API
+
+### Fixed
+- Crashes when trying to block a user if block federation is disabled
+- Not being able to start the instance without `erlang-eldap` installed
+- Users with bios over the limit getting rejected
+- Follower counters not being updated on incoming follow accepts
+
+### Upgrade notes
+
+1. Restart Pleroma
+
+## [2.0.4] - 2020-05-10
+
+### Security
+- AP C2S: Fix a potential DoS by creating nonsensical objects that break timelines
+
+### Fixed
+- Peertube user lookups not working
+- `InsertSkeletonsForDeletedUsers` migration failing on some instances
 - Healthcheck reporting the number of memory currently used, rather than allocated in total
-- `InsertSkeletonsForDeletedUsers` failing on some instances
+- LDAP not being usable in OTP releases
+- Default apache configuration having tls chain issues
+
+### Upgrade notes
+
+#### Apache only
+
+1. Remove the following line from your config:
+```
+    SSLCertificateFile      /etc/letsencrypt/live/${servername}/cert.pem
+```
+
+#### Everyone
+
+1. Restart Pleroma
 
 ## [2.0.3] - 2020-05-02
 
@@ -131,7 +315,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Follow request notifications
 <details>
   <summary>API Changes</summary>
-
 - Admin API: `GET /api/pleroma/admin/need_reboot`.
 </details>
 
@@ -166,6 +349,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Static-FE: Fix remote posts not being sanitized
 
 ### Fixed
+=======
+- Rate limiter crashes when there is no explicitly specified ip in the config
 - 500 errors when no `Accept` header is present if Static-FE is enabled
 - Instance panel not being updated immediately due to wrong `Cache-Control` headers
 - Statuses posted with BBCode/Markdown having unncessary newlines in Pleroma-FE

Dockerfile

@@ -4,7 +4,7 @@ COPY . .
 
 ENV MIX_ENV=prod
 
-RUN apk add git gcc g++ musl-dev make &&\
+RUN apk add git gcc g++ musl-dev make cmake &&\
 	echo "import Mix.Config" > config/prod.secret.exs &&\
 	mix local.hex --force &&\
 	mix local.rebar --force &&\
@@ -31,7 +31,7 @@ LABEL maintainer="ops@pleroma.social" \
 ARG HOME=/opt/pleroma
 ARG DATA=/var/lib/pleroma
 
-RUN echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\
+RUN echo "https://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\
 	apk update &&\
 	apk add exiftool imagemagick ncurses postgresql-client &&\
 	adduser --system --shell /bin/false --home ${HOME} pleroma &&\

README.md

@@ -18,15 +18,16 @@ If you are running Linux (glibc or musl) on x86/arm, the recommended way to inst
 ### From Source
 If your platform is not supported, or you just want to be able to edit the source code easily, you may install Pleroma from source.
 
-- [Debian-based](https://docs-develop.pleroma.social/backend/installation/debian_based_en/)
-- [Debian-based (jp)](https://docs-develop.pleroma.social/backend/installation/debian_based_jp/)
 - [Alpine Linux](https://docs-develop.pleroma.social/backend/installation/alpine_linux_en/)
 - [Arch Linux](https://docs-develop.pleroma.social/backend/installation/arch_linux_en/)
+- [CentOS 7](https://docs-develop.pleroma.social/backend/installation/centos7_en/)
+- [Debian-based](https://docs-develop.pleroma.social/backend/installation/debian_based_en/)
+- [Debian-based (jp)](https://docs-develop.pleroma.social/backend/installation/debian_based_jp/)
+- [FreeBSD](https://docs-develop.pleroma.social/backend/installation/freebsd_en/)
 - [Gentoo Linux](https://docs-develop.pleroma.social/backend/installation/gentoo_en/)
 - [NetBSD](https://docs-develop.pleroma.social/backend/installation/netbsd_en/)
 - [OpenBSD](https://docs-develop.pleroma.social/backend/installation/openbsd_en/)
 - [OpenBSD (fi)](https://docs-develop.pleroma.social/backend/installation/openbsd_fi/)
-- [CentOS 7](https://docs-develop.pleroma.social/backend/installation/centos7_en/)
 
 ### OS/Distro packages
 Currently Pleroma is not packaged by any OS/Distros, but if you want to package it for one, we can guide you through the process on our [community channels](#community-channels). If you want to change default options in your Pleroma package, please **discuss it with us first**.

SECURITY.md

@@ -0,0 +1,16 @@
+# Pleroma backend security policy
+
+## Supported versions
+
+Currently, Pleroma offers bugfixes and security patches only for the latest minor release.
+
+| Version | Support 
+|---------| --------
+| 2.1     | Bugfixes and security patches
+
+## Reporting a vulnerability
+
+Please use confidential issues (tick the "This issue is confidential and should only be visible to team members with at least Reporter access." box when submitting) at our [bugtracker](https://git.pleroma.social/pleroma/pleroma/-/issues/new) for reporting vulnerabilities.
+## Announcements
+
+New releases are announced at [pleroma.social](https://pleroma.social/announcements/). All security releases are tagged with ["Security"](https://pleroma.social/announcements/tags/security/). You can be notified of them by subscribing to an Atom feed at <https://pleroma.social/announcements/tags/security/feed.xml>. 

config/benchmark.exs

@@ -59,8 +59,6 @@
     "BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4",
   private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA"
 
-config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock
-
 config :pleroma, Pleroma.ScheduledActivity,
   daily_user_limit: 2,
   total_user_limit: 3,

config/config.exs

@@ -72,7 +72,8 @@
       pool: :upload
     ]
   ],
-  filename_display_max_length: 30
+  filename_display_max_length: 30,
+  default_description: nil
 
 config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
 
@@ -129,6 +130,7 @@
     dispatch: [
       {:_,
        [
+         {"/api/fedsocket/v1", Pleroma.Web.FedSockets.IncomingHandler, []},
          {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []},
          {"/websocket", Phoenix.Endpoint.CowboyWebSocket,
           {Phoenix.Transports.WebSocket,
@@ -147,6 +149,16 @@
     "SameSite=Lax"
   ]
 
+config :pleroma, :fed_sockets,
+  enabled: false,
+  connection_duration: :timer.hours(8),
+  rejection_duration: :timer.minutes(15),
+  fed_socket_fetches: [
+    default: 12_000,
+    interval: 3_000,
+    lazy: false
+  ]
+
 # Configures Elixir's Logger
 config :logger, :console,
   level: :debug,
@@ -215,7 +227,6 @@
   allow_relay: true,
   public: true,
   quarantined_instances: [],
-  managed_config: true,
   static_dir: "instance/static/",
   allowed_post_formats: [
     "text/plain",
@@ -411,6 +422,7 @@
     Pleroma.Web.RichMedia.Parsers.TwitterCard,
     Pleroma.Web.RichMedia.Parsers.OEmbed
   ],
+  failure_backoff: 60_000,
   ttl_setters: [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl]
 
 config :pleroma, :media_proxy,
@@ -422,6 +434,8 @@
   proxy_opts: [
     redirect_on_failure: false,
     max_body_length: 25 * 1_048_576,
+    # Note: max_read_duration defaults to Pleroma.ReverseProxy.max_read_duration_default/1
+    max_read_duration: 30_000,
     http: [
       follow_redirect: true,
       pool: :media
@@ -436,6 +450,14 @@
 
 config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, script_path: nil
 
+# Note: media preview proxy depends on media proxy to be enabled
+config :pleroma, :media_preview_proxy,
+  enabled: false,
+  thumbnail_max_width: 600,
+  thumbnail_max_height: 600,
+  image_quality: 85,
+  min_content_length: 100 * 1024
+
 config :pleroma, :chat, enabled: true
 
 config :phoenix, :format_encoders, json: Jason
@@ -452,9 +474,7 @@
 config :pleroma, Pleroma.Web.Metadata,
   providers: [
     Pleroma.Web.Metadata.Providers.OpenGraph,
-    Pleroma.Web.Metadata.Providers.TwitterCard,
+    Pleroma.Web.Metadata.Providers.TwitterCard
-    Pleroma.Web.Metadata.Providers.RelMe,
-    Pleroma.Web.Metadata.Providers.Feed
   ],
   unfurl_nsfw: false
 
@@ -515,7 +535,13 @@
     "user-search",
     "user_exists",
     "users",
-    "web"
+    "web",
+    "verify_credentials",
+    "update_credentials",
+    "relationships",
+    "search",
+    "confirmation_resend",
+    "mfa"
   ],
   email_blacklist: []
 
@@ -524,8 +550,10 @@
   log: false,
   queues: [
     activity_expiration: 10,
+    token_expiration: 5,
     federator_incoming: 50,
     federator_outgoing: 50,
+    ingestion_queue: 50,
     web_push: 50,
     mailer: 10,
     transmogrifier: 20,
@@ -537,9 +565,6 @@
   ],
   plugins: [Oban.Plugins.Pruner],
   crontab: [
-    {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker},
-    {"0 * * * *", Pleroma.Workers.Cron.StatsWorker},
-    {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker},
     {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker},
     {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}
   ]
@@ -650,9 +675,20 @@
   account_confirmation_resend: {8_640_000, 5},
   ap_routes: {60_000, 15}
 
-config :pleroma, Pleroma.ActivityExpiration, enabled: true
+config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true, min_lifetime: 600
 
-config :pleroma, Pleroma.Plugs.RemoteIp, enabled: true
+config :pleroma, Pleroma.Plugs.RemoteIp,
+  enabled: true,
+  headers: ["x-forwarded-for"],
+  proxies: [],
+  reserved: [
+    "127.0.0.0/8",
+    "::1/128",
+    "fc00::/7",
+    "10.0.0.0/8",
+    "172.16.0.0/12",
+    "192.168.0.0/16"
+  ]
 
 config :pleroma, :static_fe, enabled: false
 
@@ -664,7 +700,50 @@
 # With no frontend configuration, the bundled files from the `static` directory will
 # be used.
 #
-# config :pleroma, :frontends, primary: %{"name" => "pleroma", "ref" => "develop"}
+# config :pleroma, :frontends,
+# primary: %{"name" => "pleroma-fe", "ref" => "develop"},
+# admin: %{"name" => "admin-fe", "ref" => "stable"},
+# available: %{...}
+
+config :pleroma, :frontends,
+  available: %{
+    "kenoma" => %{
+      "name" => "kenoma",
+      "git" => "https://git.pleroma.social/lambadalambda/kenoma",
+      "build_url" =>
+        "https://git.pleroma.social/lambadalambda/kenoma/-/jobs/artifacts/${ref}/download?job=build",
+      "ref" => "master"
+    },
+    "pleroma-fe" => %{
+      "name" => "pleroma-fe",
+      "git" => "https://git.pleroma.social/pleroma/pleroma-fe",
+      "build_url" =>
+        "https://git.pleroma.social/pleroma/pleroma-fe/-/jobs/artifacts/${ref}/download?job=build",
+      "ref" => "develop"
+    },
+    "fedi-fe" => %{
+      "name" => "fedi-fe",
+      "git" => "https://git.pleroma.social/pleroma/fedi-fe",
+      "build_url" =>
+        "https://git.pleroma.social/pleroma/fedi-fe/-/jobs/artifacts/${ref}/download?job=build",
+      "ref" => "master"
+    },
+    "admin-fe" => %{
+      "name" => "admin-fe",
+      "git" => "https://git.pleroma.social/pleroma/admin-fe",
+      "build_url" =>
+        "https://git.pleroma.social/pleroma/admin-fe/-/jobs/artifacts/${ref}/download?job=build",
+      "ref" => "develop"
+    },
+    "soapbox-fe" => %{
+      "name" => "soapbox-fe",
+      "git" => "https://gitlab.com/soapbox-pub/soapbox-fe",
+      "build_url" =>
+        "https://gitlab.com/soapbox-pub/soapbox-fe/-/jobs/artifacts/${ref}/download?job=build-production",
+      "ref" => "v1.0.0",
+      "build_dir" => "static"
+    }
+  }
 
 config :pleroma, :web_cache_ttl,
   activity_pub: nil,
@@ -685,24 +764,28 @@
   max_connections: 250,
   max_idle_time: 30_000,
   retry: 0,
-  await_up_timeout: 5_000
+  connect_timeout: 5_000
 
 config :pleroma, :pools,
   federation: [
     size: 50,
-    max_waiting: 10
+    max_waiting: 10,
+    recv_timeout: 10_000
   ],
   media: [
     size: 50,
-    max_waiting: 10
+    max_waiting: 20,
+    recv_timeout: 15_000
   ],
   upload: [
     size: 25,
-    max_waiting: 5
+    max_waiting: 5,
+    recv_timeout: 15_000
   ],
   default: [
     size: 10,
-    max_waiting: 2
+    max_waiting: 2,
+    recv_timeout: 5_000
   ]
 
 config :pleroma, :hackney_pools,
@@ -719,10 +802,12 @@
     timeout: 300_000
   ]
 
+private_instance? = :if_instance_is_private
+
 config :pleroma, :restrict_unauthenticated,
-  timelines: %{local: false, federated: false},
+  timelines: %{local: private_instance?, federated: private_instance?},
-  profiles: %{local: false, remote: false},
+  profiles: %{local: private_instance?, remote: private_instance?},
-  activities: %{local: false, remote: false}
+  activities: %{local: private_instance?, remote: private_instance?}
 
 config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: false
 
@@ -735,8 +820,14 @@
 
 config :ex_aws, http_client: Pleroma.HTTP.ExAws
 
+config :web_push_encryption, http_client: Pleroma.HTTP.WebPush
+
 config :pleroma, :instances_favicons, enabled: false
 
+config :floki, :html_parser, Floki.HTMLParser.FastHtml
+
+config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/description.exs

@@ -12,6 +12,58 @@
   compress: false
 ]
 
+installed_frontend_options = [
+  %{
+    key: "name",
+    label: "Name",
+    type: :string,
+    description:
+      "Name of the installed frontend. Valid config must include both `Name` and `Reference` values."
+  },
+  %{
+    key: "ref",
+    label: "Reference",
+    type: :string,
+    description:
+      "Reference of the installed frontend to be used. Valid config must include both `Name` and `Reference` values."
+  }
+]
+
+frontend_options = [
+  %{
+    key: "name",
+    label: "Name",
+    type: :string,
+    description: "Name of the frontend."
+  },
+  %{
+    key: "ref",
+    label: "Reference",
+    type: :string,
+    description: "Reference of the frontend to be used."
+  },
+  %{
+    key: "git",
+    label: "Git Repository URL",
+    type: :string,
+    description: "URL of the git repository of the frontend"
+  },
+  %{
+    key: "build_url",
+    label: "Build URL",
+    type: :string,
+    description:
+      "Either an url to a zip file containing the frontend or a template to build it by inserting the `ref`. The string `${ref}` will be replaced by the configured `ref`.",
+    example: "https://some.url/builds/${ref}.zip"
+  },
+  %{
+    key: "build_dir",
+    label: "Build directory",
+    type: :string,
+    description: "The directory inside the zip file "
+  }
+]
+
 config :pleroma, :config_description, [
   %{
     group: :pleroma,
@@ -194,7 +246,7 @@
         type: [:string, {:list, :string}, {:list, :tuple}],
         description:
           "List of actions for the mogrify command. It's possible to add self-written settings as string. " <>
-            "For example `[\"auto-orient\", \"strip\", {\"resize\", \"3840x1080>\"}]` string will be parsed into list of the settings.",
+            "For example `auto-orient, strip, {\"resize\", \"3840x1080>\"}` value will be parsed into valid list of the settings.",
         suggestions: [
           "strip",
           "auto-orient",
@@ -221,6 +273,19 @@
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: :fed_sockets,
+    type: :group,
+    description: "Websocket based federation",
+    children: [
+      %{
+        key: :enabled,
+        type: :boolean,
+        description: "Enable FedSockets"
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: Pleroma.Emails.Mailer,
@@ -715,12 +780,6 @@
           "*.quarantined.com"
         ]
       },
-      %{
-        key: :managed_config,
-        type: :boolean,
-        description:
-          "Whenether the config for pleroma-fe is configured in this config or in static/config.json"
-      },
       %{
         key: :static_dir,
         type: :string,
@@ -951,7 +1010,7 @@
       },
       %{
         key: :instance_thumbnail,
-        type: :string,
+        type: {:string, :image},
         description:
           "The instance thumbnail can be any image that represents your instance and is used by some apps or services when they display information about your instance.",
         suggestions: ["/instance/thumbnail.jpeg"]
@@ -964,25 +1023,25 @@
     ]
   },
   %{
-    group: :welcome,
+    group: :pleroma,
+    key: :welcome,
     type: :group,
     description: "Welcome messages settings",
     children: [
       %{
-        group: :direct_message,
+        key: :direct_message,
-        type: :group,
+        type: :keyword,
         descpiption: "Direct message settings",
         children: [
           %{
             key: :enabled,
             type: :boolean,
-            description: "Enables sends direct message for new user after registration"
+            description: "Enables sending a direct message to newly registered users"
           },
           %{
             key: :message,
             type: :string,
-            description:
+            description: "A message that will be sent to newly registered users",
-              "A message that will be sent to a newly registered users as a direct message",
             suggestions: [
               "Hi, @username! Welcome on board!"
             ]
@@ -990,7 +1049,7 @@
           %{
             key: :sender_nickname,
             type: :string,
-            description: "The nickname of the local user that sends the welcome message",
+            description: "The nickname of the local user that sends a welcome message",
             suggestions: [
               "lain"
             ]
@@ -998,20 +1057,20 @@
         ]
       },
       %{
-        group: :chat_message,
+        key: :chat_message,
-        type: :group,
+        type: :keyword,
         descpiption: "Chat message settings",
         children: [
           %{
             key: :enabled,
             type: :boolean,
-            description: "Enables sends chat message for new user after registration"
+            description: "Enables sending a chat message to newly registered users"
           },
           %{
             key: :message,
             type: :string,
             description:
-              "A message that will be sent to a newly registered users as a chat message",
+              "A message that will be sent to newly registered users as a chat message",
             suggestions: [
               "Hello, welcome on board!"
             ]
@@ -1019,7 +1078,7 @@
           %{
             key: :sender_nickname,
             type: :string,
-            description: "The nickname of the local user that sends the welcome message",
+            description: "The nickname of the local user that sends a welcome chat message",
             suggestions: [
               "lain"
             ]
@@ -1027,20 +1086,20 @@
         ]
       },
       %{
-        group: :email,
+        key: :email,
-        type: :group,
+        type: :keyword,
         descpiption: "Email message settings",
         children: [
           %{
             key: :enabled,
             type: :boolean,
-            description: "Enables sends direct message for new user after registration"
+            description: "Enables sending an email to newly registered users"
           },
           %{
             key: :sender,
             type: [:string, :tuple],
             description:
-              "The email address or tuple with `{nickname, email}` that will use as sender to the welcome email.",
+              "Email address and/or nickname that will be used to send the welcome email.",
             suggestions: [
               {"Pleroma App", "welcome@pleroma.app"}
             ]
@@ -1049,21 +1108,21 @@
             key: :subject,
             type: :string,
             description:
-              "The subject of welcome email. Can be use EEX template with `user` and `instance_name` variables.",
+              "Subject of the welcome email. EEX template with user and instance_name variables can be used.",
             suggestions: ["Welcome to <%= instance_name%>"]
           },
           %{
             key: :html,
             type: :string,
             description:
-              "The html content of welcome email. Can be use EEX template with `user` and `instance_name` variables.",
+              "HTML content of the welcome email. EEX template with user and instance_name variables can be used.",
             suggestions: ["<h1>Hello <%= user.name%>. Welcome to <%= instance_name%></h1>"]
           },
           %{
             key: :text,
             type: :string,
             description:
-              "The text content of welcome email. Can be use EEX template with `user` and `instance_name` variables.",
+              "Text content of the welcome email. EEX template with user and instance_name variables can be used.",
             suggestions: ["Hello <%= user.name%>. \n Welcome to <%= instance_name%>\n"]
           }
         ]
@@ -1236,7 +1295,7 @@
           },
           %{
             key: :background,
-            type: :string,
+            type: {:string, :image},
             description:
               "URL of the background, unless viewing a user profile with a background that is set",
             suggestions: ["/images/city.jpg"]
@@ -1293,7 +1352,7 @@
           },
           %{
             key: :logo,
-            type: :string,
+            type: {:string, :image},
             description: "URL of the logo, defaults to Pleroma's logo",
             suggestions: ["/static/logo.png"]
           },
@@ -1325,7 +1384,7 @@
           %{
             key: :nsfwCensorImage,
             label: "NSFW Censor Image",
-            type: :string,
+            type: {:string, :image},
             description:
               "URL of the image to use for hiding NSFW media attachments in the timeline",
             suggestions: ["/static/img/nsfw.74818f9.png"]
@@ -1451,7 +1510,7 @@
       },
       %{
         key: :default_user_avatar,
-        type: :string,
+        type: {:string, :image},
         description: "URL of the default user avatar",
         suggestions: ["/images/avi.png"]
       }
@@ -1810,12 +1869,12 @@
           %{
             key: :enabled,
             type: :boolean,
-            description: "Enables invalidate media cache"
+            description: "Enables media cache object invalidation."
           },
           %{
             key: :provider,
             type: :module,
-            description: "Module which will be used to cache purge.",
+            description: "Module which will be used to purge objects from the cache.",
             suggestions: [
               Pleroma.Web.MediaProxy.Invalidation.Script,
               Pleroma.Web.MediaProxy.Invalidation.Http
@@ -1831,6 +1890,7 @@
         suggestions: [
           redirect_on_failure: false,
           max_body_length: 25 * 1_048_576,
+          max_read_duration: 30_000,
           http: [
             follow_redirect: true,
             pool: :media
@@ -1851,6 +1911,11 @@
               "Limits the content length to be approximately the " <>
                 "specified length. It is validated with the `content-length` header and also verified when proxying."
           },
+          %{
+            key: :max_read_duration,
+            type: :integer,
+            description: "Timeout (in milliseconds) of GET request to remote URI."
+          },
           %{
             key: :http,
             label: "HTTP",
@@ -1897,6 +1962,43 @@
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: :media_preview_proxy,
+    type: :group,
+    description: "Media preview proxy",
+    children: [
+      %{
+        key: :enabled,
+        type: :boolean,
+        description:
+          "Enables proxying of remote media preview to the instance's proxy. Requires enabled media proxy."
+      },
+      %{
+        key: :thumbnail_max_width,
+        type: :integer,
+        description:
+          "Max width of preview thumbnail for images (video preview always has original dimensions)."
+      },
+      %{
+        key: :thumbnail_max_height,
+        type: :integer,
+        description:
+          "Max height of preview thumbnail for images (video preview always has original dimensions)."
+      },
+      %{
+        key: :image_quality,
+        type: :integer,
+        description: "Quality of the output. Ranges from 0 (min quality) to 100 (max quality)."
+      },
+      %{
+        key: :min_content_length,
+        type: :integer,
+        description:
+          "Min content length to perform preview, in bytes. If greater than 0, media smaller in size will be served as is, without thumbnailing."
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: Pleroma.Web.MediaProxy.Invalidation.Http,
@@ -2241,9 +2343,6 @@
         type: {:list, :tuple},
         description: "Settings for cron background jobs",
         suggestions: [
-          {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker},
-          {"0 * * * *", Pleroma.Workers.Cron.StatsWorker},
-          {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker},
           {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker},
           {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}
         ]
@@ -2336,13 +2435,20 @@
         suggestions: [
           Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl
         ]
+      },
+      %{
+        key: :failure_backoff,
+        type: :integer,
+        description:
+          "Amount of milliseconds after request failure, during which the request will not be retried.",
+        suggestions: [60_000]
       }
     ]
   },
   %{
     group: :pleroma,
     key: Pleroma.Formatter,
-    label: "Auto Linker",
+    label: "Linkify",
     type: :group,
     description:
       "Configuration for Pleroma's link formatter which parses mentions, hashtags, and URLs.",
@@ -2419,14 +2525,20 @@
   },
   %{
     group: :pleroma,
-    key: Pleroma.ActivityExpiration,
+    key: Pleroma.Workers.PurgeExpiredActivity,
     type: :group,
-    description: "Expired activity settings",
+    description: "Expired activities settings",
     children: [
       %{
         key: :enabled,
         type: :boolean,
-        description: "Whether expired activities will be sent to the job queue to be deleted"
+        description: "Enables expired activities addition & deletion"
+      },
+      %{
+        key: :min_lifetime,
+        type: :integer,
+        description: "Minimum lifetime for ephemeral activity (in seconds)",
+        suggestions: [600]
       }
     ]
   },
@@ -2642,7 +2754,7 @@
     children: [
       %{
         key: :logo,
-        type: :string,
+        type: {:string, :image},
         description: "A path to a custom logo. Set it to `nil` to use the default Pleroma logo.",
         suggestions: ["some/path/logo.png"]
       },
@@ -3153,20 +3265,22 @@
       %{
         key: :headers,
         type: {:list, :string},
-        description:
+        description: """
-          "A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Default: `~w[forwarded x-forwarded-for x-client-ip x-real-ip]`."
+          A list of strings naming the HTTP headers to use when deriving the true client IP. Default: `["x-forwarded-for"]`.
+        """
       },
       %{
         key: :proxies,
         type: {:list, :string},
         description:
-          "A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Default: `[]`."
+          "A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128."
       },
       %{
         key: :reserved,
         type: {:list, :string},
-        description:
+        description: """
-          "Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network)."
+          A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`
+        """
       }
     ]
   },
@@ -3299,13 +3413,13 @@
     group: :pleroma,
     key: :connections_pool,
     type: :group,
-    description: "Advanced settings for `gun` connections pool",
+    description: "Advanced settings for `Gun` connections pool",
     children: [
       %{
         key: :connection_acquisition_wait,
         type: :integer,
         description:
-          "Timeout to acquire a connection from pool.The total max time is this value multiplied by the number of retries. Default: 250ms.",
+          "Timeout to acquire a connection from pool. The total max time is this value multiplied by the number of retries. Default: 250ms.",
         suggestions: [250]
       },
       %{
@@ -3322,7 +3436,7 @@
         suggestions: [250]
       },
       %{
-        key: :await_up_timeout,
+        key: :connect_timeout,
         type: :integer,
         description: "Timeout while `gun` will wait until connection is up. Default: 5000ms.",
         suggestions: [5000]
@@ -3340,7 +3454,7 @@
     group: :pleroma,
     key: :pools,
     type: :group,
-    description: "Advanced settings for `gun` workers pools",
+    description: "Advanced settings for `Gun` workers pools",
     children:
       Enum.map([:federation, :media, :upload, :default], fn pool_name ->
         %{
@@ -3360,6 +3474,12 @@
               description:
                 "Maximum number of requests waiting for other requests to finish. After this number is reached, the pool will start returning errrors when a new request is made",
               suggestions: [10]
+            },
+            %{
+              key: :recv_timeout,
+              type: :integer,
+              description: "Timeout for the pool while gun will wait for response",
+              suggestions: [10_000]
             }
           ]
         }
@@ -3369,7 +3489,7 @@
     group: :pleroma,
     key: :hackney_pools,
     type: :group,
-    description: "Advanced settings for `hackney` connections pools",
+    description: "Advanced settings for `Hackney` connections pools",
     children: [
       %{
         key: :federation,
@@ -3433,6 +3553,7 @@
   %{
     group: :pleroma,
     key: :restrict_unauthenticated,
+    label: "Restrict Unauthenticated",
     type: :group,
     description:
       "Disallow viewing timelines, user profiles and statuses for unauthenticated users.",
@@ -3552,17 +3673,38 @@
         key: :primary,
         type: :map,
         description: "Primary frontend, the one that is served for all pages by default",
-        children: [
+        children: installed_frontend_options
-          %{
+      },
-            key: "name",
+      %{
-            type: :string,
+        key: :admin,
-            description: "Name of the installed primary frontend"
+        type: :map,
-          },
+        description: "Admin frontend",
-          %{
+        children: installed_frontend_options
-            key: "ref",
+      },
-            type: :string,
+      %{
-            description: "reference of the installed primary frontend to be used"
+        key: :available,
-          }
+        type: :map,
+        description:
+          "A map containing available frontends and parameters for their installation.",
+        children: frontend_options
+      }
+    ]
+  },
+  %{
+    group: :pleroma,
+    key: Pleroma.Web.Preload,
+    type: :group,
+    description: "Preload-related settings",
+    children: [
+      %{
+        key: :providers,
+        type: {:list, :module},
+        description: "List of preload providers to enable",
+        suggestions: [
+          Pleroma.Web.Preload.Providers.Instance,
+          Pleroma.Web.Preload.Providers.User,
+          Pleroma.Web.Preload.Providers.Timelines,
+          Pleroma.Web.Preload.Providers.StatusNet
         ]
       }
     ]

config/test.exs

@@ -19,9 +19,17 @@
   level: :warn,
   format: "\n[$level] $message\n"
 
+config :pleroma, :fed_sockets,
+  enabled: false,
+  connection_duration: 5,
+  rejection_duration: 5
+
 config :pleroma, :auth, oauth_consumer_strategies: []
 
-config :pleroma, Pleroma.Upload, filters: [], link_name: false
+config :pleroma, Pleroma.Upload,
+  filters: [],
+  link_name: false,
+  default_description: :filename
 
 config :pleroma, Pleroma.Uploaders.Local, uploads: "test/uploads"
 
@@ -75,8 +83,6 @@
     "BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4",
   private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA"
 
-config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock
-
 config :pleroma, Oban,
   queues: false,
   crontab: false,
@@ -111,8 +117,6 @@
 
 config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true
 
-config :pleroma, :instances_favicons, enabled: true
-
 config :pleroma, Pleroma.Uploaders.S3,
   bucket: nil,
   streaming_enabled: true,

docs/API/admin_api.md

@@ -313,14 +313,37 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - On failure: `Not found`
   - On success: JSON array of user's latest statuses
 
+## `GET /api/pleroma/admin/relay`
+
+### List Relays
+
+Params: none
+Response:
+
+* On success: JSON array of relays
+
+```json
+[
+  {"actor": "https://example.com/relay", "followed_back": true},
+  {"actor": "https://example2.com/relay", "followed_back": false}
+]
+```
+
 ## `POST /api/pleroma/admin/relay`
 
 ### Follow a Relay
 
-- Params:
+Params:
-  - `relay_url`
+
-- Response:
+* `relay_url`
-  - On success: URL of the followed relay
+
+Response:
+
+* On success: relay json object
+
+```json
+{"actor": "https://example.com/relay", "followed_back": true}
+```
 
 ## `DELETE /api/pleroma/admin/relay`
 
@@ -328,16 +351,15 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 
 - Params:
   - `relay_url`
-- Response:
+  - *optional* `force`: forcefully unfollow a relay even when the relay is not available. (default is `false`)
-  - On success: URL of the unfollowed relay
 
-## `GET /api/pleroma/admin/relay`
+Response:
 
-### List Relays
+* On success: URL of the unfollowed relay
 
-- Params: none
+```json
-- Response:
+{"https://example.com/relay"}
-  - On success: JSON array of relays
+```
 
 ## `POST /api/pleroma/admin/users/invite_token`
 
@@ -1266,11 +1288,14 @@ Loads json generated from `config/descriptions.exs`.
 - Params:
 - *optional* `page`: **integer** page number
 - *optional* `page_size`: **integer** number of log entries per page (default is `50`)
+- *optional* `query`: **string** search term
 
 - Response:
 
 ``` json
 {
+  "page_size": integer,
+  "count": integer,
   "urls": [
     "http://example.com/media/a688346.jpg",
     "http://example.com/media/fb1f4d.jpg"
@@ -1290,12 +1315,7 @@ Loads json generated from `config/descriptions.exs`.
 - Response:
 
 ``` json
-{
+{ }
-  "urls": [
-    "http://example.com/media/a688346.jpg",
-    "http://example.com/media/fb1f4d.jpg"
-  ]
-}
 
 ```
 
@@ -1311,11 +1331,169 @@ Loads json generated from `config/descriptions.exs`.
 - Response:
 
 ``` json
-{
+{ }
-  "urls": [
-    "http://example.com/media/a688346.jpg",
-    "http://example.com/media/fb1f4d.jpg"
-  ]
-}
 
 ```
+
+## GET /api/pleroma/admin/users/:nickname/chats
+
+### List a user's chats
+
+- Params: None
+
+- Response:
+
+```json
+[
+   {
+      "sender": {
+        "id": "someflakeid",
+        "username": "somenick",
+        ...
+      },
+      "receiver": {
+        "id": "someflakeid",
+        "username": "somenick",
+        ...
+      },
+      "id" : "1",
+      "unread" : 2,
+      "last_message" : {...}, // The last message in that chat
+      "updated_at": "2020-04-21T15:11:46.000Z"
+   }
+]
+```
+
+## GET /api/pleroma/admin/chats/:chat_id
+
+### View a single chat
+
+- Params: None
+
+- Response:
+
+```json
+{
+  "sender": {
+    "id": "someflakeid",
+    "username": "somenick",
+    ...
+  },
+  "receiver": {
+    "id": "someflakeid",
+    "username": "somenick",
+    ...
+  },
+  "id" : "1",
+  "unread" : 2,
+  "last_message" : {...}, // The last message in that chat
+  "updated_at": "2020-04-21T15:11:46.000Z"
+}
+```
+
+## GET /api/pleroma/admin/chats/:chat_id/messages
+
+### List the messages in a chat
+
+- Params: `max_id`, `min_id`
+
+- Response:
+
+```json
+[
+  {
+    "account_id": "someflakeid",
+    "chat_id": "1",
+    "content": "Check this out :firefox:",
+    "created_at": "2020-04-21T15:11:46.000Z",
+    "emojis": [
+      {
+        "shortcode": "firefox",
+        "static_url": "https://dontbulling.me/emoji/Firefox.gif",
+        "url": "https://dontbulling.me/emoji/Firefox.gif",
+        "visible_in_picker": false
+      }
+    ],
+    "id": "13",
+    "unread": true
+  },
+  {
+    "account_id": "someflakeid",
+    "chat_id": "1",
+    "content": "Whats' up?",
+    "created_at": "2020-04-21T15:06:45.000Z",
+    "emojis": [],
+    "id": "12",
+    "unread": false
+  }
+]
+```
+
+## DELETE /api/pleroma/admin/chats/:chat_id/messages/:message_id
+
+### Delete a single message
+
+- Params: None
+
+- Response:
+
+```json
+{
+  "account_id": "someflakeid",
+  "chat_id": "1",
+  "content": "Check this out :firefox:",
+  "created_at": "2020-04-21T15:11:46.000Z",
+  "emojis": [
+    {
+      "shortcode": "firefox",
+      "static_url": "https://dontbulling.me/emoji/Firefox.gif",
+      "url": "https://dontbulling.me/emoji/Firefox.gif",
+      "visible_in_picker": false
+    }
+  ],
+  "id": "13",
+  "unread": false
+}
+```
+
+## `GET /api/pleroma/admin/instance_document/:document_name`
+
+### Get an instance document
+
+- Authentication: required
+
+- Response:
+
+Returns the content of the document
+
+```html
+<h1>Instance panel</h1>
+```
+
+## `PATCH /api/pleroma/admin/instance_document/:document_name`
+- Params:
+  - `file` (the file to be uploaded, using multipart form data.)
+
+### Update an instance document
+
+- Authentication: required
+
+- Response:
+
+``` json
+{
+  "url": "https://example.com/instance/panel.html"
+}
+```
+
+## `DELETE /api/pleroma/admin/instance_document/:document_name`
+
+### Delete an instance document
+
+- Response:
+
+``` json
+{
+  "url": "https://example.com/instance/panel.html"
+}
+```

docs/API/pleroma_api.md

@@ -44,6 +44,22 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Response: HTTP 200 on success, 500 on error
 * Note: Users that can't be followed are silently skipped.
 
+## `/api/pleroma/blocks_import`
+### Imports your blocks.
+* Method: `POST`
+* Authentication: required
+* Params:
+    * `list`: STRING or FILE containing a whitespace-separated list of accounts to block
+* Response: HTTP 200 on success, 500 on error
+
+## `/api/pleroma/mutes_import`
+### Imports your mutes.
+* Method: `POST`
+* Authentication: required
+* Params:
+    * `list`: STRING or FILE containing a whitespace-separated list of accounts to mute
+* Response: HTTP 200 on success, 500 on error
+
 ## `/api/pleroma/captcha`
 ### Get a new captcha
 * Method: `GET`
@@ -362,44 +378,43 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Params: None
 * Response: JSON, returns a list of Mastodon Conversation entities that were marked as read (200 - healthy, 503 unhealthy).
 
-## `GET /api/pleroma/emoji/packs/import`
+## `GET /api/pleroma/emoji/pack?name=:name`
-### Imports packs from filesystem
+
+### Get pack.json for the pack
+
 * Method `GET`
-* Authentication: required
+* Authentication: not required
-* Params: None
-* Response: JSON, returns a list of imported packs.
-
-## `GET /api/pleroma/emoji/packs/remote`
-### Make request to another instance for packs list
-* Method `GET`
-* Authentication: required
 * Params:
-  * `url`: url of the instance to get packs from
+  * `page`: page number for files (default 1)
-* Response: JSON with the pack list, hashmap with pack name and pack contents
+  * `page_size`: page size for files (default 30)
+* Response: JSON, pack json with `files`, `files_count` and `pack` keys with 200 status or 404 if the pack does not exist.
 
-## `POST /api/pleroma/emoji/packs/download`
+```json
-### Download pack from another instance
+{
-* Method `POST`
+  "files": {...},
-* Authentication: required
+  "files_count": 0, // emoji count in pack
-* Params:
+  "pack": {...}
-  * `url`: url of the instance to download from
+}
-  * `name`: pack to download from that instance
+```
-  * `as`: (*optional*) name how to save pack
+
-* Response: JSON, "ok" with 200 status if the pack was downloaded, or 500 if there were
+## `POST /api/pleroma/emoji/pack?name=:name`
-  errors downloading the pack
 
-## `POST /api/pleroma/emoji/packs/:name`
 ### Creates an empty pack
+
 * Method `POST`
-* Authentication: required
+* Authentication: required (admin)
-* Params: None
+* Params:
+  * `name`: pack name
 * Response: JSON, "ok" and 200 status or 409 if the pack with that name already exists
 
-## `PATCH /api/pleroma/emoji/packs/:name`
+## `PATCH /api/pleroma/emoji/pack?name=:name`
+
 ### Updates (replaces) pack metadata
+
 * Method `PATCH`
-* Authentication: required
+* Authentication: required (admin)
 * Params:
+  * `name`: pack name
   * `metadata`: metadata to replace the old one
     * `license`: Pack license
     * `homepage`: Pack home page url
@@ -410,39 +425,85 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Response: JSON, updated "metadata" section of the pack and 200 status or 400 if there was a
   problem with the new metadata (the error is specified in the "error" part of the response JSON)
 
-## `DELETE /api/pleroma/emoji/packs/:name`
+## `DELETE /api/pleroma/emoji/pack?name=:name`
+
 ### Delete a custom emoji pack
+
 * Method `DELETE`
-* Authentication: required
+* Authentication: required (admin)
-* Params: None
+* Params:
+  * `name`: pack name
 * Response: JSON, "ok" and 200 status or 500 if there was an error deleting the pack
 
-## `POST /api/pleroma/emoji/packs/:name/files`
+## `GET /api/pleroma/emoji/packs/import`
-### Add new file to the pack
+
-* Method `POST`
+### Imports packs from filesystem
-* Authentication: required
+
+* Method `GET`
+* Authentication: required (admin)
+* Params: None
+* Response: JSON, returns a list of imported packs.
+
+## `GET /api/pleroma/emoji/packs/remote`
+
+### Make request to another instance for packs list
+
+* Method `GET`
+* Authentication: required (admin)
 * Params:
+  * `url`: url of the instance to get packs from
+  * `page`: page number for packs (default 1)
+  * `page_size`: page size for packs (default 50)
+* Response: JSON with the pack list, hashmap with pack name and pack contents
+
+## `POST /api/pleroma/emoji/packs/download`
+
+### Download pack from another instance
+
+* Method `POST`
+* Authentication: required (admin)
+* Params:
+  * `url`: url of the instance to download from
+  * `name`: pack to download from that instance
+  * `as`: (*optional*) name how to save pack
+* Response: JSON, "ok" with 200 status if the pack was downloaded, or 500 if there were
+  errors downloading the pack
+
+## `POST /api/pleroma/emoji/packs/files?name=:name`
+
+### Add new file to the pack
+
+* Method `POST`
+* Authentication: required (admin)
+* Params:
+  * `name`: pack name
   * `file`: file needs to be uploaded with the multipart request or link to remote file.
   * `shortcode`: (*optional*) shortcode for new emoji, must be unique for all emoji. If not sended, shortcode will be taken from original filename.
   * `filename`: (*optional*) new emoji file name. If not specified will be taken from original filename.
 * Response: JSON, list of files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
 
-## `PATCH /api/pleroma/emoji/packs/:name/files`
+## `PATCH /api/pleroma/emoji/packs/files?name=:name`
+
 ### Update emoji file from pack
+
 * Method `PATCH`
-* Authentication: required
+* Authentication: required (admin)
 * Params:
+  * `name`: pack name
   * `shortcode`: emoji file shortcode
   * `new_shortcode`: new emoji file shortcode
   * `new_filename`: new filename for emoji file
   * `force`: (*optional*) with true value to overwrite existing emoji with new shortcode
 * Response: JSON, list with updated files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
 
-## `DELETE /api/pleroma/emoji/packs/:name/files`
+## `DELETE /api/pleroma/emoji/packs/files?name=:name`
+
 ### Delete emoji file from pack
+
 * Method `DELETE`
-* Authentication: required
+* Authentication: required (admin)
 * Params:
+  * `name`: pack name
   * `shortcode`: emoji file shortcode
 * Response: JSON, list with updated files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
 
@@ -467,30 +528,14 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 }
 ```
 
-## `GET /api/pleroma/emoji/packs/:name`
+## `GET /api/pleroma/emoji/packs/archive?name=:name`
 
-### Get pack.json for the pack
+### Requests a local pack archive from the instance
 
 * Method `GET`
 * Authentication: not required
 * Params:
-  * `page`: page number for files (default 1)
+  * `name`: pack name
-  * `page_size`: page size for files (default 30)
-* Response: JSON, pack json with `files`, `files_count` and `pack` keys with 200 status or 404 if the pack does not exist.
-
-```json
-{
-  "files": {...},
-  "files_count": 0, // emoji count in pack
-  "pack": {...}
-}
-```
-
-## `GET /api/pleroma/emoji/packs/:name/archive`
-### Requests a local pack archive from the instance
-* Method `GET`
-* Authentication: not required
-* Params: None
 * Response: the archive of the pack with a 200 status code, 403 if the pack is not set as shared,
   404 if the pack does not exist
 

docs/administration/CLI_tasks/config.md

@@ -11,14 +11,17 @@
     config :pleroma, configurable_from_database: true
     ```
 
-```sh tab="OTP"
+=== "OTP"
- ./bin/pleroma_ctl config migrate_to_db
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.config migrate_to_db
+    ./bin/pleroma_ctl config migrate_to_db
-```
+    ```
 
+=== "From Source"
+
+    ```sh
+    mix pleroma.config migrate_to_db
+    ```
 
 ## Transfer config from DB to `config/env.exported_from_db.secret.exs`
 
@@ -31,10 +34,12 @@ mix pleroma.config migrate_to_db
 
 To delete transfered settings from database optional flag `-d` can be used. `<env>` is `prod` by default.
 
-```sh tab="OTP"
+=== "OTP"
- ./bin/pleroma_ctl config migrate_from_db [--env=<env>] [-d]
+    ```sh
-```
+     ./bin/pleroma_ctl config migrate_from_db [--env=<env>] [-d]
+    ```
 
-```sh tab="From Source"
+=== "From Source"
-mix pleroma.config migrate_from_db [--env=<env>] [-d]
+    ```sh
-```
+    mix pleroma.config migrate_from_db [--env=<env>] [-d]
+    ```

docs/administration/CLI_tasks/database.md

@@ -9,13 +9,18 @@
 
 Replaces embedded objects with references to them in the `objects` table. Only needs to be ran once if the instance was created before Pleroma 1.0.5. The reason why this is not a migration is because it could significantly increase the database size after being ran, however after this `VACUUM FULL` will be able to reclaim about 20% (really depends on what is in the database, your mileage may vary) of the db size before the migration.
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl database remove_embedded_objects [option ...]
+
-```
+    ```sh
+    ./bin/pleroma_ctl database remove_embedded_objects [option ...]
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.database remove_embedded_objects [option ...]
+    ```
 
-```sh tab="From Source"
-mix pleroma.database remove_embedded_objects [option ...]
-```
 
 ### Options
 - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
@@ -27,13 +32,17 @@ This will prune remote posts older than 90 days (configurable with [`config :ple
 !!! danger
     The disk space will only be reclaimed after `VACUUM FULL`. You may run out of disk space during the execution of the task or vacuuming if you don't have about 1/3rds of the database size free.
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl database prune_objects [option ...]
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.database prune_objects [option ...]
+    ./bin/pleroma_ctl database prune_objects [option ...]
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.database prune_objects [option ...]
+    ```
 
 ### Options
 - `--vacuum` - run `VACUUM FULL` after the objects are pruned
@@ -42,33 +51,45 @@ mix pleroma.database prune_objects [option ...]
 
 Can be safely re-run
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl database bump_all_conversations
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.database bump_all_conversations
+    ./bin/pleroma_ctl database bump_all_conversations
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.database bump_all_conversations
+    ```
 
 ## Remove duplicated items from following and update followers count for all users
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl database update_users_following_followers_counts
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.database update_users_following_followers_counts
+    ./bin/pleroma_ctl database update_users_following_followers_counts
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.database update_users_following_followers_counts
+    ```
 
 ## Fix the pre-existing "likes" collections for all objects
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl database fix_likes_collections
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.database fix_likes_collections
+    ./bin/pleroma_ctl database fix_likes_collections
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.database fix_likes_collections
+    ```
 
 ## Vacuum the database
 
@@ -76,13 +97,17 @@ mix pleroma.database fix_likes_collections
 
 Running an `analyze` vacuum job can improve performance by updating statistics used by the query planner. **It is safe to cancel this.**
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl database vacuum analyze
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.database vacuum analyze
+    ./bin/pleroma_ctl database vacuum analyze
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.database vacuum analyze
+    ```
 
 ### Full
 
@@ -91,10 +116,28 @@ and more compact files with an optimized layout. This process will take a long t
 it builds the files side-by-side the existing database files. It can make your database faster and use less disk space,
 but should only be run if necessary. **It is safe to cancel this.**
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl database vacuum full
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.database vacuum full
+    ./bin/pleroma_ctl database vacuum full
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.database vacuum full
+    ```
+
+## Add expiration to all local statuses
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl database ensure_expiration
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.database ensure_expiration
+    ```

docs/administration/CLI_tasks/digest.md

@@ -4,22 +4,30 @@
 
 ## Send digest email since given date (user registration date by default) ignoring user activity status.
 
-```sh tab="OTP"
+=== "OTP"
- ./bin/pleroma_ctl digest test <nickname> [since_date]
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.digest test <nickname> [since_date]
+     ./bin/pleroma_ctl digest test <nickname> [since_date]
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.digest test <nickname> [since_date]
+    ```
 
 
 Example: 
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl digest test donaldtheduck 2019-05-20
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.digest test donaldtheduck 2019-05-20
+    ./bin/pleroma_ctl digest test donaldtheduck 2019-05-20
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.digest test donaldtheduck 2019-05-20
+    ```
 

docs/administration/CLI_tasks/email.md

@@ -1,24 +1,46 @@
-# Managing emails
+# EMail administration tasks
 
 {! backend/administration/CLI_tasks/general_cli_task_info.include !}
 
 ## Send test email (instance email by default)
 
-```sh tab="OTP"
+=== "OTP"
- ./bin/pleroma_ctl email test [--to <destination email address>]
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.email test [--to <destination email address>]
+     ./bin/pleroma_ctl email test [--to <destination email address>]
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.email test [--to <destination email address>]
+    ```
 
 
 Example: 
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl email test --to root@example.org
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.email test --to root@example.org
+    ./bin/pleroma_ctl email test --to root@example.org
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.email test --to root@example.org
+    ```
+
+## Send confirmation emails to all unconfirmed user accounts
+
+=== "OTP"
+
+    ```sh
+     ./bin/pleroma_ctl email send_confirmation_mails
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.email send_confirmation_mails
+    ```

docs/administration/CLI_tasks/emoji.md

@@ -4,13 +4,15 @@
 
 ## Lists emoji packs and metadata specified in the manifest
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl emoji ls-packs [option ...]
+    ```sh
-```
+    ./bin/pleroma_ctl emoji ls-packs [option ...]
+    ```
 
-```sh tab="From Source"
+=== "From Source"
-mix pleroma.emoji ls-packs [option ...]
+    ```sh
-```
+    mix pleroma.emoji ls-packs [option ...]
+    ```
 
 
 ### Options
@@ -18,26 +20,30 @@ mix pleroma.emoji ls-packs [option ...]
 
 ## Fetch, verify and install the specified packs from the manifest into `STATIC-DIR/emoji/PACK-NAME`
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl emoji get-packs [option ...] <pack ...>
+    ```sh
-```
+    ./bin/pleroma_ctl emoji get-packs [option ...] <pack ...>
+    ```
 
-```sh tab="From Source"
+=== "From Source"
-mix pleroma.emoji get-packs [option ...] <pack ...>
+    ```sh
-```
+    mix pleroma.emoji get-packs [option ...] <pack ...>
+    ```
 
 ### Options
 - `-m, --manifest PATH/URL` - same as [`ls-packs`](#ls-packs)
 
 ## Create a new manifest entry and a file list from the specified remote pack file
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl emoji gen-pack PACK-URL
+    ```sh
-```
+    ./bin/pleroma_ctl emoji gen-pack PACK-URL
+    ```
 
-```sh tab="From Source"
+=== "From Source"
-mix pleroma.emoji gen-pack PACK-URL
+    ```sh
-```
+    mix pleroma.emoji gen-pack PACK-URL
+    ```
 
 Currently, only .zip archives are recognized as remote pack files and packs are therefore assumed to be zip archives. This command is intended to run interactively and will first ask you some basic questions about the pack, then download the remote file and generate an SHA256 checksum for it, then generate an emoji file list for you.
 
@@ -47,8 +53,9 @@ Currently, only .zip archives are recognized as remote pack files and packs are
 
 ## Reload emoji packs
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl emoji reload
+    ```sh
-```
+    ./bin/pleroma_ctl emoji reload
+    ```
 
 This command only works with OTP releases.

docs/administration/CLI_tasks/frontend.md

@@ -0,0 +1,69 @@
+# Managing frontends
+
+`mix pleroma.frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]`
+
+Frontend can be installed either from local zip file, or automatically downloaded from the web.
+
+You can give all the options directly on the command like, but missing information will be filled out by looking at the data configured under `frontends.available` in the config files.
+
+Currently known `<frontend>` values are:
+- [admin-fe](https://git.pleroma.social/pleroma/admin-fe)
+- [kenoma](http://git.pleroma.social/lambadalambda/kenoma)
+- [pleroma-fe](http://git.pleroma.social/pleroma/pleroma-fe)
+- [fedi-fe](https://git.pleroma.social/pleroma/fedi-fe)
+- [soapbox-fe](https://gitlab.com/soapbox-pub/soapbox-fe)
+
+You can still install frontends that are not configured, see below.
+
+## Example installations for a known frontend
+
+For a frontend configured under the `available` key, it's enough to install it by name.
+
+```sh tab="OTP"
+./bin/pleroma_ctl frontend install pleroma
+```
+
+```sh tab="From Source"
+mix pleroma.frontend install pleroma
+```
+
+This will download the latest build for the the pre-configured `ref` and install it. It can then be configured as the one of the served frontends in the config file (see `primary` or `admin`).
+
+You can override any of the details. To install a pleroma build from a different url, you could do this:
+
+```sh tab="OPT"
+./bin/pleroma_ctl frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
+```
+
+```sh tab="From Source"
+mix pleroma.frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
+```
+
+Similarly, you can also install from a local zip file.
+
+```sh tab="OTP"
+./bin/pleroma_ctl frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
+```
+
+```sh tab="From Source"
+mix pleroma.frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
+```
+
+The resulting frontend will always be installed into a folder of this template: `${instance_static}/frontends/${name}/${ref}`
+
+Careful: This folder will be completely replaced on installation
+
+## Example installation for an unknown frontend
+
+The installation process is the same, but you will have to give all the needed options on the commond line. For example:
+
+```sh tab="OTP"
+./bin/pleroma_ctl frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
+```
+
+```sh tab="From Source"
+mix pleroma.frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
+```
+
+If you don't have a zip file but just want to install a frontend from a local path, you can simply copy the files over a folder of this template: `${instance_static}/frontends/${name}/${ref}`
+

docs/administration/CLI_tasks/instance.md

@@ -3,13 +3,17 @@
 {! backend/administration/CLI_tasks/general_cli_task_info.include !}
 
 ## Generate a new configuration file
-```sh tab="OTP"
+=== "OTP"
- ./bin/pleroma_ctl instance gen [option ...]
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.instance gen [option ...]
+     ./bin/pleroma_ctl instance gen [option ...]
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.instance gen [option ...]
+    ```
 
 
 If any of the options are left unspecified, you will be prompted interactively.

docs/administration/CLI_tasks/oauth_app.md

@@ -7,10 +7,14 @@
 Optional params:
   * `-s SCOPES` - scopes for app, e.g. `read,write,follow,push`.
 
-```sh tab="OTP"
+=== "OTP"
- ./bin/pleroma_ctl app create -n APP_NAME -r REDIRECT_URI
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.app create -n APP_NAME -r REDIRECT_URI
+     ./bin/pleroma_ctl app create -n APP_NAME -r REDIRECT_URI
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.app create -n APP_NAME -r REDIRECT_URI
+    ```

docs/administration/CLI_tasks/relay.md

@@ -4,30 +4,42 @@
 
 ## Follow a relay
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl relay follow <relay_url>
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.relay follow <relay_url>
+    ./bin/pleroma_ctl relay follow <relay_url>
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.relay follow <relay_url>
+    ```
 
 ## Unfollow a remote relay
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl relay unfollow <relay_url>
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.relay unfollow <relay_url>
+    ./bin/pleroma_ctl relay unfollow <relay_url>
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.relay unfollow <relay_url>
+    ```
 
 ## List relay subscriptions
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl relay list
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.relay list
+    ./bin/pleroma_ctl relay list
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.relay list
+    ```

docs/administration/CLI_tasks/release_environments.md

@@ -1,9 +0,0 @@
-# Generate release environment file
-
-```sh tab="OTP"
- ./bin/pleroma_ctl release_env gen
-```
-
-```sh tab="From Source"
-mix pleroma.release_env gen
-```

docs/administration/CLI_tasks/robots_txt.md

@@ -0,0 +1,21 @@
+# Managing robots.txt
+
+{! backend/administration/CLI_tasks/general_cli_task_info.include !}
+
+## Generate a new robots.txt file and add it to the static directory
+
+The `robots.txt` that ships by default is permissive. It allows well-behaved search engines to index all of your instance's URIs.
+
+If you want to generate a restrictive `robots.txt`, you can run the following mix task. The generated `robots.txt` will be written in your instance [static directory](../../../configuration/static_dir/).
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl robots_txt disallow_all
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.robots_txt disallow_all
+    ```

docs/administration/CLI_tasks/uploads.md

@@ -3,13 +3,17 @@
 {! backend/administration/CLI_tasks/general_cli_task_info.include !}
 
 ## Migrate uploads from local to remote storage
-```sh tab="OTP"
+=== "OTP"
- ./bin/pleroma_ctl uploads migrate_local <target_uploader> [option ...]
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.uploads migrate_local <target_uploader> [option ...]
+     ./bin/pleroma_ctl uploads migrate_local <target_uploader> [option ...]
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.uploads migrate_local <target_uploader> [option ...]
+    ```
 
 ### Options
 - `--delete` - delete local uploads after migrating them to the target uploader

docs/administration/CLI_tasks/user.md

@@ -4,13 +4,17 @@
 
 ## Create a user
 
-```sh tab="OTP"
+=== "OTP"
-./bin/pleroma_ctl user new <nickname> <email> [option ...]
-```
 
-```sh tab="From Source"
+    ```sh
-mix pleroma.user new <nickname> <email> [option ...]
+    ./bin/pleroma_ctl user new <nickname> <email> [option ...]
-```
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user new <nickname> <email> [option ...]
+    ```
 
 
 ### Options
@@ -22,23 +26,33 @@ mix pleroma.user new <nickname> <email> [option ...]
 - `-y`, `--assume-yes`/`--no-assume-yes` - whether to assume yes to all questions
 
 ## List local users
-```sh tab="OTP"
- ./bin/pleroma_ctl user list
-```
 
-```sh tab="From Source"
+=== "OTP"
-mix pleroma.user list
+
-```
+    ```sh
+     ./bin/pleroma_ctl user list
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user list
+    ```
 
 
 ## Generate an invite link
-```sh tab="OTP"
- ./bin/pleroma_ctl user invite [option ...]
-```
 
-```sh tab="From Source"
+=== "OTP"
-mix pleroma.user invite [option ...]
+
-```
+    ```sh
+     ./bin/pleroma_ctl user invite [option ...]
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user invite [option ...]
+    ```
 
 
 ### Options
@@ -46,144 +60,245 @@ mix pleroma.user invite [option ...]
 - `--max-use NUMBER` - maximum numbers of token uses
 
 ## List generated invites
-```sh tab="OTP"
- ./bin/pleroma_ctl user invites
-```
 
-```sh tab="From Source"
+=== "OTP"
-mix pleroma.user invites
+
-```
+    ```sh
+     ./bin/pleroma_ctl user invites
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user invites
+    ```
 
 
 ## Revoke invite
-```sh tab="OTP"
- ./bin/pleroma_ctl user revoke_invite <token>
-```
 
-```sh tab="From Source"
+=== "OTP"
-mix pleroma.user revoke_invite <token>
+
-```
+    ```sh
+     ./bin/pleroma_ctl user revoke_invite <token>
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user revoke_invite <token>
+    ```
 
 
 ## Delete a user
-```sh tab="OTP"
- ./bin/pleroma_ctl user rm <nickname>
-```
 
-```sh tab="From Source"
+=== "OTP"
-mix pleroma.user rm <nickname>
+
-```
+    ```sh
+     ./bin/pleroma_ctl user rm <nickname>
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user rm <nickname>
+    ```
 
 
 ## Delete user's posts and interactions
-```sh tab="OTP"
- ./bin/pleroma_ctl user delete_activities <nickname>
-```
 
-```sh tab="From Source"
+=== "OTP"
-mix pleroma.user delete_activities <nickname>
+
-```
+    ```sh
+     ./bin/pleroma_ctl user delete_activities <nickname>
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user delete_activities <nickname>
+    ```
 
 
 ## Sign user out from all applications (delete user's OAuth tokens and authorizations)
-```sh tab="OTP"
- ./bin/pleroma_ctl user sign_out <nickname>
-```
 
-```sh tab="From Source"
+=== "OTP"
-mix pleroma.user sign_out <nickname>
+
-```
+    ```sh
+     ./bin/pleroma_ctl user sign_out <nickname>
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user sign_out <nickname>
+    ```
 
 
 ## Deactivate or activate a user
-```sh tab="OTP"
- ./bin/pleroma_ctl user toggle_activated <nickname>
-```
 
-```sh tab="From Source"
+=== "OTP"
-mix pleroma.user toggle_activated <nickname>
+
-```
+    ```sh
+     ./bin/pleroma_ctl user toggle_activated <nickname>
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user toggle_activated <nickname>
+    ```
 
 
 ## Deactivate a user and unsubscribes local users from the user
-```sh tab="OTP"
- ./bin/pleroma_ctl user deactivate NICKNAME
-```
 
-```sh tab="From Source"
+=== "OTP"
-mix pleroma.user deactivate NICKNAME
+
-```
+    ```sh
+     ./bin/pleroma_ctl user deactivate NICKNAME
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user deactivate NICKNAME
+    ```
 
 
 ## Deactivate all accounts from an instance and unsubscribe local users on it
-```sh tab="OTP"
- ./bin/pleroma_ctl user deactivate_all_from_instance <instance>
-```
 
-```sh tab="From Source"
+=== "OTP"
-mix pleroma.user deactivate_all_from_instance <instance>
+
-```
+    ```sh
+     ./bin/pleroma_ctl user deactivate_all_from_instance <instance>
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user deactivate_all_from_instance <instance>
+    ```
 
 
 ## Create a password reset link for user
-```sh tab="OTP"
- ./bin/pleroma_ctl user reset_password <nickname>
-```
 
-```sh tab="From Source"
+=== "OTP"
-mix pleroma.user reset_password <nickname>
+
-```
+    ```sh
+     ./bin/pleroma_ctl user reset_password <nickname>
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user reset_password <nickname>
+    ```
 
 
 ## Disable Multi Factor Authentication (MFA/2FA) for a user
-```sh tab="OTP"
- ./bin/pleroma_ctl user reset_mfa <nickname>
-```
 
-```sh tab="From Source"
+=== "OTP"
-mix pleroma.user reset_mfa <nickname>
+
-```
+    ```sh
+     ./bin/pleroma_ctl user reset_mfa <nickname>
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user reset_mfa <nickname>
+    ```
 
 
 ## Set the value of the given user's settings
-```sh tab="OTP"
- ./bin/pleroma_ctl user set <nickname> [option ...]
-```
 
-```sh tab="From Source"
+=== "OTP"
-mix pleroma.user set <nickname> [option ...]
+
-```
+    ```sh
+     ./bin/pleroma_ctl user set <nickname> [option ...]
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user set <nickname> [option ...]
+    ```
 
 ### Options
+- `--admin`/`--no-admin` - whether the user should be an admin
+- `--confirmed`/`--no-confirmed` - whether the user account is confirmed
 - `--locked`/`--no-locked` - whether the user should be locked
 - `--moderator`/`--no-moderator` - whether the user should be a moderator
-- `--admin`/`--no-admin` - whether the user should be an admin
 
 ## Add tags to a user
-```sh tab="OTP"
- ./bin/pleroma_ctl user tag <nickname> <tags>
-```
 
-```sh tab="From Source"
+=== "OTP"
-mix pleroma.user tag <nickname> <tags>
+
-```
+    ```sh
+     ./bin/pleroma_ctl user tag <nickname> <tags>
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user tag <nickname> <tags>
+    ```
 
 
 ## Delete tags from a user
-```sh tab="OTP"
- ./bin/pleroma_ctl user untag <nickname> <tags>
-```
 
-```sh tab="From Source"
+=== "OTP"
-mix pleroma.user untag <nickname> <tags>
+
-```
+    ```sh
+     ./bin/pleroma_ctl user untag <nickname> <tags>
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user untag <nickname> <tags>
+    ```
 
 
 ## Toggle confirmation status of the user
-```sh tab="OTP"
- ./bin/pleroma_ctl user toggle_confirmed <nickname>
-```
 
-```sh tab="From Source"
+=== "OTP"
-mix pleroma.user toggle_confirmed <nickname>
+
-```
+    ```sh
+     ./bin/pleroma_ctl user toggle_confirmed <nickname>
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user toggle_confirmed <nickname>
+    ```
+
+## Set confirmation status for all regular active users
+*Admins and moderators are excluded*
+
+=== "OTP"
+
+    ```sh
+     ./bin/pleroma_ctl user confirm_all
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user confirm_all
+    ```
+
+## Revoke confirmation status for all regular active users
+*Admins and moderators are excluded*
+
+=== "OTP"
+
+    ```sh
+     ./bin/pleroma_ctl user unconfirm_all
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user unconfirm_all
+    ```

docs/administration/updating.md

@@ -18,9 +18,10 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
 
 1. Go to the working directory of Pleroma (default is `/opt/pleroma`)
 2. Run `git pull`. This pulls the latest changes from upstream.
-3. Run `mix deps.get`. This pulls in any new dependencies.
+3. Run `mix deps.get` [^1]. This pulls in any new dependencies.
 4. Stop the Pleroma service.
-5. Run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
+5. Run `mix ecto.migrate` [^1] [^2]. This task performs database migrations, if there were any.
 6. Start the Pleroma service.
 
-[^1]: Prefix with `MIX_ENV=prod` to run it using the production config file.
+[^1]: Depending on which install guide you followed (for example on Debian/Ubuntu), you want to run `mix` tasks as `pleroma` user by adding `sudo -Hu pleroma` before the command.
+[^2]: Prefix with `MIX_ENV=prod` to run it using the production config file.

docs/clients.md

@@ -6,11 +6,11 @@ Feel free to contact us to be added to this list!
 ### Roma for Desktop
 - Homepage: <https://www.pleroma.com/#desktopApp>
 - Source Code: <https://github.com/roma-apps/roma-desktop>
-- Platforms: Windows, Mac, (Linux?)
+- Platforms: Windows, Mac, Linux
 - Features: Streaming Ready
 
 ### Social
-- Source Code: <https://gitlab.gnome.org/BrainBlasted/Social>
+- Source Code: <https://gitlab.gnome.org/World/Social>
 - Contact: [@brainblasted@social.libre.fi](https://social.libre.fi/users/brainblasted)
 - Platforms: Linux (GNOME)
 - Note(2019-01-28): Not at a pre-alpha stage yet
@@ -35,7 +35,7 @@ Feel free to contact us to be added to this list!
 - Source Code: <https://framagit.org/tom79/fedilab/>
 - Contact: [@fedilab@framapiaf.org](https://framapiaf.org/users/fedilab)
 - Platforms: Android
-- Features: Streaming Ready, Moderation, Text Formatting 
+- Features: Streaming Ready, Moderation, Text Formatting
 
 ### Kyclos
 - Source Code: <https://git.pleroma.social/pleroma/harbour-kyclos>
@@ -48,16 +48,9 @@ Feel free to contact us to be added to this list!
 - Platforms: Android
 - Features: No Streaming, Emoji Reactions, Text Formatting, FE Stickers
 
-### Nekonium
-- Homepage: [F-Droid Repository](https://repo.gdgd.jp.net/), [Google Play](https://play.google.com/store/apps/details?id=com.apps.nekonium), [Amazon](https://www.amazon.co.jp/dp/B076FXPRBC/)
-- Source: <https://gogs.gdgd.jp.net/lin/nekonium>
-- Contact: [@lin@pleroma.gdgd.jp.net](https://pleroma.gdgd.jp.net/users/lin)
-- Platforms: Android
-- Features: Streaming Ready
-
 ### Fedi
 - Homepage: <https://www.fediapp.com/>
-- Source Code: Proprietary, but free
+- Source Code: Proprietary, but gratis
 - Platforms: iOS, Android
 - Features: Pleroma-specific features like Reactions
 
@@ -70,9 +63,9 @@ Feel free to contact us to be added to this list!
 
 ### Twidere
 - Homepage: <https://twidere.mariotaku.org/>
-- Source Code: <https://github.com/TwidereProject/Twidere-Android/>, <https://github.com/TwidereProject/Twidere-iOS/>
+- Source Code: <https://github.com/TwidereProject/Twidere-Android/>
 - Contact: <me@mariotaku.org>
-- Platform: Android, iOS
+- Platform: Android
 - Features: No Streaming
 
 ### Indigenous
@@ -89,11 +82,6 @@ Feel free to contact us to be added to this list!
 - Contact: [@gcupc@glitch.social](https://glitch.social/users/gcupc)
 - Features: No Streaming
 
-### Feather
-- Source Code: <https://github.com/kaniini/feather>
-- Contact: [@kaniini@pleroma.site](https://pleroma.site/kaniini)
-- Features: No Streaming
-
 ### Halcyon
 - Source Code: <https://notabug.org/halcyon-suite/halcyon>
 - Contact: [@halcyon@social.csswg.org](https://social.csswg.org/users/halcyon)
@@ -107,6 +95,15 @@ Feel free to contact us to be added to this list!
 - Features: No Streaming
 
 ### Sengi
+- Homepage: <https://nicolasconstant.github.io/sengi/>
 - Source Code: <https://github.com/NicolasConstant/sengi>
 - Contact: [@sengi_app@mastodon.social](https://mastodon.social/users/sengi_app)
-- Note(2019-01-28): The development is currently in a early stage.
+
+### DashFE
+- Source Code: <https://notabug.org/daisuke/DashboardFE>
+- Contact: [@dashfe@stereophonic.space](https://stereophonic.space/users/dashfe)
+
+### BloatFE
+- Source Code: <https://git.freesoftwareextremist.com/bloat/>
+- Contact: [@r@freesoftwareextremist.com](https://freesoftwareextremist.com/users/r)
+- Features: Does not requires JavaScript

docs/configuration/cheatsheet.md

@@ -18,7 +18,7 @@ To add configuration to your config file, you can copy it from the base config.
 * `notify_email`: Email used for notifications.
 * `description`: The instance’s description, can be seen in nodeinfo and ``/api/v1/instance``.
 * `limit`: Posts character limit (CW/Subject included in the counter).
-* `discription_limit`: The character limit for image descriptions.
+* `description_limit`: The character limit for image descriptions.
 * `chat_limit`: Character limit of the instance chat messages.
 * `remote_limit`: Hard character limit beyond which remote posts will be dropped.
 * `upload_limit`: File size limit of uploads (except for avatar, background, banner).
@@ -38,9 +38,8 @@ To add configuration to your config file, you can copy it from the base config.
 * `federation_incoming_replies_max_depth`: Max. depth of reply-to activities fetching on incoming federation, to prevent out-of-memory situations while fetching very long threads. If set to `nil`, threads of any depth will be fetched. Lower this value if you experience out-of-memory crashes.
 * `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it.
 * `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance.
-* `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. See also: `restrict_unauthenticated`.
+* `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. Note that there is a dependent setting restricting or allowing unauthenticated access to specific resources, see `restrict_unauthenticated` for more details.
-* `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send.
+* `quarantined_instances`: List of ActivityPub instances where private (DMs, followers-only) activities will not be send.
-* `managed_config`: Whenether the config for pleroma-fe is configured in [:frontend_configurations](#frontend_configurations) or in ``static/config.json``.
 * `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML).
 * `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with
     older software for theses nicknames.
@@ -87,7 +86,7 @@ To add configuration to your config file, you can copy it from the base config.
       direct_message: [
         enabled: true,
         sender_nickname: "lain",
-        message: "Hi, @username! Welcome on board!"
+        message: "Hi! Welcome on board!"
         ],
       email: [
         enabled: true,
@@ -114,6 +113,8 @@ To add configuration to your config file, you can copy it from the base config.
     * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (See [`:mrf_mention`](#mrf_mention)).
     * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (See [`:mrf_vocabulary`](#mrf_vocabulary)).
     * `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)).
+    * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.ActivityExpiration` to be enabled for processing the scheduled delections.
+    * `Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy`: Makes all bot posts to disappear from public timelines.
 * `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo).
 * `transparency_exclusions`: Exclude specific instance names from MRF transparency.  The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.
 
@@ -220,8 +221,20 @@ config :pleroma, :mrf_user_allowlist, %{
 
 ## Pleroma.ActivityExpiration
 
+Enables the worker which processes posts scheduled for deletion. Pinned posts are exempt from expiration.
+
 * `enabled`: whether expired activities will be sent to the job queue to be deleted
 
+## FedSockets
+FedSockets is an experimental feature allowing for Pleroma backends to federate using a persistant websocket connection as opposed to making each federation a seperate http connection. This feature is currently off by default. It is configurable throught he following options.
+
+### :fedsockets
+* `enabled`: Enables FedSockets for this instance. `false` by default.
+* `connection_duration`: Time an idle websocket is kept open.
+* `rejection_duration`: Failures to connect via FedSockets will not be retried for this period of time.
+* `fed_socket_fetches` and `fed_socket_rejections`: Settings passed to `cachex` for the fetch registry, and rejection stacks. See `Pleroma.Web.FedSockets` for more details.
+
+
 ## Frontends
 
 ### :frontend_configurations
@@ -311,6 +324,14 @@ This section describe PWA manifest instance-specific values. Currently this opti
   * `enabled`: Enables purge cache
   * `provider`: Which one of  the [purge cache strategy](#purge-cache-strategy) to use.
 
+## :media_preview_proxy
+
+* `enabled`: Enables proxying of remote media preview to the instance’s proxy. Requires enabled media proxy (`media_proxy/enabled`).
+* `thumbnail_max_width`: Max width of preview thumbnail for images (video preview always has original dimensions).
+* `thumbnail_max_height`: Max height of preview thumbnail for images (video preview always has original dimensions).
+* `image_quality`: Quality of the output. Ranges from 0 (min quality) to 100 (max quality).
+* `min_content_length`: Min content length to perform preview, in bytes. If greater than 0, media smaller in size will be served as is, without thumbnailing.
+
 ### Purge cache strategy
 
 #### Pleroma.Web.MediaProxy.Invalidation.Script
@@ -349,8 +370,6 @@ config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http,
 * `providers`: a list of metadata providers to enable. Providers available:
     * `Pleroma.Web.Metadata.Providers.OpenGraph`
     * `Pleroma.Web.Metadata.Providers.TwitterCard`
-    * `Pleroma.Web.Metadata.Providers.RelMe` - add links from user bio with rel=me into the `<header>` as `<link rel=me>`.
-    * `Pleroma.Web.Metadata.Providers.Feed` - add a link to a user's Atom feed into the `<header>` as `<link rel=alternate>`.
 * `unfurl_nsfw`: If set to `true` nsfw attachments will be shown in previews.
 
 ### :rich_media (consumer)
@@ -358,6 +377,7 @@ config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http,
 * `ignore_hosts`: list of hosts which will be ignored by the metadata parser. For example `["accounts.google.com", "xss.website"]`, defaults to `[]`.
 * `ignore_tld`: list TLDs (top-level domains) which will ignore for parse metadata. default is ["local", "localdomain", "lan"].
 * `parsers`: list of Rich Media parsers.
+* `failure_backoff`: Amount of milliseconds after request failure, during which the request will not be retried.
 
 ## HTTP server
 
@@ -406,9 +426,9 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start
 Available options:
 
 * `enabled` - Enable/disable the plug. Defaults to `false`.
-* `headers` - A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Defaults to `["x-forwarded-for"]`.
+* `headers` - A list of strings naming the HTTP headers to use when deriving the true client IP address. Defaults to `["x-forwarded-for"]`.
-* `proxies` - A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Defaults to `[]`.
+* `proxies` - A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128.
-* `reserved` - Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network).
+* `reserved` - A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`.
 
 
 ### :rate_limit
@@ -494,7 +514,7 @@ Settings for HTTP connection pool.
 * `:connection_acquisition_wait` - Timeout to acquire a connection from pool.The total max time is this value multiplied by the number of retries.
 * `connection_acquisition_retries` - Number of attempts to acquire the connection from the pool if it is overloaded. Each attempt is timed `:connection_acquisition_wait` apart.
 * `:max_connections` - Maximum number of connections in the pool.
-* `:await_up_timeout` - Timeout to connect to the host.
+* `:connect_timeout` - Timeout to connect to the host.
 * `:reclaim_multiplier` - Multiplied by `:max_connections` this will be the maximum number of idle connections that will be reclaimed in case the pool is overloaded.
 
 ### :pools
@@ -513,7 +533,7 @@ There are four pools used:
 For each pool, the options are:
 
 * `:size` - limit to how much requests can be concurrently executed.
-* `:timeout` - timeout while `gun` will wait for response
+* `:recv_timeout` - timeout while `gun` will wait for response
 * `:max_waiting` - limit to how much requests can be waiting for others to finish, after this is reached, subsequent requests will be dropped.
 
 ## Captcha
@@ -549,6 +569,7 @@ the source code is here: [kocaptcha](https://github.com/koto-bank/kocaptcha). Th
 * `proxy_remote`: If you're using a remote uploader, Pleroma will proxy media requests instead of redirecting to it.
 * `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation.
 * `filename_display_max_length`: Set max length of a filename to display. 0 = no limit. Default: 30.
+* `default_description`: Sets which default description an image has if none is set explicitly. Options: nil (default) - Don't set a default, :filename - use the filename of the file, a string (e.g. "attachment") - Use this string
 
 !!! warning
     `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`.
@@ -687,9 +708,8 @@ Pleroma has the following queues:
 
 Pleroma has these periodic job workers:
 
-`Pleroma.Workers.Cron.ClearOauthTokenWorker` - a job worker to cleanup expired oauth tokens.
+* `Pleroma.Workers.Cron.DigestEmailsWorker` - digest emails for users with new mentions and follows
-
+* `Pleroma.Workers.Cron.NewUsersDigestWorker` - digest emails for admins with new registrations
-Example:
 
 ```elixir
 config :pleroma, Oban,
@@ -701,7 +721,8 @@ config :pleroma, Oban,
     federator_outgoing: 50
   ],
   crontab: [
-    {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}
+    {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker},
+    {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}
   ]
 ```
 
@@ -858,9 +879,6 @@ Warning: it's discouraged to use this feature because of the associated security
 
 ### :auth
 
-* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator.
-* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication.
-
 Authentication / authorization settings.
 
 * `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`.
@@ -890,6 +908,9 @@ Pleroma account will be created with the same name as the LDAP user name.
 * `base`: LDAP base, e.g. "dc=example,dc=com"
 * `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base"
 
+Note, if your LDAP server is an Active Directory server the correct value is commonly `uid: "cn"`, but if you use an
+OpenLDAP server the value may be `uid: "uid"`.
+
 ### OAuth consumer mode
 
 OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.).
@@ -968,7 +989,7 @@ Configure OAuth 2 provider capabilities:
 
 * `token_expires_in` - The lifetime in seconds of the access token.
 * `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token.
-* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. Interval settings sets in configuration periodic jobs [`Oban.Cron`](#obancron)
+* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`.
 
 ## Link parsing
 
@@ -1048,6 +1069,8 @@ Restrict access for unauthenticated users to timelines (public and federated), u
   * `local`
   * `remote`
 
+Note: when `:instance, :public` is set to `false`, all `:restrict_unauthenticated` items be effectively set to `true` by default. If you'd like to allow unauthenticated access to specific API endpoints on a private instance, please explicitly set `:restrict_unauthenticated` to non-default value in `config/prod.secret.exs`.
+
 Note: setting `restrict_unauthenticated/timelines/local` to `true` has no practical sense if `restrict_unauthenticated/timelines/federated` is set to `false` (since local public activities will still be delivered to unauthenticated users as part of federated timeline).
 
 ## Pleroma.Web.ApiSpec.CastAndValidate
@@ -1064,11 +1087,11 @@ Control favicons for instances.
 
 Frontends in Pleroma are swappable - you can specify which one to use here.
 
-For now, you can set a frontend with the key `primary` and the options of `name` and `ref`. This will then make Pleroma serve the frontend from a folder constructed by concatenating the instance static path, `frontends` and the name and ref.
+You can set a frontends for the key `primary` and `admin` and the options of `name` and `ref`. This will then make Pleroma serve the frontend from a folder constructed by concatenating the instance static path, `frontends` and the name and ref.
 
-The key `primary` refers to the frontend that will be served by default for general requests. In the future, other frontends like the admin frontend will also be configurable here.
+The key `primary` refers to the frontend that will be served by default for general requests. The key `admin` refers to the frontend that will be served at the `/pleroma/admin` path.
 
-If you don't set anything here, the bundled frontend will be used.
+If you don't set anything here, the bundled frontends will be used.
 
 Example:
 
@@ -1077,7 +1100,18 @@ config :pleroma, :frontends,
   primary: %{
     "name" => "pleroma",
     "ref" => "stable"
+  },
+  admin: %{
+    "name" => "admin",
+    "ref" => "develop"
   }
 ```
 
 This would serve the frontend from the the folder at `$instance_static/frontends/pleroma/stable`. You have to copy the frontend into this folder yourself. You can choose the name and ref any way you like, but they will be used by mix tasks to automate installation in the future, the name referring to the project and the ref referring to a commit.
+
+## Ephemeral activities (Pleroma.Workers.PurgeExpiredActivity)
+
+Settings to enable and configure expiration for ephemeral activities
+
+* `:enabled` - enables ephemeral activities creation
+* `:min_lifetime` - minimum lifetime for ephemeral activities (in seconds). Default: 10 minutes.

docs/configuration/static_dir.md

@@ -1,45 +1,61 @@
 # Static Directory
 
-Static frontend files are shipped in `priv/static/` and tracked by version control in this repository. If you want to overwrite or update these without the possibility of merge conflicts, you can write your custom versions to `instance/static/`.
+Static frontend files are shipped with pleroma. If you want to overwrite or update these without problems during upgrades, you can write your custom versions to the static directory.
 
-```
+You can find the location of the static directory in the [configuration](../cheatsheet/#instance).
-config :pleroma, :instance,
-  static_dir: "instance/static/",
-```
 
-For example, edit `instance/static/instance/panel.html` .
+=== "OTP"
+
+    ```elixir
+    config :pleroma, :instance,
+      static_dir: "/var/lib/pleroma/static/"
+    ```
+
+=== "From Source"
+
+    ```elixir
+    config :pleroma, :instance,
+    static_dir: "instance/static/"
+    ```
 
 Alternatively, you can overwrite this value in your configuration to use a different static instance directory.
 
-This document is written assuming `instance/static/`.
+This document is written using `$static_dir` as the value of the `config :pleroma, :instance, static_dir` setting.
 
-Or, if you want to manage your custom file in git repository, basically remove the `instance/` entry from `.gitignore`.
+If you use a From Source installation and want to manage your custom files in the git repository, you can remove the `instance/` entry from `.gitignore`.
 
 ## robots.txt
 
-By default, the `robots.txt` that ships in `priv/static/` is permissive. It allows well-behaved search engines to index all of your instance's URIs.
+There's a mix tasks to [generate a new robot.txt](../../administration/CLI_tasks/robots_txt/).
 
-If you want to generate a restrictive `robots.txt`, you can run the following mix task. The generated `robots.txt` will be written in your instance static directory.
+For more complex things, you can write your own robots.txt to `$static_dir/robots.txt`.
+
+E.g. if you want to block all crawlers except for [fediverse.network](https://fediverse.network/about) you can use
 
 ```
-mix pleroma.robots_txt disallow_all
+User-Agent: *
+Disallow: /
+
+User-Agent: crawler-us-il-1.fediverse.network
+Allow: /
+
+User-Agent: makhnovtchina.random.sh
+Allow: /
 ```
 
 ## Thumbnail
 
-Put on `instance/static/instance/thumbnail.jpeg` with your selfie or other neat picture. It will appear in [Pleroma Instances](http://distsn.org/pleroma-instances.html).
+Add `$static_dir/instance/thumbnail.jpeg` with your selfie or other neat picture. It will be available on `http://your-domain.tld/instance/thumbnail.jpeg` and can be used by external applications.
 
 ## Instance-specific panel
 
-![instance-specific panel demo](/uploads/296b19ec806b130e0b49b16bfe29ce8a/image.png)
+Create and Edit your file at `$static_dir/instance/panel.html`.
-
-Create and Edit your file on `instance/static/instance/panel.html`.
 
 ## Background
 
-You can change the background of your Pleroma instance by uploading it to `instance/static/`, and then changing `background` in `config/prod.secret.exs` accordingly.
+You can change the background of your Pleroma instance by uploading it to `$static_dir/`, and then changing `background` in [your configuration](../cheatsheet/#frontend_configurations) accordingly.
 
-If you put `instance/static/images/background.jpg`
+E.g. if you put `$static_dir/images/background.jpg`
 
 ```
 config :pleroma, :frontend_configurations,
@@ -50,12 +66,14 @@ config :pleroma, :frontend_configurations,
 
 ## Logo
 
-![logo modification demo](/uploads/c70b14de60fa74245e7f0dcfa695ebff/image.png)
+!!! important
+    Note the extra `static` folder for the default logo.png location
 
-If you want to give a brand to your instance, You can change the logo of your instance by uploading it to `instance/static/`.
+If you want to give a brand to your instance, You can change the logo of your instance by uploading it to the static directory `$static_dir/static/logo.png`.
 
-Alternatively, you can specify the path with config.
+Alternatively, you can specify the path to your logo in [your configuration](../cheatsheet/#frontend_configurations).
-If you put `instance/static/static/mylogo-file.png`
+
+E.g. if you put `$static_dir/static/mylogo-file.png`
 
 ```
 config :pleroma, :frontend_configurations,
@@ -66,4 +84,7 @@ config :pleroma, :frontend_configurations,
 
 ## Terms of Service
 
-Terms of Service will be shown to all users on the registration page. It's the best place where to write down the rules for your instance. You can modify the rules by changing `instance/static/static/terms-of-service.html`.
+!!! important
+    Note the extra `static` folder for the terms-of-service.html
+
+Terms of Service will be shown to all users on the registration page. It's the best place where to write down the rules for your instance. You can modify the rules by adding and changing `$static_dir/static/terms-of-service.html`.

docs/installation/alpine_linux_en.md

@@ -14,11 +14,15 @@ It assumes that you have administrative rights, either as root or a user with [s
 * `erlang-xmerl`
 * `git`
 * Development Tools
+* `cmake`
 
 #### Optional packages used in this guide
 
 * `nginx` (preferred, example configs for other reverse proxies can be found in the repo)
 * `certbot` (or any other ACME client for Let’s Encrypt certificates)
+* `ImageMagick`
+* `ffmpeg`
+* `exiftool`
 
 ### Prepare the system
 
@@ -28,7 +32,6 @@ It assumes that you have administrative rights, either as root or a user with [s
 awk 'NR==2' /etc/apk/repositories | sed 's/main/community/' | tee -a /etc/apk/repositories
 ```
 
-
 * Then update the system, if not already done:
 
 ```shell
@@ -39,7 +42,7 @@ sudo apk upgrade
 * Install some tools, which are needed later:
 
 ```shell
-sudo apk add git build-base
+sudo apk add git build-base cmake
 ```
 
 ### Install Elixir and Erlang
@@ -55,6 +58,7 @@ sudo apk add erlang erlang-runtime-tools erlang-xmerl elixir
 ```shell
 sudo apk add erlang-eldap
 ```
+
 ### Install PostgreSQL
 
 * Install Postgresql server:
@@ -75,6 +79,12 @@ sudo /etc/init.d/postgresql start
 sudo rc-update add postgresql
 ```
 
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+
+```shell
+sudo apk add ffmpeg imagemagick exiftool
+```
+
 ### Install PleromaBE
 
 * Add a new system user for the Pleroma service:

docs/installation/arch_linux_en.md

@@ -9,11 +9,15 @@ This guide will assume that you have administrative rights, either as root or a
 * `elixir`
 * `git`
 * `base-devel`
+* `cmake`
 
 #### Optional packages used in this guide
 
 * `nginx` (preferred, example configs for other reverse proxies can be found in the repo)
 * `certbot` (or any other ACME client for Let’s Encrypt certificates)
+* `ImageMagick`
+* `ffmpeg`
+* `exiftool`
 
 ### Prepare the system
 
@@ -26,7 +30,7 @@ sudo pacman -Syu
 * Install some of the above mentioned programs:
 
 ```shell
-sudo pacman -S git base-devel elixir
+sudo pacman -S git base-devel elixir cmake
 ```
 
 ### Install PostgreSQL
@@ -51,6 +55,12 @@ sudo -iu postgres initdb -D /var/lib/postgres/data
 sudo systemctl enable --now postgresql.service
 ```
 
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+
+```shell
+sudo pacman -S ffmpeg imagemagick perl-image-exiftool
+```
+
 ### Install PleromaBE
 
 * Add a new system user for the Pleroma service:

docs/installation/debian_based_en.md

@@ -12,11 +12,15 @@ This guide will assume you are on Debian Stretch. This guide should also work wi
 * `erlang-nox`
 * `git`
 * `build-essential`
+* `cmake`
 
 #### Optional packages used in this guide
 
 * `nginx` (preferred, example configs for other reverse proxies can be found in the repo)
 * `certbot` (or any other ACME client for Let’s Encrypt certificates)
+* `ImageMagick`
+* `ffmpeg`
+* `exiftool`
 
 ### Prepare the system
 
@@ -30,7 +34,7 @@ sudo apt full-upgrade
 * Install some of the above mentioned programs:
 
 ```shell
-sudo apt install git build-essential postgresql postgresql-contrib
+sudo apt install git build-essential postgresql postgresql-contrib cmake
 ```
 
 ### Install Elixir and Erlang
@@ -49,6 +53,12 @@ sudo apt update
 sudo apt install elixir erlang-dev erlang-nox
 ```
 
+### Optional packages: [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)
+
+```shell
+sudo apt install imagemagick ffmpeg libimage-exiftool-perl
+```
+
 ### Install PleromaBE
 
 * Add a new system user for the Pleroma service:

docs/installation/debian_based_jp.md

@@ -16,11 +16,15 @@
 - `erlang-nox`
 - `git`
 - `build-essential`
+- `cmake`
 
 #### このガイドで利用している追加パッケージ
 
 - `nginx` (おすすめです。他のリバースプロキシを使う場合は、参考となる設定をこのリポジトリから探してください)
 - `certbot` (または何らかのLet's Encrypt向けACMEクライアント)
+- `ImageMagick`
+- `ffmpeg`
+- `exiftool`
 
 ### システムを準備する
 
@@ -32,10 +36,9 @@ sudo apt full-upgrade
 
 * 上記に挙げたパッケージをインストールしておきます。
 ```
-sudo apt install git build-essential postgresql postgresql-contrib
+sudo apt install git build-essential postgresql postgresql-contrib cmake ffmpeg imagemagick
 ```
 
-
 ### ElixirとErlangをインストールします
 
 * Erlangのリポジトリをダウンロードおよびインストールします。
@@ -50,6 +53,12 @@ sudo apt update
 sudo apt install elixir erlang-dev erlang-nox
 ```
 
+### オプションパッケージ: [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)
+
+```shell
+sudo apt install imagemagick ffmpeg libimage-exiftool-perl
+```
+
 ### Pleroma BE (バックエンド) をインストールします
 
 *  Pleroma用に新しいユーザーを作ります。

docs/installation/freebsd_en.md

@@ -0,0 +1,216 @@
+# Installing on FreeBSD 
+
+This document was written for FreeBSD 12.1, but should be work on future releases.
+
+## Required software 
+
+This assumes the target system has `pkg(8)`.
+
+```
+# pkg install elixir postgresql12-server postgresql12-client postgresql12-contrib git-lite sudo nginx gmake acme.sh cmake
+```
+
+Copy the rc.d scripts to the right directory:
+
+Setup the required services to automatically start at boot, using `sysrc(8)`.
+
+```
+# sysrc nginx_enable=YES
+# sysrc postgresql_enable=YES
+```
+
+## Initialize postgres
+
+```
+# service postgresql initdb
+# service postgresql start
+```
+
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+
+```shell
+# pkg install imagemagick ffmpeg p5-Image-ExifTool
+```
+
+## Configuring Pleroma
+
+Create a user for Pleroma:
+
+```
+# pw add user pleroma -m
+# echo 'export LC_ALL="en_US.UTF-8"' >> /home/pleroma/.profile
+# su -l pleroma
+```
+
+Clone the repository:
+
+```
+$ cd $HOME # Should be the same as /home/pleroma
+$ git clone -b stable https://git.pleroma.social/pleroma/pleroma.git
+```
+
+Configure Pleroma. Note that you need a domain name at this point:
+
+```
+$ cd /home/pleroma/pleroma
+$ mix deps.get # Enter "y" when asked to install Hex
+$ mix pleroma.instance gen # You will be asked a few questions here.
+$ cp config/generated_config.exs config/prod.secret.exs
+```
+
+Since Postgres is configured, we can now initialize the database. There should
+now be a file in `config/setup_db.psql` that makes this easier. Edit it, and
+*change the password* to a password of your choice. Make sure it is secure, since
+it'll be protecting your database. As root, you can now initialize the database:
+
+```
+# cd /home/pleroma/pleroma
+# sudo -Hu postgres -g postgres psql -f config/setup_db.psql
+```
+
+Postgres allows connections from all users without a password by default. To
+fix this, edit `/var/db/postgres/data12/pg_hba.conf`. Change every `trust` to
+`password`.
+
+Once this is done, restart Postgres with:
+```
+# service postgresql restart
+```
+
+Run the database migrations.
+
+Back as the pleroma user, run the following to implement any database migrations.
+
+```
+# su -l pleroma
+$ cd /home/pleroma/pleroma
+$ MIX_ENV=prod mix ecto.migrate
+```
+
+You will need to do this whenever you update with `git pull`:
+
+## Configuring acme.sh
+
+We'll be using acme.sh in Stateless Mode for TLS certificate renewal.
+
+First, as root, allow the user `acme` to have access to the acme log file, as follows:
+
+```
+# touch /var/log/acme.sh.log
+# chown acme:acme /var/log/acme.sh.log
+# chmod 600 /var/log/acme.sh.log
+```
+
+Next, obtain your account fingerprint:
+
+```
+# sudo -Hu acme -g acme acme.sh --register-account
+```
+
+You need to add the following to your nginx configuration for the server
+running on port 80:
+
+```
+  location ~ ^/\.well-known/acme-challenge/([-_a-zA-Z0-9]+)$ {
+    default_type text/plain;
+    return 200 "$1.6fXAG9VyG0IahirPEU2ZerUtItW2DHzDzD9wZaEKpqd";
+  }
+```
+
+Replace the string after after `$1.` with your fingerprint.
+
+Start nginx:
+
+```
+# service nginx start
+```
+
+It should now be possible to issue a cert (replace `example.com`
+with your domain name):
+
+```
+# sudo -Hu acme -g acme acme.sh --issue -d example.com --stateless
+```
+
+Let's add auto-renewal to `/etc/crontab`
+(replace `example.com` with your domain):
+
+```
+/usr/local/bin/sudo -Hu acme -g acme /usr/local/sbin/acme.sh -r -d example.com --stateless
+```
+
+### Configuring nginx
+
+FreeBSD's default nginx configuration does not contain an include directive, which is
+typically used for multiple sites. Therefore, you will need to first create the required
+directory as follows:
+
+
+```
+# mkdir -p /usr/local/etc/nginx/sites-available
+```
+
+Next, add an `include` directive to `/usr/local/etc/nginx/nginx.conf`, within the `http {}`
+block, as follows:
+
+
+```
+http {
+...
+	include /usr/local/etc/nginx/sites-available/*;
+}
+```
+
+As root, copy `/home/pleroma/pleroma/installation/pleroma.nginx` to
+`/usr/local/etc/nginx/sites-available/pleroma.nginx`.
+
+Edit the defaults of `/usr/local/etc/nginx/sites-available/pleroma.nginx`:
+
+* Change `ssl_trusted_certificate` to `/var/db/acme/certs/example.tld/example.tld.cer`.
+* Change `ssl_certificate` to `/var/db/acme/certs/example.tld/fullchain.cer`.
+* Change `ssl_certificate_key` to `/var/db/acme/certs/example.tld/example.tld.key`.
+* Change all references of `example.tld` to your instance's domain name.
+
+## Creating a startup script for Pleroma
+
+Pleroma will need to compile when it initially starts, which typically takes a longer
+period of time. Therefore, it is good practice to initially run pleroma from the
+command-line before utilizing the rc.d script. That is done as follows:
+
+```
+# su -l pleroma
+$ cd $HOME/pleroma
+$ MIX_ENV=prod mix phx.server
+```
+
+Copy the startup script to the correct location and make sure it's executable:
+
+```
+# cp /home/pleroma/pleroma/installation/freebsd/rc.d/pleroma /usr/local/etc/rc.d/pleroma
+# chmod +x /usr/local/etc/rc.d/pleroma
+```
+
+Update the `/etc/rc.conf` and start pleroma with the following commands:
+
+```
+# sysrc pleroma_enable=YES
+# service pleroma start
+```
+
+#### Create your first user
+
+If your instance is up and running, you can create your first user with administrative rights with the following task:
+
+```shell
+sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress> --admin
+```
+## Conclusion
+
+Restart nginx with `# service nginx restart` and you should be up and running.
+
+Make sure your time is in sync, or other instances will receive your posts with
+incorrect timestamps. You should have ntpd running.
+
+## Questions
+
+Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.

docs/installation/gentoo_en.md

@@ -28,12 +28,16 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
 * `dev-db/postgresql`
 * `dev-lang/elixir`
 * `dev-vcs/git`
+* `dev-util/cmake`
 
 #### Optional ebuilds used in this guide
 
 * `www-servers/nginx` (preferred, example configs for other reverse proxies can be found in the repo)
 * `app-crypt/certbot` (or any other ACME client for Let’s Encrypt certificates)
 * `app-crypt/certbot-nginx` (nginx certbot plugin that allows use of the all-powerful `--nginx` flag on certbot)
+* `media-gfx/imagemagick`
+* `media-video/ffmpeg`
+* `media-libs/exiftool`
 
 ### Prepare the system
 
@@ -46,7 +50,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
 * Emerge all required the required and suggested software in one go:
 
 ```shell
- # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx
+ # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake
 ```
 
 If you would not like to install the optional packages, remove them from this line. 
@@ -86,6 +90,12 @@ If you do not plan to make any modifications to your Pleroma instance, cloning d
 
 Not only does this make it much easier to deploy changes you make, as you can commit and pull from upstream and all that good stuff from the comfort of your local machine then simply `git pull` on your instance server when you're ready to deploy, it also ensures you are compliant with the Affero General Public Licence that Pleroma is licenced under, which stipulates that all network services provided with modified AGPL code must publish their changes on a publicly available internet service and for free. It also makes it much easier to ask for help from and provide help to your fellow Pleroma admins if your public repo always reflects what you are running because it is part of your deployment procedure.
 
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+
+```shell
+# emerge --ask media-video/ffmpeg media-gfx/imagemagick media-libs/exiftool
+```
+
 ### Install PleromaBE
 
 * Add a new system user for the Pleroma service and set up default directories:

docs/installation/migrating_from_source_otp_en.md

@@ -8,13 +8,15 @@ You will be running commands as root. If you aren't root already, please elevate
 
 The system needs to have `curl` and `unzip` installed for downloading and unpacking release builds.
 
-```sh tab="Alpine"
+=== "Alpine"
-apk add curl unzip
+    ```sh
-```
+    apk add curl unzip
+    ```
 
-```sh tab="Debian/Ubuntu"
+=== "Debian/Ubuntu"
-apt install curl unzip
+    ```sh
-```
+    apt install curl unzip
+    ```
 
 ## Moving content out of the application directory
 When using OTP releases the application directory changes with every version so it would be a bother to keep content there (and also dangerous unless `--no-rm` option is used when updating). Fortunately almost all paths in Pleroma are configurable, so it is possible to move them out of there.
@@ -110,27 +112,29 @@ OTP releases have different service files than from-source installs so they need
 
 **Warning:** The service files assume pleroma user's home directory is `/opt/pleroma`, please make sure all paths fit your installation.
 
-```sh tab="Alpine"
+=== "Alpine"
-# Copy the service into a proper directory
+    ```sh
-cp -f ~pleroma/installation/init.d/pleroma /etc/init.d/pleroma
+    # Copy the service into a proper directory
+    cp -f ~pleroma/installation/init.d/pleroma /etc/init.d/pleroma
 
-# Start pleroma
+    # Start pleroma
-rc-service pleroma start
+    rc-service pleroma start
-```
+    ```
 
-```sh tab="Debian/Ubuntu"
+=== "Debian/Ubuntu"
-# Copy the service into a proper directory
+    ```sh
-cp ~pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service
+    # Copy the service into a proper directory
+    cp ~pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service
 
-# Reload service files
+    # Reload service files
-systemctl daemon-reload
+    systemctl daemon-reload
 
-# Reenable pleroma to start on boot
+    # Reenable pleroma to start on boot
-systemctl reenable pleroma
+    systemctl reenable pleroma
 
-# Start pleroma
+    # Start pleroma
-systemctl start pleroma
+    systemctl start pleroma
-```
+    ```
 
 ## Running mix tasks
 Refer to [Running mix tasks](otp_en.md#running-mix-tasks) section from OTP release installation guide.

docs/installation/netbsd_en.md

@@ -10,7 +10,7 @@ Pleroma uses.
 
 The `mksh` shell is needed to run the Elixir `mix` script.
 
-`# pkgin install acmesh elixir git-base git-docs mksh nginx postgresql11-server postgresql11-client postgresql11-contrib sudo`
+`# pkgin install acmesh elixir git-base git-docs mksh nginx postgresql11-server postgresql11-client postgresql11-contrib sudo ffmpeg4 ImageMagick`
 
 You can also build these packages using pkgsrc:
 ```
@@ -19,6 +19,7 @@ databases/postgresql11-client
 databases/postgresql11-server
 devel/git-base
 devel/git-docs
+devel/cmake
 lang/elixir
 security/acmesh
 security/sudo
@@ -43,6 +44,10 @@ pgsql=YES
 
 First, run `# /etc/rc.d/pgsql start`. Then, `$ sudo -Hu pgsql -g pgsql createdb`.
 
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+
+`# pkgin install ImageMagick ffmpeg4 p5-Image-ExifTool`
+
 ## Configuring Pleroma
 
 Create a user for Pleroma:

docs/installation/openbsd_en.md

@@ -10,19 +10,34 @@ The following packages need to be installed:
 
   * elixir
   * gmake
-  * ImageMagick
   * git
   * postgresql-server
   * postgresql-contrib
+  * cmake
+  * ffmpeg
+  * ImageMagick
 
 To install them, run the following command (with doas or as root):
 
 ```
-pkg_add elixir gmake ImageMagick git postgresql-server postgresql-contrib
+pkg_add elixir gmake git postgresql-server postgresql-contrib cmake ffmpeg ImageMagick
 ```
 
 Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in this guide) and packages/ports are available for nginx (www/nginx) and apache (www/apache-httpd). Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acme-client) can be used to get a certificate from Let's Encrypt.
 
+#### Optional software
+
+Per [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md):
+  * ImageMagick
+  * ffmpeg
+  * exiftool
+
+To install the above:
+
+```
+pkg_add ImageMagick ffmpeg p5-Image-ExifTool
+```
+
 #### Creating the pleroma user
 Pleroma will be run by a dedicated user, \_pleroma. Before creating it, insert the following lines in login.conf:
 ```

docs/installation/openbsd_fi.md

@@ -16,7 +16,18 @@ Matrix-kanava #freenode_#pleroma:matrix.org ovat hyviä paikkoja löytää apua
 
 Asenna tarvittava ohjelmisto:
 
-`# pkg_add git elixir gmake postgresql-server-10.3 postgresql-contrib-10.3`
+`# pkg_add git elixir gmake postgresql-server-10.3 postgresql-contrib-10.3 cmake ffmpeg ImageMagick`
+
+#### Optional software
+
+[`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md):
+  * ImageMagick
+  * ffmpeg
+  * exiftool
+
+Asenna tarvittava ohjelmisto:
+
+`# pkg_add ImageMagick ffmpeg p5-Image-ExifTool`
 
 Luo postgresql-tietokanta:
 

docs/installation/optional/media_graphics_packages.md

@@ -0,0 +1,32 @@
+# Optional software packages needed for specific functionality
+
+For specific Pleroma functionality (which is disabled by default) some or all of the below packages are required:
+  * `ImageMagic`
+  * `ffmpeg`
+  * `exiftool`
+  
+Please refer to documentation in `docs/installation` on how to install them on specific OS.
+  
+Note: the packages are not required with the current default settings of Pleroma.
+
+## `ImageMagick`
+
+`ImageMagick` is a set of tools to create, edit, compose, or convert bitmap images.
+
+It is required for the following Pleroma features:
+  * `Pleroma.Upload.Filters.Mogrify`, `Pleroma.Upload.Filters.Mogrifun` upload filters (related config: `Plaroma.Upload/filters` in `config/config.exs`)
+  * Media preview proxy for still images (related config: `media_preview_proxy/enabled` in `config/config.exs`)
+  
+## `ffmpeg`
+
+`ffmpeg` is software to record, convert and stream audio and video.
+
+It is required for the following Pleroma features:
+  * Media preview proxy for videos (related config: `media_preview_proxy/enabled` in `config/config.exs`)
+
+## `exiftool`
+
+`exiftool` is media files metadata reader/writer.
+
+It is required for the following Pleroma features:
+  * `Pleroma.Upload.Filters.Exiftool` upload filter (related config: `Plaroma.Upload/filters` in `config/config.exs`)

docs/installation/otp_en.md

@@ -28,15 +28,36 @@ Other than things bundled in the OTP release Pleroma depends on:
 * nginx (could be swapped with another reverse proxy but this guide covers only it)
 * certbot (for Let's Encrypt certificates, could be swapped with another ACME client, but this guide covers only it)
 
-```sh tab="Alpine"
+=== "Alpine"
-echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories
+    ```
-apk update
+    echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories
-apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot
+    apk update
-```
+    apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot
+    ```
 
-```sh tab="Debian/Ubuntu"
+=== "Debian/Ubuntu"
-apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot
+    ```
-```
+    apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot
+    ```
+
+### Installing optional packages
+
+Per [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md):
+  * ImageMagick
+  * ffmpeg
+  * exiftool
+
+=== "Alpine"
+    ```
+    echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories
+    apk update
+    apk add imagemagick ffmpeg exiftool
+    ```
+
+=== "Debian/Ubuntu"
+    ```
+    apt install imagemagick ffmpeg libimage-exiftool-perl
+    ```
 
 ## Setup
 ### Configuring PostgreSQL
@@ -47,35 +68,41 @@ apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot
 
 RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. You can read more about them on the [Configuration page](../configuration/cheatsheet.md#rum-indexing-for-full-text-search). They are completely optional and most of the time are not worth it, especially if you are running a single user instance (unless you absolutely need ordered search results).
 
-```sh tab="Alpine"
+=== "Alpine"
-apk add git build-base postgresql-dev
+    ```
-git clone https://github.com/postgrespro/rum /tmp/rum
+    apk add git build-base postgresql-dev
-cd /tmp/rum
+    git clone https://github.com/postgrespro/rum /tmp/rum
-make USE_PGXS=1
+    cd /tmp/rum
-make USE_PGXS=1 install
+    make USE_PGXS=1
-cd
+    make USE_PGXS=1 install
-rm -r /tmp/rum
+    cd
-```
+    rm -r /tmp/rum
+    ```
 
-```sh tab="Debian/Ubuntu"
+=== "Debian/Ubuntu"
-# Available only on Buster/19.04
+    ```
-apt install postgresql-11-rum
+    # Available only on Buster/19.04
-```
+    apt install postgresql-11-rum
+    ```
 
 #### (Optional) Performance configuration
 It is encouraged to check [Optimizing your PostgreSQL performance](../configuration/postgresql.md) document, for tips on PostgreSQL tuning.
 
-```sh tab="Alpine"
+=== "Alpine"
-rc-service postgresql restart
+    ```
-```
+    rc-service postgresql restart
+    ```
 
-```sh tab="Debian/Ubuntu"
+=== "Debian/Ubuntu"
-systemctl restart postgresql
+    ```
-```
+    systemctl restart postgresql
+    ```
 
 If you are using PostgreSQL 12 or higher, add this to your Ecto database configuration
 
 ```elixir
+#
+config :pleroma, Pleroma.Repo,
 prepare: :named,
 parameters: [
   plan_cache_mode: "force_custom_plan"
@@ -121,9 +148,6 @@ chown -R pleroma /etc/pleroma
 # Run the config generator
 su pleroma -s $SHELL -lc "./bin/pleroma_ctl instance gen --output /etc/pleroma/config.exs --output-psql /tmp/setup_db.psql"
 
-# Run the environment file generator.
-su pleroma -s $SHELL -lc "./bin/pleroma_ctl release_env gen"
-
 # Create the postgres database
 su postgres -s $SHELL -lc "psql -f /tmp/setup_db.psql"
 
@@ -134,7 +158,7 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
 # su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
 
 # Start the instance to verify that everything is working as expected
-su pleroma -s $SHELL -lc "export $(cat /opt/pleroma/config/pleroma.env); ./bin/pleroma daemon"
+su pleroma -s $SHELL -lc "./bin/pleroma daemon"
 
 # Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly
 sleep 20 && curl http://localhost:4000/api/v1/instance
@@ -154,14 +178,16 @@ certbot certonly --standalone --preferred-challenges http -d yourinstance.tld
 
 The location of nginx configs is dependent on the distro
 
-```sh tab="Alpine"
+=== "Alpine"
-cp /opt/pleroma/installation/pleroma.nginx /etc/nginx/conf.d/pleroma.conf
+    ```
-```
+    cp /opt/pleroma/installation/pleroma.nginx /etc/nginx/conf.d/pleroma.conf
+    ```
 
-```sh tab="Debian/Ubuntu"
+=== "Debian/Ubuntu"
-cp /opt/pleroma/installation/pleroma.nginx /etc/nginx/sites-available/pleroma.conf
+    ```
-ln -s /etc/nginx/sites-available/pleroma.conf /etc/nginx/sites-enabled/pleroma.conf
+    cp /opt/pleroma/installation/pleroma.nginx /etc/nginx/sites-available/pleroma.conf
-```
+    ln -s /etc/nginx/sites-available/pleroma.conf /etc/nginx/sites-enabled/pleroma.conf
+    ```
 
 If your distro does not have either of those you can append `include /etc/nginx/pleroma.conf` to the end of the http section in /etc/nginx/nginx.conf and
 ```sh
@@ -178,36 +204,39 @@ nginx -t
 ```
 #### Start nginx
 
-```sh tab="Alpine"
+=== "Alpine"
-rc-service nginx start
+    ```
-```
+    rc-service nginx start
+    ```
 
-```sh tab="Debian/Ubuntu"
+=== "Debian/Ubuntu"
-systemctl start nginx
+    ```
-```
+    systemctl start nginx
+    ```
 
 At this point if you open your (sub)domain in a browser you should see a 502 error, that's because Pleroma is not started yet.
 
 ### Setting up a system service
 
-```sh tab="Alpine"
+=== "Alpine"
-# Copy the service into a proper directory
+    ```
-cp /opt/pleroma/installation/init.d/pleroma /etc/init.d/pleroma
+    # Copy the service into a proper directory
+    cp /opt/pleroma/installation/init.d/pleroma /etc/init.d/pleroma
 
-# Start pleroma and enable it on boot
+    # Start pleroma and enable it on boot
-rc-service pleroma start
+    rc-service pleroma start
-rc-update add pleroma
+    rc-update add pleroma
-```
+    ```
 
-```sh tab="Debian/Ubuntu"
+=== "Debian/Ubuntu"
-# Copy the service into a proper directory
+    ```
-cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service
+    # Copy the service into a proper directory
+    cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service
 
-
+    # Start pleroma and enable it on boot
-# Start pleroma and enable it on boot
+    systemctl start pleroma
-systemctl start pleroma
+    systemctl enable pleroma
-systemctl enable pleroma
+    ```
-```
 
 If everything worked, you should see Pleroma-FE when visiting your domain. If that didn't happen, try reviewing the installation steps, starting Pleroma in the foreground and seeing if there are any errrors.
 
@@ -227,43 +256,45 @@ $EDITOR path-to-nginx-config
 nginx -t
 ```
 
-```sh tab="Alpine"
+=== "Alpine"
-# Restart nginx
+    ```
-rc-service nginx restart
+    # Restart nginx
+    rc-service nginx restart
 
-# Start the cron daemon and make it start on boot
+    # Start the cron daemon and make it start on boot
-rc-service crond start
+    rc-service crond start
-rc-update add crond
+    rc-update add crond
 
-# Ensure the webroot menthod and post hook is working
+    # Ensure the webroot menthod and post hook is working
-certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --dry-run --post-hook 'rc-service nginx reload'
+    certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --dry-run --post-hook 'rc-service nginx reload'
 
-# Add it to the daily cron
+    # Add it to the daily cron
-echo '#!/bin/sh
+    echo '#!/bin/sh
-certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --post-hook "rc-service nginx reload"
+    certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --post-hook "rc-service nginx reload"
-' > /etc/periodic/daily/renew-pleroma-cert
+    ' > /etc/periodic/daily/renew-pleroma-cert
-chmod +x /etc/periodic/daily/renew-pleroma-cert
+    chmod +x /etc/periodic/daily/renew-pleroma-cert
 
-# If everything worked the output should contain /etc/cron.daily/renew-pleroma-cert
+    # If everything worked the output should contain /etc/cron.daily/renew-pleroma-cert
-run-parts --test /etc/periodic/daily
+    run-parts --test /etc/periodic/daily
-```
+    ```
 
-```sh tab="Debian/Ubuntu"
+=== "Debian/Ubuntu"
-# Restart nginx
+    ```
-systemctl restart nginx
+    # Restart nginx
+    systemctl restart nginx
 
-# Ensure the webroot menthod and post hook is working
+    # Ensure the webroot menthod and post hook is working
-certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --dry-run --post-hook 'systemctl reload nginx'
+    certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --dry-run --post-hook 'systemctl reload nginx'
 
-# Add it to the daily cron
+    # Add it to the daily cron
-echo '#!/bin/sh
+    echo '#!/bin/sh
-certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --post-hook "systemctl reload nginx"
+    certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --post-hook "systemctl reload nginx"
-' > /etc/cron.daily/renew-pleroma-cert
+    ' > /etc/cron.daily/renew-pleroma-cert
-chmod +x /etc/cron.daily/renew-pleroma-cert
+    chmod +x /etc/cron.daily/renew-pleroma-cert
 
-# If everything worked the output should contain /etc/cron.daily/renew-pleroma-cert
+    # If everything worked the output should contain /etc/cron.daily/renew-pleroma-cert
-run-parts --test /etc/cron.daily
+    run-parts --test /etc/cron.daily
-```
+    ```
 
 ## Create your first user and set as admin
 ```sh
@@ -279,3 +310,4 @@ This will create an account withe the username of 'joeuser' with the email addre
 ## Questions
 
 Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
+

installation/freebsd/rc.d/pleroma

@@ -0,0 +1,27 @@
+#!/bin/sh
+# $FreeBSD$
+# PROVIDE: pleroma
+# REQUIRE: DAEMON postgresql
+# KEYWORD: shutdown
+
+# sudo -u pleroma MIX_ENV=prod elixir --erl \"-detached\" -S mix phx.server
+
+. /etc/rc.subr
+
+name=pleroma
+rcvar=pleroma_enable
+
+desc="Pleroma Social Media Platform"
+
+load_rc_config ${name}
+
+: ${pleroma_user:=pleroma}
+: ${pleroma_home:=$(getent passwd ${pleroma_user} | awk -F: '{print $6}')}
+: ${pleroma_chdir:="${pleroma_home}/pleroma"}
+: ${pleroma_env:="HOME=${pleroma_home} MIX_ENV=prod"}
+
+command=/usr/local/bin/elixir
+command_args="--erl \"-detached\" -S /usr/local/bin/mix phx.server"
+procname="*beam.smp"
+
+run_rc_command "$1"

installation/init.d/pleroma

@@ -8,7 +8,6 @@ pidfile="/var/run/pleroma.pid"
 directory=/opt/pleroma
 healthcheck_delay=60
 healthcheck_timer=30
-export $(cat /opt/pleroma/config/pleroma.env)
 
 : ${pleroma_port:-4000}
 

installation/pleroma.nginx

@@ -9,6 +9,12 @@
 proxy_cache_path /tmp/pleroma-media-cache levels=1:2 keys_zone=pleroma_media_cache:10m max_size=10g
                  inactive=720m use_temp_path=off;
 
+# this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
+# and `localhost.` resolves to [::0] on some systems: see issue #930
+upstream phoenix {
+    server 127.0.0.1:4000 max_fails=5 fail_timeout=60s;
+}
+
 server {
     server_name    example.tld;
 
@@ -63,19 +69,16 @@ server {
 
     # the nginx default is 1m, not enough for large media uploads
     client_max_body_size 16m;
+    ignore_invalid_headers off;
+
+    proxy_http_version 1.1;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "upgrade";
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 
     location / {
-        proxy_http_version 1.1;
+        proxy_pass http://phoenix;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection "upgrade";
-        proxy_set_header Host $http_host;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-
-        # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
-        # and `localhost.` resolves to [::0] on some systems: see issue #930
-        proxy_pass http://127.0.0.1:4000;
-
-        client_max_body_size 16m;
     }
 
     location ~ ^/(media|proxy) {
@@ -83,12 +86,16 @@ server {
         slice              1m;
         proxy_cache_key    $host$uri$is_args$args$slice_range;
         proxy_set_header   Range $slice_range;
-        proxy_http_version 1.1;
         proxy_cache_valid  200 206 301 304 1h;
         proxy_cache_lock   on;
         proxy_ignore_client_abort on;
         proxy_buffering    on;
         chunked_transfer_encoding on;
-        proxy_pass         http://127.0.0.1:4000;
+        proxy_pass         http://phoenix;
+    }
+
+    location /api/fedsocket/v1 {
+        proxy_request_buffering off;
+        proxy_pass http://phoenix/api/fedsocket/v1;
     }
 }

installation/pleroma.service

@@ -17,8 +17,6 @@ Environment="MIX_ENV=prod"
 Environment="HOME=/var/lib/pleroma"
 ; Path to the folder containing the Pleroma installation.
 WorkingDirectory=/opt/pleroma
-; Path to the environment file. the file contains RELEASE_COOKIE and etc 
-EnvironmentFile=/opt/pleroma/config/pleroma.env
 ; Path to the Mix binary.
 ExecStart=/usr/bin/mix phx.server
 

installation/pleroma.vcl

@@ -1,3 +1,4 @@
+# Recommended varnishncsa logging format: '%h %l %u %t "%m %{X-Forwarded-Proto}i://%{Host}i%U%q %H" %s %b "%{Referer}i" "%{User-agent}i"'
 vcl 4.1;
 import std;
 
@@ -14,8 +15,11 @@ acl purge {
 sub vcl_recv {
     # Redirect HTTP to HTTPS
     if (std.port(server.ip) != 443) {
+      set req.http.X-Forwarded-Proto = "http";
       set req.http.x-redir = "https://" + req.http.host + req.url;
       return (synth(750, ""));
+    } else {
+      set req.http.X-Forwarded-Proto = "https";
     }
 
     # CHUNKED SUPPORT
@@ -105,7 +109,7 @@ sub vcl_hash {
 
 sub vcl_backend_fetch {
     # Be more lenient for slow servers on the fediverse
-    if bereq.url ~ "^/proxy/" {
+    if (bereq.url ~ "^/proxy/") {
       set bereq.first_byte_timeout = 300s;
     }
 

lib/mix/pleroma.ex

@@ -14,10 +14,11 @@ defmodule Mix.Pleroma do
     :swoosh,
     :timex
   ]
-  @cachex_children ["object", "user"]
+  @cachex_children ["object", "user", "scrubber"]
   @doc "Common functions to be reused in mix tasks"
   def start_pleroma do
     Pleroma.Config.Holder.save_default()
+    Pleroma.Config.Oban.warn()
     Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
 
     if Pleroma.Config.get(:env) != :test do

lib/mix/tasks/pleroma/benchmark.ex

@@ -91,20 +91,17 @@ def run(["adapters"]) do
         "Without conn and without pool" => fn ->
           {:ok, %Tesla.Env{}} =
             Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
-              adapter: [pool: :no_pool, receive_conn: false]
+              pool: :no_pool,
+              receive_conn: false
             )
         end,
         "Without conn and with pool" => fn ->
           {:ok, %Tesla.Env{}} =
-            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], receive_conn: false)
-              adapter: [receive_conn: false]
-            )
         end,
         "With reused conn and without pool" => fn ->
           {:ok, %Tesla.Env{}} =
-            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], pool: :no_pool)
-              adapter: [pool: :no_pool]
-            )
         end,
         "With reused conn and with pool" => fn ->
           {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")

lib/mix/tasks/pleroma/config.ex

@@ -32,7 +32,8 @@ def run(["migrate_from_db" | options]) do
 
   @spec migrate_to_db(Path.t() | nil) :: any()
   def migrate_to_db(file_path \\ nil) do
-    if Pleroma.Config.get([:configurable_from_database]) do
+    with true <- Pleroma.Config.get([:configurable_from_database]),
+         :ok <- Pleroma.Config.DeprecationWarnings.warn() do
       config_file =
         if file_path do
           file_path
@@ -46,7 +47,8 @@ def migrate_to_db(file_path \\ nil) do
 
       do_migrate_to_db(config_file)
     else
-      migration_error()
+      :error -> deprecation_error()
+      _ -> migration_error()
     end
   end
 
@@ -120,6 +122,10 @@ defp migration_error do
     )
   end
 
+  defp deprecation_error do
+    shell_error("Migration is not allowed until all deprecation warnings have been resolved.")
+  end
+
   if Code.ensure_loaded?(Config.Reader) do
     defp config_header, do: "import Config\r\n\r\n"
     defp read_file(config_file), do: Config.Reader.read_imports!(config_file)

lib/mix/tasks/pleroma/database.ex

@@ -10,6 +10,7 @@ defmodule Mix.Tasks.Pleroma.Database do
   alias Pleroma.User
   require Logger
   require Pleroma.Constants
+  import Ecto.Query
   import Mix.Pleroma
   use Mix.Task
 
@@ -53,8 +54,6 @@ def run(["update_users_following_followers_counts"]) do
   end
 
   def run(["prune_objects" | args]) do
-    import Ecto.Query
-
     {options, [], []} =
       OptionParser.parse(
         args,
@@ -94,15 +93,13 @@ def run(["prune_objects" | args]) do
   end
 
   def run(["fix_likes_collections"]) do
-    import Ecto.Query
-
     start_pleroma()
 
     from(object in Object,
       where: fragment("(?)->>'likes' is not null", object.data),
       select: %{id: object.id, likes: fragment("(?)->>'likes'", object.data)}
     )
-    |> Pleroma.RepoStreamer.chunk_stream(100)
+    |> Pleroma.Repo.chunk_stream(100, :batches)
     |> Stream.each(fn objects ->
       ids =
         objects
@@ -130,4 +127,38 @@ def run(["vacuum", args]) do
 
     Maintenance.vacuum(args)
   end
+
+  def run(["ensure_expiration"]) do
+    start_pleroma()
+    days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365)
+
+    Pleroma.Activity
+    |> join(:inner, [a], o in Object,
+      on:
+        fragment(
+          "(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')",
+          o.data,
+          a.data,
+          a.data
+        )
+    )
+    |> where(local: true)
+    |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
+    |> where([_a, o], fragment("?->>'type' = 'Note'", o.data))
+    |> Pleroma.Repo.chunk_stream(100, :batches)
+    |> Stream.each(fn activities ->
+      Enum.each(activities, fn activity ->
+        expires_at =
+          activity.inserted_at
+          |> DateTime.from_naive!("Etc/UTC")
+          |> Timex.shift(days: days)
+
+        Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
+          activity_id: activity.id,
+          expires_at: expires_at
+        })
+      end)
+    end)
+    |> Stream.run()
+  end
 end

lib/mix/tasks/pleroma/ecto/migrate.ex

@@ -41,6 +41,10 @@ def run(args \\ []) do
     load_pleroma()
     {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
 
+    if Application.get_env(:pleroma, Pleroma.Repo)[:ssl] do
+      Application.ensure_all_started(:ssl)
+    end
+
     opts =
       if opts[:to] || opts[:step] || opts[:all],
         do: opts,

lib/mix/tasks/pleroma/ecto/rollback.ex

@@ -40,6 +40,10 @@ def run(args \\ []) do
     load_pleroma()
     {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
 
+    if Application.get_env(:pleroma, Pleroma.Repo)[:ssl] do
+      Application.ensure_all_started(:ssl)
+    end
+
     opts =
       if opts[:to] || opts[:step] || opts[:all],
         do: opts,

lib/mix/tasks/pleroma/email.ex

@@ -2,11 +2,11 @@ defmodule Mix.Tasks.Pleroma.Email do
   use Mix.Task
   import Mix.Pleroma
 
-  @shortdoc "Simple Email test"
+  @shortdoc "Email administrative tasks"
   @moduledoc File.read!("docs/administration/CLI_tasks/email.md")
 
   def run(["test" | args]) do
-    Mix.Pleroma.start_pleroma()
+    start_pleroma()
 
     {options, [], []} =
       OptionParser.parse(
@@ -21,4 +21,20 @@ def run(["test" | args]) do
 
     shell_info("Test email has been sent to #{inspect(email.to)} from #{inspect(email.from)}")
   end
+
+  def run(["resend_confirmation_emails"]) do
+    start_pleroma()
+
+    shell_info("Sending emails to all unconfirmed users")
+
+    Pleroma.User.Query.build(%{
+      local: true,
+      deactivated: false,
+      confirmation_pending: true,
+      invisible: false
+    })
+    |> Pleroma.Repo.chunk_stream(500)
+    |> Stream.each(&Pleroma.User.try_send_confirmation_email(&1))
+    |> Stream.run()
+  end
 end

lib/mix/tasks/pleroma/emoji.ex

@@ -15,7 +15,7 @@ def run(["ls-packs" | args]) do
     {options, [], []} = parse_global_opts(args)
 
     url_or_path = options[:manifest] || default_manifest()
-    manifest = fetch_and_decode(url_or_path)
+    manifest = fetch_and_decode!(url_or_path)
 
     Enum.each(manifest, fn {name, info} ->
       to_print = [
@@ -42,7 +42,7 @@ def run(["get-packs" | args]) do
 
     url_or_path = options[:manifest] || default_manifest()
 
-    manifest = fetch_and_decode(url_or_path)
+    manifest = fetch_and_decode!(url_or_path)
 
     for pack_name <- pack_names do
       if Map.has_key?(manifest, pack_name) do
@@ -92,7 +92,7 @@ def run(["get-packs" | args]) do
           ])
         )
 
-        files = fetch_and_decode(files_loc)
+        files = fetch_and_decode!(files_loc)
 
         IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name]))
 
@@ -183,7 +183,7 @@ def run(["gen-pack" | args]) do
 
     IO.puts("Downloading the pack and generating SHA256")
 
-    binary_archive = Tesla.get!(client(), src).body
+    {:ok, %{body: binary_archive}} = Pleroma.HTTP.get(src)
     archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16()
 
     IO.puts("SHA256 is #{archive_sha}")
@@ -243,14 +243,16 @@ def run(["reload"]) do
     IO.puts("Emoji packs have been reloaded.")
   end
 
-  defp fetch_and_decode(from) do
+  defp fetch_and_decode!(from) do
     with {:ok, json} <- fetch(from) do
       Jason.decode!(json)
+    else
+      {:error, error} -> raise "#{from} cannot be fetched. Error: #{error} occur."
     end
   end
 
   defp fetch("http" <> _ = from) do
-    with {:ok, %{body: body}} <- Tesla.get(client(), from) do
+    with {:ok, %{body: body}} <- Pleroma.HTTP.get(from) do
       {:ok, body}
     end
   end
@@ -269,13 +271,5 @@ defp parse_global_opts(args) do
     )
   end
 
-  defp client do
-    middleware = [
-      {Tesla.Middleware.FollowRedirects, [max_redirects: 3]}
-    ]
-
-    Tesla.client(middleware)
-  end
-
   defp default_manifest, do: Pleroma.Config.get!([:emoji, :default_manifest])
 end

lib/mix/tasks/pleroma/frontend.ex

@@ -0,0 +1,141 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Frontend do
+  use Mix.Task
+
+  import Mix.Pleroma
+
+  @shortdoc "Manages bundled Pleroma frontends"
+
+  @moduledoc File.read!("docs/administration/CLI_tasks/frontend.md")
+
+  def run(["install", "none" | _args]) do
+    shell_info("Skipping frontend installation because none was requested")
+    "none"
+  end
+
+  def run(["install", frontend | args]) do
+    log_level = Logger.level()
+    Logger.configure(level: :warn)
+    start_pleroma()
+
+    {options, [], []} =
+      OptionParser.parse(
+        args,
+        strict: [
+          ref: :string,
+          static_dir: :string,
+          build_url: :string,
+          build_dir: :string,
+          file: :string
+        ]
+      )
+
+    instance_static_dir =
+      with nil <- options[:static_dir] do
+        Pleroma.Config.get!([:instance, :static_dir])
+      end
+
+    cmd_frontend_info = %{
+      "name" => frontend,
+      "ref" => options[:ref],
+      "build_url" => options[:build_url],
+      "build_dir" => options[:build_dir]
+    }
+
+    config_frontend_info = Pleroma.Config.get([:frontends, :available, frontend], %{})
+
+    frontend_info =
+      Map.merge(config_frontend_info, cmd_frontend_info, fn _key, config, cmd ->
+        # This only overrides things that are actually set
+        cmd || config
+      end)
+
+    ref = frontend_info["ref"]
+
+    unless ref do
+      raise "No ref given or configured"
+    end
+
+    dest =
+      Path.join([
+        instance_static_dir,
+        "frontends",
+        frontend,
+        ref
+      ])
+
+    fe_label = "#{frontend} (#{ref})"
+
+    tmp_dir = Path.join([instance_static_dir, "frontends", "tmp"])
+
+    with {_, :ok} <-
+           {:download_or_unzip, download_or_unzip(frontend_info, tmp_dir, options[:file])},
+         shell_info("Installing #{fe_label} to #{dest}"),
+         :ok <- install_frontend(frontend_info, tmp_dir, dest) do
+      File.rm_rf!(tmp_dir)
+      shell_info("Frontend #{fe_label} installed to #{dest}")
+
+      Logger.configure(level: log_level)
+    else
+      {:download_or_unzip, _} ->
+        shell_info("Could not download or unzip the frontend")
+
+      _e ->
+        shell_info("Could not install the frontend")
+    end
+  end
+
+  defp download_or_unzip(frontend_info, temp_dir, file) do
+    if file do
+      with {:ok, zip} <- File.read(Path.expand(file)) do
+        unzip(zip, temp_dir)
+      end
+    else
+      download_build(frontend_info, temp_dir)
+    end
+  end
+
+  def unzip(zip, dest) do
+    with {:ok, unzipped} <- :zip.unzip(zip, [:memory]) do
+      File.rm_rf!(dest)
+      File.mkdir_p!(dest)
+
+      Enum.each(unzipped, fn {filename, data} ->
+        path = filename
+
+        new_file_path = Path.join(dest, path)
+
+        new_file_path
+        |> Path.dirname()
+        |> File.mkdir_p!()
+
+        File.write!(new_file_path, data)
+      end)
+
+      :ok
+    end
+  end
+
+  defp download_build(frontend_info, dest) do
+    shell_info("Downloading pre-built bundle for #{frontend_info["name"]}")
+    url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
+
+    with {:ok, %{status: 200, body: zip_body}} <-
+           Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do
+      unzip(zip_body, dest)
+    else
+      e -> {:error, e}
+    end
+  end
+
+  defp install_frontend(frontend_info, source, dest) do
+    from = frontend_info["build_dir"] || "dist"
+    File.rm_rf!(dest)
+    File.mkdir_p!(dest)
+    File.cp_r!(Path.join([source, from]), dest)
+    :ok
+  end
+end

lib/mix/tasks/pleroma/relay.ex

@@ -21,10 +21,19 @@ def run(["follow", target]) do
     end
   end
 
-  def run(["unfollow", target]) do
+  def run(["unfollow", target | rest]) do
     start_pleroma()
 
-    with {:ok, _activity} <- Relay.unfollow(target) do
+    {options, [], []} =
+      OptionParser.parse(
+        rest,
+        strict: [force: :boolean],
+        aliases: [f: :force]
+      )
+
+    force = Keyword.get(options, :force, false)
+
+    with {:ok, _activity} <- Relay.unfollow(target, %{force: force}) do
       # put this task to sleep to allow the genserver to push out the messages
       :timer.sleep(500)
     else
@@ -35,10 +44,16 @@ def run(["unfollow", target]) do
   def run(["list"]) do
     start_pleroma()
 
-    with {:ok, list} <- Relay.list(true) do
+    with {:ok, list} <- Relay.list() do
-      list |> Enum.each(&shell_info(&1))
+      Enum.each(list, &print_relay_url/1)
     else
       {:error, e} -> shell_error("Error while fetching relay subscription list: #{inspect(e)}")
     end
   end
+
+  defp print_relay_url(%{followed_back: false} = relay) do
+    shell_info("#{relay.actor} - no Accept received (relay didn't follow back)")
+  end
+
+  defp print_relay_url(relay), do: shell_info(relay.actor)
 end

lib/mix/tasks/pleroma/release_env.ex

@@ -1,76 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Mix.Tasks.Pleroma.ReleaseEnv do
-  use Mix.Task
-  import Mix.Pleroma
-
-  @shortdoc "Generate Pleroma environment file."
-  @moduledoc File.read!("docs/administration/CLI_tasks/release_environments.md")
-
-  def run(["gen" | rest]) do
-    {options, [], []} =
-      OptionParser.parse(
-        rest,
-        strict: [
-          force: :boolean,
-          path: :string
-        ],
-        aliases: [
-          p: :path,
-          f: :force
-        ]
-      )
-
-    file_path =
-      get_option(
-        options,
-        :path,
-        "Environment file path",
-        "./config/pleroma.env"
-      )
-
-    env_path = Path.expand(file_path)
-
-    proceed? =
-      if File.exists?(env_path) do
-        get_option(
-          options,
-          :force,
-          "Environment file already exists. Do you want to overwrite the #{env_path} file? (y/n)",
-          "n"
-        ) === "y"
-      else
-        true
-      end
-
-    if proceed? do
-      case do_generate(env_path) do
-        {:error, reason} ->
-          shell_error(
-            File.Error.message(%{action: "write to file", reason: reason, path: env_path})
-          )
-
-        _ ->
-          shell_info("\nThe file generated: #{env_path}.\n")
-
-          shell_info("""
-          WARNING: before start pleroma app please make sure to make the file read-only and non-modifiable.
-            Example:
-              chmod 0444 #{file_path}
-              chattr +i #{file_path}
-          """)
-      end
-    else
-      shell_info("\nThe file is exist. #{env_path}.\n")
-    end
-  end
-
-  def do_generate(path) do
-    content = "RELEASE_COOKIE=#{Base.encode32(:crypto.strong_rand_bytes(32))}"
-
-    File.mkdir_p!(Path.dirname(path))
-    File.write(path, content)
-  end
-end

lib/mix/tasks/pleroma/user.ex

@@ -179,7 +179,7 @@ def run(["deactivate_all_from_instance", instance]) do
     start_pleroma()
 
     Pleroma.User.Query.build(%{nickname: "@#{instance}"})
-    |> Pleroma.RepoStreamer.chunk_stream(500)
+    |> Pleroma.Repo.chunk_stream(500, :batches)
     |> Stream.each(fn users ->
       users
       |> Enum.each(fn user ->
@@ -196,17 +196,24 @@ def run(["set", nickname | rest]) do
       OptionParser.parse(
         rest,
         strict: [
-          moderator: :boolean,
           admin: :boolean,
-          locked: :boolean
+          confirmed: :boolean,
+          locked: :boolean,
+          moderator: :boolean
         ]
       )
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
       user =
-        case Keyword.get(options, :moderator) do
+        case Keyword.get(options, :admin) do
           nil -> user
-          value -> set_moderator(user, value)
+          value -> set_admin(user, value)
+        end
+
+      user =
+        case Keyword.get(options, :confirmed) do
+          nil -> user
+          value -> set_confirmed(user, value)
         end
 
       user =
@@ -216,9 +223,9 @@ def run(["set", nickname | rest]) do
         end
 
       _user =
-        case Keyword.get(options, :admin) do
+        case Keyword.get(options, :moderator) do
           nil -> user
-          value -> set_admin(user, value)
+          value -> set_moderator(user, value)
         end
     else
       _ ->
@@ -353,6 +360,42 @@ def run(["toggle_confirmed", nickname]) do
     end
   end
 
+  def run(["confirm_all"]) do
+    start_pleroma()
+
+    Pleroma.User.Query.build(%{
+      local: true,
+      deactivated: false,
+      is_moderator: false,
+      is_admin: false,
+      invisible: false
+    })
+    |> Pleroma.Repo.chunk_stream(500, :batches)
+    |> Stream.each(fn users ->
+      users
+      |> Enum.each(fn user -> User.need_confirmation(user, false) end)
+    end)
+    |> Stream.run()
+  end
+
+  def run(["unconfirm_all"]) do
+    start_pleroma()
+
+    Pleroma.User.Query.build(%{
+      local: true,
+      deactivated: false,
+      is_moderator: false,
+      is_admin: false,
+      invisible: false
+    })
+    |> Pleroma.Repo.chunk_stream(500, :batches)
+    |> Stream.each(fn users ->
+      users
+      |> Enum.each(fn user -> User.need_confirmation(user, true) end)
+    end)
+    |> Stream.run()
+  end
+
   def run(["sign_out", nickname]) do
     start_pleroma()
 
@@ -370,7 +413,7 @@ def run(["list"]) do
     start_pleroma()
 
     Pleroma.User.Query.build(%{local: true})
-    |> Pleroma.RepoStreamer.chunk_stream(500)
+    |> Pleroma.Repo.chunk_stream(500, :batches)
     |> Stream.each(fn users ->
       users
       |> Enum.each(fn user ->
@@ -410,4 +453,11 @@ defp set_locked(user, value) do
     shell_info("Locked status of #{user.nickname}: #{user.locked}")
     user
   end
+
+  defp set_confirmed(user, value) do
+    {:ok, user} = User.need_confirmation(user, !value)
+
+    shell_info("Confirmation pending status of #{user.nickname}: #{user.confirmation_pending}")
+    user
+  end
 end

lib/pleroma/activity.ex

@@ -7,7 +7,6 @@ defmodule Pleroma.Activity do
 
   alias Pleroma.Activity
   alias Pleroma.Activity.Queries
-  alias Pleroma.ActivityExpiration
   alias Pleroma.Bookmark
   alias Pleroma.Notification
   alias Pleroma.Object
@@ -60,8 +59,6 @@ defmodule Pleroma.Activity do
     # typical case.
     has_one(:object, Object, on_delete: :nothing, foreign_key: :id)
 
-    has_one(:expiration, ActivityExpiration, on_delete: :delete_all)
-
     timestamps()
   end
 
@@ -304,14 +301,14 @@ def all_by_actor_and_id(actor, status_ids) do
     |> Repo.all()
   end
 
-  def follow_requests_for_actor(%Pleroma.User{ap_id: ap_id}) do
+  def follow_requests_for_actor(%User{ap_id: ap_id}) do
     ap_id
     |> Queries.by_object_id()
     |> Queries.by_type("Follow")
     |> where([a], fragment("? ->> 'state' = 'pending'", a.data))
   end
 
-  def following_requests_for_actor(%Pleroma.User{ap_id: ap_id}) do
+  def following_requests_for_actor(%User{ap_id: ap_id}) do
     Queries.by_type("Follow")
     |> where([a], fragment("?->>'state' = 'pending'", a.data))
     |> where([a], a.actor == ^ap_id)
@@ -340,4 +337,10 @@ def direct_conversation_id(activity, for_user) do
       _ -> nil
     end
   end
+
+  @spec pinned_by_actor?(Activity.t()) :: boolean()
+  def pinned_by_actor?(%Activity{} = activity) do
+    actor = user_actor(activity)
+    activity.id in actor.pinned_activities
+  end
 end

lib/pleroma/activity_expiration.ex

@@ -1,67 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.ActivityExpiration do
-  use Ecto.Schema
-
-  alias Pleroma.Activity
-  alias Pleroma.ActivityExpiration
-  alias Pleroma.Repo
-
-  import Ecto.Changeset
-  import Ecto.Query
-
-  @type t :: %__MODULE__{}
-  @min_activity_lifetime :timer.hours(1)
-
-  schema "activity_expirations" do
-    belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType)
-    field(:scheduled_at, :naive_datetime)
-  end
-
-  def changeset(%ActivityExpiration{} = expiration, attrs) do
-    expiration
-    |> cast(attrs, [:scheduled_at])
-    |> validate_required([:scheduled_at])
-    |> validate_scheduled_at()
-  end
-
-  def get_by_activity_id(activity_id) do
-    ActivityExpiration
-    |> where([exp], exp.activity_id == ^activity_id)
-    |> Repo.one()
-  end
-
-  def create(%Activity{} = activity, scheduled_at) do
-    %ActivityExpiration{activity_id: activity.id}
-    |> changeset(%{scheduled_at: scheduled_at})
-    |> Repo.insert()
-  end
-
-  def due_expirations(offset \\ 0) do
-    naive_datetime =
-      NaiveDateTime.utc_now()
-      |> NaiveDateTime.add(offset, :millisecond)
-
-    ActivityExpiration
-    |> where([exp], exp.scheduled_at < ^naive_datetime)
-    |> Repo.all()
-  end
-
-  def validate_scheduled_at(changeset) do
-    validate_change(changeset, :scheduled_at, fn _, scheduled_at ->
-      if not expires_late_enough?(scheduled_at) do
-        [scheduled_at: "an ephemeral activity must live for at least one hour"]
-      else
-        []
-      end
-    end)
-  end
-
-  def expires_late_enough?(scheduled_at) do
-    now = NaiveDateTime.utc_now()
-    diff = NaiveDateTime.diff(scheduled_at, now, :millisecond)
-    diff > @min_activity_lifetime
-  end
-end

lib/pleroma/application.ex

@@ -22,13 +22,18 @@ def named_version, do: @name <> " " <> @version
   def repository, do: @repository
 
   def user_agent do
-    case Config.get([:http, :user_agent], :default) do
+    if Process.whereis(Pleroma.Web.Endpoint) do
-      :default ->
+      case Config.get([:http, :user_agent], :default) do
-        info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>"
+        :default ->
-        named_version() <> "; " <> info
+          info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>"
+          named_version() <> "; " <> info
 
-      custom ->
+        custom ->
-        custom
+          custom
+      end
+    else
+      # fallback, if endpoint is not started yet
+      "Pleroma Data Loader"
     end
   end
 
@@ -39,9 +44,13 @@ def start(_type, _args) do
     # every time the application is restarted, so we disable module
     # conflicts at runtime
     Code.compiler_options(ignore_module_conflict: true)
+    # Disable warnings_as_errors at runtime, it breaks Phoenix live reload
+    # due to protocol consolidation warnings
+    Code.compiler_options(warnings_as_errors: false)
     Pleroma.Telemetry.Logger.attach()
     Config.Holder.save_default()
     Pleroma.HTML.compile_scrubbers()
+    Pleroma.Config.Oban.warn()
     Config.DeprecationWarnings.warn()
     Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
     Pleroma.ApplicationRequirements.verify!()
@@ -89,7 +98,7 @@ def start(_type, _args) do
           {Oban, Config.get(Oban)}
         ] ++
         task_children(@env) ++
-        streamer_child(@env) ++
+        dont_run_in_test(@env) ++
         chat_child(@env, chat_enabled?()) ++
         [
           Pleroma.Web.Endpoint,
@@ -178,16 +187,17 @@ def build_cachex(type, opts),
 
   defp chat_enabled?, do: Config.get([:chat, :enabled])
 
-  defp streamer_child(env) when env in [:test, :benchmark], do: []
+  defp dont_run_in_test(env) when env in [:test, :benchmark], do: []
 
-  defp streamer_child(_) do
+  defp dont_run_in_test(_) do
     [
       {Registry,
        [
          name: Pleroma.Web.Streamer.registry(),
          keys: :duplicate,
          partitions: System.schedulers_online()
-       ]}
+       ]},
+      Pleroma.Web.FedSockets.Supervisor
     ]
   end
 

lib/pleroma/application_requirements.ex

@@ -9,6 +9,9 @@ defmodule Pleroma.ApplicationRequirements do
 
   defmodule VerifyError, do: defexception([:message])
 
+  alias Pleroma.Config
+  alias Pleroma.Helpers.MediaHelper
+
   import Ecto.Query
 
   require Logger
@@ -16,7 +19,8 @@ defmodule VerifyError, do: defexception([:message])
   @spec verify!() :: :ok | VerifyError.t()
   def verify! do
     :ok
-    |> check_confirmation_accounts!
+    |> check_system_commands!()
+    |> check_confirmation_accounts!()
     |> check_migrations_applied!()
     |> check_welcome_message_config!()
     |> check_rum!()
@@ -48,7 +52,9 @@ def check_confirmation_accounts!(:ok) do
     if Pleroma.Config.get([:instance, :account_activation_required]) &&
          not Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
       Logger.error(
-        "Account activation enabled, but no Mailer settings enabled.\nPlease set config :pleroma, :instance, account_activation_required: false\nOtherwise setup and enable Mailer."
+        "Account activation enabled, but no Mailer settings enabled.\n" <>
+          "Please set config :pleroma, :instance, account_activation_required: false\n" <>
+          "Otherwise setup and enable Mailer."
       )
 
       {:error,
@@ -81,7 +87,9 @@ def check_migrations_applied!(:ok) do
               Enum.map(down_migrations, fn {:down, id, name} -> "- #{name} (#{id})\n" end)
 
             Logger.error(
-              "The following migrations were not applied:\n#{down_migrations_text}If you want to start Pleroma anyway, set\nconfig :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true"
+              "The following migrations were not applied:\n#{down_migrations_text}" <>
+                "If you want to start Pleroma anyway, set\n" <>
+                "config :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true"
             )
 
             {:error, "Unapplied Migrations detected"}
@@ -124,14 +132,22 @@ defp do_check_rum!(setting, migrate) do
     case {setting, migrate} do
       {true, false} ->
         Logger.error(
-          "Use `RUM` index is enabled, but were not applied migrations for it.\nIf you want to start Pleroma anyway, set\nconfig :pleroma, :database, rum_enabled: false\nOtherwise apply the following migrations:\n`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`"
+          "Use `RUM` index is enabled, but were not applied migrations for it.\n" <>
+            "If you want to start Pleroma anyway, set\n" <>
+            "config :pleroma, :database, rum_enabled: false\n" <>
+            "Otherwise apply the following migrations:\n" <>
+            "`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`"
         )
 
         {:error, "Unapplied RUM Migrations detected"}
 
       {false, true} ->
         Logger.error(
-          "Detected applied migrations to use `RUM` index, but `RUM` isn't enable in settings.\nIf you want to use `RUM`, set\nconfig :pleroma, :database, rum_enabled: true\nOtherwise roll `RUM` migrations back.\n`mix ecto.rollback --migrations-path priv/repo/optional_migrations/rum_indexing/`"
+          "Detected applied migrations to use `RUM` index, but `RUM` isn't enable in settings.\n" <>
+            "If you want to use `RUM`, set\n" <>
+            "config :pleroma, :database, rum_enabled: true\n" <>
+            "Otherwise roll `RUM` migrations back.\n" <>
+            "`mix ecto.rollback --migrations-path priv/repo/optional_migrations/rum_indexing/`"
         )
 
         {:error, "RUM Migrations detected"}
@@ -140,4 +156,50 @@ defp do_check_rum!(setting, migrate) do
         :ok
     end
   end
+
+  defp check_system_commands!(:ok) do
+    filter_commands_statuses = [
+      check_filter(Pleroma.Upload.Filters.Exiftool, "exiftool"),
+      check_filter(Pleroma.Upload.Filters.Mogrify, "mogrify"),
+      check_filter(Pleroma.Upload.Filters.Mogrifun, "mogrify")
+    ]
+
+    preview_proxy_commands_status =
+      if !Config.get([:media_preview_proxy, :enabled]) or
+           MediaHelper.missing_dependencies() == [] do
+        true
+      else
+        Logger.error(
+          "The following dependencies required by Media preview proxy " <>
+            "(which is currently enabled) are not installed: " <>
+            inspect(MediaHelper.missing_dependencies())
+        )
+
+        false
+      end
+
+    if Enum.all?([preview_proxy_commands_status | filter_commands_statuses], & &1) do
+      :ok
+    else
+      {:error,
+       "System commands missing. Check logs and see `docs/installation` for more details."}
+    end
+  end
+
+  defp check_system_commands!(result), do: result
+
+  defp check_filter(filter, command_required) do
+    filters = Config.get([Pleroma.Upload, :filters])
+
+    if filter in filters and not Pleroma.Utils.command_available?(command_required) do
+      Logger.error(
+        "#{filter} is specified in list of Pleroma.Upload filters, but the " <>
+          "#{command_required} command is not found"
+      )
+
+      false
+    else
+      true
+    end
+  end
 end

lib/pleroma/chat.ex

@@ -6,7 +6,9 @@ defmodule Pleroma.Chat do
   use Ecto.Schema
 
   import Ecto.Changeset
+  import Ecto.Query
 
+  alias Pleroma.Chat
   alias Pleroma.Repo
   alias Pleroma.User
 
@@ -16,6 +18,7 @@ defmodule Pleroma.Chat do
   It is a helper only, to make it easy to display a list of chats with other people, ordered by last bump. The actual messages are retrieved by querying the recipients of the ChatMessages.
   """
 
+  @type t :: %__MODULE__{}
   @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
 
   schema "chats" do
@@ -39,16 +42,28 @@ def changeset(struct, params) do
     |> unique_constraint(:user_id, name: :chats_user_id_recipient_index)
   end
 
+  @spec get_by_user_and_id(User.t(), FlakeId.Ecto.CompatType.t()) ::
+          {:ok, t()} | {:error, :not_found}
+  def get_by_user_and_id(%User{id: user_id}, id) do
+    from(c in __MODULE__,
+      where: c.id == ^id,
+      where: c.user_id == ^user_id
+    )
+    |> Repo.find_resource()
+  end
+
+  @spec get_by_id(FlakeId.Ecto.CompatType.t()) :: t() | nil
   def get_by_id(id) do
-    __MODULE__
+    Repo.get(__MODULE__, id)
-    |> Repo.get(id)
   end
 
+  @spec get(FlakeId.Ecto.CompatType.t(), String.t()) :: t() | nil
   def get(user_id, recipient) do
-    __MODULE__
+    Repo.get_by(__MODULE__, user_id: user_id, recipient: recipient)
-    |> Repo.get_by(user_id: user_id, recipient: recipient)
   end
 
+  @spec get_or_create(FlakeId.Ecto.CompatType.t(), String.t()) ::
+          {:ok, t()} | {:error, Ecto.Changeset.t()}
   def get_or_create(user_id, recipient) do
     %__MODULE__{}
     |> changeset(%{user_id: user_id, recipient: recipient})
@@ -60,6 +75,8 @@ def get_or_create(user_id, recipient) do
     )
   end
 
+  @spec bump_or_create(FlakeId.Ecto.CompatType.t(), String.t()) ::
+          {:ok, t()} | {:error, Ecto.Changeset.t()}
   def bump_or_create(user_id, recipient) do
     %__MODULE__{}
     |> changeset(%{user_id: user_id, recipient: recipient})
@@ -69,4 +86,12 @@ def bump_or_create(user_id, recipient) do
       conflict_target: [:user_id, :recipient]
     )
   end
+
+  @spec for_user_query(FlakeId.Ecto.CompatType.t()) :: Ecto.Query.t()
+  def for_user_query(user_id) do
+    from(c in Chat,
+      where: c.user_id == ^user_id,
+      order_by: [desc: c.updated_at]
+    )
+  end
 end

lib/pleroma/config.ex

@@ -81,6 +81,16 @@ def delete(key) do
     Application.delete_env(:pleroma, key)
   end
 
+  def restrict_unauthenticated_access?(resource, kind) do
+    setting = get([:restrict_unauthenticated, resource, kind])
+
+    if setting in [nil, :if_instance_is_private] do
+      !get!([:instance, :public])
+    else
+      setting
+    end
+  end
+
   def oauth_consumer_strategies, do: get([:auth, :oauth_consumer_strategies], [])
 
   def oauth_consumer_enabled?, do: oauth_consumer_strategies() != []

lib/pleroma/config/deprecation_warnings.ex

@@ -8,7 +8,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
   require Logger
   alias Pleroma.Config
 
-  @type config_namespace() :: [atom()]
+  @type config_namespace() :: atom() | [atom()]
   @type config_map() :: {config_namespace(), config_namespace(), String.t()}
 
   @mrf_config_map [
@@ -26,36 +26,25 @@ def check_hellthread_threshold do
       !!!DEPRECATION WARNING!!!
       You are using the old configuration mechanism for the hellthread filter. Please check config.md.
       """)
-    end
-  end
 
-  def mrf_user_allowlist do
+      :error
-    config = Config.get(:mrf_user_allowlist)
+    else
-
+      :ok
-    if config && Enum.any?(config, fn {k, _} -> is_atom(k) end) do
-      rewritten =
-        Enum.reduce(Config.get(:mrf_user_allowlist), Map.new(), fn {k, v}, acc ->
-          Map.put(acc, to_string(k), v)
-        end)
-
-      Config.put(:mrf_user_allowlist, rewritten)
-
-      Logger.error("""
-      !!!DEPRECATION WARNING!!!
-      As of Pleroma 2.0.7, the `mrf_user_allowlist` setting changed of format.
-      Pleroma 2.1 will remove support for the old format. Please change your configuration to match this:
-
-      config :pleroma, :mrf_user_allowlist, #{inspect(rewritten, pretty: true)}
-      """)
     end
   end
 
   def warn do
-    check_hellthread_threshold()
+    with :ok <- check_hellthread_threshold(),
-    mrf_user_allowlist()
+         :ok <- check_old_mrf_config(),
-    check_old_mrf_config()
+         :ok <- check_media_proxy_whitelist_config(),
-    check_media_proxy_whitelist_config()
+         :ok <- check_welcome_message_config(),
-    check_welcome_message_config()
+         :ok <- check_gun_pool_options(),
+         :ok <- check_activity_expiration_config() do
+      :ok
+    else
+      _ ->
+        :error
+    end
   end
 
   def check_welcome_message_config do
@@ -68,10 +57,14 @@ def check_welcome_message_config do
     if use_old_config do
       Logger.error("""
       !!!DEPRECATION WARNING!!!
-      Your config is using the old namespace for Welcome messages configuration. You need to change to the new namespace:
+      Your config is using the old namespace for Welcome messages configuration. You need to convert to the new namespace. e.g.,
-      \n* `config :pleroma, :instance, welcome_user_nickname` is now `config :pleroma, :welcome, :direct_message, :sender_nickname`
+      \n* `config :pleroma, :instance, welcome_user_nickname` and `config :pleroma, :instance, welcome_message` are now equal to:
-      \n* `config :pleroma, :instance, welcome_message` is now `config :pleroma, :welcome, :direct_message, :message`
+      \n* `config :pleroma, :welcome, direct_message: [enabled: true, sender_nickname: "NICKNAME", message: "Your welcome message"]`"
       """)
+
+      :error
+    else
+      :ok
     end
   end
 
@@ -99,8 +92,11 @@ def move_namespace_and_warn(config_map, warning_preface) do
           end
       end)
 
-    if warning != "" do
+    if warning == "" do
+      :ok
+    else
       Logger.warn(warning_preface <> warning)
+      :error
     end
   end
 
@@ -113,6 +109,71 @@ def check_media_proxy_whitelist_config do
       !!!DEPRECATION WARNING!!!
       Your config is using old format (only domain) for MediaProxy whitelist option. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
       """)
+
+      :error
+    else
+      :ok
     end
   end
+
+  def check_gun_pool_options do
+    pool_config = Config.get(:connections_pool)
+
+    if timeout = pool_config[:await_up_timeout] do
+      Logger.warn("""
+      !!!DEPRECATION WARNING!!!
+      Your config is using old setting `config :pleroma, :connections_pool, await_up_timeout`. Please change to `config :pleroma, :connections_pool, connect_timeout` to ensure compatibility with future releases.
+      """)
+
+      Config.put(:connections_pool, Keyword.put_new(pool_config, :connect_timeout, timeout))
+    end
+
+    pools_configs = Config.get(:pools)
+
+    warning_preface = """
+    !!!DEPRECATION WARNING!!!
+    Your config is using old setting name `timeout` instead of `recv_timeout` in pool settings. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
+    """
+
+    updated_config =
+      Enum.reduce(pools_configs, [], fn {pool_name, config}, acc ->
+        if timeout = config[:timeout] do
+          Keyword.put(acc, pool_name, Keyword.put_new(config, :recv_timeout, timeout))
+        else
+          acc
+        end
+      end)
+
+    if updated_config != [] do
+      pool_warnings =
+        updated_config
+        |> Keyword.keys()
+        |> Enum.map(fn pool_name ->
+          "\n* `:timeout` options in #{pool_name} pool is now `:recv_timeout`"
+        end)
+
+      Logger.warn(Enum.join([warning_preface | pool_warnings]))
+
+      Config.put(:pools, updated_config)
+      :error
+    else
+      :ok
+    end
+  end
+
+  @spec check_activity_expiration_config() :: :ok | nil
+  def check_activity_expiration_config do
+    warning_preface = """
+    !!!DEPRECATION WARNING!!!
+      Your config is using old namespace for activity expiration configuration. Setting should work for now, but you are advised to change to new namespace to prevent possible issues later:
+    """
+
+    move_namespace_and_warn(
+      [
+        {Pleroma.ActivityExpiration, Pleroma.Workers.PurgeExpiredActivity,
+         "\n* `config :pleroma, Pleroma.ActivityExpiration` is now `config :pleroma, Pleroma.Workers.PurgeExpiredActivity`"}
+      ],
+      warning_preface
+    )
+  end
 end

lib/pleroma/config/oban.ex

@@ -0,0 +1,34 @@
+defmodule Pleroma.Config.Oban do
+  require Logger
+
+  def warn do
+    oban_config = Pleroma.Config.get(Oban)
+
+    crontab =
+      [
+        Pleroma.Workers.Cron.StatsWorker,
+        Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker,
+        Pleroma.Workers.Cron.ClearOauthTokenWorker
+      ]
+      |> Enum.reduce(oban_config[:crontab], fn removed_worker, acc ->
+        with acc when is_list(acc) <- acc,
+             setting when is_tuple(setting) <-
+               Enum.find(acc, fn {_, worker} -> worker == removed_worker end) do
+          """
+          !!!OBAN CONFIG WARNING!!!
+          You are using old workers in Oban crontab settings, which were removed.
+          Please, remove setting from crontab in your config file (prod.secret.exs): #{
+            inspect(setting)
+          }
+          """
+          |> Logger.warn()
+
+          List.delete(acc, setting)
+        else
+          _ -> acc
+        end
+      end)
+
+    Pleroma.Config.put(Oban, Keyword.put(oban_config, :crontab, crontab))
+  end
+end

lib/pleroma/ecto_type/activity_pub/object_validators/emoji.ex

@@ -0,0 +1,34 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Emoji do
+  use Ecto.Type
+
+  def type, do: :map
+
+  def cast(data) when is_map(data) do
+    has_invalid_emoji? =
+      Enum.find(data, fn
+        {name, uri} when is_binary(name) and is_binary(uri) ->
+          # based on ObjectValidators.Uri.cast()
+          case URI.parse(uri) do
+            %URI{host: nil} -> true
+            %URI{host: ""} -> true
+            %URI{scheme: scheme} when scheme in ["https", "http"] -> false
+            _ -> true
+          end
+
+        {_name, _uri} ->
+          true
+      end)
+
+    if has_invalid_emoji?, do: :error, else: {:ok, data}
+  end
+
+  def cast(_data), do: :error
+
+  def dump(data), do: {:ok, data}
+
+  def load(data), do: {:ok, data}
+end

lib/pleroma/emails/mailer.ex

@@ -35,6 +35,11 @@ def perform(:deliver_async, email, config), do: deliver(email, config)
   def deliver(email, config \\ [])
 
   def deliver(email, config) do
+    # temporary hackney fix until hackney max_connections bug is fixed
+    # https://git.pleroma.social/pleroma/pleroma/-/issues/2101
+    email =
+      Swoosh.Email.put_private(email, :hackney_options, ssl_options: [versions: [:"tlsv1.2"]])
+
     case enabled?() do
       true -> Swoosh.Mailer.deliver(email, parse_config(config))
       false -> {:error, :deliveries_disabled}

lib/pleroma/emails/user_email.ex

@@ -107,25 +107,34 @@ def digest_email(user) do
       |> Enum.filter(&(&1.activity.data["type"] == "Create"))
       |> Enum.map(fn notification ->
         object = Pleroma.Object.normalize(notification.activity)
-        object = update_in(object.data["content"], &format_links/1)
 
-        %{
+        if not is_nil(object) do
-          data: notification,
+          object = update_in(object.data["content"], &format_links/1)
-          object: object,
+
-          from: User.get_by_ap_id(notification.activity.actor)
+          %{
-        }
+            data: notification,
+            object: object,
+            from: User.get_by_ap_id(notification.activity.actor)
+          }
+        end
       end)
+      |> Enum.filter(& &1)
 
     followers =
       notifications
       |> Enum.filter(&(&1.activity.data["type"] == "Follow"))
       |> Enum.map(fn notification ->
-        %{
+        from = User.get_by_ap_id(notification.activity.actor)
-          data: notification,
+
-          object: Pleroma.Object.normalize(notification.activity),
+        if not is_nil(from) do
-          from: User.get_by_ap_id(notification.activity.actor)
+          %{
-        }
+            data: notification,
+            object: Pleroma.Object.normalize(notification.activity),
+            from: User.get_by_ap_id(notification.activity.actor)
+          }
+        end
       end)
+      |> Enum.filter(& &1)
 
     unless Enum.empty?(mentions) do
       styling = Config.get([__MODULE__, :styling])

lib/pleroma/emoji.ex

@@ -56,6 +56,9 @@ def get(name) do
     end
   end
 
+  @spec exist?(String.t()) :: boolean()
+  def exist?(name), do: not is_nil(get(name))
+
   @doc "Returns all the emojos!!"
   @spec get_all() :: list({String.t(), String.t(), String.t()})
   def get_all do

lib/pleroma/emoji/pack.ex

@@ -17,6 +17,7 @@ defmodule Pleroma.Emoji.Pack do
         }
 
   alias Pleroma.Emoji
+  alias Pleroma.Emoji.Pack
 
   @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
   def create(name) do
@@ -64,24 +65,93 @@ def delete(name) do
     end
   end
 
-  @spec add_file(String.t(), String.t(), Path.t(), Plug.Upload.t() | String.t()) ::
+  @spec unpack_zip_emojies(list(tuple())) :: list(map())
-          {:ok, t()} | {:error, File.posix() | atom()}
+  defp unpack_zip_emojies(zip_files) do
-  def add_file(name, shortcode, filename, file) do
+    Enum.reduce(zip_files, [], fn
-    with :ok <- validate_not_empty([name, shortcode, filename]),
+      {_, path, s, _, _, _}, acc when elem(s, 2) == :regular ->
+        with(
+          filename <- Path.basename(path),
+          shortcode <- Path.basename(filename, Path.extname(filename)),
+          false <- Emoji.exist?(shortcode)
+        ) do
+          [%{path: path, filename: path, shortcode: shortcode} | acc]
+        else
+          _ -> acc
+        end
+
+      _, acc ->
+        acc
+    end)
+  end
+
+  @spec add_file(t(), String.t(), Path.t(), Plug.Upload.t()) ::
+          {:ok, t()}
+          | {:error, File.posix() | atom()}
+  def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"} = file) do
+    with {:ok, zip_files} <- :zip.table(to_charlist(file.path)),
+         [_ | _] = emojies <- unpack_zip_emojies(zip_files),
+         {:ok, tmp_dir} <- Pleroma.Utils.tmp_dir("emoji") do
+      try do
+        {:ok, _emoji_files} =
+          :zip.unzip(
+            to_charlist(file.path),
+            [{:file_list, Enum.map(emojies, & &1[:path])}, {:cwd, tmp_dir}]
+          )
+
+        {_, updated_pack} =
+          Enum.map_reduce(emojies, pack, fn item, emoji_pack ->
+            emoji_file = %Plug.Upload{
+              filename: item[:filename],
+              path: Path.join(tmp_dir, item[:path])
+            }
+
+            {:ok, updated_pack} =
+              do_add_file(
+                emoji_pack,
+                item[:shortcode],
+                to_string(item[:filename]),
+                emoji_file
+              )
+
+            {item, updated_pack}
+          end)
+
+        Emoji.reload()
+
+        {:ok, updated_pack}
+      after
+        File.rm_rf(tmp_dir)
+      end
+    else
+      {:error, _} = error ->
+        error
+
+      _ ->
+        {:ok, pack}
+    end
+  end
+
+  def add_file(%Pack{} = pack, shortcode, filename, %Plug.Upload{} = file) do
+    with :ok <- validate_not_empty([shortcode, filename]),
          :ok <- validate_emoji_not_exists(shortcode),
-         {:ok, pack} <- load_pack(name),
+         {:ok, updated_pack} <- do_add_file(pack, shortcode, filename, file) do
-         :ok <- save_file(file, pack, filename),
-         {:ok, updated_pack} <- pack |> put_emoji(shortcode, filename) |> save_pack() do
       Emoji.reload()
       {:ok, updated_pack}
     end
   end
 
-  @spec delete_file(String.t(), String.t()) ::
+  defp do_add_file(pack, shortcode, filename, file) do
+    with :ok <- save_file(file, pack, filename) do
+      pack
+      |> put_emoji(shortcode, filename)
+      |> save_pack()
+    end
+  end
+
+  @spec delete_file(t(), String.t()) ::
           {:ok, t()} | {:error, File.posix() | atom()}
-  def delete_file(name, shortcode) do
+  def delete_file(%Pack{} = pack, shortcode) do
-    with :ok <- validate_not_empty([name, shortcode]),
+    with :ok <- validate_not_empty([shortcode]),
-         {:ok, pack} <- load_pack(name),
          :ok <- remove_file(pack, shortcode),
          {:ok, updated_pack} <- pack |> delete_emoji(shortcode) |> save_pack() do
       Emoji.reload()
@@ -89,11 +159,10 @@ def delete_file(name, shortcode) do
     end
   end
 
-  @spec update_file(String.t(), String.t(), String.t(), String.t(), boolean()) ::
+  @spec update_file(t(), String.t(), String.t(), String.t(), boolean()) ::
           {:ok, t()} | {:error, File.posix() | atom()}
-  def update_file(name, shortcode, new_shortcode, new_filename, force) do
+  def update_file(%Pack{} = pack, shortcode, new_shortcode, new_filename, force) do
-    with :ok <- validate_not_empty([name, shortcode, new_shortcode, new_filename]),
+    with :ok <- validate_not_empty([shortcode, new_shortcode, new_filename]),
-         {:ok, pack} <- load_pack(name),
          {:ok, filename} <- get_filename(pack, shortcode),
          :ok <- validate_emoji_not_exists(new_shortcode, force),
          :ok <- rename_file(pack, filename, new_filename),
@@ -129,13 +198,13 @@ def import_from_filesystem do
     end
   end
 
-  @spec list_remote(String.t()) :: {:ok, map()} | {:error, atom()}
+  @spec list_remote(keyword()) :: {:ok, map()} | {:error, atom()}
-  def list_remote(url) do
+  def list_remote(opts) do
-    uri = url |> String.trim() |> URI.parse()
+    uri = opts[:url] |> String.trim() |> URI.parse()
 
     with :ok <- validate_shareable_packs_available(uri) do
       uri
-      |> URI.merge("/api/pleroma/emoji/packs")
+      |> URI.merge("/api/pleroma/emoji/packs?page=#{opts[:page]}&page_size=#{opts[:page_size]}")
       |> http_get()
     end
   end
@@ -175,7 +244,8 @@ def download(name, url, as) do
     uri = url |> String.trim() |> URI.parse()
 
     with :ok <- validate_shareable_packs_available(uri),
-         {:ok, remote_pack} <- uri |> URI.merge("/api/pleroma/emoji/packs/#{name}") |> http_get(),
+         {:ok, remote_pack} <-
+           uri |> URI.merge("/api/pleroma/emoji/pack?name=#{name}") |> http_get(),
          {:ok, %{sha: sha, url: url} = pack_info} <- fetch_pack_info(remote_pack, uri, name),
          {:ok, archive} <- download_archive(url, sha),
          pack <- copy_as(remote_pack, as || name),
@@ -243,9 +313,10 @@ defp validate_emoji_not_exists(shortcode, force \\ false)
   defp validate_emoji_not_exists(_shortcode, true), do: :ok
 
   defp validate_emoji_not_exists(shortcode, _) do
-    case Emoji.get(shortcode) do
+    if Emoji.exist?(shortcode) do
-      nil -> :ok
+      {:error, :already_exists}
-      _ -> {:error, :already_exists}
+    else
+      :ok
     end
   end
 
@@ -386,25 +457,18 @@ defp validate_not_empty(list) do
     end
   end
 
-  defp save_file(file, pack, filename) do
+  defp save_file(%Plug.Upload{path: upload_path}, pack, filename) do
     file_path = Path.join(pack.path, filename)
     create_subdirs(file_path)
 
-    case file do
+    with {:ok, _} <- File.copy(upload_path, file_path) do
-      %Plug.Upload{path: upload_path} ->
+      :ok
-        # Copy the uploaded file from the temporary directory
-        with {:ok, _} <- File.copy(upload_path, file_path), do: :ok
-
-      url when is_binary(url) ->
-        # Download and write the file
-        file_contents = Tesla.get!(url).body
-        File.write(file_path, file_contents)
     end
   end
 
   defp put_emoji(pack, shortcode, filename) do
     files = Map.put(pack.files, shortcode, filename)
-    %{pack | files: files}
+    %{pack | files: files, files_count: length(Map.keys(files))}
   end
 
   defp delete_emoji(pack, shortcode) do
@@ -460,7 +524,7 @@ defp get_filename(pack, shortcode) do
   defp http_get(%URI{} = url), do: url |> to_string() |> http_get()
 
   defp http_get(url) do
-    with {:ok, %{body: body}} <- url |> Pleroma.HTTP.get() do
+    with {:ok, %{body: body}} <- Pleroma.HTTP.get(url, [], pool: :default) do
       Jason.decode(body)
     end
   end
@@ -509,7 +573,7 @@ defp fetch_pack_info(remote_pack, uri, name) do
         {:ok,
          %{
            sha: sha,
-           url: URI.merge(uri, "/api/pleroma/emoji/packs/#{name}/archive") |> to_string()
+           url: URI.merge(uri, "/api/pleroma/emoji/packs/archive?name=#{name}") |> to_string()
          }}
 
       %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) ->

lib/pleroma/following_relationship.ex

@@ -264,4 +264,12 @@ defp validate_following_id_follower_id_inequality(%Changeset{} = changeset) do
       end
     end)
   end
+
+  @spec following_ap_ids(User.t()) :: [String.t()]
+  def following_ap_ids(%User{} = user) do
+    user
+    |> following_query()
+    |> select([r, u], u.ap_id)
+    |> Repo.all()
+  end
 end

lib/pleroma/gun/conn.ex

@@ -13,7 +13,7 @@ def open(%URI{} = uri, opts) do
     opts =
       opts
       |> Enum.into(%{})
-      |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
+      |> Map.put_new(:connect_timeout, pool_opts[:connect_timeout] || 5_000)
       |> Map.put_new(:supervise, false)
       |> maybe_add_tls_opts(uri)
 
@@ -50,10 +50,10 @@ defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do
 
     with open_opts <- Map.delete(opts, :tls_opts),
          {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]),
+         {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]),
          stream <- Gun.connect(conn, connect_opts),
          {:response, :fin, 200, _} <- Gun.await(conn, stream) do
-      {:ok, conn}
+      {:ok, conn, protocol}
     else
       error ->
         Logger.warn(
@@ -88,8 +88,8 @@ defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do
       |> Map.put(:socks_opts, socks_opts)
 
     with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
+         {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
-      {:ok, conn}
+      {:ok, conn, protocol}
     else
       error ->
         Logger.warn(
@@ -106,8 +106,8 @@ defp do_open(%URI{host: host, port: port} = uri, opts) do
     host = Pleroma.HTTP.AdapterHelper.parse_host(host)
 
     with {:ok, conn} <- Gun.open(host, port, opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
+         {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
-      {:ok, conn}
+      {:ok, conn, protocol}
     else
       error ->
         Logger.warn(

lib/pleroma/gun/connection_pool/worker.ex

@@ -15,7 +15,7 @@ def init([_key, _uri, _opts, _client_pid] = opts) do
 
   @impl true
   def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do
-    with {:ok, conn_pid} <- Gun.Conn.open(uri, opts),
+    with {:ok, conn_pid, protocol} <- Gun.Conn.open(uri, opts),
          Process.link(conn_pid) do
       time = :erlang.monotonic_time(:millisecond)
 
@@ -27,8 +27,12 @@ def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do
       send(client_pid, {:conn_pid, conn_pid})
 
       {:noreply,
-       %{key: key, timer: nil, client_monitors: %{client_pid => Process.monitor(client_pid)}},
+       %{
-       :hibernate}
+         key: key,
+         timer: nil,
+         client_monitors: %{client_pid => Process.monitor(client_pid)},
+         protocol: protocol
+       }, :hibernate}
     else
       err ->
         {:stop, {:shutdown, err}, nil}
@@ -53,14 +57,20 @@ def handle_cast({:remove_client, client_pid}, state) do
   end
 
   @impl true
-  def handle_call(:add_client, {client_pid, _}, %{key: key} = state) do
+  def handle_call(:add_client, {client_pid, _}, %{key: key, protocol: protocol} = state) do
     time = :erlang.monotonic_time(:millisecond)
 
-    {{conn_pid, _, _, _}, _} =
+    {{conn_pid, used_by, _, _}, _} =
       Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} ->
         {conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time}
       end)
 
+    :telemetry.execute(
+      [:pleroma, :connection_pool, :client, :add],
+      %{client_pid: client_pid, clients: used_by},
+      %{key: state.key, protocol: protocol}
+    )
+
     state =
       if state.timer != nil do
         Process.cancel_timer(state[:timer])
@@ -83,7 +93,8 @@ def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do
       end)
 
     {ref, state} = pop_in(state.client_monitors[client_pid])
-    Process.demonitor(ref)
+
+    Process.demonitor(ref, [:flush])
 
     timer =
       if used_by == [] do
@@ -103,22 +114,27 @@ def handle_info(:idle_close, state) do
     {:stop, :normal, state}
   end
 
+  @impl true
+  def handle_info({:gun_up, _pid, _protocol}, state) do
+    {:noreply, state, :hibernate}
+  end
+
   # Gracefully shutdown if the connection got closed without any streams left
   @impl true
   def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do
     {:stop, :normal, state}
   end
 
-  # Otherwise, shutdown with an error
+  # Otherwise, wait for retry
   @impl true
-  def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams} = down_message, state) do
+  def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams}, state) do
-    {:stop, {:error, down_message}, state}
+    {:noreply, state, :hibernate}
   end
 
   @impl true
   def handle_info({:DOWN, _ref, :process, pid, reason}, state) do
     :telemetry.execute(
-      [:pleroma, :connection_pool, :client_death],
+      [:pleroma, :connection_pool, :client, :dead],
       %{client_pid: pid, reason: reason},
       %{key: state.key}
     )

lib/pleroma/helpers/media_helper.ex

@@ -0,0 +1,162 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Helpers.MediaHelper do
+  @moduledoc """
+  Handles common media-related operations.
+  """
+
+  alias Pleroma.HTTP
+
+  require Logger
+
+  def missing_dependencies do
+    Enum.reduce([imagemagick: "convert", ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc ->
+      if Pleroma.Utils.command_available?(executable) do
+        acc
+      else
+        [sym | acc]
+      end
+    end)
+  end
+
+  def image_resize(url, options) do
+    with executable when is_binary(executable) <- System.find_executable("convert"),
+         {:ok, args} <- prepare_image_resize_args(options),
+         {:ok, env} <- HTTP.get(url, [], pool: :media),
+         {:ok, fifo_path} <- mkfifo() do
+      args = List.flatten([fifo_path, args])
+      run_fifo(fifo_path, env, executable, args)
+    else
+      nil -> {:error, {:convert, :command_not_found}}
+      {:error, _} = error -> error
+    end
+  end
+
+  defp prepare_image_resize_args(
+         %{max_width: max_width, max_height: max_height, format: "png"} = options
+       ) do
+    quality = options[:quality] || 85
+    resize = Enum.join([max_width, "x", max_height, ">"])
+
+    args = [
+      "-resize",
+      resize,
+      "-quality",
+      to_string(quality),
+      "png:-"
+    ]
+
+    {:ok, args}
+  end
+
+  defp prepare_image_resize_args(%{max_width: max_width, max_height: max_height} = options) do
+    quality = options[:quality] || 85
+    resize = Enum.join([max_width, "x", max_height, ">"])
+
+    args = [
+      "-interlace",
+      "Plane",
+      "-resize",
+      resize,
+      "-quality",
+      to_string(quality),
+      "jpg:-"
+    ]
+
+    {:ok, args}
+  end
+
+  defp prepare_image_resize_args(_), do: {:error, :missing_options}
+
+  # Note: video thumbnail is intentionally not resized (always has original dimensions)
+  def video_framegrab(url) do
+    with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
+         {:ok, env} <- HTTP.get(url, [], pool: :media),
+         {:ok, fifo_path} <- mkfifo(),
+         args = [
+           "-y",
+           "-i",
+           fifo_path,
+           "-vframes",
+           "1",
+           "-f",
+           "mjpeg",
+           "-loglevel",
+           "error",
+           "-"
+         ] do
+      run_fifo(fifo_path, env, executable, args)
+    else
+      nil -> {:error, {:ffmpeg, :command_not_found}}
+      {:error, _} = error -> error
+    end
+  end
+
+  defp run_fifo(fifo_path, env, executable, args) do
+    pid =
+      Port.open({:spawn_executable, executable}, [
+        :use_stdio,
+        :stream,
+        :exit_status,
+        :binary,
+        args: args
+      ])
+
+    fifo = Port.open(to_charlist(fifo_path), [:eof, :binary, :stream, :out])
+    fix = Pleroma.Helpers.QtFastStart.fix(env.body)
+    true = Port.command(fifo, fix)
+    :erlang.port_close(fifo)
+    loop_recv(pid)
+  after
+    File.rm(fifo_path)
+  end
+
+  defp mkfifo do
+    path = Path.join(System.tmp_dir!(), "pleroma-media-preview-pipe-#{Ecto.UUID.generate()}")
+
+    case System.cmd("mkfifo", [path]) do
+      {_, 0} ->
+        spawn(fifo_guard(path))
+        {:ok, path}
+
+      {_, err} ->
+        {:error, {:fifo_failed, err}}
+    end
+  end
+
+  defp fifo_guard(path) do
+    pid = self()
+
+    fn ->
+      ref = Process.monitor(pid)
+
+      receive do
+        {:DOWN, ^ref, :process, ^pid, _} ->
+          File.rm(path)
+      end
+    end
+  end
+
+  defp loop_recv(pid) do
+    loop_recv(pid, <<>>)
+  end
+
+  defp loop_recv(pid, acc) do
+    receive do
+      {^pid, {:data, data}} ->
+        loop_recv(pid, acc <> data)
+
+      {^pid, {:exit_status, 0}} ->
+        {:ok, acc}
+
+      {^pid, {:exit_status, status}} ->
+        {:error, status}
+    after
+      5000 ->
+        :erlang.port_close(pid)
+        {:error, :timeout}
+    end
+  end
+end

lib/pleroma/helpers/qt_fast_start.ex

@@ -0,0 +1,131 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Helpers.QtFastStart do
+  @moduledoc """
+  (WIP) Converts a "slow start" (data before metadatas) mov/mp4 file to a "fast start" one (metadatas before data).
+  """
+
+  # TODO: Cleanup and optimizations
+  # Inspirations: https://www.ffmpeg.org/doxygen/3.4/qt-faststart_8c_source.html
+  #               https://github.com/danielgtaylor/qtfaststart/blob/master/qtfaststart/processor.py
+  #               ISO/IEC 14496-12:2015, ISO/IEC 15444-12:2015
+  #               Paracetamol
+
+  def fix(<<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70, _::bits>> = binary) do
+    index = fix(binary, 0, nil, nil, [])
+
+    case index do
+      :abort -> binary
+      [{"ftyp", _, _, _, _}, {"mdat", _, _, _, _} | _] -> faststart(index)
+      [{"ftyp", _, _, _, _}, {"free", _, _, _, _}, {"mdat", _, _, _, _} | _] -> faststart(index)
+      _ -> binary
+    end
+  end
+
+  def fix(binary) do
+    binary
+  end
+
+  # MOOV have been seen before MDAT- abort
+  defp fix(<<_::bits>>, _, true, false, _) do
+    :abort
+  end
+
+  defp fix(
+         <<size::integer-big-size(32), fourcc::bits-size(32), rest::bits>>,
+         pos,
+         got_moov,
+         got_mdat,
+         acc
+       ) do
+    full_size = (size - 8) * 8
+    <<data::bits-size(full_size), rest::bits>> = rest
+
+    acc = [
+      {fourcc, pos, pos + size, size,
+       <<size::integer-big-size(32), fourcc::bits-size(32), data::bits>>}
+      | acc
+    ]
+
+    fix(rest, pos + size, got_moov || fourcc == "moov", got_mdat || fourcc == "mdat", acc)
+  end
+
+  defp fix(<<>>, _pos, _, _, acc) do
+    :lists.reverse(acc)
+  end
+
+  defp faststart(index) do
+    {{_ftyp, _, _, _, ftyp}, index} = List.keytake(index, "ftyp", 0)
+
+    # Skip re-writing the free fourcc as it's kind of useless.
+    # Why stream useless bytes when you can do without?
+    {free_size, index} =
+      case List.keytake(index, "free", 0) do
+        {{_, _, _, size, _}, index} -> {size, index}
+        _ -> {0, index}
+      end
+
+    {{_moov, _, _, moov_size, moov}, index} = List.keytake(index, "moov", 0)
+    offset = -free_size + moov_size
+    rest = for {_, _, _, _, data} <- index, do: data, into: []
+    <<moov_head::bits-size(64), moov_data::bits>> = moov
+    [ftyp, moov_head, fix_moov(moov_data, offset, []), rest]
+  end
+
+  defp fix_moov(
+         <<size::integer-big-size(32), fourcc::bits-size(32), rest::bits>>,
+         offset,
+         acc
+       ) do
+    full_size = (size - 8) * 8
+    <<data::bits-size(full_size), rest::bits>> = rest
+
+    data =
+      cond do
+        fourcc in ["trak", "mdia", "minf", "stbl"] ->
+          # Theses contains sto or co64 part
+          [<<size::integer-big-size(32), fourcc::bits-size(32)>>, fix_moov(data, offset, [])]
+
+        fourcc in ["stco", "co64"] ->
+          # fix the damn thing
+          <<version::integer-big-size(32), count::integer-big-size(32), rest::bits>> = data
+
+          entry_size =
+            case fourcc do
+              "stco" -> 32
+              "co64" -> 64
+            end
+
+          [
+            <<size::integer-big-size(32), fourcc::bits-size(32), version::integer-big-size(32),
+              count::integer-big-size(32)>>,
+            rewrite_entries(entry_size, offset, rest, [])
+          ]
+
+        true ->
+          [<<size::integer-big-size(32), fourcc::bits-size(32)>>, data]
+      end
+
+    acc = [acc | data]
+    fix_moov(rest, offset, acc)
+  end
+
+  defp fix_moov(<<>>, _, acc), do: acc
+
+  for size <- [32, 64] do
+    defp rewrite_entries(
+           unquote(size),
+           offset,
+           <<pos::integer-big-size(unquote(size)), rest::bits>>,
+           acc
+         ) do
+      rewrite_entries(unquote(size), offset, rest, [
+        acc | <<pos + offset::integer-big-size(unquote(size))>>
+      ])
+    end
+  end
+
+  defp rewrite_entries(_, _, <<>>, acc), do: acc
+end

lib/pleroma/helpers/uri_helper.ex

@@ -3,18 +3,22 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Helpers.UriHelper do
-  def append_uri_params(uri, appended_params) do
+  def modify_uri_params(uri, overridden_params, deleted_params \\ []) do
     uri = URI.parse(uri)
-    appended_params = for {k, v} <- appended_params, into: %{}, do: {to_string(k), v}
+
-    existing_params = URI.query_decoder(uri.query || "") |> Enum.into(%{})
+    existing_params = URI.query_decoder(uri.query || "") |> Map.new()
-    updated_params_keys = Enum.uniq(Map.keys(existing_params) ++ Map.keys(appended_params))
+    overridden_params = Map.new(overridden_params, fn {k, v} -> {to_string(k), v} end)
+    deleted_params = Enum.map(deleted_params, &to_string/1)
 
     updated_params =
-      for k <- updated_params_keys, do: {k, appended_params[k] || existing_params[k]}
+      existing_params
+      |> Map.merge(overridden_params)
+      |> Map.drop(deleted_params)
 
     uri
     |> Map.put(:query, URI.encode_query(updated_params))
     |> URI.to_string()
+    |> String.replace_suffix("?", "")
   end
 
   def maybe_add_base("/" <> uri, base), do: Path.join([base, uri])

lib/pleroma/html.ex

@@ -100,20 +100,27 @@ defp generate_scrubber_signature(scrubbers) do
     end)
   end
 
-  def extract_first_external_url(_, nil), do: {:error, "No content"}
+  def extract_first_external_url_from_object(%{data: %{"content" => content}} = object)
+      when is_binary(content) do
+    unless object.data["fake"] do
+      key = "URL|#{object.id}"
 
-  def extract_first_external_url(object, content) do
+      Cachex.fetch!(:scrubber_cache, key, fn _key ->
-    key = "URL|#{object.id}"
+        {:commit, {:ok, extract_first_external_url(content)}}
+      end)
+    else
+      {:ok, extract_first_external_url(content)}
+    end
+  end
 
-    Cachex.fetch!(:scrubber_cache, key, fn _key ->
+  def extract_first_external_url_from_object(_), do: {:error, :no_content}
-      result =
-        content
-        |> Floki.parse_fragment!()
-        |> Floki.filter_out("a.mention,a.hashtag,a.attachment,a[rel~=\"tag\"]")
-        |> Floki.attribute("a", "href")
-        |> Enum.at(0)
 
-      {:commit, {:ok, result}}
+  def extract_first_external_url(content) do
-    end)
+    content
+    |> Floki.parse_fragment!()
+    |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])")
+    |> Enum.take(1)
+    |> Floki.attribute("href")
+    |> Enum.at(0)
   end
 end

lib/pleroma/http/adapter_helper.ex

@@ -6,12 +6,11 @@ defmodule Pleroma.HTTP.AdapterHelper do
   @moduledoc """
   Configure Tesla.Client with default and customized adapter options.
   """
-  @defaults [pool: :federation]
+  @defaults [pool: :federation, connect_timeout: 5_000, recv_timeout: 5_000]
 
   @type proxy_type() :: :socks4 | :socks5
   @type host() :: charlist() | :inet.ip_address()
 
-  alias Pleroma.Config
   alias Pleroma.HTTP.AdapterHelper
   require Logger
 
@@ -20,7 +19,6 @@ defmodule Pleroma.HTTP.AdapterHelper do
           | {Connection.proxy_type(), Connection.host(), pos_integer()}
 
   @callback options(keyword(), URI.t()) :: keyword()
-  @callback get_conn(URI.t(), keyword()) :: {:ok, term()} | {:error, term()}
 
   @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil
   def format_proxy(nil), do: nil
@@ -44,27 +42,10 @@ def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy)
   @spec options(URI.t(), keyword()) :: keyword()
   def options(%URI{} = uri, opts \\ []) do
     @defaults
-    |> put_timeout()
     |> Keyword.merge(opts)
     |> adapter_helper().options(uri)
   end
 
-  # For Hackney, this is the time a connection can stay idle in the pool.
-  # For Gun, this is the timeout to receive a message from Gun.
-  defp put_timeout(opts) do
-    {config_key, default} =
-      if adapter() == Tesla.Adapter.Gun do
-        {:pools, Config.get([:pools, :default, :timeout], 5_000)}
-      else
-        {:hackney_pools, 10_000}
-      end
-
-    timeout = Config.get([config_key, opts[:pool], :timeout], default)
-
-    Keyword.merge(opts, timeout: timeout)
-  end
-
-  def get_conn(uri, opts), do: adapter_helper().get_conn(uri, opts)
   defp adapter, do: Application.get_env(:tesla, :adapter)
 
   defp adapter_helper do

lib/pleroma/http/adapter_helper/gun.ex

@@ -5,57 +5,62 @@
 defmodule Pleroma.HTTP.AdapterHelper.Gun do
   @behaviour Pleroma.HTTP.AdapterHelper
 
-  alias Pleroma.Gun.ConnectionPool
+  alias Pleroma.Config
   alias Pleroma.HTTP.AdapterHelper
 
   require Logger
 
   @defaults [
-    connect_timeout: 5_000,
+    retry: 1,
-    domain_lookup_timeout: 5_000,
+    retry_timeout: 1_000
-    tls_handshake_timeout: 5_000,
-    retry: 0,
-    retry_timeout: 1000,
-    await_up_timeout: 5_000
   ]
 
+  @type pool() :: :federation | :upload | :media | :default
+
   @spec options(keyword(), URI.t()) :: keyword()
   def options(incoming_opts \\ [], %URI{} = uri) do
     proxy =
-      Pleroma.Config.get([:http, :proxy_url])
+      [:http, :proxy_url]
+      |> Config.get()
       |> AdapterHelper.format_proxy()
 
-    config_opts = Pleroma.Config.get([:http, :adapter], [])
+    config_opts = Config.get([:http, :adapter], [])
 
     @defaults
     |> Keyword.merge(config_opts)
     |> add_scheme_opts(uri)
     |> AdapterHelper.maybe_add_proxy(proxy)
     |> Keyword.merge(incoming_opts)
+    |> put_timeout()
   end
 
   defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
 
   defp add_scheme_opts(opts, %{scheme: "https"}) do
-    opts
+    Keyword.put(opts, :certificates_verification, true)
-    |> Keyword.put(:certificates_verification, true)
   end
 
-  @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()}
+  defp put_timeout(opts) do
-  def get_conn(uri, opts) do
+    {recv_timeout, opts} = Keyword.pop(opts, :recv_timeout, pool_timeout(opts[:pool]))
-    case ConnectionPool.get_conn(uri, opts) do
+    # this is the timeout to receive a message from Gun
-      {:ok, conn_pid} -> {:ok, Keyword.merge(opts, conn: conn_pid, close_conn: false)}
+    # `:timeout` key is used in Tesla
-      err -> err
+    Keyword.put(opts, :timeout, recv_timeout)
-    end
+  end
+
+  @spec pool_timeout(pool()) :: non_neg_integer()
+  def pool_timeout(pool) do
+    default = Config.get([:pools, :default, :recv_timeout], 5_000)
+
+    Config.get([:pools, pool, :recv_timeout], default)
   end
 
   @prefix Pleroma.Gun.ConnectionPool
   def limiter_setup do
-    wait = Pleroma.Config.get([:connections_pool, :connection_acquisition_wait])
+    wait = Config.get([:connections_pool, :connection_acquisition_wait])
-    retries = Pleroma.Config.get([:connections_pool, :connection_acquisition_retries])
+    retries = Config.get([:connections_pool, :connection_acquisition_retries])
 
     :pools
-    |> Pleroma.Config.get([])
+    |> Config.get([])
     |> Enum.each(fn {name, opts} ->
       max_running = Keyword.get(opts, :size, 50)
       max_waiting = Keyword.get(opts, :max_waiting, 10)
@@ -69,7 +74,6 @@ def limiter_setup do
       case result do
         :ok -> :ok
         {:error, :existing} -> :ok
-        e -> raise e
       end
     end)
 

lib/pleroma/http/adapter_helper/hackney.ex

@@ -2,11 +2,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
   @behaviour Pleroma.HTTP.AdapterHelper
 
   @defaults [
-    connect_timeout: 10_000,
-    recv_timeout: 20_000,
     follow_redirect: true,
-    force_redirect: true,
+    force_redirect: true
-    pool: :federation
   ]
 
   @spec options(keyword(), URI.t()) :: keyword()
@@ -19,11 +16,21 @@ def options(connection_opts \\ [], %URI{} = uri) do
     |> Keyword.merge(config_opts)
     |> Keyword.merge(connection_opts)
     |> add_scheme_opts(uri)
+    |> maybe_add_with_body()
     |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
   end
 
+  defp add_scheme_opts(opts, %URI{scheme: "https"}) do
+    Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1])
+  end
+
   defp add_scheme_opts(opts, _), do: opts
 
-  @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()}
+  defp maybe_add_with_body(opts) do
-  def get_conn(_uri, opts), do: {:ok, opts}
+    if opts[:max_body] do
+      Keyword.put(opts, :with_body, true)
+    else
+      opts
+    end
+  end
 end

lib/pleroma/http/ex_aws.ex

@@ -11,6 +11,8 @@ defmodule Pleroma.HTTP.ExAws do
 
   @impl true
   def request(method, url, body \\ "", headers \\ [], http_opts \\ []) do
+    http_opts = Keyword.put_new(http_opts, :pool, :upload)
+
     case HTTP.request(method, url, body, headers, http_opts) do
       {:ok, env} ->
         {:ok, %{status_code: env.status, headers: env.headers, body: env.body}}

lib/pleroma/http/http.ex

@@ -60,30 +60,23 @@ def post(url, body, headers \\ [], options \\ []),
           {:ok, Env.t()} | {:error, any()}
   def request(method, url, body, headers, options) when is_binary(url) do
     uri = URI.parse(url)
-    adapter_opts = AdapterHelper.options(uri, options[:adapter] || [])
+    adapter_opts = AdapterHelper.options(uri, options || [])
 
-    case AdapterHelper.get_conn(uri, adapter_opts) do
+    options = put_in(options[:adapter], adapter_opts)
-      {:ok, adapter_opts} ->
+    params = options[:params] || []
-        options = put_in(options[:adapter], adapter_opts)
+    request = build_request(method, headers, options, url, body, params)
-        params = options[:params] || []
-        request = build_request(method, headers, options, url, body, params)
 
-        adapter = Application.get_env(:tesla, :adapter)
+    adapter = Application.get_env(:tesla, :adapter)
 
-        client = Tesla.client(adapter_middlewares(adapter), adapter)
+    client = Tesla.client(adapter_middlewares(adapter), adapter)
 
-        maybe_limit(
+    maybe_limit(
-          fn ->
+      fn ->
-            request(client, request)
+        request(client, request)
-          end,
+      end,
-          adapter,
+      adapter,
-          adapter_opts
+      adapter_opts
-        )
+    )
-
-      # Connection release is handled in a custom FollowRedirects middleware
-      err ->
-        err
-    end
   end
 
   @spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()}
@@ -110,7 +103,7 @@ defp maybe_limit(fun, _, _) do
   end
 
   defp adapter_middlewares(Tesla.Adapter.Gun) do
-    [Pleroma.HTTP.Middleware.FollowRedirects]
+    [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool]
   end
 
   defp adapter_middlewares(_), do: []

lib/pleroma/http/tzdata.ex

@@ -11,6 +11,8 @@ defmodule Pleroma.HTTP.Tzdata do
 
   @impl true
   def get(url, headers, options) do
+    options = Keyword.put_new(options, :pool, :default)
+
     with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers, options) do
       {:ok, {env.status, env.headers, env.body}}
     end
@@ -18,6 +20,8 @@ def get(url, headers, options) do
 
   @impl true
   def head(url, headers, options) do
+    options = Keyword.put_new(options, :pool, :default)
+
     with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers, options) do
       {:ok, {env.status, env.headers}}
     end

lib/pleroma/http/web_push.ex

@@ -0,0 +1,12 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.WebPush do
+  @moduledoc false
+
+  def post(url, payload, headers) do
+    list_headers = Map.to_list(headers)
+    Pleroma.HTTP.post(url, payload, list_headers)
+  end
+end

lib/pleroma/instances/instance.ex

@@ -14,6 +14,8 @@ defmodule Pleroma.Instances.Instance do
   import Ecto.Query
   import Ecto.Changeset
 
+  require Logger
+
   schema "instances" do
     field(:host, :string)
     field(:unreachable_since, :naive_datetime_usec)
@@ -145,25 +147,32 @@ def get_or_update_favicon(%URI{host: host} = instance_uri) do
 
       favicon
     end
+  rescue
+    e ->
+      Logger.warn("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}")
+      nil
   end
 
   defp scrape_favicon(%URI{} = instance_uri) do
     try do
       with {:ok, %Tesla.Env{body: html}} <-
-             Pleroma.HTTP.get(to_string(instance_uri), [{:Accept, "text/html"}]),
+             Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], pool: :media),
-           favicon_rel <-
+           {_, [favicon_rel | _]} when is_binary(favicon_rel) <-
-             html
+             {:parse,
-             |> Floki.parse_document!()
+              html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")},
-             |> Floki.attribute("link[rel=icon]", "href")
+           {_, favicon} when is_binary(favicon) <-
-             |> List.first(),
+             {:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do
-           favicon <- URI.merge(instance_uri, favicon_rel) |> to_string(),
-           true <- is_binary(favicon) do
         favicon
       else
         _ -> nil
       end
     rescue
-      _ -> nil
+      e ->
+        Logger.warn(
+          "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}"
+        )
+
+        nil
     end
   end
 end

lib/pleroma/job_queue_monitor.ex

@@ -15,8 +15,8 @@ def start_link(_) do
 
   @impl true
   def init(state) do
-    :telemetry.attach("oban-monitor-failure", [:oban, :failure], &handle_event/4, nil)
+    :telemetry.attach("oban-monitor-failure", [:oban, :job, :exception], &handle_event/4, nil)
-    :telemetry.attach("oban-monitor-success", [:oban, :success], &handle_event/4, nil)
+    :telemetry.attach("oban-monitor-success", [:oban, :job, :stop], &handle_event/4, nil)
 
     {:ok, state}
   end
@@ -25,8 +25,11 @@ def stats do
     GenServer.call(__MODULE__, :stats)
   end
 
-  def handle_event([:oban, status], %{duration: duration}, meta, _) do
+  def handle_event([:oban, :job, event], %{duration: duration}, meta, _) do
-    GenServer.cast(__MODULE__, {:process_event, status, duration, meta})
+    GenServer.cast(
+      __MODULE__,
+      {:process_event, mapping_status(event), duration, meta}
+    )
   end
 
   @impl true
@@ -75,4 +78,7 @@ defp update_queue(queue, status, _meta, _duration) do
     |> Map.update!(:processed_jobs, &(&1 + 1))
     |> Map.update!(status, &(&1 + 1))
   end
+
+  defp mapping_status(:stop), do: :success
+  defp mapping_status(:exception), do: :failure
 end

lib/pleroma/mfa/token.ex

@@ -10,10 +10,11 @@ defmodule Pleroma.MFA.Token do
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.OAuth.Authorization
-  alias Pleroma.Web.OAuth.Token, as: OAuthToken
 
   @expires 300
 
+  @type t() :: %__MODULE__{}
+
   schema "mfa_tokens" do
     field(:token, :string)
     field(:valid_until, :naive_datetime_usec)
@@ -24,6 +25,7 @@ defmodule Pleroma.MFA.Token do
     timestamps()
   end
 
+  @spec get_by_token(String.t()) :: {:ok, t()} | {:error, :not_found}
   def get_by_token(token) do
     from(
       t in __MODULE__,
@@ -33,33 +35,40 @@ def get_by_token(token) do
     |> Repo.find_resource()
   end
 
-  def validate(token) do
+  @spec validate(String.t()) :: {:ok, t()} | {:error, :not_found} | {:error, :expired_token}
-    with {:fetch_token, {:ok, token}} <- {:fetch_token, get_by_token(token)},
+  def validate(token_str) do
-         {:expired, false} <- {:expired, is_expired?(token)} do
+    with {:ok, token} <- get_by_token(token_str),
+         false <- expired?(token) do
       {:ok, token}
-    else
-      {:expired, _} -> {:error, :expired_token}
-      {:fetch_token, _} -> {:error, :not_found}
-      error -> {:error, error}
     end
   end
 
-  def create_token(%User{} = user) do
+  defp expired?(%__MODULE__{valid_until: valid_until}) do
-    %__MODULE__{}
+    with true <- NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) > 0 do
-    |> change
+      {:error, :expired_token}
-    |> assign_user(user)
+    end
-    |> put_token
-    |> put_valid_until
-    |> Repo.insert()
   end
 
-  def create_token(user, authorization) do
+  @spec create(User.t(), Authorization.t() | nil) :: {:ok, t()} | {:error, Ecto.Changeset.t()}
+  def create(user, authorization \\ nil) do
+    with {:ok, token} <- do_create(user, authorization) do
+      Pleroma.Workers.PurgeExpiredToken.enqueue(%{
+        token_id: token.id,
+        valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC"),
+        mod: __MODULE__
+      })
+
+      {:ok, token}
+    end
+  end
+
+  defp do_create(user, authorization) do
     %__MODULE__{}
-    |> change
+    |> change()
     |> assign_user(user)
-    |> assign_authorization(authorization)
+    |> maybe_assign_authorization(authorization)
-    |> put_token
+    |> put_token()
-    |> put_valid_until
+    |> put_valid_until()
     |> Repo.insert()
   end
 
@@ -69,15 +78,19 @@ defp assign_user(changeset, user) do
     |> validate_required([:user])
   end
 
-  defp assign_authorization(changeset, authorization) do
+  defp maybe_assign_authorization(changeset, %Authorization{} = authorization) do
     changeset
     |> put_assoc(:authorization, authorization)
     |> validate_required([:authorization])
   end
 
+  defp maybe_assign_authorization(changeset, _), do: changeset
+
   defp put_token(changeset) do
+    token = Pleroma.Web.OAuth.Token.Utils.generate_token()
+
     changeset
-    |> change(%{token: OAuthToken.Utils.generate_token()})
+    |> change(%{token: token})
     |> validate_required([:token])
     |> unique_constraint(:token)
   end
@@ -89,18 +102,4 @@ defp put_valid_until(changeset) do
     |> change(%{valid_until: expires_in})
     |> validate_required([:valid_until])
   end
-
-  def is_expired?(%__MODULE__{valid_until: valid_until}) do
-    NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) > 0
-  end
-
-  def is_expired?(_), do: false
-
-  def delete_expired_tokens do
-    from(
-      q in __MODULE__,
-      where: fragment("?", q.valid_until) < ^Timex.now()
-    )
-    |> Repo.delete_all()
-  end
 end

lib/pleroma/migration_helper/notification_backfill.ex

@@ -19,13 +19,13 @@ def fill_in_notification_types do
     query
     |> Repo.chunk_stream(100)
     |> Enum.each(fn notification ->
-      type =
+      if notification.activity do
-        notification.activity
+        type = type_from_activity(notification.activity)
-        |> type_from_activity()
 
-      notification
+        notification
-      |> Ecto.Changeset.change(%{type: type})
+        |> Ecto.Changeset.change(%{type: type})
-      |> Repo.update()
+        |> Repo.update()
+      end
     end)
   end
 
@@ -72,8 +72,7 @@ defp type_from_activity(%{data: %{"type" => type}} = activity) do
         "pleroma:emoji_reaction"
 
       "Create" ->
-        activity
+        type_from_activity_object(activity)
-        |> type_from_activity_object()
 
       t ->
         raise "No notification type for activity type #{t}"

lib/pleroma/moderation_log.ex

@@ -320,6 +320,19 @@ def insert_log(%{
     |> insert_log_entry_with_message()
   end
 
+  @spec insert_log(%{actor: User, action: String.t(), subject_id: String.t()}) ::
+          {:ok, ModerationLog} | {:error, any}
+  def insert_log(%{actor: %User{} = actor, action: "chat_message_delete", subject_id: subject_id}) do
+    %ModerationLog{
+      data: %{
+        "actor" => %{"nickname" => actor.nickname},
+        "action" => "chat_message_delete",
+        "subject_id" => subject_id
+      }
+    }
+    |> insert_log_entry_with_message()
+  end
+
   @spec insert_log_entry_with_message(ModerationLog) :: {:ok, ModerationLog} | {:error, any}
   defp insert_log_entry_with_message(entry) do
     entry.data["message"]
@@ -627,6 +640,17 @@ def get_log_entry_message(%ModerationLog{
     "@#{actor_nickname} updated users: #{users_to_nicknames_string(subjects)}"
   end
 
+  @spec get_log_entry_message(ModerationLog) :: String.t()
+  def get_log_entry_message(%ModerationLog{
+        data: %{
+          "actor" => %{"nickname" => actor_nickname},
+          "action" => "chat_message_delete",
+          "subject_id" => subject_id
+        }
+      }) do
+    "@#{actor_nickname} deleted chat message ##{subject_id}"
+  end
+
   defp nicknames_to_string(nicknames) do
     nicknames
     |> Enum.map(&"@#{&1}")

lib/pleroma/notification.ex

@@ -15,6 +15,7 @@ defmodule Pleroma.Notification do
   alias Pleroma.Repo
   alias Pleroma.ThreadMute
   alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.Push
   alias Pleroma.Web.Streamer
@@ -441,6 +442,7 @@ def create_notification(%Activity{} = activity, %User{} = user, do_send \\ true)
         |> Multi.insert(:notification, %Notification{
           user_id: user.id,
           activity: activity,
+          seen: mark_as_read?(activity, user),
           type: type_from_activity(activity)
         })
         |> Marker.multi_set_last_read_id(user, "notifications")
@@ -634,6 +636,11 @@ def skip?(:filtered, activity, user) do
 
   def skip?(_, _, _), do: false
 
+  def mark_as_read?(activity, target_user) do
+    user = Activity.user_actor(activity)
+    User.mutes_user?(target_user, user) || CommonAPI.thread_muted?(target_user, activity)
+  end
+
   def for_user_and_activity(user, activity) do
     from(n in __MODULE__,
       where: n.user_id == ^user.id,
@@ -641,4 +648,16 @@ def for_user_and_activity(user, activity) do
     )
     |> Repo.one()
   end
+
+  @spec mark_context_as_read(User.t(), String.t()) :: {integer(), nil | [term()]}
+  def mark_context_as_read(%User{id: id}, context) do
+    from(
+      n in Notification,
+      join: a in assoc(n, :activity),
+      where: n.user_id == ^id,
+      where: n.seen == false,
+      where: fragment("?->>'context'", a.data) == ^context
+    )
+    |> Repo.update_all(set: [seen: true])
+  end
 end

lib/pleroma/object.ex

@@ -255,6 +255,10 @@ def increase_replies_count(ap_id) do
     end
   end
 
+  defp poll_is_multiple?(%Object{data: %{"anyOf" => [_ | _]}}), do: true
+
+  defp poll_is_multiple?(_), do: false
+
   def decrease_replies_count(ap_id) do
     Object
     |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id)))
@@ -281,10 +285,10 @@ def decrease_replies_count(ap_id) do
   def increase_vote_count(ap_id, name, actor) do
     with %Object{} = object <- Object.normalize(ap_id),
          "Question" <- object.data["type"] do
-      multiple = Map.has_key?(object.data, "anyOf")
+      key = if poll_is_multiple?(object), do: "anyOf", else: "oneOf"
 
       options =
-        (object.data["anyOf"] || object.data["oneOf"] || [])
+        object.data[key]
         |> Enum.map(fn
           %{"name" => ^name} = option ->
             Kernel.update_in(option["replies"]["totalItems"], &(&1 + 1))
@@ -296,11 +300,8 @@ def increase_vote_count(ap_id, name, actor) do
       voters = [actor | object.data["voters"] || []] |> Enum.uniq()
 
       data =
-        if multiple do
+        object.data
-          Map.put(object.data, "anyOf", options)
+        |> Map.put(key, options)
-        else
-          Map.put(object.data, "oneOf", options)
-        end
         |> Map.put("voters", voters)
 
       object

lib/pleroma/object/containment.ex

@@ -44,18 +44,11 @@ def get_object(_) do
     nil
   end
 
-  # TODO: We explicitly allow 'tag' URIs through, due to references to legacy OStatus
-  # objects being present in the test suite environment.  Once these objects are
-  # removed, please also remove this.
-  if Mix.env() == :test do
-    defp compare_uris(_, %URI{scheme: "tag"}), do: :ok
-  end
-
   defp compare_uris(%URI{host: host} = _id_uri, %URI{host: host} = _other_uri), do: :ok
   defp compare_uris(_id_uri, _other_uri), do: :error
 
   @doc """
-  Checks that an imported AP object's actor matches the domain it came from.
+  Checks that an imported AP object's actor matches the host it came from.
   """
   def contain_origin(_id, %{"actor" => nil}), do: :error
 

lib/pleroma/object/fetcher.ex

@@ -9,8 +9,10 @@ defmodule Pleroma.Object.Fetcher do
   alias Pleroma.Repo
   alias Pleroma.Signature
   alias Pleroma.Web.ActivityPub.InternalFetchActor
+  alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Federator
+  alias Pleroma.Web.FedSockets
 
   require Logger
   require Pleroma.Constants
@@ -23,21 +25,38 @@ defp touch_changeset(changeset) do
     Ecto.Changeset.put_change(changeset, :updated_at, updated_at)
   end
 
-  defp maybe_reinject_internal_fields(data, %{data: %{} = old_data}) do
+  defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do
     internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields())
 
-    Map.merge(data, internal_fields)
+    Map.merge(new_data, internal_fields)
   end
 
-  defp maybe_reinject_internal_fields(data, _), do: data
+  defp maybe_reinject_internal_fields(_, new_data), do: new_data
 
   @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()}
-  defp reinject_object(struct, data) do
+  defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) do
-    Logger.debug("Reinjecting object #{data["id"]}")
+    Logger.debug("Reinjecting object #{new_data["id"]}")
 
-    with data <- Transmogrifier.fix_object(data),
+    with data <- maybe_reinject_internal_fields(object, new_data),
-         data <- maybe_reinject_internal_fields(data, struct),
+         {:ok, data, _} <- ObjectValidator.validate(data, %{}),
-         changeset <- Object.change(struct, %{data: data}),
+         changeset <- Object.change(object, %{data: data}),
+         changeset <- touch_changeset(changeset),
+         {:ok, object} <- Repo.insert_or_update(changeset),
+         {:ok, object} <- Object.set_cache(object) do
+      {:ok, object}
+    else
+      e ->
+        Logger.error("Error while processing object: #{inspect(e)}")
+        {:error, e}
+    end
+  end
+
+  defp reinject_object(%Object{} = object, new_data) do
+    Logger.debug("Reinjecting object #{new_data["id"]}")
+
+    with new_data <- Transmogrifier.fix_object(new_data),
+         data <- maybe_reinject_internal_fields(object, new_data),
+         changeset <- Object.change(object, %{data: data}),
          changeset <- touch_changeset(changeset),
          {:ok, object} <- Repo.insert_or_update(changeset),
          {:ok, object} <- Object.set_cache(object) do
@@ -51,8 +70,8 @@ defp reinject_object(struct, data) do
 
   def refetch_object(%Object{data: %{"id" => id}} = object) do
     with {:local, false} <- {:local, Object.local?(object)},
-         {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
+         {:ok, new_data} <- fetch_and_contain_remote_object_from_id(id),
-         {:ok, object} <- reinject_object(object, data) do
+         {:ok, object} <- reinject_object(object, new_data) do
       {:ok, object}
     else
       {:local, true} -> {:ok, object}
@@ -80,8 +99,8 @@ def fetch_object_from_id(id, options \\ []) do
       {:containment, _} ->
         {:error, "Object containment failed."}
 
-      {:transmogrifier, {:error, {:reject, nil}}} ->
+      {:transmogrifier, {:error, {:reject, e}}} ->
-        {:reject, nil}
+        {:reject, e}
 
       {:transmogrifier, _} = e ->
         {:error, e}
@@ -106,8 +125,8 @@ def fetch_object_from_id(id, options \\ []) do
   defp prepare_activity_params(data) do
     %{
       "type" => "Create",
-      "to" => data["to"],
+      "to" => data["to"] || [],
-      "cc" => data["cc"],
+      "cc" => data["cc"] || [],
       # Should we seriously keep this attributedTo thing?
       "actor" => data["actor"] || data["attributedTo"],
       "object" => data
@@ -145,12 +164,12 @@ defp make_signature(id, date) do
         date: date
       })
 
-    [{"signature", signature}]
+    {"signature", signature}
   end
 
   defp sign_fetch(headers, id, date) do
     if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
-      headers ++ make_signature(id, date)
+      [make_signature(id, date) | headers]
     else
       headers
     end
@@ -158,33 +177,26 @@ defp sign_fetch(headers, id, date) do
 
   defp maybe_date_fetch(headers, date) do
     if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
-      headers ++ [{"date", date}]
+      [{"date", date} | headers]
     else
       headers
     end
   end
 
-  def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
+  def fetch_and_contain_remote_object_from_id(prm, opts \\ [])
+
+  def fetch_and_contain_remote_object_from_id(%{"id" => id}, opts),
+    do: fetch_and_contain_remote_object_from_id(id, opts)
+
+  def fetch_and_contain_remote_object_from_id(id, opts) when is_binary(id) do
     Logger.debug("Fetching object #{id} via AP")
 
-    date = Pleroma.Signature.signed_date()
-
-    headers =
-      [{"accept", "application/activity+json"}]
-      |> maybe_date_fetch(date)
-      |> sign_fetch(id, date)
-
-    Logger.debug("Fetch headers: #{inspect(headers)}")
-
     with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
-         {:ok, %{body: body, status: code}} when code in 200..299 <- HTTP.get(id, headers),
+         {:ok, body} <- get_object(id, opts),
-         {:ok, data} <- Jason.decode(body),
+         {:ok, data} <- safe_json_decode(body),
          :ok <- Containment.contain_origin_from_id(id, data) do
       {:ok, data}
     else
-      {:ok, %{status: code}} when code in [404, 410] ->
-        {:error, "Object has been deleted"}
-
       {:scheme, _} ->
         {:error, "Unsupported URI scheme"}
 
@@ -196,8 +208,44 @@ def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
     end
   end
 
-  def fetch_and_contain_remote_object_from_id(%{"id" => id}),
+  def fetch_and_contain_remote_object_from_id(_id, _opts),
-    do: fetch_and_contain_remote_object_from_id(id)
+    do: {:error, "id must be a string"}
 
-  def fetch_and_contain_remote_object_from_id(_id), do: {:error, "id must be a string"}
+  defp get_object(id, opts) do
+    with false <- Keyword.get(opts, :force_http, false),
+         {:ok, fedsocket} <- FedSockets.get_or_create_fed_socket(id) do
+      Logger.debug("fetching via fedsocket - #{inspect(id)}")
+      FedSockets.fetch(fedsocket, id)
+    else
+      _other ->
+        Logger.debug("fetching via http - #{inspect(id)}")
+        get_object_http(id)
+    end
+  end
+
+  defp get_object_http(id) do
+    date = Pleroma.Signature.signed_date()
+
+    headers =
+      [{"accept", "application/activity+json"}]
+      |> maybe_date_fetch(date)
+      |> sign_fetch(id, date)
+
+    case HTTP.get(id, headers) do
+      {:ok, %{body: body, status: code}} when code in 200..299 ->
+        {:ok, body}
+
+      {:ok, %{status: code}} when code in [404, 410] ->
+        {:error, "Object has been deleted"}
+
+      {:error, e} ->
+        {:error, e}
+
+      e ->
+        {:error, e}
+    end
+  end
+
+  defp safe_json_decode(nil), do: {:ok, nil}
+  defp safe_json_decode(json), do: Jason.decode(json)
 end

lib/pleroma/plugs/frontend_static.ex

@@ -30,6 +30,7 @@ def init(opts) do
     opts
     |> Keyword.put(:from, "__unconfigured_frontend_static_plug")
     |> Plug.Static.init()
+    |> Map.put(:frontend_type, opts[:frontend_type])
   end
 
   def call(conn, opts) do

lib/pleroma/plugs/oauth_scopes_plug.ex

@@ -53,7 +53,7 @@ def drop_auth_info(conn) do
     |> assign(:token, nil)
   end
 
-  @doc "Filters descendants of supported scopes"
+  @doc "Keeps those of `scopes` which are descendants of `supported_scopes`"
   def filter_descendants(scopes, supported_scopes) do
     Enum.filter(
       scopes,

lib/pleroma/plugs/remote_ip.ex

@@ -7,45 +7,42 @@ defmodule Pleroma.Plugs.RemoteIp do
   This is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
   """
 
+  alias Pleroma.Config
+  import Plug.Conn
+
   @behaviour Plug
 
-  @headers ~w[
-    x-forwarded-for
-  ]
-
-  # https://en.wikipedia.org/wiki/Localhost
-  # https://en.wikipedia.org/wiki/Private_network
-  @reserved ~w[
-    127.0.0.0/8
-    ::1/128
-    fc00::/7
-    10.0.0.0/8
-    172.16.0.0/12
-    192.168.0.0/16
-  ]
-
   def init(_), do: nil
 
-  def call(conn, _) do
+  def call(%{remote_ip: original_remote_ip} = conn, _) do
-    config = Pleroma.Config.get(__MODULE__, [])
+    if Config.get([__MODULE__, :enabled]) do
-
+      %{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts())
-    if Keyword.get(config, :enabled, false) do
+      assign(conn, :remote_ip_found, original_remote_ip != new_remote_ip)
-      RemoteIp.call(conn, remote_ip_opts(config))
     else
       conn
     end
   end
 
-  defp remote_ip_opts(config) do
+  defp remote_ip_opts do
-    headers = config |> Keyword.get(:headers, @headers) |> MapSet.new()
+    headers = Config.get([__MODULE__, :headers], []) |> MapSet.new()
-    reserved = Keyword.get(config, :reserved, @reserved)
+    reserved = Config.get([__MODULE__, :reserved], [])
 
     proxies =
-      config
+      Config.get([__MODULE__, :proxies], [])
-      |> Keyword.get(:proxies, [])
       |> Enum.concat(reserved)
-      |> Enum.map(&InetCidr.parse/1)
+      |> Enum.map(&maybe_add_cidr/1)
 
     {headers, proxies}
   end
+
+  defp maybe_add_cidr(proxy) when is_binary(proxy) do
+    proxy =
+      cond do
+        "/" in String.codepoints(proxy) -> proxy
+        InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
+        InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
+      end
+
+    InetCidr.parse(proxy, true)
+  end
 end

lib/pleroma/repo.ex

@@ -49,7 +49,21 @@ def get_assoc(resource, association) do
     end
   end
 
-  def chunk_stream(query, chunk_size) do
+  @doc """
+  Returns a lazy enumerable that emits all entries from the data store matching the given query.
+
+  `returns_as` use to group records. use the `batches` option to fetch records in bulk.
+
+  ## Examples
+
+  # fetch records one-by-one
+  iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500)
+
+  # fetch records in bulk
+  iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500, :batches)
+  """
+  @spec chunk_stream(Ecto.Query.t(), integer(), atom()) :: Enumerable.t()
+  def chunk_stream(query, chunk_size, returns_as \\ :one) do
     # We don't actually need start and end funcitons of resource streaming,
     # but it seems to be the only way to not fetch records one-by-one and
     # have individual records be the elements of the stream, instead of
@@ -69,7 +83,12 @@ def chunk_stream(query, chunk_size) do
 
             records ->
               last_id = List.last(records).id
-              {records, last_id}
+
+              if returns_as == :one do
+                {records, last_id}
+              else
+                {[records], last_id}
+              end
           end
       end,
       fn _ -> :ok end