Merge remote-tracking branch 'remotes/origin/develop' into media-preview-proxy-nostream

# Conflicts:
#	CHANGELOG.md

CHANGELOG.md

@@ -9,6 +9,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
 - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.
+- The `discoverable` field in the `User` struct will now add a NOINDEX metatag to profile pages when false.
+- Users with the `discoverable` field set to false will not show up in searches.
 - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
 
 ### Added
@@ -29,6 +31,12 @@ switched to a new configuration mechanism, however it was not officially removed
 - Welcome Chat messages preventing user registration with MRF Simple Policy applied to the local instance
 - Mastodon API: the public timeline returning an error when the `reply_visibility` parameter is set to `self` for an unauthenticated user
 
+## Unreleased-patch
+
+### Security
+
+- Fix most MRF rules either crashing or not being applied to objects passed into the Common Pipeline (ChatMessage, Question, Answer, Audio, Event)
+
 ## [2.1.1] - 2020-09-08
 
 ### Security
@@ -44,6 +52,12 @@ switched to a new configuration mechanism, however it was not officially removed
 ### Added
 - Rich media failure tracking (along with `:failure_backoff` option).
 
+<details>
+  <summary>Admin API Changes</summary>
+
+- Add `PATCH /api/pleroma/admin/instance_document/:document_name` to modify the Terms of Service and Instance Panel HTML pages via Admin API
+</details>
+
 ### Fixed
 - Default HTTP adapter not respecting pool setting, leading to possible OOM.
 - Fixed uploading webp images when the Exiftool Upload Filter is enabled by skipping them

docs/API/admin_api.md

@@ -1455,3 +1455,45 @@ Loads json generated from `config/descriptions.exs`.
   "unread": false
 }
 ```
+
+## `GET /api/pleroma/admin/instance_document/:document_name`
+
+### Get an instance document
+
+- Authentication: required
+
+- Response:
+
+Returns the content of the document
+
+```html
+<h1>Instance panel</h1>
+```
+
+## `PATCH /api/pleroma/admin/instance_document/:document_name`
+- Params:
+  - `file` (the file to be uploaded, using multipart form data.)
+
+### Update an instance document
+
+- Authentication: required
+
+- Response:
+
+``` json
+{
+  "url": "https://example.com/instance/panel.html"
+}
+```
+
+## `DELETE /api/pleroma/admin/instance_document/:document_name`
+
+### Delete an instance document
+
+- Response:
+
+``` json
+{
+  "url": "https://example.com/instance/panel.html"
+}
+```

lib/mix/tasks/pleroma/config.ex

@@ -32,7 +32,8 @@ def run(["migrate_from_db" | options]) do
 
   @spec migrate_to_db(Path.t() | nil) :: any()
   def migrate_to_db(file_path \\ nil) do
-    if Pleroma.Config.get([:configurable_from_database]) do
+    with true <- Pleroma.Config.get([:configurable_from_database]),
+         :ok <- Pleroma.Config.DeprecationWarnings.warn() do
       config_file =
         if file_path do
           file_path
@@ -46,7 +47,8 @@ def migrate_to_db(file_path \\ nil) do
 
       do_migrate_to_db(config_file)
     else
-      migration_error()
+      :error -> deprecation_error()
+      _ -> migration_error()
     end
   end
 
@@ -120,6 +122,10 @@ defp migration_error do
     )
   end
 
+  defp deprecation_error do
+    shell_error("Migration is not allowed until all deprecation warnings have been resolved.")
+  end
+
   if Code.ensure_loaded?(Config.Reader) do
     defp config_header, do: "import Config\r\n\r\n"
     defp read_file(config_file), do: Config.Reader.read_imports!(config_file)

lib/pleroma/config/deprecation_warnings.ex

@@ -26,6 +26,10 @@ def check_hellthread_threshold do
       !!!DEPRECATION WARNING!!!
       You are using the old configuration mechanism for the hellthread filter. Please check config.md.
       """)
+
+      :error
+    else
+      :ok
     end
   end
 
@@ -47,17 +51,26 @@ def mrf_user_allowlist do
 
       config :pleroma, :mrf_user_allowlist, #{inspect(rewritten, pretty: true)}
       """)
+
+      :error
+    else
+      :ok
     end
   end
 
   def warn do
-    check_hellthread_threshold()
-    mrf_user_allowlist()
-    check_old_mrf_config()
-    check_media_proxy_whitelist_config()
-    check_welcome_message_config()
-    check_gun_pool_options()
-    check_activity_expiration_config()
+    with :ok <- check_hellthread_threshold(),
+         :ok <- mrf_user_allowlist(),
+         :ok <- check_old_mrf_config(),
+         :ok <- check_media_proxy_whitelist_config(),
+         :ok <- check_welcome_message_config(),
+         :ok <- check_gun_pool_options(),
+         :ok <- check_activity_expiration_config() do
+      :ok
+    else
+      _ ->
+        :error
+    end
   end
 
   def check_welcome_message_config do
@@ -74,6 +87,10 @@ def check_welcome_message_config do
       \n* `config :pleroma, :instance, welcome_user_nickname` is now `config :pleroma, :welcome, :direct_message, :sender_nickname`
       \n* `config :pleroma, :instance, welcome_message` is now `config :pleroma, :welcome, :direct_message, :message`
       """)
+
+      :error
+    else
+      :ok
     end
   end
 
@@ -101,8 +118,11 @@ def move_namespace_and_warn(config_map, warning_preface) do
           end
       end)
 
-    if warning != "" do
+    if warning == "" do
+      :ok
+    else
       Logger.warn(warning_preface <> warning)
+      :error
     end
   end
 
@@ -115,6 +135,10 @@ def check_media_proxy_whitelist_config do
       !!!DEPRECATION WARNING!!!
       Your config is using old format (only domain) for MediaProxy whitelist option. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
       """)
+
+      :error
+    else
+      :ok
     end
   end
 
@@ -157,6 +181,9 @@ def check_gun_pool_options do
       Logger.warn(Enum.join([warning_preface | pool_warnings]))
 
       Config.put(:pools, updated_config)
+      :error
+    else
+      :ok
     end
   end
 

lib/pleroma/user/search.ex

@@ -52,6 +52,7 @@ defp search_query(query_string, for_user, following) do
     |> base_query(following)
     |> filter_blocked_user(for_user)
     |> filter_invisible_users()
+    |> filter_discoverable_users()
     |> filter_internal_users()
     |> filter_blocked_domains(for_user)
     |> fts_search(query_string)
@@ -122,6 +123,10 @@ defp filter_invisible_users(query) do
     from(q in query, where: q.invisible == false)
   end
 
+  defp filter_discoverable_users(query) do
+    from(q in query, where: q.discoverable == true)
+  end
+
   defp filter_internal_users(query) do
     from(q in query, where: q.actor_type != "Application")
   end

lib/pleroma/web/activity_pub/mrf.ex

@@ -5,16 +5,34 @@
 defmodule Pleroma.Web.ActivityPub.MRF do
   @callback filter(Map.t()) :: {:ok | :reject, Map.t()}
 
-  def filter(policies, %{} = object) do
+  def filter(policies, %{} = message) do
     policies
-    |> Enum.reduce({:ok, object}, fn
-      policy, {:ok, object} -> policy.filter(object)
+    |> Enum.reduce({:ok, message}, fn
+      policy, {:ok, message} -> policy.filter(message)
       _, error -> error
     end)
   end
 
   def filter(%{} = object), do: get_policies() |> filter(object)
 
+  def pipeline_filter(%{} = message, meta) do
+    object = meta[:object_data]
+    ap_id = message["object"]
+
+    if object && ap_id do
+      with {:ok, message} <- filter(Map.put(message, "object", object)) do
+        meta = Keyword.put(meta, :object_data, message["object"])
+        {:ok, Map.put(message, "object", ap_id), meta}
+      else
+        {err, message} -> {err, message, meta}
+      end
+    else
+      {err, message} = filter(message)
+
+      {err, message, meta}
+    end
+  end
+
   def get_policies do
     Pleroma.Config.get([:mrf, :policies], []) |> get_policies()
   end

lib/pleroma/web/activity_pub/mrf/keyword_policy.ex

@@ -20,9 +20,17 @@ defp string_matches?(string, pattern) do
     String.match?(string, pattern)
   end
 
-  defp check_reject(%{"object" => %{"content" => content, "summary" => summary}} = message) do
+  defp object_payload(%{} = object) do
+    [object["content"], object["summary"], object["name"]]
+    |> Enum.filter(& &1)
+    |> Enum.join("\n")
+  end
+
+  defp check_reject(%{"object" => %{} = object} = message) do
+    payload = object_payload(object)
+
     if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern ->
-         string_matches?(content, pattern) or string_matches?(summary, pattern)
+         string_matches?(payload, pattern)
        end) do
       {:reject, "[KeywordPolicy] Matches with rejected keyword"}
     else
@@ -30,12 +38,12 @@ defp check_reject(%{"object" => %{"content" => content, "summary" => summary}} =
     end
   end
 
-  defp check_ftl_removal(
-         %{"to" => to, "object" => %{"content" => content, "summary" => summary}} = message
-       ) do
+  defp check_ftl_removal(%{"to" => to, "object" => %{} = object} = message) do
+    payload = object_payload(object)
+
     if Pleroma.Constants.as_public() in to and
          Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern ->
-           string_matches?(content, pattern) or string_matches?(summary, pattern)
+           string_matches?(payload, pattern)
          end) do
       to = List.delete(to, Pleroma.Constants.as_public())
       cc = [Pleroma.Constants.as_public() | message["cc"] || []]
@@ -51,35 +59,24 @@ defp check_ftl_removal(
     end
   end
 
-  defp check_replace(%{"object" => %{"content" => content, "summary" => summary}} = message) do
-    content =
-      if is_binary(content) do
-        content
-      else
-        ""
-      end
+  defp check_replace(%{"object" => %{} = object} = message) do
+    object =
+      ["content", "name", "summary"]
+      |> Enum.filter(fn field -> Map.has_key?(object, field) && object[field] end)
+      |> Enum.reduce(object, fn field, object ->
+        data =
+          Enum.reduce(
+            Pleroma.Config.get([:mrf_keyword, :replace]),
+            object[field],
+            fn {pat, repl}, acc -> String.replace(acc, pat, repl) end
+          )
 
-    summary =
-      if is_binary(summary) do
-        summary
-      else
-        ""
-      end
+        Map.put(object, field, data)
+      end)
 
-    {content, summary} =
-      Enum.reduce(
-        Pleroma.Config.get([:mrf_keyword, :replace]),
-        {content, summary},
-        fn {pattern, replacement}, {content_acc, summary_acc} ->
-          {String.replace(content_acc, pattern, replacement),
-           String.replace(summary_acc, pattern, replacement)}
-        end
-      )
+    message = Map.put(message, "object", object)
 
-    {:ok,
-     message
-     |> put_in(["object", "content"], content)
-     |> put_in(["object", "summary"], summary)}
+    {:ok, message}
   end
 
   @impl true

lib/pleroma/web/activity_pub/mrf/subchain_policy.ex

@@ -28,8 +28,7 @@ def filter(%{"actor" => actor} = message) do
         }"
       )
 
-      subchain
-      |> MRF.filter(message)
+      MRF.filter(subchain, message)
     else
       _e -> {:ok, message}
     end

lib/pleroma/web/activity_pub/pipeline.ex

@@ -26,13 +26,17 @@ def common_pipeline(object, meta) do
 
       {:error, e} ->
         {:error, e}
+
+      {:reject, e} ->
+        {:reject, e}
     end
   end
 
   def do_common_pipeline(object, meta) do
     with {_, {:ok, validated_object, meta}} <-
            {:validate_object, ObjectValidator.validate(object, meta)},
-         {_, {:ok, mrfd_object}} <- {:mrf_object, MRF.filter(validated_object)},
+         {_, {:ok, mrfd_object, meta}} <-
+           {:mrf_object, MRF.pipeline_filter(validated_object, meta)},
          {_, {:ok, activity, meta}} <-
            {:persist_object, ActivityPub.persist(mrfd_object, meta)},
          {_, {:ok, activity, meta}} <-
@@ -40,7 +44,7 @@ def do_common_pipeline(object, meta) do
          {_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do
       {:ok, activity, meta}
     else
-      {:mrf_object, {:reject, _}} -> {:ok, nil, meta}
+      {:mrf_object, {:reject, message, _}} -> {:reject, message}
       e -> {:error, e}
     end
   end

lib/pleroma/web/admin_api/controllers/instance_document_controller.ex

@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.InstanceDocumentController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Plugs.InstanceStatic
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.InstanceDocument
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+  action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.InstanceDocumentOperation
+
+  plug(OAuthScopesPlug, %{scopes: ["read"], admin: true} when action == :show)
+  plug(OAuthScopesPlug, %{scopes: ["write"], admin: true} when action in [:update, :delete])
+
+  def show(conn, %{name: document_name}) do
+    with {:ok, url} <- InstanceDocument.get(document_name),
+         {:ok, content} <- File.read(InstanceStatic.file_path(url)) do
+      conn
+      |> put_resp_content_type("text/html")
+      |> send_resp(200, content)
+    end
+  end
+
+  def update(%{body_params: %{file: file}} = conn, %{name: document_name}) do
+    with {:ok, url} <- InstanceDocument.put(document_name, file.path) do
+      json(conn, %{"url" => url})
+    end
+  end
+
+  def delete(conn, %{name: document_name}) do
+    with :ok <- InstanceDocument.delete(document_name) do
+      json(conn, %{})
+    end
+  end
+end

lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex

@@ -0,0 +1,115 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.InstanceDocumentOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Helpers
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def show_operation do
+    %Operation{
+      tags: ["Admin", "InstanceDocument"],
+      summary: "Get the instance document",
+      operationId: "AdminAPI.InstanceDocumentController.show",
+      security: [%{"oAuth" => ["read"]}],
+      parameters: [
+        Operation.parameter(:name, :path, %Schema{type: :string}, "The document name",
+          required: true
+        )
+        | Helpers.admin_api_params()
+      ],
+      responses: %{
+        200 => document_content(),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  def update_operation do
+    %Operation{
+      tags: ["Admin", "InstanceDocument"],
+      summary: "Update the instance document",
+      operationId: "AdminAPI.InstanceDocumentController.update",
+      security: [%{"oAuth" => ["write"]}],
+      requestBody: Helpers.request_body("Parameters", update_request()),
+      parameters: [
+        Operation.parameter(:name, :path, %Schema{type: :string}, "The document name",
+          required: true
+        )
+        | Helpers.admin_api_params()
+      ],
+      responses: %{
+        200 => Operation.response("InstanceDocument", "application/json", instance_document()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp update_request do
+    %Schema{
+      title: "UpdateRequest",
+      description: "POST body for uploading the file",
+      type: :object,
+      required: [:file],
+      properties: %{
+        file: %Schema{
+          type: :string,
+          format: :binary,
+          description: "The file to be uploaded, using multipart form data."
+        }
+      }
+    }
+  end
+
+  def delete_operation do
+    %Operation{
+      tags: ["Admin", "InstanceDocument"],
+      summary: "Get the instance document",
+      operationId: "AdminAPI.InstanceDocumentController.delete",
+      security: [%{"oAuth" => ["write"]}],
+      parameters: [
+        Operation.parameter(:name, :path, %Schema{type: :string}, "The document name",
+          required: true
+        )
+        | Helpers.admin_api_params()
+      ],
+      responses: %{
+        200 => Operation.response("InstanceDocument", "application/json", instance_document()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp instance_document do
+    %Schema{
+      title: "InstanceDocument",
+      type: :object,
+      properties: %{
+        url: %Schema{type: :string}
+      },
+      example: %{
+        "url" => "https://example.com/static/terms-of-service.html"
+      }
+    }
+  end
+
+  defp document_content do
+    Operation.response("InstanceDocumentContent", "text/html", %Schema{
+      type: :string,
+      example: "<h1>Instance panel</h1>"
+    })
+  end
+end

lib/pleroma/web/api_spec/operations/chat_operation.ex

@@ -184,7 +184,8 @@ def post_chat_message_operation do
             "application/json",
             ChatMessage
           ),
-        400 => Operation.response("Bad Request", "application/json", ApiError)
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        422 => Operation.response("MRF Rejection", "application/json", ApiError)
       },
       security: [
         %{

lib/pleroma/web/api_spec/operations/status_operation.ex

@@ -55,7 +55,7 @@ def create_operation do
             "application/json",
             %Schema{oneOf: [Status, ScheduledStatus]}
           ),
-        422 => Operation.response("Bad Request", "application/json", ApiError)
+        422 => Operation.response("Bad Request / MRF Rejection", "application/json", ApiError)
       }
     }
   end

lib/pleroma/web/common_api/common_api.ex

@@ -48,6 +48,9 @@ def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ [])
               local: true
             )} do
       {:ok, activity}
+    else
+      {:common_pipeline, {:reject, _} = e} -> e
+      e -> e
     end
   end
 

lib/pleroma/web/instance_document.ex

@@ -0,0 +1,62 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.InstanceDocument do
+  alias Pleroma.Config
+  alias Pleroma.Web.Endpoint
+
+  @instance_documents %{
+    "terms-of-service" => "/static/terms-of-service.html",
+    "instance-panel" => "/instance/panel.html"
+  }
+
+  @spec get(String.t()) :: {:ok, String.t()} | {:error, atom()}
+  def get(document_name) do
+    case Map.fetch(@instance_documents, document_name) do
+      {:ok, path} -> {:ok, path}
+      _ -> {:error, :not_found}
+    end
+  end
+
+  @spec put(String.t(), String.t()) :: {:ok, String.t()} | {:error, atom()}
+  def put(document_name, origin_path) do
+    with {_, {:ok, destination_path}} <-
+           {:instance_document, Map.fetch(@instance_documents, document_name)},
+         :ok <- put_file(origin_path, destination_path) do
+      {:ok, Path.join(Endpoint.url(), destination_path)}
+    else
+      {:instance_document, :error} -> {:error, :not_found}
+      error -> error
+    end
+  end
+
+  @spec delete(String.t()) :: :ok | {:error, atom()}
+  def delete(document_name) do
+    with {_, {:ok, path}} <- {:instance_document, Map.fetch(@instance_documents, document_name)},
+         instance_static_dir_path <- instance_static_dir(path),
+         :ok <- File.rm(instance_static_dir_path) do
+      :ok
+    else
+      {:instance_document, :error} -> {:error, :not_found}
+      {:error, :enoent} -> {:error, :not_found}
+      error -> error
+    end
+  end
+
+  defp put_file(origin_path, destination_path) do
+    with destination <- instance_static_dir(destination_path),
+         {_, :ok} <- {:mkdir_p, File.mkdir_p(Path.dirname(destination))},
+         {_, {:ok, _}} <- {:copy, File.copy(origin_path, destination)} do
+      :ok
+    else
+      {error, _} -> {:error, error}
+    end
+  end
+
+  defp instance_static_dir(filename) do
+    [:instance, :static_dir]
+    |> Config.get!()
+    |> Path.join(filename)
+  end
+end

lib/pleroma/web/metadata/restrict_indexing.ex

@@ -10,7 +10,9 @@ defmodule Pleroma.Web.Metadata.Providers.RestrictIndexing do
   """
 
   @impl true
-  def build_tags(%{user: %{local: false}}) do
+  def build_tags(%{user: %{local: true, discoverable: true}}), do: []
+
+  def build_tags(_) do
     [
       {:meta,
        [
@@ -19,7 +21,4 @@ def build_tags(%{user: %{local: false}}) do
        ], []}
     ]
   end
-
-  @impl true
-  def build_tags(%{user: %{local: true}}), do: []
 end

lib/pleroma/web/pleroma_api/controllers/chat_controller.ex

@@ -90,6 +90,16 @@ def post_chat_message(
       conn
       |> put_view(MessageReferenceView)
       |> render("show.json", chat_message_reference: cm_ref)
+    else
+      {:reject, message} ->
+        conn
+        |> put_status(:unprocessable_entity)
+        |> json(%{error: message})
+
+      {:error, message} ->
+        conn
+        |> put_status(:bad_request)
+        |> json(%{error: message})
     end
   end
 

lib/pleroma/web/router.ex

@@ -182,6 +182,10 @@ defmodule Pleroma.Web.Router do
 
     get("/instances/:instance/statuses", AdminAPIController, :list_instance_statuses)
 
+    get("/instance_document/:name", InstanceDocumentController, :show)
+    patch("/instance_document/:name", InstanceDocumentController, :update)
+    delete("/instance_document/:name", InstanceDocumentController, :delete)
+
     patch("/users/confirm_email", AdminAPIController, :confirm_email)
     patch("/users/resend_confirmation_email", AdminAPIController, :resend_confirmation_email)
 

test/fixtures/custom_instance_panel.html

@@ -0,0 +1 @@
+<h2>Custom instance panel</h2>

test/fixtures/tesla_mock/admin@mastdon.example.org.json

@@ -1,20 +1,24 @@
 {
-  "@context": ["https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1", {
-    "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
-    "sensitive": "as:sensitive",
-    "movedTo": "as:movedTo",
-    "Hashtag": "as:Hashtag",
-    "ostatus": "http://ostatus.org#",
-    "atomUri": "ostatus:atomUri",
-    "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
-    "conversation": "ostatus:conversation",
-    "toot": "http://joinmastodon.org/ns#",
-    "Emoji": "toot:Emoji",
-    "alsoKnownAs": {
-      "@id": "as:alsoKnownAs",
-      "@type": "@id"
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    "https://w3id.org/security/v1",
+    {
+      "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+      "sensitive": "as:sensitive",
+      "movedTo": "as:movedTo",
+      "Hashtag": "as:Hashtag",
+      "ostatus": "http://ostatus.org#",
+      "atomUri": "ostatus:atomUri",
+      "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+      "conversation": "ostatus:conversation",
+      "toot": "http://joinmastodon.org/ns#",
+      "Emoji": "toot:Emoji",
+      "alsoKnownAs": {
+        "@id": "as:alsoKnownAs",
+        "@type": "@id"
+      }
     }
-  }],
+  ],
   "id": "http://mastodon.example.org/users/admin",
   "type": "Person",
   "following": "http://mastodon.example.org/users/admin/following",
@@ -23,6 +27,7 @@
   "outbox": "http://mastodon.example.org/users/admin/outbox",
   "preferredUsername": "admin",
   "name": null,
+  "discoverable": "true",
   "summary": "\u003cp\u003e\u003c/p\u003e",
   "url": "http://mastodon.example.org/@admin",
   "manuallyApprovesFollowers": false,
@@ -34,7 +39,8 @@
     "owner": "http://mastodon.example.org/users/admin",
     "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtc4Tir+3ADhSNF6VKrtW\nOU32T01w7V0yshmQei38YyiVwVvFu8XOP6ACchkdxbJ+C9mZud8qWaRJKVbFTMUG\nNX4+6Q+FobyuKrwN7CEwhDALZtaN2IPbaPd6uG1B7QhWorrY+yFa8f2TBM3BxnUy\nI4T+bMIZIEYG7KtljCBoQXuTQmGtuffO0UwJksidg2ffCF5Q+K//JfQagJ3UzrR+\nZXbKMJdAw4bCVJYs4Z5EhHYBwQWiXCyMGTd7BGlmMkY6Av7ZqHKC/owp3/0EWDNz\nNqF09Wcpr3y3e8nA10X40MJqp/wR+1xtxp+YGbq/Cj5hZGBG7etFOmIpVBrDOhry\nBwIDAQAB\n-----END PUBLIC KEY-----\n"
   },
-  "attachment": [{
+  "attachment": [
+    {
       "type": "PropertyValue",
       "name": "foo",
       "value": "bar"
@@ -58,5 +64,7 @@
     "mediaType": "image/png",
     "url": "https://cdn.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png"
   },
-  "alsoKnownAs": ["http://example.org/users/foo"]
+  "alsoKnownAs": [
+    "http://example.org/users/foo"
+  ]
 }

test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json

@@ -8,6 +8,7 @@
     "preferredUsername": "mike",
     "name": "Mike Macgirvin (Osada)",
     "updated": "2018-08-29T03:09:11Z",
+    "discoverable": "true",
     "icon": {
         "type": "Image",
         "mediaType": "image/jpeg",

test/support/factory.ex

@@ -31,6 +31,7 @@ def user_factory do
       nickname: sequence(:nickname, &"nick#{&1}"),
       password_hash: Pbkdf2.hash_pwd_salt("test"),
       bio: sequence(:bio, &"Tester Number #{&1}"),
+      discoverable: true,
       last_digest_emailed_at: NaiveDateTime.utc_now(),
       last_refreshed_at: NaiveDateTime.utc_now(),
       notification_settings: %Pleroma.User.NotificationSetting{},

test/tasks/config_test.exs

@@ -40,6 +40,19 @@ test "error if file with custom settings doesn't exist" do
       on_exit(fn -> Application.put_env(:quack, :level, initial) end)
     end
 
+    @tag capture_log: true
+    test "config migration refused when deprecated settings are found" do
+      clear_config([:media_proxy, :whitelist], ["domain_without_scheme.com"])
+      assert Repo.all(ConfigDB) == []
+
+      Mix.Tasks.Pleroma.Config.migrate_to_db("test/fixtures/config/temp.secret.exs")
+
+      assert_received {:mix_shell, :error, [message]}
+
+      assert message =~
+               "Migration is not allowed until all deprecation warnings have been resolved."
+    end
+
     test "filtered settings are migrated to db" do
       assert Repo.all(ConfigDB) == []
 

test/user_search_test.exs

@@ -25,6 +25,14 @@ test "excludes invisible users from results" do
       assert found_user.id == user.id
     end
 
+    test "excludes users when discoverable is false" do
+      insert(:user, %{nickname: "john 3000", discoverable: false})
+      insert(:user, %{nickname: "john 3001"})
+
+      users = User.search("john")
+      assert Enum.count(users) == 1
+    end
+
     test "excludes service actors from results" do
       insert(:user, actor_type: "Application", nickname: "user1")
       service = insert(:user, actor_type: "Service", nickname: "user2")

test/web/activity_pub/pipeline_test.exs

@@ -26,7 +26,7 @@ test "when given an `object_data` in meta, Federation will receive a the origina
         {
           Pleroma.Web.ActivityPub.MRF,
           [],
-          [filter: fn o -> {:ok, o} end]
+          [pipeline_filter: fn o, m -> {:ok, o, m} end]
         },
         {
           Pleroma.Web.ActivityPub.ActivityPub,
@@ -51,7 +51,7 @@ test "when given an `object_data` in meta, Federation will receive a the origina
                  Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
 
         assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta))
-        assert_called(Pleroma.Web.ActivityPub.MRF.filter(activity))
+        assert_called(Pleroma.Web.ActivityPub.MRF.pipeline_filter(activity, meta))
         assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta))
         assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta))
         refute called(Pleroma.Web.Federator.publish(activity))
@@ -68,7 +68,7 @@ test "it goes through validation, filtering, persisting, side effects and federa
         {
           Pleroma.Web.ActivityPub.MRF,
           [],
-          [filter: fn o -> {:ok, o} end]
+          [pipeline_filter: fn o, m -> {:ok, o, m} end]
         },
         {
           Pleroma.Web.ActivityPub.ActivityPub,
@@ -93,7 +93,7 @@ test "it goes through validation, filtering, persisting, side effects and federa
                  Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
 
         assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta))
-        assert_called(Pleroma.Web.ActivityPub.MRF.filter(activity))
+        assert_called(Pleroma.Web.ActivityPub.MRF.pipeline_filter(activity, meta))
         assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta))
         assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta))
         assert_called(Pleroma.Web.Federator.publish(activity))
@@ -109,7 +109,7 @@ test "it goes through validation, filtering, persisting, side effects without fe
         {
           Pleroma.Web.ActivityPub.MRF,
           [],
-          [filter: fn o -> {:ok, o} end]
+          [pipeline_filter: fn o, m -> {:ok, o, m} end]
         },
         {
           Pleroma.Web.ActivityPub.ActivityPub,
@@ -131,7 +131,7 @@ test "it goes through validation, filtering, persisting, side effects without fe
                  Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
 
         assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta))
-        assert_called(Pleroma.Web.ActivityPub.MRF.filter(activity))
+        assert_called(Pleroma.Web.ActivityPub.MRF.pipeline_filter(activity, meta))
         assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta))
         assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta))
       end
@@ -148,7 +148,7 @@ test "it goes through validation, filtering, persisting, side effects without fe
         {
           Pleroma.Web.ActivityPub.MRF,
           [],
-          [filter: fn o -> {:ok, o} end]
+          [pipeline_filter: fn o, m -> {:ok, o, m} end]
         },
         {
           Pleroma.Web.ActivityPub.ActivityPub,
@@ -170,7 +170,7 @@ test "it goes through validation, filtering, persisting, side effects without fe
                  Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
 
         assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta))
-        assert_called(Pleroma.Web.ActivityPub.MRF.filter(activity))
+        assert_called(Pleroma.Web.ActivityPub.MRF.pipeline_filter(activity, meta))
         assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta))
         assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta))
       end

test/web/admin_api/controllers/instance_document_controller_test.exs

@@ -0,0 +1,106 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.InstanceDocumentControllerTest do
+  use Pleroma.Web.ConnCase, async: true
+  import Pleroma.Factory
+  alias Pleroma.Config
+
+  @dir "test/tmp/instance_static"
+  @default_instance_panel ~s(<p>Welcome to <a href="https://pleroma.social" target="_blank">Pleroma!</a></p>)
+
+  setup do
+    File.mkdir_p!(@dir)
+    on_exit(fn -> File.rm_rf(@dir) end)
+  end
+
+  setup do: clear_config([:instance, :static_dir], @dir)
+
+  setup do
+    admin = insert(:user, is_admin: true)
+    token = insert(:oauth_admin_token, user: admin)
+
+    conn =
+      build_conn()
+      |> assign(:user, admin)
+      |> assign(:token, token)
+
+    {:ok, %{admin: admin, token: token, conn: conn}}
+  end
+
+  describe "GET /api/pleroma/admin/instance_document/:name" do
+    test "return the instance document url", %{conn: conn} do
+      conn = get(conn, "/api/pleroma/admin/instance_document/instance-panel")
+
+      assert content = html_response(conn, 200)
+      assert String.contains?(content, @default_instance_panel)
+    end
+
+    test "it returns 403 if requested by a non-admin" do
+      non_admin_user = insert(:user)
+      token = insert(:oauth_token, user: non_admin_user)
+
+      conn =
+        build_conn()
+        |> assign(:user, non_admin_user)
+        |> assign(:token, token)
+        |> get("/api/pleroma/admin/instance_document/instance-panel")
+
+      assert json_response(conn, :forbidden)
+    end
+
+    test "it returns 404 if the instance document with the given name doesn't exist", %{
+      conn: conn
+    } do
+      conn = get(conn, "/api/pleroma/admin/instance_document/1234")
+
+      assert json_response_and_validate_schema(conn, 404)
+    end
+  end
+
+  describe "PATCH /api/pleroma/admin/instance_document/:name" do
+    test "uploads the instance document", %{conn: conn} do
+      image = %Plug.Upload{
+        content_type: "text/html",
+        path: Path.absname("test/fixtures/custom_instance_panel.html"),
+        filename: "custom_instance_panel.html"
+      }
+
+      conn =
+        conn
+        |> put_req_header("content-type", "multipart/form-data")
+        |> patch("/api/pleroma/admin/instance_document/instance-panel", %{
+          "file" => image
+        })
+
+      assert %{"url" => url} = json_response_and_validate_schema(conn, 200)
+      index = get(build_conn(), url)
+      assert html_response(index, 200) == "<h2>Custom instance panel</h2>"
+    end
+  end
+
+  describe "DELETE /api/pleroma/admin/instance_document/:name" do
+    test "deletes the instance document", %{conn: conn} do
+      File.mkdir!(@dir <> "/instance/")
+      File.write!(@dir <> "/instance/panel.html", "Custom instance panel")
+
+      conn_resp =
+        conn
+        |> get("/api/pleroma/admin/instance_document/instance-panel")
+
+      assert html_response(conn_resp, 200) == "Custom instance panel"
+
+      conn
+      |> delete("/api/pleroma/admin/instance_document/instance-panel")
+      |> json_response_and_validate_schema(200)
+
+      conn_resp =
+        conn
+        |> get("/api/pleroma/admin/instance_document/instance-panel")
+
+      assert content = html_response(conn_resp, 200)
+      assert String.contains?(content, @default_instance_panel)
+    end
+  end
+end

test/web/admin_api/search_test.exs

@@ -177,5 +177,14 @@ test "it returns unapproved user" do
       assert total == 3
       assert count == 1
     end
+
+    test "it returns non-discoverable users" do
+      insert(:user)
+      insert(:user, discoverable: false)
+
+      {:ok, _results, total} = Search.user()
+
+      assert total == 2
+    end
   end
 end

test/web/common_api/common_api_test.exs

@@ -217,6 +217,17 @@ test "it reject messages over the local limit" do
 
       assert message == :content_too_long
     end
+
+    test "it reject messages via MRF" do
+      clear_config([:mrf_keyword, :reject], ["GNO"])
+      clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy])
+
+      author = insert(:user)
+      recipient = insert(:user)
+
+      assert {:reject, "[KeywordPolicy] Matches with rejected keyword"} ==
+               CommonAPI.post_chat_message(author, recipient, "GNO/Linux")
+    end
   end
 
   describe "unblocking" do

test/web/mastodon_api/views/account_view_test.exs

@@ -69,7 +69,7 @@ test "Represent a user account" do
         sensitive: false,
         pleroma: %{
           actor_type: "Person",
-          discoverable: false
+          discoverable: true
         },
         fields: []
       },
@@ -167,7 +167,7 @@ test "Represent a Service(bot) account" do
         sensitive: false,
         pleroma: %{
           actor_type: "Service",
-          discoverable: false
+          discoverable: true
         },
         fields: []
       },

test/web/metadata/metadata_test.exs

@@ -16,7 +16,14 @@ test "for remote user" do
     end
 
     test "for local user" do
-      user = insert(:user)
+      user = insert(:user, discoverable: false)
+
+      assert Pleroma.Web.Metadata.build_tags(%{user: user}) =~
+               "<meta content=\"noindex, noarchive\" name=\"robots\">"
+    end
+
+    test "for local user set to discoverable" do
+      user = insert(:user, discoverable: true)
 
       refute Pleroma.Web.Metadata.build_tags(%{user: user}) =~
                "<meta content=\"noindex, noarchive\" name=\"robots\">"
@@ -24,11 +31,19 @@ test "for local user" do
   end
 
   describe "no metadata for private instances" do
-    test "for local user" do
+    test "for local user set to discoverable" do
       clear_config([:instance, :public], false)
-      user = insert(:user, bio: "This is my secret fedi account bio")
+      user = insert(:user, bio: "This is my secret fedi account bio", discoverable: true)
 
       assert "" = Pleroma.Web.Metadata.build_tags(%{user: user})
     end
+
+    test "search exclusion metadata is included" do
+      clear_config([:instance, :public], false)
+      user = insert(:user, bio: "This is my secret fedi account bio", discoverable: false)
+
+      assert ~s(<meta content="noindex, noarchive" name="robots">) ==
+               Pleroma.Web.Metadata.build_tags(%{user: user})
+    end
   end
 end

test/web/metadata/restrict_indexing_test.exs

@@ -14,8 +14,14 @@ test "for remote user" do
 
     test "for local user" do
       assert Pleroma.Web.Metadata.Providers.RestrictIndexing.build_tags(%{
-               user: %Pleroma.User{local: true}
+               user: %Pleroma.User{local: true, discoverable: true}
              }) == []
     end
+
+    test "for local user when discoverable is false" do
+      assert Pleroma.Web.Metadata.Providers.RestrictIndexing.build_tags(%{
+               user: %Pleroma.User{local: true, discoverable: false}
+             }) == [{:meta, [name: "robots", content: "noindex, noarchive"], []}]
+    end
   end
 end

test/web/pleroma_api/controllers/chat_controller_test.exs

@@ -100,7 +100,7 @@ test "it fails if there is no content", %{conn: conn, user: user} do
         |> post("/api/v1/pleroma/chats/#{chat.id}/messages")
         |> json_response_and_validate_schema(400)
 
-      assert result
+      assert %{"error" => "no_content"} == result
     end
 
     test "it works with an attachment", %{conn: conn, user: user} do
@@ -126,6 +126,23 @@ test "it works with an attachment", %{conn: conn, user: user} do
 
       assert result["attachment"]
     end
+
+    test "gets MRF reason when rejected", %{conn: conn, user: user} do
+      clear_config([:mrf_keyword, :reject], ["GNO"])
+      clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy])
+
+      other_user = insert(:user)
+
+      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+
+      result =
+        conn
+        |> put_req_header("content-type", "application/json")
+        |> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{"content" => "GNO/Linux"})
+        |> json_response_and_validate_schema(422)
+
+      assert %{"error" => "[KeywordPolicy] Matches with rejected keyword"} == result
+    end
   end
 
   describe "DELETE /api/v1/pleroma/chats/:id/messages/:message_id" do