From fcf2f38d20eed40a53b03374467d9e52b013da07 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 May 2019 17:37:00 +0200 Subject: [PATCH 001/123] Conversations: Add a function to 'import' old DMs. --- lib/pleroma/conversation.ex | 14 ++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 6 ++++++ test/conversation_test.exs | 18 ++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index 6e26c5fd4..aa73edd75 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -72,4 +72,18 @@ def create_or_bump_for(activity) do e -> {:error, e} end end + + @doc """ + This is only meant to be run by a mix task. It creates conversations/participations for all direct messages in the database. + """ + def bump_for_all_activities() do + stream = + Pleroma.Web.ActivityPub.ActivityPub.fetch_direct_messages_query() + |> Repo.stream() + + Repo.transaction(fn -> + stream + |> Enum.each(&create_or_bump_for/1) + end) + end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 8f8c23a9b..23cf4e9c4 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1061,4 +1061,10 @@ def contain_timeline(timeline, user) do contain_activity(activity, user) end) end + + def fetch_direct_messages_query() do + Activity + |> restrict_type(%{"type" => "Create"}) + |> restrict_visibility(%{visibility: "direct"}) + end end diff --git a/test/conversation_test.exs b/test/conversation_test.exs index f3300e7d1..59368b0e7 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -9,6 +9,24 @@ defmodule Pleroma.ConversationTest do import Pleroma.Factory + test "it goes through old direct conversations" do + user = insert(:user) + other_user = insert(:user) + + {:ok, _activity} = + CommonAPI.post(user, %{"visibility" => "direct", "status" => "hey @#{other_user.nickname}"}) + + Repo.delete_all(Conversation) + Repo.delete_all(Conversation.Participation) + + refute Repo.one(Conversation) + + Conversation.bump_for_all_activities() + + assert Repo.one(Conversation) + assert length(Repo.all(Conversation.Participation)) == 2 + end + test "it creates a conversation for given ap_id" do assert {:ok, %Conversation{} = conversation} = Conversation.create_for_ap_id("https://some_ap_id") From 920bd4705526d8dfa8ada516853bbb4e5438cbf1 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 May 2019 17:40:24 +0200 Subject: [PATCH 002/123] ActivityPub: Remove leftover printf debugging. --- lib/pleroma/web/activity_pub/activity_pub.ex | 42 ++++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 23cf4e9c4..cd8495035 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -533,22 +533,17 @@ def fetch_public_activities(opts \\ %{}) do defp restrict_visibility(query, %{visibility: visibility}) when is_list(visibility) do if Enum.all?(visibility, &(&1 in @valid_visibilities)) do - query = - from( - a in query, - where: - fragment( - "activity_visibility(?, ?, ?) = ANY (?)", - a.actor, - a.recipients, - a.data, - ^visibility - ) - ) - - Ecto.Adapters.SQL.to_sql(:all, Repo, query) - - query + from( + a in query, + where: + fragment( + "activity_visibility(?, ?, ?) = ANY (?)", + a.actor, + a.recipients, + a.data, + ^visibility + ) + ) else Logger.error("Could not restrict visibility to #{visibility}") end @@ -556,16 +551,11 @@ defp restrict_visibility(query, %{visibility: visibility}) defp restrict_visibility(query, %{visibility: visibility}) when visibility in @valid_visibilities do - query = - from( - a in query, - where: - fragment("activity_visibility(?, ?, ?) = ?", a.actor, a.recipients, a.data, ^visibility) - ) - - Ecto.Adapters.SQL.to_sql(:all, Repo, query) - - query + from( + a in query, + where: + fragment("activity_visibility(?, ?, ?) = ?", a.actor, a.recipients, a.data, ^visibility) + ) end defp restrict_visibility(_query, %{visibility: visibility}) From a4598b5e8bc640ffc1a052438e21f3573ff837ee Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 May 2019 18:08:50 +0200 Subject: [PATCH 003/123] Visibility: Make it more resilient. --- lib/pleroma/web/activity_pub/visibility.ex | 9 +++++---- test/web/activity_pub/visibilty_test.exs | 10 ++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index 6dee61dd6..e7613a5c8 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -13,11 +13,12 @@ def is_public?(data) do end def is_private?(activity) do - unless is_public?(activity) do - follower_address = User.get_cached_by_ap_id(activity.data["actor"]).follower_address - Enum.any?(activity.data["to"], &(&1 == follower_address)) + with false <- is_public?(activity), + %User{follower_address: follower_address} <- + User.get_cached_by_ap_id(activity.data["actor"]) do + follower_address in activity.data["to"] else - false + _ -> false end end diff --git a/test/web/activity_pub/visibilty_test.exs b/test/web/activity_pub/visibilty_test.exs index 24b96c4aa..ff0e72401 100644 --- a/test/web/activity_pub/visibilty_test.exs +++ b/test/web/activity_pub/visibilty_test.exs @@ -95,4 +95,14 @@ test "visible_for_user?", %{ refute Visibility.visible_for_user?(private, unrelated) refute Visibility.visible_for_user?(direct, unrelated) end + + test "doesn't die when the user doesn't exist", + %{ + direct: direct, + user: user + } do + Repo.delete(user) + Cachex.clear(:user_cache) + refute Visibility.is_private?(direct) + end end From 6d19bb4eae43270099a68f749519ba0f323da01a Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 May 2019 18:09:07 +0200 Subject: [PATCH 004/123] Conversations: Add mix task to 'import' old DMs. --- lib/mix/tasks/pleroma/conversations.ex | 23 +++++++++++++++++++++++ lib/pleroma/conversation.ex | 11 +++++++---- 2 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 lib/mix/tasks/pleroma/conversations.ex diff --git a/lib/mix/tasks/pleroma/conversations.ex b/lib/mix/tasks/pleroma/conversations.ex new file mode 100644 index 000000000..125d8851a --- /dev/null +++ b/lib/mix/tasks/pleroma/conversations.ex @@ -0,0 +1,23 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Conversations do + use Mix.Task + alias Mix.Tasks.Pleroma.Common + alias Pleroma.Conversation + + @shortdoc "Manages Pleroma users" + @moduledoc """ + Manages Pleroma conversations. + + ## Create a conversation for all existing DMs. Can be safely re-run. + + mix pleroma.conversations bump_all + + """ + def run(["bump_all"]) do + Common.start_pleroma() + Conversation.bump_for_all_activities() + end +end diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index aa73edd75..10c2403e8 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -81,9 +81,12 @@ def bump_for_all_activities() do Pleroma.Web.ActivityPub.ActivityPub.fetch_direct_messages_query() |> Repo.stream() - Repo.transaction(fn -> - stream - |> Enum.each(&create_or_bump_for/1) - end) + Repo.transaction( + fn -> + stream + |> Enum.each(&create_or_bump_for/1) + end, + timeout: :infinity + ) end end From e6d7f8d223b2604df38f8efa8baf09e2c607c487 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 May 2019 18:19:20 +0200 Subject: [PATCH 005/123] Credo fixes. --- lib/pleroma/conversation.ex | 2 +- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index 10c2403e8..0c6ca9f72 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -76,7 +76,7 @@ def create_or_bump_for(activity) do @doc """ This is only meant to be run by a mix task. It creates conversations/participations for all direct messages in the database. """ - def bump_for_all_activities() do + def bump_for_all_activities do stream = Pleroma.Web.ActivityPub.ActivityPub.fetch_direct_messages_query() |> Repo.stream() diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index cd8495035..8137ac83b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1052,7 +1052,7 @@ def contain_timeline(timeline, user) do end) end - def fetch_direct_messages_query() do + def fetch_direct_messages_query do Activity |> restrict_type(%{"type" => "Create"}) |> restrict_visibility(%{visibility: "direct"}) From a33bec7d58091059d578f6b7537513de11eb0679 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 9 May 2019 16:39:28 +0200 Subject: [PATCH 006/123] Conversations: Import order, import as read. --- lib/pleroma/conversation.ex | 6 +++--- lib/pleroma/conversation/participation.ex | 10 ++++++---- lib/pleroma/web/activity_pub/activity_pub.ex | 1 + test/conversation_test.exs | 4 +++- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index 0c6ca9f72..5f6ab902c 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -45,7 +45,7 @@ def get_for_ap_id(ap_id) do 2. Create a participation for all the people involved who don't have one already 3. Bump all relevant participations to 'unread' """ - def create_or_bump_for(activity) do + def create_or_bump_for(activity, opts \\ []) do with true <- Pleroma.Web.ActivityPub.Visibility.is_direct?(activity), object <- Pleroma.Object.normalize(activity), "Create" <- activity.data["type"], @@ -58,7 +58,7 @@ def create_or_bump_for(activity) do participations = Enum.map(users, fn user -> {:ok, participation} = - Participation.create_for_user_and_conversation(user, conversation) + Participation.create_for_user_and_conversation(user, conversation, opts) participation end) @@ -84,7 +84,7 @@ def bump_for_all_activities do Repo.transaction( fn -> stream - |> Enum.each(&create_or_bump_for/1) + |> Enum.each(fn a -> create_or_bump_for(a, read: true) end) end, timeout: :infinity ) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 61021fb18..2a11f9069 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -22,15 +22,17 @@ defmodule Pleroma.Conversation.Participation do def creation_cng(struct, params) do struct - |> cast(params, [:user_id, :conversation_id]) + |> cast(params, [:user_id, :conversation_id, :read]) |> validate_required([:user_id, :conversation_id]) end - def create_for_user_and_conversation(user, conversation) do + def create_for_user_and_conversation(user, conversation, opts \\ []) do + read = !!opts[:read] + %__MODULE__{} - |> creation_cng(%{user_id: user.id, conversation_id: conversation.id}) + |> creation_cng(%{user_id: user.id, conversation_id: conversation.id, read: read}) |> Repo.insert( - on_conflict: [set: [read: false, updated_at: NaiveDateTime.utc_now()]], + on_conflict: [set: [read: read, updated_at: NaiveDateTime.utc_now()]], returning: true, conflict_target: [:user_id, :conversation_id] ) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 8137ac83b..728761ebd 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1056,5 +1056,6 @@ def fetch_direct_messages_query do Activity |> restrict_type(%{"type" => "Create"}) |> restrict_visibility(%{visibility: "direct"}) + |> order_by([activity], asc: activity.id) end end diff --git a/test/conversation_test.exs b/test/conversation_test.exs index 59368b0e7..cdec18f0f 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -24,7 +24,9 @@ test "it goes through old direct conversations" do Conversation.bump_for_all_activities() assert Repo.one(Conversation) - assert length(Repo.all(Conversation.Participation)) == 2 + [participation, _p2] = Repo.all(Conversation.Participation) + + assert participation.read end test "it creates a conversation for given ap_id" do From 01c45ddc9ead715131b3c583caa14fcf20845354 Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 11 May 2019 11:26:46 +0200 Subject: [PATCH 007/123] Search: Use RUM index. --- .../mastodon_api/mastodon_api_controller.ex | 6 ++-- ...510135645_add_fts_index_to_objects_two.exs | 33 +++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 956736780..32677df95 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1019,12 +1019,12 @@ def status_search(user, query) do where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients, where: fragment( - "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", - o.data, + "? @@ plainto_tsquery('english', ?)", + o.fts_content, ^query ), limit: 20, - order_by: [desc: :id] + order_by: [fragment("? <=> now()::date", o.inserted_at)] ) Repo.all(q) ++ fetched diff --git a/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs new file mode 100644 index 000000000..14b964847 --- /dev/null +++ b/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs @@ -0,0 +1,33 @@ +defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do + use Ecto.Migration + + def up do + drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) + alter table(:objects) do + add(:fts_content, :tsvector) + end + + execute("CREATE FUNCTION objects_fts_update() RETURNS trigger AS $$ + begin + new.fts_content := to_tsvector('english', new.data->>'content'); + return new; + end + $$ LANGUAGE plpgsql") + execute("create index objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');") + + execute("CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE ON objects + FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()") + + execute("UPDATE objects SET updated_at = NOW()") + end + + def down do + execute "drop index objects_fts" + execute "drop trigger tsvectorupdate on objects" + execute "drop function objects_fts_update()" + alter table(:objects) do + remove(:fts_content, :tsvector) + end + create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) + end +end From 786f2c7a849bc4fa2bd4aac18de59ef6b2ed18c5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 13 May 2019 11:16:54 -0500 Subject: [PATCH 008/123] Update shortdoc description --- lib/mix/tasks/pleroma/conversations.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/conversations.ex b/lib/mix/tasks/pleroma/conversations.ex index 125d8851a..b52b9921a 100644 --- a/lib/mix/tasks/pleroma/conversations.ex +++ b/lib/mix/tasks/pleroma/conversations.ex @@ -7,7 +7,7 @@ defmodule Mix.Tasks.Pleroma.Conversations do alias Mix.Tasks.Pleroma.Common alias Pleroma.Conversation - @shortdoc "Manages Pleroma users" + @shortdoc "Manages Pleroma conversations." @moduledoc """ Manages Pleroma conversations. From f1e67bdc312ba16a37916024244d6cb9d4417c9e Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 15 May 2019 15:28:01 +0200 Subject: [PATCH 009/123] Search: Add optional rum indexing / searching. --- config/config.exs | 2 + docs/config.md | 15 +++++++ .../mastodon_api/mastodon_api_controller.ex | 43 ++++++++++++++----- mix.exs | 5 ++- mix.lock | 10 ++--- ...510135645_add_fts_index_to_objects_two.exs | 3 ++ 6 files changed, 62 insertions(+), 16 deletions(-) rename priv/repo/{migrations => optional_migrations/rum_indexing}/20190510135645_add_fts_index_to_objects_two.exs (91%) diff --git a/config/config.exs b/config/config.exs index 1e64b79a7..42e4cb4ce 100644 --- a/config/config.exs +++ b/config/config.exs @@ -476,6 +476,8 @@ token_expires_in: 600, issue_new_refresh_token: true +config :pleroma, :database, rum_enabled: false + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/docs/config.md b/docs/config.md index 43ea24d80..99cee25cd 100644 --- a/docs/config.md +++ b/docs/config.md @@ -537,3 +537,18 @@ Configure OAuth 2 provider capabilities: * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]` * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]` * `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays). + +## Database options + +### RUM indexing for full text search +* `rum_enabled`: If RUM indexes should be used. Defaults to `false`. + +RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. While they may eventually be mainlined, for now they have to be installed as a PostgreSQL extension from https://github.com/postgrespro/rum. + +Their advantage over the standard GIN indexes is that they allow efficient ordering of search results by timestamp, which makes search queries a lot faster on larger servers, by one or two orders of magnitude. They take up around 3 times as much space as GIN indexes. + +To enable them, both the `rum_enabled` flag has to be set and the following special migration has to be run: + +`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/` + +This will probably take a long time. diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 32677df95..fc0100b1e 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1000,6 +1000,30 @@ def unsubscribe(%{assigns: %{user: user}} = conn, %{"id" => id}) do end end + def status_search_query_with_gin(q, query) do + from([a, o] in q, + where: + fragment( + "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", + o.data, + ^query + ), + order_by: [desc: :id] + ) + end + + def status_search_query_with_rum(q, query) do + from([a, o] in q, + where: + fragment( + "? @@ plainto_tsquery('english', ?)", + o.fts_content, + ^query + ), + order_by: [fragment("? <=> now()::date", o.inserted_at)] + ) + end + def status_search(user, query) do fetched = if Regex.match?(~r/https?:/, query) do @@ -1013,20 +1037,19 @@ def status_search(user, query) do end || [] q = - from( - [a, o] in Activity.with_preloaded_object(Activity), + from([a, o] in Activity.with_preloaded_object(Activity), where: fragment("?->>'type' = 'Create'", a.data), where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients, - where: - fragment( - "? @@ plainto_tsquery('english', ?)", - o.fts_content, - ^query - ), - limit: 20, - order_by: [fragment("? <=> now()::date", o.inserted_at)] + limit: 20 ) + q = + if Pleroma.Config.get([:database, :rum_enabled]) do + status_search_query_with_rum(q, query) + else + status_search_query_with_gin(q, query) + end + Repo.all(q) ++ fetched end diff --git a/mix.exs b/mix.exs index fae21f18d..f7955c6ca 100644 --- a/mix.exs +++ b/mix.exs @@ -65,7 +65,10 @@ defp deps do {:plug_cowboy, "~> 2.0"}, {:phoenix_pubsub, "~> 1.1"}, {:phoenix_ecto, "~> 4.0"}, - {:ecto_sql, "~>3.0.5"}, + {:ecto_sql, + git: "https://github.com/elixir-ecto/ecto_sql", + ref: "e839a9a327b632d73533ac8105ba360bc831cf83", + override: true}, {:postgrex, ">= 0.13.5"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, diff --git a/mix.lock b/mix.lock index 624c0fb35..d223803ab 100644 --- a/mix.lock +++ b/mix.lock @@ -16,12 +16,12 @@ "cowlib": {:hex, :cowlib, "2.7.0", "3ef16e77562f9855a2605900cedb15c1462d76fb1be6a32fc3ae91973ee543d2", [:rebar3], [], "hexpm"}, "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, - "db_connection": {:hex, :db_connection, "2.0.5", "ddb2ba6761a08b2bb9ca0e7d260e8f4dd39067426d835c24491a321b7f92a4da", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, + "db_connection": {:hex, :db_connection, "2.0.6", "bde2f85d047969c5b5800cb8f4b3ed6316c8cb11487afedac4aa5f93fd39abfa", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, "decimal": {:hex, :decimal, "1.7.0", "30d6b52c88541f9a66637359ddf85016df9eb266170d53105f02e4a67e00c5aa", [:mix], [], "hexpm"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.0.7", "44dda84ac6b17bbbdeb8ac5dfef08b7da253b37a453c34ab1a98de7f7e5fec7f", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "e839a9a327b632d73533ac8105ba360bc831cf83", [ref: "e839a9a327b632d73533ac8105ba360bc831cf83"]}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"}, "ex_aws": {:hex, :ex_aws, "2.1.0", "b92651527d6c09c479f9013caa9c7331f19cba38a650590d82ebf2c6c16a1d8a", [:mix], [{:configparser_ex, "~> 2.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"}, @@ -61,7 +61,7 @@ "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, - "postgrex": {:hex, :postgrex, "0.14.1", "63247d4a5ad6b9de57a0bac5d807e1c32d41e39c04b8a4156a26c63bcd8a2e49", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus": {:hex, :prometheus, "4.2.2", "a830e77b79dc6d28183f4db050a7cac926a6c58f1872f9ef94a35cd989aceef8", [:mix, :rebar3], [], "hexpm"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, @@ -74,7 +74,7 @@ "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, - "telemetry": {:hex, :telemetry, "0.3.0", "099a7f3ce31e4780f971b4630a3c22ec66d22208bc090fe33a2a3a6a67754a73", [:rebar3], [], "hexpm"}, + "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, "tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, "timex": {:hex, :timex, "3.5.0", "b0a23167da02d0fe4f1a4e104d1f929a00d348502b52432c05de875d0b9cffa5", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs similarity index 91% rename from priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs rename to priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs index 14b964847..09e6cbfb1 100644 --- a/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs +++ b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs @@ -2,6 +2,7 @@ defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do use Ecto.Migration def up do + execute("create extension if not exists rum") drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) alter table(:objects) do add(:fts_content, :tsvector) @@ -19,6 +20,7 @@ def up do FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()") execute("UPDATE objects SET updated_at = NOW()") + execute("vacuum analyze") end def down do @@ -29,5 +31,6 @@ def down do remove(:fts_content, :tsvector) end create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) + execute("vacuum analyze") end end From 88cc423fa3c4baa698e99e1e62d251b6f81cc93d Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 15 May 2019 16:45:26 +0200 Subject: [PATCH 010/123] Dependencies: Update Ecto SQL. --- mix.exs | 5 ++++- mix.lock | 10 +++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/mix.exs b/mix.exs index b7b9d534d..2535fcf98 100644 --- a/mix.exs +++ b/mix.exs @@ -65,7 +65,10 @@ defp deps do {:plug_cowboy, "~> 2.0"}, {:phoenix_pubsub, "~> 1.1"}, {:phoenix_ecto, "~> 4.0"}, - {:ecto_sql, "~>3.0.5"}, + {:ecto_sql, + git: "https://github.com/elixir-ecto/ecto_sql", + ref: "e839a9a327b632d73533ac8105ba360bc831cf83", + override: true}, {:postgrex, ">= 0.13.5"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, diff --git a/mix.lock b/mix.lock index 0b24818c5..5ee5634d3 100644 --- a/mix.lock +++ b/mix.lock @@ -16,12 +16,12 @@ "cowlib": {:hex, :cowlib, "2.7.0", "3ef16e77562f9855a2605900cedb15c1462d76fb1be6a32fc3ae91973ee543d2", [:rebar3], [], "hexpm"}, "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, - "db_connection": {:hex, :db_connection, "2.0.5", "ddb2ba6761a08b2bb9ca0e7d260e8f4dd39067426d835c24491a321b7f92a4da", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, + "db_connection": {:hex, :db_connection, "2.0.6", "bde2f85d047969c5b5800cb8f4b3ed6316c8cb11487afedac4aa5f93fd39abfa", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, "decimal": {:hex, :decimal, "1.7.0", "30d6b52c88541f9a66637359ddf85016df9eb266170d53105f02e4a67e00c5aa", [:mix], [], "hexpm"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.0.7", "44dda84ac6b17bbbdeb8ac5dfef08b7da253b37a453c34ab1a98de7f7e5fec7f", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "e839a9a327b632d73533ac8105ba360bc831cf83", [ref: "e839a9a327b632d73533ac8105ba360bc831cf83"]}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, @@ -64,7 +64,7 @@ "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, - "postgrex": {:hex, :postgrex, "0.14.1", "63247d4a5ad6b9de57a0bac5d807e1c32d41e39c04b8a4156a26c63bcd8a2e49", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus": {:hex, :prometheus, "4.2.2", "a830e77b79dc6d28183f4db050a7cac926a6c58f1872f9ef94a35cd989aceef8", [:mix, :rebar3], [], "hexpm"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, @@ -77,7 +77,7 @@ "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, - "telemetry": {:hex, :telemetry, "0.3.0", "099a7f3ce31e4780f971b4630a3c22ec66d22208bc090fe33a2a3a6a67754a73", [:rebar3], [], "hexpm"}, + "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, "tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, "timex": {:hex, :timex, "3.5.0", "b0a23167da02d0fe4f1a4e104d1f929a00d348502b52432c05de875d0b9cffa5", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, From 958b4cfde916c9be71d7412fe1c90750ec578fdc Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 24 Mar 2019 23:45:57 +0000 Subject: [PATCH 011/123] migrations: add function to see if a thread can be satisfied --- ...4222404_add_thread_visibility_function.exs | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 priv/repo/migrations/20190324222404_add_thread_visibility_function.exs diff --git a/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs new file mode 100644 index 000000000..cea0322e7 --- /dev/null +++ b/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs @@ -0,0 +1,76 @@ +defmodule Pleroma.Repo.Migrations.AddThreadVisibilityFunction do + use Ecto.Migration + @disable_ddl_transaction true + + def up do + statement = """ + CREATE OR REPLACE FUNCTION thread_visibility(actor varchar, activity_id varchar) RETURNS boolean AS $$ + DECLARE + public varchar := 'https://www.w3.org/ns/activitystreams#Public'; + child objects%ROWTYPE; + activity activities%ROWTYPE; + actor_user users%ROWTYPE; + author users%ROWTYPE; + author_fa varchar; + BEGIN + --- Fetch our actor. + SELECT * INTO actor_user FROM users WHERE users.ap_id = actor; + + --- Fetch our initial activity. + SELECT * INTO activity FROM activities WHERE activities.data->>'id' = activity_id; + + LOOP + --- Ensure that we have an activity before continuing. + IF activity IS NULL THEN + RETURN true; + END IF; + + --- Normalize the child object into child. + SELECT * INTO child FROM objects + INNER JOIN activities ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' + WHERE COALESCE(activity.data->'object'->>'id', activity.data->>'object') = objects.data->>'id'; + + --- Fetch the author. + SELECT * INTO author FROM users WHERE users.ap_id = activity.actor; + + --- Prepare author's AS2 followers collection. + SELECT COALESCE(author.follower_address, '') INTO author_fa; + + --- Check visibility. + IF activity.actor = actor THEN + --- activity visible + NULL; + ELSIF ARRAY[public] && activity.recipients THEN + --- activity visible + NULL; + ELSIF ARRAY[author_fa] && activity.recipients AND ARRAY[author_fa] && actor_user.following THEN + --- activity visible + NULL; + ELSIF ARRAY[actor] && activity.recipients THEN + --- activity visible + NULL; + ELSE + --- activity not visible, break out of the loop + RETURN false; + END IF; + + --- If there's a parent, load it and do this all over again. + IF (child.data->'inReplyTo' IS NOT NULL) AND (child.data->'inReplyTo' != 'null'::jsonb) THEN + SELECT * INTO activity FROM activities + INNER JOIN objects ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' + WHERE child.data->>'inReplyTo' = objects.data->>'id'; + ELSE + RETURN true; + END IF; + END LOOP; + END; + $$ LANGUAGE plpgsql IMMUTABLE; + """ + + execute(statement) + end + + def down do + execute("drop function thread_visibility(actor varchar, activity_id varchar)") + end +end From 0387f5213805cdc4e0bf86f98797cefcd03ba61d Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 25 Mar 2019 00:06:02 +0000 Subject: [PATCH 012/123] activitypub: add restrict_thread_visibility() --- lib/pleroma/web/activity_pub/activity_pub.ex | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 233fee4fa..fec1bcd3e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -569,6 +569,20 @@ defp restrict_visibility(_query, %{visibility: visibility}) defp restrict_visibility(query, _visibility), do: query + defp restrict_thread_visibility(query, %{"user" => %User{ap_id: ap_id}}) do + query = + from( + a in query, + where: fragment("thread_visibility(?, (?)->>'id') = true", ^ap_id, a.data) + ) + + Ecto.Adapters.SQL.to_sql(:all, Repo, query) + + query + end + + defp restrict_thread_visibility(query, _), do: query + def fetch_user_activities(user, reading_user, params \\ %{}) do params = params @@ -848,6 +862,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do |> restrict_muted(opts) |> restrict_media(opts) |> restrict_visibility(opts) + |> restrict_thread_visibility(opts) |> restrict_replies(opts) |> restrict_reblogs(opts) |> restrict_pinned(opts) From de114ffbb0f92d24fd370adaaf43ff301ab04b4b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 25 Mar 2019 00:10:20 +0000 Subject: [PATCH 013/123] activitypub: remove contain_timeline() --- lib/pleroma/web/activity_pub/activity_pub.ex | 8 -------- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 1 - lib/pleroma/web/twitter_api/twitter_api_controller.ex | 4 +--- 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index fec1bcd3e..e544d0c50 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -980,12 +980,4 @@ def contain_broken_threads(%Activity{} = activity, %User{} = user) do def contain_activity(%Activity{} = activity, %User{} = user) do contain_broken_threads(activity, user) end - - # do post-processing on a timeline - def contain_timeline(timeline, user) do - timeline - |> Enum.filter(fn activity -> - contain_activity(activity, user) - end) - end end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 87e597074..66056a846 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -303,7 +303,6 @@ def home_timeline(%{assigns: %{user: user}} = conn, params) do activities = [user.ap_id | user.following] |> ActivityPub.fetch_activities(params) - |> ActivityPub.contain_timeline(user) |> Enum.reverse() conn diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 3c5a70be9..31e86685a 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -101,9 +101,7 @@ def friends_timeline(%{assigns: %{user: user}} = conn, params) do |> Map.put("blocking_user", user) |> Map.put("user", user) - activities = - ActivityPub.fetch_activities([user.ap_id | user.following], params) - |> ActivityPub.contain_timeline(user) + activities = ActivityPub.fetch_activities([user.ap_id | user.following], params) conn |> put_view(ActivityView) From 31db31c5879a2dedcc8dd4c671c4c9a79656355a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 25 Mar 2019 00:38:28 +0000 Subject: [PATCH 014/123] activitypub: visibility: use SQL thread_visibility() function instead of manually walking the thread --- lib/pleroma/web/activity_pub/visibility.ex | 26 +++++++--------------- 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index b38ee0442..46dd46575 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -1,6 +1,7 @@ defmodule Pleroma.Web.ActivityPub.Visibility do alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.Repo alias Pleroma.User def is_public?(%Object{data: %{"type" => "Tombstone"}}), do: false @@ -38,25 +39,14 @@ def visible_for_user?(activity, user) do visible_for_user?(activity, nil) || Enum.any?(x, &(&1 in y)) end - # guard - def entire_thread_visible_for_user?(nil, _user), do: false + def entire_thread_visible_for_user?(%Activity{} = activity, %User{} = user) do + {:ok, %{rows: [[result]]}} = + Ecto.Adapters.SQL.query(Repo, "SELECT thread_visibility($1, $2)", [ + user.ap_id, + activity.data["id"] + ]) - # XXX: Probably even more inefficient than the previous implementation intended to be a placeholder untill https://git.pleroma.social/pleroma/pleroma/merge_requests/971 is in develop - # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength - - def entire_thread_visible_for_user?( - %Activity{} = tail, - # %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail, - user - ) do - case Object.normalize(tail) do - %{data: %{"inReplyTo" => parent_id}} when is_binary(parent_id) -> - parent = Activity.get_in_reply_to_activity(tail) - visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user) - - _ -> - visible_for_user?(tail, user) - end + result end def get_visibility(object) do From c7644313e72520a371e4bd417b1ff852365849b6 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 25 Mar 2019 01:23:15 +0000 Subject: [PATCH 015/123] test: update obsolete test --- test/web/activity_pub/activity_pub_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 0f90aa1ac..b41f6ab81 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -968,7 +968,8 @@ test "it filters broken threads" do assert length(activities) == 3 - activities = ActivityPub.contain_timeline(activities, user1) + activities = + ActivityPub.fetch_activities([user1.ap_id | user1.following], %{"user" => user1}) assert [public_activity, private_activity_1] == activities assert length(activities) == 2 From 75ce6adcffd2dbbc2ca2f83d7fe2d7fd659cd2f4 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 25 Mar 2019 02:56:13 +0000 Subject: [PATCH 016/123] migration: only care about Create activities --- .../20190324222404_add_thread_visibility_function.exs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs index cea0322e7..11aa47e83 100644 --- a/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs +++ b/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs @@ -25,6 +25,11 @@ def up do RETURN true; END IF; + --- We only care about Create activities. + IF activity.data->>'type' != 'Create' THEN + RETURN true; + END IF; + --- Normalize the child object into child. SELECT * INTO child FROM objects INNER JOIN activities ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' From 0aada88b5594b6714b8d65f8bee9c325d77d6e7b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 8 May 2019 23:17:51 +0000 Subject: [PATCH 017/123] bbs: chase timeline containment patch --- lib/pleroma/bbs/handler.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index 106fe5d18..f34be961f 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -95,7 +95,6 @@ def handle_command(state, "home") do activities = [user.ap_id | user.following] |> ActivityPub.fetch_activities(params) - |> ActivityPub.contain_timeline(user) Enum.each(activities, fn activity -> puts_activity(activity) From 12f45e2a8907c74c6b65d866bc3bab547b31edfa Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 15 May 2019 16:22:52 +0000 Subject: [PATCH 018/123] update migration --- ...n.exs => 20190515222404_add_thread_visibility_function.exs} | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) rename priv/repo/migrations/{20190324222404_add_thread_visibility_function.exs => 20190515222404_add_thread_visibility_function.exs} (97%) diff --git a/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs similarity index 97% rename from priv/repo/migrations/20190324222404_add_thread_visibility_function.exs rename to priv/repo/migrations/20190515222404_add_thread_visibility_function.exs index 11aa47e83..a3f717b89 100644 --- a/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs +++ b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs @@ -21,8 +21,9 @@ def up do LOOP --- Ensure that we have an activity before continuing. + --- If we don't, the thread is not satisfiable. IF activity IS NULL THEN - RETURN true; + RETURN false; END IF; --- We only care about Create activities. From f09c3afdf51eea17103d1445b31b7a269c474538 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 15 May 2019 16:23:01 +0000 Subject: [PATCH 019/123] chase test failures --- lib/pleroma/filter.ex | 3 ++- test/user_test.exs | 2 -- test/web/activity_pub/activity_pub_test.exs | 9 ++++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index 79efc29f0..90457dadf 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -38,7 +38,8 @@ def get_filters(%User{id: user_id} = _user) do query = from( f in Pleroma.Filter, - where: f.user_id == ^user_id + where: f.user_id == ^user_id, + order_by: [desc: :id] ) Repo.all(query) diff --git a/test/user_test.exs b/test/user_test.exs index 0b65e89e9..bb47b4958 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -873,7 +873,6 @@ test "hide a user's statuses from timelines and notifications" do assert [activity] == ActivityPub.fetch_activities([user2.ap_id | user2.following], %{"user" => user2}) - |> ActivityPub.contain_timeline(user2) {:ok, _user} = User.deactivate(user) @@ -882,7 +881,6 @@ test "hide a user's statuses from timelines and notifications" do assert [] == ActivityPub.fetch_activities([user2.ap_id | user2.following], %{"user" => user2}) - |> ActivityPub.contain_timeline(user2) end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index b41f6ab81..34e23b852 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -960,18 +960,21 @@ test "it filters broken threads" do "in_reply_to_status_id" => private_activity_2.id }) - activities = ActivityPub.fetch_activities([user1.ap_id | user1.following]) + activities = + ActivityPub.fetch_activities([user1.ap_id | user1.following]) + |> Enum.map(fn a -> a.id end) private_activity_1 = Activity.get_by_ap_id_with_object(private_activity_1.data["id"]) - assert [public_activity, private_activity_1, private_activity_3] == activities + assert [public_activity.id, private_activity_1.id, private_activity_3.id] == activities assert length(activities) == 3 activities = ActivityPub.fetch_activities([user1.ap_id | user1.following], %{"user" => user1}) + |> Enum.map(fn a -> a.id end) - assert [public_activity, private_activity_1] == activities + assert [public_activity.id, private_activity_1.id] == activities assert length(activities) == 2 end end From 71fa7eeb6fdc7cf2087a32fb515ad11b7bf90c01 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 15 May 2019 16:54:14 +0000 Subject: [PATCH 020/123] thread visibility function: significantly improve efficiency --- ...5222404_add_thread_visibility_function.exs | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs index a3f717b89..a4daf680b 100644 --- a/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs +++ b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs @@ -10,8 +10,8 @@ def up do child objects%ROWTYPE; activity activities%ROWTYPE; actor_user users%ROWTYPE; - author users%ROWTYPE; author_fa varchar; + valid_recipients varchar[]; BEGIN --- Fetch our actor. SELECT * INTO actor_user FROM users WHERE users.ap_id = actor; @@ -36,26 +36,17 @@ def up do INNER JOIN activities ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' WHERE COALESCE(activity.data->'object'->>'id', activity.data->>'object') = objects.data->>'id'; - --- Fetch the author. - SELECT * INTO author FROM users WHERE users.ap_id = activity.actor; + --- Fetch the author's AS2 following collection. + SELECT COALESCE(author.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor; - --- Prepare author's AS2 followers collection. - SELECT COALESCE(author.follower_address, '') INTO author_fa; + --- Prepare valid recipients array. + valid_recipients := ARRAY[actor, public]; + IF ARRAY[author_fa] && actor_user.following THEN + valid_recipients := valid_recipients || author_fa; + END IF; --- Check visibility. - IF activity.actor = actor THEN - --- activity visible - NULL; - ELSIF ARRAY[public] && activity.recipients THEN - --- activity visible - NULL; - ELSIF ARRAY[author_fa] && activity.recipients AND ARRAY[author_fa] && actor_user.following THEN - --- activity visible - NULL; - ELSIF ARRAY[actor] && activity.recipients THEN - --- activity visible - NULL; - ELSE + IF NOT valid_recipients && activity.recipients THEN --- activity not visible, break out of the loop RETURN false; END IF; From a591ab6112abdf162f4d6fdfbbcdd85bbaf75058 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 15 May 2019 16:56:46 +0000 Subject: [PATCH 021/123] activity pub: remove Ecto SQL query dumps --- lib/pleroma/web/activity_pub/activity_pub.ex | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index e544d0c50..7cd5b889b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -540,8 +540,6 @@ defp restrict_visibility(query, %{visibility: visibility}) ) ) - Ecto.Adapters.SQL.to_sql(:all, Repo, query) - query else Logger.error("Could not restrict visibility to #{visibility}") @@ -557,8 +555,6 @@ defp restrict_visibility(query, %{visibility: visibility}) fragment("activity_visibility(?, ?, ?) = ?", a.actor, a.recipients, a.data, ^visibility) ) - Ecto.Adapters.SQL.to_sql(:all, Repo, query) - query end @@ -576,8 +572,6 @@ defp restrict_thread_visibility(query, %{"user" => %User{ap_id: ap_id}}) do where: fragment("thread_visibility(?, (?)->>'id') = true", ^ap_id, a.data) ) - Ecto.Adapters.SQL.to_sql(:all, Repo, query) - query end From f3971cbde3d69faec973717e1421f4a643ef947e Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 15 May 2019 17:02:40 +0000 Subject: [PATCH 022/123] thread visibility function: fix use of no longer used author variable --- .../20190515222404_add_thread_visibility_function.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs index a4daf680b..dc9abc998 100644 --- a/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs +++ b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs @@ -37,7 +37,7 @@ def up do WHERE COALESCE(activity.data->'object'->>'id', activity.data->>'object') = objects.data->>'id'; --- Fetch the author's AS2 following collection. - SELECT COALESCE(author.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor; + SELECT COALESCE(users.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor; --- Prepare valid recipients array. valid_recipients := ARRAY[actor, public]; From 64495b96e9159b88ef0b4a6486e60b54722664fb Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Wed, 15 May 2019 06:13:47 +0800 Subject: [PATCH 023/123] Add test for blocked domains --- test/web/activity_pub/activity_pub_test.exs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 0f90aa1ac..442120ffd 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -462,6 +462,20 @@ test "doesn't return announce activities concerning blocked users" do refute Enum.member?(activities, activity_three.id) end + test "doesn't return activities from blocked domains" do + domain = "dogwhistle.zone" + domain_user = insert(:user, %{ap_id: "https://#{domain}/@pundit"}) + note = insert(:note, %{data: %{"actor" => domain_user.ap_id}}) + activity = insert(:note_activity, %{note: note}) + user = insert(:user) + {:ok, user} = User.block_domain(user, domain) + + activities = + ActivityPub.fetch_activities([], %{"blocking_user" => user, "skip_preload" => true}) + + refute activity in activities + end + test "doesn't return muted activities" do activity_one = insert(:note_activity) activity_two = insert(:note_activity) From 2b6119dfbf47f118dfb102fc09f6450e59ccf15e Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Wed, 15 May 2019 07:59:24 +0800 Subject: [PATCH 024/123] Restrict reblogs of activities from blocked domains --- lib/pleroma/web/activity_pub/activity_pub.ex | 11 ++++++++++- test/web/activity_pub/activity_pub_test.exs | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 233fee4fa..6087a1cdc 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -752,6 +752,14 @@ defp restrict_blocked(query, %{"blocking_user" => %User{info: info}}) do from( activity in query, + inner_join: o in Object, + on: + fragment( + "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')", + o.data, + activity.data, + activity.data + ), where: fragment("not (? = ANY(?))", activity.actor, ^blocks), where: fragment("not (? && ?)", activity.recipients, ^blocks), where: @@ -761,7 +769,8 @@ defp restrict_blocked(query, %{"blocking_user" => %User{info: info}}) do activity.data, ^blocks ), - where: fragment("not (split_part(?, '/', 3) = ANY(?))", activity.actor, ^domain_blocks) + where: fragment("not (split_part(?, '/', 3) = ANY(?))", activity.actor, ^domain_blocks), + where: fragment("not (split_part(?->>'actor', '/', 3) = ANY(?))", o.data, ^domain_blocks) ) end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 442120ffd..dfee93f67 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -474,6 +474,15 @@ test "doesn't return activities from blocked domains" do ActivityPub.fetch_activities([], %{"blocking_user" => user, "skip_preload" => true}) refute activity in activities + + followed_user = insert(:user) + ActivityPub.follow(user, followed_user) + {:ok, repeat_activity, _} = CommonAPI.repeat(activity.id, followed_user) + + activities = + ActivityPub.fetch_activities([], %{"blocking_user" => user, "skip_preload" => true}) + + refute repeat_activity in activities end test "doesn't return muted activities" do From 793f1834d2bc37ba3d4ba11fb669078950994adb Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Thu, 16 May 2019 06:25:14 +0800 Subject: [PATCH 025/123] Use named binding to conditionally join object --- lib/pleroma/activity.ex | 19 +++++++++++-------- lib/pleroma/web/activity_pub/activity_pub.ex | 13 ++++--------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 4a0919478..3fd4003f8 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -60,21 +60,24 @@ defmodule Pleroma.Activity do timestamps() end - def with_preloaded_object(query) do - query - |> join( - :inner, - [activity], - o in Object, + def with_joined_object(query) do + join(query, :inner, [activity], o in Object, on: fragment( "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')", o.data, activity.data, activity.data - ) + ), + as: :object ) - |> preload([activity, object], object: object) + end + + def with_preloaded_object(query) do + query + |> has_named_binding?(:object) + |> if(do: query, else: with_joined_object(query)) + |> preload([activity, object: object], object: object) end def with_preloaded_bookmark(query, %User{} = user) do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 6087a1cdc..2fd073d3a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -750,16 +750,11 @@ defp restrict_blocked(query, %{"blocking_user" => %User{info: info}}) do blocks = info.blocks || [] domain_blocks = info.domain_blocks || [] + query = + if has_named_binding?(query, :object), do: query, else: Activity.with_joined_object(query) + from( - activity in query, - inner_join: o in Object, - on: - fragment( - "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')", - o.data, - activity.data, - activity.data - ), + [activity, object: o] in query, where: fragment("not (? = ANY(?))", activity.actor, ^blocks), where: fragment("not (? && ?)", activity.recipients, ^blocks), where: From 0ae15c23061413cb2b67d2be462742dccae01fb4 Mon Sep 17 00:00:00 2001 From: PolymerWitch Date: Wed, 15 May 2019 15:36:20 -0700 Subject: [PATCH 026/123] Added package to dependency list Added the erlang-ssh package to the dependency list and the installation command instructions. The project wouldn't build otherwise. --- docs/installation/debian_based_en.md | 3 ++- docs/installation/debian_based_jp.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md index 9613a329b..9c0ef92d4 100644 --- a/docs/installation/debian_based_en.md +++ b/docs/installation/debian_based_en.md @@ -12,6 +12,7 @@ This guide will assume you are on Debian Stretch. This guide should also work wi * `erlang-tools` * `erlang-parsetools` * `erlang-eldap`, if you want to enable ldap authenticator +* `erlang-ssh` * `erlang-xmerl` * `git` * `build-essential` @@ -49,7 +50,7 @@ sudo dpkg -i /tmp/erlang-solutions_1.0_all.deb ```shell sudo apt update -sudo apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools +sudo apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools erlang-ssh ``` ### Install PleromaBE diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md index ac5dcaaee..41cce6792 100644 --- a/docs/installation/debian_based_jp.md +++ b/docs/installation/debian_based_jp.md @@ -14,6 +14,7 @@ - erlang-dev - erlang-tools - erlang-parsetools +- erlang-ssh - erlang-xmerl (Jessieではバックポートからインストールすること!) - git - build-essential @@ -44,7 +45,7 @@ wget -P /tmp/ https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb * ElixirとErlangをインストールします、 ``` -apt update && apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools +apt update && apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools erlang-ssh ``` ### Pleroma BE (バックエンド) をインストールします From 1d2923e5d0893b39f41ded77d24652029f6f12ec Mon Sep 17 00:00:00 2001 From: Angelina Filippova Date: Thu, 16 May 2019 01:36:26 +0300 Subject: [PATCH 027/123] Update tag/untag docs --- docs/api/admin_api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 75fa2ee83..8f79d7f24 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -106,14 +106,14 @@ Authentication is required and the user must be an admin. - Method: `PUT` - Params: - - `nickname` + - `nicknames` - `tags` ### Untag a list of users - Method: `DELETE` - Params: - - `nickname` + - `nicknames` - `tags` ## `/api/pleroma/admin/users/:nickname/permission_group` From 0641e685c166859fe2d5994bfcd069e0a8e62acf Mon Sep 17 00:00:00 2001 From: Angelina Filippova Date: Thu, 16 May 2019 01:46:43 +0300 Subject: [PATCH 028/123] Note that nicknames is an array --- docs/api/admin_api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 8f79d7f24..94fce9c6f 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -106,14 +106,14 @@ Authentication is required and the user must be an admin. - Method: `PUT` - Params: - - `nicknames` + - `nicknames` (array) - `tags` ### Untag a list of users - Method: `DELETE` - Params: - - `nicknames` + - `nicknames` (array) - `tags` ## `/api/pleroma/admin/users/:nickname/permission_group` From 107c83ab2b186c951dc27140908d414a202119d8 Mon Sep 17 00:00:00 2001 From: Angelina Filippova Date: Thu, 16 May 2019 01:48:53 +0300 Subject: [PATCH 029/123] Note that nicknames is an array --- docs/api/admin_api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 94fce9c6f..59578f8d1 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -107,14 +107,14 @@ Authentication is required and the user must be an admin. - Method: `PUT` - Params: - `nicknames` (array) - - `tags` + - `tags` (array) ### Untag a list of users - Method: `DELETE` - Params: - `nicknames` (array) - - `tags` + - `tags` (array) ## `/api/pleroma/admin/users/:nickname/permission_group` From c31026423c8c73ab33dc0d38b9d187a0d2b68309 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 16 May 2019 04:41:27 +0000 Subject: [PATCH 030/123] publisher: use the correct queue name for outgoing federation --- lib/pleroma/web/federator/publisher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index 916bcdcba..fb4e8548d 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Web.Federator.Publisher do """ @spec enqueue_one(module(), Map.t()) :: :ok def enqueue_one(module, %{} = params), - do: PleromaJobQueue.enqueue(:federation_outgoing, __MODULE__, [:publish_one, module, params]) + do: PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish_one, module, params]) @spec perform(atom(), module(), any()) :: {:ok, any()} | {:error, any()} def perform(:publish_one, module, params) do From aa11fa4864bf6ebdd4acfdba837675abbdcffc83 Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 16 May 2019 12:49:40 +0700 Subject: [PATCH 031/123] add report uri and report to --- CHANGELOG.md | 3 +- config/test.exs | 2 + docs/config.md | 3 +- lib/pleroma/plugs/http_security_plug.ex | 31 ++++-- test/plugs/http_security_plug_test.exs | 133 ++++++++++++++---------- 5 files changed, 107 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0e849285..ea1f29304 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Configuration: `fetch_initial_posts` option - Configuration: `notify_email` option - Configuration: Media proxy `whitelist` option +- Configuration: `report_uri` option - Pleroma API: User subscriptions - Pleroma API: Healthcheck endpoint - Admin API: Endpoints for listing/revoking invite tokens @@ -98,7 +99,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`] ## Removed -- Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations` +- Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations` ## [0.9.9999] - 2019-04-05 ### Security diff --git a/config/test.exs b/config/test.exs index a0c90c371..40db66170 100644 --- a/config/test.exs +++ b/config/test.exs @@ -61,6 +61,8 @@ config :pleroma, :app_account_creation, max_requests: 5 +config :pleroma, :http_security, report_uri: "https://endpoint.com" + try do import_config "test.secret.exs" rescue diff --git a/docs/config.md b/docs/config.md index 470f71b7c..c2af5c012 100644 --- a/docs/config.md +++ b/docs/config.md @@ -286,7 +286,8 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start * ``sts``: Whether to additionally send a `Strict-Transport-Security` header * ``sts_max_age``: The maximum age for the `Strict-Transport-Security` header if sent * ``ct_max_age``: The maximum age for the `Expect-CT` header if sent -* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`. +* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"` +* ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header. ## :mrf_user_allowlist diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index a476f1d49..485ddfbc7 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -20,8 +20,9 @@ def call(conn, _options) do defp headers do referrer_policy = Config.get([:http_security, :referrer_policy]) + report_uri = Config.get([:http_security, :report_uri]) - [ + headers = [ {"x-xss-protection", "1; mode=block"}, {"x-permitted-cross-domain-policies", "none"}, {"x-frame-options", "DENY"}, @@ -30,12 +31,27 @@ defp headers do {"x-download-options", "noopen"}, {"content-security-policy", csp_string() <> ";"} ] + + if report_uri do + report_group = %{ + "group" => "csp-endpoint", + "max-age" => 10_886_400, + "endpoints" => [ + %{"url" => report_uri} + ] + } + + headers ++ [{"reply-to", Jason.encode!(report_group)}] + else + headers + end end defp csp_string do scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme] static_url = Pleroma.Web.Endpoint.static_url() websocket_url = Pleroma.Web.Endpoint.websocket_url() + report_uri = Config.get([:http_security, :report_uri]) connect_src = "connect-src 'self' #{static_url} #{websocket_url}" @@ -53,7 +69,7 @@ defp csp_string do "script-src 'self'" end - [ + main_part = [ "default-src 'none'", "base-uri 'self'", "frame-ancestors 'none'", @@ -63,11 +79,14 @@ defp csp_string do "font-src 'self'", "manifest-src 'self'", connect_src, - script_src, - if scheme == "https" do - "upgrade-insecure-requests" - end + script_src ] + + report = if report_uri, do: ["report-uri #{report_uri}; report-to csp-endpoint"], else: [] + + insecure = if scheme == "https", do: ["upgrade-insecure-requests"], else: [] + + (main_part ++ report ++ insecure) |> Enum.join("; ") end diff --git a/test/plugs/http_security_plug_test.exs b/test/plugs/http_security_plug_test.exs index 0cbb7e4b1..7dfd50c1f 100644 --- a/test/plugs/http_security_plug_test.exs +++ b/test/plugs/http_security_plug_test.exs @@ -7,28 +7,89 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do alias Pleroma.Config alias Plug.Conn - test "it sends CSP headers when enabled", %{conn: conn} do - Config.put([:http_security, :enabled], true) + describe "http security enabled" do + setup do + enabled = Config.get([:http_securiy, :enabled]) - conn = - conn - |> get("/api/v1/instance") + Config.put([:http_security, :enabled], true) - refute Conn.get_resp_header(conn, "x-xss-protection") == [] - refute Conn.get_resp_header(conn, "x-permitted-cross-domain-policies") == [] - refute Conn.get_resp_header(conn, "x-frame-options") == [] - refute Conn.get_resp_header(conn, "x-content-type-options") == [] - refute Conn.get_resp_header(conn, "x-download-options") == [] - refute Conn.get_resp_header(conn, "referrer-policy") == [] - refute Conn.get_resp_header(conn, "content-security-policy") == [] + on_exit(fn -> + Config.put([:http_security, :enabled], enabled) + end) + + :ok + end + + test "it sends CSP headers when enabled", %{conn: conn} do + conn = get(conn, "/api/v1/instance") + + refute Conn.get_resp_header(conn, "x-xss-protection") == [] + refute Conn.get_resp_header(conn, "x-permitted-cross-domain-policies") == [] + refute Conn.get_resp_header(conn, "x-frame-options") == [] + refute Conn.get_resp_header(conn, "x-content-type-options") == [] + refute Conn.get_resp_header(conn, "x-download-options") == [] + refute Conn.get_resp_header(conn, "referrer-policy") == [] + refute Conn.get_resp_header(conn, "content-security-policy") == [] + end + + test "it sends STS headers when enabled", %{conn: conn} do + Config.put([:http_security, :sts], true) + + conn = get(conn, "/api/v1/instance") + + refute Conn.get_resp_header(conn, "strict-transport-security") == [] + refute Conn.get_resp_header(conn, "expect-ct") == [] + end + + test "it does not send STS headers when disabled", %{conn: conn} do + Config.put([:http_security, :sts], false) + + conn = get(conn, "/api/v1/instance") + + assert Conn.get_resp_header(conn, "strict-transport-security") == [] + assert Conn.get_resp_header(conn, "expect-ct") == [] + end + + test "referrer-policy header reflects configured value", %{conn: conn} do + conn = get(conn, "/api/v1/instance") + + assert Conn.get_resp_header(conn, "referrer-policy") == ["same-origin"] + + Config.put([:http_security, :referrer_policy], "no-referrer") + + conn = + build_conn() + |> get("/api/v1/instance") + + assert Conn.get_resp_header(conn, "referrer-policy") == ["no-referrer"] + end + + test "it sends `report-to` & `report-uri` CSP response headers" do + conn = + build_conn() + |> get("/api/v1/instance") + + [csp] = Conn.get_resp_header(conn, "content-security-policy") + + assert csp =~ ~r|report-uri https://endpoint.com; report-to csp-endpoint;| + + [reply_to] = Conn.get_resp_header(conn, "reply-to") + + assert reply_to == + "{\"endpoints\":[{\"url\":\"https://endpoint.com\"}],\"group\":\"csp-endpoint\",\"max-age\":10886400}" + end end test "it does not send CSP headers when disabled", %{conn: conn} do + enabled = Config.get([:http_securiy, :enabled]) + Config.put([:http_security, :enabled], false) - conn = - conn - |> get("/api/v1/instance") + on_exit(fn -> + Config.put([:http_security, :enabled], enabled) + end) + + conn = get(conn, "/api/v1/instance") assert Conn.get_resp_header(conn, "x-xss-protection") == [] assert Conn.get_resp_header(conn, "x-permitted-cross-domain-policies") == [] @@ -38,46 +99,4 @@ test "it does not send CSP headers when disabled", %{conn: conn} do assert Conn.get_resp_header(conn, "referrer-policy") == [] assert Conn.get_resp_header(conn, "content-security-policy") == [] end - - test "it sends STS headers when enabled", %{conn: conn} do - Config.put([:http_security, :enabled], true) - Config.put([:http_security, :sts], true) - - conn = - conn - |> get("/api/v1/instance") - - refute Conn.get_resp_header(conn, "strict-transport-security") == [] - refute Conn.get_resp_header(conn, "expect-ct") == [] - end - - test "it does not send STS headers when disabled", %{conn: conn} do - Config.put([:http_security, :enabled], true) - Config.put([:http_security, :sts], false) - - conn = - conn - |> get("/api/v1/instance") - - assert Conn.get_resp_header(conn, "strict-transport-security") == [] - assert Conn.get_resp_header(conn, "expect-ct") == [] - end - - test "referrer-policy header reflects configured value", %{conn: conn} do - Config.put([:http_security, :enabled], true) - - conn = - conn - |> get("/api/v1/instance") - - assert Conn.get_resp_header(conn, "referrer-policy") == ["same-origin"] - - Config.put([:http_security, :referrer_policy], "no-referrer") - - conn = - build_conn() - |> get("/api/v1/instance") - - assert Conn.get_resp_header(conn, "referrer-policy") == ["no-referrer"] - end end From 8355d0607e986f5a2cc5addfb2bcb35f799f10cb Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 16 May 2019 17:03:12 +0700 Subject: [PATCH 032/123] trailing spaces --- docs/introduction.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/introduction.md b/docs/introduction.md index 4af0747fe..045dc7c05 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -1,30 +1,30 @@ # Introduction to Pleroma ## What is Pleroma? -Pleroma is a federated social networking platform, compatible with GNU social, Mastodon and other OStatus and ActivityPub implementations. It is free software licensed under the AGPLv3. -It actually consists of two components: a backend, named simply Pleroma, and a user-facing frontend, named Pleroma-FE. It also includes the Mastodon frontend, if that's your thing. -It's part of what we call the fediverse, a federated network of instances which speak common protocols and can communicate with each other. +Pleroma is a federated social networking platform, compatible with GNU social, Mastodon and other OStatus and ActivityPub implementations. It is free software licensed under the AGPLv3. +It actually consists of two components: a backend, named simply Pleroma, and a user-facing frontend, named Pleroma-FE. It also includes the Mastodon frontend, if that's your thing. +It's part of what we call the fediverse, a federated network of instances which speak common protocols and can communicate with each other. One account on a instance is enough to talk to the entire fediverse! - + ## How can I use it? -Pleroma instances are already widely deployed, a list can be found here: +Pleroma instances are already widely deployed, a list can be found here: http://distsn.org/pleroma-instances.html -If you don't feel like joining an existing instance, but instead prefer to deploy your own instance, that's easy too! -Installation instructions can be found here: +If you don't feel like joining an existing instance, but instead prefer to deploy your own instance, that's easy too! +Installation instructions can be found here: [main Pleroma wiki](/) - + ## I got an account, now what? -Great! Now you can explore the fediverse! -- Open the login page for your Pleroma instance (for ex. https://pleroma.soykaf.com) and login with your username and password. -(If you don't have one yet, click on Register) :slightly_smiling_face: +Great! Now you can explore the fediverse! +- Open the login page for your Pleroma instance (for ex. https://pleroma.soykaf.com) and login with your username and password. +(If you don't have one yet, click on Register) :slightly_smiling_face: At this point you will have two columns in front of you. ### Left column -- first block: here you can see your avatar, your nickname a bio, and statistics (Statuses, Following, Followers). -Under that you have a text form which allows you to post new statuses. The icon on the left is for uploading media files and attach them to your post. The number under the text form is a character counter, every instance can have a different character limit (the default is 5000). -If you want to mention someone, type @ + name of the person. A drop-down menu will help you in finding the right person. :slight_smile: +- first block: here you can see your avatar, your nickname a bio, and statistics (Statuses, Following, Followers). +Under that you have a text form which allows you to post new statuses. The icon on the left is for uploading media files and attach them to your post. The number under the text form is a character counter, every instance can have a different character limit (the default is 5000). +If you want to mention someone, type @ + name of the person. A drop-down menu will help you in finding the right person. :slight_smile: To post your status, simply press Submit. - second block: Here you can switch between the different timelines: @@ -38,7 +38,7 @@ To post your status, simply press Submit. - fourth block: This is the Notifications block, here you will get notified whenever somebody mentions you, follows you, repeats or favorites one of your statuses. ### Right column -This is where the interesting stuff happens! :slight_smile: +This is where the interesting stuff happens! :slight_smile: Depending on the timeline you will see different statuses, but each status has a standard structure: - Icon + name + link to profile. An optional left-arrow if it's a reply to another status (hovering will reveal the replied-to status). - A + button on the right allows you to Expand/Collapse an entire discussion thread. It also updates in realtime! @@ -47,9 +47,9 @@ Depending on the timeline you will see different statuses, but each status has a - Four buttons (left to right): Reply, Repeat, Favorite, Delete. ## Mastodon interface -If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too! :smile: -Just add a "/web" after your instance url (for ex. https://pleroma.soycaf.com/web) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC! :fireworks: -For more information on the Mastodon interface, please look here: +If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too! :smile: +Just add a "/web" after your instance url (for ex. https://pleroma.soycaf.com/web) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC! :fireworks: +For more information on the Mastodon interface, please look here: https://github.com/tootsuite/documentation/blob/master/Using-Mastodon/User-guide.md Remember, what you see is only the frontend part of Mastodon, the backend is still Pleroma. From 4711d8151c8cb7ce2024f378e17b3a113a3aac1c Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 16 May 2019 17:35:07 +0700 Subject: [PATCH 033/123] try this --- test/web/common_api/common_api_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 8d4f401ee..b0c441a37 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -279,14 +279,14 @@ test "creates a report" do test "add a reblog mute", %{muter: muter, muted: muted} do {:ok, muter} = CommonAPI.hide_reblogs(muter, muted) - assert Pleroma.User.showing_reblogs?(muter, muted) == false + assert User.showing_reblogs?(muter, muted) == false end test "remove a reblog mute", %{muter: muter, muted: muted} do {:ok, muter} = CommonAPI.hide_reblogs(muter, muted) {:ok, muter} = CommonAPI.show_reblogs(muter, muted) - assert Pleroma.User.showing_reblogs?(muter, muted) == true + assert User.showing_reblogs?(muter, muted) == true end end end From 73ae58fdfaf0b9dc9630929b0b84ae3b6083684a Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 16 May 2019 13:23:41 +0000 Subject: [PATCH 034/123] Feature/896 toggling confirmation --- CHANGELOG.md | 3 ++- lib/mix/tasks/pleroma/user.ex | 19 +++++++++++++++++++ lib/pleroma/user.ex | 13 +++++++++++++ lib/pleroma/user/info.ex | 2 +- test/tasks/user_test.exs | 27 +++++++++++++++++++++++++++ test/user_test.exs | 18 ++++++++++++++++++ 6 files changed, 80 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0e849285..b3d8e1e0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - [Prometheus](https://prometheus.io/) metrics - Support for Mastodon's remote interaction - Mix Tasks: `mix pleroma.database remove_embedded_objects` +- Mix Tasks: `mix pleroma.user toggle_confirmed` - Federation: Support for reports - Configuration: `safe_dm_mentions` option - Configuration: `link_name` option @@ -98,7 +99,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`] ## Removed -- Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations` +- Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations` ## [0.9.9999] - 2019-04-05 ### Security diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index d130ff8c9..25fc40ea7 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -77,6 +77,10 @@ defmodule Mix.Tasks.Pleroma.User do ## Delete tags from a user. mix pleroma.user untag NICKNAME TAGS + + ## Toggle confirmation of the user's account. + + mix pleroma.user toggle_confirmed NICKNAME """ def run(["new", nickname, email | rest]) do {options, [], []} = @@ -388,6 +392,21 @@ def run(["delete_activities", nickname]) do end end + def run(["toggle_confirmed", nickname]) do + Common.start_pleroma() + + with %User{} = user <- User.get_cached_by_nickname(nickname) do + {:ok, user} = User.toggle_confirmation(user) + + message = if user.info.confirmation_pending, do: "needs", else: "doesn't need" + + Mix.shell().info("#{nickname} #{message} confirmation.") + else + _ -> + Mix.shell().error("No local user #{nickname}") + end + end + defp set_moderator(user, value) do info_cng = User.Info.admin_api_update(user.info, %{is_moderator: value}) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index c6a562a61..1aa966dfc 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1378,4 +1378,17 @@ def all_superusers do def showing_reblogs?(%User{} = user, %User{} = target) do target.ap_id not in user.info.muted_reblogs end + + @spec toggle_confirmation(User.t()) :: {:ok, User.t()} | {:error, Changeset.t()} + def toggle_confirmation(%User{} = user) do + need_confirmation? = !user.info.confirmation_pending + + info_changeset = + User.Info.confirmation_changeset(user.info, need_confirmation: need_confirmation?) + + user + |> change() + |> put_embed(:info, info_changeset) + |> update_and_set_cache() + end end diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 5a50ee639..5f0cefc00 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -212,7 +212,7 @@ def profile_update(info, params) do ]) end - @spec confirmation_changeset(Info.t(), keyword()) :: Ecto.Changerset.t() + @spec confirmation_changeset(Info.t(), keyword()) :: Changeset.t() def confirmation_changeset(info, opts) do need_confirmation? = Keyword.get(opts, :need_confirmation) diff --git a/test/tasks/user_test.exs b/test/tasks/user_test.exs index eaf4ecf84..1f97740be 100644 --- a/test/tasks/user_test.exs +++ b/test/tasks/user_test.exs @@ -338,4 +338,31 @@ test "activities are deleted" do assert message == "User #{nickname} statuses deleted." end end + + describe "running toggle_confirmed" do + test "user is confirmed" do + %{id: id, nickname: nickname} = insert(:user, info: %{confirmation_pending: false}) + + assert :ok = Mix.Tasks.Pleroma.User.run(["toggle_confirmed", nickname]) + assert_received {:mix_shell, :info, [message]} + assert message == "#{nickname} needs confirmation." + + user = Repo.get(User, id) + assert user.info.confirmation_pending + assert user.info.confirmation_token + end + + test "user is not confirmed" do + %{id: id, nickname: nickname} = + insert(:user, info: %{confirmation_pending: true, confirmation_token: "some token"}) + + assert :ok = Mix.Tasks.Pleroma.User.run(["toggle_confirmed", nickname]) + assert_received {:mix_shell, :info, [message]} + assert message == "#{nickname} doesn't need confirmation." + + user = Repo.get(User, id) + refute user.info.confirmation_pending + refute user.info.confirmation_token + end + end end diff --git a/test/user_test.exs b/test/user_test.exs index 0b65e89e9..f256396c0 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1204,4 +1204,22 @@ test "follower count is updated when a follower is blocked" do assert Map.get(user_show, "followers_count") == 2 end + + describe "toggle_confirmation/1" do + test "if user is confirmed" do + user = insert(:user, info: %{confirmation_pending: false}) + {:ok, user} = User.toggle_confirmation(user) + + assert user.info.confirmation_pending + assert user.info.confirmation_token + end + + test "if user is unconfirmed" do + user = insert(:user, info: %{confirmation_pending: true, confirmation_token: "some token"}) + {:ok, user} = User.toggle_confirmation(user) + + refute user.info.confirmation_pending + refute user.info.confirmation_token + end + end end From efa61c161085fd24f7e85ccf7f32ef823e335d7b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 16 May 2019 13:14:48 -0500 Subject: [PATCH 035/123] Move to pleroma.database mix task --- lib/mix/tasks/pleroma/conversations.ex | 23 ----------------------- lib/mix/tasks/pleroma/database.ex | 11 +++++++++++ 2 files changed, 11 insertions(+), 23 deletions(-) delete mode 100644 lib/mix/tasks/pleroma/conversations.ex diff --git a/lib/mix/tasks/pleroma/conversations.ex b/lib/mix/tasks/pleroma/conversations.ex deleted file mode 100644 index b52b9921a..000000000 --- a/lib/mix/tasks/pleroma/conversations.ex +++ /dev/null @@ -1,23 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Mix.Tasks.Pleroma.Conversations do - use Mix.Task - alias Mix.Tasks.Pleroma.Common - alias Pleroma.Conversation - - @shortdoc "Manages Pleroma conversations." - @moduledoc """ - Manages Pleroma conversations. - - ## Create a conversation for all existing DMs. Can be safely re-run. - - mix pleroma.conversations bump_all - - """ - def run(["bump_all"]) do - Common.start_pleroma() - Conversation.bump_for_all_activities() - end -end diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index ab9a3a7ff..42753a1a4 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -4,6 +4,7 @@ defmodule Mix.Tasks.Pleroma.Database do alias Mix.Tasks.Pleroma.Common + alias Pleroma.Conversation require Logger use Mix.Task @@ -19,6 +20,11 @@ defmodule Mix.Tasks.Pleroma.Database do Options: - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references + + ## Create a conversation for all existing DMs. Can be safely re-run. + + mix pleroma.database bump_all_conversations + """ def run(["remove_embedded_objects" | args]) do {options, [], []} = @@ -48,4 +54,9 @@ def run(["remove_embedded_objects" | args]) do ) end end + + def run(["bump_all_conversations"]) do + Common.start_pleroma() + Conversation.bump_for_all_activities() + end end From cd127d2fa43e6b1ecd7fadfc1d6293b2a432be18 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 16 May 2019 13:17:09 -0500 Subject: [PATCH 036/123] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3d8e1e0c..fe6ab002c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - A [job queue](https://git.pleroma.social/pleroma/pleroma_job_queue) for federation, emails, web push, etc. - [Prometheus](https://prometheus.io/) metrics - Support for Mastodon's remote interaction +- Mix Tasks: `mix pleroma.database bump_all_conversations` - Mix Tasks: `mix pleroma.database remove_embedded_objects` - Mix Tasks: `mix pleroma.user toggle_confirmed` - Federation: Support for reports From e2b3a27204ca511a2e455a1151fdea36fdc0e53d Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Thu, 16 May 2019 19:09:18 +0000 Subject: [PATCH 037/123] Add Reports to Admin API --- CHANGELOG.md | 2 + docs/api/admin_api.md | 289 +++++++++++++++- lib/pleroma/activity.ex | 2 +- lib/pleroma/emails/admin_email.ex | 2 +- lib/pleroma/web/activity_pub/activity_pub.ex | 7 + lib/pleroma/web/activity_pub/utils.ex | 78 ++++- .../web/admin_api/admin_api_controller.ex | 81 +++++ .../web/admin_api/views/report_view.ex | 41 +++ lib/pleroma/web/common_api/common_api.ex | 57 +++ lib/pleroma/web/common_api/utils.ex | 10 +- lib/pleroma/web/router.ex | 8 + ...511191044_set_default_state_to_reports.exs | 19 + test/support/factory.ex | 5 +- .../admin_api/admin_api_controller_test.exs | 327 ++++++++++++++++++ test/web/common_api/common_api_test.exs | 33 +- .../mastodon_api_controller_test.exs | 4 +- 16 files changed, 945 insertions(+), 20 deletions(-) create mode 100644 lib/pleroma/web/admin_api/views/report_view.ex create mode 100644 priv/repo/migrations/20190511191044_set_default_state_to_reports.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index fe6ab002c..dc09d4f6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: Endpoints for listing/revoking invite tokens - Admin API: Endpoints for making users follow/unfollow each other - Admin API: added filters (role, tags, email, name) for users endpoint +- Admin API: Endpoints for managing reports +- Admin API: Endpoints for deleting and changing the scope of individual reported statuses - AdminFE: initial release with basic user management accessible at /pleroma/admin/ - Mastodon API: [Scheduled statuses](https://docs.joinmastodon.org/api/rest/scheduled-statuses/) - Mastodon API: `/api/v1/notifications/destroy_multiple` (glitch-soc extension) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 59578f8d1..b45c5e285 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -24,7 +24,7 @@ Authentication is required and the user must be an admin. - Example: `https://mypleroma.org/api/pleroma/admin/users?query=john&filters=local,active&page=1&page_size=10&tags[]=some_tag&tags[]=another_tag&name=display_name&email=email@example.com` - Response: -```JSON +```json { "page_size": integer, "count": integer, @@ -92,7 +92,7 @@ Authentication is required and the user must be an admin. - `nickname` - Response: User’s object -```JSON +```json { "deactivated": bool, "id": integer, @@ -124,7 +124,7 @@ Authentication is required and the user must be an admin. - Params: none - Response: -```JSON +```json { "is_moderator": bool, "is_admin": bool @@ -141,7 +141,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - Params: none - Response: -```JSON +```json { "is_moderator": bool, "is_admin": bool @@ -223,7 +223,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - Params: none - Response: -```JSON +```json { "invites": [ @@ -250,7 +250,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - `token` - Response: -```JSON +```json { "id": integer, "token": string, @@ -280,3 +280,280 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - Methods: `GET` - Params: none - Response: password reset token (base64 string) + +## `/api/pleroma/admin/reports` +### Get a list of reports +- Method `GET` +- Params: + - `state`: optional, the state of reports. Valid values are `open`, `closed` and `resolved` + - `limit`: optional, the number of records to retrieve + - `since_id`: optional, returns results that are more recent than the specified id + - `max_id`: optional, returns results that are older than the specified id +- Response: + - On failure: 403 Forbidden error `{"error": "error_msg"}` when requested by anonymous or non-admin + - On success: JSON, returns a list of reports, where: + - `account`: the user who has been reported + - `actor`: the user who has sent the report + - `statuses`: list of statuses that have been included to the report + +```json +{ + "reports": [ + { + "account": { + "acct": "user", + "avatar": "https://pleroma.example.org/images/avi.png", + "avatar_static": "https://pleroma.example.org/images/avi.png", + "bot": false, + "created_at": "2019-04-23T17:32:04.000Z", + "display_name": "User", + "emojis": [], + "fields": [], + "followers_count": 1, + "following_count": 1, + "header": "https://pleroma.example.org/images/banner.png", + "header_static": "https://pleroma.example.org/images/banner.png", + "id": "9i6dAJqSGSKMzLG2Lo", + "locked": false, + "note": "", + "pleroma": { + "confirmation_pending": false, + "hide_favorites": true, + "hide_followers": false, + "hide_follows": false, + "is_admin": false, + "is_moderator": false, + "relationship": {}, + "tags": [] + }, + "source": { + "note": "", + "pleroma": {}, + "sensitive": false + }, + "statuses_count": 3, + "url": "https://pleroma.example.org/users/user", + "username": "user" + }, + "actor": { + "acct": "lain", + "avatar": "https://pleroma.example.org/images/avi.png", + "avatar_static": "https://pleroma.example.org/images/avi.png", + "bot": false, + "created_at": "2019-03-28T17:36:03.000Z", + "display_name": "Roger Braun", + "emojis": [], + "fields": [], + "followers_count": 1, + "following_count": 1, + "header": "https://pleroma.example.org/images/banner.png", + "header_static": "https://pleroma.example.org/images/banner.png", + "id": "9hEkA5JsvAdlSrocam", + "locked": false, + "note": "", + "pleroma": { + "confirmation_pending": false, + "hide_favorites": false, + "hide_followers": false, + "hide_follows": false, + "is_admin": false, + "is_moderator": false, + "relationship": {}, + "tags": [] + }, + "source": { + "note": "", + "pleroma": {}, + "sensitive": false + }, + "statuses_count": 1, + "url": "https://pleroma.example.org/users/lain", + "username": "lain" + }, + "content": "Please delete it", + "created_at": "2019-04-29T19:48:15.000Z", + "id": "9iJGOv1j8hxuw19bcm", + "state": "open", + "statuses": [ + { + "account": { ... }, + "application": { + "name": "Web", + "website": null + }, + "bookmarked": false, + "card": null, + "content": "@lain click on my link https://www.google.com/", + "created_at": "2019-04-23T19:15:47.000Z", + "emojis": [], + "favourited": false, + "favourites_count": 0, + "id": "9i6mQ9uVrrOmOime8m", + "in_reply_to_account_id": null, + "in_reply_to_id": null, + "language": null, + "media_attachments": [], + "mentions": [ + { + "acct": "lain", + "id": "9hEkA5JsvAdlSrocam", + "url": "https://pleroma.example.org/users/lain", + "username": "lain" + }, + { + "acct": "user", + "id": "9i6dAJqSGSKMzLG2Lo", + "url": "https://pleroma.example.org/users/user", + "username": "user" + } + ], + "muted": false, + "pinned": false, + "pleroma": { + "content": { + "text/plain": "@lain click on my link https://www.google.com/" + }, + "conversation_id": 28, + "in_reply_to_account_acct": null, + "local": true, + "spoiler_text": { + "text/plain": "" + } + }, + "reblog": null, + "reblogged": false, + "reblogs_count": 0, + "replies_count": 0, + "sensitive": false, + "spoiler_text": "", + "tags": [], + "uri": "https://pleroma.example.org/objects/8717b90f-8e09-4b58-97b0-e3305472b396", + "url": "https://pleroma.example.org/notice/9i6mQ9uVrrOmOime8m", + "visibility": "direct" + } + ] + } + ] +} +``` + +## `/api/pleroma/admin/reports/:id` +### Get an individual report +- Method `GET` +- Params: + - `id` +- Response: + - On failure: + - 403 Forbidden `{"error": "error_msg"}` + - 404 Not Found `"Not found"` + - On success: JSON, Report object (see above) + +## `/api/pleroma/admin/reports/:id` +### Change the state of the report +- Method `PUT` +- Params: + - `id` + - `state`: required, the new state. Valid values are `open`, `closed` and `resolved` +- Response: + - On failure: + - 400 Bad Request `"Unsupported state"` + - 403 Forbidden `{"error": "error_msg"}` + - 404 Not Found `"Not found"` + - On success: JSON, Report object (see above) + +## `/api/pleroma/admin/reports/:id/respond` +### Respond to a report +- Method `POST` +- Params: + - `id` + - `status`: required, the message +- Response: + - On failure: + - 400 Bad Request `"Invalid parameters"` when `status` is missing + - 403 Forbidden `{"error": "error_msg"}` + - 404 Not Found `"Not found"` + - On success: JSON, created Mastodon Status entity + +```json +{ + "account": { ... }, + "application": { + "name": "Web", + "website": null + }, + "bookmarked": false, + "card": null, + "content": "Your claim is going to be closed", + "created_at": "2019-05-11T17:13:03.000Z", + "emojis": [], + "favourited": false, + "favourites_count": 0, + "id": "9ihuiSL1405I65TmEq", + "in_reply_to_account_id": null, + "in_reply_to_id": null, + "language": null, + "media_attachments": [], + "mentions": [ + { + "acct": "user", + "id": "9i6dAJqSGSKMzLG2Lo", + "url": "https://pleroma.example.org/users/user", + "username": "user" + }, + { + "acct": "admin", + "id": "9hEkA5JsvAdlSrocam", + "url": "https://pleroma.example.org/users/admin", + "username": "admin" + } + ], + "muted": false, + "pinned": false, + "pleroma": { + "content": { + "text/plain": "Your claim is going to be closed" + }, + "conversation_id": 35, + "in_reply_to_account_acct": null, + "local": true, + "spoiler_text": { + "text/plain": "" + } + }, + "reblog": null, + "reblogged": false, + "reblogs_count": 0, + "replies_count": 0, + "sensitive": false, + "spoiler_text": "", + "tags": [], + "uri": "https://pleroma.example.org/objects/cab0836d-9814-46cd-a0ea-529da9db5fcb", + "url": "https://pleroma.example.org/notice/9ihuiSL1405I65TmEq", + "visibility": "direct" +} +``` + +## `/api/pleroma/admin/statuses/:id` +### Change the scope of an individual reported status +- Method `PUT` +- Params: + - `id` + - `sensitive`: optional, valid values are `true` or `false` + - `visibility`: optional, valid values are `public`, `private` and `unlisted` +- Response: + - On failure: + - 400 Bad Request `"Unsupported visibility"` + - 403 Forbidden `{"error": "error_msg"}` + - 404 Not Found `"Not found"` + - On success: JSON, Mastodon Status entity + +## `/api/pleroma/admin/statuses/:id` +### Delete an individual reported status +- Method `DELETE` +- Params: + - `id` +- Response: + - On failure: + - 403 Forbidden `{"error": "error_msg"}` + - 404 Not Found `"Not found"` + - On success: 200 OK `{}` diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 3fd4003f8..4e54b15ba 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -111,7 +111,7 @@ def get_bookmark(_, _), do: nil def change(struct, params \\ %{}) do struct - |> cast(params, [:data]) + |> cast(params, [:data, :recipients]) |> validate_required([:data]) |> unique_constraint(:ap_id, name: :activities_unique_apid_index) end diff --git a/lib/pleroma/emails/admin_email.ex b/lib/pleroma/emails/admin_email.ex index df0f72f96..d0e254362 100644 --- a/lib/pleroma/emails/admin_email.ex +++ b/lib/pleroma/emails/admin_email.ex @@ -29,7 +29,7 @@ def report(to, reporter, account, statuses, comment) do end statuses_html = - if length(statuses) > 0 do + if is_list(statuses) && length(statuses) > 0 do statuses_list_html = statuses |> Enum.map(fn diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 6a186efbf..5c3156978 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -703,6 +703,12 @@ defp restrict_type(query, %{"type" => type}) do defp restrict_type(query, _), do: query + defp restrict_state(query, %{"state" => state}) do + from(activity in query, where: fragment("?->>'state' = ?", activity.data, ^state)) + end + + defp restrict_state(query, _), do: query + defp restrict_favorited_by(query, %{"favorited_by" => ap_id}) do from( activity in query, @@ -855,6 +861,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do |> restrict_local(opts) |> restrict_actor(opts) |> restrict_type(opts) + |> restrict_state(opts) |> restrict_favorited_by(opts) |> restrict_blocked(opts) |> restrict_muted(opts) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 236d1b4ac..ca8a0844b 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -20,6 +20,8 @@ defmodule Pleroma.Web.ActivityPub.Utils do require Logger @supported_object_types ["Article", "Note", "Video", "Page"] + @supported_report_states ~w(open closed resolved) + @valid_visibilities ~w(public unlisted private direct) # Some implementations send the actor URI as the actor field, others send the entire actor object, # so figure out what the actor's URI is based on what we have. @@ -670,7 +672,8 @@ def make_flag_data(params, additional) do "actor" => params.actor.ap_id, "content" => params.content, "object" => object, - "context" => params.context + "context" => params.context, + "state" => "open" } |> Map.merge(additional) end @@ -713,4 +716,77 @@ def fetch_ordered_collection(from, pages_left, acc \\ []) do end end end + + #### Report-related helpers + + def update_report_state(%Activity{} = activity, state) when state in @supported_report_states do + with new_data <- Map.put(activity.data, "state", state), + changeset <- Changeset.change(activity, data: new_data), + {:ok, activity} <- Repo.update(changeset) do + {:ok, activity} + end + end + + def update_report_state(_, _), do: {:error, "Unsupported state"} + + def update_activity_visibility(activity, visibility) when visibility in @valid_visibilities do + [to, cc, recipients] = + activity + |> get_updated_targets(visibility) + |> Enum.map(&Enum.uniq/1) + + object_data = + activity.object.data + |> Map.put("to", to) + |> Map.put("cc", cc) + + {:ok, object} = + activity.object + |> Object.change(%{data: object_data}) + |> Object.update_and_set_cache() + + activity_data = + activity.data + |> Map.put("to", to) + |> Map.put("cc", cc) + + activity + |> Map.put(:object, object) + |> Activity.change(%{data: activity_data, recipients: recipients}) + |> Repo.update() + end + + def update_activity_visibility(_, _), do: {:error, "Unsupported visibility"} + + defp get_updated_targets( + %Activity{data: %{"to" => to} = data, recipients: recipients}, + visibility + ) do + cc = Map.get(data, "cc", []) + follower_address = User.get_cached_by_ap_id(data["actor"]).follower_address + public = "https://www.w3.org/ns/activitystreams#Public" + + case visibility do + "public" -> + to = [public | List.delete(to, follower_address)] + cc = [follower_address | List.delete(cc, public)] + recipients = [public | recipients] + [to, cc, recipients] + + "private" -> + to = [follower_address | List.delete(to, public)] + cc = List.delete(cc, public) + recipients = List.delete(recipients, public) + [to, cc, recipients] + + "unlisted" -> + to = [follower_address | List.delete(to, public)] + cc = [public | List.delete(cc, follower_address)] + recipients = recipients ++ [follower_address, public] + [to, cc, recipients] + + _ -> + [to, cc, recipients] + end + end end diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index e00b33aba..de2a13c01 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -4,11 +4,16 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do use Pleroma.Web, :controller + alias Pleroma.Activity alias Pleroma.User alias Pleroma.UserInviteToken + alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.AdminAPI.AccountView + alias Pleroma.Web.AdminAPI.ReportView alias Pleroma.Web.AdminAPI.Search + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.MastodonAPI.StatusView import Pleroma.Web.ControllerHelper, only: [json_response: 3] @@ -287,12 +292,88 @@ def get_password_reset(conn, %{"nickname" => nickname}) do |> json(token.token) end + def list_reports(conn, params) do + params = + params + |> Map.put("type", "Flag") + |> Map.put("skip_preload", true) + + reports = + [] + |> ActivityPub.fetch_activities(params) + |> Enum.reverse() + + conn + |> put_view(ReportView) + |> render("index.json", %{reports: reports}) + end + + def report_show(conn, %{"id" => id}) do + with %Activity{} = report <- Activity.get_by_id(id) do + conn + |> put_view(ReportView) + |> render("show.json", %{report: report}) + else + _ -> {:error, :not_found} + end + end + + def report_update_state(conn, %{"id" => id, "state" => state}) do + with {:ok, report} <- CommonAPI.update_report_state(id, state) do + conn + |> put_view(ReportView) + |> render("show.json", %{report: report}) + end + end + + def report_respond(%{assigns: %{user: user}} = conn, %{"id" => id} = params) do + with false <- is_nil(params["status"]), + %Activity{} <- Activity.get_by_id(id) do + params = + params + |> Map.put("in_reply_to_status_id", id) + |> Map.put("visibility", "direct") + + {:ok, activity} = CommonAPI.post(user, params) + + conn + |> put_view(StatusView) + |> render("status.json", %{activity: activity}) + else + true -> + {:param_cast, nil} + + nil -> + {:error, :not_found} + end + end + + def status_update(conn, %{"id" => id} = params) do + with {:ok, activity} <- CommonAPI.update_activity_scope(id, params) do + conn + |> put_view(StatusView) + |> render("status.json", %{activity: activity}) + end + end + + def status_delete(%{assigns: %{user: user}} = conn, %{"id" => id}) do + with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do + json(conn, %{}) + end + end + def errors(conn, {:error, :not_found}) do conn |> put_status(404) |> json("Not found") end + def errors(conn, {:error, reason}) do + conn + |> put_status(400) + |> json(reason) + end + def errors(conn, {:param_cast, _}) do conn |> put_status(400) diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex new file mode 100644 index 000000000..47a73dc7e --- /dev/null +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -0,0 +1,41 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.ReportView do + use Pleroma.Web, :view + alias Pleroma.Activity + alias Pleroma.User + alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Web.MastodonAPI.AccountView + alias Pleroma.Web.MastodonAPI.StatusView + + def render("index.json", %{reports: reports}) do + %{ + reports: render_many(reports, __MODULE__, "show.json", as: :report) + } + end + + def render("show.json", %{report: report}) do + user = User.get_cached_by_ap_id(report.data["actor"]) + created_at = Utils.to_masto_date(report.data["published"]) + + [account_ap_id | status_ap_ids] = report.data["object"] + account = User.get_cached_by_ap_id(account_ap_id) + + statuses = + Enum.map(status_ap_ids, fn ap_id -> + Activity.get_by_ap_id_with_object(ap_id) + end) + + %{ + id: report.id, + account: AccountView.render("account.json", %{user: account}), + actor: AccountView.render("account.json", %{user: user}), + content: report.data["content"], + created_at: created_at, + statuses: StatusView.render("index.json", %{activities: statuses, as: :activity}), + state: report.data["state"] + } + end +end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 29c4c1014..208c12c7b 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -71,6 +71,9 @@ def delete(activity_id, user) do {:ok, _} <- unpin(activity_id, user), {:ok, delete} <- ActivityPub.delete(object) do {:ok, delete} + else + _ -> + {:error, "Could not delete"} end end @@ -315,6 +318,60 @@ def report(user, data) do end end + def update_report_state(activity_id, state) do + with %Activity{} = activity <- Activity.get_by_id(activity_id), + {:ok, activity} <- Utils.update_report_state(activity, state) do + {:ok, activity} + else + nil -> + {:error, :not_found} + + {:error, reason} -> + {:error, reason} + + _ -> + {:error, "Could not update state"} + end + end + + def update_activity_scope(activity_id, opts \\ %{}) do + with %Activity{} = activity <- Activity.get_by_id_with_object(activity_id), + {:ok, activity} <- toggle_sensitive(activity, opts), + {:ok, activity} <- set_visibility(activity, opts) do + {:ok, activity} + else + nil -> + {:error, :not_found} + + {:error, reason} -> + {:error, reason} + end + end + + defp toggle_sensitive(activity, %{"sensitive" => sensitive}) when sensitive in ~w(true false) do + toggle_sensitive(activity, %{"sensitive" => String.to_existing_atom(sensitive)}) + end + + defp toggle_sensitive(%Activity{object: object} = activity, %{"sensitive" => sensitive}) + when is_boolean(sensitive) do + new_data = Map.put(object.data, "sensitive", sensitive) + + {:ok, object} = + object + |> Object.change(%{data: new_data}) + |> Object.update_and_set_cache() + + {:ok, Map.put(activity, :object, object)} + end + + defp toggle_sensitive(activity, _), do: {:ok, activity} + + defp set_visibility(activity, %{"visibility" => visibility}) do + Utils.update_activity_visibility(activity, visibility) + end + + defp set_visibility(activity, _), do: {:ok, activity} + def hide_reblogs(user, muted) do ap_id = muted.ap_id diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 1dfe50b40..bee2fd159 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -237,13 +237,11 @@ def make_note_data( "tag" => tags |> Enum.map(fn {_, tag} -> tag end) |> Enum.uniq() } - if in_reply_to do - in_reply_to_object = Object.normalize(in_reply_to) - - object - |> Map.put("inReplyTo", in_reply_to_object.data["id"]) + with false <- is_nil(in_reply_to), + %Object{} = in_reply_to_object <- Object.normalize(in_reply_to) do + Map.put(object, "inReplyTo", in_reply_to_object.data["id"]) else - object + _ -> object end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 7fef82f82..6a4e4a1d4 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -194,6 +194,14 @@ defmodule Pleroma.Web.Router do get("/users", AdminAPIController, :list_users) get("/users/:nickname", AdminAPIController, :user_show) + + get("/reports", AdminAPIController, :list_reports) + get("/reports/:id", AdminAPIController, :report_show) + put("/reports/:id", AdminAPIController, :report_update_state) + post("/reports/:id/respond", AdminAPIController, :report_respond) + + put("/statuses/:id", AdminAPIController, :status_update) + delete("/statuses/:id", AdminAPIController, :status_delete) end scope "/", Pleroma.Web.TwitterAPI do diff --git a/priv/repo/migrations/20190511191044_set_default_state_to_reports.exs b/priv/repo/migrations/20190511191044_set_default_state_to_reports.exs new file mode 100644 index 000000000..0d3d253b6 --- /dev/null +++ b/priv/repo/migrations/20190511191044_set_default_state_to_reports.exs @@ -0,0 +1,19 @@ +defmodule Pleroma.Repo.Migrations.SetDefaultStateToReports do + use Ecto.Migration + + def up do + execute """ + UPDATE activities AS a + SET data = jsonb_set(data, '{state}', '"open"', true) + WHERE data->>'type' = 'Flag' + """ + end + + def down do + execute """ + UPDATE activities AS a + SET data = data #- '{state}' + WHERE data->>'type' = 'Flag' + """ + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index 2a2954ad6..90c7d80f2 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -43,7 +43,7 @@ def user_factory do def note_factory(attrs \\ %{}) do text = sequence(:text, &"This is :moominmamma: note #{&1}") - user = insert(:user) + user = attrs[:user] || insert(:user) data = %{ "type" => "Note", @@ -113,7 +113,8 @@ def direct_note_activity_factory do end def note_activity_factory(attrs \\ %{}) do - note = attrs[:note] || insert(:note) + user = attrs[:user] || insert(:user) + note = attrs[:note] || insert(:note, user: user) data = %{ "id" => Pleroma.Web.ActivityPub.Utils.generate_activity_id(), diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 6c1897b5a..ca12c7215 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -5,8 +5,10 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do use Pleroma.Web.ConnCase + alias Pleroma.Activity alias Pleroma.User alias Pleroma.UserInviteToken + alias Pleroma.Web.CommonAPI import Pleroma.Factory describe "/api/pleroma/admin/users" do @@ -949,4 +951,329 @@ test "with token" do } end end + + describe "GET /api/pleroma/admin/reports/:id" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + + %{conn: assign(conn, :user, admin)} + end + + test "returns report by its id", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended", + "status_ids" => [activity.id] + }) + + response = + conn + |> get("/api/pleroma/admin/reports/#{report_id}") + |> json_response(:ok) + + assert response["id"] == report_id + end + + test "returns 404 when report id is invalid", %{conn: conn} do + conn = get(conn, "/api/pleroma/admin/reports/test") + + assert json_response(conn, :not_found) == "Not found" + end + end + + describe "PUT /api/pleroma/admin/reports/:id" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended", + "status_ids" => [activity.id] + }) + + %{conn: assign(conn, :user, admin), id: report_id} + end + + test "mark report as resolved", %{conn: conn, id: id} do + response = + conn + |> put("/api/pleroma/admin/reports/#{id}", %{"state" => "resolved"}) + |> json_response(:ok) + + assert response["state"] == "resolved" + end + + test "closes report", %{conn: conn, id: id} do + response = + conn + |> put("/api/pleroma/admin/reports/#{id}", %{"state" => "closed"}) + |> json_response(:ok) + + assert response["state"] == "closed" + end + + test "returns 400 when state is unknown", %{conn: conn, id: id} do + conn = + conn + |> put("/api/pleroma/admin/reports/#{id}", %{"state" => "test"}) + + assert json_response(conn, :bad_request) == "Unsupported state" + end + + test "returns 404 when report is not exist", %{conn: conn} do + conn = + conn + |> put("/api/pleroma/admin/reports/test", %{"state" => "closed"}) + + assert json_response(conn, :not_found) == "Not found" + end + end + + describe "GET /api/pleroma/admin/reports" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + + %{conn: assign(conn, :user, admin)} + end + + test "returns empty response when no reports created", %{conn: conn} do + response = + conn + |> get("/api/pleroma/admin/reports") + |> json_response(:ok) + + assert Enum.empty?(response["reports"]) + end + + test "returns reports", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended", + "status_ids" => [activity.id] + }) + + response = + conn + |> get("/api/pleroma/admin/reports") + |> json_response(:ok) + + [report] = response["reports"] + + assert length(response["reports"]) == 1 + assert report["id"] == report_id + end + + test "returns reports with specified state", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: first_report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended", + "status_ids" => [activity.id] + }) + + {:ok, %{id: second_report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I don't like this user" + }) + + CommonAPI.update_report_state(second_report_id, "closed") + + response = + conn + |> get("/api/pleroma/admin/reports", %{ + "state" => "open" + }) + |> json_response(:ok) + + [open_report] = response["reports"] + + assert length(response["reports"]) == 1 + assert open_report["id"] == first_report_id + + response = + conn + |> get("/api/pleroma/admin/reports", %{ + "state" => "closed" + }) + |> json_response(:ok) + + [closed_report] = response["reports"] + + assert length(response["reports"]) == 1 + assert closed_report["id"] == second_report_id + + response = + conn + |> get("/api/pleroma/admin/reports", %{ + "state" => "resolved" + }) + |> json_response(:ok) + + assert Enum.empty?(response["reports"]) + end + + test "returns 403 when requested by a non-admin" do + user = insert(:user) + + conn = + build_conn() + |> assign(:user, user) + |> get("/api/pleroma/admin/reports") + + assert json_response(conn, :forbidden) == %{"error" => "User is not admin."} + end + + test "returns 403 when requested by anonymous" do + conn = + build_conn() + |> get("/api/pleroma/admin/reports") + + assert json_response(conn, :forbidden) == %{"error" => "Invalid credentials."} + end + end + + describe "POST /api/pleroma/admin/reports/:id/respond" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + + %{conn: assign(conn, :user, admin)} + end + + test "returns created dm", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended", + "status_ids" => [activity.id] + }) + + response = + conn + |> post("/api/pleroma/admin/reports/#{report_id}/respond", %{ + "status" => "I will check it out" + }) + |> json_response(:ok) + + recipients = Enum.map(response["mentions"], & &1["username"]) + + assert conn.assigns[:user].nickname in recipients + assert reporter.nickname in recipients + assert response["content"] == "I will check it out" + assert response["visibility"] == "direct" + end + + test "returns 400 when status is missing", %{conn: conn} do + conn = post(conn, "/api/pleroma/admin/reports/test/respond") + + assert json_response(conn, :bad_request) == "Invalid parameters" + end + + test "returns 404 when report id is invalid", %{conn: conn} do + conn = + post(conn, "/api/pleroma/admin/reports/test/respond", %{ + "status" => "foo" + }) + + assert json_response(conn, :not_found) == "Not found" + end + end + + describe "PUT /api/pleroma/admin/statuses/:id" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + activity = insert(:note_activity) + + %{conn: assign(conn, :user, admin), id: activity.id} + end + + test "toggle sensitive flag", %{conn: conn, id: id} do + response = + conn + |> put("/api/pleroma/admin/statuses/#{id}", %{"sensitive" => "true"}) + |> json_response(:ok) + + assert response["sensitive"] + + response = + conn + |> put("/api/pleroma/admin/statuses/#{id}", %{"sensitive" => "false"}) + |> json_response(:ok) + + refute response["sensitive"] + end + + test "change visibility flag", %{conn: conn, id: id} do + response = + conn + |> put("/api/pleroma/admin/statuses/#{id}", %{"visibility" => "public"}) + |> json_response(:ok) + + assert response["visibility"] == "public" + + response = + conn + |> put("/api/pleroma/admin/statuses/#{id}", %{"visibility" => "private"}) + |> json_response(:ok) + + assert response["visibility"] == "private" + + response = + conn + |> put("/api/pleroma/admin/statuses/#{id}", %{"visibility" => "unlisted"}) + |> json_response(:ok) + + assert response["visibility"] == "unlisted" + end + + test "returns 400 when visibility is unknown", %{conn: conn, id: id} do + conn = + conn + |> put("/api/pleroma/admin/statuses/#{id}", %{"visibility" => "test"}) + + assert json_response(conn, :bad_request) == "Unsupported visibility" + end + end + + describe "DELETE /api/pleroma/admin/statuses/:id" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + activity = insert(:note_activity) + + %{conn: assign(conn, :user, admin), id: activity.id} + end + + test "deletes status", %{conn: conn, id: id} do + conn + |> delete("/api/pleroma/admin/statuses/#{id}") + |> json_response(:ok) + + refute Activity.get_by_id(id) + end + + test "returns error when status is not exist", %{conn: conn} do + conn = + conn + |> delete("/api/pleroma/admin/statuses/test") + + assert json_response(conn, :bad_request) == "Could not delete" + end + end end diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 8d4f401ee..38b5aa65f 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -261,10 +261,41 @@ test "creates a report" do data: %{ "type" => "Flag", "content" => ^comment, - "object" => [^target_ap_id, ^activity_ap_id] + "object" => [^target_ap_id, ^activity_ap_id], + "state" => "open" } } = flag_activity end + + test "updates report state" do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %Activity{id: report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended", + "status_ids" => [activity.id] + }) + + {:ok, report} = CommonAPI.update_report_state(report_id, "resolved") + + assert report.data["state"] == "resolved" + end + + test "does not update report state when state is unsupported" do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %Activity{id: report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended", + "status_ids" => [activity.id] + }) + + assert CommonAPI.update_report_state(report_id, "test") == {:error, "Unsupported state"} + end end describe "reblog muting" do diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 40e7739e7..90d67a55f 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2129,7 +2129,7 @@ test "returns favorited DM only when user is logged in and he is one of recipien |> get("/api/v1/pleroma/accounts/#{user.id}/favourites") |> json_response(:ok) - assert length(anonymous_response) == 0 + assert Enum.empty?(anonymous_response) end test "does not return others' favorited DM when user is not one of recipients", %{ @@ -2153,7 +2153,7 @@ test "does not return others' favorited DM when user is not one of recipients", |> get("/api/v1/pleroma/accounts/#{user.id}/favourites") |> json_response(:ok) - assert length(response) == 0 + assert Enum.empty?(response) end test "paginates favorites using since_id and max_id", %{ From 2c303afc8b25ca92eaa229df3e9e4737856e890c Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Thu, 16 May 2019 20:04:08 +0000 Subject: [PATCH 038/123] Remove duplicated entries in users' following lists --- CHANGELOG.md | 1 + lib/mix/tasks/pleroma/database.ex | 17 +++++++++-- lib/pleroma/user.ex | 14 ++++++++- test/tasks/database_test.exs | 49 +++++++++++++++++++++++++++++++ test/tasks/user_test.exs | 1 + test/user_test.exs | 31 +++++++++++++++++++ 6 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 test/tasks/database_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index dc09d4f6b..4d314817b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Support for Mastodon's remote interaction - Mix Tasks: `mix pleroma.database bump_all_conversations` - Mix Tasks: `mix pleroma.database remove_embedded_objects` +- Mix Tasks: `mix pleroma.database update_users_following_followers_counts` - Mix Tasks: `mix pleroma.user toggle_confirmed` - Federation: Support for reports - Configuration: `safe_dm_mentions` option diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 42753a1a4..f650b447d 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -5,6 +5,8 @@ defmodule Mix.Tasks.Pleroma.Database do alias Mix.Tasks.Pleroma.Common alias Pleroma.Conversation + alias Pleroma.Repo + alias Pleroma.User require Logger use Mix.Task @@ -25,6 +27,9 @@ defmodule Mix.Tasks.Pleroma.Database do mix pleroma.database bump_all_conversations + ## Remove duplicated items from following and update followers count for all users + + mix pleroma.database update_users_following_followers_counts """ def run(["remove_embedded_objects" | args]) do {options, [], []} = @@ -38,7 +43,7 @@ def run(["remove_embedded_objects" | args]) do Common.start_pleroma() Logger.info("Removing embedded objects") - Pleroma.Repo.query!( + Repo.query!( "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;", [], timeout: :infinity @@ -47,7 +52,7 @@ def run(["remove_embedded_objects" | args]) do if Keyword.get(options, :vacuum) do Logger.info("Runnning VACUUM FULL") - Pleroma.Repo.query!( + Repo.query!( "vacuum full;", [], timeout: :infinity @@ -59,4 +64,12 @@ def run(["bump_all_conversations"]) do Common.start_pleroma() Conversation.bump_for_all_activities() end + + def run(["update_users_following_followers_counts"]) do + Common.start_pleroma() + + users = Repo.all(User) + Enum.each(users, &User.remove_duplicated_following/1) + Enum.each(users, &User.update_follower_count/1) + end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 1aa966dfc..9ffb61300 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -166,7 +166,7 @@ def remote_user_creation(params) do def update_changeset(struct, params \\ %{}) do struct - |> cast(params, [:bio, :name, :avatar]) + |> cast(params, [:bio, :name, :avatar, :following]) |> unique_constraint(:nickname) |> validate_format(:nickname, local_nickname_regex()) |> validate_length(:bio, max: 5000) @@ -709,6 +709,18 @@ def update_follower_count(%User{} = user) do end end + def remove_duplicated_following(%User{following: following} = user) do + uniq_following = Enum.uniq(following) + + if length(following) == length(uniq_following) do + {:ok, user} + else + user + |> update_changeset(%{following: uniq_following}) + |> update_and_set_cache() + end + end + @spec get_users_from_set([String.t()], boolean()) :: [User.t()] def get_users_from_set(ap_ids, local_only \\ true) do criteria = %{ap_id: ap_ids, deactivated: false} diff --git a/test/tasks/database_test.exs b/test/tasks/database_test.exs new file mode 100644 index 000000000..579130b05 --- /dev/null +++ b/test/tasks/database_test.exs @@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.DatabaseTest do + alias Pleroma.Repo + alias Pleroma.User + use Pleroma.DataCase + + import Pleroma.Factory + + setup_all do + Mix.shell(Mix.Shell.Process) + + on_exit(fn -> + Mix.shell(Mix.Shell.IO) + end) + + :ok + end + + describe "running update_users_following_followers_counts" do + test "following and followers count are updated" do + [user, user2] = insert_pair(:user) + {:ok, %User{following: following, info: info} = user} = User.follow(user, user2) + + assert length(following) == 2 + assert info.follower_count == 0 + + info_cng = Ecto.Changeset.change(info, %{follower_count: 3}) + + {:ok, user} = + user + |> Ecto.Changeset.change(%{following: following ++ following}) + |> Ecto.Changeset.put_embed(:info, info_cng) + |> Repo.update() + + assert length(user.following) == 4 + assert user.info.follower_count == 3 + + assert :ok == Mix.Tasks.Pleroma.Database.run(["update_users_following_followers_counts"]) + + user = User.get_by_id(user.id) + + assert length(user.following) == 2 + assert user.info.follower_count == 0 + end + end +end diff --git a/test/tasks/user_test.exs b/test/tasks/user_test.exs index 1f97740be..260ce0d95 100644 --- a/test/tasks/user_test.exs +++ b/test/tasks/user_test.exs @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.UserTest do + alias Pleroma.Repo alias Pleroma.User use Pleroma.DataCase diff --git a/test/user_test.exs b/test/user_test.exs index 16a014f2f..721b65693 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -626,6 +626,37 @@ test "it sets the info->follower_count property" do end end + describe "remove duplicates from following list" do + test "it removes duplicates" do + user = insert(:user) + follower = insert(:user) + + {:ok, %User{following: following} = follower} = User.follow(follower, user) + assert length(following) == 2 + + {:ok, follower} = + follower + |> User.update_changeset(%{following: following ++ following}) + |> Repo.update() + + assert length(follower.following) == 4 + + {:ok, follower} = User.remove_duplicated_following(follower) + assert length(follower.following) == 2 + end + + test "it does nothing when following is uniq" do + user = insert(:user) + follower = insert(:user) + + {:ok, follower} = User.follow(follower, user) + assert length(follower.following) == 2 + + {:ok, follower} = User.remove_duplicated_following(follower) + assert length(follower.following) == 2 + end + end + describe "follow_import" do test "it imports user followings from list" do [user1, user2, user3] = insert_list(3, :user) From 70235ce840a4ed834080ab6afc3000185590c6cb Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Fri, 17 May 2019 09:02:04 +0800 Subject: [PATCH 039/123] Fix typo: s/"tags"/"tag"/g --- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/tag_policy.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 2f105700b..50426e920 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -55,7 +55,7 @@ defp check_media_nsfw( object = if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_nsfw]), actor_host) do tags = (child_object["tag"] || []) ++ ["nsfw"] - child_object = Map.put(child_object, "tags", tags) + child_object = Map.put(child_object, "tag", tags) child_object = Map.put(child_object, "sensitive", true) Map.put(object, "object", child_object) else diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index b52be30e7..6683b8d8e 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -31,7 +31,7 @@ defp process_tag( object = object - |> Map.put("tags", tags) + |> Map.put("tag", tags) |> Map.put("sensitive", true) message = Map.put(message, "object", object) From 443cd68a4c37cc8e3a3260feff25791bd5ebabf4 Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Fri, 17 May 2019 08:11:17 +0800 Subject: [PATCH 040/123] Add tests for MRF.SimplePolicy --- .../activity_pub/mrf/simple_policy_test.exs | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 test/web/activity_pub/mrf/simple_policy_test.exs diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs new file mode 100644 index 000000000..1e0511975 --- /dev/null +++ b/test/web/activity_pub/mrf/simple_policy_test.exs @@ -0,0 +1,192 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do + use Pleroma.DataCase + import Pleroma.Factory + alias Pleroma.Config + alias Pleroma.Web.ActivityPub.MRF.SimplePolicy + + setup do + orig = Config.get!(:mrf_simple) + + Config.put(:mrf_simple, + media_removal: [], + media_nsfw: [], + federated_timeline_removal: [], + reject: [], + accept: [] + ) + + on_exit(fn -> + Config.put(:mrf_simple, orig) + end) + end + + describe "when :media_removal" do + test "is empty" do + Config.put([:mrf_simple, :media_removal], []) + media_message = build_media_message() + local_message = build_local_message() + + assert SimplePolicy.filter(media_message) == {:ok, media_message} + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + + test "has a matching host" do + Config.put([:mrf_simple, :media_removal], ["remote.instance"]) + media_message = build_media_message() + local_message = build_local_message() + + assert SimplePolicy.filter(media_message) == + {:ok, + media_message + |> Map.put("object", Map.delete(media_message["object"], "attachment"))} + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + end + + describe "when :media_nsfw" do + test "is empty" do + Config.put([:mrf_simple, :media_nsfw], []) + media_message = build_media_message() + local_message = build_local_message() + + assert SimplePolicy.filter(media_message) == {:ok, media_message} + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + + test "has a matching host" do + Config.put([:mrf_simple, :media_nsfw], ["remote.instance"]) + media_message = build_media_message() + local_message = build_local_message() + + assert SimplePolicy.filter(media_message) == + {:ok, + media_message + |> put_in(["object", "tag"], ["foo", "nsfw"]) + |> put_in(["object", "sensitive"], true)} + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + end + + defp build_media_message do + %{ + "actor" => "https://remote.instance/users/bob", + "type" => "Create", + "object" => %{ + "attachment" => [%{}], + "tag" => ["foo"], + "sensitive" => false + } + } + end + + describe "when :federated_timeline_removal" do + test "is empty" do + Config.put([:mrf_simple, :federated_timeline_removal], []) + {_, ftl_message} = build_ftl_actor_and_message() + local_message = build_local_message() + + assert SimplePolicy.filter(ftl_message) == {:ok, ftl_message} + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + + test "has a matching host" do + {actor, ftl_message} = build_ftl_actor_and_message() + + ftl_message_actor_host = + ftl_message + |> Map.fetch!("actor") + |> URI.parse() + |> Map.fetch!(:host) + + Config.put([:mrf_simple, :federated_timeline_removal], [ftl_message_actor_host]) + local_message = build_local_message() + + assert {:ok, ftl_message} = SimplePolicy.filter(ftl_message) + assert actor.follower_address in ftl_message["to"] + refute actor.follower_address in ftl_message["cc"] + refute "https://www.w3.org/ns/activitystreams#Public" in ftl_message["to"] + assert "https://www.w3.org/ns/activitystreams#Public" in ftl_message["cc"] + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + end + + defp build_ftl_actor_and_message do + actor = insert(:user) + + {actor, + %{ + "actor" => actor.ap_id, + "to" => ["https://www.w3.org/ns/activitystreams#Public", "http://foo.bar/baz"], + "cc" => [actor.follower_address, "http://foo.bar/qux"] + }} + end + + describe "when :reject" do + test "is empty" do + Config.put([:mrf_simple, :reject], []) + + remote_message = build_remote_message() + + assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + end + + test "has a matching host" do + Config.put([:mrf_simple, :reject], ["remote.instance"]) + + remote_message = build_remote_message() + + assert SimplePolicy.filter(remote_message) == {:reject, nil} + end + end + + describe "when :accept" do + test "is empty" do + Config.put([:mrf_simple, :accept], []) + + local_message = build_local_message() + remote_message = build_remote_message() + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + end + + test "is not empty but it doesn't have a matching host" do + Config.put([:mrf_simple, :accept], ["non.matching.remote"]) + + local_message = build_local_message() + remote_message = build_remote_message() + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + assert SimplePolicy.filter(remote_message) == {:reject, nil} + end + + test "has a matching host" do + Config.put([:mrf_simple, :accept], ["remote.instance"]) + + local_message = build_local_message() + remote_message = build_remote_message() + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + end + end + + defp build_local_message do + %{ + "actor" => "#{Pleroma.Web.base_url()}/users/alice", + "to" => [], + "cc" => [] + } + end + + defp build_remote_message do + %{"actor" => "https://remote.instance/users/bob"} + end +end From 68c75a08e8593d41f496a362ab1be43d054c09d5 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 17 May 2019 07:34:52 +0200 Subject: [PATCH 041/123] MastoAPI account_view.ex: requested is false when following is true Closes: https://git.pleroma.social/pleroma/pleroma/issues/903 --- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 779b9a382..606a0f2ed 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -40,7 +40,7 @@ def render("relationship.json", %{user: %User{} = user, target: %User{} = target follow_activity = Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(user, target) requested = - if follow_activity do + if follow_activity and !User.following?(target, user) do follow_activity.data["state"] == "pending" else false From 7ed682213faf88edc67b4c2a24b277ab1d992813 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 17 May 2019 07:25:20 +0000 Subject: [PATCH 042/123] Fix/902 random compile failing --- lib/pleroma/user.ex | 6 +++--- lib/pleroma/web/activity_pub/transmogrifier.ex | 1 - lib/pleroma/web/common_api/common_api.ex | 6 +++--- lib/pleroma/web/federator/publisher.ex | 8 ++++---- .../web/mastodon_api/mastodon_api_controller.ex | 2 +- lib/pleroma/web/oauth/authorization.ex | 2 +- lib/pleroma/web/oauth/token.ex | 2 +- lib/pleroma/web/web_finger/web_finger.ex | 2 +- test/formatter_test.exs | 2 +- test/plugs/legacy_authentication_plug_test.exs | 2 +- test/repo_test.exs | 7 ++++--- test/support/factory.ex | 9 +++++---- test/user_test.exs | 10 +++++----- test/web/mastodon_api/mastodon_api_controller_test.exs | 8 ++++---- test/web/ostatus/ostatus_test.exs | 4 ++-- 15 files changed, 36 insertions(+), 35 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 9ffb61300..9607c1f03 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -55,7 +55,7 @@ defmodule Pleroma.User do field(:last_refreshed_at, :naive_datetime_usec) has_many(:notifications, Notification) has_many(:registrations, Registration) - embeds_one(:info, Pleroma.User.Info) + embeds_one(:info, User.Info) timestamps() end @@ -233,7 +233,7 @@ def register_changeset(struct, params \\ %{}, opts \\ []) do |> validate_confirmation(:password) |> unique_constraint(:email) |> unique_constraint(:nickname) - |> validate_exclusion(:nickname, Pleroma.Config.get([Pleroma.User, :restricted_nicknames])) + |> validate_exclusion(:nickname, Pleroma.Config.get([User, :restricted_nicknames])) |> validate_format(:nickname, local_nickname_regex()) |> validate_format(:email, @email_regex) |> validate_length(:bio, max: 1000) @@ -278,7 +278,7 @@ def register(%Ecto.Changeset{} = changeset) do with {:ok, user} <- Repo.insert(changeset), {:ok, user} <- autofollow_users(user), {:ok, user} <- set_cache(user), - {:ok, _} <- Pleroma.User.WelcomeMessage.post_welcome_message_to_user(user), + {:ok, _} <- User.WelcomeMessage.post_welcome_message_to_user(user), {:ok, _} <- try_send_confirmation_email(user) do {:ok, user} end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 508f3532f..5edd8ccc7 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -11,7 +11,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Object.Containment alias Pleroma.Repo alias Pleroma.User - alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 208c12c7b..a599ffee5 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -200,7 +200,7 @@ def update(user) do user = with emoji <- emoji_from_profile(user), source_data <- (user.info.source_data || %{}) |> Map.put("tag", emoji), - info_cng <- Pleroma.User.Info.set_source_data(user.info, source_data), + info_cng <- User.Info.set_source_data(user.info, source_data), change <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_cng), {:ok, user} <- User.update_and_set_cache(change) do user @@ -233,7 +233,7 @@ def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do } = activity <- get_by_id_or_ap_id(id_or_ap_id), true <- Enum.member?(object_to, "https://www.w3.org/ns/activitystreams#Public"), %{valid?: true} = info_changeset <- - Pleroma.User.Info.add_pinnned_activity(user.info, activity), + User.Info.add_pinnned_activity(user.info, activity), changeset <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset), {:ok, _user} <- User.update_and_set_cache(changeset) do @@ -250,7 +250,7 @@ def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do def unpin(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), %{valid?: true} = info_changeset <- - Pleroma.User.Info.remove_pinnned_activity(user.info, activity), + User.Info.remove_pinnned_activity(user.info, activity), changeset <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset), {:ok, _user} <- User.update_and_set_cache(changeset) do diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index fb4e8548d..70f870244 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -52,9 +52,9 @@ def perform(type, _, _) do @doc """ Relays an activity to all specified peers. """ - @callback publish(Pleroma.User.t(), Pleroma.Activity.t()) :: :ok | {:error, any()} + @callback publish(User.t(), Activity.t()) :: :ok | {:error, any()} - @spec publish(Pleroma.User.t(), Pleroma.Activity.t()) :: :ok + @spec publish(User.t(), Activity.t()) :: :ok def publish(%User{} = user, %Activity{} = activity) do Config.get([:instance, :federation_publisher_modules]) |> Enum.each(fn module -> @@ -70,9 +70,9 @@ def publish(%User{} = user, %Activity{} = activity) do @doc """ Gathers links used by an outgoing federation module for WebFinger output. """ - @callback gather_webfinger_links(Pleroma.User.t()) :: list() + @callback gather_webfinger_links(User.t()) :: list() - @spec gather_webfinger_links(Pleroma.User.t()) :: list() + @spec gather_webfinger_links(User.t()) :: list() def gather_webfinger_links(%User{} = user) do Config.get([:instance, :federation_publisher_modules]) |> Enum.reduce([], fn module, links -> diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 66056a846..1b776fbca 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1222,7 +1222,7 @@ def remove_from_list(%{assigns: %{user: user}} = conn, %{"id" => id, "account_id accounts |> Enum.each(fn account_id -> with %Pleroma.List{} = list <- Pleroma.List.get(id, user), - %User{} = followed <- Pleroma.User.get_cached_by_id(account_id) do + %User{} = followed <- User.get_cached_by_id(account_id) do Pleroma.List.unfollow(list, followed) end end) diff --git a/lib/pleroma/web/oauth/authorization.ex b/lib/pleroma/web/oauth/authorization.ex index b47688de1..18973413e 100644 --- a/lib/pleroma/web/oauth/authorization.ex +++ b/lib/pleroma/web/oauth/authorization.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Web.OAuth.Authorization do field(:scopes, {:array, :string}, default: []) field(:valid_until, :naive_datetime_usec) field(:used, :boolean, default: false) - belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: Pleroma.FlakeId) belongs_to(:app, App) timestamps() diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index ef047d565..66c95c2e9 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.OAuth.Token do field(:refresh_token, :string) field(:scopes, {:array, :string}, default: []) field(:valid_until, :naive_datetime_usec) - belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: Pleroma.FlakeId) belongs_to(:app, App) timestamps() diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 3a3b98a10..1239b962a 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -99,7 +99,7 @@ def ensure_keys_present(user) do info_cng = info - |> Pleroma.User.Info.set_keys(pem) + |> User.Info.set_keys(pem) cng = Ecto.Changeset.change(user) diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 06f4f6e50..5e7011160 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -125,7 +125,7 @@ test "gives a replacement for user links, using local nicknames in user links te archaeme = insert(:user, %{ nickname: "archa_eme_", - info: %Pleroma.User.Info{source_data: %{"url" => "https://archeme/@archa_eme_"}} + info: %User.Info{source_data: %{"url" => "https://archeme/@archa_eme_"}} }) archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"}) diff --git a/test/plugs/legacy_authentication_plug_test.exs b/test/plugs/legacy_authentication_plug_test.exs index 8b0b06772..02f530058 100644 --- a/test/plugs/legacy_authentication_plug_test.exs +++ b/test/plugs/legacy_authentication_plug_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Plugs.LegacyAuthenticationPlugTest do - use Pleroma.Web.ConnCase, async: true + use Pleroma.Web.ConnCase alias Pleroma.Plugs.LegacyAuthenticationPlug alias Pleroma.User diff --git a/test/repo_test.exs b/test/repo_test.exs index 5382289c7..85085a1fa 100644 --- a/test/repo_test.exs +++ b/test/repo_test.exs @@ -1,23 +1,24 @@ defmodule Pleroma.RepoTest do use Pleroma.DataCase import Pleroma.Factory + alias Pleroma.User describe "find_resource/1" do test "returns user" do user = insert(:user) - query = from(t in Pleroma.User, where: t.id == ^user.id) + query = from(t in User, where: t.id == ^user.id) assert Repo.find_resource(query) == {:ok, user} end test "returns not_found" do - query = from(t in Pleroma.User, where: t.id == ^"9gBuXNpD2NyDmmxxdw") + query = from(t in User, where: t.id == ^"9gBuXNpD2NyDmmxxdw") assert Repo.find_resource(query) == {:error, :not_found} end end describe "get_assoc/2" do test "get assoc from preloaded data" do - user = %Pleroma.User{name: "Agent Smith"} + user = %User{name: "Agent Smith"} token = %Pleroma.Web.OAuth.Token{insert(:oauth_token) | user: user} assert Repo.get_assoc(token, :user) == {:ok, user} end diff --git a/test/support/factory.ex b/test/support/factory.ex index 90c7d80f2..be6247ca4 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Factory do use ExMachina.Ecto, repo: Pleroma.Repo + alias Pleroma.User def participation_factory do conversation = insert(:conversation) @@ -23,7 +24,7 @@ def conversation_factory do end def user_factory do - user = %Pleroma.User{ + user = %User{ name: sequence(:name, &"Test テスト User #{&1}"), email: sequence(:email, &"user#{&1}@example.com"), nickname: sequence(:nickname, &"nick#{&1}"), @@ -34,9 +35,9 @@ def user_factory do %{ user - | ap_id: Pleroma.User.ap_id(user), - follower_address: Pleroma.User.ap_followers(user), - following: [Pleroma.User.ap_id(user)] + | ap_id: User.ap_id(user), + follower_address: User.ap_followers(user), + following: [User.ap_id(user)] } end diff --git a/test/user_test.exs b/test/user_test.exs index 721b65693..10e463ff8 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -277,7 +277,7 @@ test "it requires an email, name, nickname and password, bio is optional" do end test "it restricts certain nicknames" do - [restricted_name | _] = Pleroma.Config.get([Pleroma.User, :restricted_nicknames]) + [restricted_name | _] = Pleroma.Config.get([User, :restricted_nicknames]) assert is_bitstring(restricted_name) @@ -1223,11 +1223,11 @@ test "follower count is updated when a follower is blocked" do follower2 = insert(:user) follower3 = insert(:user) - {:ok, follower} = Pleroma.User.follow(follower, user) - {:ok, _follower2} = Pleroma.User.follow(follower2, user) - {:ok, _follower3} = Pleroma.User.follow(follower3, user) + {:ok, follower} = User.follow(follower, user) + {:ok, _follower2} = User.follow(follower2, user) + {:ok, _follower3} = User.follow(follower3, user) - {:ok, _} = Pleroma.User.block(user, follower) + {:ok, _} = User.block(user, follower) user_show = Pleroma.Web.TwitterAPI.UserView.render("show.json", %{user: user}) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 90d67a55f..cbff141c8 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -446,7 +446,7 @@ test "verify_credentials", %{conn: conn} do end test "verify_credentials default scope unlisted", %{conn: conn} do - user = insert(:user, %{info: %Pleroma.User.Info{default_scope: "unlisted"}}) + user = insert(:user, %{info: %User.Info{default_scope: "unlisted"}}) conn = conn @@ -1322,7 +1322,7 @@ test "returns the relationships for the current user", %{conn: conn} do describe "locked accounts" do test "/api/v1/follow_requests works" do - user = insert(:user, %{info: %Pleroma.User.Info{locked: true}}) + user = insert(:user, %{info: %User.Info{locked: true}}) other_user = insert(:user) {:ok, _activity} = ActivityPub.follow(other_user, user) @@ -1367,7 +1367,7 @@ test "/api/v1/follow_requests/:id/authorize works" do end test "verify_credentials", %{conn: conn} do - user = insert(:user, %{info: %Pleroma.User.Info{default_scope: "private"}}) + user = insert(:user, %{info: %User.Info{default_scope: "private"}}) conn = conn @@ -1379,7 +1379,7 @@ test "verify_credentials", %{conn: conn} do end test "/api/v1/follow_requests/:id/reject works" do - user = insert(:user, %{info: %Pleroma.User.Info{locked: true}}) + user = insert(:user, %{info: %User.Info{locked: true}}) other_user = insert(:user) {:ok, _activity} = ActivityPub.follow(other_user, user) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 2916caf8d..3516a13fa 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -355,7 +355,7 @@ test "tries to use the information in poco fields" do {:ok, user} = OStatus.find_or_make_user(uri) - user = Pleroma.User.get_cached_by_id(user.id) + user = User.get_cached_by_id(user.id) assert user.name == "Constance Variable" assert user.nickname == "lambadalambda@social.heldscal.la" assert user.local == false @@ -374,7 +374,7 @@ test "find_or_make_user sets all the nessary input fields" do {:ok, user} = OStatus.find_or_make_user(uri) assert user.info == - %Pleroma.User.Info{ + %User.Info{ id: user.info.id, ap_enabled: false, background: %{}, From 718b59a2631af6245570a073e3c5b2464b810a27 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 11:19:30 +0200 Subject: [PATCH 043/123] Dependencies: Update ecto sql once more. --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 2535fcf98..2888b69e2 100644 --- a/mix.exs +++ b/mix.exs @@ -67,7 +67,7 @@ defp deps do {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, git: "https://github.com/elixir-ecto/ecto_sql", - ref: "e839a9a327b632d73533ac8105ba360bc831cf83", + ref: "14cb065a74c488d737d973f7a91bc036c6245f78", override: true}, {:postgrex, ">= 0.13.5"}, {:gettext, "~> 0.15"}, diff --git a/mix.lock b/mix.lock index 5ee5634d3..714a8a2b1 100644 --- a/mix.lock +++ b/mix.lock @@ -21,7 +21,7 @@ "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"}, "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "e839a9a327b632d73533ac8105ba360bc831cf83", [ref: "e839a9a327b632d73533ac8105ba360bc831cf83"]}, + "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "14cb065a74c488d737d973f7a91bc036c6245f78", [ref: "14cb065a74c488d737d973f7a91bc036c6245f78"]}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, From e39c190f046bb345e80a2195278bb59ac81b8002 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 11:30:35 +0200 Subject: [PATCH 044/123] Configuration: disable migration lock. --- config/config.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 3d11c3a87..92d3e0dcd 100644 --- a/config/config.exs +++ b/config/config.exs @@ -48,7 +48,8 @@ config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes, - telemetry_event: [Pleroma.Repo.Instrumenter] + telemetry_event: [Pleroma.Repo.Instrumenter], + migration_lock: nil config :pleroma, Pleroma.Captcha, enabled: false, From 249b31ffe53115ad187235890c405138ba190e14 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 11:49:13 +0200 Subject: [PATCH 045/123] Fix specs. --- lib/pleroma/user.ex | 1 - test/web/ostatus/ostatus_test.exs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 9607c1f03..28da310ee 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1144,7 +1144,6 @@ def delete_user_activities(%User{ap_id: ap_id} = user) do stream = ap_id |> Activity.query_by_actor() - |> Activity.with_preloaded_object() |> Repo.stream() Repo.transaction(fn -> Enum.each(stream, &delete_activity(&1)) end, timeout: :infinity) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 3516a13fa..f6be16862 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -407,7 +407,7 @@ test "find_make_or_update_user takes an author element and returns an updated us {:ok, user} = OStatus.find_or_make_user(uri) old_name = user.name old_bio = user.bio - change = Ecto.Changeset.change(user, %{avatar: nil, bio: nil, old_name: nil}) + change = Ecto.Changeset.change(user, %{avatar: nil, bio: nil, name: nil}) {:ok, user} = Repo.update(change) refute user.avatar From 5743124c7727af3fbcd8243f1f8e71de91c9b470 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 11:51:50 +0200 Subject: [PATCH 046/123] Dependencies: Update tzdata. This fixes the bug that tz data could not be loaded. Closes #910. --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index cd2d2370a..1dc19532d 100644 --- a/mix.lock +++ b/mix.lock @@ -83,7 +83,7 @@ "tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, "timex": {:hex, :timex, "3.5.0", "b0a23167da02d0fe4f1a4e104d1f929a00d348502b52432c05de875d0b9cffa5", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, - "tzdata": {:hex, :tzdata, "0.5.17", "50793e3d85af49736701da1a040c415c97dc1caf6464112fd9bd18f425d3053b", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, + "tzdata": {:hex, :tzdata, "0.5.20", "304b9e98a02840fb32a43ec111ffbe517863c8566eb04a061f1c4dbb90b4d84c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, "ueberauth": {:hex, :ueberauth, "0.6.1", "9e90d3337dddf38b1ca2753aca9b1e53d8a52b890191cdc55240247c89230412", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"}, "unsafe": {:hex, :unsafe, "1.0.0", "7c21742cd05380c7875546b023481d3a26f52df8e5dfedcb9f958f322baae305", [:mix], [], "hexpm"}, From 46684db84d0583cb5d88f8fcbd6b970ef95c84de Mon Sep 17 00:00:00 2001 From: lambda Date: Fri, 17 May 2019 09:53:44 +0000 Subject: [PATCH 047/123] Update account_view.ex --- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 606a0f2ed..134c07b7e 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -40,7 +40,7 @@ def render("relationship.json", %{user: %User{} = user, target: %User{} = target follow_activity = Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(user, target) requested = - if follow_activity and !User.following?(target, user) do + if follow_activity && !User.following?(target, user) do follow_activity.data["state"] == "pending" else false From a3fc7294da43a7e2d17626200e8d3bf5c05808d3 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 12:47:38 +0200 Subject: [PATCH 048/123] CI: Add rum variant testing. --- .gitlab-ci.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f9745122a..3ea275127 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,7 @@ cache: stages: - build - test + - test-rum - deploy before_script: @@ -45,7 +46,7 @@ docs-build: unit-testing: stage: test services: - - name: postgres:9.6.2 + - name: lainsoykaf/postgres-with-rum command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] script: - mix deps.get @@ -54,6 +55,20 @@ unit-testing: - mix test --trace --preload-modules - mix coveralls +unit-testing-rum: + stage: test-rum + services: + - name: lainsoykaf/postgres-with-rum + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + script: + - "echo 'config :pleroma, :database, rum_enabled: true' >> config/test.exs" + - mix deps.get + - mix ecto.create + - mix ecto.migrate + - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/" + - mix test --trace --preload-modules + - mix coveralls + lint: stage: test script: @@ -65,7 +80,6 @@ analysis: - mix deps.get - mix credo --strict --only=warnings,todo,fixme,consistency,readability - docs-deploy: stage: deploy image: alpine:3.9 From 022e6e4b44229be70f0ec6720a66610a0e2a403a Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 14:59:01 +0200 Subject: [PATCH 049/123] RUM: Remove vacuum analyze from migration Can't be run in a trnasaction. --- .../20190510135645_add_fts_index_to_objects_two.exs | 2 -- 1 file changed, 2 deletions(-) diff --git a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs index 09e6cbfb1..b6a24441a 100644 --- a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs +++ b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs @@ -20,7 +20,6 @@ def up do FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()") execute("UPDATE objects SET updated_at = NOW()") - execute("vacuum analyze") end def down do @@ -31,6 +30,5 @@ def down do remove(:fts_content, :tsvector) end create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) - execute("vacuum analyze") end end From 8784a7d1b40394cb1b4d5ce2d04b8cd057a57ceb Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 15:06:51 +0200 Subject: [PATCH 050/123] RUM: Set rum status by the environment. --- config/test.exs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/test.exs b/config/test.exs index 40db66170..e1785d10f 100644 --- a/config/test.exs +++ b/config/test.exs @@ -63,6 +63,10 @@ config :pleroma, :http_security, report_uri: "https://endpoint.com" +rum_enabled = System.get_env("RUM_ENABLED") == "true" +config :pleroma, :database, rum_enabled: rum_enabled +IO.puts("RUM enabled: #{rum_enabled}") + try do import_config "test.secret.exs" rescue From ef63cf70883f4c7e3814ebac907f000da835ea10 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 15:21:29 +0200 Subject: [PATCH 051/123] CI: Use the correct image with the correct hostname. --- .gitlab-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3ea275127..8b5131dc3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,7 +15,6 @@ cache: stages: - build - test - - test-rum - deploy before_script: @@ -47,6 +46,7 @@ unit-testing: stage: test services: - name: lainsoykaf/postgres-with-rum + alias: postgres command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] script: - mix deps.get @@ -56,18 +56,19 @@ unit-testing: - mix coveralls unit-testing-rum: - stage: test-rum + stage: test services: - name: lainsoykaf/postgres-with-rum + alias: postgres command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + variables: + RUM_ENABLED: "true" script: - - "echo 'config :pleroma, :database, rum_enabled: true' >> config/test.exs" - mix deps.get - mix ecto.create - mix ecto.migrate - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/" - mix test --trace --preload-modules - - mix coveralls lint: stage: test From f959bf7aa6b878ee5b669c4caabd5cdc4cc2dc9e Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 18:21:11 +0200 Subject: [PATCH 052/123] MongooseIM: Add basic integration endpoints. --- .../web/mongooseim/mongoose_im_controller.ex | 41 +++++++++++++ lib/pleroma/web/router.ex | 5 ++ .../mongoose_im_controller_test.exs | 59 +++++++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 lib/pleroma/web/mongooseim/mongoose_im_controller.ex create mode 100644 test/web/mongooseim/mongoose_im_controller_test.exs diff --git a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex new file mode 100644 index 000000000..f8c634653 --- /dev/null +++ b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex @@ -0,0 +1,41 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MongooseIM.MongooseIMController do + use Pleroma.Web, :controller + alias Comeonin.Pbkdf2 + alias Pleroma.User + alias Pleroma.Repo + + def user_exists(conn, %{"user" => username}) do + with %User{} <- Repo.get_by(User, nickname: username, local: true) do + conn + |> json(true) + else + _ -> + conn + |> put_status(:not_found) + |> json(false) + end + end + + def check_password(conn, %{"user" => username, "pass" => password}) do + with %User{password_hash: password_hash} <- + Repo.get_by(User, nickname: username, local: true), + true <- Pbkdf2.checkpw(password, password_hash) do + conn + |> json(true) + else + false -> + conn + |> put_status(403) + |> json(false) + + _ -> + conn + |> put_status(:not_found) + |> json(false) + end + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6a4e4a1d4..552778c92 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -704,6 +704,11 @@ defmodule Pleroma.Web.Router do end end + scope "/", Pleroma.Web.MongooseIM do + get("/user_exists", MongooseIMController, :user_exists) + get("/check_password", MongooseIMController, :check_password) + end + scope "/", Fallback do get("/registration/:token", RedirectController, :registration_page) get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta) diff --git a/test/web/mongooseim/mongoose_im_controller_test.exs b/test/web/mongooseim/mongoose_im_controller_test.exs new file mode 100644 index 000000000..eb83999bb --- /dev/null +++ b/test/web/mongooseim/mongoose_im_controller_test.exs @@ -0,0 +1,59 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MongooseIMController do + use Pleroma.Web.ConnCase + import Pleroma.Factory + + test "/user_exists", %{conn: conn} do + _user = insert(:user, nickname: "lain") + _remote_user = insert(:user, nickname: "alice", local: false) + + res = + conn + |> get(mongoose_im_path(conn, :user_exists), user: "lain") + |> json_response(200) + + assert res == true + + res = + conn + |> get(mongoose_im_path(conn, :user_exists), user: "alice") + |> json_response(404) + + assert res == false + + res = + conn + |> get(mongoose_im_path(conn, :user_exists), user: "bob") + |> json_response(404) + + assert res == false + end + + test "/check_password", %{conn: conn} do + user = insert(:user, password_hash: Comeonin.Pbkdf2.hashpwsalt("cool")) + + res = + conn + |> get(mongoose_im_path(conn, :check_password), user: user.nickname, pass: "cool") + |> json_response(200) + + assert res == true + + res = + conn + |> get(mongoose_im_path(conn, :check_password), user: user.nickname, pass: "uncool") + |> json_response(403) + + assert res == false + + res = + conn + |> get(mongoose_im_path(conn, :check_password), user: "nobody", pass: "cool") + |> json_response(404) + + assert res == false + end +end From 075eecec907b0a623a90eed44a0378a6812d8037 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 18:32:30 +0200 Subject: [PATCH 053/123] Linting. --- lib/pleroma/web/mongooseim/mongoose_im_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex index f8c634653..489d5d3a5 100644 --- a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex +++ b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex @@ -5,8 +5,8 @@ defmodule Pleroma.Web.MongooseIM.MongooseIMController do use Pleroma.Web, :controller alias Comeonin.Pbkdf2 - alias Pleroma.User alias Pleroma.Repo + alias Pleroma.User def user_exists(conn, %{"user" => username}) do with %User{} <- Repo.get_by(User, nickname: username, local: true) do From 78588dbd80580fbef53819dd87ee8fcc26cb09e9 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 17 May 2019 18:49:10 +0000 Subject: [PATCH 054/123] mrf: simple policy: mark all posts instead of posts with media as sensitive if they match media_nsfw --- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 50426e920..9627c3400 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -48,10 +48,9 @@ defp check_media_nsfw( %{host: actor_host} = _actor_info, %{ "type" => "Create", - "object" => %{"attachment" => child_attachment} = child_object + "object" => child_object } = object - ) - when length(child_attachment) > 0 do + ) do object = if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_nsfw]), actor_host) do tags = (child_object["tag"] || []) ++ ["nsfw"] From 0da1233e8e093ff7c69994f9e81d58611be60507 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 17 May 2019 18:49:43 +0000 Subject: [PATCH 055/123] rich media: suppress link previews if post is marked as sensitive --- lib/pleroma/web/rich_media/helpers.ex | 1 + test/web/rich_media/helpers_test.exs | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 0162a5be9..9bc8f2559 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -24,6 +24,7 @@ defp validate_page_url(_), do: :error def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do with true <- Pleroma.Config.get([:rich_media, :enabled]), %Object{} = object <- Object.normalize(activity), + false <- object.data["sensitive"] || false, {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]), :ok <- validate_page_url(page_url), {:ok, rich_media} <- Parser.parse(page_url) do diff --git a/test/web/rich_media/helpers_test.exs b/test/web/rich_media/helpers_test.exs index 60d93768f..6e23392ca 100644 --- a/test/web/rich_media/helpers_test.exs +++ b/test/web/rich_media/helpers_test.exs @@ -1,6 +1,7 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do use Pleroma.DataCase + alias Pleroma.Object alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -59,4 +60,24 @@ test "crawls valid, complete URLs" do Pleroma.Config.put([:rich_media, :enabled], false) end + + test "refuses to crawl URLs from posts marked sensitive" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "http://example.com/ogp", + "spoiler_text" => "." + }) + + %Object{} = object = Object.normalize(activity) + + assert object.data["sensitive"] + + Pleroma.Config.put([:rich_media, :enabled], true) + + assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) + + Pleroma.Config.put([:rich_media, :enabled], false) + end end From d3b8cd342ff6d1fe87bcfe6305424faf49666252 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 17 May 2019 19:03:19 +0000 Subject: [PATCH 056/123] http: request builder: send user-agent when making requests --- lib/pleroma/http/request_builder.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 5f2cff2c0..522728da1 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -45,8 +45,9 @@ def url(request, u) do Add headers to the request """ @spec headers(map(), list(tuple)) :: map() - def headers(request, h) do - Map.put_new(request, :headers, h) + def headers(request, header_list) do + header_list = header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}] + Map.put_new(request, :headers, header_list) end @doc """ From 290f5b2cfe91dd2acba56f79ef137f15c68a0db0 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 17 May 2019 20:28:58 +0000 Subject: [PATCH 057/123] config: make sending the user agent configurable, disable sending the user agent in tests --- config/config.exs | 1 + config/test.exs | 2 ++ lib/pleroma/http/request_builder.ex | 8 +++++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index e82f08e07..9a10b0ff7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -192,6 +192,7 @@ # Configures http settings, upstream proxy etc. config :pleroma, :http, proxy_url: nil, + send_user_agent: true, adapter: [ ssl_options: [ # We don't support TLS v1.3 yet diff --git a/config/test.exs b/config/test.exs index e1785d10f..6100989c4 100644 --- a/config/test.exs +++ b/config/test.exs @@ -63,6 +63,8 @@ config :pleroma, :http_security, report_uri: "https://endpoint.com" +config :pleroma, :http, send_user_agent: false + rum_enabled = System.get_env("RUM_ENABLED") == "true" config :pleroma, :database, rum_enabled: rum_enabled IO.puts("RUM enabled: #{rum_enabled}") diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 522728da1..e23457999 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -46,7 +46,13 @@ def url(request, u) do """ @spec headers(map(), list(tuple)) :: map() def headers(request, header_list) do - header_list = header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}] + header_list = + if Pleroma.Config.get([:http, :send_user_agent]) do + header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}] + else + header_list + end + Map.put_new(request, :headers, header_list) end From c234ce546a769747f436c19fee99bed2a7a58f3b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 17 May 2019 20:31:39 +0000 Subject: [PATCH 058/123] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12c439135..5de9ae292 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Correct `reblogged`, `favourited`, and `bookmarked` values in the reblog status JSON - Mastodon API: Exposing default scope of the user to anyone - Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`] +- User-Agent is now sent correctly for all HTTP requests. ## Removed - Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations` From dc081595385084fe6b382e4b38c17cb51cf0a611 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 17 May 2019 20:42:51 +0000 Subject: [PATCH 059/123] also suppress link previews from posts marked #nsfw --- lib/pleroma/web/common_api/common_api.ex | 4 +++- lib/pleroma/web/common_api/utils.ex | 4 +++- test/web/rich_media/helpers_test.exs | 21 ++++++++++++++++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index a599ffee5..5a312d673 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -157,6 +157,7 @@ def post(user, %{"status" => status} = data) do {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility), context <- make_context(in_reply_to), cw <- data["spoiler_text"] || "", + sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}), full_payload <- String.trim(status <> cw), length when length in 1..limit <- String.length(full_payload), object <- @@ -169,7 +170,8 @@ def post(user, %{"status" => status} = data) do in_reply_to, tags, cw, - cc + cc, + sensitive ), object <- Map.put( diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index bee2fd159..8d6160976 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -223,7 +223,8 @@ def make_note_data( in_reply_to, tags, cw \\ nil, - cc \\ [] + cc \\ [], + sensitive \\ false ) do object = %{ "type" => "Note", @@ -231,6 +232,7 @@ def make_note_data( "cc" => cc, "content" => content_html, "summary" => cw, + "sensitive" => sensitive, "context" => context, "attachment" => attachments, "actor" => actor, diff --git a/test/web/rich_media/helpers_test.exs b/test/web/rich_media/helpers_test.exs index 6e23392ca..53b0596f5 100644 --- a/test/web/rich_media/helpers_test.exs +++ b/test/web/rich_media/helpers_test.exs @@ -67,7 +67,26 @@ test "refuses to crawl URLs from posts marked sensitive" do {:ok, activity} = CommonAPI.post(user, %{ "status" => "http://example.com/ogp", - "spoiler_text" => "." + "sensitive" => true + }) + + %Object{} = object = Object.normalize(activity) + + assert object.data["sensitive"] + + Pleroma.Config.put([:rich_media, :enabled], true) + + assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) + + Pleroma.Config.put([:rich_media, :enabled], false) + end + + test "refuses to crawl URLs from posts tagged NSFW" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "http://example.com/ogp #nsfw" }) %Object{} = object = Object.normalize(activity) From c4a55e167afcfd25cf4c1efb46886f2defe72c22 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 17 May 2019 20:43:31 +0000 Subject: [PATCH 060/123] add Changelog entry --- CHANGELOG.md | 1 + lib/pleroma/web/common_api/utils.ex | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12c439135..a73bf47d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Deps: Updated Ecto to 3.0.7 - Don't ship finmoji by default, they can be installed as an emoji pack - Hide deactivated users and their statuses +- Posts which are marked sensitive or tagged nsfw no longer have link previews. ### Fixed - Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended. diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 8d6160976..d93c0d46e 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -232,7 +232,7 @@ def make_note_data( "cc" => cc, "content" => content_html, "summary" => cw, - "sensitive" => sensitive, + "sensitive" => !Enum.member?(["false", "False", "0", false], sensitive), "context" => context, "attachment" => attachments, "actor" => actor, From 2375e9a95ba9042958ff7e8f75830df4ab53fed2 Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Mon, 20 May 2019 06:02:50 +0800 Subject: [PATCH 061/123] Add report filtering to MRF.SimplePolicy --- config/config.exs | 1 + .../web/activity_pub/mrf/simple_policy.ex | 13 ++++++++- .../activity_pub/mrf/simple_policy_test.exs | 28 +++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 9a10b0ff7..bab47a8a2 100644 --- a/config/config.exs +++ b/config/config.exs @@ -298,6 +298,7 @@ media_removal: [], media_nsfw: [], federated_timeline_removal: [], + report_removal: [], reject: [], accept: [] diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 9627c3400..7190652d2 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -94,6 +94,16 @@ defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do {:ok, object} end + defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do + if actor_host in Pleroma.Config.get([:mrf_simple, :report_removal]) do + {:reject, nil} + else + {:ok, object} + end + end + + defp check_report_removal(_actor_info, object), do: {:ok, object} + @impl true def filter(object) do actor_info = URI.parse(object["actor"]) @@ -102,7 +112,8 @@ def filter(object) do {:ok, object} <- check_reject(actor_info, object), {:ok, object} <- check_media_removal(actor_info, object), {:ok, object} <- check_media_nsfw(actor_info, object), - {:ok, object} <- check_ftl_removal(actor_info, object) do + {:ok, object} <- check_ftl_removal(actor_info, object), + {:ok, object} <- check_report_removal(actor_info, object) do {:ok, object} else _e -> {:reject, nil} diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs index 1e0511975..74af7dcde 100644 --- a/test/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/web/activity_pub/mrf/simple_policy_test.exs @@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do media_removal: [], media_nsfw: [], federated_timeline_removal: [], + report_removal: [], reject: [], accept: [] ) @@ -85,6 +86,33 @@ defp build_media_message do } end + describe "when :report_removal" do + test "is empty" do + Config.put([:mrf_simple, :report_removal], []) + report_message = build_report_message() + local_message = build_local_message() + + assert SimplePolicy.filter(report_message) == {:ok, report_message} + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + + test "has a matching host" do + Config.put([:mrf_simple, :report_removal], ["remote.instance"]) + report_message = build_report_message() + local_message = build_local_message() + + assert SimplePolicy.filter(report_message) == {:reject, nil} + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + end + + defp build_report_message do + %{ + "actor" => "https://remote.instance/users/bob", + "type" => "Flag" + } + end + describe "when :federated_timeline_removal" do test "is empty" do Config.put([:mrf_simple, :federated_timeline_removal], []) From 54e9cb5c2db580bc12441f3651fa87a7b976137d Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 20 May 2019 12:39:23 +0100 Subject: [PATCH 062/123] Add API endpoints for a custom user mascot --- docs/api/pleroma_api.md | 39 +++++++++++++++++++ lib/pleroma/user/info.ex | 21 ++++++++++ .../mastodon_api/mastodon_api_controller.ex | 36 ++++++++++++++++- lib/pleroma/web/router.ex | 3 ++ 4 files changed, 98 insertions(+), 1 deletion(-) diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index dd0b6ca73..4d99a2d2b 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -252,6 +252,45 @@ See [Admin-API](Admin-API.md) ] ``` +## `/api/v1/pleroma/mascot` +### Gets user mascot image +* Method `GET` +* Authentication: required + +* Response: JSON. Returns a mastodon media attachment entity. +* Example response: +```json +{ + "id": "abcdefg", + "url": "https://pleroma.example.org/media/abcdefg.png", + "type": "image", + "pleroma": { + "mime_type": "image/png" + } +} +``` + +### Updates user mascot image +* Method `PUT` +* Authentication: required +* Params: + * `image`: Multipart image +* Response: JSON. Returns a mastodon media attachment entity + when successful, otherwise returns HTTP 415 `{"error": "error_msg"}` +* Example response: +```json +{ + "id": "abcdefg", + "url": "https://pleroma.example.org/media/abcdefg.png", + "type": "image", + "pleroma": { + "mime_type": "image/png" + } +} +``` +* Note: Behaves exactly the same as `POST /api/v1/upload`. + Can only accept images - any attempt to upload non-image files will be met with `HTTP 415 Unsupported Media Type`. + ## `/api/pleroma/notification_settings` ### Updates user notification settings * Method `PUT` diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 5f0cefc00..ffcd06e3e 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -43,6 +43,19 @@ defmodule Pleroma.User.Info do field(:hide_favorites, :boolean, default: true) field(:pinned_activities, {:array, :string}, default: []) field(:flavour, :string, default: nil) + + field(:mascot, :map, + default: %{ + id: "pleromatan", + url: "/images/pleroma-fox-tan-smol.png", + type: "image", + preview_url: "/images/pleroma-fox-tan-smol.png", + pleroma: %{ + mime_type: "image/png" + } + } + ) + field(:emoji, {:array, :map}, default: []) field(:notification_settings, :map, @@ -248,6 +261,14 @@ def mastodon_flavour_update(info, flavour) do |> validate_required([:flavour]) end + def mascot_update(info, url) do + params = %{mascot: url} + + info + |> cast(params, [:mascot]) + |> validate_required([:mascot]) + end + def set_source_data(info, source_data) do params = %{source_data: source_data} diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 1051861ff..67f363859 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -707,6 +707,40 @@ def upload(%{assigns: %{user: user}} = conn, %{"file" => file} = data) do end end + def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do + with {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)), + %{} = attachment_data <- Map.put(object.data, "id", object.id), + %{type: type} = rendered <- + StatusView.render("attachment.json", %{attachment: attachment_data}) do + # Reject if not an image + if type == "image" do + # Sure! + # Save to the user's info + info_changeset = User.Info.mascot_update(user.info, rendered) + + user_changeset = + user + |> Ecto.Changeset.change() + |> Ecto.Changeset.put_embed(:info, info_changeset) + + {:ok, _user} = User.update_and_set_cache(user_changeset) + + conn + |> json(rendered) + else + conn + |> send_resp(415, Jason.encode!(%{"error" => "mascots can only be images"})) + end + end + end + + def get_mascot(%{assigns: %{user: user}} = conn, _params) do + %{info: %{mascot: mascot}} = user + + conn + |> json(mascot) + end + def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id), %Object{data: %{"likes" => likes}} <- Object.normalize(object) do @@ -1329,7 +1363,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do display_sensitive_media: false, reduce_motion: false, max_toot_chars: limit, - mascot: "/images/pleroma-fox-tan-smol.png" + mascot: Map.get(user.info.mascot, "url", "/images/pleroma-fox-tan-smol.png") }, rights: %{ delete_others_notice: present?(user.info.is_moderator), diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6a4e4a1d4..4c29b24eb 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -352,6 +352,9 @@ defmodule Pleroma.Web.Router do post("/pleroma/flavour/:flavour", MastodonAPIController, :set_flavour) + get("/pleroma/mascot", MastodonAPIController, :get_mascot) + put("/pleroma/mascot", MastodonAPIController, :set_mascot) + post("/reports", MastodonAPIController, :reports) end From e81f0fc6d45249dd70656c58af926c21c70c482f Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 20 May 2019 12:58:06 +0100 Subject: [PATCH 063/123] Add mascot get/set tests --- .../mastodon_api/mastodon_api_controller.ex | 1 + test/fixtures/sound.mp3 | Bin 0 -> 521 bytes .../mastodon_api_controller_test.exs | 65 ++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 test/fixtures/sound.mp3 diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 67f363859..d7f095a1f 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -729,6 +729,7 @@ def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do |> json(rendered) else conn + |> put_resp_content_type("application/json") |> send_resp(415, Jason.encode!(%{"error" => "mascots can only be images"})) end end diff --git a/test/fixtures/sound.mp3 b/test/fixtures/sound.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..9f0f661a3bc34b6366304e9fd1d4fe8513411083 GIT binary patch literal 521 zcmezWd%_V0bP$o5mkt!;2VzDB1}091|Fj1{y8?V1eO-<93=IrecEX$_s-VK;;K;!E q0OXGES_-dW5+jBF|6AY)1M>j}#w9=>D=;vaG%zr*zym6jY5)LcXO3V1 literal 0 HcmV?d00001 diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index cbff141c8..87e1c105d 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1455,6 +1455,71 @@ test "media upload", %{conn: conn} do assert object.data["actor"] == User.ap_id(user) end + test "mascot upload", %{conn: conn} do + user = insert(:user) + + non_image_file = %Plug.Upload{ + content_type: "audio/mpeg", + path: Path.absname("test/fixtures/sound.mp3"), + filename: "sound.mp3" + } + + conn = + conn + |> assign(:user, user) + |> put("/api/v1/pleroma/mascot", %{"file" => non_image_file}) + + assert json_response(conn, 415) + + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + conn = + build_conn() + |> assign(:user, user) + |> put("/api/v1/pleroma/mascot", %{"file" => file}) + + assert %{"id" => _, "type" => image} = json_response(conn, 200) + end + + test "mascot retrieving", %{conn: conn} do + user = insert(:user) + # When user hasn't set a mascot, we should just get pleroma tan back + conn = + conn + |> assign(:user, user) + |> get("/api/v1/pleroma/mascot") + + assert %{"url" => url} = json_response(conn, 200) + assert url =~ "pleroma-fox-tan-smol" + + # When a user sets their mascot, we should get that back + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + conn = + build_conn() + |> assign(:user, user) + |> put("/api/v1/pleroma/mascot", %{"file" => file}) + assert json_response(conn, 200) + + user = User.get_cached_by_id(user.id) + + conn = + build_conn() + |> assign(:user, user) + |> get("/api/v1/pleroma/mascot") + + assert %{"url" => url, "type" => "image"} = json_response(conn, 200) + assert url =~ "an_image" + end + test "hashtag timeline", %{conn: conn} do following = insert(:user) From dc916ba15f0fd77afa015084849f082065ed6f74 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 20 May 2019 12:58:17 +0100 Subject: [PATCH 064/123] Format mascot tests --- test/web/mastodon_api/mastodon_api_controller_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 87e1c105d..1d9f5a816 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1503,10 +1503,11 @@ test "mascot retrieving", %{conn: conn} do filename: "an_image.jpg" } - conn = + conn = build_conn() |> assign(:user, user) |> put("/api/v1/pleroma/mascot", %{"file" => file}) + assert json_response(conn, 200) user = User.get_cached_by_id(user.id) From 3d0d9e7a5680bcb1b79e5f4aab0e8c514fc9e5ff Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 20 May 2019 13:10:04 +0100 Subject: [PATCH 065/123] Use string map for default mascot --- lib/pleroma/user/info.ex | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index ffcd06e3e..e76d04d7f 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -46,12 +46,12 @@ defmodule Pleroma.User.Info do field(:mascot, :map, default: %{ - id: "pleromatan", - url: "/images/pleroma-fox-tan-smol.png", - type: "image", - preview_url: "/images/pleroma-fox-tan-smol.png", - pleroma: %{ - mime_type: "image/png" + "id" => "pleromatan", + "url" => "/images/pleroma-fox-tan-smol.png", + "type" => "image", + "preview_url" => "/images/pleroma-fox-tan-smol.png", + "pleroma" => %{ + "mime_type" => "image/png" } } ) From d835810610e5e7660a56d305bc8509e38c642942 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 20 May 2019 14:19:42 +0100 Subject: [PATCH 066/123] Add changelog entry for mascot config --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87101db30..07e6bce31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Configuration: `report_uri` option - Pleroma API: User subscriptions - Pleroma API: Healthcheck endpoint +- Pleroma API: `/api/v1/pleroma/mascot` per-user frontend mascot configuration endpoints - Admin API: Endpoints for listing/revoking invite tokens - Admin API: Endpoints for making users follow/unfollow each other - Admin API: added filters (role, tags, email, name) for users endpoint From daeae8e2e7c506b72c66dea6ac790408f948ec16 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 20 May 2019 16:12:55 +0100 Subject: [PATCH 067/123] Move default mascot configuration to `config/` --- config/config.exs | 13 ++++++++++++ docs/config.md | 10 ++++++++++ lib/pleroma/user.ex | 20 +++++++++++++++++++ lib/pleroma/user/info.ex | 14 +------------ .../mastodon_api/mastodon_api_controller.ex | 4 ++-- 5 files changed, 46 insertions(+), 15 deletions(-) diff --git a/config/config.exs b/config/config.exs index bab47a8a2..72908266d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -276,6 +276,19 @@ showInstanceSpecificPanel: true } +config :pleroma, :assets, + mascots: [ + pleroma_fox_tan: %{ + url: "/images/pleroma-fox-tan-smol.png", + mime_type: "image/png" + }, + pleroma_fox_tan_shy: %{ + url: "/images/pleroma-fox-tan-shy.png", + mime_type: "image/png" + } + ], + default_mascot: :pleroma_fox_tan + config :pleroma, :activitypub, accept_blocks: true, unfollow_blocked: true, diff --git a/docs/config.md b/docs/config.md index 450d73fda..197326bbd 100644 --- a/docs/config.md +++ b/docs/config.md @@ -203,6 +203,16 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i * `hide_post_stats`: Hide notices statistics(repeats, favorites, …) * `hide_user_stats`: Hide profile statistics(posts, posts per day, followers, followings, …) +## :assets + +This section configures assets to be used with various frontends. Currently the only option +relates to mascots on the mastodon frontend + +* `mascots`: KeywordList of mascots, each element __MUST__ contain both a `url` and a + `mime_type` key. +* `default_mascot`: An element from `mascots` - This will be used as the default mascot + on MastoFE (default: `:pleroma_fox_tan`) + ## :mrf_simple * `media_removal`: List of instances to remove medias from * `media_nsfw`: List of instances to put medias as NSFW(sensitive) from diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 28da310ee..05fe58f7c 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1402,4 +1402,24 @@ def toggle_confirmation(%User{} = user) do |> put_embed(:info, info_changeset) |> update_and_set_cache() end + + def get_mascot(%{info: %{mascot: %{} = mascot}}) when not is_nil(mascot) do + mascot + end + + def get_mascot(%{info: %{mascot: mascot}}) when is_nil(mascot) do + # use instance-default + config = Pleroma.Config.get([:assets, :mascots]) + default_mascot = Pleroma.Config.get([:assets, :default_mascot]) + mascot = Keyword.get(config, default_mascot) + + %{ + "id" => "default-mascot", + "url" => mascot[:url], + "preview_url" => mascot[:url], + "pleroma" => %{ + "mime_type" => mascot[:mime_type] + } + } + end end diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index e76d04d7f..6397e2737 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -43,19 +43,7 @@ defmodule Pleroma.User.Info do field(:hide_favorites, :boolean, default: true) field(:pinned_activities, {:array, :string}, default: []) field(:flavour, :string, default: nil) - - field(:mascot, :map, - default: %{ - "id" => "pleromatan", - "url" => "/images/pleroma-fox-tan-smol.png", - "type" => "image", - "preview_url" => "/images/pleroma-fox-tan-smol.png", - "pleroma" => %{ - "mime_type" => "image/png" - } - } - ) - + field(:mascot, :map, default: nil) field(:emoji, {:array, :map}, default: []) field(:notification_settings, :map, diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index d7f095a1f..1ec0f30a1 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -736,7 +736,7 @@ def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do end def get_mascot(%{assigns: %{user: user}} = conn, _params) do - %{info: %{mascot: mascot}} = user + mascot = User.get_mascot(user) conn |> json(mascot) @@ -1364,7 +1364,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do display_sensitive_media: false, reduce_motion: false, max_toot_chars: limit, - mascot: Map.get(user.info.mascot, "url", "/images/pleroma-fox-tan-smol.png") + mascot: User.get_mascot(user)["url"] }, rights: %{ delete_others_notice: present?(user.info.is_moderator), From eb02edcad9cb0d65fc216408960aec63713e5d2b Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Tue, 21 May 2019 00:35:46 +0800 Subject: [PATCH 068/123] Add virtual :thread_muted? field that may be set when fetching activities --- lib/pleroma/activity.ex | 12 +++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 8 ++++++ .../web/mastodon_api/views/status_view.ex | 8 +++++- .../web/twitter_api/views/activity_view.ex | 8 +++++- test/activity_test.exs | 26 +++++++++++++++++++ test/user_test.exs | 2 +- 6 files changed, 61 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 4e54b15ba..99589590c 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Activity do alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.ThreadMute alias Pleroma.User import Ecto.Changeset @@ -37,6 +38,7 @@ defmodule Pleroma.Activity do field(:local, :boolean, default: true) field(:actor, :string) field(:recipients, {:array, :string}, default: []) + field(:thread_muted?, :boolean, virtual: true) # This is a fake relation, do not use outside of with_preloaded_bookmark/get_bookmark has_one(:bookmark, Bookmark) has_many(:notifications, Notification, on_delete: :delete_all) @@ -90,6 +92,16 @@ def with_preloaded_bookmark(query, %User{} = user) do def with_preloaded_bookmark(query, _), do: query + def with_set_thread_muted_field(query, %User{} = user) do + from([a] in query, + left_join: tm in ThreadMute, + on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data), + select: %Activity{a | thread_muted?: not is_nil(tm.id)} + ) + end + + def with_set_thread_muted_field(query, _), do: query + def get_by_ap_id(ap_id) do Repo.one( from( diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 5c3156978..3d9679ec0 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -834,6 +834,13 @@ defp maybe_preload_bookmarks(query, opts) do |> Activity.with_preloaded_bookmark(opts["user"]) end + defp maybe_set_thread_muted_field(query, %{"skip_preload" => true}), do: query + + defp maybe_set_thread_muted_field(query, opts) do + query + |> Activity.with_set_thread_muted_field(opts["user"]) + end + defp maybe_order(query, %{order: :desc}) do query |> order_by(desc: :id) @@ -852,6 +859,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do base_query |> maybe_preload_objects(opts) |> maybe_preload_bookmarks(opts) + |> maybe_set_thread_muted_field(opts) |> maybe_order(opts) |> restrict_recipients(recipients, opts["user"]) |> restrict_tag(opts) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index c93d915e5..e55f9b96e 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -157,6 +157,12 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity bookmarked = Activity.get_bookmark(activity, opts[:for]) != nil + thread_muted? = + case activity.thread_muted? do + thread_muted? when is_boolean(thread_muted?) -> thread_muted? + nil -> CommonAPI.thread_muted?(user, activity) + end + attachment_data = object.data["attachment"] || [] attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment) @@ -228,7 +234,7 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity reblogged: reblogged?(activity, opts[:for]), favourited: present?(favorited), bookmarked: present?(bookmarked), - muted: CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user), + muted: thread_muted? || User.mutes?(opts[:for], user), pinned: pinned?(activity, user), sensitive: sensitive, spoiler_text: summary_html, diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index 44bcafe0e..e84af84dc 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -284,6 +284,12 @@ def render( Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) ) + thread_muted? = + case activity.thread_muted? do + thread_muted? when is_boolean(thread_muted?) -> thread_muted? + nil -> CommonAPI.thread_muted?(user, activity) + end + %{ "id" => activity.id, "uri" => object.data["id"], @@ -314,7 +320,7 @@ def render( "summary" => summary, "summary_html" => summary |> Formatter.emojify(object.data["emoji"]), "card" => card, - "muted" => CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user) + "muted" => thread_muted? || User.mutes?(opts[:for], user) } end diff --git a/test/activity_test.exs b/test/activity_test.exs index 7e91d534b..15c95502a 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.ActivityTest do use Pleroma.DataCase alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.ThreadMute import Pleroma.Factory test "returns an activity by it's AP id" do @@ -47,6 +48,31 @@ test "preloading a bookmark" do assert queried_activity.bookmark == bookmark3 end + test "setting thread_muted?" do + activity = insert(:note_activity) + user = insert(:user) + annoyed_user = insert(:user) + {:ok, _} = ThreadMute.add_mute(annoyed_user.id, activity.data["context"]) + + activity_with_unset_thread_muted_field = + Ecto.Query.from(Activity) + |> Repo.one() + + activity_for_user = + Ecto.Query.from(Activity) + |> Activity.with_set_thread_muted_field(user) + |> Repo.one() + + activity_for_annoyed_user = + Ecto.Query.from(Activity) + |> Activity.with_set_thread_muted_field(annoyed_user) + |> Repo.one() + + assert activity_with_unset_thread_muted_field.thread_muted? == nil + assert activity_for_user.thread_muted? == false + assert activity_for_annoyed_user.thread_muted? == true + end + describe "getting a bookmark" do test "when association is loaded" do user = insert(:user) diff --git a/test/user_test.exs b/test/user_test.exs index 10e463ff8..cb6afbe07 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -902,7 +902,7 @@ test "hide a user's statuses from timelines and notifications" do assert [activity] == ActivityPub.fetch_public_activities(%{}) |> Repo.preload(:bookmark) - assert [activity] == + assert [%{activity | thread_muted?: CommonAPI.thread_muted?(user2, activity)}] == ActivityPub.fetch_activities([user2.ap_id | user2.following], %{"user" => user2}) {:ok, _user} = User.deactivate(user) From 5aa107d9126c484f0ae06eddec6995b535623b32 Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Tue, 21 May 2019 00:59:12 +0800 Subject: [PATCH 069/123] Document MRF.Simple :report_removal --- docs/config/mrf.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/config/mrf.md b/docs/config/mrf.md index 2cc16cef0..45be18fc5 100644 --- a/docs/config/mrf.md +++ b/docs/config/mrf.md @@ -5,11 +5,12 @@ Possible uses include: * marking incoming messages with media from a given account or instance as sensitive * rejecting messages from a specific instance +* rejecting reports (flags) from a specific instance * removing/unlisting messages from the public timelines * removing media from messages * sending only public messages to a specific instance -The MRF provides user-configurable policies. The default policy is `NoOpPolicy`, which disables the MRF functionality. Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module. +The MRF provides user-configurable policies. The default policy is `NoOpPolicy`, which disables the MRF functionality. Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module. It is possible to use multiple, active MRF policies at the same time. ## Quarantine Instances @@ -41,12 +42,13 @@ Once `SimplePolicy` is enabled, you can configure various groups in the `:mrf_si * `media_nsfw`: Servers in this group will have the #nsfw tag and sensitive setting injected into incoming messages which contain media. * `reject`: Servers in this group will have their messages rejected. * `federated_timeline_removal`: Servers in this group will have their messages unlisted from the public timelines by flipping the `to` and `cc` fields. +* `report_removal`: Servers in this group will have their reports (flags) rejected. Servers should be configured as lists. ### Example -This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com` and remove messages from `spam.university` from the federated timeline: +This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com`, remove messages from `spam.university` from the federated timeline and block reports (flags) from `whiny.whiner`: ``` config :pleroma, :instance, @@ -56,7 +58,8 @@ config :pleroma, :mrf_simple, media_removal: ["illegalporn.biz"], media_nsfw: ["porn.biz", "porn.business"], reject: ["spam.com"], - federated_timeline_removal: ["spam.university"] + federated_timeline_removal: ["spam.university"], + report_removal: ["whiny.whiner"] ``` From 75c7bb9289066bfaebe7d96aaa474d34ec4d5a5f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 20 May 2019 17:18:59 -0500 Subject: [PATCH 070/123] Additional reserved usernames --- config/config.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/config.exs b/config/config.exs index 61e2648a9..387c1a5a7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -369,6 +369,7 @@ "activities", "api", "auth", + "check_password", "dev", "friend-requests", "inbox", @@ -389,6 +390,7 @@ "status", "tag", "user-search", + "user_exists", "users", "web" ] From f96e9b28bb5ee241a3f0ca6a622b925ca560c141 Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Tue, 21 May 2019 07:30:18 +0800 Subject: [PATCH 071/123] Fix prometheus-ecto error when not configured --- lib/pleroma/application.ex | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index eeb415084..dab45a0b2 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -131,19 +131,22 @@ def start(_type, _args) do defp setup_instrumenters do require Prometheus.Registry - :ok = - :telemetry.attach( - "prometheus-ecto", - [:pleroma, :repo, :query], - &Pleroma.Repo.Instrumenter.handle_event/4, - %{} - ) + if Application.get_env(:prometheus, Pleroma.Repo.Instrumenter) do + :ok = + :telemetry.attach( + "prometheus-ecto", + [:pleroma, :repo, :query], + &Pleroma.Repo.Instrumenter.handle_event/4, + %{} + ) + + Pleroma.Repo.Instrumenter.setup() + end Prometheus.Registry.register_collector(:prometheus_process_collector) Pleroma.Web.Endpoint.MetricsExporter.setup() Pleroma.Web.Endpoint.PipelineInstrumenter.setup() Pleroma.Web.Endpoint.Instrumenter.setup() - Pleroma.Repo.Instrumenter.setup() end def enabled_hackney_pools do From c972d0bb14ee5a65f053b8c9629d93fc9b94ca78 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 21 May 2019 04:58:26 +0000 Subject: [PATCH 072/123] http: bump connection timeout to 10 seconds --- CHANGELOG.md | 1 + lib/pleroma/http/connection.ex | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07e6bce31..256df91b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Don't ship finmoji by default, they can be installed as an emoji pack - Hide deactivated users and their statuses - Posts which are marked sensitive or tagged nsfw no longer have link previews. +- HTTP connection timeout is now set to 10 seconds. ### Fixed - Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended. diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index c0173465a..558005c19 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -8,7 +8,7 @@ defmodule Pleroma.HTTP.Connection do """ @hackney_options [ - connect_timeout: 2_000, + connect_timeout: 10_000, recv_timeout: 20_000, follow_redirect: true, pool: :federation From d378b342ba0d7f54be3b47f031c602a578191dfd Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 21 May 2019 18:57:36 +0200 Subject: [PATCH 073/123] MongooseIM: Add documentation. --- CHANGELOG.md | 1 + docs/config/howto_mongooseim.md | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 docs/config/howto_mongooseim.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 12c439135..b50ca895c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Added +- [MongooseIM](https://github.com/esl/MongooseIM) http authentication support. - LDAP authentication - External OAuth provider authentication - A [job queue](https://git.pleroma.social/pleroma/pleroma_job_queue) for federation, emails, web push, etc. diff --git a/docs/config/howto_mongooseim.md b/docs/config/howto_mongooseim.md new file mode 100644 index 000000000..a33e590a1 --- /dev/null +++ b/docs/config/howto_mongooseim.md @@ -0,0 +1,10 @@ +# Configuring MongooseIM (XMPP Server) to use Pleroma for authentication + +If you want to give your Pleroma users an XMPP (chat) account, you can configure [MongooseIM](https://github.com/esl/MongooseIM) to use your Pleroma server for user authentication, automatically giving every local user an XMPP account. + +In general, you just have to follow the configuration described at [https://mongooseim.readthedocs.io/en/latest/authentication-backends/HTTP-authentication-module/](https://mongooseim.readthedocs.io/en/latest/authentication-backends/HTTP-authentication-module/) and do these changes to your mongooseim.cfg. + +1. Set the auth_method to `{auth_method, http}`. +2. Add the http auth pool like this: `{http, global, auth, [{workers, 50}], [{server, "https://yourpleromainstance.com"}]}` + +Restart your MongooseIM server, your users should now be able to connect with their Pleroma credentials. From c2b0b82e6a6d40f945feacc001ad984a17e23336 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 21 May 2019 00:41:40 +0000 Subject: [PATCH 074/123] object: add Object.prune() --- lib/pleroma/object.ex | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 740d687a3..cc6fc9c5d 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -130,6 +130,13 @@ def delete(%Object{data: %{"id" => id}} = object) do end end + def prune(%Object{data: %{"id" => id}} = object) do + with {:ok, object} <- Repo.delete(object), + {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do + {:ok, object} + end + end + def set_cache(%Object{data: %{"id" => ap_id}} = object) do Cachex.put(:object_cache, "object:#{ap_id}", object) {:ok, object} From 73df9d690d5c1a9c11f0f04b8d877c0677022591 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 21 May 2019 00:41:58 +0000 Subject: [PATCH 075/123] object: fetcher: add support for reinjecting pruned objects --- lib/pleroma/object/fetcher.ex | 22 ++++++++++++++++++++-- test/object/fetcher_test.exs | 19 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 8d4bcc95e..bb9388d4f 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -8,6 +8,19 @@ defmodule Pleroma.Object.Fetcher do @httpoison Application.get_env(:pleroma, :httpoison) + defp reinject_object(data) do + Logger.debug("Reinjecting object #{data["id"]}") + + with data <- Transmogrifier.fix_object(data), + {:ok, object} <- Object.create(data) do + {:ok, object} + else + e -> + Logger.error("Error while processing object: #{inspect(e)}") + {:error, e} + end + end + # TODO: # This will create a Create activity, which we need internally at the moment. def fetch_object_from_id(id) do @@ -26,12 +39,17 @@ def fetch_object_from_id(id) do "object" => data }, :ok <- Containment.contain_origin(id, params), - {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, Object.normalize(activity, false)} + {:ok, activity} <- Transmogrifier.handle_incoming(params), + {:object, _data, %Object{} = object} <- + {:object, data, Object.normalize(activity, false)} do + {:ok, object} else {:error, {:reject, nil}} -> {:reject, nil} + {:object, data, nil} -> + reinject_object(data) + object = %Object{} -> {:ok, object} diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs index 72f616782..d604fd5f5 100644 --- a/test/object/fetcher_test.exs +++ b/test/object/fetcher_test.exs @@ -87,4 +87,23 @@ test "all objects with fake directions are rejected by the object fetcher" do ) end end + + describe "pruning" do + test "it can refetch pruned objects" do + object_id = "http://mastodon.example.org/@admin/99541947525187367" + + {:ok, object} = Fetcher.fetch_object_from_id(object_id) + + assert object + + {:ok, _object} = Object.prune(object) + + refute Object.get_by_ap_id(object_id) + + {:ok, %Object{} = object_two} = Fetcher.fetch_object_from_id(object_id) + + assert object.data["id"] == object_two.data["id"] + assert object.id != object_two.id + end + end end From 16b260fb19cca02463766c2e36a41bfcc823af9b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 21 May 2019 01:21:28 +0000 Subject: [PATCH 076/123] add mix task to prune the object database using a configured retention period --- config/config.exs | 3 ++- docs/config.md | 1 + lib/mix/tasks/pleroma/database.ex | 40 +++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 72908266d..466a6e9b7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -239,7 +239,8 @@ welcome_message: nil, max_report_comment_size: 1000, safe_dm_mentions: false, - healthcheck: false + healthcheck: false, + remote_post_retention_days: 90 config :pleroma, :app_account_creation, enabled: true, max_requests: 25, interval: 1800 diff --git a/docs/config.md b/docs/config.md index 197326bbd..a050068f4 100644 --- a/docs/config.md +++ b/docs/config.md @@ -104,6 +104,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`) * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`) * `healthcheck`: if set to true, system data will be shown on ``/api/pleroma/healthcheck``. +* `remote_post_retention_days`: the default amount of days to retain remote posts when pruning the database ## :app_account_creation REST API for creating an account settings diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index f650b447d..fdb216037 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -23,6 +23,10 @@ defmodule Mix.Tasks.Pleroma.Database do Options: - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references + ## Prune old objects from the database + + mix pleroma.database prune_objects + ## Create a conversation for all existing DMs. Can be safely re-run. mix pleroma.database bump_all_conversations @@ -72,4 +76,40 @@ def run(["update_users_following_followers_counts"]) do Enum.each(users, &User.remove_duplicated_following/1) Enum.each(users, &User.update_follower_count/1) end + + def run(["prune_objects" | args]) do + {options, [], []} = + OptionParser.parse( + args, + strict: [ + vacuum: :boolean + ] + ) + + Common.start_pleroma() + + deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + + Logger.info("Pruning objects older than #{deadline} days") + + time_deadline = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(-(deadline * 86_400)) + + Repo.query!( + "DELETE FROM objects WHERE inserted_at < $1 AND split_part(data->>'actor', '/', 3) != $2", + [time_deadline, Pleroma.Web.Endpoint.host()], + timeout: :infinity + ) + + if Keyword.get(options, :vacuum) do + Logger.info("Runnning VACUUM FULL") + + Repo.query!( + "vacuum full;", + [], + timeout: :infinity + ) + end + end end From f446f94290a6cdae531859c2efa55f55dbaeb7e8 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 21 May 2019 01:22:27 +0000 Subject: [PATCH 077/123] add changelog entry for object pruning --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 256df91b7..a21c4bff2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Metadata: RelMe provider - OAuth: added support for refresh tokens - Emoji packs and emoji pack manager +- Object pruning (`mix pleroma.database prune_objects`) ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer From a13d449b241b6e5fa14fb741694e63cd21569b2c Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Tue, 21 May 2019 09:39:32 +0800 Subject: [PATCH 078/123] Add tests for fallback routes --- test/web/fallback_test.exs | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 test/web/fallback_test.exs diff --git a/test/web/fallback_test.exs b/test/web/fallback_test.exs new file mode 100644 index 000000000..514923a20 --- /dev/null +++ b/test/web/fallback_test.exs @@ -0,0 +1,46 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.FallbackTest do + use Pleroma.Web.ConnCase + import Pleroma.Factory + + test "GET /registration/:token", %{conn: conn} do + assert conn + |> get("/registration/foo") + |> html_response(200) =~ "" + end + + test "GET /:maybe_nickname_or_id", %{conn: conn} do + user = insert(:user) + + assert conn + |> get("/foo") + |> html_response(200) =~ "" + + refute conn + |> get("/" <> user.nickname) + |> html_response(200) =~ "" + end + + test "GET /*path", %{conn: conn} do + assert conn + |> get("/foo") + |> html_response(200) =~ "" + + assert conn + |> get("/foo/bar") + |> html_response(200) =~ "" + end + + test "OPTIONS /*path", %{conn: conn} do + assert conn + |> options("/foo") + |> response(204) == "" + + assert conn + |> options("/foo/bar") + |> response(204) == "" + end +end From 3ab9255eda21a5f8a25047375af9608e0c0c7592 Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Tue, 21 May 2019 09:40:29 +0800 Subject: [PATCH 079/123] Respond with a 404 Not implemented JSON error message when requested API is not implemented --- CHANGELOG.md | 1 + lib/pleroma/web/router.ex | 7 +++++++ test/web/fallback_test.exs | 6 ++++++ 3 files changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 256df91b7..2ed380102 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Hide deactivated users and their statuses - Posts which are marked sensitive or tagged nsfw no longer have link previews. - HTTP connection timeout is now set to 10 seconds. +- Respond with a 404 Not implemented JSON error message when requested API is not implemented ### Fixed - Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended. diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 4c29b24eb..49e28cc2d 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -710,6 +710,7 @@ defmodule Pleroma.Web.Router do scope "/", Fallback do get("/registration/:token", RedirectController, :registration_page) get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta) + get("/api*path", RedirectController, :api_not_implemented) get("/*path", RedirectController, :redirector) options("/*path", RedirectController, :empty) @@ -721,6 +722,12 @@ defmodule Fallback.RedirectController do alias Pleroma.User alias Pleroma.Web.Metadata + def api_not_implemented(conn, _params) do + conn + |> put_status(404) + |> json(%{error: "Not implemented"}) + end + def redirector(conn, _params, code \\ 200) do conn |> put_resp_content_type("text/html") diff --git a/test/web/fallback_test.exs b/test/web/fallback_test.exs index 514923a20..cc78b3ae1 100644 --- a/test/web/fallback_test.exs +++ b/test/web/fallback_test.exs @@ -24,6 +24,12 @@ test "GET /:maybe_nickname_or_id", %{conn: conn} do |> html_response(200) =~ "" end + test "GET /api*path", %{conn: conn} do + assert conn + |> get("/api/foo") + |> json_response(404) == %{"error" => "Not implemented"} + end + test "GET /*path", %{conn: conn} do assert conn |> get("/foo") From f76268135c014c20a482d30a7c9596ec2e7d6a69 Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Wed, 22 May 2019 07:11:09 +0800 Subject: [PATCH 080/123] Fix failing test --- test/web/admin_api/admin_api_controller_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index ca12c7215..c15c67e31 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -397,14 +397,14 @@ test "it returns 500 if `registrations_open` is enabled", %{conn: conn, user: us end end - test "/api/pleroma/admin/invite_token" do + test "/api/pleroma/admin/users/invite_token" do admin = insert(:user, info: %{is_admin: true}) conn = build_conn() |> assign(:user, admin) |> put_req_header("accept", "application/json") - |> get("/api/pleroma/admin/invite_token") + |> get("/api/pleroma/admin/users/invite_token") assert conn.status == 200 end From a023ca004cbd90e330cab35e4dfda16346d08668 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 03:12:48 +0000 Subject: [PATCH 081/123] prune objects task: use Repo.delete_all() --- lib/mix/tasks/pleroma/database.ex | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index fdb216037..f9bafb277 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -5,6 +5,7 @@ defmodule Mix.Tasks.Pleroma.Database do alias Mix.Tasks.Pleroma.Common alias Pleroma.Conversation + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User require Logger @@ -78,6 +79,8 @@ def run(["update_users_following_followers_counts"]) do end def run(["prune_objects" | args]) do + import Ecto.Query + {options, [], []} = OptionParser.parse( args, @@ -96,11 +99,15 @@ def run(["prune_objects" | args]) do NaiveDateTime.utc_now() |> NaiveDateTime.add(-(deadline * 86_400)) - Repo.query!( - "DELETE FROM objects WHERE inserted_at < $1 AND split_part(data->>'actor', '/', 3) != $2", - [time_deadline, Pleroma.Web.Endpoint.host()], - timeout: :infinity + public = "https://www.w3.org/ns/activitystreams#Public" + + from(o in Object, + where: fragment("?->'to' \\? ? OR ?->'cc' \\? ?", o.data, ^public, o.data, ^public), + where: o.inserted_at < ^time_deadline, + where: + fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host()) ) + |> Repo.delete_all() if Keyword.get(options, :vacuum) do Logger.info("Runnning VACUUM FULL") From 045803346d70c1f9c6ea770485904fd7cc52969a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 03:58:15 +0000 Subject: [PATCH 082/123] move key generation functions into Pleroma.Keys module --- lib/pleroma/keys.ex | 44 +++++++++++++++++++ lib/pleroma/signature.ex | 7 ++- lib/pleroma/user.ex | 21 +++++++++ .../activity_pub/activity_pub_controller.ex | 14 +++--- .../web/activity_pub/views/user_view.ex | 11 +++-- lib/pleroma/web/federator/federator.ex | 6 +-- lib/pleroma/web/salmon/salmon.ex | 44 ++----------------- lib/pleroma/web/web_finger/web_finger.ex | 26 +---------- test/keys_test.exs | 20 +++++++++ test/user_test.exs | 15 +++++++ test/web/activity_pub/activity_pub_test.exs | 2 +- .../web/activity_pub/views/user_view_test.exs | 13 +++--- test/web/salmon/salmon_test.exs | 19 ++------ test/web/web_finger/web_finger_test.exs | 15 ------- 14 files changed, 133 insertions(+), 124 deletions(-) create mode 100644 lib/pleroma/keys.ex create mode 100644 test/keys_test.exs diff --git a/lib/pleroma/keys.ex b/lib/pleroma/keys.ex new file mode 100644 index 000000000..b7bc7a4da --- /dev/null +++ b/lib/pleroma/keys.ex @@ -0,0 +1,44 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Keys do + # Native generation of RSA keys is only available since OTP 20+ and in default build conditions + # We try at compile time to generate natively an RSA key otherwise we fallback on the old way. + try do + _ = :public_key.generate_key({:rsa, 2048, 65_537}) + + def generate_rsa_pem do + key = :public_key.generate_key({:rsa, 2048, 65_537}) + entry = :public_key.pem_entry_encode(:RSAPrivateKey, key) + pem = :public_key.pem_encode([entry]) |> String.trim_trailing() + {:ok, pem} + end + rescue + _ -> + def generate_rsa_pem do + port = Port.open({:spawn, "openssl genrsa"}, [:binary]) + + {:ok, pem} = + receive do + {^port, {:data, pem}} -> {:ok, pem} + end + + Port.close(port) + + if Regex.match?(~r/RSA PRIVATE KEY/, pem) do + {:ok, pem} + else + :error + end + end + end + + def keys_from_pem(pem) do + [private_key_code] = :public_key.pem_decode(pem) + private_key = :public_key.pem_entry_decode(private_key_code) + {:RSAPrivateKey, _, modulus, exponent, _, _, _, _, _, _, _} = private_key + public_key = {:RSAPublicKey, modulus, exponent} + {:ok, private_key, public_key} + end +end diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index b7ecf00a0..1a4d54c62 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -5,11 +5,10 @@ defmodule Pleroma.Signature do @behaviour HTTPSignatures.Adapter + alias Pleroma.Keys alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Web.Salmon - alias Pleroma.Web.WebFinger def fetch_public_key(conn) do with actor_id <- Utils.get_ap_id(conn.params["actor"]), @@ -33,8 +32,8 @@ def refetch_public_key(conn) do end def sign(%User{} = user, headers) do - with {:ok, %{info: %{keys: keys}}} <- WebFinger.ensure_keys_present(user), - {:ok, private_key, _} <- Salmon.keys_from_pem(keys) do + with {:ok, %{info: %{keys: keys}}} <- User.ensure_keys_present(user), + {:ok, private_key, _} <- Keys.keys_from_pem(keys) do HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers) end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 05fe58f7c..653dec95f 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -10,6 +10,7 @@ defmodule Pleroma.User do alias Comeonin.Pbkdf2 alias Pleroma.Activity + alias Pleroma.Keys alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Registration @@ -1422,4 +1423,24 @@ def get_mascot(%{info: %{mascot: mascot}}) when is_nil(mascot) do } } end + + def ensure_keys_present(user) do + info = user.info + + if info.keys do + {:ok, user} + else + {:ok, pem} = Keys.generate_rsa_pem() + + info_cng = + info + |> User.Info.set_keys(pem) + + cng = + Ecto.Changeset.change(user) + |> Ecto.Changeset.put_embed(:info, info_cng) + + update_and_set_cache(cng) + end + end end diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index c967ab7a9..ad2ca1e54 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -39,7 +39,7 @@ def relay_active?(conn, _) do def user(conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_header("content-type", "application/activity+json") |> json(UserView.render("user.json", %{user: user})) @@ -106,7 +106,7 @@ def activity(conn, %{"uuid" => uuid}) do def following(conn, %{"nickname" => nickname, "page" => page}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do {page, _} = Integer.parse(page) conn @@ -117,7 +117,7 @@ def following(conn, %{"nickname" => nickname, "page" => page}) do def following(conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_header("content-type", "application/activity+json") |> json(UserView.render("following.json", %{user: user})) @@ -126,7 +126,7 @@ def following(conn, %{"nickname" => nickname}) do def followers(conn, %{"nickname" => nickname, "page" => page}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do {page, _} = Integer.parse(page) conn @@ -137,7 +137,7 @@ def followers(conn, %{"nickname" => nickname, "page" => page}) do def followers(conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_header("content-type", "application/activity+json") |> json(UserView.render("followers.json", %{user: user})) @@ -146,7 +146,7 @@ def followers(conn, %{"nickname" => nickname}) do def outbox(conn, %{"nickname" => nickname} = params) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_header("content-type", "application/activity+json") |> json(UserView.render("outbox.json", %{user: user, max_id: params["max_id"]})) @@ -195,7 +195,7 @@ def inbox(conn, params) do def relay(conn, _params) do with %User{} = user <- Relay.get_actor(), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_header("content-type", "application/activity+json") |> json(UserView.render("user.json", %{user: user})) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 1254fdf6c..327e0e05b 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do use Pleroma.Web, :view + alias Pleroma.Keys alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -12,8 +13,6 @@ defmodule Pleroma.Web.ActivityPub.UserView do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Endpoint alias Pleroma.Web.Router.Helpers - alias Pleroma.Web.Salmon - alias Pleroma.Web.WebFinger import Ecto.Query @@ -34,8 +33,8 @@ def render("endpoints.json", _), do: %{} # the instance itself is not a Person, but instead an Application def render("user.json", %{user: %{nickname: nil} = user}) do - {:ok, user} = WebFinger.ensure_keys_present(user) - {:ok, _, public_key} = Salmon.keys_from_pem(user.info.keys) + {:ok, user} = User.ensure_keys_present(user) + {:ok, _, public_key} = Keys.keys_from_pem(user.info.keys) public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key) public_key = :public_key.pem_encode([public_key]) @@ -62,8 +61,8 @@ def render("user.json", %{user: %{nickname: nil} = user}) do end def render("user.json", %{user: user}) do - {:ok, user} = WebFinger.ensure_keys_present(user) - {:ok, _, public_key} = Salmon.keys_from_pem(user.info.keys) + {:ok, user} = User.ensure_keys_present(user) + {:ok, _, public_key} = Keys.keys_from_pem(user.info.keys) public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key) public_key = :public_key.pem_encode([public_key]) diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 169fdf4dc..6b0b75284 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -11,7 +11,6 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Federator.Publisher alias Pleroma.Web.Federator.RetryQueue - alias Pleroma.Web.WebFinger alias Pleroma.Web.Websub require Logger @@ -77,9 +76,8 @@ def perform(:request_subscription, websub) do def perform(:publish, activity) do Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end) - with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do - {:ok, actor} = WebFinger.ensure_keys_present(actor) - + with %User{} = actor <- User.get_cached_by_ap_id(activity.data["actor"]), + {:ok, actor} <- User.ensure_keys_present(actor) do Publisher.publish(actor, activity) end end diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 42709ab47..fa30f73cd 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.Salmon do use Bitwise alias Pleroma.Activity + alias Pleroma.Keys alias Pleroma.Instances alias Pleroma.User alias Pleroma.Web.ActivityPub.Visibility @@ -89,45 +90,6 @@ def encode_key({:RSAPublicKey, modulus, exponent}) do "RSA.#{modulus_enc}.#{exponent_enc}" end - # Native generation of RSA keys is only available since OTP 20+ and in default build conditions - # We try at compile time to generate natively an RSA key otherwise we fallback on the old way. - try do - _ = :public_key.generate_key({:rsa, 2048, 65_537}) - - def generate_rsa_pem do - key = :public_key.generate_key({:rsa, 2048, 65_537}) - entry = :public_key.pem_entry_encode(:RSAPrivateKey, key) - pem = :public_key.pem_encode([entry]) |> String.trim_trailing() - {:ok, pem} - end - rescue - _ -> - def generate_rsa_pem do - port = Port.open({:spawn, "openssl genrsa"}, [:binary]) - - {:ok, pem} = - receive do - {^port, {:data, pem}} -> {:ok, pem} - end - - Port.close(port) - - if Regex.match?(~r/RSA PRIVATE KEY/, pem) do - {:ok, pem} - else - :error - end - end - end - - def keys_from_pem(pem) do - [private_key_code] = :public_key.pem_decode(pem) - private_key = :public_key.pem_entry_decode(private_key_code) - {:RSAPrivateKey, _, modulus, exponent, _, _, _, _, _, _, _} = private_key - public_key = {:RSAPublicKey, modulus, exponent} - {:ok, private_key, public_key} - end - def encode(private_key, doc) do type = "application/atom+xml" encoding = "base64url" @@ -227,7 +189,7 @@ def publish(%{info: %{keys: keys}} = user, %{data: %{"type" => type}} = activity |> :xmerl.export_simple(:xmerl_xml) |> to_string - {:ok, private, _} = keys_from_pem(keys) + {:ok, private, _} = Keys.keys_from_pem(keys) {:ok, feed} = encode(private, feed) remote_users = remote_users(activity) @@ -253,7 +215,7 @@ def publish(%{info: %{keys: keys}} = user, %{data: %{"type" => type}} = activity def publish(%{id: id}, _), do: Logger.debug(fn -> "Keys missing for user #{id}" end) def gather_webfinger_links(%User{} = user) do - {:ok, _private, public} = keys_from_pem(user.info.keys) + {:ok, _private, public} = Keys.keys_from_pem(user.info.keys) magic_key = encode_key(public) [ diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 1239b962a..c5b7d4acb 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.WebFinger do alias Pleroma.User alias Pleroma.Web alias Pleroma.Web.Federator.Publisher - alias Pleroma.Web.Salmon alias Pleroma.Web.XML alias Pleroma.XmlBuilder require Jason @@ -61,7 +60,7 @@ defp gather_links(%User{} = user) do end def represent_user(user, "JSON") do - {:ok, user} = ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) %{ "subject" => "acct:#{user.nickname}@#{Pleroma.Web.Endpoint.host()}", @@ -71,7 +70,7 @@ def represent_user(user, "JSON") do end def represent_user(user, "XML") do - {:ok, user} = ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) links = gather_links(user) @@ -88,27 +87,6 @@ def represent_user(user, "XML") do |> XmlBuilder.to_doc() end - # This seems a better fit in Salmon - def ensure_keys_present(user) do - info = user.info - - if info.keys do - {:ok, user} - else - {:ok, pem} = Salmon.generate_rsa_pem() - - info_cng = - info - |> User.Info.set_keys(pem) - - cng = - Ecto.Changeset.change(user) - |> Ecto.Changeset.put_embed(:info, info_cng) - - User.update_and_set_cache(cng) - end - end - defp get_magic_key(magic_key) do "data:application/magic-public-key," <> magic_key = magic_key {:ok, magic_key} diff --git a/test/keys_test.exs b/test/keys_test.exs new file mode 100644 index 000000000..776fdea6f --- /dev/null +++ b/test/keys_test.exs @@ -0,0 +1,20 @@ +defmodule Pleroma.KeysTest do + use Pleroma.DataCase + + alias Pleroma.Keys + + test "generates an RSA private key pem" do + {:ok, key} = Keys.generate_rsa_pem() + + assert is_binary(key) + assert Regex.match?(~r/RSA/, key) + end + + test "returns a public and private key from a pem" do + pem = File.read!("test/fixtures/private_key.pem") + {:ok, private, public} = Keys.keys_from_pem(pem) + + assert elem(private, 0) == :RSAPrivateKey + assert elem(public, 0) == :RSAPublicKey + end +end diff --git a/test/user_test.exs b/test/user_test.exs index cb6afbe07..019f2b56d 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1251,4 +1251,19 @@ test "if user is unconfirmed" do refute user.info.confirmation_token end end + + describe "ensure_keys_present" do + test "it creates keys for a user and stores them in info" do + user = insert(:user) + refute is_binary(user.info.keys) + {:ok, user} = User.ensure_keys_present(user) + assert is_binary(user.info.keys) + end + + test "it doesn't create keys if there already are some" do + user = insert(:user, %{info: %{keys: "xxx"}}) + {:ok, user} = User.ensure_keys_present(user) + assert user.info.keys == "xxx" + end + end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index c18e0ab5f..f743f380b 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -1005,7 +1005,7 @@ test "it filters broken threads" do describe "update" do test "it creates an update activity with the new user data" do user = insert(:user) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) user_data = Pleroma.Web.ActivityPub.UserView.render("user.json", %{user: user}) {:ok, update} = diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 9fb9455d2..e6483db8b 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -2,11 +2,12 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do use Pleroma.DataCase import Pleroma.Factory + alias Pleroma.User alias Pleroma.Web.ActivityPub.UserView test "Renders a user, including the public key" do user = insert(:user) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) @@ -18,7 +19,7 @@ test "Renders a user, including the public key" do test "Does not add an avatar image if the user hasn't set one" do user = insert(:user) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) refute result["icon"] @@ -32,7 +33,7 @@ test "Does not add an avatar image if the user hasn't set one" do } ) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) assert result["icon"]["url"] == "https://someurl" @@ -42,7 +43,7 @@ test "Does not add an avatar image if the user hasn't set one" do describe "endpoints" do test "local users have a usable endpoints structure" do user = insert(:user) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) @@ -58,7 +59,7 @@ test "local users have a usable endpoints structure" do test "remote users have an empty endpoints structure" do user = insert(:user, local: false) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) @@ -68,7 +69,7 @@ test "remote users have an empty endpoints structure" do test "instance users do not expose oAuth endpoints" do user = insert(:user, nickname: nil, local: true) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs index 232082779..e86e76fe9 100644 --- a/test/web/salmon/salmon_test.exs +++ b/test/web/salmon/salmon_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.Web.Salmon.SalmonTest do use Pleroma.DataCase alias Pleroma.Activity + alias Pleroma.Keys alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.Federator.Publisher @@ -34,12 +35,6 @@ test "errors on wrong magic key" do assert Salmon.decode_and_validate(@wrong_magickey, salmon) == :error end - test "generates an RSA private key pem" do - {:ok, key} = Salmon.generate_rsa_pem() - assert is_binary(key) - assert Regex.match?(~r/RSA/, key) - end - test "it encodes a magic key from a public key" do key = Salmon.decode_key(@magickey) magic_key = Salmon.encode_key(key) @@ -51,18 +46,10 @@ test "it decodes a friendica public key" do _key = Salmon.decode_key(@magickey_friendica) end - test "returns a public and private key from a pem" do - pem = File.read!("test/fixtures/private_key.pem") - {:ok, private, public} = Salmon.keys_from_pem(pem) - - assert elem(private, 0) == :RSAPrivateKey - assert elem(public, 0) == :RSAPublicKey - end - test "encodes an xml payload with a private key" do doc = File.read!("test/fixtures/incoming_note_activity.xml") pem = File.read!("test/fixtures/private_key.pem") - {:ok, private, public} = Salmon.keys_from_pem(pem) + {:ok, private, public} = Keys.keys_from_pem(pem) # Let's try a roundtrip. {:ok, salmon} = Salmon.encode(private, doc) @@ -105,7 +92,7 @@ test "it gets a magic key" do {:ok, activity} = Repo.insert(%Activity{data: activity_data, recipients: activity_data["to"]}) user = User.get_cached_by_ap_id(activity.data["actor"]) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) Salmon.publish(user, activity) diff --git a/test/web/web_finger/web_finger_test.exs b/test/web/web_finger/web_finger_test.exs index 6b20d8d56..335c95b18 100644 --- a/test/web/web_finger/web_finger_test.exs +++ b/test/web/web_finger/web_finger_test.exs @@ -105,19 +105,4 @@ test "it gets the xrd endpoint for statusnet" do assert template == "http://status.alpicola.com/main/xrd?uri={uri}" end end - - describe "ensure_keys_present" do - test "it creates keys for a user and stores them in info" do - user = insert(:user) - refute is_binary(user.info.keys) - {:ok, user} = WebFinger.ensure_keys_present(user) - assert is_binary(user.info.keys) - end - - test "it doesn't create keys if there already are some" do - user = insert(:user, %{info: %{keys: "xxx"}}) - {:ok, user} = WebFinger.ensure_keys_present(user) - assert user.info.keys == "xxx" - end - end end From 913484817076bf5ca4bdbe7c3c1ff34f7debd3e5 Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Wed, 22 May 2019 04:04:20 +0000 Subject: [PATCH 083/123] Do not truncate DM when it contains newlines and safe_dm_mentions is set to true --- lib/pleroma/formatter.ex | 2 +- test/formatter_test.exs | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 3d7c36d21..3e3b9fe97 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Formatter do alias Pleroma.User alias Pleroma.Web.MediaProxy - @safe_mention_regex ~r/^(\s*(?@.+?\s+)+)(?.*)/ + @safe_mention_regex ~r/^(\s*(?@.+?\s+)+)(?.*)/s @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/ diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 5e7011160..47b91b121 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -206,6 +206,15 @@ test "given the 'safe_mention' option, it will still work without any mention" d assert mentions == [] assert expected_text == text end + + test "given the 'safe_mention' option, it will keep text after newlines" do + user = insert(:user) + text = " @#{user.nickname}\n hey dude\n\nhow are you doing?" + + {expected_text, _, _} = Formatter.linkify(text, safe_mention: true) + + assert expected_text =~ "how are you doing?" + end end describe ".parse_tags" do From 0e2c215a006c7ca5756e80a357ff6395a4325946 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 22 May 2019 07:22:19 +0200 Subject: [PATCH 084/123] MastoAPI AccountView: fill source.note with plaintext version of note Closes: https://git.pleroma.social/pleroma/pleroma/issues/926 --- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- test/web/mastodon_api/account_view_test.exs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 134c07b7e..b82d3319b 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -112,7 +112,7 @@ defp do_render("account.json", %{user: user} = opts) do fields: fields, bot: bot, source: %{ - note: "", + note: HTML.strip_tags((user.bio || "") |> String.replace("
", "\n")), sensitive: false, pleroma: %{} }, diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index a24f2a050..aaf2261bb 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -55,7 +55,7 @@ test "Represent a user account" do fields: [], bot: false, source: %{ - note: "", + note: "valid html", sensitive: false, pleroma: %{} }, @@ -120,7 +120,7 @@ test "Represent a Service(bot) account" do fields: [], bot: true, source: %{ - note: "", + note: user.bio, sensitive: false, pleroma: %{} }, @@ -209,7 +209,7 @@ test "represent an embedded relationship" do fields: [], bot: true, source: %{ - note: "", + note: user.bio, sensitive: false, pleroma: %{} }, From 1344c85e2fe636fd6b9d033eb7add1c3a9701c7f Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 05:58:51 +0000 Subject: [PATCH 085/123] salmon: fix credo --- lib/pleroma/web/salmon/salmon.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index fa30f73cd..f25d92fad 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -10,8 +10,8 @@ defmodule Pleroma.Web.Salmon do use Bitwise alias Pleroma.Activity - alias Pleroma.Keys alias Pleroma.Instances + alias Pleroma.Keys alias Pleroma.User alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Federator.Publisher From f9e0d09ec0082a096dcd4980bc5ffebe8e3139ae Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 10:17:32 +0200 Subject: [PATCH 086/123] Changelog: Add SSH mode. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ff70e6e5..3d1e7640d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Added +- Optional SSH access mode. - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support. - LDAP authentication - External OAuth provider authentication From b6cf62ddeab04db6bd2695c5537c81e0fb1aecaf Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 10:28:50 +0200 Subject: [PATCH 087/123] Mix: Don't start esshd application if we don't need it. --- mix.exs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 95c052c34..bc5b6204f 100644 --- a/mix.exs +++ b/mix.exs @@ -40,9 +40,16 @@ def project do # # Type `mix help compile.app` for more information. def application do + extra_applications = [:logger, :runtime_tools, :comeonin, :quack] + extra_applications = if Application.get_env(:esshd, :enabled, false) do + [:esshd | extra_applications] + else + extra_applications + end + [ mod: {Pleroma.Application, []}, - extra_applications: [:logger, :runtime_tools, :comeonin, :esshd, :quack], + extra_applications: extra_applications, included_applications: [:ex_syslogger] ] end From db9a82d168cfc452611a44d92df2b81a5e6d1e69 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 10:40:15 +0200 Subject: [PATCH 088/123] Linting. --- mix.exs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mix.exs b/mix.exs index bc5b6204f..25ec46a46 100644 --- a/mix.exs +++ b/mix.exs @@ -41,11 +41,13 @@ def project do # Type `mix help compile.app` for more information. def application do extra_applications = [:logger, :runtime_tools, :comeonin, :quack] - extra_applications = if Application.get_env(:esshd, :enabled, false) do - [:esshd | extra_applications] - else - extra_applications - end + + extra_applications = + if Application.get_env(:esshd, :enabled, false) do + [:esshd | extra_applications] + else + extra_applications + end [ mod: {Pleroma.Application, []}, From b22145cbc40b57cf83f6389f063a76a03625ff16 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 10:44:26 +0200 Subject: [PATCH 089/123] Documentation: Specify PEM format for SSH keys. Otherwise openssh-client 7.9 will generate a different format that can't be used by esshd. --- docs/config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.md b/docs/config.md index 197326bbd..63ca61d1e 100644 --- a/docs/config.md +++ b/docs/config.md @@ -477,7 +477,7 @@ config :esshd, password_authenticator: "Pleroma.BBS.Authenticator" ``` -Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT` +Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -m PEM -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT` ## :auth From 3b12e1ba7c99382c678ce17629352135f44dcb9f Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 11:01:10 +0200 Subject: [PATCH 090/123] Changelog: Add tip for debian users. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d1e7640d..b88edd072 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Added -- Optional SSH access mode. +- Optional SSH access mode. (Needs `erlang-ssh` package on Debian). - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support. - LDAP authentication - External OAuth provider authentication From f4cfcead8868481c19ebd93b0fcd2b942dc0e477 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 11:44:17 +0200 Subject: [PATCH 091/123] Mix: Bring ecto-sql back to mainline. --- mix.exs | 5 +---- mix.lock | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/mix.exs b/mix.exs index 95c052c34..9149b241f 100644 --- a/mix.exs +++ b/mix.exs @@ -66,10 +66,7 @@ defp deps do {:plug_cowboy, "~> 2.0"}, {:phoenix_pubsub, "~> 1.1"}, {:phoenix_ecto, "~> 4.0"}, - {:ecto_sql, - git: "https://github.com/elixir-ecto/ecto_sql", - ref: "14cb065a74c488d737d973f7a91bc036c6245f78", - override: true}, + {:ecto_sql, "~> 3.1"}, {:postgrex, ">= 0.13.5"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, diff --git a/mix.lock b/mix.lock index bacc09787..857bfca79 100644 --- a/mix.lock +++ b/mix.lock @@ -21,7 +21,7 @@ "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"}, "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "14cb065a74c488d737d973f7a91bc036c6245f78", [ref: "14cb065a74c488d737d973f7a91bc036c6245f78"]}, + "ecto_sql": {:hex, :ecto_sql, "3.1.3", "2c536139190492d9de33c5fefac7323c5eaaa82e1b9bf93482a14649042f7cd9", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, From f323031927ecaf155e661b17cc9b96333fb9e4ad Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 12:57:20 +0200 Subject: [PATCH 092/123] Mix: Only start sshd when needed, second try. --- mix.exs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/mix.exs b/mix.exs index 25ec46a46..b2c075c85 100644 --- a/mix.exs +++ b/mix.exs @@ -40,18 +40,9 @@ def project do # # Type `mix help compile.app` for more information. def application do - extra_applications = [:logger, :runtime_tools, :comeonin, :quack] - - extra_applications = - if Application.get_env(:esshd, :enabled, false) do - [:esshd | extra_applications] - else - extra_applications - end - [ mod: {Pleroma.Application, []}, - extra_applications: extra_applications, + extra_applications: [:logger, :runtime_tools, :comeonin, :quack], included_applications: [:ex_syslogger] ] end @@ -129,7 +120,7 @@ defp deps do {:recon, github: "ferd/recon", tag: "2.4.0"}, {:quack, "~> 0.1.1"}, {:benchee, "~> 1.0"}, - {:esshd, "~> 0.1.0"}, + {:esshd, "~> 0.1.0", runtime: Application.get_env(:esshd, :enabled, false)}, {:ex_rated, "~> 1.2"}, {:plug_static_index_html, "~> 1.0.0"}, {:excoveralls, "~> 0.11.1", only: :test} From 78ac8ee56139ed98625c54ce627eb37047a361f0 Mon Sep 17 00:00:00 2001 From: lambda Date: Wed, 22 May 2019 11:07:51 +0000 Subject: [PATCH 093/123] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b88edd072..bb2306fc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Added -- Optional SSH access mode. (Needs `erlang-ssh` package on Debian). +- Optional SSH access mode. (Needs `erlang-ssh` package on some distributions). - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support. - LDAP authentication - External OAuth provider authentication From 620908a2db86942a00bc0ba9c71c037061e26967 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 22 May 2019 15:44:50 +0000 Subject: [PATCH 094/123] [#699] add worker to clean expired oauth tokens --- CHANGELOG.md | 1 + config/config.exs | 4 +- docs/config.md | 2 + lib/pleroma/application.ex | 1 + lib/pleroma/web/oauth/token.ex | 36 ++++++-------- lib/pleroma/web/oauth/token/clean_worker.ex | 41 +++++++++++++++ lib/pleroma/web/oauth/token/query.ex | 55 +++++++++++++++++++++ test/web/oauth/token_test.exs | 13 +++++ 8 files changed, 132 insertions(+), 21 deletions(-) create mode 100644 lib/pleroma/web/oauth/token/clean_worker.ex create mode 100644 lib/pleroma/web/oauth/token/query.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index b5c42d1fd..02d64a850 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - OAuth: added support for refresh tokens - Emoji packs and emoji pack manager - Object pruning (`mix pleroma.database prune_objects`) +- OAuth: added job to clean expired access tokens ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer diff --git a/config/config.exs b/config/config.exs index a05f8b1d2..33b7e713d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -481,7 +481,9 @@ config :pleroma, :oauth2, token_expires_in: 600, - issue_new_refresh_token: true + issue_new_refresh_token: true, + clean_expired_tokens: false, + clean_expired_tokens_interval: 86_400_000 config :pleroma, :database, rum_enabled: false diff --git a/docs/config.md b/docs/config.md index a050068f4..264b65499 100644 --- a/docs/config.md +++ b/docs/config.md @@ -550,6 +550,8 @@ Configure OAuth 2 provider capabilities: * `token_expires_in` - The lifetime in seconds of the access token. * `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token. +* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. +* `clean_expired_tokens_interval` - Interval to run the job to clean expired tokens. Defaults to `86_400_000` (24 hours). ## :emoji * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]` diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index dab45a0b2..76df3945e 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -110,6 +110,7 @@ def start(_type, _args) do hackney_pool_children() ++ [ worker(Pleroma.Web.Federator.RetryQueue, []), + worker(Pleroma.Web.OAuth.Token.CleanWorker, []), worker(Pleroma.Stats, []), worker(Task, [&Pleroma.Web.Push.init/0], restart: :temporary, id: :web_push_init), worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary, id: :federator_init) diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index 66c95c2e9..f412f7eb2 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.OAuth.Token do use Ecto.Schema - import Ecto.Query import Ecto.Changeset alias Pleroma.Repo @@ -13,6 +12,7 @@ defmodule Pleroma.Web.OAuth.Token do alias Pleroma.Web.OAuth.App alias Pleroma.Web.OAuth.Authorization alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.OAuth.Token.Query @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600) @type t :: %__MODULE__{} @@ -31,17 +31,17 @@ defmodule Pleroma.Web.OAuth.Token do @doc "Gets token for app by access token" @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found} def get_by_token(%App{id: app_id} = _app, token) do - from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token) + Query.get_by_app(app_id) + |> Query.get_by_token(token) |> Repo.find_resource() end @doc "Gets token for app by refresh token" @spec get_by_refresh_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found} def get_by_refresh_token(%App{id: app_id} = _app, token) do - from(t in __MODULE__, - where: t.app_id == ^app_id and t.refresh_token == ^token, - preload: [:user] - ) + Query.get_by_app(app_id) + |> Query.get_by_refresh_token(token) + |> Query.preload([:user]) |> Repo.find_resource() end @@ -97,29 +97,25 @@ def create_token(%App{} = app, %User{} = user, attrs \\ %{}) do end def delete_user_tokens(%User{id: user_id}) do - from( - t in Token, - where: t.user_id == ^user_id - ) + Query.get_by_user(user_id) |> Repo.delete_all() end def delete_user_token(%User{id: user_id}, token_id) do - from( - t in Token, - where: t.user_id == ^user_id, - where: t.id == ^token_id - ) + Query.get_by_user(user_id) + |> Query.get_by_id(token_id) + |> Repo.delete_all() + end + + def delete_expired_tokens do + Query.get_expired_tokens() |> Repo.delete_all() end def get_user_tokens(%User{id: user_id}) do - from( - t in Token, - where: t.user_id == ^user_id - ) + Query.get_by_user(user_id) + |> Query.preload([:app]) |> Repo.all() - |> Repo.preload(:app) end def is_expired?(%__MODULE__{valid_until: valid_until}) do diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex new file mode 100644 index 000000000..dca852449 --- /dev/null +++ b/lib/pleroma/web/oauth/token/clean_worker.ex @@ -0,0 +1,41 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.OAuth.Token.CleanWorker do + @moduledoc """ + The module represents functions to clean an expired oauth tokens. + """ + + # 10 seconds + @start_interval 10_000 + @interval Pleroma.Config.get( + # 24 hours + [:oauth2, :clean_expired_tokens_interval], + 86_400_000 + ) + @queue :background + + alias Pleroma.Web.OAuth.Token + + def start_link, do: GenServer.start_link(__MODULE__, nil) + + def init(_) do + if Pleroma.Config.get([:oauth2, :clean_expired_tokens], false) do + Process.send_after(self(), :perform, @start_interval) + {:ok, nil} + else + :ignore + end + end + + @doc false + def handle_info(:perform, state) do + Process.send_after(self(), :perform, @interval) + PleromaJobQueue.enqueue(@queue, __MODULE__, [:clean]) + {:noreply, state} + end + + # Job Worker Callbacks + def perform(:clean), do: Token.delete_expired_tokens() +end diff --git a/lib/pleroma/web/oauth/token/query.ex b/lib/pleroma/web/oauth/token/query.ex new file mode 100644 index 000000000..d92e1f071 --- /dev/null +++ b/lib/pleroma/web/oauth/token/query.ex @@ -0,0 +1,55 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.OAuth.Token.Query do + @moduledoc """ + Contains queries for OAuth Token. + """ + + import Ecto.Query, only: [from: 2] + + @type query :: Ecto.Queryable.t() | Token.t() + + alias Pleroma.Web.OAuth.Token + + @spec get_by_refresh_token(query, String.t()) :: query + def get_by_refresh_token(query \\ Token, refresh_token) do + from(q in query, where: q.refresh_token == ^refresh_token) + end + + @spec get_by_token(query, String.t()) :: query + def get_by_token(query \\ Token, token) do + from(q in query, where: q.token == ^token) + end + + @spec get_by_app(query, String.t()) :: query + def get_by_app(query \\ Token, app_id) do + from(q in query, where: q.app_id == ^app_id) + end + + @spec get_by_id(query, String.t()) :: query + def get_by_id(query \\ Token, id) do + from(q in query, where: q.id == ^id) + end + + @spec get_expired_tokens(query, DateTime.t() | nil) :: query + def get_expired_tokens(query \\ Token, date \\ nil) do + expired_date = date || Timex.now() + from(q in query, where: fragment("?", q.valid_until) < ^expired_date) + end + + @spec get_by_user(query, String.t()) :: query + def get_by_user(query \\ Token, user_id) do + from(q in query, where: q.user_id == ^user_id) + end + + @spec preload(query, any) :: query + def preload(query \\ Token, assoc_preload \\ []) + + def preload(query, assoc_preload) when is_list(assoc_preload) do + from(q in query, preload: ^assoc_preload) + end + + def preload(query, _assoc_preload), do: query +end diff --git a/test/web/oauth/token_test.exs b/test/web/oauth/token_test.exs index ad2a49f09..3c07309b7 100644 --- a/test/web/oauth/token_test.exs +++ b/test/web/oauth/token_test.exs @@ -69,4 +69,17 @@ test "deletes all tokens of a user" do assert tokens == 2 end + + test "deletes expired tokens" do + insert(:oauth_token, valid_until: Timex.shift(Timex.now(), days: -3)) + insert(:oauth_token, valid_until: Timex.shift(Timex.now(), days: -3)) + t3 = insert(:oauth_token) + t4 = insert(:oauth_token, valid_until: Timex.shift(Timex.now(), minutes: 10)) + {tokens, _} = Token.delete_expired_tokens() + assert tokens == 2 + available_tokens = Pleroma.Repo.all(Token) + + token_ids = available_tokens |> Enum.map(& &1.id) + assert token_ids == [t3.id, t4.id] + end end From 54e10a3e55fe46b71ef7f330605baf8bcccd5a44 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 22 May 2019 20:10:52 +0300 Subject: [PATCH 095/123] Disable timeouts for object pruning query --- lib/mix/tasks/pleroma/database.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index f9bafb277..4d480ac3f 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -107,7 +107,7 @@ def run(["prune_objects" | args]) do where: fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host()) ) - |> Repo.delete_all() + |> Repo.delete_all(timeout: :infinity) if Keyword.get(options, :vacuum) do Logger.info("Runnning VACUUM FULL") From 75b6c4b00433560fb5ee502f13e8261b4b8a246a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 04:15:59 +0000 Subject: [PATCH 096/123] mrf: defang policy modules for filtering user profile objects --- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 6 ++++-- lib/pleroma/web/activity_pub/mrf/user_allowlist.ex | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 7190652d2..ffaa4b7db 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -105,8 +105,8 @@ defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} defp check_report_removal(_actor_info, object), do: {:ok, object} @impl true - def filter(object) do - actor_info = URI.parse(object["actor"]) + def filter(%{"actor" => actor} = object) do + actor_info = URI.parse(actor) with {:ok, object} <- check_accept(actor_info, object), {:ok, object} <- check_reject(actor_info, object), @@ -119,4 +119,6 @@ def filter(object) do _e -> {:reject, nil} end end + + def filter(object), do: {:ok, object} end diff --git a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex index f5078d818..47663414a 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex @@ -19,10 +19,12 @@ defp filter_by_list(%{"actor" => actor} = object, allow_list) do end @impl true - def filter(object) do - actor_info = URI.parse(object["actor"]) + def filter(%{"actor" => actor} = object) do + actor_info = URI.parse(actor) allow_list = Config.get([:mrf_user_allowlist, String.to_atom(actor_info.host)], []) filter_by_list(object, allow_list) end + + def filter(object), do: {:ok, object} end From 60f882b09f5f837546d59f8eef56b905e065ec60 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 04:33:10 +0000 Subject: [PATCH 097/123] activitypub: run user objects through MRF filters --- lib/pleroma/web/activity_pub/activity_pub.ex | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 3d9679ec0..aa0229db7 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -909,7 +909,7 @@ def upload(file, opts \\ []) do end end - def user_data_from_user_object(data) do + defp object_to_user_data(data) do avatar = data["icon"]["url"] && %{ @@ -956,9 +956,19 @@ def user_data_from_user_object(data) do {:ok, user_data} end + def user_data_from_user_object(data) do + with {:ok, data} <- MRF.filter(data), + {:ok, data} <- object_to_user_data(data) do + {:ok, data} + else + e -> {:error, e} + end + end + def fetch_and_prepare_user_from_ap_id(ap_id) do - with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do - user_data_from_user_object(data) + with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id), + {:ok, data} <- user_data_from_user_object(data) do + {:ok, data} else e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}") end From baf72d6c580e5c05ef5fea8a57c57150a5d38589 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 04:55:16 +0000 Subject: [PATCH 098/123] mrf: simple policy: add the ability to strip avatars and banners from user profiles --- config/config.exs | 4 ++- .../web/activity_pub/mrf/simple_policy.ex | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 33b7e713d..e90821d66 100644 --- a/config/config.exs +++ b/config/config.exs @@ -314,7 +314,9 @@ federated_timeline_removal: [], report_removal: [], reject: [], - accept: [] + accept: [], + avatar_removal: [], + banner_removal: [] config :pleroma, :mrf_keyword, reject: [], diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index ffaa4b7db..890d70a7a 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -104,6 +104,26 @@ defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} defp check_report_removal(_actor_info, object), do: {:ok, object} + defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = object) do + if actor_host in Pleroma.Config.get([:mrf_simple, :avatar_removal]) do + {:ok, Map.delete(object, "icon")} + else + {:ok, object} + end + end + + defp check_avatar_removal(_actor_info, object), do: {:ok, object} + + defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = object) do + if actor_host in Pleroma.Config.get([:mrf_simple, :banner_removal]) do + {:ok, Map.delete(object, "image")} + else + {:ok, object} + end + end + + defp check_banner_removal(_actor_info, object), do: {:ok, object} + @impl true def filter(%{"actor" => actor} = object) do actor_info = URI.parse(actor) @@ -120,5 +140,17 @@ def filter(%{"actor" => actor} = object) do end end + def filter(%{"id" => actor, "type" => obj_type} = object) + when obj_type in ["Application", "Group", "Organization", "Person", "Service"] do + actor_info = URI.parse(actor) + + with {:ok, object} <- check_avatar_removal(actor_info, object), + {:ok, object} <- check_banner_removal(actor_info, object) do + {:ok, object} + else + _e -> {:reject, nil} + end + end + def filter(object), do: {:ok, object} end From 8086c7aed6cdc3b2ac1c09c6c40344e47be08ed9 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 05:08:37 +0000 Subject: [PATCH 099/123] tests: add tests for banner and avatar removal --- .../activity_pub/mrf/simple_policy_test.exs | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs index 74af7dcde..3d1f26e60 100644 --- a/test/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/web/activity_pub/mrf/simple_policy_test.exs @@ -17,7 +17,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do federated_timeline_removal: [], report_removal: [], reject: [], - accept: [] + accept: [], + avatar_removal: [], + banner_removal: [] ) on_exit(fn -> @@ -206,6 +208,60 @@ test "has a matching host" do end end + describe "when :avatar_removal" do + test "is empty" do + Config.put([:mrf_simple, :avatar_removal], []) + + remote_user = build_remote_user() + + assert SimplePolicy.filter(remote_user) == {:ok, remote_user} + end + + test "is not empty but it doesn't have a matching host" do + Config.put([:mrf_simple, :avatar_removal], ["non.matching.remote"]) + + remote_user = build_remote_user() + + assert SimplePolicy.filter(remote_user) == {:ok, remote_user} + end + + test "has a matching host" do + Config.put([:mrf_simple, :avatar_removal], ["remote.instance"]) + + remote_user = build_remote_user() + {:ok, filtered} = SimplePolicy.filter(remote_user) + + refute filtered["icon"] + end + end + + describe "when :banner_removal" do + test "is empty" do + Config.put([:mrf_simple, :banner_removal], []) + + remote_user = build_remote_user() + + assert SimplePolicy.filter(remote_user) == {:ok, remote_user} + end + + test "is not empty but it doesn't have a matching host" do + Config.put([:mrf_simple, :banner_removal], ["non.matching.remote"]) + + remote_user = build_remote_user() + + assert SimplePolicy.filter(remote_user) == {:ok, remote_user} + end + + test "has a matching host" do + Config.put([:mrf_simple, :banner_removal], ["remote.instance"]) + + remote_user = build_remote_user() + {:ok, filtered} = SimplePolicy.filter(remote_user) + + refute filtered["image"] + end + end + defp build_local_message do %{ "actor" => "#{Pleroma.Web.base_url()}/users/alice", @@ -217,4 +273,19 @@ defp build_local_message do defp build_remote_message do %{"actor" => "https://remote.instance/users/bob"} end + + defp build_remote_user do + %{ + "id" => "https://remote.instance/users/bob", + "icon" => %{ + "url" => "http://example.com/image.jpg", + "type" => "Image" + }, + "image" => %{ + "url" => "http://example.com/image.jpg", + "type" => "Image" + }, + "type" => "Person" + } + end end From 7d9b33b3cebeed451210f754a8c34cc14a9e969b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 05:55:09 +0000 Subject: [PATCH 100/123] update documentation for the new MRF features [no-ci] --- CHANGELOG.md | 2 ++ docs/config.md | 3 +++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02d64a850..2a2b11ddf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Emoji packs and emoji pack manager - Object pruning (`mix pleroma.database prune_objects`) - OAuth: added job to clean expired access tokens +- MRF: Support for rejecting reports from specific instances (`mrf_simple`) +- MRF: Support for stripping avatars and banner images from specific instances (`mrf_simple`) ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer diff --git a/docs/config.md b/docs/config.md index 264b65499..1d1d24c32 100644 --- a/docs/config.md +++ b/docs/config.md @@ -220,6 +220,9 @@ relates to mascots on the mastodon frontend * `federated_timeline_removal`: List of instances to remove from Federated (aka The Whole Known Network) Timeline * `reject`: List of instances to reject any activities from * `accept`: List of instances to accept any activities from +* `report_removal`: List of instances to reject reports from +* `avatar_removal`: List of instances to strip avatars from +* `banner_removal`: List of instances to strip banners from ## :mrf_rejectnonpublic * `allow_followersonly`: whether to allow followers-only posts From 356c047759735bcd984ebd44059a21a3cf22af0e Mon Sep 17 00:00:00 2001 From: Alfie Pates Date: Thu, 23 May 2019 22:33:27 +0100 Subject: [PATCH 101/123] explicitly set reverse proxy upstream to IPv4 since Pleroma.Web.Endpoint binds on IPv4 only and `localhost.` resolves to [::0] on some systems fixes #930. --- installation/caddyfile-pleroma.example | 4 +++- installation/pleroma-apache.conf | 6 ++++-- installation/pleroma.nginx | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/installation/caddyfile-pleroma.example b/installation/caddyfile-pleroma.example index fcf76718e..7985d9c67 100644 --- a/installation/caddyfile-pleroma.example +++ b/installation/caddyfile-pleroma.example @@ -10,7 +10,9 @@ example.tld { gzip - proxy / localhost:4000 { + # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only + # and `localhost.` resolves to [::0] on some systems: see issue #930 + proxy / 127.0.0.1:4000 { websocket transparent } diff --git a/installation/pleroma-apache.conf b/installation/pleroma-apache.conf index 2beb7c4cc..b5640ac3d 100644 --- a/installation/pleroma-apache.conf +++ b/installation/pleroma-apache.conf @@ -58,8 +58,10 @@ CustomLog ${APACHE_LOG_DIR}/access.log combined RewriteRule /(.*) ws://localhost:4000/$1 [P,L] ProxyRequests off - ProxyPass / http://localhost:4000/ - ProxyPassReverse / http://localhost:4000/ + # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only + # and `localhost.` resolves to [::0] on some systems: see issue #930 + ProxyPass / http://127.0.0.1:4000/ + ProxyPassReverse / http://127.0.0.1:4000/ RequestHeader set Host ${servername} ProxyPreserveHost On diff --git a/installation/pleroma.nginx b/installation/pleroma.nginx index cc75d78b2..7425da33f 100644 --- a/installation/pleroma.nginx +++ b/installation/pleroma.nginx @@ -69,7 +69,9 @@ server { proxy_set_header Connection "upgrade"; proxy_set_header Host $http_host; - proxy_pass http://localhost:4000; + # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only + # and `localhost.` resolves to [::0] on some systems: see issue #930 + proxy_pass http://127.0.0.1:4000; client_max_body_size 16m; } From f916e4cdd9a502b83c615146c598be135f47e57a Mon Sep 17 00:00:00 2001 From: feld Date: Fri, 24 May 2019 20:33:55 +0000 Subject: [PATCH 102/123] Move the Cache Control header test to its own file We can consolidate our cache control header tests here --- lib/pleroma/web/endpoint.ex | 19 +++++++++++++++++-- test/plugs/cache_control_test.exs | 20 ++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 test/plugs/cache_control_test.exs diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 9ef30e885..8cd7a2270 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -16,17 +16,32 @@ defmodule Pleroma.Web.Endpoint do plug(Pleroma.Plugs.UploadedMedia) + @static_cache_control "public, no-cache" + # InstanceStatic needs to be before Plug.Static to be able to override shipped-static files # If you're adding new paths to `only:` you'll need to configure them in InstanceStatic as well - plug(Pleroma.Plugs.InstanceStatic, at: "/") + # Cache-control headers are duplicated in case we turn off etags in the future + plug(Pleroma.Plugs.InstanceStatic, + at: "/", + gzip: true, + cache_control_for_etags: @static_cache_control, + headers: %{ + "cache-control" => @static_cache_control + } + ) plug( Plug.Static, at: "/", from: :pleroma, only: - ~w(index.html robots.txt static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc) + ~w(index.html robots.txt static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc), # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength + gzip: true, + cache_control_for_etags: @static_cache_control, + headers: %{ + "cache-control" => @static_cache_control + } ) plug(Plug.Static.IndexHtml, at: "/pleroma/admin/") diff --git a/test/plugs/cache_control_test.exs b/test/plugs/cache_control_test.exs new file mode 100644 index 000000000..45151b289 --- /dev/null +++ b/test/plugs/cache_control_test.exs @@ -0,0 +1,20 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.CacheControlTest do + use Pleroma.Web.ConnCase + alias Plug.Conn + + test "Verify Cache-Control header on static assets", %{conn: conn} do + conn = get(conn, "/index.html") + + assert Conn.get_resp_header(conn, "cache-control") == ["public, no-cache"] + end + + test "Verify Cache-Control header on the API", %{conn: conn} do + conn = get(conn, "/api/v1/instance") + + assert Conn.get_resp_header(conn, "cache-control") == ["max-age=0, private, must-revalidate"] + end +end From bbea5691da67916151a883f09e24da7c2e27d9ba Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Fri, 24 May 2019 20:34:23 +0000 Subject: [PATCH 103/123] Mention all people in the beginning of DM --- lib/pleroma/formatter.ex | 2 +- test/formatter_test.exs | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 3e3b9fe97..607843a5b 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Formatter do alias Pleroma.User alias Pleroma.Web.MediaProxy - @safe_mention_regex ~r/^(\s*(?@.+?\s+)+)(?.*)/s + @safe_mention_regex ~r/^(\s*(?(@.+?\s+){1,})+)(?.*)/s @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/ diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 47b91b121..bfa673049 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -184,17 +184,19 @@ test "does not give a replacement for single-character local nicknames who don't test "given the 'safe_mention' option, it will only mention people in the beginning" do user = insert(:user) - _other_user = insert(:user) + other_user = insert(:user) third_user = insert(:user) - text = " @#{user.nickname} hey dude i hate @#{third_user.nickname}" + text = " @#{user.nickname} @#{other_user.nickname} hey dudes i hate @#{third_user.nickname}" {expected_text, mentions, [] = _tags} = Formatter.linkify(text, safe_mention: true) - assert mentions == [{"@#{user.nickname}", user}] + assert mentions == [{"@#{user.nickname}", user}, {"@#{other_user.nickname}", other_user}] assert expected_text == "@#{user.nickname} hey dude i hate @#{user.nickname} @#{other_user.nickname} hey dudes i hate @#{third_user.nickname}" end From 9415932af5829f2aa19e362076e0653dd1ce9c5a Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Sat, 25 May 2019 08:15:12 +0800 Subject: [PATCH 104/123] Keep nodeinfo available when not federating --- .../web/nodeinfo/nodeinfo_controller.ex | 2 - test/web/node_info_test.exs | 80 ++++--------------- 2 files changed, 16 insertions(+), 66 deletions(-) diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 3bf2a0fbc..45f90c579 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -12,8 +12,6 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.Federator.Publisher - plug(Pleroma.Web.FederatingPlug) - def schemas(conn, _params) do response = %{ links: [ diff --git a/test/web/node_info_test.exs b/test/web/node_info_test.exs index 2fc42b7cc..be1173513 100644 --- a/test/web/node_info_test.exs +++ b/test/web/node_info_test.exs @@ -7,6 +7,22 @@ defmodule Pleroma.Web.NodeInfoTest do import Pleroma.Factory + test "GET /.well-known/nodeinfo", %{conn: conn} do + links = + conn + |> get("/.well-known/nodeinfo") + |> json_response(200) + |> Map.fetch!("links") + + Enum.each(links, fn link -> + href = Map.fetch!(link, "href") + + conn + |> get(href) + |> json_response(200) + end) + end + test "nodeinfo shows staff accounts", %{conn: conn} do moderator = insert(:user, %{local: true, info: %{is_moderator: true}}) admin = insert(:user, %{local: true, info: %{is_admin: true}}) @@ -32,70 +48,6 @@ test "nodeinfo shows restricted nicknames", %{conn: conn} do result["metadata"]["restrictedNicknames"] end - test "returns 404 when federation is disabled", %{conn: conn} do - instance = - Application.get_env(:pleroma, :instance) - |> Keyword.put(:federating, false) - - Application.put_env(:pleroma, :instance, instance) - - conn - |> get("/.well-known/nodeinfo") - |> json_response(404) - - conn - |> get("/nodeinfo/2.1.json") - |> json_response(404) - - instance = - Application.get_env(:pleroma, :instance) - |> Keyword.put(:federating, true) - - Application.put_env(:pleroma, :instance, instance) - end - - test "returns 200 when federation is enabled", %{conn: conn} do - conn - |> get("/.well-known/nodeinfo") - |> json_response(200) - - conn - |> get("/nodeinfo/2.1.json") - |> json_response(200) - end - - test "returns 404 when federation is disabled (nodeinfo 2.0)", %{conn: conn} do - instance = - Application.get_env(:pleroma, :instance) - |> Keyword.put(:federating, false) - - Application.put_env(:pleroma, :instance, instance) - - conn - |> get("/.well-known/nodeinfo") - |> json_response(404) - - conn - |> get("/nodeinfo/2.0.json") - |> json_response(404) - - instance = - Application.get_env(:pleroma, :instance) - |> Keyword.put(:federating, true) - - Application.put_env(:pleroma, :instance, instance) - end - - test "returns 200 when federation is enabled (nodeinfo 2.0)", %{conn: conn} do - conn - |> get("/.well-known/nodeinfo") - |> json_response(200) - - conn - |> get("/nodeinfo/2.0.json") - |> json_response(200) - end - test "returns software.repository field in nodeinfo 2.1", %{conn: conn} do conn |> get("/.well-known/nodeinfo") From 9bec891eb4d5d06e6bd84dd2c95259d2c1a4f563 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 04:24:21 +0000 Subject: [PATCH 105/123] kill @httpoison --- config/config.exs | 1 - lib/pleroma/object/fetcher.ex | 5 ++--- lib/pleroma/reverse_proxy.ex | 5 +++-- lib/pleroma/uploaders/mdii.ex | 5 ++--- lib/pleroma/web/activity_pub/publisher.ex | 5 ++--- .../web/mastodon_api/mastodon_api_controller.ex | 4 ++-- lib/pleroma/web/ostatus/ostatus.ex | 7 +++---- lib/pleroma/web/salmon/salmon.ex | 5 ++--- lib/pleroma/web/web_finger/web_finger.ex | 9 ++++----- lib/pleroma/web/websub/websub.ex | 11 +++++------ 10 files changed, 25 insertions(+), 32 deletions(-) diff --git a/config/config.exs b/config/config.exs index e90821d66..01827fb5f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -186,7 +186,6 @@ config :pleroma, :websub, Pleroma.Web.Websub config :pleroma, :ostatus, Pleroma.Web.OStatus -config :pleroma, :httpoison, Pleroma.HTTP config :tesla, adapter: Tesla.Adapter.Hackney # Configures http settings, upstream proxy etc. diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index bb9388d4f..ca980c629 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -1,4 +1,5 @@ defmodule Pleroma.Object.Fetcher do + alias Pleroma.HTTP alias Pleroma.Object alias Pleroma.Object.Containment alias Pleroma.Web.ActivityPub.Transmogrifier @@ -6,8 +7,6 @@ defmodule Pleroma.Object.Fetcher do require Logger - @httpoison Application.get_env(:pleroma, :httpoison) - defp reinject_object(data) do Logger.debug("Reinjecting object #{data["id"]}") @@ -78,7 +77,7 @@ def fetch_and_contain_remote_object_from_id(id) do with true <- String.starts_with?(id, "http"), {:ok, %{body: body, status: code}} when code in 200..299 <- - @httpoison.get( + HTTP.get( id, [{:Accept, "application/activity+json"}] ), diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index a3f177fec..6e5feb4c3 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -3,6 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy do + alias Pleroma.HTTP + @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++ ~w(if-unmodified-since if-none-match if-range range) @resp_cache_headers ~w(etag date last-modified cache-control) @@ -60,7 +62,6 @@ defmodule Pleroma.ReverseProxy do """ @hackney Application.get_env(:pleroma, :hackney, :hackney) - @httpoison Application.get_env(:pleroma, :httpoison, HTTPoison) @default_hackney_options [] @@ -97,7 +98,7 @@ def call(conn = %{method: method}, url, opts) when method in @methods do hackney_opts = @default_hackney_options |> Keyword.merge(Keyword.get(opts, :http, [])) - |> @httpoison.process_request_options() + |> HTTP.process_request_options() req_headers = build_req_headers(conn.req_headers, opts) diff --git a/lib/pleroma/uploaders/mdii.ex b/lib/pleroma/uploaders/mdii.ex index 190ed9f3a..237544337 100644 --- a/lib/pleroma/uploaders/mdii.ex +++ b/lib/pleroma/uploaders/mdii.ex @@ -4,11 +4,10 @@ defmodule Pleroma.Uploaders.MDII do alias Pleroma.Config + alias Pleroma.HTTP @behaviour Pleroma.Uploaders.Uploader - @httpoison Application.get_env(:pleroma, :httpoison) - # MDII-hosted images are never passed through the MediaPlug; only local media. # Delegate to Pleroma.Uploaders.Local def get_file(file) do @@ -25,7 +24,7 @@ def put_file(upload) do query = "#{cgi}?#{extension}" with {:ok, %{status: 200, body: body}} <- - @httpoison.post(query, file_data, [], adapter: [pool: :default]) do + HTTP.post(query, file_data, [], adapter: [pool: :default]) do remote_file_name = String.split(body) |> List.first() public_url = "#{files}/#{remote_file_name}.#{extension}" {:ok, {:url, public_url}} diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 11dba87de..8f1399ce6 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do alias Pleroma.Activity alias Pleroma.Config + alias Pleroma.HTTP alias Pleroma.Instances alias Pleroma.User alias Pleroma.Web.ActivityPub.Relay @@ -16,8 +17,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do require Logger - @httpoison Application.get_env(:pleroma, :httpoison) - @moduledoc """ ActivityPub outgoing federation module. """ @@ -63,7 +62,7 @@ def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = pa with {:ok, %{status: code}} when code in 200..299 <- result = - @httpoison.post( + HTTP.post( inbox, json, [ diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 1ec0f30a1..0fe09c285 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Pleroma.Conversation.Participation alias Pleroma.Filter alias Pleroma.Formatter + alias Pleroma.HTTP alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Object.Fetcher @@ -55,7 +56,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do when action in [:account_register] ) - @httpoison Application.get_env(:pleroma, :httpoison) @local_mastodon_name "Mastodon-Local" action_fallback(:errors) @@ -1691,7 +1691,7 @@ def suggestions(%{assigns: %{user: user}} = conn, _) do |> String.replace("{{user}}", user) with {:ok, %{status: 200, body: body}} <- - @httpoison.get( + HTTP.get( url, [], adapter: [ diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 61515b31e..6ed089d84 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -3,13 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.OStatus do - @httpoison Application.get_env(:pleroma, :httpoison) - import Ecto.Query import Pleroma.Web.XML require Logger alias Pleroma.Activity + alias Pleroma.HTTP alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User @@ -363,7 +362,7 @@ def get_atom_url(body) do def fetch_activity_from_atom_url(url) do with true <- String.starts_with?(url, "http"), {:ok, %{body: body, status: code}} when code in 200..299 <- - @httpoison.get( + HTTP.get( url, [{:Accept, "application/atom+xml"}] ) do @@ -380,7 +379,7 @@ def fetch_activity_from_html_url(url) do Logger.debug("Trying to fetch #{url}") with true <- String.starts_with?(url, "http"), - {:ok, %{body: body}} <- @httpoison.get(url, []), + {:ok, %{body: body}} <- HTTP.get(url, []), {:ok, atom_url} <- get_atom_url(body) do fetch_activity_from_atom_url(atom_url) else diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index f25d92fad..9e91a5a40 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -5,11 +5,10 @@ defmodule Pleroma.Web.Salmon do @behaviour Pleroma.Web.Federator.Publisher - @httpoison Application.get_env(:pleroma, :httpoison) - use Bitwise alias Pleroma.Activity + alias Pleroma.HTTP alias Pleroma.Instances alias Pleroma.Keys alias Pleroma.User @@ -138,7 +137,7 @@ def publish_one(%{recipient: %{info: %{salmon: salmon}}} = params), def publish_one(%{recipient: url, feed: feed} = params) when is_binary(url) do with {:ok, %{status: code}} when code in 200..299 <- - @httpoison.post( + HTTP.post( url, feed, [{"Content-Type", "application/magic-envelope+xml"}] diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index c5b7d4acb..3fca72de8 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -3,8 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.WebFinger do - @httpoison Application.get_env(:pleroma, :httpoison) - + alias Pleroma.HTTP alias Pleroma.User alias Pleroma.Web alias Pleroma.Web.Federator.Publisher @@ -176,11 +175,11 @@ def get_template_from_xml(body) do def find_lrdd_template(domain) do with {:ok, %{status: status, body: body}} when status in 200..299 <- - @httpoison.get("http://#{domain}/.well-known/host-meta", []) do + HTTP.get("http://#{domain}/.well-known/host-meta", []) do get_template_from_xml(body) else _ -> - with {:ok, %{body: body}} <- @httpoison.get("https://#{domain}/.well-known/host-meta", []) do + with {:ok, %{body: body}} <- HTTP.get("https://#{domain}/.well-known/host-meta", []) do get_template_from_xml(body) else e -> {:error, "Can't find LRDD template: #{inspect(e)}"} @@ -209,7 +208,7 @@ def finger(account) do end with response <- - @httpoison.get( + HTTP.get( address, Accept: "application/xrd+xml,application/jrd+json" ), diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 7ad0414ab..b61f388b8 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.Websub do alias Ecto.Changeset alias Pleroma.Activity + alias Pleroma.HTTP alias Pleroma.Instances alias Pleroma.Repo alias Pleroma.User @@ -24,9 +25,7 @@ defmodule Pleroma.Web.Websub do @behaviour Pleroma.Web.Federator.Publisher - @httpoison Application.get_env(:pleroma, :httpoison) - - def verify(subscription, getter \\ &@httpoison.get/3) do + def verify(subscription, getter \\ &HTTP.get/3) do challenge = Base.encode16(:crypto.strong_rand_bytes(8)) lease_seconds = NaiveDateTime.diff(subscription.valid_until, subscription.updated_at) lease_seconds = lease_seconds |> to_string @@ -207,7 +206,7 @@ def subscribe(subscriber, subscribed, requester \\ &request_subscription/1) do requester.(subscription) end - def gather_feed_data(topic, getter \\ &@httpoison.get/1) do + def gather_feed_data(topic, getter \\ &HTTP.get/1) do with {:ok, response} <- getter.(topic), status when status in 200..299 <- response.status, body <- response.body, @@ -236,7 +235,7 @@ def gather_feed_data(topic, getter \\ &@httpoison.get/1) do end end - def request_subscription(websub, poster \\ &@httpoison.post/3, timeout \\ 10_000) do + def request_subscription(websub, poster \\ &HTTP.post/3, timeout \\ 10_000) do data = [ "hub.mode": "subscribe", "hub.topic": websub.topic, @@ -294,7 +293,7 @@ def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret} = Logger.info(fn -> "Pushing #{topic} to #{callback}" end) with {:ok, %{status: code}} when code in 200..299 <- - @httpoison.post( + HTTP.post( callback, xml, [ From 80d55d428fedfe4ebc44569c6134908d269db698 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 04:34:16 +0000 Subject: [PATCH 106/123] tests: websub: check only that signature validation succeeds or fails --- config/test.exs | 4 ++-- test/web/websub/websub_controller_test.exs | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/config/test.exs b/config/test.exs index 6100989c4..3bc407840 100644 --- a/config/test.exs +++ b/config/test.exs @@ -39,8 +39,8 @@ # Reduce hash rounds for testing config :pbkdf2_elixir, rounds: 1 -config :pleroma, :websub, Pleroma.Web.WebsubMock -config :pleroma, :ostatus, Pleroma.Web.OStatusMock +#config :pleroma, :websub, Pleroma.Web.WebsubMock +#config :pleroma, :ostatus, Pleroma.Web.OStatusMock config :tesla, adapter: Tesla.Mock config :pleroma, :rich_media, enabled: false diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs index 1e69ed01a..bf2ee31ee 100644 --- a/test/web/websub/websub_controller_test.exs +++ b/test/web/websub/websub_controller_test.exs @@ -52,7 +52,7 @@ test "websub subscription confirmation", %{conn: conn} do end describe "websub_incoming" do - test "handles incoming feed updates", %{conn: conn} do + test "accepts incoming feed updates", %{conn: conn} do websub = insert(:websub_client_subscription) doc = "some stuff" signature = Websub.sign(websub.secret, doc) @@ -64,8 +64,6 @@ test "handles incoming feed updates", %{conn: conn} do |> post("/push/subscriptions/#{websub.id}", doc) assert response(conn, 200) == "OK" - - assert length(Repo.all(Activity)) == 1 end test "rejects incoming feed updates with the wrong signature", %{conn: conn} do @@ -80,8 +78,6 @@ test "rejects incoming feed updates with the wrong signature", %{conn: conn} do |> post("/push/subscriptions/#{websub.id}", doc) assert response(conn, 500) == "Error" - - assert Enum.empty?(Repo.all(Activity)) end end end From 56fd7dbdd7e3f1a0bb05430ed4805d8715e54935 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 04:39:32 +0000 Subject: [PATCH 107/123] remove @websub and @ostatus module-level constants --- config/config.exs | 2 -- config/test.exs | 2 -- lib/pleroma/web/federator/federator.ex | 8 +++----- test/support/ostatus_mock.ex | 11 ----------- test/support/websub_mock.ex | 9 --------- test/web/websub/websub_controller_test.exs | 1 - 6 files changed, 3 insertions(+), 30 deletions(-) delete mode 100644 test/support/ostatus_mock.ex delete mode 100644 test/support/websub_mock.ex diff --git a/config/config.exs b/config/config.exs index 01827fb5f..68168b279 100644 --- a/config/config.exs +++ b/config/config.exs @@ -184,8 +184,6 @@ "application/ld+json" => ["activity+json"] } -config :pleroma, :websub, Pleroma.Web.Websub -config :pleroma, :ostatus, Pleroma.Web.OStatus config :tesla, adapter: Tesla.Adapter.Hackney # Configures http settings, upstream proxy etc. diff --git a/config/test.exs b/config/test.exs index 3bc407840..41cddb9bd 100644 --- a/config/test.exs +++ b/config/test.exs @@ -39,8 +39,6 @@ # Reduce hash rounds for testing config :pbkdf2_elixir, rounds: 1 -#config :pleroma, :websub, Pleroma.Web.WebsubMock -#config :pleroma, :ostatus, Pleroma.Web.OStatusMock config :tesla, adapter: Tesla.Mock config :pleroma, :rich_media, enabled: false diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 6b0b75284..f4c9fe284 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -11,13 +11,11 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Federator.Publisher alias Pleroma.Web.Federator.RetryQueue + alias Pleroma.Web.OStatus alias Pleroma.Web.Websub require Logger - @websub Application.get_env(:pleroma, :websub) - @ostatus Application.get_env(:pleroma, :ostatus) - def init do # 1 minute Process.sleep(1000 * 60) @@ -87,12 +85,12 @@ def perform(:verify_websub, websub) do "Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})" end) - @websub.verify(websub) + Websub.verify(websub) end def perform(:incoming_doc, doc) do Logger.info("Got document, trying to parse") - @ostatus.handle_incoming(doc) + OStatus.handle_incoming(doc) end def perform(:incoming_ap_doc, params) do diff --git a/test/support/ostatus_mock.ex b/test/support/ostatus_mock.ex deleted file mode 100644 index 9c0f2f323..000000000 --- a/test/support/ostatus_mock.ex +++ /dev/null @@ -1,11 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OStatusMock do - import Pleroma.Factory - - def handle_incoming(_doc) do - insert(:note_activity) - end -end diff --git a/test/support/websub_mock.ex b/test/support/websub_mock.ex deleted file mode 100644 index e3d5a5b16..000000000 --- a/test/support/websub_mock.ex +++ /dev/null @@ -1,9 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.WebsubMock do - def verify(sub) do - {:ok, sub} - end -end diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs index bf2ee31ee..f79745d58 100644 --- a/test/web/websub/websub_controller_test.exs +++ b/test/web/websub/websub_controller_test.exs @@ -5,7 +5,6 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory - alias Pleroma.Activity alias Pleroma.Repo alias Pleroma.Web.Websub alias Pleroma.Web.Websub.WebsubClientSubscription From 79503ce90f6d85f00ee9e2dbc6358df2237d5036 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 26 May 2019 01:57:22 +0000 Subject: [PATCH 108/123] mrf: simple policy: fix matching imported activitypub and ostatus statuses --- CHANGELOG.md | 1 + .../web/activity_pub/mrf/simple_policy.ex | 3 +-- .../activity_pub/mrf/simple_policy_test.exs | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7869e299b..f689160e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -113,6 +113,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Exposing default scope of the user to anyone - Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`] - User-Agent is now sent correctly for all HTTP requests. +- MRF: Simple policy now properly delists imported or relayed statuses ## Removed - Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations` diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 890d70a7a..433d23c5f 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -74,8 +74,7 @@ defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do actor_host ), user <- User.get_cached_by_ap_id(object["actor"]), - true <- "https://www.w3.org/ns/activitystreams#Public" in object["to"], - true <- user.follower_address in object["cc"] do + true <- "https://www.w3.org/ns/activitystreams#Public" in object["to"] do to = List.delete(object["to"], "https://www.w3.org/ns/activitystreams#Public") ++ [user.follower_address] diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs index 3d1f26e60..0fd68e103 100644 --- a/test/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/web/activity_pub/mrf/simple_policy_test.exs @@ -145,6 +145,24 @@ test "has a matching host" do assert SimplePolicy.filter(local_message) == {:ok, local_message} end + + test "has a matching host but only as:Public in to" do + {_actor, ftl_message} = build_ftl_actor_and_message() + + ftl_message_actor_host = + ftl_message + |> Map.fetch!("actor") + |> URI.parse() + |> Map.fetch!(:host) + + ftl_message = Map.put(ftl_message, "cc", []) + + Config.put([:mrf_simple, :federated_timeline_removal], [ftl_message_actor_host]) + + assert {:ok, ftl_message} = SimplePolicy.filter(ftl_message) + refute "https://www.w3.org/ns/activitystreams#Public" in ftl_message["to"] + assert "https://www.w3.org/ns/activitystreams#Public" in ftl_message["cc"] + end end defp build_ftl_actor_and_message do From 9f3bcf0efe7fa06e2b8970386c099c1ea2974d0a Mon Sep 17 00:00:00 2001 From: jeff Date: Tue, 28 May 2019 06:49:53 +0000 Subject: [PATCH 109/123] Respect proxy settings federation --- lib/pleroma/http/connection.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 558005c19..c216cdcb1 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -32,9 +32,11 @@ def new(opts \\ []) do defp hackney_options(opts) do options = Keyword.get(opts, :adapter, []) adapter_options = Pleroma.Config.get([:http, :adapter], []) + proxy_url = Pleroma.Config.get([:http, :proxy_url], nil) @hackney_options |> Keyword.merge(adapter_options) |> Keyword.merge(options) + |> Keyword.merge(proxy: proxy_url) end end From abc15b6dcca39f62f043fcaf0292443a3dcb3d76 Mon Sep 17 00:00:00 2001 From: feld Date: Tue, 28 May 2019 21:20:24 +0000 Subject: [PATCH 110/123] Improve Varnish config. We set sane headers from the backend now. --- installation/pleroma.vcl | 70 ++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/installation/pleroma.vcl b/installation/pleroma.vcl index 92153d8ef..154747aa6 100644 --- a/installation/pleroma.vcl +++ b/installation/pleroma.vcl @@ -1,4 +1,4 @@ -vcl 4.0; +vcl 4.1; import std; backend default { @@ -35,24 +35,6 @@ sub vcl_recv { } return(purge); } - - # Pleroma MediaProxy - strip headers that will affect caching - if (req.url ~ "^/proxy/") { - unset req.http.Cookie; - unset req.http.Authorization; - unset req.http.Accept; - return (hash); - } - - # Strip headers that will affect caching from all other static content - # This also permits caching of individual toots and AP Activities - if ((req.url ~ "^/(media|static)/") || - (req.url ~ "(?i)\.(html|js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|mp4|ogg|webm|svg|swf|ttf|pdf|woff|woff2)$")) - { - unset req.http.Cookie; - unset req.http.Authorization; - return (hash); - } } sub vcl_backend_response { @@ -61,6 +43,12 @@ sub vcl_backend_response { set beresp.do_gzip = true; } + # Retry broken backend responses. + if (beresp.status == 503) { + set bereq.http.X-Varnish-Backend-503 = "1"; + return (retry); + } + # CHUNKED SUPPORT if (bereq.http.x-range ~ "bytes=" && beresp.status == 206) { set beresp.ttl = 10m; @@ -73,8 +61,6 @@ sub vcl_backend_response { return (deliver); } - # Default object caching of 86400s; - set beresp.ttl = 86400s; # Allow serving cached content for 6h in case backend goes down set beresp.grace = 6h; @@ -90,20 +76,6 @@ sub vcl_backend_response { set beresp.ttl = 30s; return (deliver); } - - # Pleroma MediaProxy internally sets headers properly - if (bereq.url ~ "^/proxy/") { - return (deliver); - } - - # Strip cache-restricting headers from Pleroma on static content that we want to cache - if (bereq.url ~ "(?i)\.(js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|mp4|ogg|webm|svg|swf|ttf|pdf|woff|woff2)$") - { - unset beresp.http.set-cookie; - unset beresp.http.Cache-Control; - unset beresp.http.x-request-id; - set beresp.http.Cache-Control = "public, max-age=86400"; - } } # The synthetic response for 301 redirects @@ -132,10 +104,32 @@ sub vcl_hash { } sub vcl_backend_fetch { + # Be more lenient for slow servers on the fediverse + if bereq.url ~ "^/proxy/" { + set bereq.first_byte_timeout = 300s; + } + # CHUNKED SUPPORT if (bereq.http.x-range) { set bereq.http.Range = bereq.http.x-range; } + + if (bereq.retries == 0) { + # Clean up the X-Varnish-Backend-503 flag that is used internally + # to mark broken backend responses that should be retried. + unset bereq.http.X-Varnish-Backend-503; + } else { + if (bereq.http.X-Varnish-Backend-503) { + if (bereq.method != "POST" && + std.healthy(bereq.backend) && + bereq.retries <= 4) { + # Flush broken backend response flag & try again. + unset bereq.http.X-Varnish-Backend-503; + } else { + return (abandon); + } + } + } } sub vcl_deliver { @@ -145,3 +139,9 @@ sub vcl_deliver { unset resp.http.CR; } } + +sub vcl_backend_error { + # Retry broken backend responses. + set bereq.http.X-Varnish-Backend-503 = "1"; + return (retry); +} From 0159a6dbe97330150d2913c7d7a060151f83f7eb Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 29 May 2019 10:58:45 +0000 Subject: [PATCH 111/123] router: require oauth_read for searching Search calls are generally expensive and allow unauthenticated users to crawl the instance for user profiles or posts which contain specified keywords. An adversary can build a distributed search engine which not only will consume significant instance resources, but also can be used for undesirable purposes such as datamining. Accordingly, require authenticated access to use the search API endpoints. This acts as a nice balance as it allows guest users to make use of most functionality available in Pleroma FE while ensuring that Pleroma instances are reasonably protected from resource exhaustion. It also removes Pleroma as a potential vector in distributed search engines. --- lib/pleroma/web/router.ex | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 352268b96..08c74a742 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -414,7 +414,12 @@ defmodule Pleroma.Web.Router do get("/trends", MastodonAPIController, :empty_array) - get("/accounts/search", MastodonAPIController, :account_search) + scope [] do + pipe_through(:oauth_read) + + get("/search", MastodonAPIController, :search) + get("/accounts/search", MastodonAPIController, :account_search) + end scope [] do pipe_through(:oauth_read_or_public) @@ -431,14 +436,12 @@ defmodule Pleroma.Web.Router do get("/accounts/:id/following", MastodonAPIController, :following) get("/accounts/:id", MastodonAPIController, :user) - get("/search", MastodonAPIController, :search) - get("/pleroma/accounts/:id/favourites", MastodonAPIController, :user_favourites) end end scope "/api/v2", Pleroma.Web.MastodonAPI do - pipe_through([:api, :oauth_read_or_public]) + pipe_through([:api, :oauth_read]) get("/search", MastodonAPIController, :search2) end @@ -480,9 +483,14 @@ defmodule Pleroma.Web.Router do get("/statuses/show/:id", TwitterAPI.Controller, :fetch_status) get("/statusnet/conversation/:id", TwitterAPI.Controller, :fetch_conversation) - get("/search", TwitterAPI.Controller, :search) get("/statusnet/tags/timeline/:tag", TwitterAPI.Controller, :public_and_external_timeline) end + + scope [] do + pipe_through(:oauth_read) + + get("/search", TwitterAPI.Controller, :search) + end end scope "/api", Pleroma.Web do @@ -500,7 +508,7 @@ defmodule Pleroma.Web.Router do end scope "/api", Pleroma.Web, as: :twitter_api_search do - pipe_through([:api, :oauth_read_or_public]) + pipe_through([:api, :oauth_read]) get("/pleroma/search_user", TwitterAPI.Controller, :search_user) end From 672fddb7213b69f3ee8b9a7728426373090847ed Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 29 May 2019 08:06:26 -0500 Subject: [PATCH 112/123] Default search limit should be 40 https://docs.joinmastodon.org/api/rest/search/ --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 0fe09c285..2110027c3 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1084,7 +1084,7 @@ def status_search(user, query) do from([a, o] in Activity.with_preloaded_object(Activity), where: fragment("?->>'type' = 'Create'", a.data), where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients, - limit: 20 + limit: 40 ) q = From db94294dfb6f25ce0f08acfb78b3c4cc2241e9f5 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 29 May 2019 14:04:58 +0000 Subject: [PATCH 113/123] [#936] fix tests --- .../admin_api/admin_api_controller_test.exs | 176 ++++++++++-------- 1 file changed, 96 insertions(+), 80 deletions(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index c15c67e31..43dcf945a 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -437,27 +437,31 @@ test "renders users array for the first page", %{conn: conn, admin: admin} do user = insert(:user, local: false, tags: ["foo", "bar"]) conn = get(conn, "/api/pleroma/admin/users?page=1") + users = + [ + %{ + "deactivated" => admin.info.deactivated, + "id" => admin.id, + "nickname" => admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "local" => true, + "tags" => [] + }, + %{ + "deactivated" => user.info.deactivated, + "id" => user.id, + "nickname" => user.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => false, + "tags" => ["foo", "bar"] + } + ] + |> Enum.sort_by(& &1["nickname"]) + assert json_response(conn, 200) == %{ "count" => 2, "page_size" => 50, - "users" => [ - %{ - "deactivated" => admin.info.deactivated, - "id" => admin.id, - "nickname" => admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => true, - "tags" => [] - }, - %{ - "deactivated" => user.info.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => false, - "tags" => ["foo", "bar"] - } - ] + "users" => users } end @@ -659,35 +663,39 @@ test "only local users with no query", %{admin: old_admin} do |> assign(:user, admin) |> get("/api/pleroma/admin/users?filters=local") + users = + [ + %{ + "deactivated" => user.info.deactivated, + "id" => user.id, + "nickname" => user.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => true, + "tags" => [] + }, + %{ + "deactivated" => admin.info.deactivated, + "id" => admin.id, + "nickname" => admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "local" => true, + "tags" => [] + }, + %{ + "deactivated" => false, + "id" => old_admin.id, + "local" => true, + "nickname" => old_admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "tags" => [] + } + ] + |> Enum.sort_by(& &1["nickname"]) + assert json_response(conn, 200) == %{ "count" => 3, "page_size" => 50, - "users" => [ - %{ - "deactivated" => user.info.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [] - }, - %{ - "deactivated" => admin.info.deactivated, - "id" => admin.id, - "nickname" => admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => true, - "tags" => [] - }, - %{ - "deactivated" => false, - "id" => old_admin.id, - "local" => true, - "nickname" => old_admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "tags" => [] - } - ] + "users" => users } end @@ -698,27 +706,31 @@ test "load only admins", %{conn: conn, admin: admin} do conn = get(conn, "/api/pleroma/admin/users?filters=is_admin") + users = + [ + %{ + "deactivated" => false, + "id" => admin.id, + "nickname" => admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "local" => admin.local, + "tags" => [] + }, + %{ + "deactivated" => false, + "id" => second_admin.id, + "nickname" => second_admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "local" => second_admin.local, + "tags" => [] + } + ] + |> Enum.sort_by(& &1["nickname"]) + assert json_response(conn, 200) == %{ "count" => 2, "page_size" => 50, - "users" => [ - %{ - "deactivated" => false, - "id" => admin.id, - "nickname" => admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => admin.local, - "tags" => [] - }, - %{ - "deactivated" => false, - "id" => second_admin.id, - "nickname" => second_admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => second_admin.local, - "tags" => [] - } - ] + "users" => users } end @@ -753,27 +765,31 @@ test "load users with tags list", %{conn: conn} do conn = get(conn, "/api/pleroma/admin/users?tags[]=first&tags[]=second") + users = + [ + %{ + "deactivated" => false, + "id" => user1.id, + "nickname" => user1.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => user1.local, + "tags" => ["first"] + }, + %{ + "deactivated" => false, + "id" => user2.id, + "nickname" => user2.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => user2.local, + "tags" => ["second"] + } + ] + |> Enum.sort_by(& &1["nickname"]) + assert json_response(conn, 200) == %{ "count" => 2, "page_size" => 50, - "users" => [ - %{ - "deactivated" => false, - "id" => user1.id, - "nickname" => user1.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => user1.local, - "tags" => ["first"] - }, - %{ - "deactivated" => false, - "id" => user2.id, - "nickname" => user2.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => user2.local, - "tags" => ["second"] - } - ] + "users" => users } end From 6aec0d1b58c12db2fa669fcff043338174290a35 Mon Sep 17 00:00:00 2001 From: kaniini Date: Wed, 29 May 2019 22:10:16 +0000 Subject: [PATCH 114/123] Revert "Merge branch 'feature/search-authenticated-only' into 'develop'" This reverts merge request !1209 --- lib/pleroma/web/router.ex | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 08c74a742..352268b96 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -414,12 +414,7 @@ defmodule Pleroma.Web.Router do get("/trends", MastodonAPIController, :empty_array) - scope [] do - pipe_through(:oauth_read) - - get("/search", MastodonAPIController, :search) - get("/accounts/search", MastodonAPIController, :account_search) - end + get("/accounts/search", MastodonAPIController, :account_search) scope [] do pipe_through(:oauth_read_or_public) @@ -436,12 +431,14 @@ defmodule Pleroma.Web.Router do get("/accounts/:id/following", MastodonAPIController, :following) get("/accounts/:id", MastodonAPIController, :user) + get("/search", MastodonAPIController, :search) + get("/pleroma/accounts/:id/favourites", MastodonAPIController, :user_favourites) end end scope "/api/v2", Pleroma.Web.MastodonAPI do - pipe_through([:api, :oauth_read]) + pipe_through([:api, :oauth_read_or_public]) get("/search", MastodonAPIController, :search2) end @@ -483,13 +480,8 @@ defmodule Pleroma.Web.Router do get("/statuses/show/:id", TwitterAPI.Controller, :fetch_status) get("/statusnet/conversation/:id", TwitterAPI.Controller, :fetch_conversation) - get("/statusnet/tags/timeline/:tag", TwitterAPI.Controller, :public_and_external_timeline) - end - - scope [] do - pipe_through(:oauth_read) - get("/search", TwitterAPI.Controller, :search) + get("/statusnet/tags/timeline/:tag", TwitterAPI.Controller, :public_and_external_timeline) end end @@ -508,7 +500,7 @@ defmodule Pleroma.Web.Router do end scope "/api", Pleroma.Web, as: :twitter_api_search do - pipe_through([:api, :oauth_read]) + pipe_through([:api, :oauth_read_or_public]) get("/pleroma/search_user", TwitterAPI.Controller, :search_user) end From 99f70c7e2011ab29746a8a61d3a4b354f4513045 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 30 May 2019 15:33:58 +0700 Subject: [PATCH 115/123] Use Pleroma.Config everywhere --- lib/healthcheck.ex | 4 +- lib/pleroma/emoji.ex | 4 +- lib/pleroma/html.ex | 20 ++---- lib/pleroma/http/http.ex | 7 +- lib/pleroma/plugs/federating_plug.ex | 2 +- lib/pleroma/reverse_proxy.ex | 2 +- lib/pleroma/user.ex | 4 +- lib/pleroma/web/activity_pub/activity_pub.ex | 12 ++-- .../activity_pub/activity_pub_controller.ex | 2 +- lib/pleroma/web/activity_pub/mrf.ex | 4 +- lib/pleroma/web/endpoint.ex | 2 +- lib/pleroma/web/media_proxy/media_proxy.ex | 34 +++++----- .../web/nodeinfo/nodeinfo_controller.ex | 66 ++++++++----------- lib/pleroma/web/templates/layout/app.html.eex | 4 +- .../mastodon_api/mastodon/index.html.eex | 2 +- .../web/twitter_api/twitter_api_controller.ex | 2 +- test/web/plugs/federating_plug_test.exs | 12 +--- .../twitter_api_controller_test.exs | 48 +++----------- 18 files changed, 80 insertions(+), 151 deletions(-) diff --git a/lib/healthcheck.ex b/lib/healthcheck.ex index 646fb3b9d..32aafc210 100644 --- a/lib/healthcheck.ex +++ b/lib/healthcheck.ex @@ -29,13 +29,13 @@ def system_info do end defp assign_db_info(healthcheck) do - database = Application.get_env(:pleroma, Repo)[:database] + database = Pleroma.Config.get([Repo, :database]) query = "select state, count(pid) from pg_stat_activity where datname = '#{database}' group by state;" result = Repo.query!(query) - pool_size = Application.get_env(:pleroma, Repo)[:pool_size] + pool_size = Pleroma.Config.get([Repo, :pool_size]) db_info = Enum.reduce(result.rows, %{active: 0, idle: 0}, fn [state, cnt], states -> diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 6390cce4c..7d12eff7f 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Emoji do @ets __MODULE__.Ets @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}] - @groups Application.get_env(:pleroma, :emoji)[:groups] + @groups Pleroma.Config.get([:emoji, :groups]) @doc false def start_link do @@ -112,7 +112,7 @@ defp load do # Compat thing for old custom emoji handling & default emoji, # it should run even if there are no emoji packs - shortcode_globs = Application.get_env(:pleroma, :emoji)[:shortcode_globs] || [] + shortcode_globs = Pleroma.Config.get([:emoji, :shortcode_globs], []) emojis = (load_from_file("config/emoji.txt") ++ diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index d1da746de..e5e78ee4f 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -104,7 +104,6 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do paragraphs, breaks and links are allowed through the filter. """ - @markup Application.get_env(:pleroma, :markup) @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], []) require HtmlSanitizeEx.Scrubber.Meta @@ -142,9 +141,7 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do Meta.allow_tag_with_these_attributes("span", []) # allow inline images for custom emoji - @allow_inline_images Keyword.get(@markup, :allow_inline_images) - - if @allow_inline_images do + if Pleroma.Config.get([:markup, :allow_inline_images]) do # restrict img tags to http/https only, because of MediaProxy. Meta.allow_tag_with_uri_attributes("img", ["src"], ["http", "https"]) @@ -168,7 +165,6 @@ defmodule Pleroma.HTML.Scrubber.Default do # credo:disable-for-previous-line # No idea how to fix this one… - @markup Application.get_env(:pleroma, :markup) @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], []) Meta.remove_cdata_sections_before_scrub() @@ -213,7 +209,7 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_this_attribute_values("span", "class", ["h-card"]) Meta.allow_tag_with_these_attributes("span", []) - @allow_inline_images Keyword.get(@markup, :allow_inline_images) + @allow_inline_images Pleroma.Config.get([:markup, :allow_inline_images]) if @allow_inline_images do # restrict img tags to http/https only, because of MediaProxy. @@ -228,9 +224,7 @@ defmodule Pleroma.HTML.Scrubber.Default do ]) end - @allow_tables Keyword.get(@markup, :allow_tables) - - if @allow_tables do + if Pleroma.Config.get([:markup, :allow_tables]) do Meta.allow_tag_with_these_attributes("table", []) Meta.allow_tag_with_these_attributes("tbody", []) Meta.allow_tag_with_these_attributes("td", []) @@ -239,9 +233,7 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes("tr", []) end - @allow_headings Keyword.get(@markup, :allow_headings) - - if @allow_headings do + if Pleroma.Config.get([:markup, :allow_headings]) do Meta.allow_tag_with_these_attributes("h1", []) Meta.allow_tag_with_these_attributes("h2", []) Meta.allow_tag_with_these_attributes("h3", []) @@ -249,9 +241,7 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes("h5", []) end - @allow_fonts Keyword.get(@markup, :allow_fonts) - - if @allow_fonts do + if Pleroma.Config.get([:markup, :allow_fonts]) do Meta.allow_tag_with_these_attributes("font", ["face"]) end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index c5f720bc9..c96ee7353 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -65,12 +65,9 @@ defp process_sni_options(options, url) do end def process_request_options(options) do - config = Application.get_env(:pleroma, :http, []) - proxy = Keyword.get(config, :proxy_url, nil) - - case proxy do + case Pleroma.Config.get([:http, :proxy_url]) do nil -> options - _ -> options ++ [proxy: proxy] + proxy -> options ++ [proxy: proxy] end end diff --git a/lib/pleroma/plugs/federating_plug.ex b/lib/pleroma/plugs/federating_plug.ex index effc154bf..4dc4e9279 100644 --- a/lib/pleroma/plugs/federating_plug.ex +++ b/lib/pleroma/plugs/federating_plug.ex @@ -10,7 +10,7 @@ def init(options) do end def call(conn, _opts) do - if Keyword.get(Application.get_env(:pleroma, :instance), :federating) do + if Pleroma.Config.get([:instance, :federating]) do conn else conn diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index 6e5feb4c3..983e156f5 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -61,7 +61,7 @@ defmodule Pleroma.ReverseProxy do * `http`: options for [hackney](https://github.com/benoitc/hackney). """ - @hackney Application.get_env(:pleroma, :hackney, :hackney) + @hackney Pleroma.Config.get(:hackney, :hackney) @default_hackney_options [] diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 653dec95f..5c91dc7d4 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -366,9 +366,7 @@ def follow_all(follower, followeds) do end def follow(%User{} = follower, %User{info: info} = followed) do - user_config = Application.get_env(:pleroma, :user) - deny_follow_blocked = Keyword.get(user_config, :deny_follow_blocked) - + deny_follow_blocked = Pleroma.Config.get([:user, :deny_follow_blocked]) ap_followers = followed.follower_address cond do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index aa0229db7..bdd7e78d2 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -399,16 +399,12 @@ def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ tru end def block(blocker, blocked, activity_id \\ nil, local \\ true) do - ap_config = Application.get_env(:pleroma, :activitypub) - unfollow_blocked = Keyword.get(ap_config, :unfollow_blocked) - outgoing_blocks = Keyword.get(ap_config, :outgoing_blocks) + outgoing_blocks = Pleroma.Config.get([:activitypub, :outgoing_blocks]) + unfollow_blocked = Pleroma.Config.get([:activitypub, :unfollow_blocked]) - with true <- unfollow_blocked do + if unfollow_blocked do follow_activity = fetch_latest_follow(blocker, blocked) - - if follow_activity do - unfollow(blocker, blocked, nil, local) - end + if follow_activity, do: unfollow(blocker, blocked, nil, local) end with true <- outgoing_blocks, diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index ad2ca1e54..0182bda46 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do plug(:relay_active? when action in [:relay]) def relay_active?(conn, _) do - if Keyword.get(Application.get_env(:pleroma, :instance), :allow_relay) do + if Pleroma.Config.get([:instance, :allow_relay]) do conn else conn diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 1aaa20050..3bf7955f3 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -17,9 +17,7 @@ def filter(object) do end def get_policies do - Application.get_env(:pleroma, :instance, []) - |> Keyword.get(:rewrite_policy, []) - |> get_policies() + Pleroma.Config.get([:instance, :rewrite_policy], []) |> get_policies() end defp get_policies(policy) when is_atom(policy), do: [policy] diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 8cd7a2270..bd76e4295 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -66,7 +66,7 @@ defmodule Pleroma.Web.Endpoint do parsers: [:urlencoded, :multipart, :json], pass: ["*/*"], json_decoder: Jason, - length: Application.get_env(:pleroma, :instance) |> Keyword.get(:upload_limit), + length: Pleroma.Config.get([:instance, :upload_limit]), body_reader: {Pleroma.Web.Plugs.DigestPlug, :read_body, []} ) diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index 5762e767b..cee6d8481 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -12,25 +12,27 @@ def url(""), do: nil def url("/" <> _ = url), do: url def url(url) do - config = Application.get_env(:pleroma, :media_proxy, []) - domain = URI.parse(url).host - - cond do - !Keyword.get(config, :enabled, false) or String.starts_with?(url, Pleroma.Web.base_url()) -> - url - - Enum.any?(Pleroma.Config.get([:media_proxy, :whitelist]), fn pattern -> - String.equivalent?(domain, pattern) - end) -> - url - - true -> - encode_url(url) + if !enabled?() or local?(url) or whitelisted?(url) do + url + else + encode_url(url) end end + defp enabled?, do: Pleroma.Config.get([:media_proxy, :enabled], false) + + defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url()) + + defp whitelisted?(url) do + %{host: domain} = URI.parse(url) + + Enum.any?(Pleroma.Config.get([:media_proxy, :whitelist]), fn pattern -> + String.equivalent?(domain, pattern) + end) + end + def encode_url(url) do - secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base] + secret = Pleroma.Config.get([Pleroma.Web.Endpoint, :secret_key_base]) # Must preserve `%2F` for compatibility with S3 # https://git.pleroma.social/pleroma/pleroma/issues/580 @@ -52,7 +54,7 @@ def encode_url(url) do end def decode_url(sig, url) do - secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base] + secret = Pleroma.Config.get([Pleroma.Web.Endpoint, :secret_key_base]) sig = Base.url_decode64!(sig, @base64_opts) local_sig = :crypto.hmac(:sha, secret, url) diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 45f90c579..59f3d4e11 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -32,20 +32,15 @@ def schemas(conn, _params) do # returns a nodeinfo 2.0 map, since 2.1 just adds a repository field # under software. def raw_nodeinfo do - instance = Application.get_env(:pleroma, :instance) - media_proxy = Application.get_env(:pleroma, :media_proxy) - suggestions = Application.get_env(:pleroma, :suggestions) - chat = Application.get_env(:pleroma, :chat) - gopher = Application.get_env(:pleroma, :gopher) stats = Stats.get_stats() mrf_simple = - Application.get_env(:pleroma, :mrf_simple) + Config.get(:mrf_simple) |> Enum.into(%{}) # This horror is needed to convert regex sigils to strings mrf_keyword = - Application.get_env(:pleroma, :mrf_keyword, []) + Config.get(:mrf_keyword, []) |> Enum.map(fn {key, value} -> {key, Enum.map(value, fn @@ -74,14 +69,7 @@ def raw_nodeinfo do MRF.get_policies() |> Enum.map(fn policy -> to_string(policy) |> String.split(".") |> List.last() end) - quarantined = Keyword.get(instance, :quarantined_instances) - - quarantined = - if is_list(quarantined) do - quarantined - else - [] - end + quarantined = Config.get([:instance, :quarantined_instances], []) staff_accounts = User.all_superusers() @@ -92,7 +80,7 @@ def raw_nodeinfo do |> Enum.into(%{}, fn {k, v} -> {k, length(v)} end) federation_response = - if Keyword.get(instance, :mrf_transparency) do + if Config.get([:instance, :mrf_transparency]) do %{ mrf_policies: mrf_policies, mrf_simple: mrf_simple, @@ -109,22 +97,22 @@ def raw_nodeinfo do "pleroma_api", "mastodon_api", "mastodon_api_streaming", - if Keyword.get(media_proxy, :enabled) do + if Config.get([:media_proxy, :enabled]) do "media_proxy" end, - if Keyword.get(gopher, :enabled) do + if Config.get([:gopher, :enabled]) do "gopher" end, - if Keyword.get(chat, :enabled) do + if Config.get([:chat, :enabled]) do "chat" end, - if Keyword.get(suggestions, :enabled) do + if Config.get([:suggestions, :enabled]) do "suggestions" end, - if Keyword.get(instance, :allow_relay) do + if Config.get([:instance, :allow_relay]) do "relay" end, - if Keyword.get(instance, :safe_dm_mentions) do + if Config.get([:instance, :safe_dm_mentions]) do "safe_dm_mentions" end ] @@ -141,7 +129,7 @@ def raw_nodeinfo do inbound: [], outbound: [] }, - openRegistrations: Keyword.get(instance, :registrations_open), + openRegistrations: Config.get([:instance, :registrations_open]), usage: %{ users: %{ total: stats.user_count || 0 @@ -149,29 +137,29 @@ def raw_nodeinfo do localPosts: stats.status_count || 0 }, metadata: %{ - nodeName: Keyword.get(instance, :name), - nodeDescription: Keyword.get(instance, :description), - private: !Keyword.get(instance, :public, true), + nodeName: Config.get([:instance, :name]), + nodeDescription: Config.get([:instance, :description]), + private: !Config.get([:instance, :public], true), suggestions: %{ - enabled: Keyword.get(suggestions, :enabled, false), - thirdPartyEngine: Keyword.get(suggestions, :third_party_engine, ""), - timeout: Keyword.get(suggestions, :timeout, 5000), - limit: Keyword.get(suggestions, :limit, 23), - web: Keyword.get(suggestions, :web, "") + enabled: Config.get([:suggestions, :enabled], false), + thirdPartyEngine: Config.get([:suggestions, :third_party_engine], ""), + timeout: Config.get([:suggestions, :timeout], 5000), + limit: Config.get([:suggestions, :limit], 23), + web: Config.get([:suggestions, :web], "") }, staffAccounts: staff_accounts, federation: federation_response, - postFormats: Keyword.get(instance, :allowed_post_formats), + postFormats: Config.get([:instance, :allowed_post_formats]), uploadLimits: %{ - general: Keyword.get(instance, :upload_limit), - avatar: Keyword.get(instance, :avatar_upload_limit), - banner: Keyword.get(instance, :banner_upload_limit), - background: Keyword.get(instance, :background_upload_limit) + general: Config.get([:instance, :upload_limit]), + avatar: Config.get([:instance, :avatar_upload_limit]), + banner: Config.get([:instance, :banner_upload_limit]), + background: Config.get([:instance, :background_upload_limit]) }, - accountActivationRequired: Keyword.get(instance, :account_activation_required, false), - invitesEnabled: Keyword.get(instance, :invites_enabled, false), + accountActivationRequired: Config.get([:instance, :account_activation_required], false), + invitesEnabled: Config.get([:instance, :invites_enabled], false), features: features, - restrictedNicknames: Pleroma.Config.get([Pleroma.User, :restricted_nicknames]) + restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames]) } } end diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index 3389c91cc..85ec4d76c 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -4,7 +4,7 @@ - <%= Application.get_env(:pleroma, :instance)[:name] %> + <%= Pleroma.Config.get([:instance, :name]) %>