alyssa
/
pleroma
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into remake-remodel-dms

This commit is contained in:
lain 2020-04-06 17:50:31 +02:00
parent fd97b0e634 1d4ab06505
commit 19335be24c
266 changed files with 7618 additions and 2416 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -12,6 +12,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Added
- NodeInfo: `pleroma:api/v1/notifications:include_types_filter` to the `features` list.
- NodeInfo: `pleroma_emoji_reactions` to the `features` list.
- Configuration: `:restrict_unauthenticated` setting, restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses.
- New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires minimum OTP version of 22.2 otherwise Pleroma won’t start. For hackney OTP update is not required.
<details>
<summary>API Changes</summary>
- Mastodon API: Support for `include_types` in `/api/v1/notifications`.
@ -77,6 +80,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Mastodon API: User timelines will now respect blocks, unless you are getting the user timeline of somebody you blocked (which would be empty otherwise).
- Mastodon API: Favoriting / Repeating a post multiple times will now return the identical response every time. Before, executing that action twice would return an error ("already favorited") on the second try.
- Mastodon API: Limit timeline requests to 3 per timeline per 500ms per user/ip by default.
- Admin API: `PATCH /api/pleroma/admin/users/:nickname/credentials` and `GET /api/pleroma/admin/users/:nickname/credentials`
</details>
### Added

4
Dockerfile
View File

@ -12,7 +12,7 @@ RUN apk add git gcc g++ musl-dev make &&\
mkdir release &&\
mix release --path release
FROM alpine:3.9
FROM alpine:3.11
ARG BUILD_DATE
ARG VCS_REF
@ -33,7 +33,7 @@ ARG DATA=/var/lib/pleroma
RUN echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\
apk update &&\
apk add ncurses postgresql-client &&\
apk add imagemagick ncurses postgresql-client &&\
adduser --system --shell /bin/false --home ${HOME} pleroma &&\
mkdir -p ${DATA}/uploads &&\
mkdir -p ${DATA}/static &&\

557
benchmarks/load_testing/activities.ex Normal file
View File

@ -0,0 +1,557 @@
defmodule Pleroma.LoadTesting.Activities do
@moduledoc """
Module for generating different activities.
"""
import Ecto.Query
import Pleroma.LoadTesting.Helper, only: [to_sec: 1]
alias Ecto.UUID
alias Pleroma.Constants
alias Pleroma.LoadTesting.Users
alias Pleroma.Repo
alias Pleroma.Web.CommonAPI
require Constants
@defaults [
iterations: 170,
friends_used: 20,
non_friends_used: 20
]
@max_concurrency 10
@visibility ~w(public private direct unlisted)
@types ~w(simple emoji mentions hell_thread attachment tag like reblog simple_thread remote)
@groups ~w(user friends non_friends)
@spec generate(User.t(), keyword()) :: :ok
def generate(user, opts \\ []) do
{:ok, _} =
Agent.start_link(fn -> %{} end,
name: :benchmark_state
)
opts = Keyword.merge(@defaults, opts)
friends =
user
|> Users.get_users(limit: opts[:friends_used], local: :local, friends?: true)
|> Enum.shuffle()
non_friends =
user
|> Users.get_users(limit: opts[:non_friends_used], local: :local, friends?: false)
|> Enum.shuffle()
task_data =
for visibility <- @visibility,
type <- @types,
group <- @groups,
do: {visibility, type, group}
IO.puts("Starting generating #{opts[:iterations]} iterations of activities...")
friends_thread = Enum.take(friends, 5)
non_friends_thread = Enum.take(friends, 5)
public_long_thread = fn ->
generate_long_thread("public", user, friends_thread, non_friends_thread, opts)
end
private_long_thread = fn ->
generate_long_thread("private", user, friends_thread, non_friends_thread, opts)
end
iterations = opts[:iterations]
{time, _} =
:timer.tc(fn ->
Enum.each(
1..iterations,
fn
i when i == iterations - 2 ->
spawn(public_long_thread)
spawn(private_long_thread)
generate_activities(user, friends, non_friends, Enum.shuffle(task_data), opts)
_ ->
generate_activities(user, friends, non_friends, Enum.shuffle(task_data), opts)
end
)
end)
IO.puts("Generating iterations of activities took #{to_sec(time)} sec.\n")
:ok
end
def generate_power_intervals(opts \\ []) do
count = Keyword.get(opts, :count, 20)
power = Keyword.get(opts, :power, 2)
IO.puts("Generating #{count} intervals for a power #{power} series...")
counts = Enum.map(1..count, fn n -> :math.pow(n, power) end)
sum = Enum.sum(counts)
densities =
Enum.map(counts, fn c ->
c / sum
end)
densities
|> Enum.reduce(0, fn density, acc ->
if acc == 0 do
[{0, density}]
else
[{_, lower} | _] = acc
[{lower, lower + density} | acc]
end
end)
|> Enum.reverse()
end
def generate_tagged_activities(opts \\ []) do
tag_count = Keyword.get(opts, :tag_count, 20)
users = Keyword.get(opts, :users, Repo.all(Pleroma.User))
activity_count = Keyword.get(opts, :count, 200_000)
intervals = generate_power_intervals(count: tag_count)
IO.puts(
"Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0"
)
Enum.each(1..activity_count, fn _ ->
random = :rand.uniform()
i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end)
CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"})
end)
end
defp generate_long_thread(visibility, user, friends, non_friends, _opts) do
group =
if visibility == "public",
do: "friends",
else: "user"
tasks = get_reply_tasks(visibility, group) |> Stream.cycle() |> Enum.take(50)
{:ok, activity} =
CommonAPI.post(user, %{
"status" => "Start of #{visibility} long thread",
"visibility" => visibility
})
Agent.update(:benchmark_state, fn state ->
key =
if visibility == "public",
do: :public_thread,
else: :private_thread
Map.put(state, key, activity)
end)
acc = {activity.id, ["@" <> user.nickname, "reply to long thread"]}
insert_replies_for_long_thread(tasks, visibility, user, friends, non_friends, acc)
IO.puts("Generating #{visibility} long thread ended\n")
end
defp insert_replies_for_long_thread(tasks, visibility, user, friends, non_friends, acc) do
Enum.reduce(tasks, acc, fn
"friend", {id, data} ->
friend = Enum.random(friends)
insert_reply(friend, List.delete(data, "@" <> friend.nickname), id, visibility)
"non_friend", {id, data} ->
non_friend = Enum.random(non_friends)
insert_reply(non_friend, List.delete(data, "@" <> non_friend.nickname), id, visibility)
"user", {id, data} ->
insert_reply(user, List.delete(data, "@" <> user.nickname), id, visibility)
end)
end
defp generate_activities(user, friends, non_friends, task_data, opts) do
Task.async_stream(
task_data,
fn {visibility, type, group} ->
insert_activity(type, visibility, group, user, friends, non_friends, opts)
end,
max_concurrency: @max_concurrency,
timeout: 30_000
)
|> Stream.run()
end
defp insert_activity("simple", visibility, group, user, friends, non_friends, _opts) do
{:ok, _activity} =
group
|> get_actor(user, friends, non_friends)
|> CommonAPI.post(%{"status" => "Simple status", "visibility" => visibility})
end
defp insert_activity("emoji", visibility, group, user, friends, non_friends, _opts) do
{:ok, _activity} =
group
|> get_actor(user, friends, non_friends)
|> CommonAPI.post(%{
"status" => "Simple status with emoji :firefox:",
"visibility" => visibility
})
end
defp insert_activity("mentions", visibility, group, user, friends, non_friends, _opts) do
user_mentions =
get_random_mentions(friends, Enum.random(0..3)) ++
get_random_mentions(non_friends, Enum.random(0..3))
user_mentions =
if Enum.random([true, false]),
do: ["@" <> user.nickname | user_mentions],
else: user_mentions
{:ok, _activity} =
group
|> get_actor(user, friends, non_friends)
|> CommonAPI.post(%{
"status" => Enum.join(user_mentions, ", ") <> " simple status with mentions",
"visibility" => visibility
})
end
defp insert_activity("hell_thread", visibility, group, user, friends, non_friends, _opts) do
mentions =
with {:ok, nil} <- Cachex.get(:user_cache, "hell_thread_mentions") do
cached =
([user | Enum.take(friends, 10)] ++ Enum.take(non_friends, 10))
|> Enum.map(&"@#{&1.nickname}")
|> Enum.join(", ")
Cachex.put(:user_cache, "hell_thread_mentions", cached)
cached
else
{:ok, cached} -> cached
end
{:ok, _activity} =
group
|> get_actor(user, friends, non_friends)
|> CommonAPI.post(%{
"status" => mentions <> " hell thread status",
"visibility" => visibility
})
end
defp insert_activity("attachment", visibility, group, user, friends, non_friends, _opts) do
actor = get_actor(group, user, friends, non_friends)
obj_data = %{
"actor" => actor.ap_id,
"name" => "4467-11.jpg",
"type" => "Document",
"url" => [
%{
"href" =>
"#{Pleroma.Web.base_url()}/media/b1b873552422a07bf53af01f3c231c841db4dfc42c35efde681abaf0f2a4eab7.jpg",
"mediaType" => "image/jpeg",
"type" => "Link"
}
]
}
object = Repo.insert!(%Pleroma.Object{data: obj_data})
{:ok, _activity} =
CommonAPI.post(actor, %{
"status" => "Post with attachment",
"visibility" => visibility,
"media_ids" => [object.id]
})
end
defp insert_activity("tag", visibility, group, user, friends, non_friends, _opts) do
{:ok, _activity} =
group
|> get_actor(user, friends, non_friends)
|> CommonAPI.post(%{"status" => "Status with #tag", "visibility" => visibility})
end
defp insert_activity("like", visibility, group, user, friends, non_friends, opts) do
actor = get_actor(group, user, friends, non_friends)
with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(),
{:ok, _activity, _object} <- CommonAPI.favorite(activity_id, actor) do
:ok
else
{:error, _} ->
insert_activity("like", visibility, group, user, friends, non_friends, opts)
nil ->
Process.sleep(15)
insert_activity("like", visibility, group, user, friends, non_friends, opts)
end
end
defp insert_activity("reblog", visibility, group, user, friends, non_friends, opts) do
actor = get_actor(group, user, friends, non_friends)
with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(),
{:ok, _activity, _object} <- CommonAPI.repeat(activity_id, actor) do
:ok
else
{:error, _} ->
insert_activity("reblog", visibility, group, user, friends, non_friends, opts)
nil ->
Process.sleep(15)
insert_activity("reblog", visibility, group, user, friends, non_friends, opts)
end
end
defp insert_activity("simple_thread", visibility, group, user, friends, non_friends, _opts)
when visibility in ["public", "unlisted", "private"] do
actor = get_actor(group, user, friends, non_friends)
tasks = get_reply_tasks(visibility, group)
{:ok, activity} =
CommonAPI.post(user, %{"status" => "Simple status", "visibility" => "unlisted"})
acc = {activity.id, ["@" <> actor.nickname, "reply to status"]}
insert_replies(tasks, visibility, user, friends, non_friends, acc)
end
defp insert_activity("simple_thread", "direct", group, user, friends, non_friends, _opts) do
actor = get_actor(group, user, friends, non_friends)
tasks = get_reply_tasks("direct", group)
list =
case group do
"non_friends" ->
Enum.take(non_friends, 3)
_ ->
Enum.take(friends, 3)
end
data = Enum.map(list, &("@" <> &1.nickname))
{:ok, activity} =
CommonAPI.post(actor, %{
"status" => Enum.join(data, ", ") <> "simple status",
"visibility" => "direct"
})
acc = {activity.id, ["@" <> user.nickname | data] ++ ["reply to status"]}
insert_direct_replies(tasks, user, list, acc)
end
defp insert_activity("remote", _, "user", _, _, _, _), do: :ok
defp insert_activity("remote", visibility, group, user, _friends, _non_friends, opts) do
remote_friends =
Users.get_users(user, limit: opts[:friends_used], local: :external, friends?: true)
remote_non_friends =
Users.get_users(user, limit: opts[:non_friends_used], local: :external, friends?: false)
actor = get_actor(group, user, remote_friends, remote_non_friends)
{act_data, obj_data} = prepare_activity_data(actor, visibility, user)
{activity_data, object_data} = other_data(actor)
activity_data
|> Map.merge(act_data)
|> Map.put("object", Map.merge(object_data, obj_data))
|> Pleroma.Web.ActivityPub.ActivityPub.insert(false)
end
defp get_actor("user", user, _friends, _non_friends), do: user
defp get_actor("friends", _user, friends, _non_friends), do: Enum.random(friends)
defp get_actor("non_friends", _user, _friends, non_friends), do: Enum.random(non_friends)
defp other_data(actor) do
%{host: host} = URI.parse(actor.ap_id)
datetime = DateTime.utc_now()
context_id = "http://#{host}:4000/contexts/#{UUID.generate()}"
activity_id = "http://#{host}:4000/activities/#{UUID.generate()}"
object_id = "http://#{host}:4000/objects/#{UUID.generate()}"
activity_data = %{
"actor" => actor.ap_id,
"context" => context_id,
"id" => activity_id,
"published" => datetime,
"type" => "Create",
"directMessage" => false
}
object_data = %{
"actor" => actor.ap_id,
"attachment" => [],
"attributedTo" => actor.ap_id,
"bcc" => [],
"bto" => [],
"content" => "Remote post",
"context" => context_id,
"conversation" => context_id,
"emoji" => %{},
"id" => object_id,
"published" => datetime,
"sensitive" => false,
"summary" => "",
"tag" => [],
"to" => ["https://www.w3.org/ns/activitystreams#Public"],
"type" => "Note"
}
{activity_data, object_data}
end
defp prepare_activity_data(actor, "public", _mention) do
obj_data = %{
"cc" => [actor.follower_address],
"to" => [Constants.as_public()]
}
act_data = %{
"cc" => [actor.follower_address],
"to" => [Constants.as_public()]
}
{act_data, obj_data}
end
defp prepare_activity_data(actor, "private", _mention) do
obj_data = %{
"cc" => [],
"to" => [actor.follower_address]
}
act_data = %{
"cc" => [],
"to" => [actor.follower_address]
}
{act_data, obj_data}
end
defp prepare_activity_data(actor, "unlisted", _mention) do
obj_data = %{
"cc" => [Constants.as_public()],
"to" => [actor.follower_address]
}
act_data = %{
"cc" => [Constants.as_public()],
"to" => [actor.follower_address]
}
{act_data, obj_data}
end
defp prepare_activity_data(_actor, "direct", mention) do
%{host: mentioned_host} = URI.parse(mention.ap_id)
obj_data = %{
"cc" => [],
"content" =>
"<span class=\"h-card\"><a class=\"u-url mention\" href=\"#{mention.ap_id}\" rel=\"ugc\">@<span>#{
mention.nickname
}</span></a></span> direct message",
"tag" => [
%{
"href" => mention.ap_id,
"name" => "@#{mention.nickname}@#{mentioned_host}",
"type" => "Mention"
}
],
"to" => [mention.ap_id]
}
act_data = %{
"cc" => [],
"directMessage" => true,
"to" => [mention.ap_id]
}
{act_data, obj_data}
end
defp get_reply_tasks("public", "user"), do: ~w(friend non_friend user)
defp get_reply_tasks("public", "friends"), do: ~w(non_friend user friend)
defp get_reply_tasks("public", "non_friends"), do: ~w(user friend non_friend)
defp get_reply_tasks(visibility, "user") when visibility in ["unlisted", "private"],
do: ~w(friend user friend)
defp get_reply_tasks(visibility, "friends") when visibility in ["unlisted", "private"],
do: ~w(user friend user)
defp get_reply_tasks(visibility, "non_friends") when visibility in ["unlisted", "private"],
do: []
defp get_reply_tasks("direct", "user"), do: ~w(friend user friend)
defp get_reply_tasks("direct", "friends"), do: ~w(user friend user)
defp get_reply_tasks("direct", "non_friends"), do: ~w(user non_friend user)
defp insert_replies(tasks, visibility, user, friends, non_friends, acc) do
Enum.reduce(tasks, acc, fn
"friend", {id, data} ->
friend = Enum.random(friends)
insert_reply(friend, data, id, visibility)
"non_friend", {id, data} ->
non_friend = Enum.random(non_friends)
insert_reply(non_friend, data, id, visibility)
"user", {id, data} ->
insert_reply(user, data, id, visibility)
end)
end
defp insert_direct_replies(tasks, user, list, acc) do
Enum.reduce(tasks, acc, fn
group, {id, data} when group in ["friend", "non_friend"] ->
actor = Enum.random(list)
{reply_id, _} =
insert_reply(actor, List.delete(data, "@" <> actor.nickname), id, "direct")
{reply_id, data}
"user", {id, data} ->
{reply_id, _} = insert_reply(user, List.delete(data, "@" <> user.nickname), id, "direct")
{reply_id, data}
end)
end
defp insert_reply(actor, data, activity_id, visibility) do
{:ok, reply} =
CommonAPI.post(actor, %{
"status" => Enum.join(data, ", "),
"visibility" => visibility,
"in_reply_to_status_id" => activity_id
})
{reply.id, ["@" <> actor.nickname | data]}
end
defp get_random_mentions(_users, count) when count == 0, do: []
defp get_random_mentions(users, count) do
users
|> Enum.shuffle()
|> Enum.take(count)
|> Enum.map(&"@#{&1.nickname}")
end
defp get_random_create_activity_id do
Repo.one(
from(a in Pleroma.Activity,
where: fragment("(?)->>'type' = ?", a.data, ^"Create"),
order_by: fragment("RANDOM()"),
limit: 1,
select: a.id
)
)
end
end

661
benchmarks/load_testing/fetcher.ex
View File

@ -1,260 +1,489 @@
defmodule Pleroma.LoadTesting.Fetcher do
use Pleroma.LoadTesting.Helper
alias Pleroma.Activity
alias Pleroma.Pagination
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.MastodonAPI.MastodonAPI
alias Pleroma.Web.MastodonAPI.StatusView
def fetch_user(user) do
Benchee.run(%{
@spec run_benchmarks(User.t()) :: any()
def run_benchmarks(user) do
fetch_user(user)
fetch_timelines(user)
render_views(user)
end
defp formatters do
[
Benchee.Formatters.Console
]
end
defp fetch_user(user) do
Benchee.run(
%{
"By id" => fn -> Repo.get_by(User, id: user.id) end,
"By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end,
"By email" => fn -> Repo.get_by(User, email: user.email) end,
"By nickname" => fn -> Repo.get_by(User, nickname: user.nickname) end
})
},
formatters: formatters()
)
end
def query_timelines(user) do
home_timeline_params = %{
"count" => 20,
"with_muted" => true,
"type" => ["Create", "Announce"],
defp fetch_timelines(user) do
fetch_home_timeline(user)
fetch_direct_timeline(user)
fetch_public_timeline(user)
fetch_public_timeline(user, :local)
fetch_public_timeline(user, :tag)
fetch_notifications(user)
fetch_favourites(user)
fetch_long_thread(user)
end
defp render_views(user) do
render_timelines(user)
render_long_thread(user)
end
defp opts_for_home_timeline(user) do
%{
"blocking_user" => user,
"count" => "20",
"muting_user" => user,
"type" => ["Create", "Announce"],
"user" => user,
"with_muted" => "true"
}
end
defp fetch_home_timeline(user) do
opts = opts_for_home_timeline(user)
recipients = [user.ap_id | User.following(user)]
first_page_last =
ActivityPub.fetch_activities(recipients, opts) |> Enum.reverse() |> List.last()
second_page_last =
ActivityPub.fetch_activities(recipients, Map.put(opts, "max_id", first_page_last.id))
|> Enum.reverse()
|> List.last()
third_page_last =
ActivityPub.fetch_activities(recipients, Map.put(opts, "max_id", second_page_last.id))
|> Enum.reverse()
|> List.last()
forth_page_last =
ActivityPub.fetch_activities(recipients, Map.put(opts, "max_id", third_page_last.id))
|> Enum.reverse()
|> List.last()
Benchee.run(
%{
"home timeline" => fn opts -> ActivityPub.fetch_activities(recipients, opts) end
},
inputs: %{
"1 page" => opts,
"2 page" => Map.put(opts, "max_id", first_page_last.id),
"3 page" => Map.put(opts, "max_id", second_page_last.id),
"4 page" => Map.put(opts, "max_id", third_page_last.id),
"5 page" => Map.put(opts, "max_id", forth_page_last.id),
"1 page only media" => Map.put(opts, "only_media", "true"),
"2 page only media" =>
Map.put(opts, "max_id", first_page_last.id) |> Map.put("only_media", "true"),
"3 page only media" =>
Map.put(opts, "max_id", second_page_last.id) |> Map.put("only_media", "true"),
"4 page only media" =>
Map.put(opts, "max_id", third_page_last.id) |> Map.put("only_media", "true"),
"5 page only media" =>
Map.put(opts, "max_id", forth_page_last.id) |> Map.put("only_media", "true")
},
formatters: formatters()
)
end
defp opts_for_direct_timeline(user) do
%{
:visibility => "direct",
"blocking_user" => user,
"count" => "20",
"type" => "Create",
"user" => user,
"with_muted" => "true"
}
end
defp fetch_direct_timeline(user) do
recipients = [user.ap_id]
opts = opts_for_direct_timeline(user)
first_page_last =
recipients
|> ActivityPub.fetch_activities_query(opts)
|> Pagination.fetch_paginated(opts)
|> List.last()
opts2 = Map.put(opts, "max_id", first_page_last.id)
second_page_last =
recipients
|> ActivityPub.fetch_activities_query(opts2)
|> Pagination.fetch_paginated(opts2)
|> List.last()
opts3 = Map.put(opts, "max_id", second_page_last.id)
third_page_last =
recipients
|> ActivityPub.fetch_activities_query(opts3)
|> Pagination.fetch_paginated(opts3)
|> List.last()
opts4 = Map.put(opts, "max_id", third_page_last.id)
forth_page_last =
recipients
|> ActivityPub.fetch_activities_query(opts4)
|> Pagination.fetch_paginated(opts4)
|> List.last()
Benchee.run(
%{
"direct timeline" => fn opts ->
ActivityPub.fetch_activities_query(recipients, opts) |> Pagination.fetch_paginated(opts)
end
},
inputs: %{
"1 page" => opts,
"2 page" => opts2,
"3 page" => opts3,
"4 page" => opts4,
"5 page" => Map.put(opts4, "max_id", forth_page_last.id)
},
formatters: formatters()
)
end
defp opts_for_public_timeline(user) do
%{
"type" => ["Create", "Announce"],
"local_only" => false,
"blocking_user" => user,
"muting_user" => user
}
end
defp opts_for_public_timeline(user, :local) do
%{
"type" => ["Create", "Announce"],
"local_only" => true,
"blocking_user" => user,
"muting_user" => user
}
end
defp opts_for_public_timeline(user, :tag) do
%{
"blocking_user" => user,
"count" => "20",
"local_only" => nil,
"muting_user" => user,
"tag" => ["tag"],
"tag_all" => [],
"tag_reject" => [],
"type" => "Create",
"user" => user,
"with_muted" => "true"
}
end
defp fetch_public_timeline(user) do
opts = opts_for_public_timeline(user)
fetch_public_timeline(opts, "public timeline")
end
defp fetch_public_timeline(user, :local) do
opts = opts_for_public_timeline(user, :local)
fetch_public_timeline(opts, "public timeline only local")
end
defp fetch_public_timeline(user, :tag) do
opts = opts_for_public_timeline(user, :tag)
fetch_public_timeline(opts, "hashtag timeline")
end
defp fetch_public_timeline(user, :only_media) do
opts = opts_for_public_timeline(user) |> Map.put("only_media", "true")
fetch_public_timeline(opts, "public timeline only media")
end
defp fetch_public_timeline(opts, title) when is_binary(title) do
first_page_last = ActivityPub.fetch_public_activities(opts) |> List.last()
second_page_last =
ActivityPub.fetch_public_activities(Map.put(opts, "max_id", first_page_last.id))
|> List.last()
third_page_last =
ActivityPub.fetch_public_activities(Map.put(opts, "max_id", second_page_last.id))
|> List.last()
forth_page_last =
ActivityPub.fetch_public_activities(Map.put(opts, "max_id", third_page_last.id))
|> List.last()
Benchee.run(
%{
title => fn opts ->
ActivityPub.fetch_public_activities(opts)
end
},
inputs: %{
"1 page" => opts,
"2 page" => Map.put(opts, "max_id", first_page_last.id),
"3 page" => Map.put(opts, "max_id", second_page_last.id),
"4 page" => Map.put(opts, "max_id", third_page_last.id),
"5 page" => Map.put(opts, "max_id", forth_page_last.id)
},
formatters: formatters()
)
end
defp opts_for_notifications do
%{"count" => "20", "with_muted" => "true"}
end
defp fetch_notifications(user) do
opts = opts_for_notifications()
first_page_last = MastodonAPI.get_notifications(user, opts) |> List.last()
second_page_last =
MastodonAPI.get_notifications(user, Map.put(opts, "max_id", first_page_last.id))
|> List.last()
third_page_last =
MastodonAPI.get_notifications(user, Map.put(opts, "max_id", second_page_last.id))
|> List.last()
forth_page_last =
MastodonAPI.get_notifications(user, Map.put(opts, "max_id", third_page_last.id))
|> List.last()
Benchee.run(
%{
"Notifications" => fn opts ->
MastodonAPI.get_notifications(user, opts)
end
},
inputs: %{
"1 page" => opts,
"2 page" => Map.put(opts, "max_id", first_page_last.id),
"3 page" => Map.put(opts, "max_id", second_page_last.id),
"4 page" => Map.put(opts, "max_id", third_page_last.id),
"5 page" => Map.put(opts, "max_id", forth_page_last.id)
},
formatters: formatters()
)
end
defp fetch_favourites(user) do
first_page_last = ActivityPub.fetch_favourites(user) |> List.last()
second_page_last =
ActivityPub.fetch_favourites(user, %{"max_id" => first_page_last.id}) |> List.last()
third_page_last =
ActivityPub.fetch_favourites(user, %{"max_id" => second_page_last.id}) |> List.last()
forth_page_last =
ActivityPub.fetch_favourites(user, %{"max_id" => third_page_last.id}) |> List.last()
Benchee.run(
%{
"Favourites" => fn opts ->
ActivityPub.fetch_favourites(user, opts)
end
},
inputs: %{
"1 page" => %{},
"2 page" => %{"max_id" => first_page_last.id},
"3 page" => %{"max_id" => second_page_last.id},
"4 page" => %{"max_id" => third_page_last.id},
"5 page" => %{"max_id" => forth_page_last.id}
},
formatters: formatters()
)
end
defp opts_for_long_thread(user) do
%{
"blocking_user" => user,
"user" => user
}
end
mastodon_public_timeline_params = %{
"count" => 20,
"local_only" => true,
"only_media" => "false",
"type" => ["Create", "Announce"],
"with_muted" => "true",
"blocking_user" => user,
"muting_user" => user
}
defp fetch_long_thread(user) do
%{public_thread: public, private_thread: private} =
Agent.get(:benchmark_state, fn state -> state end)
mastodon_federated_timeline_params = %{
"count" => 20,
"only_media" => "false",
"type" => ["Create", "Announce"],
"with_muted" => "true",
"blocking_user" => user,
"muting_user" => user
}
opts = opts_for_long_thread(user)
following = User.following(user)
private_input = {private.data["context"], Map.put(opts, "exclude_id", private.id)}
Benchee.run(%{
"User home timeline" => fn ->
Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
following,
home_timeline_params
)
end,
"User mastodon public timeline" => fn ->
Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
mastodon_public_timeline_params
)
end,
"User mastodon federated public timeline" => fn ->
Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
mastodon_federated_timeline_params
public_input = {public.data["context"], Map.put(opts, "exclude_id", public.id)}
Benchee.run(
%{
"fetch context" => fn {context, opts} ->
ActivityPub.fetch_activities_for_context(context, opts)
end
},
inputs: %{
"Private long thread" => private_input,
"Public long thread" => public_input
},
formatters: formatters()
)
end
})
home_activities =
Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
following,
home_timeline_params
)
defp render_timelines(user) do
opts = opts_for_home_timeline(user)
public_activities =
Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params)
recipients = [user.ap_id | User.following(user)]
public_federated_activities =
Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
mastodon_federated_timeline_params
)
home_activities = ActivityPub.fetch_activities(recipients, opts) |> Enum.reverse()
Benchee.run(%{
recipients = [user.ap_id]
opts = opts_for_direct_timeline(user)
direct_activities =
recipients
|> ActivityPub.fetch_activities_query(opts)
|> Pagination.fetch_paginated(opts)
opts = opts_for_public_timeline(user)
public_activities = ActivityPub.fetch_public_activities(opts)
opts = opts_for_public_timeline(user, :tag)
tag_activities = ActivityPub.fetch_public_activities(opts)
opts = opts_for_notifications()
notifications = MastodonAPI.get_notifications(user, opts)
favourites = ActivityPub.fetch_favourites(user)
Benchee.run(
%{
"Rendering home timeline" => fn ->
Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
StatusView.render("index.json", %{
activities: home_activities,
for: user,
as: :activity
})
end,
"Rendering direct timeline" => fn ->
StatusView.render("index.json", %{
activities: direct_activities,
for: user,
as: :activity
})
end,
"Rendering public timeline" => fn ->
Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
StatusView.render("index.json", %{
activities: public_activities,
for: user,
as: :activity
})
end,
"Rendering public federated timeline" => fn ->
Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
activities: public_federated_activities,
"Rendering tag timeline" => fn ->
StatusView.render("index.json", %{
activities: tag_activities,
for: user,
as: :activity
})
end,
"Rendering favorites timeline" => fn ->
conn = Phoenix.ConnTest.build_conn(:get, "http://localhost:4001/api/v1/favourites", nil)
Pleroma.Web.MastodonAPI.StatusController.favourites(
%Plug.Conn{conn |
assigns: %{user: user},
query_params: %{"limit" => "0"},
body_params: %{},
cookies: %{},
params: %{},
path_params: %{},
private: %{
Pleroma.Web.Router => {[], %{}},
phoenix_router: Pleroma.Web.Router,
phoenix_action: :favourites,
phoenix_controller: Pleroma.Web.MastodonAPI.StatusController,
phoenix_endpoint: Pleroma.Web.Endpoint,
phoenix_format: "json",
phoenix_layout: {Pleroma.Web.LayoutView, "app.html"},
phoenix_recycled: true,
"Rendering notifications" => fn ->
Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
notifications: notifications,
for: user
})
end,
"Rendering favourites timeline" => fn ->
StatusView.render("index.json", %{
activities: favourites,
for: user,
as: :activity
})
end
},
formatters: formatters()
)
end
phoenix_view: Pleroma.Web.MastodonAPI.StatusView,
plug_session: %{"user_id" => user.id},
plug_session_fetch: :done,
plug_session_info: :write,
plug_skip_csrf_protection: true
defp render_long_thread(user) do
%{public_thread: public, private_thread: private} =
Agent.get(:benchmark_state, fn state -> state end)
opts = %{for: user}
public_activity = Activity.get_by_id_with_object(public.id)
private_activity = Activity.get_by_id_with_object(private.id)
Benchee.run(
%{
"render" => fn opts ->
StatusView.render("show.json", opts)
end
},
inputs: %{
"Public root" => Map.put(opts, :activity, public_activity),
"Private root" => Map.put(opts, :activity, private_activity)
},
formatters: formatters()
)
fetch_opts = opts_for_long_thread(user)
public_context =
ActivityPub.fetch_activities_for_context(
public.data["context"],
Map.put(fetch_opts, "exclude_id", public.id)
)
private_context =
ActivityPub.fetch_activities_for_context(
private.data["context"],
Map.put(fetch_opts, "exclude_id", private.id)
)
Benchee.run(
%{
"render" => fn opts ->
StatusView.render("context.json", opts)
end
},
inputs: %{
"Public context" => %{user: user, activity: public_activity, activities: public_context},
"Private context" => %{
user: user,
activity: private_activity,
activities: private_context
}
},
%{})
end,
})
end
def query_notifications(user) do
without_muted_params = %{"count" => "20", "with_muted" => "false"}
with_muted_params = %{"count" => "20", "with_muted" => "true"}
Benchee.run(%{
"Notifications without muted" => fn ->
Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
end,
"Notifications with muted" => fn ->
Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
end
})
without_muted_notifications =
Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
with_muted_notifications =
Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
Benchee.run(%{
"Render notifications without muted" => fn ->
Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
notifications: without_muted_notifications,
for: user
})
end,
"Render notifications with muted" => fn ->
Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
notifications: with_muted_notifications,
for: user
})
end
})
end
def query_dms(user) do
params = %{
"count" => "20",
"with_muted" => "true",
"type" => "Create",
"blocking_user" => user,
"user" => user,
visibility: "direct"
}
Benchee.run(%{
"Direct messages with muted" => fn ->
Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
|> Pleroma.Pagination.fetch_paginated(params)
end,
"Direct messages without muted" => fn ->
Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
|> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
end
})
dms_with_muted =
Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
|> Pleroma.Pagination.fetch_paginated(params)
dms_without_muted =
Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
|> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
Benchee.run(%{
"Rendering dms with muted" => fn ->
Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
activities: dms_with_muted,
for: user,
as: :activity
})
end,
"Rendering dms without muted" => fn ->
Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
activities: dms_without_muted,
for: user,
as: :activity
})
end
})
end
def query_long_thread(user, activity) do
Benchee.run(%{
"Fetch main post" => fn ->
Pleroma.Activity.get_by_id_with_object(activity.id)
end,
"Fetch context of main post" => fn ->
Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
activity.data["context"],
%{
"blocking_user" => user,
"user" => user,
"exclude_id" => activity.id
}
formatters: formatters()
)
end
})
activity = Pleroma.Activity.get_by_id_with_object(activity.id)
context =
Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
activity.data["context"],
%{
"blocking_user" => user,
"user" => user,
"exclude_id" => activity.id
}
)
Benchee.run(%{
"Render status" => fn ->
Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{
activity: activity,
for: user
})
end,
"Render context" => fn ->
Pleroma.Web.MastodonAPI.StatusView.render(
"index.json",
for: user,
activities: context,
as: :activity
)
|> Enum.reverse()
end
})
end
end

409
benchmarks/load_testing/generator.ex
View File

@ -1,409 +0,0 @@
defmodule Pleroma.LoadTesting.Generator do
use Pleroma.LoadTesting.Helper
alias Pleroma.Web.CommonAPI
def generate_like_activities(user, posts) do
count_likes = Kernel.trunc(length(posts) / 4)
IO.puts("Starting generating #{count_likes} like activities...")
{time, _} =
:timer.tc(fn ->
Task.async_stream(
Enum.take_random(posts, count_likes),
fn post -> {:ok, _, _} = CommonAPI.favorite(post.id, user) end,
max_concurrency: 10,
timeout: 30_000
)
|> Stream.run()
end)
IO.puts("Inserting like activities take #{to_sec(time)} sec.\n")
end
def generate_users(opts) do
IO.puts("Starting generating #{opts[:users_max]} users...")
{time, _} = :timer.tc(fn -> do_generate_users(opts) end)
IO.puts("Inserting users take #{to_sec(time)} sec.\n")
end
defp do_generate_users(opts) do
max = Keyword.get(opts, :users_max)
Task.async_stream(
1..max,
&generate_user_data(&1),
max_concurrency: 10,
timeout: 30_000
)
|> Enum.to_list()
end
defp generate_user_data(i) do
remote = Enum.random([true, false])
user = %User{
name: "Test テスト User #{i}",
email: "user#{i}@example.com",
nickname: "nick#{i}",
password_hash:
"$pbkdf2-sha512$160000$bU.OSFI7H/yqWb5DPEqyjw$uKp/2rmXw12QqnRRTqTtuk2DTwZfF8VR4MYW2xMeIlqPR/UX1nT1CEKVUx2CowFMZ5JON8aDvURrZpJjSgqXrg",
bio: "Tester Number #{i}",
local: remote
}
user_urls =
if remote do
base_url =
Enum.random(["https://domain1.com", "https://domain2.com", "https://domain3.com"])
ap_id = "#{base_url}/users/#{user.nickname}"
%{
ap_id: ap_id,
follower_address: ap_id <> "/followers",
following_address: ap_id <> "/following"
}
else
%{
ap_id: User.ap_id(user),
follower_address: User.ap_followers(user),
following_address: User.ap_following(user)
}
end
user = Map.merge(user, user_urls)
Repo.insert!(user)
end
def generate_activities(user, users) do
do_generate_activities(user, users)
end
defp do_generate_activities(user, users) do
IO.puts("Starting generating 20000 common activities...")
{time, _} =
:timer.tc(fn ->
Task.async_stream(
1..20_000,
fn _ ->
do_generate_activity([user | users])
end,
max_concurrency: 10,
timeout: 30_000
)
|> Stream.run()
end)
IO.puts("Inserting common activities take #{to_sec(time)} sec.\n")
IO.puts("Starting generating 20000 activities with mentions...")
{time, _} =
:timer.tc(fn ->
Task.async_stream(
1..20_000,
fn _ ->
do_generate_activity_with_mention(user, users)
end,
max_concurrency: 10,
timeout: 30_000
)
|> Stream.run()
end)
IO.puts("Inserting activities with menthions take #{to_sec(time)} sec.\n")
IO.puts("Starting generating 10000 activities with threads...")
{time, _} =
:timer.tc(fn ->
Task.async_stream(
1..10_000,
fn _ ->
do_generate_threads([user | users])
end,
max_concurrency: 10,
timeout: 30_000
)
|> Stream.run()
end)
IO.puts("Inserting activities with threads take #{to_sec(time)} sec.\n")
end
defp do_generate_activity(users) do
post = %{
"status" => "Some status without mention with random user"
}
CommonAPI.post(Enum.random(users), post)
end
def generate_power_intervals(opts \\ []) do
count = Keyword.get(opts, :count, 20)
power = Keyword.get(opts, :power, 2)
IO.puts("Generating #{count} intervals for a power #{power} series...")
counts = Enum.map(1..count, fn n -> :math.pow(n, power) end)
sum = Enum.sum(counts)
densities =
Enum.map(counts, fn c ->
c / sum
end)
densities
|> Enum.reduce(0, fn density, acc ->
if acc == 0 do
[{0, density}]
else
[{_, lower} | _] = acc
[{lower, lower + density} | acc]
end
end)
|> Enum.reverse()
end
def generate_tagged_activities(opts \\ []) do
tag_count = Keyword.get(opts, :tag_count, 20)
users = Keyword.get(opts, :users, Repo.all(User))
activity_count = Keyword.get(opts, :count, 200_000)
intervals = generate_power_intervals(count: tag_count)
IO.puts(
"Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0"
)
Enum.each(1..activity_count, fn _ ->
random = :rand.uniform()
i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end)
CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"})
end)
end
defp do_generate_activity_with_mention(user, users) do
mentions_cnt = Enum.random([2, 3, 4, 5])
with_user = Enum.random([true, false])
users = Enum.shuffle(users)
mentions_users = Enum.take(users, mentions_cnt)
mentions_users = if with_user, do: [user | mentions_users], else: mentions_users
mentions_str =
Enum.map(mentions_users, fn user -> "@" <> user.nickname end) |> Enum.join(", ")
post = %{
"status" => mentions_str <> "some status with mentions random users"
}
CommonAPI.post(Enum.random(users), post)
end
defp do_generate_threads(users) do
thread_length = Enum.random([2, 3, 4, 5])
actor = Enum.random(users)
post = %{
"status" => "Start of the thread"
}
{:ok, activity} = CommonAPI.post(actor, post)
Enum.each(1..thread_length, fn _ ->
user = Enum.random(users)
post = %{
"status" => "@#{actor.nickname} reply to thread",
"in_reply_to_status_id" => activity.id
}
CommonAPI.post(user, post)
end)
end
def generate_remote_activities(user, users) do
do_generate_remote_activities(user, users)
end
defp do_generate_remote_activities(user, users) do
IO.puts("Starting generating 10000 remote activities...")
{time, _} =
:timer.tc(fn ->
Task.async_stream(
1..10_000,
fn i ->
do_generate_remote_activity(i, user, users)
end,
max_concurrency: 10,
timeout: 30_000
)
|> Stream.run()
end)
IO.puts("Inserting remote activities take #{to_sec(time)} sec.\n")
end
defp do_generate_remote_activity(i, user, users) do
actor = Enum.random(users)
%{host: host} = URI.parse(actor.ap_id)
date = Date.utc_today()
datetime = DateTime.utc_now()
map = %{
"actor" => actor.ap_id,
"cc" => [actor.follower_address, user.ap_id],
"context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
"id" => actor.ap_id <> "/statuses/#{i}/activity",
"object" => %{
"actor" => actor.ap_id,
"atomUri" => actor.ap_id <> "/statuses/#{i}",
"attachment" => [],
"attributedTo" => actor.ap_id,
"bcc" => [],
"bto" => [],
"cc" => [actor.follower_address, user.ap_id],
"content" =>
"<p><span class=\"h-card\"><a href=\"" <>
user.ap_id <>
"\" class=\"u-url mention\">@<span>" <> user.nickname <> "</span></a></span></p>",
"context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
"conversation" =>
"tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
"emoji" => %{},
"id" => actor.ap_id <> "/statuses/#{i}",
"inReplyTo" => nil,
"inReplyToAtomUri" => nil,
"published" => datetime,
"sensitive" => true,
"summary" => "cw",
"tag" => [
%{
"href" => user.ap_id,
"name" => "@#{user.nickname}@#{host}",
"type" => "Mention"
}
],
"to" => ["https://www.w3.org/ns/activitystreams#Public"],
"type" => "Note",
"url" => "http://#{host}/@#{actor.nickname}/#{i}"
},
"published" => datetime,
"to" => ["https://www.w3.org/ns/activitystreams#Public"],
"type" => "Create"
}
Pleroma.Web.ActivityPub.ActivityPub.insert(map, false)
end
def generate_dms(user, users, opts) do
IO.puts("Starting generating #{opts[:dms_max]} DMs")
{time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end)
IO.puts("Inserting dms take #{to_sec(time)} sec.\n")
end
defp do_generate_dms(user, users, opts) do
Task.async_stream(
1..opts[:dms_max],
fn _ ->
do_generate_dm(user, users)
end,
max_concurrency: 10,
timeout: 30_000
)
|> Stream.run()
end
defp do_generate_dm(user, users) do
post = %{
"status" => "@#{user.nickname} some direct message",
"visibility" => "direct"
}
CommonAPI.post(Enum.random(users), post)
end
def generate_long_thread(user, users, opts) do
IO.puts("Starting generating long thread with #{opts[:thread_length]} replies")
{time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end)
IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
{:ok, activity}
end
defp do_generate_long_thread(user, users, opts) do
{:ok, %{id: id} = activity} = CommonAPI.post(user, %{"status" => "Start of long thread"})
Task.async_stream(
1..opts[:thread_length],
fn _ -> do_generate_thread(users, id) end,
max_concurrency: 10,
timeout: 30_000
)
|> Stream.run()
activity
end
defp do_generate_thread(users, activity_id) do
CommonAPI.post(Enum.random(users), %{
"status" => "reply to main post",
"in_reply_to_status_id" => activity_id
})
end
def generate_non_visible_message(user, users) do
IO.puts("Starting generating 1000 non visible posts")
{time, _} =
:timer.tc(fn ->
do_generate_non_visible_posts(user, users)
end)
IO.puts("Inserting non visible posts take #{to_sec(time)} sec.\n")
end
defp do_generate_non_visible_posts(user, users) do
[not_friend | users] = users
make_friends(user, users)
Task.async_stream(1..1000, fn _ -> do_generate_non_visible_post(not_friend, users) end,
max_concurrency: 10,
timeout: 30_000
)
|> Stream.run()
end
defp make_friends(_user, []), do: nil
defp make_friends(user, [friend | users]) do
{:ok, _} = User.follow(user, friend)
{:ok, _} = User.follow(friend, user)
make_friends(user, users)
end
defp do_generate_non_visible_post(not_friend, users) do
post = %{
"status" => "some non visible post",
"visibility" => "private"
}
{:ok, activity} = CommonAPI.post(not_friend, post)
thread_length = Enum.random([2, 3, 4, 5])
Enum.each(1..thread_length, fn _ ->
user = Enum.random(users)
post = %{
"status" => "@#{not_friend.nickname} reply to non visible post",
"in_reply_to_status_id" => activity.id,
"visibility" => "private"
}
CommonAPI.post(user, post)
end)
end
end

15
benchmarks/load_testing/helper.ex
View File

@ -1,11 +1,14 @@
defmodule Pleroma.LoadTesting.Helper do
defmacro __using__(_) do
quote do
import Ecto.Query
alias Ecto.Adapters.SQL
alias Pleroma.Repo
alias Pleroma.User
defp to_sec(microseconds), do: microseconds / 1_000_000
end
def to_sec(microseconds), do: microseconds / 1_000_000
def clean_tables do
IO.puts("Deleting old data...\n")
SQL.query!(Repo, "TRUNCATE users CASCADE;")
SQL.query!(Repo, "TRUNCATE activities CASCADE;")
SQL.query!(Repo, "TRUNCATE objects CASCADE;")
SQL.query!(Repo, "TRUNCATE oban_jobs CASCADE;")
end
end

169
benchmarks/load_testing/users.ex Normal file
View File

@ -0,0 +1,169 @@
defmodule Pleroma.LoadTesting.Users do
@moduledoc """
Module for generating users with friends.
"""
import Ecto.Query
import Pleroma.LoadTesting.Helper, only: [to_sec: 1]
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.User.Query
@defaults [
users: 20_000,
friends: 100
]
@max_concurrency 10
@spec generate(keyword()) :: User.t()
def generate(opts \\ []) do
opts = Keyword.merge(@defaults, opts)
generate_users(opts[:users])
main_user =
Repo.one(from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1))
make_friends(main_user, opts[:friends])
Repo.get(User, main_user.id)
end
def generate_users(max) do
IO.puts("Starting generating #{max} users...")
{time, users} =
:timer.tc(fn ->
Task.async_stream(
1..max,
&generate_user(&1),
max_concurrency: @max_concurrency,
timeout: 30_000
)
|> Enum.to_list()
end)
IO.puts("Generating users took #{to_sec(time)} sec.\n")
users
end
defp generate_user(i) do
remote = Enum.random([true, false])
%User{
name: "Test テスト User #{i}",
email: "user#{i}@example.com",
nickname: "nick#{i}",
password_hash: Comeonin.Pbkdf2.hashpwsalt("test"),
bio: "Tester Number #{i}",
local: !remote
}
|> user_urls()
|> Repo.insert!()
end
defp user_urls(%{local: true} = user) do
urls = %{
ap_id: User.ap_id(user),
follower_address: User.ap_followers(user),
following_address: User.ap_following(user)
}
Map.merge(user, urls)
end
defp user_urls(%{local: false} = user) do
base_domain = Enum.random(["domain1.com", "domain2.com", "domain3.com"])
ap_id = "https://#{base_domain}/users/#{user.nickname}"
urls = %{
ap_id: ap_id,
follower_address: ap_id <> "/followers",
following_address: ap_id <> "/following"
}
Map.merge(user, urls)
end
def make_friends(main_user, max) when is_integer(max) do
IO.puts("Starting making friends for #{max} users...")
{time, _} =
:timer.tc(fn ->
number_of_users =
(max / 2)
|> Kernel.trunc()
main_user
|> get_users(%{limit: number_of_users, local: :local})
|> run_stream(main_user)
main_user
|> get_users(%{limit: number_of_users, local: :external})
|> run_stream(main_user)
end)
IO.puts("Making friends took #{to_sec(time)} sec.\n")
end
def make_friends(%User{} = main_user, %User{} = user) do
{:ok, _} = User.follow(main_user, user)
{:ok, _} = User.follow(user, main_user)
end
@spec get_users(User.t(), keyword()) :: [User.t()]
def get_users(user, opts) do
criteria = %{limit: opts[:limit]}
criteria =
if opts[:local] do
Map.put(criteria, opts[:local], true)
else
criteria
end
criteria =
if opts[:friends?] do
Map.put(criteria, :friends, user)
else
criteria
end
query =
criteria
|> Query.build()
|> random_without_user(user)
query =
if opts[:friends?] == false do
friends_ids =
%{friends: user}
|> Query.build()
|> Repo.all()
|> Enum.map(& &1.id)
from(u in query, where: u.id not in ^friends_ids)
else
query
end
Repo.all(query)
end
defp random_without_user(query, user) do
from(u in query,
where: u.id != ^user.id,
order_by: fragment("RANDOM()")
)
end
defp run_stream(users, main_user) do
Task.async_stream(users, &make_friends(main_user, &1),
max_concurrency: @max_concurrency,
timeout: 30_000
)
|> Stream.run()
end
end

24
benchmarks/mix/tasks/pleroma/benchmarks/tags.ex
View File

@ -1,9 +1,12 @@
defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
use Mix.Task
alias Pleroma.Repo
alias Pleroma.LoadTesting.Generator
import Pleroma.LoadTesting.Helper, only: [clean_tables: 0]
import Ecto.Query
alias Pleroma.Repo
alias Pleroma.Web.MastodonAPI.TimelineController
def run(_args) do
Mix.Pleroma.start_pleroma()
activities_count = Repo.aggregate(from(a in Pleroma.Activity), :count, :id)
@ -11,8 +14,8 @@ def run(_args) do
if activities_count == 0 do
IO.puts("Did not find any activities, cleaning and generating")
clean_tables()
Generator.generate_users(users_max: 10)
Generator.generate_tagged_activities()
Pleroma.LoadTesting.Users.generate_users(10)
Pleroma.LoadTesting.Activities.generate_tagged_activities()
else
IO.puts("Found #{activities_count} activities, won't generate new ones")
end
@ -34,7 +37,7 @@ def run(_args) do
Benchee.run(
%{
"Hashtag fetching, any" => fn tags ->
Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
TimelineController.hashtag_fetching(
%{
"any" => tags
},
@ -44,7 +47,7 @@ def run(_args) do
end,
# Will always return zero results because no overlapping hashtags are generated.
"Hashtag fetching, all" => fn tags ->
Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
TimelineController.hashtag_fetching(
%{
"all" => tags
},
@ -64,7 +67,7 @@ def run(_args) do
Benchee.run(
%{
"Hashtag fetching" => fn tag ->
Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
TimelineController.hashtag_fetching(
%{
"tag" => tag
},
@ -77,11 +80,4 @@ def run(_args) do
time: 5
)
end
defp clean_tables do
IO.puts("Deleting old data...\n")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
end
end

70
benchmarks/mix/tasks/pleroma/benchmarks/timelines.ex Normal file
View File

@ -0,0 +1,70 @@
defmodule Mix.Tasks.Pleroma.Benchmarks.Timelines do
use Mix.Task
import Pleroma.LoadTesting.Helper, only: [clean_tables: 0]
alias Pleroma.Web.CommonAPI
alias Plug.Conn
def run(_args) do
Mix.Pleroma.start_pleroma()
# Cleaning tables
clean_tables()
[{:ok, user} | users] = Pleroma.LoadTesting.Users.generate_users(1000)
# Let the user make 100 posts
1..100
|> Enum.each(fn i -> CommonAPI.post(user, %{"status" => to_string(i)}) end)
# Let 10 random users post
posts =
users
|> Enum.take_random(10)
|> Enum.map(fn {:ok, random_user} ->
{:ok, activity} = CommonAPI.post(random_user, %{"status" => "."})
activity
end)
# let our user repeat them
posts
|> Enum.each(fn activity ->
CommonAPI.repeat(activity.id, user)
end)
Benchee.run(
%{
"user timeline, no followers" => fn reading_user ->
conn =
Phoenix.ConnTest.build_conn()
|> Conn.assign(:user, reading_user)
|> Conn.assign(:skip_link_headers, true)
Pleroma.Web.MastodonAPI.AccountController.statuses(conn, %{"id" => user.id})
end
},
inputs: %{"user" => user, "no user" => nil},
time: 60
)
users
|> Enum.each(fn {:ok, follower} -> Pleroma.User.follow(follower, user) end)
Benchee.run(
%{
"user timeline, all following" => fn reading_user ->
conn =
Phoenix.ConnTest.build_conn()
|> Conn.assign(:user, reading_user)
|> Conn.assign(:skip_link_headers, true)
Pleroma.Web.MastodonAPI.AccountController.statuses(conn, %{"id" => user.id})
end
},
inputs: %{"user" => user, "no user" => nil},
time: 60
)
end
end

136
benchmarks/mix/tasks/pleroma/load_testing.ex
View File

@ -1,114 +1,55 @@
defmodule Mix.Tasks.Pleroma.LoadTesting do
use Mix.Task
use Pleroma.LoadTesting.Helper
import Mix.Pleroma
import Pleroma.LoadTesting.Generator
import Pleroma.LoadTesting.Fetcher
import Ecto.Query
import Pleroma.LoadTesting.Helper, only: [clean_tables: 0]
alias Pleroma.Repo
alias Pleroma.User
@shortdoc "Factory for generation data"
@moduledoc """
Generates data like:
- local/remote users
- local/remote activities with notifications
- direct messages
- long thread
- non visible posts
- local/remote activities with differrent visibility:
- simple activiities
- with emoji
- with mentions
- hellthreads
- with attachments
- with tags
- likes
- reblogs
- simple threads
- long threads
## Generate data
MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --dms 20000 --thread_length 2000
MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -d 20000 -t 2000
MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --friends 1000 --iterations 170 --friends_used 20 --non_friends_used 20
MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -f 1000 -i 170 -fu 20 -nfu 20
Options:
- `--users NUMBER` - number of users to generate. Defaults to: 20000. Alias: `-u`
- `--dms NUMBER` - number of direct messages to generate. Defaults to: 20000. Alias `-d`
- `--thread_length` - number of messages in thread. Defaults to: 2000. ALias `-t`
- `--friends NUMBER` - number of friends for main user. Defaults to: 1000. Alias: `-f`
- `--iterations NUMBER` - number of iterations to generate activities. For each iteration in database is inserted about 120+ activities with different visibility, actors and types.Defaults to: 170. Alias: `-i`
- `--friends_used NUMBER` - number of main user friends used in activity generation. Defaults to: 20. Alias: `-fu`
- `--non_friends_used NUMBER` - number of non friends used in activity generation. Defaults to: 20. Alias: `-nfu`
"""
@aliases [u: :users, d: :dms, t: :thread_length]
@aliases [u: :users, f: :friends, i: :iterations, fu: :friends_used, nfu: :non_friends_used]
@switches [
users: :integer,
dms: :integer,
thread_length: :integer
friends: :integer,
iterations: :integer,
friends_used: :integer,
non_friends_used: :integer
]
@users_default 20_000
@dms_default 1_000
@thread_length_default 2_000
def run(args) do
start_pleroma()
Pleroma.Config.put([:instance, :skip_thread_containment], true)
Mix.Pleroma.start_pleroma()
clean_tables()
{opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
users_max = Keyword.get(opts, :users, @users_default)
dms_max = Keyword.get(opts, :dms, @dms_default)
thread_length = Keyword.get(opts, :thread_length, @thread_length_default)
clean_tables()
opts =
Keyword.put(opts, :users_max, users_max)
|> Keyword.put(:dms_max, dms_max)
|> Keyword.put(:thread_length, thread_length)
generate_users(opts)
# main user for queries
IO.puts("Fetching local main user...")
{time, user} =
:timer.tc(fn ->
Repo.one(
from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)
)
end)
IO.puts("Fetching main user take #{to_sec(time)} sec.\n")
IO.puts("Fetching local users...")
{time, users} =
:timer.tc(fn ->
Repo.all(
from(u in User,
where: u.id != ^user.id,
where: u.local == true,
order_by: fragment("RANDOM()"),
limit: 10
)
)
end)
IO.puts("Fetching local users take #{to_sec(time)} sec.\n")
IO.puts("Fetching remote users...")
{time, remote_users} =
:timer.tc(fn ->
Repo.all(
from(u in User,
where: u.id != ^user.id,
where: u.local == false,
order_by: fragment("RANDOM()"),
limit: 10
)
)
end)
IO.puts("Fetching remote users take #{to_sec(time)} sec.\n")
generate_activities(user, users)
generate_remote_activities(user, remote_users)
generate_like_activities(
user, Pleroma.Repo.all(Pleroma.Activity.Queries.by_type("Create"))
)
generate_dms(user, users, opts)
{:ok, activity} = generate_long_thread(user, users, opts)
generate_non_visible_message(user, users)
user = Pleroma.LoadTesting.Users.generate(opts)
Pleroma.LoadTesting.Activities.generate(user, opts)
IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}")
@ -120,19 +61,6 @@ def run(args) do
"Notifications in DB: #{Repo.aggregate(from(n in Pleroma.Notification), :count, :id)}"
)
fetch_user(user)
query_timelines(user)
query_notifications(user)
query_dms(user)
query_long_thread(user, activity)
Pleroma.Config.put([:instance, :skip_thread_containment], false)
query_timelines(user)
end
defp clean_tables do
IO.puts("Deleting old data...\n")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
Pleroma.LoadTesting.Fetcher.run_benchmarks(user)
end
end

2
config/benchmark.exs
View File

@ -39,7 +39,7 @@
adapter: Ecto.Adapters.Postgres,
username: "postgres",
password: "postgres",
database: "pleroma_test",
database: "pleroma_benchmark",
hostname: System.get_env("DB_HOST") || "localhost",
pool_size: 10

74
config/config.exs
View File

@ -58,20 +58,6 @@
config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
config :pleroma, :hackney_pools,
federation: [
max_connections: 50,
timeout: 150_000
],
media: [
max_connections: 50,
timeout: 150_000
],
upload: [
max_connections: 25,
timeout: 300_000
]
# Upload configuration
config :pleroma, Pleroma.Upload,
uploader: Pleroma.Uploaders.Local,
@ -184,21 +170,13 @@
"application/ld+json" => ["activity+json"]
}
config :tesla, adapter: Tesla.Adapter.Hackney
config :tesla, adapter: Tesla.Adapter.Gun
# Configures http settings, upstream proxy etc.
config :pleroma, :http,
proxy_url: nil,
send_user_agent: true,
user_agent: :default,
adapter: [
ssl_options: [
# Workaround for remote server certificate chain issues
partial_chain: &:hackney_connect.partial_chain/1,
# We don't support TLS v1.3 yet
versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"]
]
]
adapter: []
config :pleroma, :instance,
name: "Pleroma",
@ -624,6 +602,54 @@
parameters: [gin_fuzzy_search_limit: "500"],
prepare: :unnamed
config :pleroma, :connections_pool,
checkin_timeout: 250,
max_connections: 250,
retry: 1,
retry_timeout: 1000,
await_up_timeout: 5_000
config :pleroma, :pools,
federation: [
size: 50,
max_overflow: 10,
timeout: 150_000
],
media: [
size: 50,
max_overflow: 10,
timeout: 150_000
],
upload: [
size: 25,
max_overflow: 5,
timeout: 300_000
],
default: [
size: 10,
max_overflow: 2,
timeout: 10_000
]
config :pleroma, :hackney_pools,
federation: [
max_connections: 50,
timeout: 150_000
],
media: [
max_connections: 50,
timeout: 150_000
],
upload: [
max_connections: 25,
timeout: 300_000
]
config :pleroma, :restrict_unauthenticated,
timelines: %{local: false, federated: false},
profiles: %{local: false, remote: false},
activities: %{local: false, remote: false}
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"

275
config/description.exs
View File

@ -2442,7 +2442,7 @@
%{
key: :relations_actions,
type: [:tuple, {:list, :tuple}],
description: "For actions on relations with all users (follow, unfollow)",
description: "For actions on relationships with all users (follow, unfollow)",
suggestions: [{1000, 10}, [{10_000, 10}, {10_000, 50}]]
},
%{
@ -2915,5 +2915,278 @@
suggestions: [2]
}
]
},
%{
group: :pleroma,
key: :connections_pool,
type: :group,
description: "Advanced settings for `gun` connections pool",
children: [
%{
key: :checkin_timeout,
type: :integer,
description: "Timeout to checkin connection from pool. Default: 250ms.",
suggestions: [250]
},
%{
key: :max_connections,
type: :integer,
description: "Maximum number of connections in the pool. Default: 250 connections.",
suggestions: [250]
},
%{
key: :retry,
type: :integer,
description:
"Number of retries, while `gun` will try to reconnect if connection goes down. Default: 1.",
suggestions: [1]
},
%{
key: :retry_timeout,
type: :integer,
description:
"Time between retries when `gun` will try to reconnect in milliseconds. Default: 1000ms.",
suggestions: [1000]
},
%{
key: :await_up_timeout,
type: :integer,
description: "Timeout while `gun` will wait until connection is up. Default: 5000ms.",
suggestions: [5000]
}
]
},
%{
group: :pleroma,
key: :pools,
type: :group,
description: "Advanced settings for `gun` workers pools",
children: [
%{
key: :federation,
type: :keyword,
description: "Settings for federation pool.",
children: [
%{
key: :size,
type: :integer,
description: "Number workers in the pool.",
suggestions: [50]
},
%{
key: :max_overflow,
type: :integer,
description: "Number of additional workers if pool is under load.",
suggestions: [10]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `gun` will wait for response.",
suggestions: [150_000]
}
]
},
%{
key: :media,
type: :keyword,
description: "Settings for media pool.",
children: [
%{
key: :size,
type: :integer,
description: "Number workers in the pool.",
suggestions: [50]
},
%{
key: :max_overflow,
type: :integer,
description: "Number of additional workers if pool is under load.",
suggestions: [10]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `gun` will wait for response.",
suggestions: [150_000]
}
]
},
%{
key: :upload,
type: :keyword,
description: "Settings for upload pool.",
children: [
%{
key: :size,
type: :integer,
description: "Number workers in the pool.",
suggestions: [25]
},
%{
key: :max_overflow,
type: :integer,
description: "Number of additional workers if pool is under load.",
suggestions: [5]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `gun` will wait for response.",
suggestions: [300_000]
}
]
},
%{
key: :default,
type: :keyword,
description: "Settings for default pool.",
children: [
%{
key: :size,
type: :integer,
description: "Number workers in the pool.",
suggestions: [10]
},
%{
key: :max_overflow,
type: :integer,
description: "Number of additional workers if pool is under load.",
suggestions: [2]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `gun` will wait for response.",
suggestions: [10_000]
}
]
}
]
},
%{
group: :pleroma,
key: :hackney_pools,
type: :group,
description: "Advanced settings for `hackney` connections pools",
children: [
%{
key: :federation,
type: :keyword,
description: "Settings for federation pool.",
children: [
%{
key: :max_connections,
type: :integer,
description: "Number workers in the pool.",
suggestions: [50]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `hackney` will wait for response.",
suggestions: [150_000]
}
]
},
%{
key: :media,
type: :keyword,
description: "Settings for media pool.",
children: [
%{
key: :max_connections,
type: :integer,
description: "Number workers in the pool.",
suggestions: [50]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `hackney` will wait for response.",
suggestions: [150_000]
}
]
},
%{
key: :upload,
type: :keyword,
description: "Settings for upload pool.",
children: [
%{
key: :max_connections,
type: :integer,
description: "Number workers in the pool.",
suggestions: [25]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `hackney` will wait for response.",
suggestions: [300_000]
}
]
}
]
},
%{
group: :pleroma,
key: :restrict_unauthenticated,
type: :group,
description:
"Disallow viewing timelines, user profiles and statuses for unauthenticated users.",
children: [
%{
key: :timelines,
type: :map,
description: "Settings for public and federated timelines.",
children: [
%{
key: :local,
type: :boolean,
description: "Disallow view public timeline."
},
%{
key: :federated,
type: :boolean,
description: "Disallow view federated timeline."
}
]
},
%{
key: :profiles,
type: :map,
description: "Settings for user profiles.",
children: [
%{
key: :local,
type: :boolean,
description: "Disallow view local user profiles."
},
%{
key: :remote,
type: :boolean,
description: "Disallow view remote user profiles."
}
]
},
%{
key: :activities,
type: :map,
description: "Settings for statuses.",
children: [
%{
key: :local,
type: :boolean,
description: "Disallow view local statuses."
},
%{
key: :remote,
type: :boolean,
description: "Disallow view remote statuses."
}
]
}
]
}
]

2
config/test.exs
View File

@ -90,6 +90,8 @@
config :pleroma, :modules, runtime_dir: "test/fixtures/modules"
config :pleroma, Pleroma.Gun, Pleroma.GunMock
config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: true
config :pleroma, Pleroma.Plugs.RemoteIp, enabled: false

6
coveralls.json Normal file
View File

@ -0,0 +1,6 @@
{
"skip_files": [
"test/support",
"lib/mix/tasks/pleroma/benchmark.ex"
]
}

79
docs/API/admin_api.md
View File

@ -414,6 +414,83 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
- `nicknames`
- Response: none (code `204`)
## `GET /api/pleroma/admin/users/:nickname/credentials`
### Get the user's email, password, display and settings-related fields
- Params:
- `nickname`
- Response:
```json
{
"actor_type": "Person",
"allow_following_move": true,
"avatar": "https://pleroma.social/media/7e8e7508fd545ef580549b6881d80ec0ff2c81ed9ad37b9bdbbdf0e0d030159d.jpg",
"background": "https://pleroma.social/media/4de34c0bd10970d02cbdef8972bef0ebbf55f43cadc449554d4396156162fe9a.jpg",
"banner": "https://pleroma.social/media/8d92ba2bd244b613520abf557dd448adcd30f5587022813ee9dd068945986946.jpg",
"bio": "bio",
"default_scope": "public",
"discoverable": false,
"email": "user@example.com",
"fields": [
{
"name": "example",
"value": "<a href=\"https://example.com\" rel=\"ugc\">https://example.com</a>"
}
],
"hide_favorites": false,
"hide_followers": false,
"hide_followers_count": false,
"hide_follows": false,
"hide_follows_count": false,
"id": "9oouHaEEUR54hls968",
"locked": true,
"name": "user",
"no_rich_text": true,
"pleroma_settings_store": {},
"raw_fields": [
{
"id": 1,
"name": "example",
"value": "https://example.com"
},
],
"show_role": true,
"skip_thread_containment": false
}
```
## `PATCH /api/pleroma/admin/users/:nickname/credentials`
### Change the user's email, password, display and settings-related fields
- Params:
- `email`
- `password`
- `name`
- `bio`
- `avatar`
- `locked`
- `no_rich_text`
- `default_scope`
- `banner`
- `hide_follows`
- `hide_followers`
- `hide_followers_count`
- `hide_follows_count`
- `hide_favorites`
- `allow_following_move`
- `background`
- `show_role`
- `skip_thread_containment`
- `fields`
- `discoverable`
- `actor_type`
- Response: none (code `200`)
## `GET /api/pleroma/admin/reports`
### Get a list of reports
@ -764,6 +841,8 @@ Some modifications are necessary to save the config settings correctly:
Most of the settings will be applied in `runtime`, this means that you don't need to restart the instance. But some settings are applied in `compile time` and require a reboot of the instance, such as:
- all settings inside these keys:
- `:hackney_pools`
- `:connections_pool`
- `:pools`
- `:chat`
- partially settings inside these keys:
- `:seconds_valid` in `Pleroma.Captcha`

18
docs/API/differences_in_mastoapi_responses.md
View File

@ -164,6 +164,7 @@ Additional parameters can be added to the JSON body/Form data:
- `actor_type` - the type of this account.
### Pleroma Settings Store
Pleroma has mechanism that allows frontends to save blobs of json for each user on the backend. This can be used to save frontend-specific settings for a user that the backend does not need to know about.
The parameter should have a form of `{frontend_name: {...}}`, with `frontend_name` identifying your type of client, e.g. `pleroma_fe`. It will overwrite everything under this property, but will not overwrite other frontend's settings.
@ -172,17 +173,20 @@ This information is returned in the `verify_credentials` endpoint.
## Authentication
*Pleroma supports refreshing tokens.
*Pleroma supports refreshing tokens.*
`POST /oauth/token`
Post here request with grant_type=refresh_token to obtain new access token. Returns an access token.
Post here request with `grant_type=refresh_token` to obtain new access token. Returns an access token.
## Account Registration
`POST /api/v1/accounts`
Has theses additional parameters (which are the same as in Pleroma-API):
* `fullname`: optional
* `bio`: optional
* `captcha_solution`: optional, contains provider-specific captcha solution,
* `captcha_token`: optional, contains provider-specific captcha token
* `token`: invite token required when the registerations aren't public.
- `fullname`: optional
- `bio`: optional
- `captcha_solution`: optional, contains provider-specific captcha solution,
- `captcha_token`: optional, contains provider-specific captcha token
- `token`: invite token required when the registrations aren't public.

2
docs/API/pleroma_api.md
View File

@ -431,7 +431,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
# Emoji Reactions
Emoji reactions work a lot like favourites do. They make it possible to react to a post with a single emoji character.
Emoji reactions work a lot like favourites do. They make it possible to react to a post with a single emoji character. To detect the presence of this feature, you can check `pleroma_emoji_reactions` entry in the features list of nodeinfo.
## `PUT /api/v1/pleroma/statuses/:id/reactions/:emoji`
### React to a post with a unicode emoji

2
docs/administration/CLI_tasks/emoji.md
View File

@ -41,6 +41,6 @@ mix pleroma.emoji gen-pack PACK-URL
Currently, only .zip archives are recognized as remote pack files and packs are therefore assumed to be zip archives. This command is intended to run interactively and will first ask you some basic questions about the pack, then download the remote file and generate an SHA256 checksum for it, then generate an emoji file list for you.
The manifest entry will either be written to a newly created `index.json` file or appended to the existing one, *replacing* the old pack with the same name if it was in the file previously.
The manifest entry will either be written to a newly created `pack_name.json` file (pack name is asked in questions) or appended to the existing one, *replacing* the old pack with the same name if it was in the file previously.
The file list will be written to the file specified previously, *replacing* that file. You _should_ check that the file list doesn't contain anything you don't need in the pack, that is, anything that is not an emoji (the whole pack is downloaded, but only emoji files are extracted).

57
docs/configuration/cheatsheet.md
View File

@ -369,8 +369,7 @@ Available caches:
* `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`)
* `send_user_agent`: should we include a user agent with HTTP requests? (default: `true`)
* `user_agent`: what user agent should we use? (default: `:default`), must be string or `:default`
* `adapter`: array of hackney options
* `adapter`: array of adapter options
### :hackney_pools
@ -389,6 +388,42 @@ For each pool, the options are:
* `timeout` - retention duration for connections
### :connections_pool
*For `gun` adapter*
Advanced settings for connections pool. Pool with opened connections. These connections can be reused in worker pools.
For big instances it's recommended to increase `config :pleroma, :connections_pool, max_connections: 500` up to 500-1000.
It will increase memory usage, but federation would work faster.
* `:checkin_timeout` - timeout to checkin connection from pool. Default: 250ms.
* `:max_connections` - maximum number of connections in the pool. Default: 250 connections.
* `:retry` - number of retries, while `gun` will try to reconnect if connection goes down. Default: 1.
* `:retry_timeout` - time between retries when `gun` will try to reconnect in milliseconds. Default: 1000ms.
* `:await_up_timeout` - timeout while `gun` will wait until connection is up. Default: 5000ms.
### :pools
*For `gun` adapter*
Advanced settings for workers pools.
There are four pools used:
* `:federation` for the federation jobs.
You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs.
* `:media` for rich media, media proxy
* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`)
* `:default` for other requests
For each pool, the options are:
* `:size` - how much workers the pool can hold
* `:timeout` - timeout while `gun` will wait for response
* `:max_overflow` - additional workers if pool is under load
## Captcha
### Pleroma.Captcha
@ -872,3 +907,21 @@ config :auto_linker,
## :configurable_from_database
Boolean, enables/disables in-database configuration. Read [Transfering the config to/from the database](../administration/CLI_tasks/config.md) for more information.
## Restrict entities access for unauthenticated users
### :restrict_unauthenticated
Restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses.
* `timelines` - public and federated timelines
* `local` - public timeline
* `federated`
* `profiles` - user profiles
* `local`
* `remote`
* `activities` - statuses
* `local`
* `remote`

1
lib/mix/pleroma.ex
View File

@ -5,6 +5,7 @@
defmodule Mix.Pleroma do
@doc "Common functions to be reused in mix tasks"
def start_pleroma do
Mix.Task.run("app.start")
Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
if Pleroma.Config.get(:env) != :test do

39
lib/mix/tasks/pleroma/benchmark.ex
View File

@ -74,4 +74,43 @@ def run(["render_timeline", nickname | _] = args) do
inputs: inputs
)
end
def run(["adapters"]) do
start_pleroma()
:ok =
Pleroma.Gun.Conn.open(
"https://httpbin.org/stream-bytes/1500",
:gun_connections
)
Process.sleep(1_500)
Benchee.run(
%{
"Without conn and without pool" => fn ->
{:ok, %Tesla.Env{}} =
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
adapter: [pool: :no_pool, receive_conn: false]
)
end,
"Without conn and with pool" => fn ->
{:ok, %Tesla.Env{}} =
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
adapter: [receive_conn: false]
)
end,
"With reused conn and without pool" => fn ->
{:ok, %Tesla.Env{}} =
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
adapter: [pool: :no_pool]
)
end,
"With reused conn and with pool" => fn ->
{:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
end
},
parallel: 10
)
end
end

99
lib/mix/tasks/pleroma/emoji.ex
View File

@ -4,18 +4,18 @@
defmodule Mix.Tasks.Pleroma.Emoji do
use Mix.Task
import Mix.Pleroma
@shortdoc "Manages emoji packs"
@moduledoc File.read!("docs/administration/CLI_tasks/emoji.md")
def run(["ls-packs" | args]) do
Mix.Pleroma.start_pleroma()
Application.ensure_all_started(:hackney)
start_pleroma()
{options, [], []} = parse_global_opts(args)
manifest =
fetch_manifest(if options[:manifest], do: options[:manifest], else: default_manifest())
url_or_path = options[:manifest] || default_manifest()
manifest = fetch_manifest(url_or_path)
Enum.each(manifest, fn {name, info} ->
to_print = [
@ -36,14 +36,13 @@ def run(["ls-packs" | args]) do
end
def run(["get-packs" | args]) do
Mix.Pleroma.start_pleroma()
Application.ensure_all_started(:hackney)
start_pleroma()
{options, pack_names, []} = parse_global_opts(args)
manifest_url = if options[:manifest], do: options[:manifest], else: default_manifest()
url_or_path = options[:manifest] || default_manifest()
manifest = fetch_manifest(manifest_url)
manifest = fetch_manifest(url_or_path)
for pack_name <- pack_names do
if Map.has_key?(manifest, pack_name) do
@ -76,7 +75,10 @@ def run(["get-packs" | args]) do
end
# The url specified in files should be in the same directory
files_url = Path.join(Path.dirname(manifest_url), pack["files"])
files_url =
url_or_path
|> Path.dirname()
|> Path.join(pack["files"])
IO.puts(
IO.ANSI.format([
@ -134,38 +136,51 @@ def run(["get-packs" | args]) do
end
end
def run(["gen-pack", src]) do
Application.ensure_all_started(:hackney)
def run(["gen-pack" | args]) do
start_pleroma()
proposed_name = Path.basename(src) |> Path.rootname()
name = String.trim(IO.gets("Pack name [#{proposed_name}]: "))
# If there's no name, use the default one
name = if String.length(name) > 0, do: name, else: proposed_name
license = String.trim(IO.gets("License: "))
homepage = String.trim(IO.gets("Homepage: "))
description = String.trim(IO.gets("Description: "))
proposed_files_name = "#{name}.json"
files_name = String.trim(IO.gets("Save file list to [#{proposed_files_name}]: "))
files_name = if String.length(files_name) > 0, do: files_name, else: proposed_files_name
default_exts = [".png", ".gif"]
default_exts_str = Enum.join(default_exts, " ")
exts =
String.trim(
IO.gets("Emoji file extensions (separated with spaces) [#{default_exts_str}]: ")
{opts, [src], []} =
OptionParser.parse(
args,
strict: [
name: :string,
license: :string,
homepage: :string,
description: :string,
files: :string,
extensions: :string
]
)
proposed_name = Path.basename(src) |> Path.rootname()
name = get_option(opts, :name, "Pack name:", proposed_name)
license = get_option(opts, :license, "License:")
homepage = get_option(opts, :homepage, "Homepage:")
description = get_option(opts, :description, "Description:")
proposed_files_name = "#{name}_files.json"
files_name = get_option(opts, :files, "Save file list to:", proposed_files_name)
default_exts = [".png", ".gif"]
custom_exts =
get_option(
opts,
:extensions,
"Emoji file extensions (separated with spaces):",
Enum.join(default_exts, " ")
)
|> String.split(" ", trim: true)
exts =
if String.length(exts) > 0 do
String.split(exts, " ")
|> Enum.filter(fn e -> e |> String.trim() |> String.length() > 0 end)
else
if MapSet.equal?(MapSet.new(default_exts), MapSet.new(custom_exts)) do
default_exts
else
custom_exts
end
IO.puts("Using #{Enum.join(exts, " ")} extensions")
IO.puts("Downloading the pack and generating SHA256")
binary_archive = Tesla.get!(client(), src).body
@ -195,14 +210,16 @@ def run(["gen-pack", src]) do
IO.puts("""
#{files_name} has been created and contains the list of all found emojis in the pack.
Please review the files in the remove those not needed.
Please review the files in the pack and remove those not needed.
""")
if File.exists?("index.json") do
existing_data = File.read!("index.json") |> Jason.decode!()
pack_file = "#{name}.json"
if File.exists?(pack_file) do
existing_data = File.read!(pack_file) |> Jason.decode!()
File.write!(
"index.json",
pack_file,
Jason.encode!(
Map.merge(
existing_data,
@ -212,11 +229,11 @@ def run(["gen-pack", src]) do
)
)
IO.puts("index.json file has been update with the #{name} pack")
IO.puts("#{pack_file} has been updated with the #{name} pack")
else
File.write!("index.json", Jason.encode!(pack_json, pretty: true))
File.write!(pack_file, Jason.encode!(pack_json, pretty: true))
IO.puts("index.json has been created with the #{name} pack")
IO.puts("#{pack_file} has been created with the #{name} pack")
end
end

11
lib/pleroma/activity.ex
View File

@ -95,6 +95,17 @@ def with_preloaded_object(query, join_type \\ :inner) do
|> preload([activity, object: object], object: object)
end
# Note: applies to fake activities (ActivityPub.Utils.get_notified_from_object/1 etc.)
def user_actor(%Activity{actor: nil}), do: nil
def user_actor(%Activity{} = activity) do
with %User{} <- activity.user_actor do
activity.user_actor
else
_ -> User.get_cached_by_ap_id(activity.actor)
end
end
def with_joined_user_actor(query, join_type \\ :inner) do
join(query, join_type, [activity], u in User,
on: u.ap_id == activity.actor,

7
lib/pleroma/activity/queries.ex
View File

@ -35,6 +35,13 @@ def by_author(query \\ Activity, %User{ap_id: ap_id}) do
from(a in query, where: a.actor == ^ap_id)
end
def find_by_object_ap_id(activities, object_ap_id) do
Enum.find(
activities,
&(object_ap_id in [is_map(&1.data["object"]) && &1.data["object"]["id"], &1.data["object"]])
)
end
@spec by_object_id(query, String.t() | [String.t()]) :: query
def by_object_id(query \\ Activity, object_id)

98
lib/pleroma/application.ex
View File

@ -3,8 +3,12 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Application do
import Cachex.Spec
use Application
import Cachex.Spec
alias Pleroma.Config
require Logger
@name Mix.Project.config()[:name]
@ -18,9 +22,9 @@ def named_version, do: @name <> " " <> @version
def repository, do: @repository
def user_agent do
case Pleroma.Config.get([:http, :user_agent], :default) do
case Config.get([:http, :user_agent], :default) do
:default ->
info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"
info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>"
named_version() <> "; " <> info
custom ->
@ -33,27 +37,51 @@ def user_agent do
def start(_type, _args) do
Pleroma.Config.Holder.save_default()
Pleroma.HTML.compile_scrubbers()
Pleroma.Config.DeprecationWarnings.warn()
Config.DeprecationWarnings.warn()
Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
Pleroma.Repo.check_migrations_applied!()
setup_instrumenters()
load_custom_modules()
adapter = Application.get_env(:tesla, :adapter)
if adapter == Tesla.Adapter.Gun do
if version = Pleroma.OTPVersion.version() do
[major, minor] =
version
|> String.split(".")
|> Enum.map(&String.to_integer/1)
|> Enum.take(2)
if (major == 22 and minor < 2) or major < 22 do
raise "
!!!OTP VERSION WARNING!!!
You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains.
"
end
else
raise "
!!!OTP VERSION WARNING!!!
To support correct handling of unordered certificates chains - OTP version must be > 22.2.
"
end
end
# Define workers and child supervisors to be supervised
children =
[
Pleroma.Repo,
Pleroma.Config.TransferTask,
Config.TransferTask,
Pleroma.Emoji,
Pleroma.Captcha,
Pleroma.Plugs.RateLimiter.Supervisor
] ++
cachex_children() ++
hackney_pool_children() ++
http_children(adapter, @env) ++
[
Pleroma.Stats,
Pleroma.JobQueueMonitor,
{Oban, Pleroma.Config.get(Oban)}
{Oban, Config.get(Oban)}
] ++
task_children(@env) ++
streamer_child(@env) ++
@ -70,7 +98,7 @@ def start(_type, _args) do
end
def load_custom_modules do
dir = Pleroma.Config.get([:modules, :runtime_dir])
dir = Config.get([:modules, :runtime_dir])
if dir && File.exists?(dir) do
dir
@ -111,20 +139,6 @@ defp setup_instrumenters do
Pleroma.Web.Endpoint.Instrumenter.setup()
end
def enabled_hackney_pools do
[:media] ++
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
[:federation]
else
[]
end ++
if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do
[:upload]
else
[]
end
end
defp cachex_children do
[
build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),
@ -146,7 +160,7 @@ defp idempotency_expiration,
do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
defp seconds_valid_interval,
do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid]))
do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
defp build_cachex(type, opts),
do: %{
@ -155,9 +169,9 @@ defp build_cachex(type, opts),
type: :worker
}
defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled])
defp chat_enabled?, do: Config.get([:chat, :enabled])
defp streamer_child(:test), do: []
defp streamer_child(env) when env in [:test, :benchmark], do: []
defp streamer_child(_) do
[Pleroma.Web.Streamer.supervisor()]
@ -169,13 +183,6 @@ defp chat_child(_env, true) do
defp chat_child(_, _), do: []
defp hackney_pool_children do
for pool <- enabled_hackney_pools() do
options = Pleroma.Config.get([:hackney_pools, pool])
:hackney_pool.child_spec(pool, options)
end
end
defp task_children(:test) do
[
%{
@ -200,4 +207,31 @@ defp task_children(_) do
}
]
end
# start hackney and gun pools in tests
defp http_children(_, :test) do
hackney_options = Config.get([:hackney_pools, :federation])
hackney_pool = :hackney_pool.child_spec(:federation, hackney_options)
[hackney_pool, Pleroma.Pool.Supervisor]
end
defp http_children(Tesla.Adapter.Hackney, _) do
pools = [:federation, :media]
pools =
if Config.get([Pleroma.Upload, :proxy_remote]) do
[:upload | pools]
else
pools
end
for pool <- pools do
options = Config.get([:hackney_pools, pool])
:hackney_pool.child_spec(pool, options)
end
end
defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor]
defp http_children(_, _), do: []
end

11
lib/pleroma/config/config_db.ex
View File

@ -278,8 +278,6 @@ defp do_convert({:proxy_url, {type, host, port}}) do
}
end
defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]}
defp do_convert(entity) when is_tuple(entity) do
value =
entity
@ -323,15 +321,6 @@ defp do_transform(%{"tuple" => [":proxy_url", %{"tuple" => [type, host, port]}]}
{:proxy_url, {do_transform_string(type), parse_host(host), port}}
end
defp do_transform(%{"tuple" => [":partial_chain", entity]}) do
{partial_chain, []} =
entity
|> String.replace(~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "")
|> Code.eval_string()
{:partial_chain, partial_chain}
end
defp do_transform(%{"tuple" => entity}) do
Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end)
end

105
lib/pleroma/config/transfer_task.ex
View File

@ -5,6 +5,7 @@
defmodule Pleroma.Config.TransferTask do
use Task
alias Pleroma.Config
alias Pleroma.ConfigDB
alias Pleroma.Repo
@ -18,7 +19,9 @@ defmodule Pleroma.Config.TransferTask do
{:pleroma, Oban},
{:pleroma, :rate_limit},
{:pleroma, :markup},
{:plerome, :streamer}
{:pleroma, :streamer},
{:pleroma, :pools},
{:pleroma, :connections_pool}
]
@reboot_time_subkeys [
@ -32,45 +35,33 @@ defmodule Pleroma.Config.TransferTask do
{:pleroma, :gopher, [:enabled]}
]
@reject [nil, :prometheus]
def start_link(_) do
load_and_update_env()
if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo)
if Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo)
:ignore
end
@spec load_and_update_env([ConfigDB.t()]) :: :ok | false
def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do
with {:configurable, true} <-
{:configurable, Pleroma.Config.get(:configurable_from_database)},
true <- Ecto.Adapters.SQL.table_exists?(Repo, "config"),
started_applications <- Application.started_applications() do
@spec load_and_update_env([ConfigDB.t()], boolean()) :: :ok
def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do
with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do
# We need to restart applications for loaded settings take effect
in_db = Repo.all(ConfigDB)
with_deleted = in_db ++ deleted
reject_for_restart = if restart_pleroma?, do: @reject, else: [:pleroma | @reject]
applications =
with_deleted
|> Enum.map(&merge_and_update(&1))
|> Enum.uniq()
# TODO: some problem with prometheus after restart!
|> Enum.reject(&(&1 in reject_for_restart))
# to be ensured that pleroma will be restarted last
applications =
if :pleroma in applications do
List.delete(applications, :pleroma) ++ [:pleroma]
reject_restart =
if restart_pleroma? do
[nil, :prometheus]
else
Restarter.Pleroma.rebooted()
applications
[:pleroma, nil, :prometheus]
end
Enum.each(applications, &restart(started_applications, &1, Pleroma.Config.get(:env)))
started_applications = Application.started_applications()
(Repo.all(ConfigDB) ++ deleted_settings)
|> Enum.map(&merge_and_update/1)
|> Enum.uniq()
|> Enum.reject(&(&1 in reject_restart))
|> maybe_set_pleroma_last()
|> Enum.each(&restart(started_applications, &1, Config.get(:env)))
:ok
else
@ -78,32 +69,19 @@ def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do
end
end
defp merge_and_update(setting) do
try do
key = ConfigDB.from_string(setting.key)
group = ConfigDB.from_string(setting.group)
default = Pleroma.Config.Holder.default_config(group, key)
value = ConfigDB.from_binary(setting.value)
merged_value =
if Ecto.get_meta(setting, :state) == :deleted do
default
defp maybe_set_pleroma_last(apps) do
# to be ensured that pleroma will be restarted last
if :pleroma in apps do
apps
|> List.delete(:pleroma)
|> List.insert_at(-1, :pleroma)
else
if can_be_merged?(default, value) do
ConfigDB.merge_group(group, key, default, value)
else
value
Restarter.Pleroma.rebooted()
apps
end
end
:ok = update_env(group, key, merged_value)
if group != :logger do
if group != :pleroma or pleroma_need_restart?(group, key, value) do
group
end
else
defp group_for_restart(:logger, key, _, merged_value) do
# change logger configuration in runtime, without restart
if Keyword.keyword?(merged_value) and
key not in [:compile_time_application, :backends, :compile_time_purge_matching] do
@ -114,6 +92,31 @@ defp merge_and_update(setting) do
nil
end
defp group_for_restart(group, _, _, _) when group != :pleroma, do: group
defp group_for_restart(group, key, value, _) do
if pleroma_need_restart?(group, key, value), do: group
end
defp merge_and_update(setting) do
try do
key = ConfigDB.from_string(setting.key)
group = ConfigDB.from_string(setting.group)
default = Config.Holder.default_config(group, key)
value = ConfigDB.from_binary(setting.value)
merged_value =
cond do
Ecto.get_meta(setting, :state) == :deleted -> default
can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value)
true -> value
end
:ok = update_env(group, key, merged_value)
group_for_restart(group, key, value, merged_value)
rescue
error ->
error_msg =

11
lib/pleroma/conversation/participation.ex
View File

@ -129,21 +129,18 @@ def for_user(user, params \\ %{}) do
end
def restrict_recipients(query, user, %{"recipients" => user_ids}) do
user_ids =
user_binary_ids =
[user.id | user_ids]
|> Enum.uniq()
|> Enum.reduce([], fn user_id, acc ->
{:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id)
[user_id | acc]
end)
|> User.binary_id()
conversation_subquery =
__MODULE__
|> group_by([p], p.conversation_id)
|> having(
[p],
count(p.user_id) == ^length(user_ids) and
fragment("array_agg(?) @> ?", p.user_id, ^user_ids)
count(p.user_id) == ^length(user_binary_ids) and
fragment("array_agg(?) @> ?", p.user_id, ^user_binary_ids)
)
|> select([p], %{id: p.conversation_id})

28
lib/pleroma/following_relationship.ex
View File

@ -129,4 +129,32 @@ def move_following(origin, target) do
move_following(origin, target)
end
end
def all_between_user_sets(
source_users,
target_users
)
when is_list(source_users) and is_list(target_users) do
source_user_ids = User.binary_id(source_users)
target_user_ids = User.binary_id(target_users)
__MODULE__
|> where(
fragment(
"(follower_id = ANY(?) AND following_id = ANY(?)) OR \
(follower_id = ANY(?) AND following_id = ANY(?))",
^source_user_ids,
^target_user_ids,
^target_user_ids,
^source_user_ids
)
)
|> Repo.all()
end
def find(following_relationships, follower, following) do
Enum.find(following_relationships, fn
fr -> fr.follower_id == follower.id and fr.following_id == following.id
end)
end
end

45
lib/pleroma/gun/api.ex Normal file
View File

@ -0,0 +1,45 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Gun.API do
@behaviour Pleroma.Gun
alias Pleroma.Gun
@gun_keys [
:connect_timeout,
:http_opts,
:http2_opts,
:protocols,
:retry,
:retry_timeout,
:trace,
:transport,
:tls_opts,
:tcp_opts,
:socks_opts,
:ws_opts
]
@impl Gun
def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun_keys))
@impl Gun
defdelegate info(pid), to: :gun
@impl Gun
defdelegate close(pid), to: :gun
@impl Gun
defdelegate await_up(pid, timeout \\ 5_000), to: :gun
@impl Gun
defdelegate connect(pid, opts), to: :gun
@impl Gun
defdelegate await(pid, ref), to: :gun
@impl Gun
defdelegate set_owner(pid, owner), to: :gun
end

196
lib/pleroma/gun/conn.ex Normal file
View File

@ -0,0 +1,196 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Gun.Conn do
@moduledoc """
Struct for gun connection data
"""
alias Pleroma.Gun
alias Pleroma.Pool.Connections
require Logger
@type gun_state :: :up | :down
@type conn_state :: :active | :idle
@type t :: %__MODULE__{
conn: pid(),
gun_state: gun_state(),
conn_state: conn_state(),
used_by: [pid()],
last_reference: pos_integer(),
crf: float(),
retries: pos_integer()
}
defstruct conn: nil,
gun_state: :open,
conn_state: :init,
used_by: [],
last_reference: 0,
crf: 1,
retries: 0
@spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil
def open(url, name, opts \\ [])
def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts)
def open(%URI{} = uri, name, opts) do
pool_opts = Pleroma.Config.get([:connections_pool], [])
opts =
opts
|> Enum.into(%{})
|> Map.put_new(:retry, pool_opts[:retry] || 1)
|> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000)
|> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
|> maybe_add_tls_opts(uri)
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
conn_pid =
if Connections.count(name) < opts[:max_connection] do
do_open(uri, opts)
else
close_least_used_and_do_open(name, uri, opts)
end
if is_pid(conn_pid) do
conn = %Pleroma.Gun.Conn{
conn: conn_pid,
gun_state: :up,
conn_state: :active,
last_reference: :os.system_time(:second)
}
:ok = Gun.set_owner(conn_pid, Process.whereis(name))
Connections.add_conn(name, key, conn)
end
end
defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts
defp maybe_add_tls_opts(opts, %URI{scheme: "https", host: host}) do
tls_opts = [
verify: :verify_peer,
cacertfile: CAStore.file_path(),
depth: 20,
reuse_sessions: false,
verify_fun:
{&:ssl_verify_hostname.verify_fun/3,
[check_hostname: Pleroma.HTTP.Connection.format_host(host)]}
]
tls_opts =
if Keyword.keyword?(opts[:tls_opts]) do
Keyword.merge(tls_opts, opts[:tls_opts])
else
tls_opts
end
Map.put(opts, :tls_opts, tls_opts)
end
defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do
connect_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
with open_opts <- Map.delete(opts, :tls_opts),
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]),
stream <- Gun.connect(conn, connect_opts),
{:response, :fin, 200, _} <- Gun.await(conn, stream) do
conn
else
error ->
Logger.warn(
"Opening proxied connection to #{compose_uri_log(uri)} failed with error #{
inspect(error)
}"
)
error
end
end
defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do
version =
proxy_type
|> to_string()
|> String.last()
|> case do
"4" -> 4
_ -> 5
end
socks_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|> Map.put(:version, version)
opts =
opts
|> Map.put(:protocols, [:socks])
|> Map.put(:socks_opts, socks_opts)
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
conn
else
error ->
Logger.warn(
"Opening socks proxied connection to #{compose_uri_log(uri)} failed with error #{
inspect(error)
}"
)
error
end
end
defp do_open(%URI{host: host, port: port} = uri, opts) do
host = Pleroma.HTTP.Connection.parse_host(host)
with {:ok, conn} <- Gun.open(host, port, opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
conn
else
error ->
Logger.warn(
"Opening connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}"
)
error
end
end
defp destination_opts(%URI{host: host, port: port}) do
host = Pleroma.HTTP.Connection.parse_host(host)
%{host: host, port: port}
end
defp add_http2_opts(opts, "https", tls_opts) do
Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts})
end
defp add_http2_opts(opts, _, _), do: opts
defp close_least_used_and_do_open(name, uri, opts) do
with [{key, conn} | _conns] <- Connections.get_unused_conns(name),
:ok <- Gun.close(conn.conn) do
Connections.remove_conn(name, key)
do_open(uri, opts)
else
[] -> {:error, :pool_overflowed}
end
end
def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
"#{scheme}://#{host}#{path}"
end
end

31
lib/pleroma/gun/gun.ex Normal file
View File

@ -0,0 +1,31 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Gun do
@callback open(charlist(), pos_integer(), map()) :: {:ok, pid()}
@callback info(pid()) :: map()
@callback close(pid()) :: :ok
@callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()}
@callback connect(pid(), map()) :: reference()
@callback await(pid(), reference()) :: {:response, :fin, 200, []}
@callback set_owner(pid(), pid()) :: :ok
@api Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API)
defp api, do: @api
def open(host, port, opts), do: api().open(host, port, opts)
def info(pid), do: api().info(pid)
def close(pid), do: api().close(pid)
def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout)
def connect(pid, opts), do: api().connect(pid, opts)
def await(pid, ref), do: api().await(pid, ref)
def set_owner(pid, owner), do: api().set_owner(pid, owner)
end

41
lib/pleroma/http/adapter_helper.ex Normal file
View File

@ -0,0 +1,41 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.AdapterHelper do
alias Pleroma.HTTP.Connection
@type proxy ::
{Connection.host(), pos_integer()}
| {Connection.proxy_type(), Connection.host(), pos_integer()}
@callback options(keyword(), URI.t()) :: keyword()
@callback after_request(keyword()) :: :ok
@spec options(keyword(), URI.t()) :: keyword()
def options(opts, _uri) do
proxy = Pleroma.Config.get([:http, :proxy_url], nil)
maybe_add_proxy(opts, format_proxy(proxy))
end
@spec maybe_get_conn(URI.t(), keyword()) :: keyword()
def maybe_get_conn(_uri, opts), do: opts
@spec after_request(keyword()) :: :ok
def after_request(_opts), do: :ok
@spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil
def format_proxy(nil), do: nil
def format_proxy(proxy_url) do
case Connection.parse_proxy(proxy_url) do
{:ok, host, port} -> {host, port}
{:ok, type, host, port} -> {type, host, port}
_ -> nil
end
end
@spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword()
def maybe_add_proxy(opts, nil), do: opts
def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy)
end

77
lib/pleroma/http/adapter_helper/gun.ex Normal file
View File

@ -0,0 +1,77 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.AdapterHelper.Gun do
@behaviour Pleroma.HTTP.AdapterHelper
alias Pleroma.HTTP.AdapterHelper
alias Pleroma.Pool.Connections
require Logger
@defaults [
connect_timeout: 5_000,
domain_lookup_timeout: 5_000,
tls_handshake_timeout: 5_000,
retry: 1,
retry_timeout: 1000,
await_up_timeout: 5_000
]
@spec options(keyword(), URI.t()) :: keyword()
def options(incoming_opts \\ [], %URI{} = uri) do
proxy =
Pleroma.Config.get([:http, :proxy_url])
|> AdapterHelper.format_proxy()
config_opts = Pleroma.Config.get([:http, :adapter], [])
@defaults
|> Keyword.merge(config_opts)
|> add_scheme_opts(uri)
|> AdapterHelper.maybe_add_proxy(proxy)
|> maybe_get_conn(uri, incoming_opts)
end
@spec after_request(keyword()) :: :ok
def after_request(opts) do
if opts[:conn] && opts[:body_as] != :chunks do
Connections.checkout(opts[:conn], self(), :gun_connections)
end
:ok
end
defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
defp add_scheme_opts(opts, %{scheme: "https"}) do
opts
|> Keyword.put(:certificates_verification, true)
|> Keyword.put(:tls_opts, log_level: :warning)
end
defp maybe_get_conn(adapter_opts, uri, incoming_opts) do
{receive_conn?, opts} =
adapter_opts
|> Keyword.merge(incoming_opts)
|> Keyword.pop(:receive_conn, true)
if Connections.alive?(:gun_connections) and receive_conn? do
checkin_conn(uri, opts)
else
opts
end
end
defp checkin_conn(uri, opts) do
case Connections.checkin(uri, :gun_connections) do
nil ->
Task.start(Pleroma.Gun.Conn, :open, [uri, :gun_connections, opts])
opts
conn when is_pid(conn) ->
Keyword.merge(opts, conn: conn, close_conn: false)
end
end
end

43
lib/pleroma/http/adapter_helper/hackney.ex Normal file
View File

@ -0,0 +1,43 @@
defmodule Pleroma.HTTP.AdapterHelper.Hackney do
@behaviour Pleroma.HTTP.AdapterHelper
@defaults [
connect_timeout: 10_000,
recv_timeout: 20_000,
follow_redirect: true,
force_redirect: true,
pool: :federation
]
@spec options(keyword(), URI.t()) :: keyword()
def options(connection_opts \\ [], %URI{} = uri) do
proxy = Pleroma.Config.get([:http, :proxy_url])
config_opts = Pleroma.Config.get([:http, :adapter], [])
@defaults
|> Keyword.merge(config_opts)
|> Keyword.merge(connection_opts)
|> add_scheme_opts(uri)
|> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
end
defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts
defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do
ssl_opts = [
ssl_options: [
# Workaround for remote server certificate chain issues
partial_chain: &:hackney_connect.partial_chain/1,
# We don't support TLS v1.3 yet
versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"],
server_name_indication: to_charlist(host)
]
]
Keyword.merge(opts, ssl_opts)
end
def after_request(_), do: :ok
end

135
lib/pleroma/http/connection.ex
View File

@ -4,40 +4,121 @@
defmodule Pleroma.HTTP.Connection do
@moduledoc """
Connection for http-requests.
Configure Tesla.Client with default and customized adapter options.
"""
@hackney_options [
connect_timeout: 10_000,
recv_timeout: 20_000,
follow_redirect: true,
force_redirect: true,
pool: :federation
]
@adapter Application.get_env(:tesla, :adapter)
alias Pleroma.Config
alias Pleroma.HTTP.AdapterHelper
require Logger
@defaults [pool: :federation]
@type ip_address :: ipv4_address() | ipv6_address()
@type ipv4_address :: {0..255, 0..255, 0..255, 0..255}
@type ipv6_address ::
{0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535}
@type proxy_type() :: :socks4 | :socks5
@type host() :: charlist() | ip_address()
@doc """
Configure a client connection
# Returns
Tesla.Env.client
Merge default connection & adapter options with received ones.
"""
@spec new(Keyword.t()) :: Tesla.Env.client()
def new(opts \\ []) do
Tesla.client([], {@adapter, hackney_options(opts)})
@spec options(URI.t(), keyword()) :: keyword()
def options(%URI{} = uri, opts \\ []) do
@defaults
|> pool_timeout()
|> Keyword.merge(opts)
|> adapter_helper().options(uri)
end
# fetch Hackney options
#
def hackney_options(opts) do
options = Keyword.get(opts, :adapter, [])
adapter_options = Pleroma.Config.get([:http, :adapter], [])
proxy_url = Pleroma.Config.get([:http, :proxy_url], nil)
defp pool_timeout(opts) do
{config_key, default} =
if adapter() == Tesla.Adapter.Gun do
{:pools, Config.get([:pools, :default, :timeout])}
else
{:hackney_pools, 10_000}
end
@hackney_options
|> Keyword.merge(adapter_options)
|> Keyword.merge(options)
|> Keyword.merge(proxy: proxy_url)
timeout = Config.get([config_key, opts[:pool], :timeout], default)
Keyword.merge(opts, timeout: timeout)
end
@spec after_request(keyword()) :: :ok
def after_request(opts), do: adapter_helper().after_request(opts)
defp adapter, do: Application.get_env(:tesla, :adapter)
defp adapter_helper do
case adapter() do
Tesla.Adapter.Gun -> AdapterHelper.Gun
Tesla.Adapter.Hackney -> AdapterHelper.Hackney
_ -> AdapterHelper
end
end
@spec parse_proxy(String.t() | tuple() | nil) ::
{:ok, host(), pos_integer()}
| {:ok, proxy_type(), host(), pos_integer()}
| {:error, atom()}
| nil
def parse_proxy(nil), do: nil
def parse_proxy(proxy) when is_binary(proxy) do
with [host, port] <- String.split(proxy, ":"),
{port, ""} <- Integer.parse(port) do
{:ok, parse_host(host), port}
else
{_, _} ->
Logger.warn("Parsing port failed #{inspect(proxy)}")
{:error, :invalid_proxy_port}
:error ->
Logger.warn("Parsing port failed #{inspect(proxy)}")
{:error, :invalid_proxy_port}
_ ->
Logger.warn("Parsing proxy failed #{inspect(proxy)}")
{:error, :invalid_proxy}
end
end
def parse_proxy(proxy) when is_tuple(proxy) do
with {type, host, port} <- proxy do
{:ok, type, parse_host(host), port}
else
_ ->
Logger.warn("Parsing proxy failed #{inspect(proxy)}")
{:error, :invalid_proxy}
end
end
@spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address()
def parse_host(host) when is_list(host), do: host
def parse_host(host) when is_atom(host), do: to_charlist(host)
def parse_host(host) when is_binary(host) do
host = to_charlist(host)
case :inet.parse_address(host) do
{:error, :einval} -> host
{:ok, ip} -> ip
end
end
@spec format_host(String.t()) :: charlist()
def format_host(host) do
host_charlist = to_charlist(host)
case :inet.parse_address(host_charlist) do
{:error, :einval} ->
:idna.encode(host_charlist)
{:ok, _ip} ->
host_charlist
end
end
end

135
lib/pleroma/http/http.ex
View File

@ -4,21 +4,47 @@
defmodule Pleroma.HTTP do
@moduledoc """
Wrapper for `Tesla.request/2`.
"""
alias Pleroma.HTTP.Connection
alias Pleroma.HTTP.Request
alias Pleroma.HTTP.RequestBuilder, as: Builder
alias Tesla.Client
alias Tesla.Env
require Logger
@type t :: __MODULE__
@doc """
Builds and perform http request.
Performs GET request.
See `Pleroma.HTTP.request/5`
"""
@spec get(Request.url() | nil, Request.headers(), keyword()) ::
nil | {:ok, Env.t()} | {:error, any()}
def get(url, headers \\ [], options \\ [])
def get(nil, _, _), do: nil
def get(url, headers, options), do: request(:get, url, "", headers, options)
@doc """
Performs POST request.
See `Pleroma.HTTP.request/5`
"""
@spec post(Request.url(), String.t(), Request.headers(), keyword()) ::
{:ok, Env.t()} | {:error, any()}
def post(url, body, headers \\ [], options \\ []),
do: request(:post, url, body, headers, options)
@doc """
Builds and performs http request.
# Arguments:
`method` - :get, :post, :put, :delete
`url`
`body`
`url` - full url
`body` - request body
`headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]`
`options` - custom, per-request middleware or adapter options
@ -26,61 +52,66 @@ defmodule Pleroma.HTTP do
`{:ok, %Tesla.Env{}}` or `{:error, error}`
"""
def request(method, url, body \\ "", headers \\ [], options \\ []) do
try do
options =
process_request_options(options)
|> process_sni_options(url)
@spec request(atom(), Request.url(), String.t(), Request.headers(), keyword()) ::
{:ok, Env.t()} | {:error, any()}
def request(method, url, body, headers, options) when is_binary(url) do
uri = URI.parse(url)
adapter_opts = Connection.options(uri, options[:adapter] || [])
options = put_in(options[:adapter], adapter_opts)
params = options[:params] || []
request = build_request(method, headers, options, url, body, params)
params = Keyword.get(options, :params, [])
adapter = Application.get_env(:tesla, :adapter)
client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter)
%{}
pid = Process.whereis(adapter_opts[:pool])
pool_alive? =
if adapter == Tesla.Adapter.Gun && pid do
Process.alive?(pid)
else
false
end
request_opts =
adapter_opts
|> Enum.into(%{})
|> Map.put(:env, Pleroma.Config.get([:env]))
|> Map.put(:pool_alive?, pool_alive?)
response = request(client, request, request_opts)
Connection.after_request(adapter_opts)
response
end
@spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()}
def request(%Client{} = client, request, %{env: :test}), do: request(client, request)
def request(%Client{} = client, request, %{body_as: :chunks}), do: request(client, request)
def request(%Client{} = client, request, %{pool_alive?: false}), do: request(client, request)
def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do
:poolboy.transaction(
pool,
&Pleroma.Pool.Request.execute(&1, client, request, timeout),
timeout
)
end
@spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()}
def request(client, request), do: Tesla.request(client, request)
defp build_request(method, headers, options, url, body, params) do
Builder.new()
|> Builder.method(method)
|> Builder.headers(headers)
|> Builder.opts(options)
|> Builder.url(url)
|> Builder.add_param(:body, :body, body)
|> Builder.add_param(:query, :query, params)
|> Enum.into([])
|> (&Tesla.request(Connection.new(options), &1)).()
rescue
e ->
{:error, e}
catch
:exit, e ->
{:error, e}
|> Builder.convert_to_keyword()
end
end
defp process_sni_options(options, nil), do: options
defp process_sni_options(options, url) do
uri = URI.parse(url)
host = uri.host |> to_charlist()
case uri.scheme do
"https" -> options ++ [ssl: [server_name_indication: host]]
_ -> options
end
end
def process_request_options(options) do
Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options)
end
@doc """
Performs GET request.
See `Pleroma.HTTP.request/5`
"""
def get(url, headers \\ [], options \\ []),
do: request(:get, url, "", headers, options)
@doc """
Performs POST request.
See `Pleroma.HTTP.request/5`
"""
def post(url, body, headers \\ [], options \\ []),
do: request(:post, url, body, headers, options)
end

23
lib/pleroma/http/request.ex Normal file
View File

@ -0,0 +1,23 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.Request do
@moduledoc """
Request struct.
"""
defstruct method: :get, url: "", query: [], headers: [], body: "", opts: []
@type method :: :head | :get | :delete | :trace | :options | :post | :put | :patch
@type url :: String.t()
@type headers :: [{String.t(), String.t()}]
@type t :: %__MODULE__{
method: method(),
url: url(),
query: keyword(),
headers: headers(),
body: String.t(),
opts: keyword()
}
end

123
lib/pleroma/http/request_builder.ex
View File

@ -7,136 +7,87 @@ defmodule Pleroma.HTTP.RequestBuilder do
Helper functions for building Tesla requests
"""
alias Pleroma.HTTP.Request
alias Tesla.Multipart
@doc """
Specify the request method when building a request
## Parameters
- request (Map) - Collected request options
- m (atom) - Request method
## Returns
Map
Creates new request
"""
@spec method(map(), atom) :: map()
def method(request, m) do
Map.put_new(request, :method, m)
end
@spec new(Request.t()) :: Request.t()
def new(%Request{} = request \\ %Request{}), do: request
@doc """
Specify the request method when building a request
## Parameters
- request (Map) - Collected request options
- u (String) - Request URL
## Returns
Map
"""
@spec url(map(), String.t()) :: map()
def url(request, u) do
Map.put_new(request, :url, u)
end
@spec method(Request.t(), Request.method()) :: Request.t()
def method(request, m), do: %{request | method: m}
@doc """
Specify the request method when building a request
"""
@spec url(Request.t(), Request.url()) :: Request.t()
def url(request, u), do: %{request | url: u}
@doc """
Add headers to the request
"""
@spec headers(map(), list(tuple)) :: map()
def headers(request, header_list) do
header_list =
@spec headers(Request.t(), Request.headers()) :: Request.t()
def headers(request, headers) do
headers_list =
if Pleroma.Config.get([:http, :send_user_agent]) do
header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}]
[{"user-agent", Pleroma.Application.user_agent()} | headers]
else
header_list
headers
end
Map.put_new(request, :headers, header_list)
%{request | headers: headers_list}
end
@doc """
Add custom, per-request middleware or adapter options to the request
"""
@spec opts(map(), Keyword.t()) :: map()
def opts(request, options) do
Map.put_new(request, :opts, options)
end
@spec opts(Request.t(), keyword()) :: Request.t()
def opts(request, options), do: %{request | opts: options}
@doc """
Add optional parameters to the request
## Parameters
- request (Map) - Collected request options
- definitions (Map) - Map of parameter name to parameter location.
- options (KeywordList) - The provided optional parameters
## Returns
Map
"""
@spec add_optional_params(map(), %{optional(atom) => atom}, keyword()) :: map()
def add_optional_params(request, _, []), do: request
@spec add_param(Request.t(), atom(), atom(), any()) :: Request.t()
def add_param(request, :query, :query, values), do: %{request | query: values}
def add_optional_params(request, definitions, [{key, value} | tail]) do
case definitions do
%{^key => location} ->
request
|> add_param(location, key, value)
|> add_optional_params(definitions, tail)
_ ->
add_optional_params(request, definitions, tail)
end
end
@doc """
Add optional parameters to the request
## Parameters
- request (Map) - Collected request options
- location (atom) - Where to put the parameter
- key (atom) - The name of the parameter
- value (any) - The value of the parameter
## Returns
Map
"""
@spec add_param(map(), atom, atom, any()) :: map()
def add_param(request, :query, :query, values), do: Map.put(request, :query, values)
def add_param(request, :body, :body, value), do: Map.put(request, :body, value)
def add_param(request, :body, :body, value), do: %{request | body: value}
def add_param(request, :body, key, value) do
request
|> Map.put_new_lazy(:body, &Tesla.Multipart.new/0)
|> Map.put(:body, Multipart.new())
|> Map.update!(
:body,
&Tesla.Multipart.add_field(
&Multipart.add_field(
&1,
key,
Jason.encode!(value),
headers: [{:"Content-Type", "application/json"}]
headers: [{"content-type", "application/json"}]
)
)
end
def add_param(request, :file, name, path) do
request
|> Map.put_new_lazy(:body, &Tesla.Multipart.new/0)
|> Map.update!(:body, &Tesla.Multipart.add_file(&1, path, name: name))
|> Map.put(:body, Multipart.new())
|> Map.update!(:body, &Multipart.add_file(&1, path, name: name))
end
def add_param(request, :form, name, value) do
request
|> Map.update(:body, %{name => value}, &Map.put(&1, name, value))
Map.update(request, :body, %{name => value}, &Map.put(&1, name, value))
end
def add_param(request, location, key, value) do
Map.update(request, location, [{key, value}], &(&1 ++ [{key, value}]))
end
def convert_to_keyword(request) do
request
|> Map.from_struct()
|> Enum.into([])
end
end

11
lib/pleroma/moderation_log.ex
View File

@ -605,6 +605,17 @@ def get_log_entry_message(%ModerationLog{
}"
end
@spec get_log_entry_message(ModerationLog) :: String.t()
def get_log_entry_message(%ModerationLog{
data: %{
"actor" => %{"nickname" => actor_nickname},
"action" => "updated_users",
"subject" => subjects
}
}) do
"@#{actor_nickname} updated users: #{users_to_nicknames_string(subjects)}"
end
defp nicknames_to_string(nicknames) do
nicknames
|> Enum.map(&"@#{&1}")

122
lib/pleroma/notification.ex
View File

@ -10,6 +10,7 @@ defmodule Pleroma.Notification do
alias Pleroma.Object
alias Pleroma.Pagination
alias Pleroma.Repo
alias Pleroma.ThreadMute
alias Pleroma.User
alias Pleroma.Web.CommonAPI.Utils
alias Pleroma.Web.Push
@ -17,6 +18,7 @@ defmodule Pleroma.Notification do
import Ecto.Query
import Ecto.Changeset
require Logger
@type t :: %__MODULE__{}
@ -37,11 +39,11 @@ def changeset(%Notification{} = notification, attrs) do
end
defp for_user_query_ap_id_opts(user, opts) do
ap_id_relations =
ap_id_relationships =
[:block] ++
if opts[@include_muted_option], do: [], else: [:notification_mute]
preloaded_ap_ids = User.outgoing_relations_ap_ids(user, ap_id_relations)
preloaded_ap_ids = User.outgoing_relationships_ap_ids(user, ap_id_relationships)
exclude_blocked_opts = Map.merge(%{blocked_users_ap_ids: preloaded_ap_ids[:block]}, opts)
@ -100,7 +102,7 @@ defp exclude_notification_muted(query, user, opts) do
query
|> where([n, a], a.actor not in ^notification_muted_ap_ids)
|> join(:left, [n, a], tm in Pleroma.ThreadMute,
|> join(:left, [n, a], tm in ThreadMute,
on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data)
)
|> where([n, a, o, tm], is_nil(tm.user_id))
@ -275,58 +277,111 @@ def dismiss(%{id: user_id} = _user, id) do
def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do
object = Object.normalize(activity)
unless object && object.data["type"] == "Answer" do
users = get_notified_from_activity(activity)
notifications = Enum.map(users, fn user -> create_notification(activity, user) end)
{:ok, notifications}
else
if object && object.data["type"] == "Answer" do
{:ok, []}
else
do_create_notifications(activity)
end
end
def create_notifications(%Activity{data: %{"type" => type}} = activity)
when type in ["Like", "Announce", "Follow", "Move", "EmojiReact"] do
notifications =
activity
|> get_notified_from_activity()
|> Enum.map(&create_notification(activity, &1))
{:ok, notifications}
do_create_notifications(activity)
end
def create_notifications(_), do: {:ok, []}
defp do_create_notifications(%Activity{} = activity) do
{enabled_receivers, disabled_receivers} = get_notified_from_activity(activity)
potential_receivers = enabled_receivers ++ disabled_receivers
notifications =
Enum.map(potential_receivers, fn user ->
do_send = user in enabled_receivers
create_notification(activity, user, do_send)
end)
{:ok, notifications}
end
# TODO move to sql, too.
def create_notification(%Activity{} = activity, %User{} = user) do
def create_notification(%Activity{} = activity, %User{} = user, do_send \\ true) do
unless skip?(activity, user) do
notification = %Notification{user_id: user.id, activity: activity}
{:ok, notification} = Repo.insert(notification)
["user", "user:notification"]
|> Streamer.stream(notification)
if do_send do
Streamer.stream(["user", "user:notification"], notification)
Push.send(notification)
end
notification
end
end
@doc """
Returns a tuple with 2 elements:
{enabled notification receivers, currently disabled receivers (blocking / [thread] muting)}
NOTE: might be called for FAKE Activities, see ActivityPub.Utils.get_notified_from_object/1
"""
def get_notified_from_activity(activity, local_only \\ true)
def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only)
when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact"] do
potential_receiver_ap_ids =
[]
|> Utils.maybe_notify_to_recipients(activity)
|> Utils.maybe_notify_mentioned_recipients(activity)
|> Utils.maybe_notify_subscribers(activity)
|> Utils.maybe_notify_followers(activity)
|> Enum.uniq()
# Since even subscribers and followers can mute / thread-mute, filtering all above AP IDs
notification_enabled_ap_ids =
potential_receiver_ap_ids
|> exclude_relationship_restricted_ap_ids(activity)
|> exclude_thread_muter_ap_ids(activity)
potential_receivers =
potential_receiver_ap_ids
|> Enum.uniq()
|> User.get_users_from_set(local_only)
notification_enabled_users =
Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
{notification_enabled_users, potential_receivers -- notification_enabled_users}
end
def get_notified_from_activity(_, _local_only), do: []
def get_notified_from_activity(_, _local_only), do: {[], []}
@doc "Filters out AP IDs of users basing on their relationships with activity actor user"
def exclude_relationship_restricted_ap_ids([], _activity), do: []
def exclude_relationship_restricted_ap_ids(ap_ids, %Activity{} = activity) do
relationship_restricted_ap_ids =
activity
|> Activity.user_actor()
|> User.incoming_relationships_ungrouped_ap_ids([
:block,
:notification_mute
])
Enum.uniq(ap_ids) -- relationship_restricted_ap_ids
end
@doc "Filters out AP IDs of users who mute activity thread"
def exclude_thread_muter_ap_ids([], _activity), do: []
def exclude_thread_muter_ap_ids(ap_ids, %Activity{} = activity) do
thread_muter_ap_ids = ThreadMute.muter_ap_ids(activity.data["context"])
Enum.uniq(ap_ids) -- thread_muter_ap_ids
end
@spec skip?(Activity.t(), User.t()) :: boolean()
def skip?(activity, user) do
def skip?(%Activity{} = activity, %User{} = user) do
[
:self,
:followers,
@ -335,18 +390,20 @@ def skip?(activity, user) do
:non_follows,
:recently_followed
]
|> Enum.any?(&skip?(&1, activity, user))
|> Enum.find(&skip?(&1, activity, user))
end
def skip?(_, _), do: false
@spec skip?(atom(), Activity.t(), User.t()) :: boolean()
def skip?(:self, activity, user) do
def skip?(:self, %Activity{} = activity, %User{} = user) do
activity.data["actor"] == user.ap_id
end
def skip?(
:followers,
activity,
%{notification_settings: %{followers: false}} = user
%Activity{} = activity,
%User{notification_settings: %{followers: false}} = user
) do
actor = activity.data["actor"]
follower = User.get_cached_by_ap_id(actor)
@ -355,15 +412,19 @@ def skip?(
def skip?(
:non_followers,
activity,
%{notification_settings: %{non_followers: false}} = user
%Activity{} = activity,
%User{notification_settings: %{non_followers: false}} = user
) do
actor = activity.data["actor"]
follower = User.get_cached_by_ap_id(actor)
!User.following?(follower, user)
end
def skip?(:follows, activity, %{notification_settings: %{follows: false}} = user) do
def skip?(
:follows,
%Activity{} = activity,
%User{notification_settings: %{follows: false}} = user
) do
actor = activity.data["actor"]
followed = User.get_cached_by_ap_id(actor)
User.following?(user, followed)
@ -371,15 +432,16 @@ def skip?(:follows, activity, %{notification_settings: %{follows: false}} = user
def skip?(
:non_follows,
activity,
%{notification_settings: %{non_follows: false}} = user
%Activity{} = activity,
%User{notification_settings: %{non_follows: false}} = user
) do
actor = activity.data["actor"]
followed = User.get_cached_by_ap_id(actor)
!User.following?(user, followed)
end
def skip?(:recently_followed, %{data: %{"type" => "Follow"}} = activity, user) do
# To do: consider defining recency in hours and checking FollowingRelationship with a single SQL
def skip?(:recently_followed, %Activity{data: %{"type" => "Follow"}} = activity, %User{} = user) do
actor = activity.data["actor"]
Notification.for_user(user)

6
lib/pleroma/object/fetcher.ex
View File

@ -141,7 +141,7 @@ defp make_signature(id, date) do
date: date
})
[{:Signature, signature}]
[{"signature", signature}]
end
defp sign_fetch(headers, id, date) do
@ -154,7 +154,7 @@ defp sign_fetch(headers, id, date) do
defp maybe_date_fetch(headers, date) do
if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
headers ++ [{:Date, date}]
headers ++ [{"date", date}]
else
headers
end
@ -166,7 +166,7 @@ def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
date = Pleroma.Signature.signed_date()
headers =
[{:Accept, "application/activity+json"}]
[{"accept", "application/activity+json"}]
|> maybe_date_fetch(date)
|> sign_fetch(id, date)

28
lib/pleroma/otp_version.ex Normal file
View File

@ -0,0 +1,28 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.OTPVersion do
@spec version() :: String.t() | nil
def version do
# OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version
[
Path.join(:code.root_dir(), "OTP_VERSION"),
Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"])
]
|> get_version_from_files()
end
@spec get_version_from_files([Path.t()]) :: String.t() | nil
def get_version_from_files([]), do: nil
def get_version_from_files([path | paths]) do
if File.exists?(path) do
path
|> File.read!()
|> String.replace(~r/\r|\n|\s/, "")
else
get_version_from_files(paths)
end
end
end

283
lib/pleroma/pool/connections.ex Normal file
View File

@ -0,0 +1,283 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Pool.Connections do
use GenServer
alias Pleroma.Config
alias Pleroma.Gun
require Logger
@type domain :: String.t()
@type conn :: Pleroma.Gun.Conn.t()
@type t :: %__MODULE__{
conns: %{domain() => conn()},
opts: keyword()
}
defstruct conns: %{}, opts: []
@spec start_link({atom(), keyword()}) :: {:ok, pid()}
def start_link({name, opts}) do
GenServer.start_link(__MODULE__, opts, name: name)
end
@impl true
def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}}
@spec checkin(String.t() | URI.t(), atom()) :: pid() | nil
def checkin(url, name)
def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name)
def checkin(%URI{} = uri, name) do
timeout = Config.get([:connections_pool, :checkin_timeout], 250)
GenServer.call(name, {:checkin, uri}, timeout)
end
@spec alive?(atom()) :: boolean()
def alive?(name) do
if pid = Process.whereis(name) do
Process.alive?(pid)
else
false
end
end
@spec get_state(atom()) :: t()
def get_state(name) do
GenServer.call(name, :state)
end
@spec count(atom()) :: pos_integer()
def count(name) do
GenServer.call(name, :count)
end
@spec get_unused_conns(atom()) :: [{domain(), conn()}]
def get_unused_conns(name) do
GenServer.call(name, :unused_conns)
end
@spec checkout(pid(), pid(), atom()) :: :ok
def checkout(conn, pid, name) do
GenServer.cast(name, {:checkout, conn, pid})
end
@spec add_conn(atom(), String.t(), Pleroma.Gun.Conn.t()) :: :ok
def add_conn(name, key, conn) do
GenServer.cast(name, {:add_conn, key, conn})
end
@spec remove_conn(atom(), String.t()) :: :ok
def remove_conn(name, key) do
GenServer.cast(name, {:remove_conn, key})
end
@impl true
def handle_cast({:add_conn, key, conn}, state) do
state = put_in(state.conns[key], conn)
Process.monitor(conn.conn)
{:noreply, state}
end
@impl true
def handle_cast({:checkout, conn_pid, pid}, state) do
state =
with true <- Process.alive?(conn_pid),
{key, conn} <- find_conn(state.conns, conn_pid),
used_by <- List.keydelete(conn.used_by, pid, 0) do
conn_state = if used_by == [], do: :idle, else: conn.conn_state
put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by})
else
false ->
Logger.debug("checkout for closed conn #{inspect(conn_pid)}")
state
nil ->
Logger.debug("checkout for alive conn #{inspect(conn_pid)}, but is not in state")
state
end
{:noreply, state}
end
@impl true
def handle_cast({:remove_conn, key}, state) do
state = put_in(state.conns, Map.delete(state.conns, key))
{:noreply, state}
end
@impl true
def handle_call({:checkin, uri}, from, state) do
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
case state.conns[key] do
%{conn: pid, gun_state: :up} = conn ->
time = :os.system_time(:second)
last_reference = time - conn.last_reference
crf = crf(last_reference, 100, conn.crf)
state =
put_in(state.conns[key], %{
conn
| last_reference: time,
crf: crf,
conn_state: :active,
used_by: [from | conn.used_by]
})
{:reply, pid, state}
%{gun_state: :down} ->
{:reply, nil, state}
nil ->
{:reply, nil, state}
end
end
@impl true
def handle_call(:state, _from, state), do: {:reply, state, state}
@impl true
def handle_call(:count, _from, state) do
{:reply, Enum.count(state.conns), state}
end
@impl true
def handle_call(:unused_conns, _from, state) do
unused_conns =
state.conns
|> Enum.filter(&filter_conns/1)
|> Enum.sort(&sort_conns/2)
{:reply, unused_conns, state}
end
defp filter_conns({_, %{conn_state: :idle, used_by: []}}), do: true
defp filter_conns(_), do: false
defp sort_conns({_, c1}, {_, c2}) do
c1.crf <= c2.crf and c1.last_reference <= c2.last_reference
end
@impl true
def handle_info({:gun_up, conn_pid, _protocol}, state) do
%{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid)
host =
case :inet.ntoa(host) do
{:error, :einval} -> host
ip -> ip
end
key = "#{scheme}:#{host}:#{port}"
state =
with {key, conn} <- find_conn(state.conns, conn_pid, key),
{true, key} <- {Process.alive?(conn_pid), key} do
put_in(state.conns[key], %{
conn
| gun_state: :up,
conn_state: :active,
retries: 0
})
else
{false, key} ->
put_in(
state.conns,
Map.delete(state.conns, key)
)
nil ->
:ok = Gun.close(conn_pid)
state
end
{:noreply, state}
end
@impl true
def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do
retries = Config.get([:connections_pool, :retry], 1)
# we can't get info on this pid, because pid is dead
state =
with {key, conn} <- find_conn(state.conns, conn_pid),
{true, key} <- {Process.alive?(conn_pid), key} do
if conn.retries == retries do
:ok = Gun.close(conn.conn)
put_in(
state.conns,
Map.delete(state.conns, key)
)
else
put_in(state.conns[key], %{
conn
| gun_state: :down,
retries: conn.retries + 1
})
end
else
{false, key} ->
put_in(
state.conns,
Map.delete(state.conns, key)
)
nil ->
Logger.debug(":gun_down for conn which isn't found in state")
state
end
{:noreply, state}
end
@impl true
def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do
Logger.debug("received DOWM message for #{inspect(conn_pid)} reason -> #{inspect(reason)}")
state =
with {key, conn} <- find_conn(state.conns, conn_pid) do
Enum.each(conn.used_by, fn {pid, _ref} ->
Process.exit(pid, reason)
end)
put_in(
state.conns,
Map.delete(state.conns, key)
)
else
nil ->
Logger.debug(":DOWN for conn which isn't found in state")
state
end
{:noreply, state}
end
defp find_conn(conns, conn_pid) do
Enum.find(conns, fn {_key, conn} ->
conn.conn == conn_pid
end)
end
defp find_conn(conns, conn_pid, conn_key) do
Enum.find(conns, fn {key, conn} ->
key == conn_key and conn.conn == conn_pid
end)
end
def crf(current, steps, crf) do
1 + :math.pow(0.5, current / steps) * crf
end
end

22
lib/pleroma/pool/pool.ex Normal file
View File

@ -0,0 +1,22 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Pool do
def child_spec(opts) do
poolboy_opts =
opts
|> Keyword.put(:worker_module, Pleroma.Pool.Request)
|> Keyword.put(:name, {:local, opts[:name]})
|> Keyword.put(:size, opts[:size])
|> Keyword.put(:max_overflow, opts[:max_overflow])
%{
id: opts[:id] || {__MODULE__, make_ref()},
start: {:poolboy, :start_link, [poolboy_opts, [name: opts[:name]]]},
restart: :permanent,
shutdown: 5000,
type: :worker
}
end
end

65
lib/pleroma/pool/request.ex Normal file
View File

@ -0,0 +1,65 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Pool.Request do
use GenServer
require Logger
def start_link(args) do
GenServer.start_link(__MODULE__, args)
end
@impl true
def init(_), do: {:ok, []}
@spec execute(pid() | atom(), Tesla.Client.t(), keyword(), pos_integer()) ::
{:ok, Tesla.Env.t()} | {:error, any()}
def execute(pid, client, request, timeout) do
GenServer.call(pid, {:execute, client, request}, timeout)
end
@impl true
def handle_call({:execute, client, request}, _from, state) do
response = Pleroma.HTTP.request(client, request)
{:reply, response, state}
end
@impl true
def handle_info({:gun_data, _conn, _stream, _, _}, state) do
{:noreply, state}
end
@impl true
def handle_info({:gun_up, _conn, _protocol}, state) do
{:noreply, state}
end
@impl true
def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do
{:noreply, state}
end
@impl true
def handle_info({:gun_error, _conn, _stream, _error}, state) do
{:noreply, state}
end
@impl true
def handle_info({:gun_push, _conn, _stream, _new_stream, _method, _uri, _headers}, state) do
{:noreply, state}
end
@impl true
def handle_info({:gun_response, _conn, _stream, _, _status, _headers}, state) do
{:noreply, state}
end
@impl true
def handle_info(msg, state) do
Logger.warn("Received unexpected message #{inspect(__MODULE__)} #{inspect(msg)}")
{:noreply, state}
end
end

42
lib/pleroma/pool/supervisor.ex Normal file
View File

@ -0,0 +1,42 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Pool.Supervisor do
use Supervisor
alias Pleroma.Config
alias Pleroma.Pool
def start_link(args) do
Supervisor.start_link(__MODULE__, args, name: __MODULE__)
end
def init(_) do
conns_child = %{
id: Pool.Connections,
start:
{Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]}
}
Supervisor.init([conns_child | pools()], strategy: :one_for_one)
end
defp pools do
pools = Config.get(:pools)
pools =
if Config.get([Pleroma.Upload, :proxy_remote]) == false do
Keyword.delete(pools, :upload)
else
pools
end
for {pool_name, pool_opts} <- pools do
pool_opts
|> Keyword.put(:id, {Pool, pool_name})
|> Keyword.put(:name, pool_name)
|> Pool.child_spec()
end
end
end

26
lib/pleroma/reverse_proxy/client.ex
View File

@ -3,19 +3,23 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxy.Client do
@callback request(atom(), String.t(), [tuple()], String.t(), list()) ::
{:ok, pos_integer(), [tuple()], reference() | map()}
| {:ok, pos_integer(), [tuple()]}
@type status :: pos_integer()
@type header_name :: String.t()
@type header_value :: String.t()
@type headers :: [{header_name(), header_value()}]
@callback request(atom(), String.t(), headers(), String.t(), list()) ::
{:ok, status(), headers(), reference() | map()}
| {:ok, status(), headers()}
| {:ok, reference()}
| {:error, term()}
@callback stream_body(reference() | pid() | map()) ::
{:ok, binary()} | :done | {:error, String.t()}
@callback stream_body(map()) :: {:ok, binary(), map()} | :done | {:error, atom() | String.t()}
@callback close(reference() | pid() | map()) :: :ok
def request(method, url, headers, "", opts \\ []) do
client().request(method, url, headers, "", opts)
def request(method, url, headers, body \\ "", opts \\ []) do
client().request(method, url, headers, body, opts)
end
def stream_body(ref), do: client().stream_body(ref)
@ -23,6 +27,12 @@ def stream_body(ref), do: client().stream_body(ref)
def close(ref), do: client().close(ref)
defp client do
Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney)
:tesla
|> Application.get_env(:adapter)
|> client()
end
defp client(Tesla.Adapter.Hackney), do: Pleroma.ReverseProxy.Client.Hackney
defp client(Tesla.Adapter.Gun), do: Pleroma.ReverseProxy.Client.Tesla
defp client(_), do: Pleroma.Config.get!(Pleroma.ReverseProxy.Client)
end

24
lib/pleroma/reverse_proxy/client/hackney.ex Normal file
View File

@ -0,0 +1,24 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxy.Client.Hackney do
@behaviour Pleroma.ReverseProxy.Client
@impl true
def request(method, url, headers, body, opts \\ []) do
:hackney.request(method, url, headers, body, opts)
end
@impl true
def stream_body(ref) do
case :hackney.stream_body(ref) do
:done -> :done
{:ok, data} -> {:ok, data, ref}
{:error, error} -> {:error, error}
end
end
@impl true
def close(ref), do: :hackney.close(ref)
end

90
lib/pleroma/reverse_proxy/client/tesla.ex Normal file
View File

@ -0,0 +1,90 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxy.Client.Tesla do
@behaviour Pleroma.ReverseProxy.Client
@type headers() :: [{String.t(), String.t()}]
@type status() :: pos_integer()
@spec request(atom(), String.t(), headers(), String.t(), keyword()) ::
{:ok, status(), headers}
| {:ok, status(), headers, map()}
| {:error, atom() | String.t()}
| no_return()
@impl true
def request(method, url, headers, body, opts \\ []) do
check_adapter()
opts = Keyword.put(opts, :body_as, :chunks)
with {:ok, response} <-
Pleroma.HTTP.request(
method,
url,
body,
headers,
Keyword.put(opts, :adapter, opts)
) do
if is_map(response.body) and method != :head do
{:ok, response.status, response.headers, response.body}
else
{:ok, response.status, response.headers}
end
else
{:error, error} -> {:error, error}
end
end
@impl true
@spec stream_body(map()) ::
{:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return()
def stream_body(%{pid: pid, opts: opts, fin: true}) do
# if connection was reused, but in tesla were redirects,
# tesla returns new opened connection, which must be closed manually
if opts[:old_conn], do: Tesla.Adapter.Gun.close(pid)
# if there were redirects we need to checkout old conn
conn = opts[:old_conn] || opts[:conn]
if conn, do: :ok = Pleroma.Pool.Connections.checkout(conn, self(), :gun_connections)
:done
end
def stream_body(client) do
case read_chunk!(client) do
{:fin, body} ->
{:ok, body, Map.put(client, :fin, true)}
{:nofin, part} ->
{:ok, part, client}
{:error, error} ->
{:error, error}
end
end
defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do
adapter = check_adapter()
adapter.read_chunk(pid, stream, opts)
end
@impl true
@spec close(map) :: :ok | no_return()
def close(%{pid: pid}) do
adapter = check_adapter()
adapter.close(pid)
end
defp check_adapter do
adapter = Application.get_env(:tesla, :adapter)
unless adapter == Tesla.Adapter.Gun do
raise "#{adapter} doesn't support reading body in chunks"
end
adapter
end
end

20
lib/pleroma/reverse_proxy/reverse_proxy.ex
View File

@ -3,8 +3,6 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxy do
alias Pleroma.HTTP
@keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++
~w(if-unmodified-since if-none-match if-range range)
@resp_cache_headers ~w(etag date last-modified)
@ -58,10 +56,10 @@ defmodule Pleroma.ReverseProxy do
* `req_headers`, `resp_headers` additional headers.
* `http`: options for [hackney](https://github.com/benoitc/hackney).
* `http`: options for [hackney](https://github.com/benoitc/hackney) or [gun](https://github.com/ninenines/gun).
"""
@default_hackney_options [pool: :media]
@default_options [pool: :media]
@inline_content_types [
"image/gif",
@ -94,11 +92,7 @@ defmodule Pleroma.ReverseProxy do
def call(_conn, _url, _opts \\ [])
def call(conn = %{method: method}, url, opts) when method in @methods do
hackney_opts =
Pleroma.HTTP.Connection.hackney_options([])
|> Keyword.merge(@default_hackney_options)
|> Keyword.merge(Keyword.get(opts, :http, []))
|> HTTP.process_request_options()
client_opts = Keyword.merge(@default_options, Keyword.get(opts, :http, []))
req_headers = build_req_headers(conn.req_headers, opts)
@ -110,7 +104,7 @@ def call(conn = %{method: method}, url, opts) when method in @methods do
end
with {:ok, nil} <- Cachex.get(:failed_proxy_url_cache, url),
{:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts),
{:ok, code, headers, client} <- request(method, url, req_headers, client_opts),
:ok <-
header_length_constraint(
headers,
@ -156,11 +150,11 @@ def call(conn, _, _) do
|> halt()
end
defp request(method, url, headers, hackney_opts) do
defp request(method, url, headers, opts) do
Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
method = method |> String.downcase() |> String.to_existing_atom()
case client().request(method, url, headers, "", hackney_opts) do
case client().request(method, url, headers, "", opts) do
{:ok, code, headers, client} when code in @valid_resp_codes ->
{:ok, code, downcase_headers(headers), client}
@ -210,7 +204,7 @@ defp chunk_reply(conn, client, opts, sent_so_far, duration) do
duration,
Keyword.get(opts, :max_read_duration, @max_read_duration)
),
{:ok, data} <- client().stream_body(client),
{:ok, data, client} <- client().stream_body(client),
{:ok, duration} <- increase_read_duration(duration),
sent_so_far = sent_so_far + byte_size(data),
:ok <-

44
lib/pleroma/thread_mute.ex
View File

@ -9,7 +9,8 @@ defmodule Pleroma.ThreadMute do
alias Pleroma.ThreadMute
alias Pleroma.User
require Ecto.Query
import Ecto.Changeset
import Ecto.Query
schema "thread_mutes" do
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
@ -18,19 +19,44 @@ defmodule Pleroma.ThreadMute do
def changeset(mute, params \\ %{}) do
mute
|> Ecto.Changeset.cast(params, [:user_id, :context])
|> Ecto.Changeset.foreign_key_constraint(:user_id)
|> Ecto.Changeset.unique_constraint(:user_id, name: :unique_index)
|> cast(params, [:user_id, :context])
|> foreign_key_constraint(:user_id)
|> unique_constraint(:user_id, name: :unique_index)
end
def query(user_id, context) do
{:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id)
user_binary_id = User.binary_id(user_id)
ThreadMute
|> Ecto.Query.where(user_id: ^user_id)
|> Ecto.Query.where(context: ^context)
|> where(user_id: ^user_binary_id)
|> where(context: ^context)
end
def muters_query(context) do
ThreadMute
|> join(:inner, [tm], u in assoc(tm, :user))
|> where([tm], tm.context == ^context)
|> select([tm, u], u.ap_id)
end
def muter_ap_ids(context, ap_ids \\ nil)
# Note: applies to fake activities (ActivityPub.Utils.get_notified_from_object/1 etc.)
def muter_ap_ids(context, _ap_ids) when is_nil(context), do: []
def muter_ap_ids(context, ap_ids) do
context
|> muters_query()
|> maybe_filter_on_ap_id(ap_ids)
|> Repo.all()
end
defp maybe_filter_on_ap_id(query, ap_ids) when is_list(ap_ids) do
where(query, [tm, u], u.ap_id in ^ap_ids)
end
defp maybe_filter_on_ap_id(query, _ap_ids), do: query
def add_mute(user_id, context) do
%ThreadMute{}
|> changeset(%{user_id: user_id, context: context})
@ -42,8 +68,8 @@ def remove_mute(user_id, context) do
|> Repo.delete_all()
end
def check_muted(user_id, context) do
def exists?(user_id, context) do
query(user_id, context)
|> Repo.all()
|> Repo.exists?()
end
end

212
lib/pleroma/user.ex
View File

@ -150,22 +150,26 @@ defmodule Pleroma.User do
{outgoing_relation, outgoing_relation_target},
{incoming_relation, incoming_relation_source}
]} <- @user_relationships_config do
# Definitions of `has_many :blocker_blocks`, `has_many :muter_mutes` etc.
# Definitions of `has_many` relations: :blocker_blocks, :muter_mutes, :reblog_muter_mutes,
# :notification_muter_mutes, :subscribee_subscriptions
has_many(outgoing_relation, UserRelationship,
foreign_key: :source_id,
where: [relationship_type: relationship_type]
)
# Definitions of `has_many :blockee_blocks`, `has_many :mutee_mutes` etc.
# Definitions of `has_many` relations: :blockee_blocks, :mutee_mutes, :reblog_mutee_mutes,
# :notification_mutee_mutes, :subscriber_subscriptions
has_many(incoming_relation, UserRelationship,
foreign_key: :target_id,
where: [relationship_type: relationship_type]
)
# Definitions of `has_many :blocked_users`, `has_many :muted_users` etc.
# Definitions of `has_many` relations: :blocked_users, :muted_users, :reblog_muted_users,
# :notification_muted_users, :subscriber_users
has_many(outgoing_relation_target, through: [outgoing_relation, :target])
# Definitions of `has_many :blocker_users`, `has_many :muter_users` etc.
# Definitions of `has_many` relations: :blocker_users, :muter_users, :reblog_muter_users,
# :notification_muter_users, :subscribee_users
has_many(incoming_relation_source, through: [incoming_relation, :source])
end
@ -185,7 +189,9 @@ defmodule Pleroma.User do
for {_relationship_type, [{_outgoing_relation, outgoing_relation_target}, _]} <-
@user_relationships_config do
# Definitions of `blocked_users_relation/1`, `muted_users_relation/1`, etc.
# `def blocked_users_relation/2`, `def muted_users_relation/2`,
# `def reblog_muted_users_relation/2`, `def notification_muted_users/2`,
# `def subscriber_users/2`
def unquote(:"#{outgoing_relation_target}_relation")(user, restrict_deactivated? \\ false) do
target_users_query = assoc(user, unquote(outgoing_relation_target))
@ -196,7 +202,8 @@ def unquote(:"#{outgoing_relation_target}_relation")(user, restrict_deactivated?
end
end
# Definitions of `blocked_users/1`, `muted_users/1`, etc.
# `def blocked_users/2`, `def muted_users/2`, `def reblog_muted_users/2`,
# `def notification_muted_users/2`, `def subscriber_users/2`
def unquote(outgoing_relation_target)(user, restrict_deactivated? \\ false) do
__MODULE__
|> apply(unquote(:"#{outgoing_relation_target}_relation"), [
@ -206,7 +213,8 @@ def unquote(outgoing_relation_target)(user, restrict_deactivated? \\ false) do
|> Repo.all()
end
# Definitions of `blocked_users_ap_ids/1`, `muted_users_ap_ids/1`, etc.
# `def blocked_users_ap_ids/2`, `def muted_users_ap_ids/2`, `def reblog_muted_users_ap_ids/2`,
# `def notification_muted_users_ap_ids/2`, `def subscriber_users_ap_ids/2`
def unquote(:"#{outgoing_relation_target}_ap_ids")(user, restrict_deactivated? \\ false) do
__MODULE__
|> apply(unquote(:"#{outgoing_relation_target}_relation"), [
@ -218,6 +226,24 @@ def unquote(:"#{outgoing_relation_target}_ap_ids")(user, restrict_deactivated? \
end
end
@doc """
Dumps Flake Id to SQL-compatible format (16-byte UUID).
E.g. "9pQtDGXuq4p3VlcJEm" -> <<0, 0, 1, 110, 179, 218, 42, 92, 213, 41, 44, 227, 95, 213, 0, 0>>
"""
def binary_id(source_id) when is_binary(source_id) do
with {:ok, dumped_id} <- FlakeId.Ecto.CompatType.dump(source_id) do
dumped_id
else
_ -> source_id
end
end
def binary_id(source_ids) when is_list(source_ids) do
Enum.map(source_ids, &binary_id/1)
end
def binary_id(%User{} = user), do: binary_id(user.id)
@doc "Returns status account"
@spec account_status(User.t()) :: account_status()
def account_status(%User{deactivated: true}), do: :deactivated
@ -237,7 +263,18 @@ def visible_for?(user, for_user \\ nil)
def visible_for?(%User{invisible: true}, _), do: false
def visible_for?(%User{id: user_id}, %User{id: for_id}) when user_id == for_id, do: true
def visible_for?(%User{id: user_id}, %User{id: user_id}), do: true
def visible_for?(%User{local: local} = user, nil) do
cfg_key =
if local,
do: :local,
else: :remote
if Config.get([:restrict_unauthenticated, :profiles, cfg_key]),
do: false,
else: account_status(user) == :active
end
def visible_for?(%User{} = user, for_user) do
account_status(user) == :active || superuser?(for_user)
@ -268,37 +305,15 @@ def banner_url(user, options \\ []) do
end
end
def profile_url(%User{source_data: %{"url" => url}}), do: url
def profile_url(%User{ap_id: ap_id}), do: ap_id
def profile_url(_), do: nil
def ap_id(%User{nickname: nickname}), do: "#{Web.base_url()}/users/#{nickname}"
def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa
def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers"
@spec ap_following(User.t()) :: Sring.t()
@spec ap_following(User.t()) :: String.t()
def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa
def ap_following(%User{} = user), do: "#{ap_id(user)}/following"
def follow_state(%User{} = user, %User{} = target) do
case Utils.fetch_latest_follow(user, target) do
%{data: %{"state" => state}} -> state
# Ideally this would be nil, but then Cachex does not commit the value
_ -> false
end
end
def get_cached_follow_state(user, target) do
key = "follow_state:#{user.ap_id}|#{target.ap_id}"
Cachex.fetch!(:user_cache, key, fn _ -> {:commit, follow_state(user, target)} end)
end
@spec set_follow_state_cache(String.t(), String.t(), String.t()) :: {:ok | :error, boolean()}
def set_follow_state_cache(user_ap_id, target_ap_id, state) do
Cachex.put(:user_cache, "follow_state:#{user_ap_id}|#{target_ap_id}", state)
end
@spec restrict_deactivated(Ecto.Query.t()) :: Ecto.Query.t()
def restrict_deactivated(query) do
from(u in query, where: u.deactivated != ^true)
@ -417,9 +432,55 @@ def update_changeset(struct, params \\ %{}) do
|> validate_format(:nickname, local_nickname_regex())
|> validate_length(:bio, max: bio_limit)
|> validate_length(:name, min: 1, max: name_limit)
|> put_fields()
|> put_change_if_present(:bio, &{:ok, parse_bio(&1, struct)})
|> put_change_if_present(:avatar, &put_upload(&1, :avatar))
|> put_change_if_present(:banner, &put_upload(&1, :banner))
|> put_change_if_present(:background, &put_upload(&1, :background))
|> put_change_if_present(
:pleroma_settings_store,
&{:ok, Map.merge(struct.pleroma_settings_store, &1)}
)
|> validate_fields(false)
end
defp put_fields(changeset) do
if raw_fields = get_change(changeset, :raw_fields) do
raw_fields =
raw_fields
|> Enum.filter(fn %{"name" => n} -> n != "" end)
fields =
raw_fields
|> Enum.map(fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end)
changeset
|> put_change(:raw_fields, raw_fields)
|> put_change(:fields, fields)
else
changeset
end
end
defp put_change_if_present(changeset, map_field, value_function) do
if value = get_change(changeset, map_field) do
with {:ok, new_value} <- value_function.(value) do
put_change(changeset, map_field, new_value)
else
_ -> changeset
end
else
changeset
end
end
defp put_upload(value, type) do
with %Plug.Upload{} <- value,
{:ok, object} <- ActivityPub.upload(value, type: type) do
{:ok, object.data}
end
end
def upgrade_changeset(struct, params \\ %{}, remote? \\ false) do
bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000)
name_limit = Pleroma.Config.get([:instance, :user_name_length], 100)
@ -463,6 +524,27 @@ def upgrade_changeset(struct, params \\ %{}, remote? \\ false) do
|> validate_fields(remote?)
end
def update_as_admin_changeset(struct, params) do
struct
|> update_changeset(params)
|> cast(params, [:email])
|> delete_change(:also_known_as)
|> unique_constraint(:email)
|> validate_format(:email, @email_regex)
end
@spec update_as_admin(%User{}, map) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
def update_as_admin(user, params) do
params = Map.put(params, "password_confirmation", params["password"])
changeset = update_as_admin_changeset(user, params)
if params["password"] do
reset_password(user, changeset, params)
else
User.update_and_set_cache(changeset)
end
end
def password_update_changeset(struct, params) do
struct
|> cast(params, [:password, :password_confirmation])
@ -473,10 +555,14 @@ def password_update_changeset(struct, params) do
end
@spec reset_password(User.t(), map) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
def reset_password(%User{id: user_id} = user, data) do
def reset_password(%User{} = user, params) do
reset_password(user, user, params)
end
def reset_password(%User{id: user_id} = user, struct, params) do
multi =
Multi.new()
|> Multi.update(:user, password_update_changeset(user, data))
|> Multi.update(:user, password_update_changeset(struct, params))
|> Multi.delete_all(:tokens, OAuth.Token.Query.get_by_user(user_id))
|> Multi.delete_all(:auth, OAuth.Authorization.delete_by_user_query(user))
@ -681,7 +767,14 @@ def unfollow(%User{} = follower, %User{} = followed) do
def get_follow_state(%User{} = follower, %User{} = following) do
following_relationship = FollowingRelationship.get(follower, following)
get_follow_state(follower, following, following_relationship)
end
def get_follow_state(
%User{} = follower,
%User{} = following,
following_relationship
) do
case {following_relationship, following.local} do
{nil, false} ->
case Utils.fetch_latest_follow(follower, following) do
@ -1214,13 +1307,15 @@ def subscribed_to?(%User{} = user, %{ap_id: ap_id}) do
end
@doc """
Returns map of outgoing (blocked, muted etc.) relations' user AP IDs by relation type.
E.g. `outgoing_relations_ap_ids(user, [:block])` -> `%{block: ["https://some.site/users/userapid"]}`
Returns map of outgoing (blocked, muted etc.) relationships' user AP IDs by relation type.
E.g. `outgoing_relationships_ap_ids(user, [:block])` -> `%{block: ["https://some.site/users/userapid"]}`
"""
@spec outgoing_relations_ap_ids(User.t(), list(atom())) :: %{atom() => list(String.t())}
def outgoing_relations_ap_ids(_, []), do: %{}
@spec outgoing_relationships_ap_ids(User.t(), list(atom())) :: %{atom() => list(String.t())}
def outgoing_relationships_ap_ids(_user, []), do: %{}
def outgoing_relations_ap_ids(%User{} = user, relationship_types)
def outgoing_relationships_ap_ids(nil, _relationship_types), do: %{}
def outgoing_relationships_ap_ids(%User{} = user, relationship_types)
when is_list(relationship_types) do
db_result =
user
@ -1239,6 +1334,30 @@ def outgoing_relations_ap_ids(%User{} = user, relationship_types)
)
end
def incoming_relationships_ungrouped_ap_ids(user, relationship_types, ap_ids \\ nil)
def incoming_relationships_ungrouped_ap_ids(_user, [], _ap_ids), do: []
def incoming_relationships_ungrouped_ap_ids(nil, _relationship_types, _ap_ids), do: []
def incoming_relationships_ungrouped_ap_ids(%User{} = user, relationship_types, ap_ids)
when is_list(relationship_types) do
user
|> assoc(:incoming_relationships)
|> join(:inner, [user_rel], u in assoc(user_rel, :source))
|> where([user_rel, u], user_rel.relationship_type in ^relationship_types)
|> maybe_filter_on_ap_id(ap_ids)
|> select([user_rel, u], u.ap_id)
|> distinct(true)
|> Repo.all()
end
defp maybe_filter_on_ap_id(query, ap_ids) when is_list(ap_ids) do
where(query, [user_rel, u], u.ap_id in ^ap_ids)
end
defp maybe_filter_on_ap_id(query, _ap_ids), do: query
def deactivate_async(user, status \\ true) do
BackgroundWorker.enqueue("deactivate_user", %{"user_id" => user.id, "status" => status})
end
@ -1649,8 +1768,12 @@ def all_superusers do
|> Repo.all()
end
def muting_reblogs?(%User{} = user, %User{} = target) do
UserRelationship.reblog_mute_exists?(user, target)
end
def showing_reblogs?(%User{} = user, %User{} = target) do
not UserRelationship.reblog_mute_exists?(user, target)
not muting_reblogs?(user, target)
end
@doc """
@ -1856,6 +1979,17 @@ def fields(%{fields: nil}), do: []
def fields(%{fields: fields}), do: fields
def sanitized_fields(%User{} = user) do
user
|> User.fields()
|> Enum.map(fn %{"name" => name, "value" => value} ->
%{
"name" => name,
"value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly)
}
end)
end
def validate_fields(changeset, remote? \\ false) do
limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields
limit = Pleroma.Config.get([:instance, limit_name], 0)

81
lib/pleroma/user_relationship.ex
View File

@ -8,6 +8,7 @@ defmodule Pleroma.UserRelationship do
import Ecto.Changeset
import Ecto.Query
alias Pleroma.FollowingRelationship
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.UserRelationship
@ -21,19 +22,26 @@ defmodule Pleroma.UserRelationship do
end
for relationship_type <- Keyword.keys(UserRelationshipTypeEnum.__enum_map__()) do
# Definitions of `create_block/2`, `create_mute/2` etc.
# `def create_block/2`, `def create_mute/2`, `def create_reblog_mute/2`,
# `def create_notification_mute/2`, `def create_inverse_subscription/2`
def unquote(:"create_#{relationship_type}")(source, target),
do: create(unquote(relationship_type), source, target)
# Definitions of `delete_block/2`, `delete_mute/2` etc.
# `def delete_block/2`, `def delete_mute/2`, `def delete_reblog_mute/2`,
# `def delete_notification_mute/2`, `def delete_inverse_subscription/2`
def unquote(:"delete_#{relationship_type}")(source, target),
do: delete(unquote(relationship_type), source, target)
# Definitions of `block_exists?/2`, `mute_exists?/2` etc.
# `def block_exists?/2`, `def mute_exists?/2`, `def reblog_mute_exists?/2`,
# `def notification_mute_exists?/2`, `def inverse_subscription_exists?/2`
def unquote(:"#{relationship_type}_exists?")(source, target),
do: exists?(unquote(relationship_type), source, target)
end
def user_relationship_types, do: Keyword.keys(user_relationship_mappings())
def user_relationship_mappings, do: UserRelationshipTypeEnum.__enum_map__()
def changeset(%UserRelationship{} = user_relationship, params \\ %{}) do
user_relationship
|> cast(params, [:relationship_type, :source_id, :target_id])
@ -72,6 +80,73 @@ def delete(relationship_type, %User{} = source, %User{} = target) do
end
end
def dictionary(
source_users,
target_users,
source_to_target_rel_types \\ nil,
target_to_source_rel_types \\ nil
)
when is_list(source_users) and is_list(target_users) do
source_user_ids = User.binary_id(source_users)
target_user_ids = User.binary_id(target_users)
get_rel_type_codes = fn rel_type -> user_relationship_mappings()[rel_type] end
source_to_target_rel_types =
Enum.map(source_to_target_rel_types || user_relationship_types(), &get_rel_type_codes.(&1))
target_to_source_rel_types =
Enum.map(target_to_source_rel_types || user_relationship_types(), &get_rel_type_codes.(&1))
__MODULE__
|> where(
fragment(
"(source_id = ANY(?) AND target_id = ANY(?) AND relationship_type = ANY(?)) OR \
(source_id = ANY(?) AND target_id = ANY(?) AND relationship_type = ANY(?))",
^source_user_ids,
^target_user_ids,
^source_to_target_rel_types,
^target_user_ids,
^source_user_ids,
^target_to_source_rel_types
)
)
|> select([ur], [ur.relationship_type, ur.source_id, ur.target_id])
|> Repo.all()
end
def exists?(dictionary, rel_type, source, target, func) do
cond do
is_nil(source) or is_nil(target) ->
false
dictionary ->
[rel_type, source.id, target.id] in dictionary
true ->
func.(source, target)
end
end
@doc ":relationships option for StatusView / AccountView / NotificationView"
def view_relationships_option(nil = _reading_user, _actors) do
%{user_relationships: [], following_relationships: []}
end
def view_relationships_option(%User{} = reading_user, actors) do
user_relationships =
UserRelationship.dictionary(
[reading_user],
actors,
[:block, :mute, :notification_mute, :reblog_mute],
[:block, :inverse_subscription]
)
following_relationships = FollowingRelationship.all_between_user_sets([reading_user], actors)
%{user_relationships: user_relationships, following_relationships: following_relationships}
end
defp validate_not_self_relationship(%Ecto.Changeset{} = changeset) do
changeset
|> validate_change(:target_id, fn _, target_id ->

34
lib/pleroma/web/activity_pub/activity_pub.ex
View File

@ -518,8 +518,7 @@ def follow(follower, followed, activity_id \\ nil, local \\ true) do
defp do_follow(follower, followed, activity_id, local) do
with data <- make_follow_data(follower, followed, activity_id),
{:ok, activity} <- insert(data, local),
:ok <- maybe_federate(activity),
_ <- User.set_follow_state_cache(follower.ap_id, followed.ap_id, activity.data["state"]) do
:ok <- maybe_federate(activity) do
{:ok, activity}
else
{:error, error} -> Repo.rollback(error)
@ -599,6 +598,16 @@ defp do_delete(%Object{data: %{"id" => id, "actor" => actor}} = object, options)
end
end
defp do_delete(%Object{data: %{"type" => "Tombstone", "id" => ap_id}}, _) do
activity =
ap_id
|> Activity.Queries.by_object_id()
|> Activity.Queries.by_type("Delete")
|> Repo.one()
{:ok, activity}
end
@spec block(User.t(), User.t(), String.t() | nil, boolean()) ::
{:ok, Activity.t()} | {:error, any()}
def block(blocker, blocked, activity_id \\ nil, local \\ true) do
@ -1245,17 +1254,17 @@ defp maybe_order(query, _), do: query
defp fetch_activities_query_ap_ids_ops(opts) do
source_user = opts["muting_user"]
ap_id_relations = if source_user, do: [:mute, :reblog_mute], else: []
ap_id_relationships = if source_user, do: [:mute, :reblog_mute], else: []
ap_id_relations =
ap_id_relations ++
ap_id_relationships =
ap_id_relationships ++
if opts["blocking_user"] && opts["blocking_user"] == source_user do
[:block]
else
[]
end
preloaded_ap_ids = User.outgoing_relations_ap_ids(source_user, ap_id_relations)
preloaded_ap_ids = User.outgoing_relationships_ap_ids(source_user, ap_id_relationships)
restrict_blocked_opts = Map.merge(%{"blocked_users_ap_ids" => preloaded_ap_ids[:block]}, opts)
restrict_muted_opts = Map.merge(%{"muted_users_ap_ids" => preloaded_ap_ids[:mute]}, opts)
@ -1385,6 +1394,18 @@ def upload(file, opts \\ []) do
end
end
@spec get_actor_url(any()) :: binary() | nil
defp get_actor_url(url) when is_binary(url), do: url
defp get_actor_url(%{"href" => href}) when is_binary(href), do: href
defp get_actor_url(url) when is_list(url) do
url
|> List.first()
|> get_actor_url()
end
defp get_actor_url(_url), do: nil
defp object_to_user_data(data) do
avatar =
data["icon"]["url"] &&
@ -1414,6 +1435,7 @@ defp object_to_user_data(data) do
user_data = %{
ap_id: data["id"],
uri: get_actor_url(data["url"]),
ap_enabled: true,
source_data: data,
banner: banner,

2
lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex
View File

@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server
# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do

14
lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
View File

@ -12,17 +12,23 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
require Logger
@hackney_options [
pool: :media,
recv_timeout: 10_000
@options [
pool: :media
]
def perform(:prefetch, url) do
Logger.debug("Prefetching #{inspect(url)}")
opts =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.put(@options, :recv_timeout, 10_000)
else
@options
end
url
|> MediaProxy.url()
|> HTTP.get([], adapter: @hackney_options)
|> HTTP.get([], adapter: opts)
end
def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do

2
lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex
View File

@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server
# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do

14
lib/pleroma/web/activity_pub/object_validators/types/object_id.ex
View File

@ -4,14 +4,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID do
def type, do: :string
def cast(object) when is_binary(object) do
with %URI{
scheme: scheme,
host: host
}
when scheme in ["https", "http"] and not is_nil(host) <-
URI.parse(object) do
# Host has to be present and scheme has to be an http scheme (for now)
case URI.parse(object) do
%URI{host: nil} ->
:error
%URI{scheme: scheme} when scheme in ["https", "http"] ->
{:ok, object}
else
_ ->
:error
end

29
lib/pleroma/web/activity_pub/transmogrifier.ex
View File

@ -232,7 +232,8 @@ def fix_url(%{"url" => url} = object) when is_map(url) do
Map.put(object, "url", url["href"])
end
def fix_url(%{"type" => "Video", "url" => url} = object) when is_list(url) do
def fix_url(%{"type" => object_type, "url" => url} = object)
when object_type in ["Video", "Audio"] and is_list(url) do
first_element = Enum.at(url, 0)
link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end)
@ -401,7 +402,7 @@ def handle_incoming(
%{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
options
)
when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer"] do
when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer", "Audio"] do
actor = Containment.get_actor(data)
data =
@ -617,9 +618,9 @@ def handle_incoming(%{"type" => "Like"} = data, _options) do
data |> LikeValidator.cast_data() |> Ecto.Changeset.apply_action(:insert)},
cast_data = ObjectValidator.stringify_keys(Map.from_struct(cast_data_sym)),
:ok <- ObjectValidator.fetch_actor_and_object(cast_data),
{_, {:ok, cast_data}} <- {:maybe_add_context, maybe_add_context_from_object(cast_data)},
{_, {:ok, cast_data}} <- {:ensure_context_presence, ensure_context_presence(cast_data)},
{_, {:ok, cast_data}} <-
{:maybe_add_recipients, maybe_add_recipients_from_object(cast_data)},
{:ensure_recipients_presence, ensure_recipients_presence(cast_data)},
{_, {:ok, activity, _meta}} <-
{:common_pipeline, Pipeline.common_pipeline(cast_data, local: false)} do
{:ok, activity}
@ -1114,13 +1115,11 @@ def add_hashtags(object) do
end
def add_mention_tags(object) do
mentions =
object
|> Utils.get_notified_from_object()
|> Enum.map(&build_mention_tag/1)
{enabled_receivers, disabled_receivers} = Utils.get_notified_from_object(object)
potential_receivers = enabled_receivers ++ disabled_receivers
mentions = Enum.map(potential_receivers, &build_mention_tag/1)
tags = object["tag"] || []
Map.put(object, "tag", tags ++ mentions)
end
@ -1251,10 +1250,10 @@ def maybe_fix_user_url(data), do: data
def maybe_fix_user_object(data), do: maybe_fix_user_url(data)
defp maybe_add_context_from_object(%{"context" => context} = data) when is_binary(context),
defp ensure_context_presence(%{"context" => context} = data) when is_binary(context),
do: {:ok, data}
defp maybe_add_context_from_object(%{"object" => object} = data) when is_binary(object) do
defp ensure_context_presence(%{"object" => object} = data) when is_binary(object) do
with %{data: %{"context" => context}} when is_binary(context) <- Object.normalize(object) do
{:ok, Map.put(data, "context", context)}
else
@ -1263,14 +1262,14 @@ defp maybe_add_context_from_object(%{"object" => object} = data) when is_binary(
end
end
defp maybe_add_context_from_object(_) do
defp ensure_context_presence(_) do
{:error, :no_context}
end
defp maybe_add_recipients_from_object(%{"to" => [_ | _], "cc" => [_ | _]} = data),
defp ensure_recipients_presence(%{"to" => [_ | _], "cc" => [_ | _]} = data),
do: {:ok, data}
defp maybe_add_recipients_from_object(%{"object" => object} = data) do
defp ensure_recipients_presence(%{"object" => object} = data) do
case Object.normalize(object) do
%{data: %{"actor" => actor}} ->
data =
@ -1288,7 +1287,7 @@ defp maybe_add_recipients_from_object(%{"object" => object} = data) do
end
end
defp maybe_add_recipients_from_object(_) do
defp ensure_recipients_presence(_) do
{:error, :no_object}
end
end

101
lib/pleroma/web/activity_pub/utils.ex
View File

@ -440,22 +440,19 @@ def update_follow_state_for_all(
|> update(set: [data: fragment("jsonb_set(data, '{state}', ?)", ^state)])
|> Repo.update_all([])
User.set_follow_state_cache(actor, object, state)
activity = Activity.get_by_id(activity.id)
{:ok, activity}
end
def update_follow_state(
%Activity{data: %{"actor" => actor, "object" => object}} = activity,
%Activity{} = activity,
state
) do
new_data = Map.put(activity.data, "state", state)
changeset = Changeset.change(activity, data: new_data)
with {:ok, activity} <- Repo.update(changeset) do
User.set_follow_state_cache(actor, object, state)
{:ok, activity}
end
end
@ -798,102 +795,6 @@ def get_reports(params, page, page_size) do
ActivityPub.fetch_activities([], params, :offset)
end
def parse_report_group(activity) do
reports = get_reports_by_status_id(activity["id"])
max_date = Enum.max_by(reports, &NaiveDateTime.from_iso8601!(&1.data["published"]))
actors = Enum.map(reports, & &1.user_actor)
[%{data: %{"object" => [account_id | _]}} | _] = reports
account =
AccountView.render("show.json", %{
user: User.get_by_ap_id(account_id)
})
status = get_status_data(activity)
%{
date: max_date.data["published"],
account: account,
status: status,
actors: Enum.uniq(actors),
reports: reports
}
end
defp get_status_data(status) do
case status["deleted"] do
true ->
%{
"id" => status["id"],
"deleted" => true
}
_ ->
Activity.get_by_ap_id(status["id"])
end
end
def get_reports_by_status_id(ap_id) do
from(a in Activity,
where: fragment("(?)->>'type' = 'Flag'", a.data),
where: fragment("(?)->'object' @> ?", a.data, ^[%{id: ap_id}]),
or_where: fragment("(?)->'object' @> ?", a.data, ^[ap_id])
)
|> Activity.with_preloaded_user_actor()
|> Repo.all()
end
@spec get_reports_grouped_by_status([String.t()]) :: %{
required(:groups) => [
%{
required(:date) => String.t(),
required(:account) => %{},
required(:status) => %{},
required(:actors) => [%User{}],
required(:reports) => [%Activity{}]
}
]
}
def get_reports_grouped_by_status(activity_ids) do
parsed_groups =
activity_ids
|> Enum.map(fn id ->
id
|> build_flag_object()
|> parse_report_group()
end)
%{
groups: parsed_groups
}
end
@spec get_reported_activities() :: [
%{
required(:activity) => String.t(),
required(:date) => String.t()
}
]
def get_reported_activities do
reported_activities_query =
from(a in Activity,
where: fragment("(?)->>'type' = 'Flag'", a.data),
select: %{
activity: fragment("jsonb_array_elements((? #- '{object,0}')->'object')", a.data)
},
group_by: fragment("activity")
)
from(a in subquery(reported_activities_query),
distinct: true,
select: %{
id: fragment("COALESCE(?->>'id'::text, ? #>> '{}')", a.activity, a.activity)
}
)
|> Repo.all()
|> Enum.map(& &1.id)
end
def update_report_state(%Activity{} = activity, state)
when state in @strip_status_report_states do
{:ok, stripped_activity} = strip_report_status_data(activity)

14
lib/pleroma/web/activity_pub/visibility.ex
View File

@ -44,6 +44,7 @@ def is_direct?(activity) do
def is_list?(%{data: %{"listMessage" => _}}), do: true
def is_list?(_), do: false
@spec visible_for_user?(Activity.t(), User.t() | nil) :: boolean()
def visible_for_user?(%{actor: ap_id}, %User{ap_id: ap_id}), do: true
def visible_for_user?(%{data: %{"listMessage" => list_ap_id}} = activity, %User{} = user) do
@ -55,14 +56,21 @@ def visible_for_user?(%{data: %{"listMessage" => list_ap_id}} = activity, %User{
def visible_for_user?(%{data: %{"listMessage" => _}}, nil), do: false
def visible_for_user?(activity, nil) do
is_public?(activity)
def visible_for_user?(%{local: local} = activity, nil) do
cfg_key =
if local,
do: :local,
else: :remote
if Pleroma.Config.get([:restrict_unauthenticated, :activities, cfg_key]),
do: false,
else: is_public?(activity)
end
def visible_for_user?(activity, user) do
x = [user.ap_id | User.following(user)]
y = [activity.actor] ++ activity.data["to"] ++ (activity.data["cc"] || [])
visible_for_user?(activity, nil) || Enum.any?(x, &(&1 in y))
is_public?(activity) || Enum.any?(x, &(&1 in y))
end
def entire_thread_visible_for_user?(%Activity{} = activity, %User{} = user) do

59
lib/pleroma/web/admin_api/admin_api_controller.ex
View File

@ -38,7 +38,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
plug(
OAuthScopesPlug,
%{scopes: ["read:accounts"], admin: true}
when action in [:list_users, :user_show, :right_get]
when action in [:list_users, :user_show, :right_get, :show_user_credentials]
)
plug(
@ -54,7 +54,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
:tag_users,
:untag_users,
:right_add,
:right_delete
:right_delete,
:update_user_credentials
]
)
@ -658,6 +659,52 @@ def force_password_reset(%{assigns: %{user: admin}} = conn, %{"nicknames" => nic
json_response(conn, :no_content, "")
end
@doc "Show a given user's credentials"
def show_user_credentials(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do
conn
|> put_view(AccountView)
|> render("credentials.json", %{user: user, for: admin})
else
_ -> {:error, :not_found}
end
end
@doc "Updates a given user"
def update_user_credentials(
%{assigns: %{user: admin}} = conn,
%{"nickname" => nickname} = params
) do
with {_, user} <- {:user, User.get_cached_by_nickname(nickname)},
{:ok, _user} <-
User.update_as_admin(user, params) do
ModerationLog.insert_log(%{
actor: admin,
subject: [user],
action: "updated_users"
})
if params["password"] do
User.force_password_reset_async(user)
end
ModerationLog.insert_log(%{
actor: admin,
subject: [user],
action: "force_password_reset"
})
json(conn, %{status: "success"})
else
{:error, changeset} ->
{_, {error, _}} = Enum.at(changeset.errors, 0)
json(conn, %{error: "New password #{error}."})
_ ->
json(conn, %{error: "Unable to change password."})
end
end
def list_reports(conn, params) do
{page, page_size} = page_params(params)
@ -668,14 +715,6 @@ def list_reports(conn, params) do
|> render("index.json", %{reports: reports})
end
def list_grouped_reports(conn, _params) do
statuses = Utils.get_reported_activities()
conn
|> put_view(ReportView)
|> render("index_grouped.json", Utils.get_reports_grouped_by_status(statuses))
end
def report_show(conn, %{"id" => id}) do
with %Activity{} = report <- Activity.get_by_id(id) do
conn

40
lib/pleroma/web/admin_api/views/account_view.ex
View File

@ -23,6 +23,43 @@ def render("index.json", %{users: users}) do
}
end
def render("credentials.json", %{user: user, for: for_user}) do
user = User.sanitize_html(user, User.html_filter_policy(for_user))
avatar = User.avatar_url(user) |> MediaProxy.url()
banner = User.banner_url(user) |> MediaProxy.url()
background = image_url(user.background) |> MediaProxy.url()
user
|> Map.take([
:id,
:bio,
:email,
:fields,
:name,
:nickname,
:locked,
:no_rich_text,
:default_scope,
:hide_follows,
:hide_followers_count,
:hide_follows_count,
:hide_followers,
:hide_favorites,
:allow_following_move,
:show_role,
:skip_thread_containment,
:pleroma_settings_store,
:raw_fields,
:discoverable,
:actor_type
])
|> Map.merge(%{
"avatar" => avatar,
"banner" => banner,
"background" => background
})
end
def render("show.json", %{user: user}) do
avatar = User.avatar_url(user) |> MediaProxy.url()
display_name = Pleroma.HTML.strip_tags(user.name || user.nickname)
@ -104,4 +141,7 @@ defp parse_error(errors) do
""
end
end
defp image_url(%{"url" => [%{"href" => href} | _]}), do: href
defp image_url(_), do: nil
end

28
lib/pleroma/web/admin_api/views/report_view.ex
View File

@ -4,7 +4,7 @@
defmodule Pleroma.Web.AdminAPI.ReportView do
use Pleroma.Web, :view
alias Pleroma.Activity
alias Pleroma.HTML
alias Pleroma.User
alias Pleroma.Web.AdminAPI.Report
@ -44,32 +44,6 @@ def render("show.json", %{report: report, user: user, account: account, statuses
}
end
def render("index_grouped.json", %{groups: groups}) do
reports =
Enum.map(groups, fn group ->
status =
case group.status do
%Activity{} = activity -> StatusView.render("show.json", %{activity: activity})
_ -> group.status
end
%{
date: group[:date],
account: group[:account],
status: Map.put_new(status, "deleted", false),
actors: Enum.map(group[:actors], &merge_account_views/1),
reports:
group[:reports]
|> Enum.map(&Report.extract_report_info(&1))
|> Enum.map(&render(__MODULE__, "show.json", &1))
}
end)
%{
reports: reports
}
end
def render("index_notes.json", %{notes: notes}) when is_list(notes) do
Enum.map(notes, &render(__MODULE__, "show_note.json", &1))
end

2
lib/pleroma/web/common_api/common_api.ex
View File

@ -394,7 +394,7 @@ def remove_mute(user, activity) do
def thread_muted?(%{id: nil} = _user, _activity), do: false
def thread_muted?(user, activity) do
ThreadMute.check_muted(user.id, activity.data["context"]) != []
ThreadMute.exists?(user.id, activity.data["context"])
end
def report(user, %{"account_id" => account_id} = data) do

14
lib/pleroma/web/controller_helper.ex
View File

@ -34,7 +34,12 @@ defp param_to_integer(val, default) when is_binary(val) do
defp param_to_integer(_, default), do: default
def add_link_headers(conn, activities, extra_params \\ %{}) do
def add_link_headers(conn, activities, extra_params \\ %{})
def add_link_headers(%{assigns: %{skip_link_headers: true}} = conn, _activities, _extra_params),
do: conn
def add_link_headers(conn, activities, extra_params) do
case List.last(activities) do
%{id: max_id} ->
params =
@ -87,7 +92,8 @@ def try_render(conn, _, _) do
render_error(conn, :not_implemented, "Can't display this activity")
end
@spec put_in_if_exist(map(), atom() | String.t(), any) :: map()
def put_in_if_exist(map, _key, nil), do: map
def put_in_if_exist(map, key, value), do: put_in(map, key, value)
@spec put_if_exist(map(), atom() | String.t(), any) :: map()
def put_if_exist(map, _key, nil), do: map
def put_if_exist(map, key, value), do: Map.put(map, key, value)
end

6
lib/pleroma/web/feed/tag_controller.ex
View File

@ -9,18 +9,18 @@ defmodule Pleroma.Web.Feed.TagController do
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.Feed.FeedView
import Pleroma.Web.ControllerHelper, only: [put_in_if_exist: 3]
import Pleroma.Web.ControllerHelper, only: [put_if_exist: 3]
def feed(conn, %{"tag" => raw_tag} = params) do
{format, tag} = parse_tag(raw_tag)
activities =
%{"type" => ["Create"], "tag" => tag}
|> put_in_if_exist("max_id", params["max_id"])
|> put_if_exist("max_id", params["max_id"])
|> ActivityPub.fetch_public_activities()
conn
|> put_resp_content_type("application/atom+xml")
|> put_resp_content_type("application/#{format}+xml")
|> put_view(FeedView)
|> render("tag.#{format}",
activities: activities,

17
lib/pleroma/web/feed/user_controller.ex
View File

@ -11,7 +11,7 @@ defmodule Pleroma.Web.Feed.UserController do
alias Pleroma.Web.ActivityPub.ActivityPubController
alias Pleroma.Web.Feed.FeedView
import Pleroma.Web.ControllerHelper, only: [put_in_if_exist: 3]
import Pleroma.Web.ControllerHelper, only: [put_if_exist: 3]
plug(Pleroma.Plugs.SetFormatPlug when action in [:feed_redirect])
@ -40,19 +40,28 @@ def feed_redirect(conn, %{"nickname" => nickname}) do
end
def feed(conn, %{"nickname" => nickname} = params) do
format = get_format(conn)
format =
if format in ["rss", "atom"] do
format
else
"atom"
end
with {_, %User{} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)} do
activities =
%{
"type" => ["Create"],
"actor_id" => user.ap_id
}
|> put_in_if_exist("max_id", params["max_id"])
|> put_if_exist("max_id", params["max_id"])
|> ActivityPub.fetch_public_activities()
conn
|> put_resp_content_type("application/atom+xml")
|> put_resp_content_type("application/#{format}+xml")
|> put_view(FeedView)
|> render("user.xml",
|> render("user.#{format}",
user: user,
activities: activities,
feed_config: Pleroma.Config.get([:feed])

77
lib/pleroma/web/mastodon_api/controllers/account_controller.ex
View File

@ -8,7 +8,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
import Pleroma.Web.ControllerHelper,
only: [add_link_headers: 2, truthy_param?: 1, assign_account_by_id: 2, json_response: 3]
alias Pleroma.Emoji
alias Pleroma.Plugs.OAuthScopesPlug
alias Pleroma.Plugs.RateLimiter
alias Pleroma.User
@ -60,14 +59,18 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
plug(
Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug
when action != :create
when action not in [:create, :show, :statuses]
)
@relations [:follow, :unfollow]
@relationship_actions [:follow, :unfollow]
@needs_account ~W(followers following lists follow unfollow mute unmute block unblock)a
plug(RateLimiter, [name: :relations_id_action, params: ["id", "uri"]] when action in @relations)
plug(RateLimiter, [name: :relations_actions] when action in @relations)
plug(
RateLimiter,
[name: :relation_id_action, params: ["id", "uri"]] when action in @relationship_actions
)
plug(RateLimiter, [name: :relations_actions] when action in @relationship_actions)
plug(RateLimiter, [name: :app_account_creation] when action == :create)
plug(:assign_account_by_id when action in @needs_account)
@ -140,17 +143,6 @@ def verify_credentials(%{assigns: %{user: user}} = conn, _) do
def update_credentials(%{assigns: %{user: original_user}} = conn, params) do
user = original_user
params =
if Map.has_key?(params, "fields_attributes") do
Map.update!(params, "fields_attributes", fn fields ->
fields
|> normalize_fields_attributes()
|> Enum.filter(fn %{"name" => n} -> n != "" end)
end)
else
params
end
user_params =
[
:no_rich_text,
@ -169,46 +161,20 @@ def update_credentials(%{assigns: %{user: original_user}} = conn, params) do
add_if_present(acc, params, to_string(key), key, &{:ok, truthy_param?(&1)})
end)
|> add_if_present(params, "display_name", :name)
|> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value, user)} end)
|> add_if_present(params, "avatar", :avatar, fn value ->
with %Plug.Upload{} <- value,
{:ok, object} <- ActivityPub.upload(value, type: :avatar) do
{:ok, object.data}
end
end)
|> add_if_present(params, "header", :banner, fn value ->
with %Plug.Upload{} <- value,
{:ok, object} <- ActivityPub.upload(value, type: :banner) do
{:ok, object.data}
end
end)
|> add_if_present(params, "pleroma_background_image", :background, fn value ->
with %Plug.Upload{} <- value,
{:ok, object} <- ActivityPub.upload(value, type: :background) do
{:ok, object.data}
end
end)
|> add_if_present(params, "fields_attributes", :fields, fn fields ->
fields = Enum.map(fields, fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end)
{:ok, fields}
end)
|> add_if_present(params, "fields_attributes", :raw_fields)
|> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store, fn value ->
{:ok, Map.merge(user.pleroma_settings_store, value)}
end)
|> add_if_present(params, "note", :bio)
|> add_if_present(params, "avatar", :avatar)
|> add_if_present(params, "header", :banner)
|> add_if_present(params, "pleroma_background_image", :background)
|> add_if_present(
params,
"fields_attributes",
:raw_fields,
&{:ok, normalize_fields_attributes(&1)}
)
|> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store)
|> add_if_present(params, "default_scope", :default_scope)
|> add_if_present(params, "actor_type", :actor_type)
emojis_text = (user_params["display_name"] || "") <> (user_params["note"] || "")
user_emojis =
user
|> Map.get(:emoji, [])
|> Enum.concat(Emoji.Formatter.get_emoji_map(emojis_text))
|> Enum.dedup()
user_params = Map.put(user_params, :emoji, user_emojis)
changeset = User.update_changeset(user, user_params)
with {:ok, user} <- User.update_and_set_cache(changeset) do
@ -259,7 +225,8 @@ def show(%{assigns: %{user: for_user}} = conn, %{"id" => nickname_or_id}) do
@doc "GET /api/v1/accounts/:id/statuses"
def statuses(%{assigns: %{user: reading_user}} = conn, params) do
with %User{} = user <- User.get_cached_by_nickname_or_id(params["id"], for: reading_user) do
with %User{} = user <- User.get_cached_by_nickname_or_id(params["id"], for: reading_user),
true <- User.visible_for?(user, reading_user) do
params =
params
|> Map.put("tag", params["tagged"])
@ -271,6 +238,8 @@ def statuses(%{assigns: %{user: reading_user}} = conn, params) do
|> add_link_headers(activities)
|> put_view(StatusView)
|> render("index.json", activities: activities, for: reading_user, as: :activity)
else
_e -> render_error(conn, :not_found, "Can't find user")
end
end

2
lib/pleroma/web/mastodon_api/controllers/status_controller.ex
View File

@ -76,7 +76,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
%{scopes: ["write:bookmarks"]} when action in [:bookmark, :unbookmark]
)
plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug)
plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug when action not in [:index, :show])
@rate_limited_status_actions ~w(reblog unreblog favourite unfavourite create delete)a

15
lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
View File

@ -27,7 +27,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do
plug(OAuthScopesPlug, %{scopes: ["read:statuses"]} when action in [:home, :direct])
plug(OAuthScopesPlug, %{scopes: ["read:lists"]} when action == :list)
plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug)
plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug when action != :public)
plug(:put_view, Pleroma.Web.MastodonAPI.StatusView)
@ -75,6 +75,16 @@ def direct(%{assigns: %{user: user}} = conn, params) do
def public(%{assigns: %{user: user}} = conn, params) do
local_only = truthy_param?(params["local"])
cfg_key =
if local_only do
:local
else
:federated
end
restrict? = Pleroma.Config.get([:restrict_unauthenticated, :timelines, cfg_key])
if not (restrict? and is_nil(user)) do
activities =
params
|> Map.put("type", ["Create", "Announce"])
@ -86,6 +96,9 @@ def public(%{assigns: %{user: user}} = conn, params) do
conn
|> add_link_headers(activities, %{"local" => local_only})
|> render("index.json", activities: activities, for: user, as: :activity)
else
render_error(conn, :unauthorized, "authorization required for timeline view")
end
end
def hashtag_fetching(params, user, local_only) do

140
lib/pleroma/web/mastodon_api/views/account_view.ex
View File

@ -5,12 +5,30 @@
defmodule Pleroma.Web.MastodonAPI.AccountView do
use Pleroma.Web, :view
alias Pleroma.FollowingRelationship
alias Pleroma.User
alias Pleroma.UserRelationship
alias Pleroma.Web.CommonAPI.Utils
alias Pleroma.Web.MastodonAPI.AccountView
alias Pleroma.Web.MediaProxy
def render("index.json", %{users: users} = opts) do
reading_user = opts[:for]
relationships_opt =
cond do
Map.has_key?(opts, :relationships) ->
opts[:relationships]
is_nil(reading_user) ->
UserRelationship.view_relationships_option(nil, [])
true ->
UserRelationship.view_relationships_option(reading_user, users)
end
opts = Map.put(opts, :relationships, relationships_opt)
users
|> render_many(AccountView, "show.json", opts)
|> Enum.filter(&Enum.any?/1)
@ -27,7 +45,7 @@ def render("mention.json", %{user: user}) do
id: to_string(user.id),
acct: user.nickname,
username: username_from_nickname(user.nickname),
url: User.profile_url(user)
url: user.uri || user.ap_id
}
end
@ -35,34 +53,107 @@ def render("relationship.json", %{user: nil, target: _target}) do
%{}
end
def render("relationship.json", %{user: %User{} = user, target: %User{} = target}) do
follow_state = User.get_cached_follow_state(user, target)
def render(
"relationship.json",
%{user: %User{} = reading_user, target: %User{} = target} = opts
) do
user_relationships = get_in(opts, [:relationships, :user_relationships])
following_relationships = get_in(opts, [:relationships, :following_relationships])
requested =
if follow_state && !User.following?(user, target) do
follow_state == "pending"
follow_state =
if following_relationships do
user_to_target_following_relation =
FollowingRelationship.find(following_relationships, reading_user, target)
User.get_follow_state(reading_user, target, user_to_target_following_relation)
else
false
User.get_follow_state(reading_user, target)
end
followed_by =
if following_relationships do
case FollowingRelationship.find(following_relationships, target, reading_user) do
%{state: "accept"} -> true
_ -> false
end
else
User.following?(target, reading_user)
end
# NOTE: adjust UserRelationship.view_relationships_option/2 on new relation-related flags
%{
id: to_string(target.id),
following: User.following?(user, target),
followed_by: User.following?(target, user),
blocking: User.blocks_user?(user, target),
blocked_by: User.blocks_user?(target, user),
muting: User.mutes?(user, target),
muting_notifications: User.muted_notifications?(user, target),
subscribing: User.subscribed_to?(user, target),
requested: requested,
domain_blocking: User.blocks_domain?(user, target),
showing_reblogs: User.showing_reblogs?(user, target),
following: follow_state == "accept",
followed_by: followed_by,
blocking:
UserRelationship.exists?(
user_relationships,
:block,
reading_user,
target,
&User.blocks_user?(&1, &2)
),
blocked_by:
UserRelationship.exists?(
user_relationships,
:block,
target,
reading_user,
&User.blocks_user?(&1, &2)
),
muting:
UserRelationship.exists?(
user_relationships,
:mute,
reading_user,
target,
&User.mutes?(&1, &2)
),
muting_notifications:
UserRelationship.exists?(
user_relationships,
:notification_mute,
reading_user,
target,
&User.muted_notifications?(&1, &2)
),
subscribing:
UserRelationship.exists?(
user_relationships,
:inverse_subscription,
target,
reading_user,
&User.subscribed_to?(&2, &1)
),
requested: follow_state == "pending",
domain_blocking: User.blocks_domain?(reading_user, target),
showing_reblogs:
not UserRelationship.exists?(
user_relationships,
:reblog_mute,
reading_user,
target,
&User.muting_reblogs?(&1, &2)
),
endorsed: false
}
end
def render("relationships.json", %{user: user, targets: targets}) do
render_many(targets, AccountView, "relationship.json", user: user, as: :target)
def render("relationships.json", %{user: user, targets: targets} = opts) do
relationships_opt =
cond do
Map.has_key?(opts, :relationships) ->
opts[:relationships]
is_nil(user) ->
UserRelationship.view_relationships_option(nil, [])
true ->
UserRelationship.view_relationships_option(user, targets)
end
render_opts = %{as: :target, user: user, relationships: relationships_opt}
render_many(targets, AccountView, "relationship.json", render_opts)
end
defp do_render("show.json", %{user: user} = opts) do
@ -100,7 +191,12 @@ defp do_render("show.json", %{user: user} = opts) do
}
end)
relationship = render("relationship.json", %{user: opts[:for], target: user})
relationship =
render("relationship.json", %{
user: opts[:for],
target: user,
relationships: opts[:relationships]
})
%{
id: to_string(user.id),
@ -113,7 +209,7 @@ defp do_render("show.json", %{user: user} = opts) do
following_count: following_count,
statuses_count: user.note_count,
note: user.bio || "",
url: User.profile_url(user),
url: user.uri || user.ap_id,
avatar: image,
avatar_static: image,
header: header,
@ -122,7 +218,7 @@ defp do_render("show.json", %{user: user} = opts) do
fields: user.fields,
bot: bot,
source: %{
note: Pleroma.HTML.strip_tags((user.bio || "") |> String.replace("<br>", "\n")),
note: (user.bio || "") |> String.replace(~r(<br */?>), "\n") |> Pleroma.HTML.strip_tags(),
sensitive: false,
fields: user.raw_fields,
pleroma: %{

109
lib/pleroma/web/mastodon_api/views/notification_view.ex
View File

@ -8,24 +8,86 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
alias Pleroma.Activity
alias Pleroma.Notification
alias Pleroma.User
alias Pleroma.UserRelationship
alias Pleroma.Web.CommonAPI
alias Pleroma.Web.MastodonAPI.AccountView
alias Pleroma.Web.MastodonAPI.NotificationView
alias Pleroma.Web.MastodonAPI.StatusView
def render("index.json", %{notifications: notifications, for: user}) do
safe_render_many(notifications, NotificationView, "show.json", %{for: user})
def render("index.json", %{notifications: notifications, for: reading_user} = opts) do
activities = Enum.map(notifications, & &1.activity)
parent_activities =
activities
|> Enum.filter(
&(Activity.mastodon_notification_type(&1) in [
"favourite",
"reblog",
"pleroma:emoji_reaction"
])
)
|> Enum.map(& &1.data["object"])
|> Activity.create_by_object_ap_id()
|> Activity.with_preloaded_object(:left)
|> Pleroma.Repo.all()
relationships_opt =
cond do
Map.has_key?(opts, :relationships) ->
opts[:relationships]
is_nil(reading_user) ->
UserRelationship.view_relationships_option(nil, [])
true ->
move_activities_targets =
activities
|> Enum.filter(&(Activity.mastodon_notification_type(&1) == "move"))
|> Enum.map(&User.get_cached_by_ap_id(&1.data["target"]))
actors =
activities
|> Enum.map(fn a -> User.get_cached_by_ap_id(a.data["actor"]) end)
|> Enum.filter(& &1)
|> Kernel.++(move_activities_targets)
UserRelationship.view_relationships_option(reading_user, actors)
end
def render("show.json", %{
opts = %{
for: reading_user,
parent_activities: parent_activities,
relationships: relationships_opt
}
safe_render_many(notifications, NotificationView, "show.json", opts)
end
def render(
"show.json",
%{
notification: %Notification{activity: activity} = notification,
for: user
}) do
for: reading_user
} = opts
) do
actor = User.get_cached_by_ap_id(activity.data["actor"])
parent_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
parent_activity_fn = fn ->
if opts[:parent_activities] do
Activity.Queries.find_by_object_ap_id(opts[:parent_activities], activity.data["object"])
else
Activity.get_create_by_object_ap_id(activity.data["object"])
end
end
mastodon_type = Activity.mastodon_notification_type(activity)
with %{id: _} = account <- AccountView.render("show.json", %{user: actor, for: user}) do
with %{id: _} = account <-
AccountView.render("show.json", %{
user: actor,
for: reading_user,
relationships: opts[:relationships]
}) do
response = %{
id: to_string(notification.id),
type: mastodon_type,
@ -36,24 +98,28 @@ def render("show.json", %{
}
}
render_opts = %{relationships: opts[:relationships]}
case mastodon_type do
"mention" ->
put_status(response, activity, user)
put_status(response, activity, reading_user, render_opts)
"favourite" ->
put_status(response, parent_activity, user)
put_status(response, parent_activity_fn.(), reading_user, render_opts)
"reblog" ->
put_status(response, parent_activity, user)
put_status(response, parent_activity_fn.(), reading_user, render_opts)
"move" ->
put_target(response, activity, user)
put_target(response, activity, reading_user, render_opts)
"follow" ->
response
"pleroma:emoji_reaction" ->
put_status(response, parent_activity, user) |> put_emoji(activity)
response
|> put_status(parent_activity_fn.(), reading_user, render_opts)
|> put_emoji(activity)
_ ->
nil
@ -64,16 +130,21 @@ def render("show.json", %{
end
defp put_emoji(response, activity) do
response
|> Map.put(:emoji, activity.data["content"])
Map.put(response, :emoji, activity.data["content"])
end
defp put_status(response, activity, user) do
Map.put(response, :status, StatusView.render("show.json", %{activity: activity, for: user}))
defp put_status(response, activity, reading_user, opts) do
status_render_opts = Map.merge(opts, %{activity: activity, for: reading_user})
status_render = StatusView.render("show.json", status_render_opts)
Map.put(response, :status, status_render)
end
defp put_target(response, activity, user) do
target = User.get_cached_by_ap_id(activity.data["target"])
Map.put(response, :target, AccountView.render("show.json", %{user: target, for: user}))
defp put_target(response, activity, reading_user, opts) do
target_user = User.get_cached_by_ap_id(activity.data["target"])
target_render_opts = Map.merge(opts, %{user: target_user, for: reading_user})
target_render = AccountView.render("show.json", target_render_opts)
Map.put(response, :target, target_render)
end
end

91
lib/pleroma/web/mastodon_api/views/status_view.ex
View File

@ -13,6 +13,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
alias Pleroma.Object
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.UserRelationship
alias Pleroma.Web.CommonAPI
alias Pleroma.Web.CommonAPI.Utils
alias Pleroma.Web.MastodonAPI.AccountView
@ -71,10 +72,43 @@ defp reblogged?(activity, user) do
end
def render("index.json", opts) do
replied_to_activities = get_replied_to_activities(opts.activities)
opts = Map.put(opts, :replied_to_activities, replied_to_activities)
reading_user = opts[:for]
safe_render_many(opts.activities, StatusView, "show.json", opts)
# To do: check AdminAPIControllerTest on the reasons behind nil activities in the list
activities = Enum.filter(opts.activities, & &1)
replied_to_activities = get_replied_to_activities(activities)
parent_activities =
activities
|> Enum.filter(&(&1.data["type"] == "Announce" && &1.data["object"]))
|> Enum.map(&Object.normalize(&1).data["id"])
|> Activity.create_by_object_ap_id()
|> Activity.with_preloaded_object(:left)
|> Activity.with_preloaded_bookmark(reading_user)
|> Activity.with_set_thread_muted_field(reading_user)
|> Repo.all()
relationships_opt =
cond do
Map.has_key?(opts, :relationships) ->
opts[:relationships]
is_nil(reading_user) ->
UserRelationship.view_relationships_option(nil, [])
true ->
actors = Enum.map(activities ++ parent_activities, &get_user(&1.data["actor"]))
UserRelationship.view_relationships_option(reading_user, actors)
end
opts =
opts
|> Map.put(:replied_to_activities, replied_to_activities)
|> Map.put(:parent_activities, parent_activities)
|> Map.put(:relationships, relationships_opt)
safe_render_many(activities, StatusView, "show.json", opts)
end
def render(
@ -85,17 +119,25 @@ def render(
created_at = Utils.to_masto_date(activity.data["published"])
activity_object = Object.normalize(activity)
reblogged_activity =
reblogged_parent_activity =
if opts[:parent_activities] do
Activity.Queries.find_by_object_ap_id(
opts[:parent_activities],
activity_object.data["id"]
)
else
Activity.create_by_object_ap_id(activity_object.data["id"])
|> Activity.with_preloaded_bookmark(opts[:for])
|> Activity.with_set_thread_muted_field(opts[:for])
|> Repo.one()
end
reblogged = render("show.json", Map.put(opts, :activity, reblogged_activity))
reblog_rendering_opts = Map.put(opts, :activity, reblogged_parent_activity)
reblogged = render("show.json", reblog_rendering_opts)
favorited = opts[:for] && opts[:for].ap_id in (activity_object.data["likes"] || [])
bookmarked = Activity.get_bookmark(reblogged_activity, opts[:for]) != nil
bookmarked = Activity.get_bookmark(reblogged_parent_activity, opts[:for]) != nil
mentions =
activity.recipients
@ -107,7 +149,12 @@ def render(
id: to_string(activity.id),
uri: activity_object.data["id"],
url: activity_object.data["id"],
account: AccountView.render("show.json", %{user: user, for: opts[:for]}),
account:
AccountView.render("show.json", %{
user: user,
for: opts[:for],
relationships: opts[:relationships]
}),
in_reply_to_id: nil,
in_reply_to_account_id: nil,
reblog: reblogged,
@ -116,7 +163,7 @@ def render(
reblogs_count: 0,
replies_count: 0,
favourites_count: 0,
reblogged: reblogged?(reblogged_activity, opts[:for]),
reblogged: reblogged?(reblogged_parent_activity, opts[:for]),
favourited: present?(favorited),
bookmarked: present?(bookmarked),
muted: false,
@ -183,9 +230,10 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
end
thread_muted? =
case activity.thread_muted? do
thread_muted? when is_boolean(thread_muted?) -> thread_muted?
nil -> (opts[:for] && CommonAPI.thread_muted?(opts[:for], activity)) || false
cond do
is_nil(opts[:for]) -> false
is_boolean(activity.thread_muted?) -> activity.thread_muted?
true -> CommonAPI.thread_muted?(opts[:for], activity)
end
attachment_data = object.data["attachment"] || []
@ -253,11 +301,26 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
_ -> []
end
muted =
thread_muted? ||
UserRelationship.exists?(
get_in(opts, [:relationships, :user_relationships]),
:mute,
opts[:for],
user,
fn for_user, user -> User.mutes?(for_user, user) end
)
%{
id: to_string(activity.id),
uri: object.data["id"],
url: url,
account: AccountView.render("show.json", %{user: user, for: opts[:for]}),
account:
AccountView.render("show.json", %{
user: user,
for: opts[:for],
relationships: opts[:relationships]
}),
in_reply_to_id: reply_to && to_string(reply_to.id),
in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id),
reblog: nil,
@ -270,7 +333,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
reblogged: reblogged?(activity, opts[:for]),
favourited: present?(favorited),
bookmarked: present?(bookmarked),
muted: thread_muted? || User.mutes?(opts[:for], user),
muted: muted,
pinned: pinned?(activity, user),
sensitive: sensitive,
spoiler_text: summary,
@ -421,7 +484,7 @@ def get_reply_to(%{data: %{"object" => _object}} = activity, _) do
end
def render_content(%{data: %{"type" => object_type}} = object)
when object_type in ["Video", "Event"] do
when object_type in ["Video", "Event", "Audio"] do
with name when not is_nil(name) and name != "" <- object.data["name"] do
"<p><a href=\"#{object.data["id"]}\">#{name}</a></p>#{object.data["content"]}"
else

7
lib/pleroma/web/metadata.ex
View File

@ -6,7 +6,12 @@ defmodule Pleroma.Web.Metadata do
alias Phoenix.HTML
def build_tags(params) do
Enum.reduce(Pleroma.Config.get([__MODULE__, :providers], []), "", fn parser, acc ->
providers = [
Pleroma.Web.Metadata.Providers.RestrictIndexing
| Pleroma.Config.get([__MODULE__, :providers], [])
]
Enum.reduce(providers, "", fn parser, acc ->
rendered_html =
params
|> parser.build_tags()

2
lib/pleroma/web/metadata/opengraph.ex
View File

@ -68,7 +68,7 @@ def build_tags(%{user: user}) do
property: "og:title",
content: Utils.user_name_string(user)
], []},
{:meta, [property: "og:url", content: User.profile_url(user)], []},
{:meta, [property: "og:url", content: user.uri || user.ap_id], []},
{:meta, [property: "og:description", content: truncated_bio], []},
{:meta, [property: "og:type", content: "website"], []},
{:meta, [property: "og:image", content: Utils.attachment_url(User.avatar_url(user))], []},

25
lib/pleroma/web/metadata/restrict_indexing.ex Normal file
View File

@ -0,0 +1,25 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Metadata.Providers.RestrictIndexing do
@behaviour Pleroma.Web.Metadata.Providers.Provider
@moduledoc """
Restricts indexing of remote users.
"""
@impl true
def build_tags(%{user: %{local: false}}) do
[
{:meta,
[
name: "robots",
content: "noindex, noarchive"
], []}
]
end
@impl true
def build_tags(%{user: %{local: true}}), do: []
end

3
lib/pleroma/web/nodeinfo/nodeinfo_controller.ex
View File

@ -75,7 +75,8 @@ def raw_nodeinfo do
end,
if Config.get([:instance, :safe_dm_mentions]) do
"safe_dm_mentions"
end
end,
"pleroma_emoji_reactions"
]
|> Enum.filter(& &1)

18
lib/pleroma/web/rel_me.ex
View File

@ -3,11 +3,9 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RelMe do
@hackney_options [
@options [
pool: :media,
recv_timeout: 2_000,
max_body: 2_000_000,
with_body: true
max_body: 2_000_000
]
if Pleroma.Config.get(:env) == :test do
@ -25,8 +23,18 @@ def parse(url) when is_binary(url) do
def parse(_), do: {:error, "No URL provided"}
defp parse_url(url) do
opts =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.merge(@options,
recv_timeout: 2_000,
with_body: true
)
else
@options
end
with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <-
Pleroma.HTTP.get(url, [], adapter: @hackney_options),
Pleroma.HTTP.get(url, [], adapter: opts),
{:ok, html_tree} <- Floki.parse_document(html),
data <-
Floki.attribute(html_tree, "link[rel~=me]", "href") ++

18
lib/pleroma/web/rich_media/parser.ex
View File

@ -3,11 +3,9 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parser do
@hackney_options [
@options [
pool: :media,
recv_timeout: 2_000,
max_body: 2_000_000,
with_body: true
max_body: 2_000_000
]
defp parsers do
@ -77,8 +75,18 @@ defp get_ttl_from_image(data, url) do
end
defp parse_url(url) do
opts =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.merge(@options,
recv_timeout: 2_000,
with_body: true
)
else
@options
end
try do
{:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options)
{:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: opts)
html
|> parse_html()

5
lib/pleroma/web/router.ex
View File

@ -173,6 +173,8 @@ defmodule Pleroma.Web.Router do
get("/users/:nickname/password_reset", AdminAPIController, :get_password_reset)
patch("/users/force_password_reset", AdminAPIController, :force_password_reset)
get("/users/:nickname/credentials", AdminAPIController, :show_user_credentials)
patch("/users/:nickname/credentials", AdminAPIController, :update_user_credentials)
get("/users", AdminAPIController, :list_users)
get("/users/:nickname", AdminAPIController, :user_show)
@ -184,7 +186,6 @@ defmodule Pleroma.Web.Router do
patch("/users/resend_confirmation_email", AdminAPIController, :resend_confirmation_email)
get("/reports", AdminAPIController, :list_reports)
get("/grouped_reports", AdminAPIController, :list_grouped_reports)
get("/reports/:id", AdminAPIController, :report_show)
patch("/reports", AdminAPIController, :reports_update)
post("/reports/:id/notes", AdminAPIController, :report_notes_create)
@ -513,7 +514,7 @@ defmodule Pleroma.Web.Router do
end
pipeline :ostatus do
plug(:accepts, ["html", "xml", "atom", "activity+json", "json"])
plug(:accepts, ["html", "xml", "rss", "atom", "activity+json", "json"])
plug(Pleroma.Plugs.StaticFEPlug)
end

4
lib/pleroma/web/static_fe/static_fe_controller.ex
View File

@ -60,7 +60,9 @@ defp represent(%Activity{object: %Object{data: data}} = activity, selected) do
content =
if data["content"] do
Pleroma.HTML.filter_tags(data["content"])
data["content"]
|> Pleroma.HTML.filter_tags()
|> Pleroma.Emoji.Formatter.emojify(Map.get(data, "emoji", %{}))
else
nil
end

2
lib/pleroma/web/streamer/worker.ex
View File

@ -130,7 +130,7 @@ defp do_stream(%{topic: topic, item: item}) do
defp should_send?(%User{} = user, %Activity{} = item) do
%{block: blocked_ap_ids, mute: muted_ap_ids, reblog_mute: reblog_muted_ap_ids} =
User.outgoing_relations_ap_ids(user, [:block, :mute, :reblog_mute])
User.outgoing_relationships_ap_ids(user, [:block, :mute, :reblog_mute])
recipient_blocks = MapSet.new(blocked_ap_ids ++ muted_ap_ids)
recipients = MapSet.new(item.recipients)

0
lib/pleroma/web/templates/feed/feed/_activity.xml.eex → lib/pleroma/web/templates/feed/feed/_activity.atom.eex
View File

49
lib/pleroma/web/templates/feed/feed/_activity.rss.eex Normal file
View File

@ -0,0 +1,49 @@
<item>
<activity:object-type>http://activitystrea.ms/schema/1.0/note</activity:object-type>
<activity:verb>http://activitystrea.ms/schema/1.0/post</activity:verb>
<guid><%= @data["id"] %></guid>
<title><%= activity_title(@object, Keyword.get(@feed_config, :post_title, %{})) %></title>
<description><%= activity_content(@object) %></description>
<pubDate><%= @data["published"] %></pubDate>
<updated><%= @data["published"] %></updated>
<ostatus:conversation ref="<%= activity_context(@activity) %>">
<%= activity_context(@activity) %>
</ostatus:conversation>
<link rel="ostatus:conversation"><%= activity_context(@activity) %></link>
<%= if @data["summary"] do %>
<description><%= @data["summary"] %></description>
<% end %>
<%= if @activity.local do %>
<link><%= @data["id"] %></link>
<% else %>
<link><%= @data["external_url"] %></link>
<% end %>
<%= for tag <- @data["tag"] || [] do %>
<category term="<%= tag %>"></category>
<% end %>
<%= for attachment <- @data["attachment"] || [] do %>
<link type="<%= attachment_type(attachment) %>"><%= attachment_href(attachment) %></link>
<% end %>
<%= if @data["inReplyTo"] do %>
<thr:in-reply-to ref='<%= @data["inReplyTo"] %>' href='<%= get_href(@data["inReplyTo"]) %>'/>
<% end %>
<%= for id <- @activity.recipients do %>
<%= if id == Pleroma.Constants.as_public() do %>
<link rel="mentioned" ostatus:object-type="http://activitystrea.ms/schema/1.0/collection">http://activityschema.org/collection/public</link>
<% else %>
<%= unless Regex.match?(~r/^#{Pleroma.Web.base_url()}.+followers$/, id) do %>
<link rel="mentioned" ostatus:object-type="http://activitystrea.ms/schema/1.0/person"><%= id %></link>
<% end %>
<% end %>
<% end %>
<%= for {emoji, file} <- @data["emoji"] || %{} do %>
<link name="<%= emoji %>" rel="emoji"><%= file %></link>
<% end %>
</item>

0
lib/pleroma/web/templates/feed/feed/_author.xml.eex → lib/pleroma/web/templates/feed/feed/_author.atom.eex
View File

17
lib/pleroma/web/templates/feed/feed/_author.rss.eex Normal file
View File

@ -0,0 +1,17 @@
<managingEditor>
<guid><%= @user.ap_id %></guid>
<activity:object>http://activitystrea.ms/schema/1.0/person</activity:object>
<uri><%= @user.ap_id %></uri>
<poco:preferredUsername><%= @user.nickname %></poco:preferredUsername>
<poco:displayName><%= @user.name %></poco:displayName>
<poco:note><%= escape(@user.bio) %></poco:note>
<description><%= escape(@user.bio) %></description>
<name><%= @user.nickname %></name>
<link rel="avatar"><%= User.avatar_url(@user) %></link>
<%= if User.banner_url(@user) do %>
<link rel="header"><%= User.banner_url(@user) %></link>
<% end %>
<%= if @user.local do %>
<ap_enabled>true</ap_enabled>
<% end %>
</managingEditor>

4
lib/pleroma/web/templates/feed/feed/user.xml.eex → lib/pleroma/web/templates/feed/feed/user.atom.eex
View File

@ -12,13 +12,13 @@
<logo><%= logo(@user) %></logo>
<link rel="self" href="<%= '#{user_feed_url(@conn, :feed, @user.nickname)}.atom' %>" type="application/atom+xml"/>
<%= render @view_module, "_author.xml", assigns %>
<%= render @view_module, "_author.atom", assigns %>
<%= if last_activity(@activities) do %>
<link rel="next" href="<%= '#{user_feed_url(@conn, :feed, @user.nickname)}.atom?max_id=#{last_activity(@activities).id}' %>" type="application/atom+xml"/>
<% end %>
<%= for activity <- @activities do %>
<%= render @view_module, "_activity.xml", Map.merge(assigns, prepare_activity(activity)) %>
<%= render @view_module, "_activity.atom", Map.merge(assigns, prepare_activity(activity)) %>
<% end %>
</feed>

20
lib/pleroma/web/templates/feed/feed/user.rss.eex Normal file
View File

@ -0,0 +1,20 @@
<?xml version="1.0" encoding="UTF-8" ?>
<rss version="2.0">
<channel>
<guid><%= user_feed_url(@conn, :feed, @user.nickname) <> ".rss" %></guid>
<title><%= @user.nickname <> "'s timeline" %></title>
<updated><%= most_recent_update(@activities, @user) %></updated>
<image><%= logo(@user) %></image>
<link><%= '#{user_feed_url(@conn, :feed, @user.nickname)}.rss' %></link>
<%= render @view_module, "_author.rss", assigns %>
<%= if last_activity(@activities) do %>
<link rel="next"><%= '#{user_feed_url(@conn, :feed, @user.nickname)}.rss?max_id=#{last_activity(@activities).id}' %></link>
<% end %>
<%= for activity <- @activities do %>
<%= render @view_module, "_activity.rss", Map.merge(assigns, prepare_activity(activity)) %>
<% end %>
</channel>
</rss>

2
lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex
View File

@ -1,5 +1,5 @@
<div class="p-author h-card">
<a class="u-url" rel="author noopener" href="<%= User.profile_url(@user) %>">
<a class="u-url" rel="author noopener" href="<%= (@user.uri || @user.ap_id) %>">
<div class="avatar">
<img src="<%= User.avatar_url(@user) |> MediaProxy.url %>" width="48" height="48" alt="">
</div>

2
lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex
View File

@ -8,7 +8,7 @@
<button type="submit" class="collapse">Remote follow</button>
</form>
<%= raw Formatter.emojify(@user.name, emoji_for_user(@user)) %> |
<%= link "@#{@user.nickname}@#{Endpoint.host()}", to: User.profile_url(@user) %>
<%= link "@#{@user.nickname}@#{Endpoint.host()}", to: (@user.uri || @user.ap_id) %>
</h3>
<p><%= raw @user.bio %></p>
</header>

5
lib/pleroma/web/web_finger/web_finger.ex
View File

@ -173,7 +173,8 @@ def find_lrdd_template(domain) do
get_template_from_xml(body)
else
_ ->
with {:ok, %{body: body}} <- HTTP.get("https://#{domain}/.well-known/host-meta", []) do
with {:ok, %{body: body, status: status}} when status in 200..299 <-
HTTP.get("https://#{domain}/.well-known/host-meta", []) do
get_template_from_xml(body)
else
e -> {:error, "Can't find LRDD template: #{inspect(e)}"}
@ -205,7 +206,7 @@ def finger(account) do
with response <-
HTTP.get(
address,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
),
{:ok, %{status: status, body: body}} when status in 200..299 <- response do
doc = XML.parse_document(body)

12
mix.exs
View File

@ -63,7 +63,7 @@ def copy_nginx_config(%{path: target_path} = release) do
def application do
[
mod: {Pleroma.Application, []},
extra_applications: [:logger, :runtime_tools, :comeonin, :quack, :fast_sanitize],
extra_applications: [:logger, :runtime_tools, :comeonin, :quack, :fast_sanitize, :ssl],
included_applications: [:ex_syslogger]
]
end
@ -119,7 +119,15 @@ defp deps do
{:calendar, "~> 0.17.4"},
{:cachex, "~> 3.2"},
{:poison, "~> 3.0", override: true},
{:tesla, "~> 1.3", override: true},
# {:tesla, "~> 1.3", override: true},
{:tesla,
git: "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git",
ref: "61b7503cef33f00834f78ddfafe0d5d9dec2270b",
override: true},
{:castore, "~> 0.1"},
{:cowlib, "~> 2.8", override: true},
{:gun,
github: "ninenines/gun", ref: "e1a69b36b180a574c0ac314ced9613fdd52312cc", override: true},
{:jason, "~> 1.0"},
{:mogrify, "~> 0.6.1"},
{:ex_aws, "~> 2.1"},

4
mix.lock
View File

@ -10,6 +10,7 @@
"cachex": {:hex, :cachex, "3.2.0", "a596476c781b0646e6cb5cd9751af2e2974c3e0d5498a8cab71807618b74fe2f", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "aef93694067a43697ae0531727e097754a9e992a1e7946296f5969d6dd9ac986"},
"calendar": {:hex, :calendar, "0.17.6", "ec291cb2e4ba499c2e8c0ef5f4ace974e2f9d02ae9e807e711a9b0c7850b9aee", [:mix], [{:tzdata, "~> 0.5.20 or ~> 0.1.201603 or ~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "738d0e17a93c2ccfe4ddc707bdc8e672e9074c8569498483feb1c4530fb91b2b"},
"captcha": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", "e0f16822d578866e186a0974d65ad58cddc1e2ab", [ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"]},
"castore": {:hex, :castore, "0.1.5", "591c763a637af2cc468a72f006878584bc6c306f8d111ef8ba1d4c10e0684010", [:mix], [], "hexpm", "6db356b2bc6cc22561e051ff545c20ad064af57647e436650aa24d7d06cd941a"},
"certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "805abd97539caf89ec6d4732c91e62ba9da0cda51ac462380bbd28ee697a8c42"},
"combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
"comeonin": {:hex, :comeonin, "4.1.2", "3eb5620fd8e35508991664b4c2b04dd41e52f1620b36957be837c1d7784b7592", [:mix], [{:argon2_elixir, "~> 1.2", [hex: :argon2_elixir, repo: "hexpm", optional: true]}, {:bcrypt_elixir, "~> 0.12.1 or ~> 1.0", [hex: :bcrypt_elixir, repo: "hexpm", optional: true]}, {:pbkdf2_elixir, "~> 0.12", [hex: :pbkdf2_elixir, repo: "hexpm", optional: true]}], "hexpm", "d8700a0ca4dbb616c22c9b3f6dd539d88deaafec3efe66869d6370c9a559b3e9"},
@ -46,6 +47,7 @@
"gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm"},
"gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"},
"gettext": {:hex, :gettext, "0.17.4", "f13088e1ec10ce01665cf25f5ff779e7df3f2dc71b37084976cf89d1aa124d5c", [:mix], [], "hexpm", "3c75b5ea8288e2ee7ea503ff9e30dfe4d07ad3c054576a6e60040e79a801e14d"},
"gun": {:git, "https://github.com/ninenines/gun.git", "e1a69b36b180a574c0ac314ced9613fdd52312cc", [ref: "e1a69b36b180a574c0ac314ced9613fdd52312cc"]},
"hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e0100f8ef7d1124222c11ad362c857d3df7cb5f4204054f9f0f4a728666591fc"},
"html_entities": {:hex, :html_entities, "0.5.1", "1c9715058b42c35a2ab65edc5b36d0ea66dd083767bef6e3edb57870ef556549", [:mix], [], "hexpm", "30efab070904eb897ff05cd52fa61c1025d7f8ef3a9ca250bc4e6513d16c32de"},
"html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"},
@ -101,7 +103,7 @@
"swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "e3928e1d2889a308aaf3e42755809ac21cffd77cb58eef01cbfdab4ce2fd1e21"},
"syslog": {:hex, :syslog, "1.0.6", "995970c9aa7feb380ac493302138e308d6e04fd57da95b439a6df5bb3bf75076", [:rebar3], [], "hexpm", "769ddfabd0d2a16f3f9c17eb7509951e0ca4f68363fb26f2ee51a8ec4a49881a"},
"telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm", "4738382e36a0a9a2b6e25d67c960e40e1a2c95560b9f936d8e29de8cd858480f"},
"tesla": {:hex, :tesla, "1.3.2", "deb92c5c9ce35e747a395ba413ca78593a4f75bf0e1545630ee2e3d34264021e", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "7567704c4790e21bd9a961b56d0b6a988ff68cc4dacfe6b2106e258da1d5cdda"},
"tesla": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git", "61b7503cef33f00834f78ddfafe0d5d9dec2270b", [ref: "61b7503cef33f00834f78ddfafe0d5d9dec2270b"]},
"timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "f354efb2400dd7a80fd9eb6c8419068c4f632da4ac47f3d8822d6e33f08bc852"},
"trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"},
"tzdata": {:hex, :tzdata, "0.5.22", "f2ba9105117ee0360eae2eca389783ef7db36d533899b2e84559404dbc77ebb8", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "cd66c8a1e6a9e121d1f538b01bef459334bb4029a1ffb4eeeb5e4eae0337e7b6"},

0
priv/static/adminfe/app.c836e084.css → priv/static/adminfe/app.85534e14.css
View File

BIN
priv/static/adminfe/chunk-0d8f.650c8e81.css → priv/static/adminfe/chunk-0d8f.d85f5a29.css
View File

Binary file not shown.

BIN
priv/static/adminfe/chunk-136a.3936457d.css → priv/static/adminfe/chunk-136a.f1130f8e.css
View File

Binary file not shown.

BIN
priv/static/adminfe/chunk-13e9.98eaadba.css Normal file
View File

Binary file not shown.

Some files were not shown because too many files have changed in this diff Show More