Merge remote-tracking branch 'remotes/origin/develop' into ostatus-controller-no-auth-check-on-non-federating-instances

This commit is contained in:
Ivan Tashkinov 2020-10-11 22:34:48 +03:00
commit 2498e569f1
13 changed files with 190 additions and 90 deletions

View File

@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Changed ### Changed
- **Breaking:** Pleroma Admin API: emoji packs and files routes changed. - **Breaking:** Pleroma Admin API: emoji packs and files routes changed.
- **Breaking:** Sensitive/NSFW statuses no longer disable link previews.
- Search: Users are now findable by their urls. - Search: Users are now findable by their urls.
- Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated. - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
- Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated. - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.

View File

@ -677,7 +677,18 @@
config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true, min_lifetime: 600 config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true, min_lifetime: 600
config :pleroma, Pleroma.Plugs.RemoteIp, enabled: true config :pleroma, Pleroma.Plugs.RemoteIp,
enabled: true,
headers: ["x-forwarded-for"],
proxies: [],
reserved: [
"127.0.0.0/8",
"::1/128",
"fc00::/7",
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16"
]
config :pleroma, :static_fe, enabled: false config :pleroma, :static_fe, enabled: false

View File

@ -44,11 +44,13 @@
}, },
%{ %{
key: "git", key: "git",
label: "Git Repository URL",
type: :string, type: :string,
description: "URL of the git repository of the frontend" description: "URL of the git repository of the frontend"
}, },
%{ %{
key: "build_url", key: "build_url",
label: "Build URL",
type: :string, type: :string,
description: description:
"Either an url to a zip file containing the frontend or a template to build it by inserting the `ref`. The string `${ref}` will be replaced by the configured `ref`.", "Either an url to a zip file containing the frontend or a template to build it by inserting the `ref`. The string `${ref}` will be replaced by the configured `ref`.",
@ -56,6 +58,7 @@
}, },
%{ %{
key: "build_dir", key: "build_dir",
label: "Build directory",
type: :string, type: :string,
description: "The directory inside the zip file " description: "The directory inside the zip file "
} }
@ -3262,20 +3265,22 @@
%{ %{
key: :headers, key: :headers,
type: {:list, :string}, type: {:list, :string},
description: description: """
"A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Default: `~w[forwarded x-forwarded-for x-client-ip x-real-ip]`." A list of strings naming the HTTP headers to use when deriving the true client IP. Default: `["x-forwarded-for"]`.
"""
}, },
%{ %{
key: :proxies, key: :proxies,
type: {:list, :string}, type: {:list, :string},
description: description:
"A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Default: `[]`." "A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128."
}, },
%{ %{
key: :reserved, key: :reserved,
type: {:list, :string}, type: {:list, :string},
description: description: """
"Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network)." A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`
"""
} }
] ]
}, },
@ -3681,9 +3686,7 @@
type: :map, type: :map,
description: description:
"A map containing available frontends and parameters for their installation.", "A map containing available frontends and parameters for their installation.",
children: [ children: frontend_options
frontend_options
]
} }
] ]
}, },

View File

@ -5,20 +5,24 @@
1. Stop the Pleroma service. 1. Stop the Pleroma service.
2. Go to the working directory of Pleroma (default is `/opt/pleroma`) 2. Go to the working directory of Pleroma (default is `/opt/pleroma`)
3. Run `sudo -Hu postgres pg_dump -d <pleroma_db> --format=custom -f </path/to/backup_location/pleroma.pgdump>` (make sure the postgres user has write access to the destination file) 3. Run `sudo -Hu postgres pg_dump -d <pleroma_db> --format=custom -f </path/to/backup_location/pleroma.pgdump>` (make sure the postgres user has write access to the destination file)
4. Copy `pleroma.pgdump`, `config/prod.secret.exs` and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too. 4. Copy `pleroma.pgdump`, `config/prod.secret.exs`, `config/setup_db.psql` (if still available) and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too.
5. Restart the Pleroma service. 5. Restart the Pleroma service.
## Restore/Move ## Restore/Move
1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers). Try to use the same database name. 1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers).
2. Stop the Pleroma service. 2. Stop the Pleroma service.
3. Go to the working directory of Pleroma (default is `/opt/pleroma`) 3. Go to the working directory of Pleroma (default is `/opt/pleroma`)
4. Copy the above mentioned files back to their original position. 4. Copy the above mentioned files back to their original position.
5. Drop the existing database and recreate an empty one `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;';` `sudo -Hu postgres psql -c 'CREATE DATABASE <pleroma_db>;';` 5. Drop the existing database if restoring in-place. `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;'`
6. Run `sudo -Hu postgres pg_restore -d <pleroma_db> -v -1 </path/to/backup_location/pleroma.pgdump>` 6. Restore the database schema and pleroma postgres role the with the original `setup_db.psql` if you have it: `sudo -Hu postgres psql -f config/setup_db.psql`.
7. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
8. Restart the Pleroma service. Alternatively, run the `mix pleroma.instance gen` task again. You can ignore most of the questions, but make the database user, name, and password the same as found in your backup of `config/prod.secret.exs`. Then run the restoration of the pleroma role and schema with of the generated `config/setup_db.psql` as instructed above. You may delete the `config/generated_config.exs` file as it is not needed.
9. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries.
7. Now restore the Pleroma instance's data into the empty database schema: `sudo -Hu postgres pg_restore -d <pleroma_db> -v -1 </path/to/backup_location/pleroma.pgdump>`
8. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
9. Restart the Pleroma service.
10. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries.
[^1]: Prefix with `MIX_ENV=prod` to run it using the production config file. [^1]: Prefix with `MIX_ENV=prod` to run it using the production config file.

View File

@ -426,9 +426,9 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start
Available options: Available options:
* `enabled` - Enable/disable the plug. Defaults to `false`. * `enabled` - Enable/disable the plug. Defaults to `false`.
* `headers` - A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Defaults to `["x-forwarded-for"]`. * `headers` - A list of strings naming the HTTP headers to use when deriving the true client IP address. Defaults to `["x-forwarded-for"]`.
* `proxies` - A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Defaults to `[]`. * `proxies` - A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128.
* `reserved` - Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network). * `reserved` - A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`.
### :rate_limit ### :rate_limit

View File

@ -88,7 +88,7 @@ def new_unapproved_registration(to, account) do
html_body = """ html_body = """
<p>New account for review: <a href="#{user_url(account)}">@#{account.nickname}</a></p> <p>New account for review: <a href="#{user_url(account)}">@#{account.nickname}</a></p>
<blockquote>#{HTML.strip_tags(account.registration_reason)}</blockquote> <blockquote>#{HTML.strip_tags(account.registration_reason)}</blockquote>
<a href="#{Pleroma.Web.base_url()}/pleroma/admin">Visit AdminFE</a> <a href="#{Pleroma.Web.base_url()}/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a>
""" """
new() new()

View File

@ -7,48 +7,42 @@ defmodule Pleroma.Plugs.RemoteIp do
This is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration. This is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
""" """
alias Pleroma.Config
import Plug.Conn import Plug.Conn
@behaviour Plug @behaviour Plug
@headers ~w[
x-forwarded-for
]
# https://en.wikipedia.org/wiki/Localhost
# https://en.wikipedia.org/wiki/Private_network
@reserved ~w[
127.0.0.0/8
::1/128
fc00::/7
10.0.0.0/8
172.16.0.0/12
192.168.0.0/16
]
def init(_), do: nil def init(_), do: nil
def call(%{remote_ip: original_remote_ip} = conn, _) do def call(%{remote_ip: original_remote_ip} = conn, _) do
config = Pleroma.Config.get(__MODULE__, []) if Config.get([__MODULE__, :enabled]) do
%{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts())
if Keyword.get(config, :enabled, false) do
%{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts(config))
assign(conn, :remote_ip_found, original_remote_ip != new_remote_ip) assign(conn, :remote_ip_found, original_remote_ip != new_remote_ip)
else else
conn conn
end end
end end
defp remote_ip_opts(config) do defp remote_ip_opts do
headers = config |> Keyword.get(:headers, @headers) |> MapSet.new() headers = Config.get([__MODULE__, :headers], []) |> MapSet.new()
reserved = Keyword.get(config, :reserved, @reserved) reserved = Config.get([__MODULE__, :reserved], [])
proxies = proxies =
config Config.get([__MODULE__, :proxies], [])
|> Keyword.get(:proxies, [])
|> Enum.concat(reserved) |> Enum.concat(reserved)
|> Enum.map(&InetCidr.parse/1) |> Enum.map(&maybe_add_cidr/1)
{headers, proxies} {headers, proxies}
end end
defp maybe_add_cidr(proxy) when is_binary(proxy) do
proxy =
cond do
"/" in String.codepoints(proxy) -> proxy
InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
end
InetCidr.parse(proxy, true)
end
end end

View File

@ -790,7 +790,17 @@ defp restrict_replies(query, %{
[activity, object] in query, [activity, object] in query,
where: where:
fragment( fragment(
"?->>'inReplyTo' is null OR ? && array_remove(?, ?) OR ? = ?", """
?->>'type' != 'Create' -- This isn't a Create
OR ?->>'inReplyTo' is null -- this isn't a reply
OR ? && array_remove(?, ?) -- The recipient is us or one of our friends,
-- unless they are the author (because authors
-- are also part of the recipients). This leads
-- to a bug that self-replies by friends won't
-- show up.
OR ? = ? -- The actor is us
""",
activity.data,
object.data, object.data,
^[user.ap_id | User.get_cached_user_friends_ap_ids(user)], ^[user.ap_id | User.get_cached_user_friends_ap_ids(user)],
activity.recipients, activity.recipients,

View File

@ -57,7 +57,6 @@ defp get_tld(host) do
def fetch_data_for_object(object) do def fetch_data_for_object(object) do
with true <- Config.get([:rich_media, :enabled]), with true <- Config.get([:rich_media, :enabled]),
false <- object.data["sensitive"] || false,
{:ok, page_url} <- {:ok, page_url} <-
HTML.extract_first_external_url_from_object(object), HTML.extract_first_external_url_from_object(object),
:ok <- validate_page_url(page_url), :ok <- validate_page_url(page_url),

View File

@ -63,7 +63,7 @@ test "new unapproved registration email" do
assert res.html_body == """ assert res.html_body == """
<p>New account for review: <a href="#{account_url}">@#{account.nickname}</a></p> <p>New account for review: <a href="#{account_url}">@#{account.nickname}</a></p>
<blockquote>Plz let me in</blockquote> <blockquote>Plz let me in</blockquote>
<a href="http://localhost:4001/pleroma/admin">Visit AdminFE</a> <a href="http://localhost:4001/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a>
""" """
end end
end end

View File

@ -3,13 +3,27 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Plugs.RemoteIpTest do defmodule Pleroma.Plugs.RemoteIpTest do
use ExUnit.Case, async: true use ExUnit.Case
use Plug.Test use Plug.Test
alias Pleroma.Plugs.RemoteIp alias Pleroma.Plugs.RemoteIp
import Pleroma.Tests.Helpers, only: [clear_config: 1, clear_config: 2] import Pleroma.Tests.Helpers, only: [clear_config: 2]
setup do: clear_config(RemoteIp)
setup do:
clear_config(RemoteIp,
enabled: true,
headers: ["x-forwarded-for"],
proxies: [],
reserved: [
"127.0.0.0/8",
"::1/128",
"fc00::/7",
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16"
]
)
test "disabled" do test "disabled" do
Pleroma.Config.put(RemoteIp, enabled: false) Pleroma.Config.put(RemoteIp, enabled: false)
@ -25,8 +39,6 @@ test "disabled" do
end end
test "enabled" do test "enabled" do
Pleroma.Config.put(RemoteIp, enabled: true)
conn = conn =
conn(:get, "/") conn(:get, "/")
|> put_req_header("x-forwarded-for", "1.1.1.1") |> put_req_header("x-forwarded-for", "1.1.1.1")
@ -54,8 +66,6 @@ test "custom headers" do
end end
test "custom proxies" do test "custom proxies" do
Pleroma.Config.put(RemoteIp, enabled: true)
conn = conn =
conn(:get, "/") conn(:get, "/")
|> put_req_header("x-forwarded-for", "173.245.48.1, 1.1.1.1, 173.245.48.2") |> put_req_header("x-forwarded-for", "173.245.48.1, 1.1.1.1, 173.245.48.2")
@ -72,4 +82,27 @@ test "custom proxies" do
assert conn.remote_ip == {1, 1, 1, 1} assert conn.remote_ip == {1, 1, 1, 1}
end end
test "proxies set without CIDR format" do
Pleroma.Config.put([RemoteIp, :proxies], ["173.245.48.1"])
conn =
conn(:get, "/")
|> put_req_header("x-forwarded-for", "173.245.48.1, 1.1.1.1")
|> RemoteIp.call(nil)
assert conn.remote_ip == {1, 1, 1, 1}
end
test "proxies set `nonsensical` CIDR" do
Pleroma.Config.put([RemoteIp, :reserved], ["127.0.0.0/8"])
Pleroma.Config.put([RemoteIp, :proxies], ["10.0.0.3/24"])
conn =
conn(:get, "/")
|> put_req_header("x-forwarded-for", "10.0.0.3, 1.1.1.1")
|> RemoteIp.call(nil)
assert conn.remote_ip == {1, 1, 1, 1}
end
end end

View File

@ -2177,4 +2177,84 @@ test "does nothing with a clashing nickname and the same ap id" do
assert user.nickname == orig_user.nickname assert user.nickname == orig_user.nickname
end end
end end
describe "reply filtering" do
test "`following` still contains announcements by friends" do
user = insert(:user)
followed = insert(:user)
not_followed = insert(:user)
User.follow(user, followed)
{:ok, followed_post} = CommonAPI.post(followed, %{status: "Hello"})
{:ok, not_followed_to_followed} =
CommonAPI.post(not_followed, %{
status: "Also hello",
in_reply_to_status_id: followed_post.id
})
{:ok, retoot} = CommonAPI.repeat(not_followed_to_followed.id, followed)
params =
%{}
|> Map.put(:type, ["Create", "Announce"])
|> Map.put(:blocking_user, user)
|> Map.put(:muting_user, user)
|> Map.put(:reply_filtering_user, user)
|> Map.put(:reply_visibility, "following")
|> Map.put(:announce_filtering_user, user)
|> Map.put(:user, user)
activities =
[user.ap_id | User.following(user)]
|> ActivityPub.fetch_activities(params)
followed_post_id = followed_post.id
retoot_id = retoot.id
assert [%{id: ^followed_post_id}, %{id: ^retoot_id}] = activities
assert length(activities) == 2
end
# This test is skipped because, while this is the desired behavior,
# there seems to be no good way to achieve it with the method that
# we currently use for detecting to who a reply is directed.
# This is a TODO and should be fixed by a later rewrite of the code
# in question.
@tag skip: true
test "`following` still contains self-replies by friends" do
user = insert(:user)
followed = insert(:user)
not_followed = insert(:user)
User.follow(user, followed)
{:ok, followed_post} = CommonAPI.post(followed, %{status: "Hello"})
{:ok, not_followed_post} = CommonAPI.post(not_followed, %{status: "Also hello"})
{:ok, _followed_to_not_followed} =
CommonAPI.post(followed, %{status: "sup", in_reply_to_status_id: not_followed_post.id})
{:ok, _followed_self_reply} =
CommonAPI.post(followed, %{status: "Also cofe", in_reply_to_status_id: followed_post.id})
params =
%{}
|> Map.put(:type, ["Create", "Announce"])
|> Map.put(:blocking_user, user)
|> Map.put(:muting_user, user)
|> Map.put(:reply_filtering_user, user)
|> Map.put(:reply_visibility, "following")
|> Map.put(:announce_filtering_user, user)
|> Map.put(:user, user)
activities =
[user.ap_id | User.following(user)]
|> ActivityPub.fetch_activities(params)
assert length(activities) == 2
end
end
end end

View File

@ -64,41 +64,6 @@ test "crawls valid, complete URLs" do
Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
end end
test "refuses to crawl URLs from posts marked sensitive" do
user = insert(:user)
{:ok, activity} =
CommonAPI.post(user, %{
status: "http://example.com/ogp",
sensitive: true
})
%Object{} = object = Object.normalize(activity)
assert object.data["sensitive"]
Config.put([:rich_media, :enabled], true)
assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
end
test "refuses to crawl URLs from posts tagged NSFW" do
user = insert(:user)
{:ok, activity} =
CommonAPI.post(user, %{
status: "http://example.com/ogp #nsfw"
})
%Object{} = object = Object.normalize(activity)
assert object.data["sensitive"]
Config.put([:rich_media, :enabled], true)
assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
end
test "refuses to crawl URLs of private network from posts" do test "refuses to crawl URLs of private network from posts" do
user = insert(:user) user = insert(:user)