From 514c899275a32e6ef63305f9424c50344d41b12e Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 11 Feb 2020 10:12:57 +0300 Subject: [PATCH 01/73] adding gun adapter --- CHANGELOG.md | 1 + config/config.exs | 67 +- config/description.exs | 2 +- config/test.exs | 2 + docs/API/admin_api.md | 2 + docs/configuration/cheatsheet.md | 36 +- lib/mix/tasks/pleroma/benchmark.ex | 39 + lib/mix/tasks/pleroma/emoji.ex | 9 +- lib/pleroma/application.ex | 90 +- lib/pleroma/config/config_db.ex | 11 - lib/pleroma/config/transfer_task.ex | 43 +- lib/pleroma/gun/api.ex | 26 + lib/pleroma/gun/api/mock.ex | 151 +++ lib/pleroma/gun/conn.ex | 29 + lib/pleroma/gun/gun.ex | 45 + lib/pleroma/http/adapter.ex | 64 ++ lib/pleroma/http/adapter/gun.ex | 123 +++ lib/pleroma/http/adapter/hackney.ex | 41 + lib/pleroma/http/connection.ex | 113 ++- lib/pleroma/http/http.ex | 156 ++- lib/pleroma/http/request.ex | 23 + lib/pleroma/http/request_builder.ex | 107 +- lib/pleroma/object/fetcher.ex | 6 +- lib/pleroma/otp_version.ex | 63 ++ lib/pleroma/pool/connections.ex | 415 ++++++++ lib/pleroma/pool/pool.ex | 22 + lib/pleroma/pool/request.ex | 72 ++ lib/pleroma/pool/supervisor.ex | 36 + lib/pleroma/reverse_proxy/client.ex | 26 +- lib/pleroma/reverse_proxy/client/hackney.ex | 24 + lib/pleroma/reverse_proxy/client/tesla.ex | 87 ++ lib/pleroma/reverse_proxy/reverse_proxy.ex | 20 +- .../mrf/media_proxy_warming_policy.ex | 14 +- lib/pleroma/web/rel_me.ex | 18 +- lib/pleroma/web/rich_media/parser.ex | 18 +- lib/pleroma/web/web_finger/web_finger.ex | 2 +- mix.exs | 4 + mix.lock | 2 + test/activity/ir/topics_test.exs | 2 +- test/config/config_db_test.exs | 8 - test/fixtures/warnings/otp_version/21.1 | 1 + test/fixtures/warnings/otp_version/22.1 | 1 + test/fixtures/warnings/otp_version/22.4 | 1 + test/fixtures/warnings/otp_version/23.0 | 1 + test/fixtures/warnings/otp_version/error | 1 + test/fixtures/warnings/otp_version/undefined | 1 + test/gun/gun_test.exs | 33 + test/http/adapter/gun_test.exs | 266 +++++ test/http/adapter/hackney_test.exs | 54 + test/http/adapter_test.exs | 65 ++ test/http/connection_test.exs | 142 +++ test/http/request_builder_test.exs | 30 +- test/http_test.exs | 35 +- test/notification_test.exs | 7 + test/otp_version_test.exs | 58 ++ test/pool/connections_test.exs | 959 ++++++++++++++++++ test/reverse_proxy/client/tesla_test.exs | 93 ++ .../reverse_proxy_test.exs | 121 ++- test/support/http_request_mock.ex | 94 +- test/user_invite_token_test.exs | 4 - .../admin_api/admin_api_controller_test.exs | 9 +- test/web/common_api/common_api_utils_test.exs | 7 + test/web/push/impl_test.exs | 2 +- 63 files changed, 3615 insertions(+), 389 deletions(-) create mode 100644 lib/pleroma/gun/api.ex create mode 100644 lib/pleroma/gun/api/mock.ex create mode 100644 lib/pleroma/gun/conn.ex create mode 100644 lib/pleroma/gun/gun.ex create mode 100644 lib/pleroma/http/adapter.ex create mode 100644 lib/pleroma/http/adapter/gun.ex create mode 100644 lib/pleroma/http/adapter/hackney.ex create mode 100644 lib/pleroma/http/request.ex create mode 100644 lib/pleroma/otp_version.ex create mode 100644 lib/pleroma/pool/connections.ex create mode 100644 lib/pleroma/pool/pool.ex create mode 100644 lib/pleroma/pool/request.ex create mode 100644 lib/pleroma/pool/supervisor.ex create mode 100644 lib/pleroma/reverse_proxy/client/hackney.ex create mode 100644 lib/pleroma/reverse_proxy/client/tesla.ex create mode 100644 test/fixtures/warnings/otp_version/21.1 create mode 100644 test/fixtures/warnings/otp_version/22.1 create mode 100644 test/fixtures/warnings/otp_version/22.4 create mode 100644 test/fixtures/warnings/otp_version/23.0 create mode 100644 test/fixtures/warnings/otp_version/error create mode 100644 test/fixtures/warnings/otp_version/undefined create mode 100644 test/gun/gun_test.exs create mode 100644 test/http/adapter/gun_test.exs create mode 100644 test/http/adapter/hackney_test.exs create mode 100644 test/http/adapter_test.exs create mode 100644 test/http/connection_test.exs create mode 100644 test/otp_version_test.exs create mode 100644 test/pool/connections_test.exs create mode 100644 test/reverse_proxy/client/tesla_test.exs rename test/{ => reverse_proxy}/reverse_proxy_test.exs (79%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e838983b..48080503a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Support for custom Elixir modules (such as MRF policies) - User settings: Add _This account is a_ option. - OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`). +- New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires OTP version older that 22.2, otherwise pleroma won’t start. For hackney OTP update is not required.
API Changes diff --git a/config/config.exs b/config/config.exs index ccc0c4e52..27091393b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -58,20 +58,6 @@ config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch" -config :pleroma, :hackney_pools, - federation: [ - max_connections: 50, - timeout: 150_000 - ], - media: [ - max_connections: 50, - timeout: 150_000 - ], - upload: [ - max_connections: 25, - timeout: 300_000 - ] - # Upload configuration config :pleroma, Pleroma.Upload, uploader: Pleroma.Uploaders.Local, @@ -185,20 +171,12 @@ } config :tesla, adapter: Tesla.Adapter.Hackney - # Configures http settings, upstream proxy etc. config :pleroma, :http, proxy_url: nil, send_user_agent: true, user_agent: :default, - adapter: [ - ssl_options: [ - # Workaround for remote server certificate chain issues - partial_chain: &:hackney_connect.partial_chain/1, - # We don't support TLS v1.3 yet - versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"] - ] - ] + adapter: [] config :pleroma, :instance, name: "Pleroma", @@ -612,6 +590,49 @@ config :pleroma, configurable_from_database: false +config :pleroma, :connections_pool, + receive_connection_timeout: 250, + max_connections: 250, + retry: 5, + retry_timeout: 100, + await_up_timeout: 5_000 + +config :pleroma, :pools, + federation: [ + size: 50, + max_overflow: 10, + timeout: 150_000 + ], + media: [ + size: 50, + max_overflow: 10, + timeout: 150_000 + ], + upload: [ + size: 25, + max_overflow: 5, + timeout: 300_000 + ], + default: [ + size: 10, + max_overflow: 2, + timeout: 10_000 + ] + +config :pleroma, :hackney_pools, + federation: [ + max_connections: 50, + timeout: 150_000 + ], + media: [ + max_connections: 50, + timeout: 150_000 + ], + upload: [ + max_connections: 25, + timeout: 300_000 + ] + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/description.exs b/config/description.exs index efea7c137..d5322fa33 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2728,7 +2728,7 @@ key: :adapter, type: :module, description: "Tesla adapter", - suggestions: [Tesla.Adapter.Hackney] + suggestions: [Tesla.Adapter.Hackney, Tesla.Adapter.Gun] } ] }, diff --git a/config/test.exs b/config/test.exs index 078c46205..83783cf8f 100644 --- a/config/test.exs +++ b/config/test.exs @@ -94,6 +94,8 @@ config :pleroma, :modules, runtime_dir: "test/fixtures/modules" +config :pleroma, Pleroma.Gun.API, Pleroma.Gun.API.Mock + if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" else diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index fb6dfcb08..cd8123c5d 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -731,6 +731,8 @@ Some modifications are necessary to save the config settings correctly: Most of the settings will be applied in `runtime`, this means that you don't need to restart the instance. But some settings are applied in `compile time` and require a reboot of the instance, such as: - all settings inside these keys: - `:hackney_pools` + - `:connections_pool` + - `:pools` - `:chat` - partially settings inside these keys: - `:seconds_valid` in `Pleroma.Captcha` diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 2bd935983..1c67eca35 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -368,8 +368,7 @@ Available caches: * `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`) * `send_user_agent`: should we include a user agent with HTTP requests? (default: `true`) * `user_agent`: what user agent should we use? (default: `:default`), must be string or `:default` -* `adapter`: array of hackney options - +* `adapter`: array of adapter options ### :hackney_pools @@ -388,6 +387,39 @@ For each pool, the options are: * `timeout` - retention duration for connections +### :connections_pool + +*For `gun` adapter* + +Advanced settings for connections pool. Pool with opened connections. These connections can be reused in worker pools. + +* `:receive_connection_timeout` - timeout to receive connection from pool. Default: 250ms. +* `:max_connections` - maximum number of connections in the pool. Default: 250 connections. +* `:retry` - number of retries, while `gun` will try to reconnect if connections goes down. Default: 5. +* `:retry_timeout` - timeout while `gun` will try to reconnect. Default: 100ms. +* `:await_up_timeout` - timeout while `gun` will wait until connection is up. Default: 5000ms. + +### :pools + +*For `gun` adapter* + +Advanced settings for workers pools. + +There's four pools used: + +* `:federation` for the federation jobs. + You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs. +* `:media` for rich media, media proxy +* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`) +* `:default` for other requests + +For each pool, the options are: + +* `:size` - how much workers the pool can hold +* `:timeout` - timeout while `gun` will wait for response +* `:max_overflow` - additional workers if pool is under load + + ## Captcha ### Pleroma.Captcha diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index 84dccf7f3..01e079136 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -74,4 +74,43 @@ def run(["render_timeline", nickname | _] = args) do inputs: inputs ) end + + def run(["adapters"]) do + start_pleroma() + + :ok = + Pleroma.Pool.Connections.open_conn( + "https://httpbin.org/stream-bytes/1500", + :gun_connections + ) + + Process.sleep(1_500) + + Benchee.run( + %{ + "Without conn and without pool" => fn -> + {:ok, %Tesla.Env{}} = + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], + adapter: [pool: :no_pool, receive_conn: false] + ) + end, + "Without conn and with pool" => fn -> + {:ok, %Tesla.Env{}} = + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], + adapter: [receive_conn: false] + ) + end, + "With reused conn and without pool" => fn -> + {:ok, %Tesla.Env{}} = + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], + adapter: [pool: :no_pool] + ) + end, + "With reused conn and with pool" => fn -> + {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500") + end + }, + parallel: 10 + ) + end end diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 24d999707..b4e8d3a0b 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -4,13 +4,13 @@ defmodule Mix.Tasks.Pleroma.Emoji do use Mix.Task + import Mix.Pleroma @shortdoc "Manages emoji packs" @moduledoc File.read!("docs/administration/CLI_tasks/emoji.md") def run(["ls-packs" | args]) do - Mix.Pleroma.start_pleroma() - Application.ensure_all_started(:hackney) + start_pleroma() {options, [], []} = parse_global_opts(args) @@ -36,8 +36,7 @@ def run(["ls-packs" | args]) do end def run(["get-packs" | args]) do - Mix.Pleroma.start_pleroma() - Application.ensure_all_started(:hackney) + start_pleroma() {options, pack_names, []} = parse_global_opts(args) @@ -135,7 +134,7 @@ def run(["get-packs" | args]) do end def run(["gen-pack", src]) do - Application.ensure_all_started(:hackney) + start_pleroma() proposed_name = Path.basename(src) |> Path.rootname() name = String.trim(IO.gets("Pack name [#{proposed_name}]: ")) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 27758cf94..df6d3a98d 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -3,8 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Application do - import Cachex.Spec use Application + + import Cachex.Spec + + alias Pleroma.Config + require Logger @name Mix.Project.config()[:name] @@ -18,9 +22,9 @@ def named_version, do: @name <> " " <> @version def repository, do: @repository def user_agent do - case Pleroma.Config.get([:http, :user_agent], :default) do + case Config.get([:http, :user_agent], :default) do :default -> - info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>" + info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>" named_version() <> "; " <> info custom -> @@ -32,7 +36,7 @@ def user_agent do # for more information on OTP Applications def start(_type, _args) do Pleroma.HTML.compile_scrubbers() - Pleroma.Config.DeprecationWarnings.warn() + Config.DeprecationWarnings.warn() Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled() Pleroma.Repo.check_migrations_applied!() setup_instrumenters() @@ -42,17 +46,17 @@ def start(_type, _args) do children = [ Pleroma.Repo, - Pleroma.Config.TransferTask, + Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, Pleroma.Plugs.RateLimiter.Supervisor ] ++ cachex_children() ++ - hackney_pool_children() ++ + http_pools_children(Config.get(:env)) ++ [ Pleroma.Stats, Pleroma.JobQueueMonitor, - {Oban, Pleroma.Config.get(Oban)} + {Oban, Config.get(Oban)} ] ++ task_children(@env) ++ streamer_child(@env) ++ @@ -62,6 +66,18 @@ def start(_type, _args) do Pleroma.Gopher.Server ] + case Pleroma.OTPVersion.check_version() do + :ok -> :ok + {:error, version} -> raise " + !!!OTP VERSION WARNING!!! + You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. + " + :undefined -> raise " + !!!OTP VERSION WARNING!!! + To support correct handling of unordered certificates chains - OTP version must be > 22.2. + " + end + # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options opts = [strategy: :one_for_one, name: Pleroma.Supervisor] @@ -69,7 +85,7 @@ def start(_type, _args) do end def load_custom_modules do - dir = Pleroma.Config.get([:modules, :runtime_dir]) + dir = Config.get([:modules, :runtime_dir]) if dir && File.exists?(dir) do dir @@ -110,20 +126,6 @@ defp setup_instrumenters do Pleroma.Web.Endpoint.Instrumenter.setup() end - def enabled_hackney_pools do - [:media] ++ - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do - [:federation] - else - [] - end ++ - if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do - [:upload] - else - [] - end - end - defp cachex_children do [ build_cachex("used_captcha", ttl_interval: seconds_valid_interval()), @@ -145,7 +147,7 @@ defp idempotency_expiration, do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) defp seconds_valid_interval, - do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])) + do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid])) defp build_cachex(type, opts), do: %{ @@ -154,7 +156,7 @@ defp build_cachex(type, opts), type: :worker } - defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled]) + defp chat_enabled?, do: Config.get([:chat, :enabled]) defp streamer_child(:test), do: [] @@ -168,13 +170,6 @@ defp chat_child(_env, true) do defp chat_child(_, _), do: [] - defp hackney_pool_children do - for pool <- enabled_hackney_pools() do - options = Pleroma.Config.get([:hackney_pools, pool]) - :hackney_pool.child_spec(pool, options) - end - end - defp task_children(:test) do [ %{ @@ -199,4 +194,37 @@ defp task_children(_) do } ] end + + # start hackney and gun pools in tests + defp http_pools_children(:test) do + hackney_options = Config.get([:hackney_pools, :federation]) + hackney_pool = :hackney_pool.child_spec(:federation, hackney_options) + [hackney_pool, Pleroma.Pool.Supervisor] + end + + defp http_pools_children(_) do + :tesla + |> Application.get_env(:adapter) + |> http_pools() + end + + defp http_pools(Tesla.Adapter.Hackney) do + pools = [:federation, :media] + + pools = + if Config.get([Pleroma.Upload, :proxy_remote]) do + [:upload | pools] + else + pools + end + + for pool <- pools do + options = Config.get([:hackney_pools, pool]) + :hackney_pool.child_spec(pool, options) + end + end + + defp http_pools(Tesla.Adapter.Gun), do: [Pleroma.Pool.Supervisor] + + defp http_pools(_), do: [] end diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex index 119251bee..bdacefa97 100644 --- a/lib/pleroma/config/config_db.ex +++ b/lib/pleroma/config/config_db.ex @@ -278,8 +278,6 @@ defp do_convert({:proxy_url, {type, host, port}}) do } end - defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]} - defp do_convert(entity) when is_tuple(entity) do value = entity @@ -323,15 +321,6 @@ defp do_transform(%{"tuple" => [":proxy_url", %{"tuple" => [type, host, port]}]} {:proxy_url, {do_transform_string(type), parse_host(host), port}} end - defp do_transform(%{"tuple" => [":partial_chain", entity]}) do - {partial_chain, []} = - entity - |> String.replace(~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "") - |> Code.eval_string() - - {:partial_chain, partial_chain} - end - defp do_transform(%{"tuple" => entity}) do Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end) end diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 6c5ba1f95..251074aaa 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -18,7 +18,10 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, Oban}, {:pleroma, :rate_limit}, {:pleroma, :markup}, - {:plerome, :streamer} + {:pleroma, :streamer}, + {:pleroma, :pools}, + {:pleroma, :connections_pool}, + {:tesla, :adapter} ] @reboot_time_subkeys [ @@ -74,6 +77,28 @@ def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do end end + defp group_for_restart(:logger, key, _, merged_value) do + # change logger configuration in runtime, without restart + if Keyword.keyword?(merged_value) and + key not in [:compile_time_application, :backends, :compile_time_purge_matching] do + Logger.configure_backend(key, merged_value) + else + Logger.configure([{key, merged_value}]) + end + + nil + end + + defp group_for_restart(:tesla, _, _, _), do: :pleroma + + defp group_for_restart(group, _, _, _) when group != :pleroma, do: group + + defp group_for_restart(group, key, value, _) do + if pleroma_need_restart?(group, key, value) do + group + end + end + defp merge_and_update(setting) do try do key = ConfigDB.from_string(setting.key) @@ -95,21 +120,7 @@ defp merge_and_update(setting) do :ok = update_env(group, key, merged_value) - if group != :logger do - if group != :pleroma or pleroma_need_restart?(group, key, value) do - group - end - else - # change logger configuration in runtime, without restart - if Keyword.keyword?(merged_value) and - key not in [:compile_time_application, :backends, :compile_time_purge_matching] do - Logger.configure_backend(key, merged_value) - else - Logger.configure([{key, merged_value}]) - end - - nil - end + group_for_restart(group, key, value, merged_value) rescue error -> error_msg = diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex new file mode 100644 index 000000000..a0c3c5415 --- /dev/null +++ b/lib/pleroma/gun/api.ex @@ -0,0 +1,26 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.API do + @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} + @callback info(pid()) :: map() + @callback close(pid()) :: :ok + @callback await_up(pid) :: {:ok, atom()} | {:error, atom()} + @callback connect(pid(), map()) :: reference() + @callback await(pid(), reference()) :: {:response, :fin, 200, []} + + def open(host, port, opts), do: api().open(host, port, opts) + + def info(pid), do: api().info(pid) + + def close(pid), do: api().close(pid) + + def await_up(pid), do: api().await_up(pid) + + def connect(pid, opts), do: api().connect(pid, opts) + + def await(pid, ref), do: api().await(pid, ref) + + defp api, do: Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun) +end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex new file mode 100644 index 000000000..0134b016e --- /dev/null +++ b/lib/pleroma/gun/api/mock.ex @@ -0,0 +1,151 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.API.Mock do + @behaviour Pleroma.Gun.API + + alias Pleroma.Gun.API + + @impl API + def open('some-domain.com', 443, _) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "https", + origin_host: 'some-domain.com', + origin_port: 443 + }) + + {:ok, conn_pid} + end + + @impl API + def open(ip, port, _) + when ip in [{10_755, 10_368, 61_708, 131, 64_206, 45_068, 0, 9_694}, {127, 0, 0, 1}] and + port in [80, 443] do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + scheme = if port == 443, do: "https", else: "http" + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: scheme, + origin_host: ip, + origin_port: port + }) + + {:ok, conn_pid} + end + + @impl API + def open('localhost', 1234, %{ + protocols: [:socks], + proxy: {:socks5, 'localhost', 1234}, + socks_opts: %{host: 'proxy-socks.com', port: 80, version: 5} + }) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "http", + origin_host: 'proxy-socks.com', + origin_port: 80 + }) + + {:ok, conn_pid} + end + + @impl API + def open('localhost', 1234, %{ + protocols: [:socks], + proxy: {:socks4, 'localhost', 1234}, + socks_opts: %{ + host: 'proxy-socks.com', + port: 443, + protocols: [:http2], + tls_opts: [], + transport: :tls, + version: 4 + } + }) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "https", + origin_host: 'proxy-socks.com', + origin_port: 443 + }) + + {:ok, conn_pid} + end + + @impl API + def open('gun-not-up.com', 80, _opts), do: {:error, :timeout} + + @impl API + def open('example.com', port, _) when port in [443, 115] do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "https", + origin_host: 'example.com', + origin_port: 443 + }) + + {:ok, conn_pid} + end + + @impl API + def open(domain, 80, _) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "http", + origin_host: domain, + origin_port: 80 + }) + + {:ok, conn_pid} + end + + @impl API + def open({127, 0, 0, 1}, 8123, _) do + Task.start_link(fn -> Process.sleep(1_000) end) + end + + @impl API + def open('localhost', 9050, _) do + Task.start_link(fn -> Process.sleep(1_000) end) + end + + @impl API + def await_up(_pid), do: {:ok, :http} + + @impl API + def connect(pid, %{host: _, port: 80}) do + ref = make_ref() + Registry.register(API.Mock, ref, pid) + ref + end + + @impl API + def connect(pid, %{host: _, port: 443, protocols: [:http2], transport: :tls}) do + ref = make_ref() + Registry.register(API.Mock, ref, pid) + ref + end + + @impl API + def await(pid, ref) do + [{_, ^pid}] = Registry.lookup(API.Mock, ref) + {:response, :fin, 200, []} + end + + @impl API + def info(pid) do + [{_, info}] = Registry.lookup(API.Mock, pid) + info + end + + @impl API + def close(_pid), do: :ok +end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex new file mode 100644 index 000000000..2474829d6 --- /dev/null +++ b/lib/pleroma/gun/conn.ex @@ -0,0 +1,29 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.Conn do + @moduledoc """ + Struct for gun connection data + """ + @type gun_state :: :up | :down + @type conn_state :: :active | :idle + + @type t :: %__MODULE__{ + conn: pid(), + gun_state: gun_state(), + conn_state: conn_state(), + used_by: [pid()], + last_reference: pos_integer(), + crf: float(), + retries: pos_integer() + } + + defstruct conn: nil, + gun_state: :open, + conn_state: :init, + used_by: [], + last_reference: 0, + crf: 1, + retries: 0 +end diff --git a/lib/pleroma/gun/gun.ex b/lib/pleroma/gun/gun.ex new file mode 100644 index 000000000..4a1bbc95f --- /dev/null +++ b/lib/pleroma/gun/gun.ex @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun do + @behaviour Pleroma.Gun.API + + alias Pleroma.Gun.API + + @gun_keys [ + :connect_timeout, + :http_opts, + :http2_opts, + :protocols, + :retry, + :retry_timeout, + :trace, + :transport, + :tls_opts, + :tcp_opts, + :socks_opts, + :ws_opts + ] + + @impl API + def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun_keys)) + + @impl API + defdelegate info(pid), to: :gun + + @impl API + defdelegate close(pid), to: :gun + + @impl API + defdelegate await_up(pid), to: :gun + + @impl API + defdelegate connect(pid, opts), to: :gun + + @impl API + defdelegate await(pid, ref), to: :gun + + @spec flush(pid() | reference()) :: :ok + defdelegate flush(pid), to: :gun +end diff --git a/lib/pleroma/http/adapter.ex b/lib/pleroma/http/adapter.ex new file mode 100644 index 000000000..6166a3eb4 --- /dev/null +++ b/lib/pleroma/http/adapter.ex @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Adapter do + alias Pleroma.HTTP.Connection + + @type proxy :: + {Connection.host(), pos_integer()} + | {Connection.proxy_type(), pos_integer()} + @type host_type :: :domain | :ip + + @callback options(keyword(), URI.t()) :: keyword() + @callback after_request(keyword()) :: :ok + + @spec options(keyword(), URI.t()) :: keyword() + def options(opts, _uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + maybe_add_proxy(opts, format_proxy(proxy)) + end + + @spec maybe_get_conn(URI.t(), keyword()) :: keyword() + def maybe_get_conn(_uri, opts), do: opts + + @spec after_request(keyword()) :: :ok + def after_request(_opts), do: :ok + + @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil + def format_proxy(nil), do: nil + + def format_proxy(proxy_url) do + with {:ok, host, port} <- Connection.parse_proxy(proxy_url) do + {host, port} + else + {:ok, type, host, port} -> {type, host, port} + _ -> nil + end + end + + @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() + def maybe_add_proxy(opts, nil), do: opts + def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) + + @spec domain_or_fallback(String.t()) :: charlist() + def domain_or_fallback(host) do + case domain_or_ip(host) do + {:domain, domain} -> domain + {:ip, _ip} -> to_charlist(host) + end + end + + @spec domain_or_ip(String.t()) :: {host_type(), Connection.host()} + def domain_or_ip(host) do + charlist = to_charlist(host) + + case :inet.parse_address(charlist) do + {:error, :einval} -> + {:domain, :idna.encode(charlist)} + + {:ok, ip} when is_tuple(ip) and tuple_size(ip) in [4, 8] -> + {:ip, ip} + end + end +end diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex new file mode 100644 index 000000000..f25afeda7 --- /dev/null +++ b/lib/pleroma/http/adapter/gun.ex @@ -0,0 +1,123 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Adapter.Gun do + @behaviour Pleroma.HTTP.Adapter + + alias Pleroma.HTTP.Adapter + + require Logger + + alias Pleroma.Pool.Connections + + @defaults [ + connect_timeout: 20_000, + domain_lookup_timeout: 5_000, + tls_handshake_timeout: 5_000, + retry_timeout: 100, + await_up_timeout: 5_000 + ] + + @spec options(keyword(), URI.t()) :: keyword() + def options(connection_opts \\ [], %URI{} = uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + + @defaults + |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) + |> add_original(uri) + |> add_scheme_opts(uri) + |> Adapter.maybe_add_proxy(Adapter.format_proxy(proxy)) + |> maybe_get_conn(uri, connection_opts) + end + + @spec after_request(keyword()) :: :ok + def after_request(opts) do + with conn when not is_nil(conn) <- opts[:conn], + body_as when body_as != :chunks <- opts[:body_as] do + Connections.checkout(conn, self(), :gun_connections) + end + + :ok + end + + defp add_original(opts, %URI{host: host, port: port}) do + formatted_host = Adapter.domain_or_fallback(host) + + Keyword.put(opts, :original, "#{formatted_host}:#{port}") + end + + defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts + + defp add_scheme_opts(opts, %URI{scheme: "https", host: host, port: port}) do + adapter_opts = [ + certificates_verification: true, + tls_opts: [ + verify: :verify_peer, + cacertfile: CAStore.file_path(), + depth: 20, + reuse_sessions: false, + verify_fun: + {&:ssl_verify_hostname.verify_fun/3, [check_hostname: Adapter.domain_or_fallback(host)]} + ] + ] + + adapter_opts = + if port != 443 do + Keyword.put(adapter_opts, :transport, :tls) + else + adapter_opts + end + + Keyword.merge(opts, adapter_opts) + end + + defp maybe_get_conn(adapter_opts, uri, connection_opts) do + {receive_conn?, opts} = + adapter_opts + |> Keyword.merge(connection_opts) + |> Keyword.pop(:receive_conn, true) + + if Connections.alive?(:gun_connections) and receive_conn? do + try_to_get_conn(uri, opts) + else + opts + end + end + + defp try_to_get_conn(uri, opts) do + try do + case Connections.checkin(uri, :gun_connections) do + nil -> + Logger.info( + "Gun connections pool checkin was not succesfull. Trying to open conn for next request." + ) + + :ok = Connections.open_conn(uri, :gun_connections, opts) + opts + + conn when is_pid(conn) -> + Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri(uri)}") + + opts + |> Keyword.put(:conn, conn) + |> Keyword.put(:close_conn, false) + end + rescue + error -> + Logger.warn("Gun connections pool checkin caused error #{inspect(error)}") + opts + catch + :exit, {:timeout, _} -> + Logger.info( + "Gun connections pool checkin with timeout error #{Connections.compose_uri(uri)}" + ) + + opts + + :exit, error -> + Logger.warn("Gun pool checkin exited with error #{inspect(error)}") + opts + end + end +end diff --git a/lib/pleroma/http/adapter/hackney.ex b/lib/pleroma/http/adapter/hackney.ex new file mode 100644 index 000000000..00db30083 --- /dev/null +++ b/lib/pleroma/http/adapter/hackney.ex @@ -0,0 +1,41 @@ +defmodule Pleroma.HTTP.Adapter.Hackney do + @behaviour Pleroma.HTTP.Adapter + + @defaults [ + connect_timeout: 10_000, + recv_timeout: 20_000, + follow_redirect: true, + force_redirect: true, + pool: :federation + ] + + @spec options(keyword(), URI.t()) :: keyword() + def options(connection_opts \\ [], %URI{} = uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + + @defaults + |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) + |> Keyword.merge(connection_opts) + |> add_scheme_opts(uri) + |> Pleroma.HTTP.Adapter.maybe_add_proxy(proxy) + end + + defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts + + defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do + ssl_opts = [ + ssl_options: [ + # Workaround for remote server certificate chain issues + partial_chain: &:hackney_connect.partial_chain/1, + + # We don't support TLS v1.3 yet + versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"], + server_name_indication: to_charlist(host) + ] + ] + + Keyword.merge(opts, ssl_opts) + end + + def after_request(_), do: :ok +end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 7e2c6f5e8..85918341a 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -4,40 +4,99 @@ defmodule Pleroma.HTTP.Connection do @moduledoc """ - Connection for http-requests. + Configure Tesla.Client with default and customized adapter options. """ + @type ip_address :: ipv4_address() | ipv6_address() + @type ipv4_address :: {0..255, 0..255, 0..255, 0..255} + @type ipv6_address :: + {0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535} + @type proxy_type() :: :socks4 | :socks5 + @type host() :: charlist() | ip_address() - @hackney_options [ - connect_timeout: 10_000, - recv_timeout: 20_000, - follow_redirect: true, - force_redirect: true, - pool: :federation - ] - @adapter Application.get_env(:tesla, :adapter) + @defaults [pool: :federation] + + require Logger + + alias Pleroma.Config + alias Pleroma.HTTP.Adapter @doc """ - Configure a client connection - - # Returns - - Tesla.Env.client + Merge default connection & adapter options with received ones. """ - @spec new(Keyword.t()) :: Tesla.Env.client() - def new(opts \\ []) do - Tesla.client([], {@adapter, hackney_options(opts)}) + + @spec options(URI.t(), keyword()) :: keyword() + def options(%URI{} = uri, opts \\ []) do + @defaults + |> pool_timeout() + |> Keyword.merge(opts) + |> adapter().options(uri) end - # fetch Hackney options - # - def hackney_options(opts) do - options = Keyword.get(opts, :adapter, []) - adapter_options = Pleroma.Config.get([:http, :adapter], []) - proxy_url = Pleroma.Config.get([:http, :proxy_url], nil) + defp pool_timeout(opts) do + timeout = + Config.get([:pools, opts[:pool], :timeout]) || Config.get([:pools, :default, :timeout]) - @hackney_options - |> Keyword.merge(adapter_options) - |> Keyword.merge(options) - |> Keyword.merge(proxy: proxy_url) + Keyword.merge(opts, timeout: timeout) + end + + @spec after_request(keyword()) :: :ok + def after_request(opts), do: adapter().after_request(opts) + + defp adapter do + case Application.get_env(:tesla, :adapter) do + Tesla.Adapter.Gun -> Adapter.Gun + Tesla.Adapter.Hackney -> Adapter.Hackney + _ -> Adapter + end + end + + @spec parse_proxy(String.t() | tuple() | nil) :: + {:ok, host(), pos_integer()} + | {:ok, proxy_type(), host(), pos_integer()} + | {:error, atom()} + | nil + + def parse_proxy(nil), do: nil + + def parse_proxy(proxy) when is_binary(proxy) do + with [host, port] <- String.split(proxy, ":"), + {port, ""} <- Integer.parse(port) do + {:ok, parse_host(host), port} + else + {_, _} -> + Logger.warn("parsing port in proxy fail #{inspect(proxy)}") + {:error, :error_parsing_port_in_proxy} + + :error -> + Logger.warn("parsing port in proxy fail #{inspect(proxy)}") + {:error, :error_parsing_port_in_proxy} + + _ -> + Logger.warn("parsing proxy fail #{inspect(proxy)}") + {:error, :error_parsing_proxy} + end + end + + def parse_proxy(proxy) when is_tuple(proxy) do + with {type, host, port} <- proxy do + {:ok, type, parse_host(host), port} + else + _ -> + Logger.warn("parsing proxy fail #{inspect(proxy)}") + {:error, :error_parsing_proxy} + end + end + + @spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address() + def parse_host(host) when is_list(host), do: host + def parse_host(host) when is_atom(host), do: to_charlist(host) + + def parse_host(host) when is_binary(host) do + host = to_charlist(host) + + case :inet.parse_address(host) do + {:error, :einval} -> host + {:ok, ip} -> ip + end end end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index dec24458a..ad47dc936 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -4,21 +4,47 @@ defmodule Pleroma.HTTP do @moduledoc """ - + Wrapper for `Tesla.request/2`. """ alias Pleroma.HTTP.Connection + alias Pleroma.HTTP.Request alias Pleroma.HTTP.RequestBuilder, as: Builder + alias Tesla.Client + alias Tesla.Env + + require Logger @type t :: __MODULE__ @doc """ - Builds and perform http request. + Performs GET request. + + See `Pleroma.HTTP.request/5` + """ + @spec get(Request.url() | nil, Request.headers(), keyword()) :: + nil | {:ok, Env.t()} | {:error, any()} + def get(url, headers \\ [], options \\ []) + def get(nil, _, _), do: nil + def get(url, headers, options), do: request(:get, url, "", headers, options) + + @doc """ + Performs POST request. + + See `Pleroma.HTTP.request/5` + """ + @spec post(Request.url(), String.t(), Request.headers(), keyword()) :: + {:ok, Env.t()} | {:error, any()} + def post(url, body, headers \\ [], options \\ []), + do: request(:post, url, body, headers, options) + + @doc """ + Builds and performs http request. # Arguments: `method` - :get, :post, :put, :delete - `url` - `body` + `url` - full url + `body` - request body `headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]` `options` - custom, per-request middleware or adapter options @@ -26,23 +52,78 @@ defmodule Pleroma.HTTP do `{:ok, %Tesla.Env{}}` or `{:error, error}` """ - def request(method, url, body \\ "", headers \\ [], options \\ []) do + @spec request(atom(), Request.url(), String.t(), Request.headers(), keyword()) :: + {:ok, Env.t()} | {:error, any()} + def request(method, url, body, headers, options) when is_binary(url) do + with uri <- URI.parse(url), + received_adapter_opts <- Keyword.get(options, :adapter, []), + adapter_opts <- Connection.options(uri, received_adapter_opts), + options <- put_in(options[:adapter], adapter_opts), + params <- Keyword.get(options, :params, []), + request <- build_request(method, headers, options, url, body, params), + client <- Tesla.client([Tesla.Middleware.FollowRedirects], tesla_adapter()), + pid <- Process.whereis(adapter_opts[:pool]) do + pool_alive? = + if tesla_adapter() == Tesla.Adapter.Gun do + if pid, do: Process.alive?(pid), else: false + else + false + end + + request_opts = + adapter_opts + |> Enum.into(%{}) + |> Map.put(:env, Pleroma.Config.get([:env])) + |> Map.put(:pool_alive?, pool_alive?) + + response = + request( + client, + request, + request_opts + ) + + Connection.after_request(adapter_opts) + + response + end + end + + @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()} + def request(%Client{} = client, request, %{env: :test}), do: request_try(client, request) + + def request(%Client{} = client, request, %{body_as: :chunks}) do + request_try(client, request) + end + + def request(%Client{} = client, request, %{pool_alive?: false}) do + request_try(client, request) + end + + def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do try do - options = - process_request_options(options) - |> process_sni_options(url) + :poolboy.transaction( + pool, + &Pleroma.Pool.Request.execute(&1, client, request, timeout + 500), + timeout + 1_000 + ) + rescue + e -> + {:error, e} + catch + :exit, {:timeout, _} -> + Logger.warn("Receive response from pool failed #{request[:url]}") + {:error, :recv_pool_timeout} - params = Keyword.get(options, :params, []) + :exit, e -> + {:error, e} + end + end - %{} - |> Builder.method(method) - |> Builder.headers(headers) - |> Builder.opts(options) - |> Builder.url(url) - |> Builder.add_param(:body, :body, body) - |> Builder.add_param(:query, :query, params) - |> Enum.into([]) - |> (&Tesla.request(Connection.new(options), &1)).() + @spec request_try(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()} + def request_try(client, request) do + try do + Tesla.request(client, request) rescue e -> {:error, e} @@ -52,35 +133,16 @@ def request(method, url, body \\ "", headers \\ [], options \\ []) do end end - defp process_sni_options(options, nil), do: options - - defp process_sni_options(options, url) do - uri = URI.parse(url) - host = uri.host |> to_charlist() - - case uri.scheme do - "https" -> options ++ [ssl: [server_name_indication: host]] - _ -> options - end + defp build_request(method, headers, options, url, body, params) do + Builder.new() + |> Builder.method(method) + |> Builder.headers(headers) + |> Builder.opts(options) + |> Builder.url(url) + |> Builder.add_param(:body, :body, body) + |> Builder.add_param(:query, :query, params) + |> Builder.convert_to_keyword() end - def process_request_options(options) do - Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options) - end - - @doc """ - Performs GET request. - - See `Pleroma.HTTP.request/5` - """ - def get(url, headers \\ [], options \\ []), - do: request(:get, url, "", headers, options) - - @doc """ - Performs POST request. - - See `Pleroma.HTTP.request/5` - """ - def post(url, body, headers \\ [], options \\ []), - do: request(:post, url, body, headers, options) + defp tesla_adapter, do: Application.get_env(:tesla, :adapter) end diff --git a/lib/pleroma/http/request.ex b/lib/pleroma/http/request.ex new file mode 100644 index 000000000..891d88d53 --- /dev/null +++ b/lib/pleroma/http/request.ex @@ -0,0 +1,23 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Request do + @moduledoc """ + Request struct. + """ + defstruct method: :get, url: "", query: [], headers: [], body: "", opts: [] + + @type method :: :head | :get | :delete | :trace | :options | :post | :put | :patch + @type url :: String.t() + @type headers :: [{String.t(), String.t()}] + + @type t :: %__MODULE__{ + method: method(), + url: url(), + query: keyword(), + headers: headers(), + body: String.t(), + opts: keyword() + } +end diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index e23457999..491acd0f9 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -7,77 +7,54 @@ defmodule Pleroma.HTTP.RequestBuilder do Helper functions for building Tesla requests """ + alias Pleroma.HTTP.Request + alias Tesla.Multipart + @doc """ - Specify the request method when building a request - - ## Parameters - - - request (Map) - Collected request options - - m (atom) - Request method - - ## Returns - - Map + Creates new request """ - @spec method(map(), atom) :: map() - def method(request, m) do - Map.put_new(request, :method, m) - end + @spec new(Request.t()) :: Request.t() + def new(%Request{} = request \\ %Request{}), do: request @doc """ Specify the request method when building a request - - ## Parameters - - - request (Map) - Collected request options - - u (String) - Request URL - - ## Returns - - Map """ - @spec url(map(), String.t()) :: map() - def url(request, u) do - Map.put_new(request, :url, u) - end + @spec method(Request.t(), Request.method()) :: Request.t() + def method(request, m), do: %{request | method: m} + + @doc """ + Specify the request method when building a request + """ + @spec url(Request.t(), Request.url()) :: Request.t() + def url(request, u), do: %{request | url: u} @doc """ Add headers to the request """ - @spec headers(map(), list(tuple)) :: map() - def headers(request, header_list) do - header_list = + @spec headers(Request.t(), Request.headers()) :: Request.t() + def headers(request, headers) do + headers_list = if Pleroma.Config.get([:http, :send_user_agent]) do - header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}] + headers ++ [{"user-agent", Pleroma.Application.user_agent()}] else - header_list + headers end - Map.put_new(request, :headers, header_list) + %{request | headers: headers_list} end @doc """ Add custom, per-request middleware or adapter options to the request """ - @spec opts(map(), Keyword.t()) :: map() - def opts(request, options) do - Map.put_new(request, :opts, options) - end + @spec opts(Request.t(), keyword()) :: Request.t() + def opts(request, options), do: %{request | opts: options} + # NOTE: isn't used anywhere @doc """ Add optional parameters to the request - ## Parameters - - - request (Map) - Collected request options - - definitions (Map) - Map of parameter name to parameter location. - - options (KeywordList) - The provided optional parameters - - ## Returns - - Map """ - @spec add_optional_params(map(), %{optional(atom) => atom}, keyword()) :: map() + @spec add_optional_params(Request.t(), %{optional(atom) => atom}, keyword()) :: map() def add_optional_params(request, _, []), do: request def add_optional_params(request, definitions, [{key, value} | tail]) do @@ -94,49 +71,43 @@ def add_optional_params(request, definitions, [{key, value} | tail]) do @doc """ Add optional parameters to the request - - ## Parameters - - - request (Map) - Collected request options - - location (atom) - Where to put the parameter - - key (atom) - The name of the parameter - - value (any) - The value of the parameter - - ## Returns - - Map """ - @spec add_param(map(), atom, atom, any()) :: map() - def add_param(request, :query, :query, values), do: Map.put(request, :query, values) + @spec add_param(Request.t(), atom(), atom(), any()) :: Request.t() + def add_param(request, :query, :query, values), do: %{request | query: values} - def add_param(request, :body, :body, value), do: Map.put(request, :body, value) + def add_param(request, :body, :body, value), do: %{request | body: value} def add_param(request, :body, key, value) do request - |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0) + |> Map.put(:body, Multipart.new()) |> Map.update!( :body, - &Tesla.Multipart.add_field( + &Multipart.add_field( &1, key, Jason.encode!(value), - headers: [{:"Content-Type", "application/json"}] + headers: [{"content-type", "application/json"}] ) ) end def add_param(request, :file, name, path) do request - |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0) - |> Map.update!(:body, &Tesla.Multipart.add_file(&1, path, name: name)) + |> Map.put(:body, Multipart.new()) + |> Map.update!(:body, &Multipart.add_file(&1, path, name: name)) end def add_param(request, :form, name, value) do - request - |> Map.update(:body, %{name => value}, &Map.put(&1, name, value)) + Map.update(request, :body, %{name => value}, &Map.put(&1, name, value)) end def add_param(request, location, key, value) do Map.update(request, location, [{key, value}], &(&1 ++ [{key, value}])) end + + def convert_to_keyword(request) do + request + |> Map.from_struct() + |> Enum.into([]) + end end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 037c42339..5e9bf1574 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -137,7 +137,7 @@ defp make_signature(id, date) do date: date }) - [{:Signature, signature}] + [{"signature", signature}] end defp sign_fetch(headers, id, date) do @@ -150,7 +150,7 @@ defp sign_fetch(headers, id, date) do defp maybe_date_fetch(headers, date) do if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do - headers ++ [{:Date, date}] + headers ++ [{"date", date}] else headers end @@ -162,7 +162,7 @@ def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do date = Pleroma.Signature.signed_date() headers = - [{:Accept, "application/activity+json"}] + [{"accept", "application/activity+json"}] |> maybe_date_fetch(date) |> sign_fetch(id, date) diff --git a/lib/pleroma/otp_version.ex b/lib/pleroma/otp_version.ex new file mode 100644 index 000000000..0be189304 --- /dev/null +++ b/lib/pleroma/otp_version.ex @@ -0,0 +1,63 @@ +defmodule Pleroma.OTPVersion do + @type check_status() :: :undefined | {:error, String.t()} | :ok + + require Logger + + @spec check_version() :: check_status() + def check_version do + # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version + paths = [ + Path.join(:code.root_dir(), "OTP_VERSION"), + Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"]) + ] + + :tesla + |> Application.get_env(:adapter) + |> get_and_check_version(paths) + end + + @spec get_and_check_version(module(), [Path.t()]) :: check_status() + def get_and_check_version(Tesla.Adapter.Gun, paths) do + paths + |> check_files() + |> check_version() + end + + def get_and_check_version(_, _), do: :ok + + defp check_files([]), do: nil + + defp check_files([path | paths]) do + if File.exists?(path) do + File.read!(path) + else + check_files(paths) + end + end + + defp check_version(nil), do: :undefined + + defp check_version(version) do + try do + version = String.replace(version, ~r/\r|\n|\s/, "") + + formatted = + version + |> String.split(".") + |> Enum.map(&String.to_integer/1) + |> Enum.take(2) + + with [major, minor] when length(formatted) == 2 <- formatted, + true <- (major == 22 and minor >= 2) or major > 22 do + :ok + else + false -> {:error, version} + _ -> :undefined + end + rescue + _ -> :undefined + catch + _ -> :undefined + end + end +end diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex new file mode 100644 index 000000000..1ed16d1c1 --- /dev/null +++ b/lib/pleroma/pool/connections.ex @@ -0,0 +1,415 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool.Connections do + use GenServer + + require Logger + + @type domain :: String.t() + @type conn :: Pleroma.Gun.Conn.t() + + @type t :: %__MODULE__{ + conns: %{domain() => conn()}, + opts: keyword() + } + + defstruct conns: %{}, opts: [] + + alias Pleroma.Gun.API + alias Pleroma.Gun.Conn + + @spec start_link({atom(), keyword()}) :: {:ok, pid()} + def start_link({name, opts}) do + GenServer.start_link(__MODULE__, opts, name: name) + end + + @impl true + def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}} + + @spec checkin(String.t() | URI.t(), atom()) :: pid() | nil + def checkin(url, name) + def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name) + + def checkin(%URI{} = uri, name) do + timeout = Pleroma.Config.get([:connections_pool, :receive_connection_timeout], 250) + + GenServer.call( + name, + {:checkin, uri}, + timeout + ) + end + + @spec open_conn(String.t() | URI.t(), atom(), keyword()) :: :ok + def open_conn(url, name, opts \\ []) + def open_conn(url, name, opts) when is_binary(url), do: open_conn(URI.parse(url), name, opts) + + def open_conn(%URI{} = uri, name, opts) do + pool_opts = Pleroma.Config.get([:connections_pool], []) + + opts = + opts + |> Enum.into(%{}) + |> Map.put_new(:receive, false) + |> Map.put_new(:retry, pool_opts[:retry] || 5) + |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100) + |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) + + GenServer.cast(name, {:open_conn, %{opts: opts, uri: uri}}) + end + + @spec alive?(atom()) :: boolean() + def alive?(name) do + pid = Process.whereis(name) + if pid, do: Process.alive?(pid), else: false + end + + @spec get_state(atom()) :: t() + def get_state(name) do + GenServer.call(name, :state) + end + + @spec checkout(pid(), pid(), atom()) :: :ok + def checkout(conn, pid, name) do + GenServer.cast(name, {:checkout, conn, pid}) + end + + @impl true + def handle_cast({:open_conn, %{opts: opts, uri: uri}}, state) do + Logger.debug("opening new #{compose_uri(uri)}") + max_connections = state.opts[:max_connections] + + key = compose_key(uri) + + if Enum.count(state.conns) < max_connections do + open_conn(key, uri, state, opts) + else + try_to_open_conn(key, uri, state, opts) + end + end + + @impl true + def handle_cast({:checkout, conn_pid, pid}, state) do + Logger.debug("checkout #{inspect(conn_pid)}") + + state = + with true <- Process.alive?(conn_pid), + {key, conn} <- find_conn(state.conns, conn_pid), + used_by <- List.keydelete(conn.used_by, pid, 0) do + conn_state = + if used_by == [] do + :idle + else + conn.conn_state + end + + put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by}) + else + false -> + Logger.warn("checkout for closed conn #{inspect(conn_pid)}") + state + + nil -> + Logger.info("checkout for alive conn #{inspect(conn_pid)}, but is not in state") + state + end + + {:noreply, state} + end + + @impl true + def handle_call({:checkin, uri}, from, state) do + Logger.debug("checkin #{compose_uri(uri)}") + key = compose_key(uri) + + case state.conns[key] do + %{conn: conn, gun_state: gun_state} = current_conn when gun_state == :up -> + Logger.debug("reusing conn #{compose_uri(uri)}") + + with time <- :os.system_time(:second), + last_reference <- time - current_conn.last_reference, + current_crf <- crf(last_reference, 100, current_conn.crf), + state <- + put_in(state.conns[key], %{ + current_conn + | last_reference: time, + crf: current_crf, + conn_state: :active, + used_by: [from | current_conn.used_by] + }) do + {:reply, conn, state} + end + + %{gun_state: gun_state} when gun_state == :down -> + {:reply, nil, state} + + nil -> + {:reply, nil, state} + end + end + + @impl true + def handle_call(:state, _from, state), do: {:reply, state, state} + + @impl true + def handle_info({:gun_up, conn_pid, _protocol}, state) do + state = + with true <- Process.alive?(conn_pid), + conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid), + {key, conn} <- find_conn(state.conns, conn_pid, conn_key), + time <- :os.system_time(:second), + last_reference <- time - conn.last_reference, + current_crf <- crf(last_reference, 100, conn.crf) do + put_in(state.conns[key], %{ + conn + | gun_state: :up, + last_reference: time, + crf: current_crf, + conn_state: :active, + retries: 0 + }) + else + :error_gun_info -> + Logger.warn(":gun.info caused error") + state + + false -> + Logger.warn(":gun_up message for closed conn #{inspect(conn_pid)}") + state + + nil -> + Logger.warn( + ":gun_up message for alive conn #{inspect(conn_pid)}, but deleted from state" + ) + + :ok = API.close(conn_pid) + + state + end + + {:noreply, state} + end + + @impl true + def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do + # we can't get info on this pid, because pid is dead + state = + with true <- Process.alive?(conn_pid), + {key, conn} <- find_conn(state.conns, conn_pid) do + if conn.retries == 5 do + Logger.debug("closing conn if retries is eq 5 #{inspect(conn_pid)}") + :ok = API.close(conn.conn) + + put_in( + state.conns, + Map.delete(state.conns, key) + ) + else + put_in(state.conns[key], %{ + conn + | gun_state: :down, + retries: conn.retries + 1 + }) + end + else + false -> + # gun can send gun_down for closed conn, maybe connection is not closed yet + Logger.warn(":gun_down message for closed conn #{inspect(conn_pid)}") + state + + nil -> + Logger.warn( + ":gun_down message for alive conn #{inspect(conn_pid)}, but deleted from state" + ) + + :ok = API.close(conn_pid) + + state + end + + {:noreply, state} + end + + defp compose_key(%URI{scheme: scheme, host: host, port: port}), do: "#{scheme}:#{host}:#{port}" + + defp compose_key_gun_info(pid) do + try do + # sometimes :gun.info can raise MatchError, which lead to pool terminate + %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = API.info(pid) + + host = + case :inet.ntoa(origin_host) do + {:error, :einval} -> origin_host + ip -> ip + end + + "#{scheme}:#{host}:#{port}" + rescue + _ -> :error_gun_info + end + end + + defp find_conn(conns, conn_pid) do + Enum.find(conns, fn {_key, conn} -> + conn.conn == conn_pid + end) + end + + defp find_conn(conns, conn_pid, conn_key) do + Enum.find(conns, fn {key, conn} -> + key == conn_key and conn.conn == conn_pid + end) + end + + defp open_conn(key, uri, state, %{proxy: {proxy_host, proxy_port}} = opts) do + connect_opts = + uri + |> destination_opts() + |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) + + with open_opts <- Map.delete(opts, :tls_opts), + {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), + {:ok, _} <- API.await_up(conn), + stream <- API.connect(conn, connect_opts), + {:response, :fin, 200, _} <- API.await(conn, stream), + state <- + put_in(state.conns[key], %Conn{ + conn: conn, + gun_state: :up, + conn_state: :active, + last_reference: :os.system_time(:second) + }) do + {:noreply, state} + else + error -> + Logger.warn( + "Received error on opening connection with http proxy #{uri.scheme}://#{ + compose_uri(uri) + }: #{inspect(error)}" + ) + + {:noreply, state} + end + end + + defp open_conn(key, uri, state, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do + version = + proxy_type + |> to_string() + |> String.last() + |> case do + "4" -> 4 + _ -> 5 + end + + socks_opts = + uri + |> destination_opts() + |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) + |> Map.put(:version, version) + + opts = + opts + |> Map.put(:protocols, [:socks]) + |> Map.put(:socks_opts, socks_opts) + + with {:ok, conn} <- API.open(proxy_host, proxy_port, opts), + {:ok, _} <- API.await_up(conn), + state <- + put_in(state.conns[key], %Conn{ + conn: conn, + gun_state: :up, + conn_state: :active, + last_reference: :os.system_time(:second) + }) do + {:noreply, state} + else + error -> + Logger.warn( + "Received error on opening connection with socks proxy #{uri.scheme}://#{ + compose_uri(uri) + }: #{inspect(error)}" + ) + + {:noreply, state} + end + end + + defp open_conn(key, %URI{host: host, port: port} = uri, state, opts) do + Logger.debug("opening conn #{compose_uri(uri)}") + {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + + with {:ok, conn} <- API.open(host, port, opts), + {:ok, _} <- API.await_up(conn), + state <- + put_in(state.conns[key], %Conn{ + conn: conn, + gun_state: :up, + conn_state: :active, + last_reference: :os.system_time(:second) + }) do + Logger.debug("new conn opened #{compose_uri(uri)}") + Logger.debug("replying to the call #{compose_uri(uri)}") + {:noreply, state} + else + error -> + Logger.warn( + "Received error on opening connection #{uri.scheme}://#{compose_uri(uri)}: #{ + inspect(error) + }" + ) + + {:noreply, state} + end + end + + defp destination_opts(%URI{host: host, port: port}) do + {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + %{host: host, port: port} + end + + defp add_http2_opts(opts, "https", tls_opts) do + Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts}) + end + + defp add_http2_opts(opts, _, _), do: opts + + @spec get_unused_conns(map()) :: [{domain(), conn()}] + def get_unused_conns(conns) do + conns + |> Enum.filter(fn {_k, v} -> + v.conn_state == :idle and v.used_by == [] + end) + |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> + x.crf <= y.crf and x.last_reference <= y.last_reference + end) + end + + defp try_to_open_conn(key, uri, state, opts) do + Logger.debug("try to open conn #{compose_uri(uri)}") + + with [{close_key, least_used} | _conns] <- get_unused_conns(state.conns), + :ok <- API.close(least_used.conn), + state <- + put_in( + state.conns, + Map.delete(state.conns, close_key) + ) do + Logger.debug( + "least used conn found and closed #{inspect(least_used.conn)} #{compose_uri(uri)}" + ) + + open_conn(key, uri, state, opts) + else + [] -> {:noreply, state} + end + end + + def crf(current, steps, crf) do + 1 + :math.pow(0.5, current / steps) * crf + end + + def compose_uri(%URI{} = uri), do: "#{uri.host}#{uri.path}" +end diff --git a/lib/pleroma/pool/pool.ex b/lib/pleroma/pool/pool.ex new file mode 100644 index 000000000..a7ae64ce4 --- /dev/null +++ b/lib/pleroma/pool/pool.ex @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool do + def child_spec(opts) do + poolboy_opts = + opts + |> Keyword.put(:worker_module, Pleroma.Pool.Request) + |> Keyword.put(:name, {:local, opts[:name]}) + |> Keyword.put(:size, opts[:size]) + |> Keyword.put(:max_overflow, opts[:max_overflow]) + + %{ + id: opts[:id] || {__MODULE__, make_ref()}, + start: {:poolboy, :start_link, [poolboy_opts, [name: opts[:name]]]}, + restart: :permanent, + shutdown: 5000, + type: :worker + } + end +end diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex new file mode 100644 index 000000000..2c3574561 --- /dev/null +++ b/lib/pleroma/pool/request.ex @@ -0,0 +1,72 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool.Request do + use GenServer + + require Logger + + def start_link(args) do + GenServer.start_link(__MODULE__, args) + end + + @impl true + def init(_), do: {:ok, []} + + @spec execute(pid() | atom(), Tesla.Client.t(), keyword(), pos_integer()) :: + {:ok, Tesla.Env.t()} | {:error, any()} + def execute(pid, client, request, timeout) do + GenServer.call(pid, {:execute, client, request}, timeout) + end + + @impl true + def handle_call({:execute, client, request}, _from, state) do + response = Pleroma.HTTP.request_try(client, request) + + {:reply, response, state} + end + + @impl true + def handle_info({:gun_data, _conn, stream, _, _}, state) do + # in some cases if we reuse conn and got {:error, :body_too_large} + # gun continues to send messages to this process, + # so we flush messages for this request + :ok = :gun.flush(stream) + + {:noreply, state} + end + + @impl true + def handle_info({:gun_up, _conn, _protocol}, state) do + {:noreply, state} + end + + @impl true + def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do + # don't flush messages here, because gun can reconnect + {:noreply, state} + end + + @impl true + def handle_info({:gun_error, _conn, stream, _error}, state) do + :ok = :gun.flush(stream) + {:noreply, state} + end + + @impl true + def handle_info({:gun_push, _conn, _stream, _new_stream, _method, _uri, _headers}, state) do + {:noreply, state} + end + + @impl true + def handle_info({:gun_response, _conn, _stream, _, _status, _headers}, state) do + {:noreply, state} + end + + @impl true + def handle_info(msg, state) do + Logger.warn("Received unexpected message #{inspect(__MODULE__)} #{inspect(msg)}") + {:noreply, state} + end +end diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex new file mode 100644 index 000000000..32be2264d --- /dev/null +++ b/lib/pleroma/pool/supervisor.ex @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool.Supervisor do + use Supervisor + + alias Pleroma.Pool + + def start_link(args) do + Supervisor.start_link(__MODULE__, args, name: __MODULE__) + end + + def init(_) do + children = + [ + %{ + id: Pool.Connections, + start: + {Pool.Connections, :start_link, + [{:gun_connections, Pleroma.Config.get([:connections_pool])}]} + } + ] ++ pools() + + Supervisor.init(children, strategy: :one_for_one) + end + + defp pools do + for {pool_name, pool_opts} <- Pleroma.Config.get([:pools]) do + pool_opts + |> Keyword.put(:id, {Pool, pool_name}) + |> Keyword.put(:name, pool_name) + |> Pool.child_spec() + end + end +end diff --git a/lib/pleroma/reverse_proxy/client.ex b/lib/pleroma/reverse_proxy/client.ex index 776c4794c..63261b94c 100644 --- a/lib/pleroma/reverse_proxy/client.ex +++ b/lib/pleroma/reverse_proxy/client.ex @@ -3,19 +3,23 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client do - @callback request(atom(), String.t(), [tuple()], String.t(), list()) :: - {:ok, pos_integer(), [tuple()], reference() | map()} - | {:ok, pos_integer(), [tuple()]} + @type status :: pos_integer() + @type header_name :: String.t() + @type header_value :: String.t() + @type headers :: [{header_name(), header_value()}] + + @callback request(atom(), String.t(), headers(), String.t(), list()) :: + {:ok, status(), headers(), reference() | map()} + | {:ok, status(), headers()} | {:ok, reference()} | {:error, term()} - @callback stream_body(reference() | pid() | map()) :: - {:ok, binary()} | :done | {:error, String.t()} + @callback stream_body(map()) :: {:ok, binary(), map()} | :done | {:error, atom() | String.t()} @callback close(reference() | pid() | map()) :: :ok - def request(method, url, headers, "", opts \\ []) do - client().request(method, url, headers, "", opts) + def request(method, url, headers, body \\ "", opts \\ []) do + client().request(method, url, headers, body, opts) end def stream_body(ref), do: client().stream_body(ref) @@ -23,6 +27,12 @@ def stream_body(ref), do: client().stream_body(ref) def close(ref), do: client().close(ref) defp client do - Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney) + :tesla + |> Application.get_env(:adapter) + |> client() end + + defp client(Tesla.Adapter.Hackney), do: Pleroma.ReverseProxy.Client.Hackney + defp client(Tesla.Adapter.Gun), do: Pleroma.ReverseProxy.Client.Tesla + defp client(_), do: Pleroma.Config.get!(Pleroma.ReverseProxy.Client) end diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex new file mode 100644 index 000000000..e41560ab0 --- /dev/null +++ b/lib/pleroma/reverse_proxy/client/hackney.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ReverseProxy.Client.Hackney do + @behaviour Pleroma.ReverseProxy.Client + + @impl true + def request(method, url, headers, body, opts \\ []) do + :hackney.request(method, url, headers, body, opts) + end + + @impl true + def stream_body(ref) do + case :hackney.stream_body(ref) do + :done -> :done + {:ok, data} -> {:ok, data, ref} + {:error, error} -> {:error, error} + end + end + + @impl true + def close(ref), do: :hackney.close(ref) +end diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex new file mode 100644 index 000000000..55a11b4a8 --- /dev/null +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -0,0 +1,87 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ReverseProxy.Client.Tesla do + @type headers() :: [{String.t(), String.t()}] + @type status() :: pos_integer() + + @behaviour Pleroma.ReverseProxy.Client + + @spec request(atom(), String.t(), headers(), String.t(), keyword()) :: + {:ok, status(), headers} + | {:ok, status(), headers, map()} + | {:error, atom() | String.t()} + | no_return() + + @impl true + def request(method, url, headers, body, opts \\ []) do + _adapter = check_adapter() + + with opts <- Keyword.merge(opts, body_as: :chunks, mode: :passive), + {:ok, response} <- + Pleroma.HTTP.request( + method, + url, + body, + headers, + Keyword.put(opts, :adapter, opts) + ) do + if is_map(response.body) and method != :head do + {:ok, response.status, response.headers, response.body} + else + {:ok, response.status, response.headers} + end + else + {:error, error} -> {:error, error} + end + end + + @impl true + @spec stream_body(map()) :: {:ok, binary(), map()} | {:error, atom() | String.t()} | :done + def stream_body(%{pid: pid, opts: opts, fin: true}) do + # if connection was sended and there were redirects, we need to close new conn - pid manually + if opts[:old_conn], do: Tesla.Adapter.Gun.close(pid) + # if there were redirects we need to checkout old conn + conn = opts[:old_conn] || opts[:conn] + + if conn, do: :ok = Pleroma.Pool.Connections.checkout(conn, self(), :gun_connections) + + :done + end + + def stream_body(client) do + case read_chunk!(client) do + {:fin, body} -> + {:ok, body, Map.put(client, :fin, true)} + + {:nofin, part} -> + {:ok, part, client} + + {:error, error} -> + {:error, error} + end + end + + defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do + adapter = check_adapter() + adapter.read_chunk(pid, stream, opts) + end + + @impl true + @spec close(map) :: :ok | no_return() + def close(%{pid: pid}) do + adapter = check_adapter() + adapter.close(pid) + end + + defp check_adapter do + adapter = Application.get_env(:tesla, :adapter) + + unless adapter == Tesla.Adapter.Gun do + raise "#{adapter} doesn't support reading body in chunks" + end + + adapter + end +end diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 2ed719315..9f5710c92 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -3,8 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy do - alias Pleroma.HTTP - @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++ ~w(if-unmodified-since if-none-match if-range range) @resp_cache_headers ~w(etag date last-modified cache-control) @@ -61,10 +59,10 @@ defmodule Pleroma.ReverseProxy do * `req_headers`, `resp_headers` additional headers. - * `http`: options for [hackney](https://github.com/benoitc/hackney). + * `http`: options for [gun](https://github.com/ninenines/gun). """ - @default_hackney_options [pool: :media] + @default_options [pool: :media] @inline_content_types [ "image/gif", @@ -97,11 +95,7 @@ defmodule Pleroma.ReverseProxy do def call(_conn, _url, _opts \\ []) def call(conn = %{method: method}, url, opts) when method in @methods do - hackney_opts = - Pleroma.HTTP.Connection.hackney_options([]) - |> Keyword.merge(@default_hackney_options) - |> Keyword.merge(Keyword.get(opts, :http, [])) - |> HTTP.process_request_options() + client_opts = Keyword.merge(@default_options, Keyword.get(opts, :http, [])) req_headers = build_req_headers(conn.req_headers, opts) @@ -113,7 +107,7 @@ def call(conn = %{method: method}, url, opts) when method in @methods do end with {:ok, nil} <- Cachex.get(:failed_proxy_url_cache, url), - {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts), + {:ok, code, headers, client} <- request(method, url, req_headers, client_opts), :ok <- header_length_constraint( headers, @@ -159,11 +153,11 @@ def call(conn, _, _) do |> halt() end - defp request(method, url, headers, hackney_opts) do + defp request(method, url, headers, opts) do Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}") method = method |> String.downcase() |> String.to_existing_atom() - case client().request(method, url, headers, "", hackney_opts) do + case client().request(method, url, headers, "", opts) do {:ok, code, headers, client} when code in @valid_resp_codes -> {:ok, code, downcase_headers(headers), client} @@ -213,7 +207,7 @@ defp chunk_reply(conn, client, opts, sent_so_far, duration) do duration, Keyword.get(opts, :max_read_duration, @max_read_duration) ), - {:ok, data} <- client().stream_body(client), + {:ok, data, client} <- client().stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), :ok <- diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index df774b0f7..ade87daf2 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -12,17 +12,23 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger - @hackney_options [ - pool: :media, - recv_timeout: 10_000 + @options [ + pool: :media ] def perform(:prefetch, url) do Logger.debug("Prefetching #{inspect(url)}") + opts = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do + Keyword.put(@options, :recv_timeout, 10_000) + else + @options + end + url |> MediaProxy.url() - |> HTTP.get([], adapter: @hackney_options) + |> HTTP.get([], adapter: opts) end def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex index 16b1a53d2..0ae926375 100644 --- a/lib/pleroma/web/rel_me.ex +++ b/lib/pleroma/web/rel_me.ex @@ -3,11 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RelMe do - @hackney_options [ + @options [ pool: :media, - recv_timeout: 2_000, - max_body: 2_000_000, - with_body: true + max_body: 2_000_000 ] if Pleroma.Config.get(:env) == :test do @@ -25,8 +23,18 @@ def parse(url) when is_binary(url) do def parse(_), do: {:error, "No URL provided"} defp parse_url(url) do + opts = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do + Keyword.merge(@options, + recv_timeout: 2_000, + with_body: true + ) + else + @options + end + with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <- - Pleroma.HTTP.get(url, [], adapter: @hackney_options), + Pleroma.HTTP.get(url, [], adapter: opts), data <- Floki.attribute(html, "link[rel~=me]", "href") ++ Floki.attribute(html, "a[rel~=me]", "href") do diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index c06b0a0f2..9deb03845 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -3,11 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Parser do - @hackney_options [ + @options [ pool: :media, - recv_timeout: 2_000, - max_body: 2_000_000, - with_body: true + max_body: 2_000_000 ] defp parsers do @@ -77,8 +75,18 @@ defp get_ttl_from_image(data, url) do end defp parse_url(url) do + opts = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do + Keyword.merge(@options, + recv_timeout: 2_000, + with_body: true + ) + else + @options + end + try do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) + {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: opts) html |> parse_html diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index b4cc80179..91e9e2271 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -205,7 +205,7 @@ def finger(account) do with response <- HTTP.get( address, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ), {:ok, %{status: status, body: body}} when status in 200..299 <- response do doc = XML.parse_document(body) diff --git a/mix.exs b/mix.exs index b28c65694..7c6de5423 100644 --- a/mix.exs +++ b/mix.exs @@ -120,6 +120,10 @@ defp deps do {:cachex, "~> 3.0.2"}, {:poison, "~> 3.0", override: true}, {:tesla, "~> 1.3", override: true}, + {:castore, "~> 0.1"}, + {:cowlib, "~> 2.8", override: true}, + {:gun, + github: "ninenines/gun", ref: "bd6425ab87428cf4c95f4d23e0a48fd065fbd714", override: true}, {:jason, "~> 1.0"}, {:mogrify, "~> 0.6.1"}, {:ex_aws, "~> 2.1"}, diff --git a/mix.lock b/mix.lock index 9c811a974..158a87e47 100644 --- a/mix.lock +++ b/mix.lock @@ -9,6 +9,7 @@ "cachex": {:hex, :cachex, "3.0.3", "4e2d3e05814a5738f5ff3903151d5c25636d72a3527251b753f501ad9c657967", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "3aadb1e605747122f60aa7b0b121cca23c14868558157563b3f3e19ea929f7d0"}, "calendar": {:hex, :calendar, "0.17.6", "ec291cb2e4ba499c2e8c0ef5f4ace974e2f9d02ae9e807e711a9b0c7850b9aee", [:mix], [{:tzdata, "~> 0.5.20 or ~> 0.1.201603 or ~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "738d0e17a93c2ccfe4ddc707bdc8e672e9074c8569498483feb1c4530fb91b2b"}, "captcha": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", "e0f16822d578866e186a0974d65ad58cddc1e2ab", [ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"]}, + "castore": {:hex, :castore, "0.1.5", "591c763a637af2cc468a72f006878584bc6c306f8d111ef8ba1d4c10e0684010", [:mix], [], "hexpm", "6db356b2bc6cc22561e051ff545c20ad064af57647e436650aa24d7d06cd941a"}, "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "805abd97539caf89ec6d4732c91e62ba9da0cda51ac462380bbd28ee697a8c42"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "comeonin": {:hex, :comeonin, "4.1.2", "3eb5620fd8e35508991664b4c2b04dd41e52f1620b36957be837c1d7784b7592", [:mix], [{:argon2_elixir, "~> 1.2", [hex: :argon2_elixir, repo: "hexpm", optional: true]}, {:bcrypt_elixir, "~> 0.12.1 or ~> 1.0", [hex: :bcrypt_elixir, repo: "hexpm", optional: true]}, {:pbkdf2_elixir, "~> 0.12", [hex: :pbkdf2_elixir, repo: "hexpm", optional: true]}], "hexpm", "d8700a0ca4dbb616c22c9b3f6dd539d88deaafec3efe66869d6370c9a559b3e9"}, @@ -45,6 +46,7 @@ "gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm"}, "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.4", "f13088e1ec10ce01665cf25f5ff779e7df3f2dc71b37084976cf89d1aa124d5c", [:mix], [], "hexpm", "3c75b5ea8288e2ee7ea503ff9e30dfe4d07ad3c054576a6e60040e79a801e14d"}, + "gun": {:git, "https://github.com/ninenines/gun.git", "bd6425ab87428cf4c95f4d23e0a48fd065fbd714", [ref: "bd6425ab87428cf4c95f4d23e0a48fd065fbd714"]}, "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e0100f8ef7d1124222c11ad362c857d3df7cb5f4204054f9f0f4a728666591fc"}, "html_entities": {:hex, :html_entities, "0.5.1", "1c9715058b42c35a2ab65edc5b36d0ea66dd083767bef6e3edb57870ef556549", [:mix], [], "hexpm", "30efab070904eb897ff05cd52fa61c1025d7f8ef3a9ca250bc4e6513d16c32de"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/activity/ir/topics_test.exs b/test/activity/ir/topics_test.exs index e75f83586..8729e5746 100644 --- a/test/activity/ir/topics_test.exs +++ b/test/activity/ir/topics_test.exs @@ -83,7 +83,7 @@ test "converts tags to hash tags", %{activity: %{object: %{data: data} = object} assert Enum.member?(topics, "hashtag:bar") end - test "only converts strinngs to hash tags", %{ + test "only converts strings to hash tags", %{ activity: %{object: %{data: data} = object} = activity } do tagged_data = Map.put(data, "tag", [2]) diff --git a/test/config/config_db_test.exs b/test/config/config_db_test.exs index 812709fd8..394040a59 100644 --- a/test/config/config_db_test.exs +++ b/test/config/config_db_test.exs @@ -478,14 +478,6 @@ test "simple keyword" do assert ConfigDB.from_binary(binary) == [key: "value"] end - test "keyword with partial_chain key" do - binary = - ConfigDB.transform([%{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}]) - - assert binary == :erlang.term_to_binary(partial_chain: &:hackney_connect.partial_chain/1) - assert ConfigDB.from_binary(binary) == [partial_chain: &:hackney_connect.partial_chain/1] - end - test "keyword" do binary = ConfigDB.transform([ diff --git a/test/fixtures/warnings/otp_version/21.1 b/test/fixtures/warnings/otp_version/21.1 new file mode 100644 index 000000000..90cd64c4f --- /dev/null +++ b/test/fixtures/warnings/otp_version/21.1 @@ -0,0 +1 @@ +21.1 \ No newline at end of file diff --git a/test/fixtures/warnings/otp_version/22.1 b/test/fixtures/warnings/otp_version/22.1 new file mode 100644 index 000000000..d9b314368 --- /dev/null +++ b/test/fixtures/warnings/otp_version/22.1 @@ -0,0 +1 @@ +22.1 \ No newline at end of file diff --git a/test/fixtures/warnings/otp_version/22.4 b/test/fixtures/warnings/otp_version/22.4 new file mode 100644 index 000000000..1da8ccd28 --- /dev/null +++ b/test/fixtures/warnings/otp_version/22.4 @@ -0,0 +1 @@ +22.4 \ No newline at end of file diff --git a/test/fixtures/warnings/otp_version/23.0 b/test/fixtures/warnings/otp_version/23.0 new file mode 100644 index 000000000..4266d8634 --- /dev/null +++ b/test/fixtures/warnings/otp_version/23.0 @@ -0,0 +1 @@ +23.0 \ No newline at end of file diff --git a/test/fixtures/warnings/otp_version/error b/test/fixtures/warnings/otp_version/error new file mode 100644 index 000000000..8fdd954df --- /dev/null +++ b/test/fixtures/warnings/otp_version/error @@ -0,0 +1 @@ +22 \ No newline at end of file diff --git a/test/fixtures/warnings/otp_version/undefined b/test/fixtures/warnings/otp_version/undefined new file mode 100644 index 000000000..66dc9051d --- /dev/null +++ b/test/fixtures/warnings/otp_version/undefined @@ -0,0 +1 @@ +undefined \ No newline at end of file diff --git a/test/gun/gun_test.exs b/test/gun/gun_test.exs new file mode 100644 index 000000000..7f185617c --- /dev/null +++ b/test/gun/gun_test.exs @@ -0,0 +1,33 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.GunTest do + use ExUnit.Case + alias Pleroma.Gun + + @moduletag :integration + + test "opens connection and receive response" do + {:ok, conn} = Gun.open('httpbin.org', 443) + assert is_pid(conn) + {:ok, _protocol} = Gun.await_up(conn) + ref = :gun.get(conn, '/get?a=b&c=d') + assert is_reference(ref) + + assert {:response, :nofin, 200, _} = Gun.await(conn, ref) + assert json = receive_response(conn, ref) + + assert %{"args" => %{"a" => "b", "c" => "d"}} = Jason.decode!(json) + end + + defp receive_response(conn, ref, acc \\ "") do + case Gun.await(conn, ref) do + {:data, :nofin, body} -> + receive_response(conn, ref, acc <> body) + + {:data, :fin, body} -> + acc <> body + end + end +end diff --git a/test/http/adapter/gun_test.exs b/test/http/adapter/gun_test.exs new file mode 100644 index 000000000..37489e1a4 --- /dev/null +++ b/test/http/adapter/gun_test.exs @@ -0,0 +1,266 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Adapter.GunTest do + use ExUnit.Case, async: true + use Pleroma.Tests.Helpers + import ExUnit.CaptureLog + alias Pleroma.Config + alias Pleroma.HTTP.Adapter.Gun + alias Pleroma.Pool.Connections + + setup_all do + {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.Gun.API.Mock) + :ok + end + + describe "options/1" do + clear_config([:http, :adapter]) do + Config.put([:http, :adapter], a: 1, b: 2) + end + + test "https url with default port" do + uri = URI.parse("https://example.com") + + opts = Gun.options(uri) + assert opts[:certificates_verification] + tls_opts = opts[:tls_opts] + assert tls_opts[:verify] == :verify_peer + assert tls_opts[:depth] == 20 + assert tls_opts[:reuse_sessions] == false + + assert tls_opts[:verify_fun] == + {&:ssl_verify_hostname.verify_fun/3, [check_hostname: 'example.com']} + + assert File.exists?(tls_opts[:cacertfile]) + + assert opts[:original] == "example.com:443" + end + + test "https ipv4 with default port" do + uri = URI.parse("https://127.0.0.1") + + opts = Gun.options(uri) + + assert opts[:tls_opts][:verify_fun] == + {&:ssl_verify_hostname.verify_fun/3, [check_hostname: '127.0.0.1']} + + assert opts[:original] == "127.0.0.1:443" + end + + test "https ipv6 with default port" do + uri = URI.parse("https://[2a03:2880:f10c:83:face:b00c:0:25de]") + + opts = Gun.options(uri) + + assert opts[:tls_opts][:verify_fun] == + {&:ssl_verify_hostname.verify_fun/3, + [check_hostname: '2a03:2880:f10c:83:face:b00c:0:25de']} + + assert opts[:original] == "2a03:2880:f10c:83:face:b00c:0:25de:443" + end + + test "https url with non standart port" do + uri = URI.parse("https://example.com:115") + + opts = Gun.options(uri) + + assert opts[:certificates_verification] + assert opts[:transport] == :tls + end + + test "receive conn by default" do + uri = URI.parse("http://another-domain.com") + :ok = Connections.open_conn(uri, :gun_connections) + + received_opts = Gun.options(uri) + assert received_opts[:close_conn] == false + assert is_pid(received_opts[:conn]) + end + + test "don't receive conn if receive_conn is false" do + uri = URI.parse("http://another-domain2.com") + :ok = Connections.open_conn(uri, :gun_connections) + + opts = [receive_conn: false] + received_opts = Gun.options(opts, uri) + assert received_opts[:close_conn] == nil + assert received_opts[:conn] == nil + end + + test "get conn on next request" do + level = Application.get_env(:logger, :level) + Logger.configure(level: :info) + on_exit(fn -> Logger.configure(level: level) end) + uri = URI.parse("http://some-domain2.com") + + assert capture_log(fn -> + opts = Gun.options(uri) + + assert opts[:conn] == nil + assert opts[:close_conn] == nil + end) =~ + "Gun connections pool checkin was not succesfull. Trying to open conn for next request." + + opts = Gun.options(uri) + + assert is_pid(opts[:conn]) + assert opts[:close_conn] == false + end + + test "merges with defaul http adapter config" do + defaults = Gun.options(URI.parse("https://example.com")) + assert Keyword.has_key?(defaults, :a) + assert Keyword.has_key?(defaults, :b) + end + + test "default ssl adapter opts with connection" do + uri = URI.parse("https://some-domain.com") + + :ok = Connections.open_conn(uri, :gun_connections) + + opts = Gun.options(uri) + + assert opts[:certificates_verification] + tls_opts = opts[:tls_opts] + assert tls_opts[:verify] == :verify_peer + assert tls_opts[:depth] == 20 + assert tls_opts[:reuse_sessions] == false + + assert opts[:original] == "some-domain.com:443" + assert opts[:close_conn] == false + assert is_pid(opts[:conn]) + end + + test "parses string proxy host & port" do + proxy = Config.get([:http, :proxy_url]) + Config.put([:http, :proxy_url], "localhost:8123") + on_exit(fn -> Config.put([:http, :proxy_url], proxy) end) + + uri = URI.parse("https://some-domain.com") + opts = Gun.options([receive_conn: false], uri) + assert opts[:proxy] == {'localhost', 8123} + end + + test "parses tuple proxy scheme host and port" do + proxy = Config.get([:http, :proxy_url]) + Config.put([:http, :proxy_url], {:socks, 'localhost', 1234}) + on_exit(fn -> Config.put([:http, :proxy_url], proxy) end) + + uri = URI.parse("https://some-domain.com") + opts = Gun.options([receive_conn: false], uri) + assert opts[:proxy] == {:socks, 'localhost', 1234} + end + + test "passed opts have more weight than defaults" do + proxy = Config.get([:http, :proxy_url]) + Config.put([:http, :proxy_url], {:socks5, 'localhost', 1234}) + on_exit(fn -> Config.put([:http, :proxy_url], proxy) end) + uri = URI.parse("https://some-domain.com") + opts = Gun.options([receive_conn: false, proxy: {'example.com', 4321}], uri) + + assert opts[:proxy] == {'example.com', 4321} + end + end + + describe "after_request/1" do + test "body_as not chunks" do + uri = URI.parse("http://some-domain.com") + :ok = Connections.open_conn(uri, :gun_connections) + opts = Gun.options(uri) + :ok = Gun.after_request(opts) + conn = opts[:conn] + + assert %Connections{ + conns: %{ + "http:some-domain.com:80" => %Pleroma.Gun.Conn{ + conn: ^conn, + conn_state: :idle, + used_by: [] + } + } + } = Connections.get_state(:gun_connections) + end + + test "body_as chunks" do + uri = URI.parse("http://some-domain.com") + :ok = Connections.open_conn(uri, :gun_connections) + opts = Gun.options([body_as: :chunks], uri) + :ok = Gun.after_request(opts) + conn = opts[:conn] + self = self() + + assert %Connections{ + conns: %{ + "http:some-domain.com:80" => %Pleroma.Gun.Conn{ + conn: ^conn, + conn_state: :active, + used_by: [{^self, _}] + } + } + } = Connections.get_state(:gun_connections) + end + + test "with no connection" do + uri = URI.parse("http://uniq-domain.com") + + :ok = Connections.open_conn(uri, :gun_connections) + + opts = Gun.options([body_as: :chunks], uri) + conn = opts[:conn] + opts = Keyword.delete(opts, :conn) + self = self() + + :ok = Gun.after_request(opts) + + assert %Connections{ + conns: %{ + "http:uniq-domain.com:80" => %Pleroma.Gun.Conn{ + conn: ^conn, + conn_state: :active, + used_by: [{^self, _}] + } + } + } = Connections.get_state(:gun_connections) + end + + test "with ipv4" do + uri = URI.parse("http://127.0.0.1") + :ok = Connections.open_conn(uri, :gun_connections) + opts = Gun.options(uri) + send(:gun_connections, {:gun_up, opts[:conn], :http}) + :ok = Gun.after_request(opts) + conn = opts[:conn] + + assert %Connections{ + conns: %{ + "http:127.0.0.1:80" => %Pleroma.Gun.Conn{ + conn: ^conn, + conn_state: :idle, + used_by: [] + } + } + } = Connections.get_state(:gun_connections) + end + + test "with ipv6" do + uri = URI.parse("http://[2a03:2880:f10c:83:face:b00c:0:25de]") + :ok = Connections.open_conn(uri, :gun_connections) + opts = Gun.options(uri) + send(:gun_connections, {:gun_up, opts[:conn], :http}) + :ok = Gun.after_request(opts) + conn = opts[:conn] + + assert %Connections{ + conns: %{ + "http:2a03:2880:f10c:83:face:b00c:0:25de:80" => %Pleroma.Gun.Conn{ + conn: ^conn, + conn_state: :idle, + used_by: [] + } + } + } = Connections.get_state(:gun_connections) + end + end +end diff --git a/test/http/adapter/hackney_test.exs b/test/http/adapter/hackney_test.exs new file mode 100644 index 000000000..35cb58125 --- /dev/null +++ b/test/http/adapter/hackney_test.exs @@ -0,0 +1,54 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Adapter.HackneyTest do + use ExUnit.Case + use Pleroma.Tests.Helpers + + alias Pleroma.Config + alias Pleroma.HTTP.Adapter.Hackney + + setup_all do + uri = URI.parse("http://domain.com") + {:ok, uri: uri} + end + + describe "options/2" do + clear_config([:http, :adapter]) do + Config.put([:http, :adapter], a: 1, b: 2) + end + + test "add proxy and opts from config", %{uri: uri} do + proxy = Config.get([:http, :proxy_url]) + Config.put([:http, :proxy_url], "localhost:8123") + on_exit(fn -> Config.put([:http, :proxy_url], proxy) end) + + opts = Hackney.options(uri) + + assert opts[:a] == 1 + assert opts[:b] == 2 + assert opts[:proxy] == "localhost:8123" + end + + test "respect connection opts and no proxy", %{uri: uri} do + opts = Hackney.options([a: 2, b: 1], uri) + + assert opts[:a] == 2 + assert opts[:b] == 1 + refute Keyword.has_key?(opts, :proxy) + end + + test "add opts for https" do + uri = URI.parse("https://domain.com") + + opts = Hackney.options(uri) + + assert opts[:ssl_options] == [ + partial_chain: &:hackney_connect.partial_chain/1, + versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"], + server_name_indication: 'domain.com' + ] + end + end +end diff --git a/test/http/adapter_test.exs b/test/http/adapter_test.exs new file mode 100644 index 000000000..37e47dabe --- /dev/null +++ b/test/http/adapter_test.exs @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.AdapterTest do + use ExUnit.Case, async: true + + alias Pleroma.HTTP.Adapter + + describe "domain_or_ip/1" do + test "with domain" do + assert Adapter.domain_or_ip("example.com") == {:domain, 'example.com'} + end + + test "with idna domain" do + assert Adapter.domain_or_ip("ですexample.com") == {:domain, 'xn--example-183fne.com'} + end + + test "with ipv4" do + assert Adapter.domain_or_ip("127.0.0.1") == {:ip, {127, 0, 0, 1}} + end + + test "with ipv6" do + assert Adapter.domain_or_ip("2a03:2880:f10c:83:face:b00c:0:25de") == + {:ip, {10_755, 10_368, 61_708, 131, 64_206, 45_068, 0, 9_694}} + end + end + + describe "domain_or_fallback/1" do + test "with domain" do + assert Adapter.domain_or_fallback("example.com") == 'example.com' + end + + test "with idna domain" do + assert Adapter.domain_or_fallback("ですexample.com") == 'xn--example-183fne.com' + end + + test "with ipv4" do + assert Adapter.domain_or_fallback("127.0.0.1") == '127.0.0.1' + end + + test "with ipv6" do + assert Adapter.domain_or_fallback("2a03:2880:f10c:83:face:b00c:0:25de") == + '2a03:2880:f10c:83:face:b00c:0:25de' + end + end + + describe "format_proxy/1" do + test "with nil" do + assert Adapter.format_proxy(nil) == nil + end + + test "with string" do + assert Adapter.format_proxy("127.0.0.1:8123") == {{127, 0, 0, 1}, 8123} + end + + test "localhost with port" do + assert Adapter.format_proxy("localhost:8123") == {'localhost', 8123} + end + + test "tuple" do + assert Adapter.format_proxy({:socks4, :localhost, 9050}) == {:socks4, 'localhost', 9050} + end + end +end diff --git a/test/http/connection_test.exs b/test/http/connection_test.exs new file mode 100644 index 000000000..c1ff0cc21 --- /dev/null +++ b/test/http/connection_test.exs @@ -0,0 +1,142 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.ConnectionTest do + use ExUnit.Case + use Pleroma.Tests.Helpers + import ExUnit.CaptureLog + alias Pleroma.Config + alias Pleroma.HTTP.Connection + + setup_all do + {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.Gun.API.Mock) + :ok + end + + describe "parse_host/1" do + test "as atom to charlist" do + assert Connection.parse_host(:localhost) == 'localhost' + end + + test "as string to charlist" do + assert Connection.parse_host("localhost.com") == 'localhost.com' + end + + test "as string ip to tuple" do + assert Connection.parse_host("127.0.0.1") == {127, 0, 0, 1} + end + end + + describe "parse_proxy/1" do + test "ip with port" do + assert Connection.parse_proxy("127.0.0.1:8123") == {:ok, {127, 0, 0, 1}, 8123} + end + + test "host with port" do + assert Connection.parse_proxy("localhost:8123") == {:ok, 'localhost', 8123} + end + + test "as tuple" do + assert Connection.parse_proxy({:socks4, :localhost, 9050}) == + {:ok, :socks4, 'localhost', 9050} + end + + test "as tuple with string host" do + assert Connection.parse_proxy({:socks5, "localhost", 9050}) == + {:ok, :socks5, 'localhost', 9050} + end + end + + describe "parse_proxy/1 errors" do + test "ip without port" do + capture_log(fn -> + assert Connection.parse_proxy("127.0.0.1") == {:error, :error_parsing_proxy} + end) =~ "parsing proxy fail \"127.0.0.1\"" + end + + test "host without port" do + capture_log(fn -> + assert Connection.parse_proxy("localhost") == {:error, :error_parsing_proxy} + end) =~ "parsing proxy fail \"localhost\"" + end + + test "host with bad port" do + capture_log(fn -> + assert Connection.parse_proxy("localhost:port") == {:error, :error_parsing_port_in_proxy} + end) =~ "parsing port in proxy fail \"localhost:port\"" + end + + test "ip with bad port" do + capture_log(fn -> + assert Connection.parse_proxy("127.0.0.1:15.9") == {:error, :error_parsing_port_in_proxy} + end) =~ "parsing port in proxy fail \"127.0.0.1:15.9\"" + end + + test "as tuple without port" do + capture_log(fn -> + assert Connection.parse_proxy({:socks5, :localhost}) == {:error, :error_parsing_proxy} + end) =~ "parsing proxy fail {:socks5, :localhost}" + end + + test "with nil" do + assert Connection.parse_proxy(nil) == nil + end + end + + describe "options/3" do + clear_config([:http, :proxy_url]) + + test "without proxy_url in config" do + Config.delete([:http, :proxy_url]) + + opts = Connection.options(%URI{}) + refute Keyword.has_key?(opts, :proxy) + end + + test "parses string proxy host & port" do + Config.put([:http, :proxy_url], "localhost:8123") + + opts = Connection.options(%URI{}) + assert opts[:proxy] == {'localhost', 8123} + end + + test "parses tuple proxy scheme host and port" do + Config.put([:http, :proxy_url], {:socks, 'localhost', 1234}) + + opts = Connection.options(%URI{}) + assert opts[:proxy] == {:socks, 'localhost', 1234} + end + + test "passed opts have more weight than defaults" do + Config.put([:http, :proxy_url], {:socks5, 'localhost', 1234}) + + opts = Connection.options(%URI{}, proxy: {'example.com', 4321}) + + assert opts[:proxy] == {'example.com', 4321} + end + + test "default ssl adapter opts with connection" do + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) + + uri = URI.parse("https://some-domain.com") + + pid = Process.whereis(:federation) + :ok = Pleroma.Pool.Connections.open_conn(uri, :gun_connections, genserver_pid: pid) + + opts = Connection.options(uri) + + assert opts[:certificates_verification] + tls_opts = opts[:tls_opts] + assert tls_opts[:verify] == :verify_peer + assert tls_opts[:depth] == 20 + assert tls_opts[:reuse_sessions] == false + + assert opts[:original] == "some-domain.com:443" + assert opts[:close_conn] == false + assert is_pid(opts[:conn]) + end + end +end diff --git a/test/http/request_builder_test.exs b/test/http/request_builder_test.exs index 80ef25d7b..27ca651be 100644 --- a/test/http/request_builder_test.exs +++ b/test/http/request_builder_test.exs @@ -5,30 +5,32 @@ defmodule Pleroma.HTTP.RequestBuilderTest do use ExUnit.Case, async: true use Pleroma.Tests.Helpers + alias Pleroma.Config + alias Pleroma.HTTP.Request alias Pleroma.HTTP.RequestBuilder describe "headers/2" do clear_config([:http, :send_user_agent]) test "don't send pleroma user agent" do - assert RequestBuilder.headers(%{}, []) == %{headers: []} + assert RequestBuilder.headers(%Request{}, []) == %Request{headers: []} end test "send pleroma user agent" do - Pleroma.Config.put([:http, :send_user_agent], true) - Pleroma.Config.put([:http, :user_agent], :default) + Config.put([:http, :send_user_agent], true) + Config.put([:http, :user_agent], :default) - assert RequestBuilder.headers(%{}, []) == %{ - headers: [{"User-Agent", Pleroma.Application.user_agent()}] + assert RequestBuilder.headers(%Request{}, []) == %Request{ + headers: [{"user-agent", Pleroma.Application.user_agent()}] } end test "send custom user agent" do - Pleroma.Config.put([:http, :send_user_agent], true) - Pleroma.Config.put([:http, :user_agent], "totally-not-pleroma") + Config.put([:http, :send_user_agent], true) + Config.put([:http, :user_agent], "totally-not-pleroma") - assert RequestBuilder.headers(%{}, []) == %{ - headers: [{"User-Agent", "totally-not-pleroma"}] + assert RequestBuilder.headers(%Request{}, []) == %Request{ + headers: [{"user-agent", "totally-not-pleroma"}] } end end @@ -40,19 +42,19 @@ test "don't add if keyword is empty" do test "add query parameter" do assert RequestBuilder.add_optional_params( - %{}, + %Request{}, %{query: :query, body: :body, another: :val}, [ {:query, "param1=val1¶m2=val2"}, {:body, "some body"} ] - ) == %{query: "param1=val1¶m2=val2", body: "some body"} + ) == %Request{query: "param1=val1¶m2=val2", body: "some body"} end end describe "add_param/4" do test "add file parameter" do - %{ + %Request{ body: %Tesla.Multipart{ boundary: _, content_type_params: [], @@ -69,7 +71,7 @@ test "add file parameter" do } ] } - } = RequestBuilder.add_param(%{}, :file, "filename.png", "some-path/filename.png") + } = RequestBuilder.add_param(%Request{}, :file, "filename.png", "some-path/filename.png") end test "add key to body" do @@ -81,7 +83,7 @@ test "add key to body" do %Tesla.Multipart.Part{ body: "\"someval\"", dispositions: [name: "somekey"], - headers: ["Content-Type": "application/json"] + headers: [{"content-type", "application/json"}] } ] } diff --git a/test/http_test.exs b/test/http_test.exs index 5f9522cf0..d80b96496 100644 --- a/test/http_test.exs +++ b/test/http_test.exs @@ -3,8 +3,10 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTPTest do - use Pleroma.DataCase + use ExUnit.Case + use Pleroma.Tests.Helpers import Tesla.Mock + alias Pleroma.HTTP setup do mock(fn @@ -27,7 +29,7 @@ defmodule Pleroma.HTTPTest do describe "get/1" do test "returns successfully result" do - assert Pleroma.HTTP.get("http://example.com/hello") == { + assert HTTP.get("http://example.com/hello") == { :ok, %Tesla.Env{status: 200, body: "hello"} } @@ -36,7 +38,7 @@ test "returns successfully result" do describe "get/2 (with headers)" do test "returns successfully result for json content-type" do - assert Pleroma.HTTP.get("http://example.com/hello", [{"content-type", "application/json"}]) == + assert HTTP.get("http://example.com/hello", [{"content-type", "application/json"}]) == { :ok, %Tesla.Env{ @@ -50,10 +52,35 @@ test "returns successfully result for json content-type" do describe "post/2" do test "returns successfully result" do - assert Pleroma.HTTP.post("http://example.com/world", "") == { + assert HTTP.post("http://example.com/world", "") == { :ok, %Tesla.Env{status: 200, body: "world"} } end end + + describe "connection pools" do + @describetag :integration + clear_config([Pleroma.Gun.API]) do + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun) + end + + test "gun" do + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + + on_exit(fn -> + Application.put_env(:tesla, :adapter, adapter) + end) + + options = [adapter: [pool: :federation]] + + assert {:ok, resp} = HTTP.get("https://httpbin.org/user-agent", [], options) + + assert resp.status == 200 + + state = Pleroma.Pool.Connections.get_state(:gun_connections) + assert state.conns["https:httpbin.org:443"] + end + end end diff --git a/test/notification_test.exs b/test/notification_test.exs index 04bf5b41a..1de3c6e3b 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -649,6 +649,13 @@ test "notifications are deleted if a remote user is deleted" do "object" => remote_user.ap_id } + remote_user_url = remote_user.ap_id + + Tesla.Mock.mock(fn + %{method: :get, url: ^remote_user_url} -> + %Tesla.Env{status: 404, body: ""} + end) + {:ok, _delete_activity} = Transmogrifier.handle_incoming(delete_user_message) ObanHelpers.perform_all() diff --git a/test/otp_version_test.exs b/test/otp_version_test.exs new file mode 100644 index 000000000..f26b90f61 --- /dev/null +++ b/test/otp_version_test.exs @@ -0,0 +1,58 @@ +defmodule Pleroma.OTPVersionTest do + use ExUnit.Case, async: true + + alias Pleroma.OTPVersion + + describe "get_and_check_version/2" do + test "22.4" do + assert OTPVersion.get_and_check_version(Tesla.Adapter.Gun, [ + "test/fixtures/warnings/otp_version/22.4" + ]) == :ok + end + + test "22.1" do + assert OTPVersion.get_and_check_version(Tesla.Adapter.Gun, [ + "test/fixtures/warnings/otp_version/22.1" + ]) == {:error, "22.1"} + end + + test "21.1" do + assert OTPVersion.get_and_check_version(Tesla.Adapter.Gun, [ + "test/fixtures/warnings/otp_version/21.1" + ]) == {:error, "21.1"} + end + + test "23.0" do + assert OTPVersion.get_and_check_version(Tesla.Adapter.Gun, [ + "test/fixtures/warnings/otp_version/23.0" + ]) == :ok + end + + test "undefined" do + assert OTPVersion.get_and_check_version(Tesla.Adapter.Gun, [ + "test/fixtures/warnings/otp_version/undefined" + ]) == :undefined + end + + test "not parsable" do + assert OTPVersion.get_and_check_version(Tesla.Adapter.Gun, [ + "test/fixtures/warnings/otp_version/error" + ]) == :undefined + end + + test "with non existance file" do + assert OTPVersion.get_and_check_version(Tesla.Adapter.Gun, [ + "test/fixtures/warnings/otp_version/non-exising", + "test/fixtures/warnings/otp_version/22.4" + ]) == :ok + end + + test "empty paths" do + assert OTPVersion.get_and_check_version(Tesla.Adapter.Gun, []) == :undefined + end + + test "another adapter" do + assert OTPVersion.get_and_check_version(Tesla.Adapter.Hackney, []) == :ok + end + end +end diff --git a/test/pool/connections_test.exs b/test/pool/connections_test.exs new file mode 100644 index 000000000..6f0e041ae --- /dev/null +++ b/test/pool/connections_test.exs @@ -0,0 +1,959 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool.ConnectionsTest do + use ExUnit.Case + use Pleroma.Tests.Helpers + import ExUnit.CaptureLog + alias Pleroma.Gun.API + alias Pleroma.Gun.Conn + alias Pleroma.Pool.Connections + + setup_all do + {:ok, _} = Registry.start_link(keys: :unique, name: API.Mock) + :ok + end + + setup do + name = :test_connections + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) + + {:ok, _pid} = + Connections.start_link({name, [max_connections: 2, receive_connection_timeout: 1_500]}) + + {:ok, name: name} + end + + describe "alive?/2" do + test "is alive", %{name: name} do + assert Connections.alive?(name) + end + + test "returns false if not started" do + refute Connections.alive?(:some_random_name) + end + end + + test "opens connection and reuse it on next request", %{name: name} do + url = "http://some-domain.com" + key = "http:some-domain.com:80" + refute Connections.checkin(url, name) + :ok = Connections.open_conn(url, name) + + conn = Connections.checkin(url, name) + assert is_pid(conn) + assert Process.alive?(conn) + + self = self() + + %Connections{ + conns: %{ + ^key => %Conn{ + conn: ^conn, + gun_state: :up, + used_by: [{^self, _}], + conn_state: :active + } + } + } = Connections.get_state(name) + + reused_conn = Connections.checkin(url, name) + + assert conn == reused_conn + + %Connections{ + conns: %{ + ^key => %Conn{ + conn: ^conn, + gun_state: :up, + used_by: [{^self, _}, {^self, _}], + conn_state: :active + } + } + } = Connections.get_state(name) + + :ok = Connections.checkout(conn, self, name) + + %Connections{ + conns: %{ + ^key => %Conn{ + conn: ^conn, + gun_state: :up, + used_by: [{^self, _}], + conn_state: :active + } + } + } = Connections.get_state(name) + + :ok = Connections.checkout(conn, self, name) + + %Connections{ + conns: %{ + ^key => %Conn{ + conn: ^conn, + gun_state: :up, + used_by: [], + conn_state: :idle + } + } + } = Connections.get_state(name) + end + + test "reuse connection for idna domains", %{name: name} do + url = "http://ですsome-domain.com" + refute Connections.checkin(url, name) + + :ok = Connections.open_conn(url, name) + + conn = Connections.checkin(url, name) + assert is_pid(conn) + assert Process.alive?(conn) + + self = self() + + %Connections{ + conns: %{ + "http:ですsome-domain.com:80" => %Conn{ + conn: ^conn, + gun_state: :up, + used_by: [{^self, _}], + conn_state: :active + } + } + } = Connections.get_state(name) + + reused_conn = Connections.checkin(url, name) + + assert conn == reused_conn + end + + test "reuse for ipv4", %{name: name} do + url = "http://127.0.0.1" + + refute Connections.checkin(url, name) + + :ok = Connections.open_conn(url, name) + + conn = Connections.checkin(url, name) + assert is_pid(conn) + assert Process.alive?(conn) + + self = self() + + %Connections{ + conns: %{ + "http:127.0.0.1:80" => %Conn{ + conn: ^conn, + gun_state: :up, + used_by: [{^self, _}], + conn_state: :active + } + } + } = Connections.get_state(name) + + reused_conn = Connections.checkin(url, name) + + assert conn == reused_conn + + :ok = Connections.checkout(conn, self, name) + :ok = Connections.checkout(reused_conn, self, name) + + %Connections{ + conns: %{ + "http:127.0.0.1:80" => %Conn{ + conn: ^conn, + gun_state: :up, + used_by: [], + conn_state: :idle + } + } + } = Connections.get_state(name) + end + + test "reuse for ipv6", %{name: name} do + url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]" + + refute Connections.checkin(url, name) + + :ok = Connections.open_conn(url, name) + + conn = Connections.checkin(url, name) + assert is_pid(conn) + assert Process.alive?(conn) + + self = self() + + %Connections{ + conns: %{ + "http:2a03:2880:f10c:83:face:b00c:0:25de:80" => %Conn{ + conn: ^conn, + gun_state: :up, + used_by: [{^self, _}], + conn_state: :active + } + } + } = Connections.get_state(name) + + reused_conn = Connections.checkin(url, name) + + assert conn == reused_conn + end + + test "up and down ipv4", %{name: name} do + self = self() + url = "http://127.0.0.1" + :ok = Connections.open_conn(url, name) + conn = Connections.checkin(url, name) + send(name, {:gun_down, conn, nil, nil, nil}) + send(name, {:gun_up, conn, nil}) + + %Connections{ + conns: %{ + "http:127.0.0.1:80" => %Conn{ + conn: ^conn, + gun_state: :up, + used_by: [{^self, _}], + conn_state: :active + } + } + } = Connections.get_state(name) + end + + test "up and down ipv6", %{name: name} do + self = self() + url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]" + :ok = Connections.open_conn(url, name) + conn = Connections.checkin(url, name) + send(name, {:gun_down, conn, nil, nil, nil}) + send(name, {:gun_up, conn, nil}) + + %Connections{ + conns: %{ + "http:2a03:2880:f10c:83:face:b00c:0:25de:80" => %Conn{ + conn: ^conn, + gun_state: :up, + used_by: [{^self, _}], + conn_state: :active + } + } + } = Connections.get_state(name) + end + + test "reuses connection based on protocol", %{name: name} do + http_url = "http://some-domain.com" + http_key = "http:some-domain.com:80" + https_url = "https://some-domain.com" + https_key = "https:some-domain.com:443" + + refute Connections.checkin(http_url, name) + :ok = Connections.open_conn(http_url, name) + conn = Connections.checkin(http_url, name) + assert is_pid(conn) + assert Process.alive?(conn) + + refute Connections.checkin(https_url, name) + :ok = Connections.open_conn(https_url, name) + https_conn = Connections.checkin(https_url, name) + + refute conn == https_conn + + reused_https = Connections.checkin(https_url, name) + + refute conn == reused_https + + assert reused_https == https_conn + + %Connections{ + conns: %{ + ^http_key => %Conn{ + conn: ^conn, + gun_state: :up + }, + ^https_key => %Conn{ + conn: ^https_conn, + gun_state: :up + } + } + } = Connections.get_state(name) + end + + test "connection can't get up", %{name: name} do + url = "http://gun-not-up.com" + + assert capture_log(fn -> + :ok = Connections.open_conn(url, name) + refute Connections.checkin(url, name) + end) =~ + "Received error on opening connection http://gun-not-up.com: {:error, :timeout}" + end + + test "process gun_down message and then gun_up", %{name: name} do + self = self() + url = "http://gun-down-and-up.com" + key = "http:gun-down-and-up.com:80" + :ok = Connections.open_conn(url, name) + conn = Connections.checkin(url, name) + + assert is_pid(conn) + assert Process.alive?(conn) + + %Connections{ + conns: %{ + ^key => %Conn{ + conn: ^conn, + gun_state: :up, + used_by: [{^self, _}] + } + } + } = Connections.get_state(name) + + send(name, {:gun_down, conn, :http, nil, nil}) + + %Connections{ + conns: %{ + ^key => %Conn{ + conn: ^conn, + gun_state: :down, + used_by: [{^self, _}] + } + } + } = Connections.get_state(name) + + send(name, {:gun_up, conn, :http}) + + conn2 = Connections.checkin(url, name) + assert conn == conn2 + + assert is_pid(conn2) + assert Process.alive?(conn2) + + %Connections{ + conns: %{ + ^key => %Conn{ + conn: _, + gun_state: :up, + used_by: [{^self, _}, {^self, _}] + } + } + } = Connections.get_state(name) + end + + test "async processes get same conn for same domain", %{name: name} do + url = "http://some-domain.com" + :ok = Connections.open_conn(url, name) + + tasks = + for _ <- 1..5 do + Task.async(fn -> + Connections.checkin(url, name) + end) + end + + tasks_with_results = Task.yield_many(tasks) + + results = + Enum.map(tasks_with_results, fn {task, res} -> + res || Task.shutdown(task, :brutal_kill) + end) + + conns = for {:ok, value} <- results, do: value + + %Connections{ + conns: %{ + "http:some-domain.com:80" => %Conn{ + conn: conn, + gun_state: :up + } + } + } = Connections.get_state(name) + + assert Enum.all?(conns, fn res -> res == conn end) + end + + test "remove frequently used and idle", %{name: name} do + self = self() + http_url = "http://some-domain.com" + https_url = "https://some-domain.com" + :ok = Connections.open_conn(https_url, name) + :ok = Connections.open_conn(http_url, name) + + conn1 = Connections.checkin(https_url, name) + + [conn2 | _conns] = + for _ <- 1..4 do + Connections.checkin(http_url, name) + end + + http_key = "http:some-domain.com:80" + + %Connections{ + conns: %{ + ^http_key => %Conn{ + conn: ^conn2, + gun_state: :up, + conn_state: :active, + used_by: [{^self, _}, {^self, _}, {^self, _}, {^self, _}] + }, + "https:some-domain.com:443" => %Conn{ + conn: ^conn1, + gun_state: :up, + conn_state: :active, + used_by: [{^self, _}] + } + } + } = Connections.get_state(name) + + :ok = Connections.checkout(conn1, self, name) + + another_url = "http://another-domain.com" + :ok = Connections.open_conn(another_url, name) + conn = Connections.checkin(another_url, name) + + %Connections{ + conns: %{ + "http:another-domain.com:80" => %Conn{ + conn: ^conn, + gun_state: :up + }, + ^http_key => %Conn{ + conn: _, + gun_state: :up + } + } + } = Connections.get_state(name) + end + + describe "integration test" do + @describetag :integration + + clear_config([API]) do + Pleroma.Config.put([API], Pleroma.Gun) + end + + test "opens connection and reuse it on next request", %{name: name} do + url = "http://httpbin.org" + :ok = Connections.open_conn(url, name) + Process.sleep(250) + conn = Connections.checkin(url, name) + + assert is_pid(conn) + assert Process.alive?(conn) + + reused_conn = Connections.checkin(url, name) + + assert conn == reused_conn + + %Connections{ + conns: %{ + "http:httpbin.org:80" => %Conn{ + conn: ^conn, + gun_state: :up + } + } + } = Connections.get_state(name) + end + + test "opens ssl connection and reuse it on next request", %{name: name} do + url = "https://httpbin.org" + :ok = Connections.open_conn(url, name) + Process.sleep(1_000) + conn = Connections.checkin(url, name) + + assert is_pid(conn) + assert Process.alive?(conn) + + reused_conn = Connections.checkin(url, name) + + assert conn == reused_conn + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: ^conn, + gun_state: :up + } + } + } = Connections.get_state(name) + end + + test "remove frequently used and idle", %{name: name} do + self = self() + https1 = "https://www.google.com" + https2 = "https://httpbin.org" + + :ok = Connections.open_conn(https1, name) + :ok = Connections.open_conn(https2, name) + Process.sleep(1_500) + conn = Connections.checkin(https1, name) + + for _ <- 1..4 do + Connections.checkin(https2, name) + end + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: _, + gun_state: :up + }, + "https:www.google.com:443" => %Conn{ + conn: _, + gun_state: :up + } + } + } = Connections.get_state(name) + + :ok = Connections.checkout(conn, self, name) + http = "http://httpbin.org" + Process.sleep(1_000) + :ok = Connections.open_conn(http, name) + conn = Connections.checkin(http, name) + + %Connections{ + conns: %{ + "http:httpbin.org:80" => %Conn{ + conn: ^conn, + gun_state: :up + }, + "https:httpbin.org:443" => %Conn{ + conn: _, + gun_state: :up + } + } + } = Connections.get_state(name) + end + + test "remove earlier used and idle", %{name: name} do + self = self() + + https1 = "https://www.google.com" + https2 = "https://httpbin.org" + :ok = Connections.open_conn(https1, name) + :ok = Connections.open_conn(https2, name) + Process.sleep(1_500) + + Connections.checkin(https1, name) + conn = Connections.checkin(https1, name) + + Process.sleep(1_000) + Connections.checkin(https2, name) + Connections.checkin(https2, name) + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: _, + gun_state: :up + }, + "https:www.google.com:443" => %Conn{ + conn: ^conn, + gun_state: :up + } + } + } = Connections.get_state(name) + + :ok = Connections.checkout(conn, self, name) + :ok = Connections.checkout(conn, self, name) + + http = "http://httpbin.org" + :ok = Connections.open_conn(http, name) + Process.sleep(1_000) + + conn = Connections.checkin(http, name) + + %Connections{ + conns: %{ + "http:httpbin.org:80" => %Conn{ + conn: ^conn, + gun_state: :up + }, + "https:httpbin.org:443" => %Conn{ + conn: _, + gun_state: :up + } + } + } = Connections.get_state(name) + end + + test "doesn't open new conn on pool overflow", %{name: name} do + self = self() + + https1 = "https://www.google.com" + https2 = "https://httpbin.org" + :ok = Connections.open_conn(https1, name) + :ok = Connections.open_conn(https2, name) + Process.sleep(1_000) + Connections.checkin(https1, name) + conn1 = Connections.checkin(https1, name) + conn2 = Connections.checkin(https2, name) + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: ^conn2, + gun_state: :up, + conn_state: :active, + used_by: [{^self, _}] + }, + "https:www.google.com:443" => %Conn{ + conn: ^conn1, + gun_state: :up, + conn_state: :active, + used_by: [{^self, _}, {^self, _}] + } + } + } = Connections.get_state(name) + + refute Connections.checkin("http://httpbin.org", name) + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: ^conn2, + gun_state: :up, + conn_state: :active, + used_by: [{^self, _}] + }, + "https:www.google.com:443" => %Conn{ + conn: ^conn1, + gun_state: :up, + conn_state: :active, + used_by: [{^self, _}, {^self, _}] + } + } + } = Connections.get_state(name) + end + + test "get idle connection with the smallest crf", %{ + name: name + } do + self = self() + + https1 = "https://www.google.com" + https2 = "https://httpbin.org" + + :ok = Connections.open_conn(https1, name) + :ok = Connections.open_conn(https2, name) + Process.sleep(1_500) + Connections.checkin(https1, name) + Connections.checkin(https2, name) + Connections.checkin(https1, name) + conn1 = Connections.checkin(https1, name) + conn2 = Connections.checkin(https2, name) + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: ^conn2, + gun_state: :up, + conn_state: :active, + used_by: [{^self, _}, {^self, _}], + crf: crf2 + }, + "https:www.google.com:443" => %Conn{ + conn: ^conn1, + gun_state: :up, + conn_state: :active, + used_by: [{^self, _}, {^self, _}, {^self, _}], + crf: crf1 + } + } + } = Connections.get_state(name) + + assert crf1 > crf2 + + :ok = Connections.checkout(conn1, self, name) + :ok = Connections.checkout(conn1, self, name) + :ok = Connections.checkout(conn1, self, name) + + :ok = Connections.checkout(conn2, self, name) + :ok = Connections.checkout(conn2, self, name) + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: ^conn2, + gun_state: :up, + conn_state: :idle, + used_by: [] + }, + "https:www.google.com:443" => %Conn{ + conn: ^conn1, + gun_state: :up, + conn_state: :idle, + used_by: [] + } + } + } = Connections.get_state(name) + + http = "http://httpbin.org" + :ok = Connections.open_conn(http, name) + Process.sleep(1_000) + conn = Connections.checkin(http, name) + + %Connections{ + conns: %{ + "https:www.google.com:443" => %Conn{ + conn: ^conn1, + gun_state: :up, + conn_state: :idle, + used_by: [], + crf: crf1 + }, + "http:httpbin.org:80" => %Conn{ + conn: ^conn, + gun_state: :up, + conn_state: :active, + used_by: [{^self, _}], + crf: crf + } + } + } = Connections.get_state(name) + + assert crf1 > crf + end + end + + describe "with proxy" do + test "as ip", %{name: name} do + url = "http://proxy-string.com" + key = "http:proxy-string.com:80" + :ok = Connections.open_conn(url, name, proxy: {{127, 0, 0, 1}, 8123}) + + conn = Connections.checkin(url, name) + + %Connections{ + conns: %{ + ^key => %Conn{ + conn: ^conn, + gun_state: :up + } + } + } = Connections.get_state(name) + + reused_conn = Connections.checkin(url, name) + + assert reused_conn == conn + end + + test "as host", %{name: name} do + url = "http://proxy-tuple-atom.com" + :ok = Connections.open_conn(url, name, proxy: {'localhost', 9050}) + conn = Connections.checkin(url, name) + + %Connections{ + conns: %{ + "http:proxy-tuple-atom.com:80" => %Conn{ + conn: ^conn, + gun_state: :up + } + } + } = Connections.get_state(name) + + reused_conn = Connections.checkin(url, name) + + assert reused_conn == conn + end + + test "as ip and ssl", %{name: name} do + url = "https://proxy-string.com" + + :ok = Connections.open_conn(url, name, proxy: {{127, 0, 0, 1}, 8123}) + conn = Connections.checkin(url, name) + + %Connections{ + conns: %{ + "https:proxy-string.com:443" => %Conn{ + conn: ^conn, + gun_state: :up + } + } + } = Connections.get_state(name) + + reused_conn = Connections.checkin(url, name) + + assert reused_conn == conn + end + + test "as host and ssl", %{name: name} do + url = "https://proxy-tuple-atom.com" + :ok = Connections.open_conn(url, name, proxy: {'localhost', 9050}) + conn = Connections.checkin(url, name) + + %Connections{ + conns: %{ + "https:proxy-tuple-atom.com:443" => %Conn{ + conn: ^conn, + gun_state: :up + } + } + } = Connections.get_state(name) + + reused_conn = Connections.checkin(url, name) + + assert reused_conn == conn + end + + test "with socks type", %{name: name} do + url = "http://proxy-socks.com" + + :ok = Connections.open_conn(url, name, proxy: {:socks5, 'localhost', 1234}) + + conn = Connections.checkin(url, name) + + %Connections{ + conns: %{ + "http:proxy-socks.com:80" => %Conn{ + conn: ^conn, + gun_state: :up + } + } + } = Connections.get_state(name) + + reused_conn = Connections.checkin(url, name) + + assert reused_conn == conn + end + + test "with socks4 type and ssl", %{name: name} do + url = "https://proxy-socks.com" + + :ok = Connections.open_conn(url, name, proxy: {:socks4, 'localhost', 1234}) + + conn = Connections.checkin(url, name) + + %Connections{ + conns: %{ + "https:proxy-socks.com:443" => %Conn{ + conn: ^conn, + gun_state: :up + } + } + } = Connections.get_state(name) + + reused_conn = Connections.checkin(url, name) + + assert reused_conn == conn + end + end + + describe "crf/3" do + setup do + crf = Connections.crf(1, 10, 1) + {:ok, crf: crf} + end + + test "more used will have crf higher", %{crf: crf} do + # used 3 times + crf1 = Connections.crf(1, 10, crf) + crf1 = Connections.crf(1, 10, crf1) + + # used 2 times + crf2 = Connections.crf(1, 10, crf) + + assert crf1 > crf2 + end + + test "recently used will have crf higher on equal references", %{crf: crf} do + # used 3 sec ago + crf1 = Connections.crf(3, 10, crf) + + # used 4 sec ago + crf2 = Connections.crf(4, 10, crf) + + assert crf1 > crf2 + end + + test "equal crf on equal reference and time", %{crf: crf} do + # used 2 times + crf1 = Connections.crf(1, 10, crf) + + # used 2 times + crf2 = Connections.crf(1, 10, crf) + + assert crf1 == crf2 + end + + test "recently used will have higher crf", %{crf: crf} do + crf1 = Connections.crf(2, 10, crf) + crf1 = Connections.crf(1, 10, crf1) + + crf2 = Connections.crf(3, 10, crf) + crf2 = Connections.crf(4, 10, crf2) + assert crf1 > crf2 + end + end + + describe "get_unused_conns/1" do + test "crf is equalent, sorting by reference" do + conns = %{ + "1" => %Conn{ + conn_state: :idle, + last_reference: now() - 1 + }, + "2" => %Conn{ + conn_state: :idle, + last_reference: now() + } + } + + assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(conns) + end + + test "reference is equalent, sorting by crf" do + conns = %{ + "1" => %Conn{ + conn_state: :idle, + crf: 1.999 + }, + "2" => %Conn{ + conn_state: :idle, + crf: 2 + } + } + + assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(conns) + end + + test "higher crf and lower reference" do + conns = %{ + "1" => %Conn{ + conn_state: :idle, + crf: 3, + last_reference: now() - 1 + }, + "2" => %Conn{ + conn_state: :idle, + crf: 2, + last_reference: now() + } + } + + assert [{"2", _unused_conn} | _others] = Connections.get_unused_conns(conns) + end + + test "lower crf and lower reference" do + conns = %{ + "1" => %Conn{ + conn_state: :idle, + crf: 1.99, + last_reference: now() - 1 + }, + "2" => %Conn{ + conn_state: :idle, + crf: 2, + last_reference: now() + } + } + + assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(conns) + end + end + + defp now do + :os.system_time(:second) + end +end diff --git a/test/reverse_proxy/client/tesla_test.exs b/test/reverse_proxy/client/tesla_test.exs new file mode 100644 index 000000000..75a70988c --- /dev/null +++ b/test/reverse_proxy/client/tesla_test.exs @@ -0,0 +1,93 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ReverseProxy.Client.TeslaTest do + use ExUnit.Case + use Pleroma.Tests.Helpers + alias Pleroma.ReverseProxy.Client + @moduletag :integration + + clear_config_all([Pleroma.Gun.API]) do + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun) + end + + setup do + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + + on_exit(fn -> + Application.put_env(:tesla, :adapter, Tesla.Mock) + end) + end + + test "get response body stream" do + {:ok, status, headers, ref} = + Client.Tesla.request( + :get, + "http://httpbin.org/stream-bytes/10", + [{"accept", "application/octet-stream"}], + "", + [] + ) + + assert status == 200 + assert headers != [] + + {:ok, response, ref} = Client.Tesla.stream_body(ref) + check_ref(ref) + assert is_binary(response) + assert byte_size(response) == 10 + + assert :done == Client.Tesla.stream_body(ref) + assert :ok = Client.Tesla.close(ref) + end + + test "head response" do + {:ok, status, headers} = Client.Tesla.request(:head, "https://httpbin.org/get", [], "") + + assert status == 200 + assert headers != [] + end + + test "get error response" do + {:ok, status, headers, _body} = + Client.Tesla.request( + :get, + "https://httpbin.org/status/500", + [], + "" + ) + + assert status == 500 + assert headers != [] + end + + describe "client error" do + setup do + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Hackney) + + on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) + :ok + end + + test "adapter doesn't support reading body in chunks" do + assert_raise RuntimeError, + "Elixir.Tesla.Adapter.Hackney doesn't support reading body in chunks", + fn -> + Client.Tesla.request( + :get, + "http://httpbin.org/stream-bytes/10", + [{"accept", "application/octet-stream"}], + "" + ) + end + end + end + + defp check_ref(%{pid: pid, stream: stream} = ref) do + assert is_pid(pid) + assert is_reference(stream) + assert ref[:fin] + end +end diff --git a/test/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs similarity index 79% rename from test/reverse_proxy_test.exs rename to test/reverse_proxy/reverse_proxy_test.exs index 0672f57db..1ab3cc4bb 100644 --- a/test/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxyTest do - use Pleroma.Web.ConnCase, async: true + use Pleroma.Web.ConnCase import ExUnit.CaptureLog import Mox alias Pleroma.ReverseProxy @@ -29,11 +29,11 @@ defp user_agent_mock(user_agent, invokes) do {"content-length", byte_size(json) |> to_string()} ], %{url: url}} end) - |> expect(:stream_body, invokes, fn %{url: url} -> + |> expect(:stream_body, invokes, fn %{url: url} = client -> case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do [{_, 0}] -> Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) - {:ok, json} + {:ok, json, client} [{_, 1}] -> Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) @@ -78,7 +78,39 @@ test "closed connection", %{conn: conn} do assert conn.halted end - describe "max_body " do + defp stream_mock(invokes, with_close? \\ false) do + ClientMock + |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ -> + Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0) + + {:ok, 200, [{"content-type", "application/octet-stream"}], + %{url: "/stream-bytes/" <> length}} + end) + |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} = client -> + max = String.to_integer(length) + + case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do + [{_, current}] when current < max -> + Registry.update_value( + Pleroma.ReverseProxy.ClientMock, + "/stream-bytes/" <> length, + &(&1 + 10) + ) + + {:ok, "0123456789", client} + + [{_, ^max}] -> + Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) + :done + end + end) + + if with_close? do + expect(ClientMock, :close, fn _ -> :ok end) + end + end + + describe "max_body" do test "length returns error if content-length more than option", %{conn: conn} do user_agent_mock("hackney/1.15.1", 0) @@ -94,38 +126,6 @@ test "length returns error if content-length more than option", %{conn: conn} do end) == "" end - defp stream_mock(invokes, with_close? \\ false) do - ClientMock - |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ -> - Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0) - - {:ok, 200, [{"content-type", "application/octet-stream"}], - %{url: "/stream-bytes/" <> length}} - end) - |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} -> - max = String.to_integer(length) - - case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do - [{_, current}] when current < max -> - Registry.update_value( - Pleroma.ReverseProxy.ClientMock, - "/stream-bytes/" <> length, - &(&1 + 10) - ) - - {:ok, "0123456789"} - - [{_, ^max}] -> - Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) - :done - end - end) - - if with_close? do - expect(ClientMock, :close, fn _ -> :ok end) - end - end - test "max_body_length returns error if streaming body more than that option", %{conn: conn} do stream_mock(3, true) @@ -223,12 +223,12 @@ defp headers_mock(_) do Registry.register(Pleroma.ReverseProxy.ClientMock, "/headers", 0) {:ok, 200, [{"content-type", "application/json"}], %{url: "/headers", headers: headers}} end) - |> expect(:stream_body, 2, fn %{url: url, headers: headers} -> + |> expect(:stream_body, 2, fn %{url: url, headers: headers} = client -> case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do [{_, 0}] -> Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) headers = for {k, v} <- headers, into: %{}, do: {String.capitalize(k), v} - {:ok, Jason.encode!(%{headers: headers})} + {:ok, Jason.encode!(%{headers: headers}), client} [{_, 1}] -> Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) @@ -305,11 +305,11 @@ defp disposition_headers_mock(headers) do {:ok, 200, headers, %{url: "/disposition"}} end) - |> expect(:stream_body, 2, fn %{url: "/disposition"} -> + |> expect(:stream_body, 2, fn %{url: "/disposition"} = client -> case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/disposition") do [{_, 0}] -> Registry.update_value(Pleroma.ReverseProxy.ClientMock, "/disposition", &(&1 + 1)) - {:ok, ""} + {:ok, "", client} [{_, 1}] -> Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/disposition") @@ -341,4 +341,45 @@ test "with content-disposition header", %{conn: conn} do assert {"content-disposition", "attachment; filename=\"filename.jpg\""} in conn.resp_headers end end + + describe "tesla client using gun integration" do + @describetag :integration + + clear_config([Pleroma.ReverseProxy.Client]) do + Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) + end + + clear_config([Pleroma.Gun.API]) do + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun) + end + + setup do + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + + on_exit(fn -> + Application.put_env(:tesla, :adapter, adapter) + end) + end + + test "common", %{conn: conn} do + conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") + assert byte_size(conn.resp_body) == 10 + assert conn.state == :chunked + assert conn.status == 200 + end + + test "ssl", %{conn: conn} do + conn = ReverseProxy.call(conn, "https://httpbin.org/stream-bytes/10") + assert byte_size(conn.resp_body) == 10 + assert conn.state == :chunked + assert conn.status == 200 + end + + test "follow redirects", %{conn: conn} do + conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/5") + assert conn.state == :chunked + assert conn.status == 200 + end + end end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index ba3341327..5727871ea 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -107,7 +107,7 @@ def get( "https://osada.macgirvin.com/.well-known/webfinger?resource=acct:mike@osada.macgirvin.com", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -120,7 +120,7 @@ def get( "https://social.heldscal.la/.well-known/webfinger?resource=https://social.heldscal.la/user/29191", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -141,7 +141,7 @@ def get( "https://pawoo.net/.well-known/webfinger?resource=acct:https://pawoo.net/users/pekorino", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -167,7 +167,7 @@ def get( "https://social.stopwatchingus-heidelberg.de/.well-known/webfinger?resource=acct:https://social.stopwatchingus-heidelberg.de/user/18330", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -188,7 +188,7 @@ def get( "https://mamot.fr/.well-known/webfinger?resource=acct:https://mamot.fr/users/Skruyb", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -201,7 +201,7 @@ def get( "https://social.heldscal.la/.well-known/webfinger?resource=nonexistant@social.heldscal.la", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -214,7 +214,7 @@ def get( "https://squeet.me/xrd/?uri=lain@squeet.me", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -227,7 +227,7 @@ def get( "https://mst3k.interlinked.me/users/luciferMysticus", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -248,7 +248,7 @@ def get( "https://hubzilla.example.org/channel/kaniini", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -257,7 +257,7 @@ def get( }} end - def get("https://niu.moe/users/rye", _, _, Accept: "application/activity+json") do + def get("https://niu.moe/users/rye", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -265,7 +265,7 @@ def get("https://niu.moe/users/rye", _, _, Accept: "application/activity+json") }} end - def get("https://n1u.moe/users/rye", _, _, Accept: "application/activity+json") do + def get("https://n1u.moe/users/rye", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -284,7 +284,7 @@ def get("http://mastodon.example.org/users/admin/statuses/100787282858396771", _ }} end - def get("https://puckipedia.com/", _, _, Accept: "application/activity+json") do + def get("https://puckipedia.com/", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -308,9 +308,9 @@ def get("https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" }} end - def get("https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39", _, _, - Accept: "application/activity+json" - ) do + def get("https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, @@ -318,7 +318,7 @@ def get("https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39", _, }} end - def get("https://mobilizon.org/@tcit", _, _, Accept: "application/activity+json") do + def get("https://mobilizon.org/@tcit", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -358,7 +358,7 @@ def get("https://wedistribute.org/wp-json/pterotype/v1/actor/-blog", _, _, _) do }} end - def get("http://mastodon.example.org/users/admin", _, _, Accept: "application/activity+json") do + def get("http://mastodon.example.org/users/admin", _, _, _) do {:ok, %Tesla.Env{ status: 200, @@ -366,7 +366,9 @@ def get("http://mastodon.example.org/users/admin", _, _, Accept: "application/ac }} end - def get("http://mastodon.example.org/users/relay", _, _, Accept: "application/activity+json") do + def get("http://mastodon.example.org/users/relay", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, @@ -374,7 +376,9 @@ def get("http://mastodon.example.org/users/relay", _, _, Accept: "application/ac }} end - def get("http://mastodon.example.org/users/gargron", _, _, Accept: "application/activity+json") do + def get("http://mastodon.example.org/users/gargron", _, _, [ + {"accept", "application/activity+json"} + ]) do {:error, :nxdomain} end @@ -557,7 +561,7 @@ def get( "http://mastodon.example.org/@admin/99541947525187367", _, _, - Accept: "application/activity+json" + _ ) do {:ok, %Tesla.Env{ @@ -582,7 +586,7 @@ def get("https://shitposter.club/notice/7369654", _, _, _) do }} end - def get("https://mstdn.io/users/mayuutann", _, _, Accept: "application/activity+json") do + def get("https://mstdn.io/users/mayuutann", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -594,7 +598,7 @@ def get( "https://mstdn.io/users/mayuutann/statuses/99568293732299394", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -614,7 +618,7 @@ def get("https://pleroma.soykaf.com/users/lain/feed.atom", _, _, _) do }} end - def get(url, _, _, Accept: "application/xrd+xml,application/jrd+json") + def get(url, _, _, [{"accept", "application/xrd+xml,application/jrd+json"}]) when url in [ "https://pleroma.soykaf.com/.well-known/webfinger?resource=acct:https://pleroma.soykaf.com/users/lain", "https://pleroma.soykaf.com/.well-known/webfinger?resource=https://pleroma.soykaf.com/users/lain" @@ -641,7 +645,7 @@ def get( "https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/1", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -685,7 +689,7 @@ def get( "https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/5381", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -738,7 +742,7 @@ def get( "https://social.sakamoto.gq/.well-known/webfinger?resource=https://social.sakamoto.gq/users/eal", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -751,7 +755,7 @@ def get( "https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056", _, _, - Accept: "application/atom+xml" + [{"accept", "application/atom+xml"}] ) do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sakamoto.atom")}} end @@ -768,7 +772,7 @@ def get( "https://mastodon.social/.well-known/webfinger?resource=https://mastodon.social/users/lambadalambda", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -790,7 +794,7 @@ def get( "http://gs.example.org/.well-known/webfinger?resource=http://gs.example.org:4040/index.php/user/1", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -804,7 +808,7 @@ def get( "http://gs.example.org:4040/index.php/user/1", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{status: 406, body: ""}} end @@ -840,7 +844,7 @@ def get( "https://squeet.me/xrd?uri=lain@squeet.me", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -853,7 +857,7 @@ def get( "https://social.heldscal.la/.well-known/webfinger?resource=shp@social.heldscal.la", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -866,7 +870,7 @@ def get( "https://social.heldscal.la/.well-known/webfinger?resource=invalid_content@social.heldscal.la", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{status: 200, body: ""}} end @@ -883,7 +887,7 @@ def get( "http://framatube.org/main/xrd?uri=framasoft@framatube.org", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -905,7 +909,7 @@ def get( "http://gnusocial.de/main/xrd?uri=winterdienst@gnusocial.de", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -942,7 +946,7 @@ def get( "https://gerzilla.de/xrd/?uri=kaniini@gerzilla.de", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -1005,7 +1009,7 @@ def get("https://apfed.club/channel/indio", _, _, _) do %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/osada-user-indio.json")}} end - def get("https://social.heldscal.la/user/23211", _, _, Accept: "application/activity+json") do + def get("https://social.heldscal.la/user/23211", _, _, [{"accept", "application/activity+json"}]) do {:ok, Tesla.Mock.json(%{"id" => "https://social.heldscal.la/user/23211"}, status: 200)} end @@ -1138,7 +1142,7 @@ def get( "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=lain@zetsubou.xn--q9jyb4c", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -1151,7 +1155,7 @@ def get( "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=https://zetsubou.xn--q9jyb4c/users/lain", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -1173,7 +1177,9 @@ def get( }} end - def get("https://info.pleroma.site/activity.json", _, _, Accept: "application/activity+json") do + def get("https://info.pleroma.site/activity.json", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, @@ -1185,7 +1191,9 @@ def get("https://info.pleroma.site/activity.json", _, _, _) do {:ok, %Tesla.Env{status: 404, body: ""}} end - def get("https://info.pleroma.site/activity2.json", _, _, Accept: "application/activity+json") do + def get("https://info.pleroma.site/activity2.json", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, @@ -1197,7 +1205,9 @@ def get("https://info.pleroma.site/activity2.json", _, _, _) do {:ok, %Tesla.Env{status: 404, body: ""}} end - def get("https://info.pleroma.site/activity3.json", _, _, Accept: "application/activity+json") do + def get("https://info.pleroma.site/activity3.json", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, diff --git a/test/user_invite_token_test.exs b/test/user_invite_token_test.exs index 111e40361..671560e41 100644 --- a/test/user_invite_token_test.exs +++ b/test/user_invite_token_test.exs @@ -4,7 +4,6 @@ defmodule Pleroma.UserInviteTokenTest do use ExUnit.Case, async: true - use Pleroma.DataCase alias Pleroma.UserInviteToken describe "valid_invite?/1 one time invites" do @@ -64,7 +63,6 @@ test "expires today returns true", %{invite: invite} do test "expires yesterday returns false", %{invite: invite} do invite = %{invite | expires_at: Date.add(Date.utc_today(), -1)} - invite = Repo.insert!(invite) refute UserInviteToken.valid_invite?(invite) end end @@ -82,7 +80,6 @@ test "not overdue date and less uses returns true", %{invite: invite} do test "overdue date and less uses returns false", %{invite: invite} do invite = %{invite | expires_at: Date.add(Date.utc_today(), -1)} - invite = Repo.insert!(invite) refute UserInviteToken.valid_invite?(invite) end @@ -93,7 +90,6 @@ test "not overdue date with more uses returns false", %{invite: invite} do test "overdue date with more uses returns false", %{invite: invite} do invite = %{invite | expires_at: Date.add(Date.utc_today(), -1), uses: 5} - invite = Repo.insert!(invite) refute UserInviteToken.valid_invite?(invite) end end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 5fbdf96f6..02ffbfa0b 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -2439,7 +2439,8 @@ test "saving full setting if value is not keyword", %{conn: conn} do "value" => "Tesla.Adapter.Httpc", "db" => [":adapter"] } - ] + ], + "need_reboot" => true } end @@ -2526,7 +2527,6 @@ test "common config example", %{conn: conn} do %{"tuple" => [":seconds_valid", 60]}, %{"tuple" => [":path", ""]}, %{"tuple" => [":key1", nil]}, - %{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}, %{"tuple" => [":regex1", "~r/https:\/\/example.com/"]}, %{"tuple" => [":regex2", "~r/https:\/\/example.com/u"]}, %{"tuple" => [":regex3", "~r/https:\/\/example.com/i"]}, @@ -2556,7 +2556,6 @@ test "common config example", %{conn: conn} do %{"tuple" => [":seconds_valid", 60]}, %{"tuple" => [":path", ""]}, %{"tuple" => [":key1", nil]}, - %{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}, %{"tuple" => [":regex1", "~r/https:\\/\\/example.com/"]}, %{"tuple" => [":regex2", "~r/https:\\/\\/example.com/u"]}, %{"tuple" => [":regex3", "~r/https:\\/\\/example.com/i"]}, @@ -2569,7 +2568,6 @@ test "common config example", %{conn: conn} do ":seconds_valid", ":path", ":key1", - ":partial_chain", ":regex1", ":regex2", ":regex3", @@ -2583,7 +2581,8 @@ test "common config example", %{conn: conn} do "value" => "Tesla.Adapter.Httpc", "db" => [":adapter"] } - ] + ], + "need_reboot" => true } end diff --git a/test/web/common_api/common_api_utils_test.exs b/test/web/common_api/common_api_utils_test.exs index 848300ef3..759501a67 100644 --- a/test/web/common_api/common_api_utils_test.exs +++ b/test/web/common_api/common_api_utils_test.exs @@ -474,6 +474,13 @@ test "returns recipients when object not found" do activity = insert(:note_activity, user: user, note: object) Pleroma.Repo.delete(object) + obj_url = activity.data["object"] + + Tesla.Mock.mock(fn + %{method: :get, url: ^obj_url} -> + %Tesla.Env{status: 404, body: ""} + end) + assert Utils.maybe_notify_mentioned_recipients(["test-test"], activity) == [ "test-test" ] diff --git a/test/web/push/impl_test.exs b/test/web/push/impl_test.exs index acae7a734..737976f1f 100644 --- a/test/web/push/impl_test.exs +++ b/test/web/push/impl_test.exs @@ -126,7 +126,7 @@ test "renders title and body for follow activity" do user = insert(:user, nickname: "Bob") other_user = insert(:user) {:ok, _, _, activity} = CommonAPI.follow(user, other_user) - object = Object.normalize(activity) + object = Object.normalize(activity, false) assert Impl.format_body(%{activity: activity}, user, object) == "@Bob has followed you" From 2a219f5e86bea076b1bc93f1a9205c764d43a380 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 18 Feb 2020 09:12:46 -0600 Subject: [PATCH 02/73] Improve changelog message --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48080503a..e4bce5c02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,7 +73,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Support for custom Elixir modules (such as MRF policies) - User settings: Add _This account is a_ option. - OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`). -- New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires OTP version older that 22.2, otherwise pleroma won’t start. For hackney OTP update is not required. +- New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires minimum OTP version of 22.2 otherwise Pleroma won’t start. For hackney OTP update is not required.
API Changes From 7d73e7a09a72354acf526652e307149afbf5b1a3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 18 Feb 2020 09:18:09 -0600 Subject: [PATCH 03/73] Spelling --- lib/pleroma/http/adapter/gun.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index f25afeda7..ec6475e96 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -90,7 +90,7 @@ defp try_to_get_conn(uri, opts) do case Connections.checkin(uri, :gun_connections) do nil -> Logger.info( - "Gun connections pool checkin was not succesfull. Trying to open conn for next request." + "Gun connections pool checkin was not successful. Trying to open conn for next request." ) :ok = Connections.open_conn(uri, :gun_connections, opts) From 138a3c1fe48bbace79c0121d4571db3c2a827860 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 18 Feb 2020 09:30:18 -0600 Subject: [PATCH 04/73] Spelling was wrong in test as well --- test/http/adapter/gun_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/http/adapter/gun_test.exs b/test/http/adapter/gun_test.exs index 37489e1a4..1d7977c83 100644 --- a/test/http/adapter/gun_test.exs +++ b/test/http/adapter/gun_test.exs @@ -101,7 +101,7 @@ test "get conn on next request" do assert opts[:conn] == nil assert opts[:close_conn] == nil end) =~ - "Gun connections pool checkin was not succesfull. Trying to open conn for next request." + "Gun connections pool checkin was not successful. Trying to open conn for next request." opts = Gun.options(uri) From c9db0507f8d49aee9988b0b63477672f5df9c0b2 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 19 Feb 2020 12:19:03 +0300 Subject: [PATCH 05/73] removing retry option and changing some logger messages levels --- lib/pleroma/http/adapter/gun.ex | 28 +++++++++++++++++++++------- lib/pleroma/pool/connections.ex | 17 ++++++++--------- test/http/adapter/gun_test.exs | 2 +- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index ec6475e96..f1018dd8d 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -15,7 +15,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do connect_timeout: 20_000, domain_lookup_timeout: 5_000, tls_handshake_timeout: 5_000, - retry_timeout: 100, + retry: 0, await_up_timeout: 5_000 ] @@ -89,7 +89,7 @@ defp try_to_get_conn(uri, opts) do try do case Connections.checkin(uri, :gun_connections) do nil -> - Logger.info( + Logger.debug( "Gun connections pool checkin was not successful. Trying to open conn for next request." ) @@ -97,7 +97,9 @@ defp try_to_get_conn(uri, opts) do opts conn when is_pid(conn) -> - Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri(uri)}") + Logger.debug( + "received conn #{inspect(conn)} #{uri.scheme}://#{Connections.compose_uri(uri)}" + ) opts |> Keyword.put(:conn, conn) @@ -105,18 +107,30 @@ defp try_to_get_conn(uri, opts) do end rescue error -> - Logger.warn("Gun connections pool checkin caused error #{inspect(error)}") + Logger.warn( + "Gun connections pool checkin caused error #{uri.scheme}://#{ + Connections.compose_uri(uri) + } #{inspect(error)}" + ) + opts catch :exit, {:timeout, _} -> - Logger.info( - "Gun connections pool checkin with timeout error #{Connections.compose_uri(uri)}" + Logger.warn( + "Gun connections pool checkin with timeout error #{uri.scheme}://#{ + Connections.compose_uri(uri) + }" ) opts :exit, error -> - Logger.warn("Gun pool checkin exited with error #{inspect(error)}") + Logger.warn( + "Gun pool checkin exited with error #{uri.scheme}://#{Connections.compose_uri(uri)} #{ + inspect(error) + }" + ) + opts end end diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 1ed16d1c1..c7136e0e0 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -52,8 +52,7 @@ def open_conn(%URI{} = uri, name, opts) do opts = opts |> Enum.into(%{}) - |> Map.put_new(:receive, false) - |> Map.put_new(:retry, pool_opts[:retry] || 5) + |> Map.put_new(:retry, pool_opts[:retry] || 0) |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100) |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) @@ -108,11 +107,11 @@ def handle_cast({:checkout, conn_pid, pid}, state) do put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by}) else false -> - Logger.warn("checkout for closed conn #{inspect(conn_pid)}") + Logger.debug("checkout for closed conn #{inspect(conn_pid)}") state nil -> - Logger.info("checkout for alive conn #{inspect(conn_pid)}, but is not in state") + Logger.debug("checkout for alive conn #{inspect(conn_pid)}, but is not in state") state end @@ -172,15 +171,15 @@ def handle_info({:gun_up, conn_pid, _protocol}, state) do }) else :error_gun_info -> - Logger.warn(":gun.info caused error") + Logger.debug(":gun.info caused error") state false -> - Logger.warn(":gun_up message for closed conn #{inspect(conn_pid)}") + Logger.debug(":gun_up message for closed conn #{inspect(conn_pid)}") state nil -> - Logger.warn( + Logger.debug( ":gun_up message for alive conn #{inspect(conn_pid)}, but deleted from state" ) @@ -216,11 +215,11 @@ def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do else false -> # gun can send gun_down for closed conn, maybe connection is not closed yet - Logger.warn(":gun_down message for closed conn #{inspect(conn_pid)}") + Logger.debug(":gun_down message for closed conn #{inspect(conn_pid)}") state nil -> - Logger.warn( + Logger.debug( ":gun_down message for alive conn #{inspect(conn_pid)}, but deleted from state" ) diff --git a/test/http/adapter/gun_test.exs b/test/http/adapter/gun_test.exs index 1d7977c83..ef1b4a882 100644 --- a/test/http/adapter/gun_test.exs +++ b/test/http/adapter/gun_test.exs @@ -91,7 +91,7 @@ test "don't receive conn if receive_conn is false" do test "get conn on next request" do level = Application.get_env(:logger, :level) - Logger.configure(level: :info) + Logger.configure(level: :debug) on_exit(fn -> Logger.configure(level: level) end) uri = URI.parse("http://some-domain2.com") From 3849bbb60d9085bced717fef1f09216d570af287 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 21 Feb 2020 10:15:56 +0300 Subject: [PATCH 06/73] temp using tesla from fork --- mix.exs | 6 +++++- mix.lock | 46 +++++++++++++++++++++++----------------------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/mix.exs b/mix.exs index 273307bbe..18e33b214 100644 --- a/mix.exs +++ b/mix.exs @@ -119,7 +119,11 @@ defp deps do {:calendar, "~> 0.17.4"}, {:cachex, "~> 3.0.2"}, {:poison, "~> 3.0", override: true}, - {:tesla, "~> 1.3", override: true}, + # {:tesla, "~> 1.3", override: true}, + {:tesla, + github: "alex-strizhakov/tesla", + ref: "922cc3db13b421763edbea76246b8ea61c38c6fa", + override: true}, {:castore, "~> 0.1"}, {:cowlib, "~> 2.8", override: true}, {:gun, diff --git a/mix.lock b/mix.lock index 12ce1afac..10b2fe30d 100644 --- a/mix.lock +++ b/mix.lock @@ -21,42 +21,42 @@ "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, - "db_connection": {:hex, :db_connection, "2.2.1", "caee17725495f5129cb7faebde001dc4406796f12a62b8949f4ac69315080566", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, + "db_connection": {:hex, :db_connection, "2.2.1", "caee17725495f5129cb7faebde001dc4406796f12a62b8949f4ac69315080566", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "2b02ece62d9f983fcd40954e443b7d9e6589664380e5546b2b9b523cd0fb59e1"}, "decimal": {:hex, :decimal, "1.8.1", "a4ef3f5f3428bdbc0d35374029ffcf4ede8533536fa79896dd450168d9acdf3c", [:mix], [], "hexpm", "3cb154b00225ac687f6cbd4acc4b7960027c757a5152b369923ead9ddbca7aec"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, - "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.3.3", "0830bf3aebcbf3d8c1a1811cd581773b6866886c012f52c0f027031fa96a0b53", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"}, + "ecto": {:hex, :ecto, "3.3.3", "0830bf3aebcbf3d8c1a1811cd581773b6866886c012f52c0f027031fa96a0b53", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "12e368e3c2a2938d7776defaabdae40e82900fc4d8d66120ec1e01dfd8b93c3a"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, - "ecto_sql": {:hex, :ecto_sql, "3.3.4", "aa18af12eb875fbcda2f75e608b3bd534ebf020fc4f6448e4672fcdcbb081244", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4 or ~> 3.3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, - "esshd": {:hex, :esshd, "0.1.1", "d4dd4c46698093a40a56afecce8a46e246eb35463c457c246dacba2e056f31b5", [:mix], [], "hexpm"}, + "ecto_sql": {:hex, :ecto_sql, "3.3.4", "aa18af12eb875fbcda2f75e608b3bd534ebf020fc4f6448e4672fcdcbb081244", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4 or ~> 3.3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5eccbdbf92e3c6f213007a82d5dbba4cd9bb659d1a21331f89f408e4c0efd7a8"}, + "esshd": {:hex, :esshd, "0.1.1", "d4dd4c46698093a40a56afecce8a46e246eb35463c457c246dacba2e056f31b5", [:mix], [], "hexpm", "d73e341e3009d390aa36387dc8862860bf9f874c94d9fd92ade2926376f49981"}, "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm", "b14f1dc204321429479c569cfbe8fb287541184ed040956c8862cb7a677b8406"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, "ex_aws": {:hex, :ex_aws, "2.1.1", "1e4de2106cfbf4e837de41be41cd15813eabc722315e388f0d6bb3732cec47cd", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "06b6fde12b33bb6d65d5d3493e903ba5a56d57a72350c15285a4298338089e10"}, "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.2", "c0258bbdfea55de4f98f0b2f0ca61fe402cc696f573815134beb1866e778f47b", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "0569f5b211b1a3b12b705fe2a9d0e237eb1360b9d76298028df2346cad13097a"}, "ex_const": {:hex, :ex_const, "0.2.4", "d06e540c9d834865b012a17407761455efa71d0ce91e5831e86881b9c9d82448", [:mix], [], "hexpm", "96fd346610cc992b8f896ed26a98be82ac4efb065a0578f334a32d60a3ba9767"}, - "ex_doc": {:hex, :ex_doc, "0.21.3", "857ec876b35a587c5d9148a2512e952e24c24345552259464b98bfbb883c7b42", [:mix], [{:earmark, "~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, + "ex_doc": {:hex, :ex_doc, "0.21.3", "857ec876b35a587c5d9148a2512e952e24c24345552259464b98bfbb883c7b42", [:mix], [{:earmark, "~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "0db1ee8d1547ab4877c5b5dffc6604ef9454e189928d5ba8967d4a58a801f161"}, "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "b84f6af156264530b312a8ab98ac6088f6b77ae5fe2058305c81434aa01fbaf9"}, - "ex_syslogger": {:hex, :ex_syslogger, "1.5.0", "bc936ee3fd13d9e592cb4c3a1e8a55fccd33b05e3aa7b185f211f3ed263ff8f0", [:mix], [{:poison, ">= 1.5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:syslog, "~> 1.0.5", [hex: :syslog, repo: "hexpm", optional: false]}], "hexpm"}, - "excoveralls": {:hex, :excoveralls, "0.12.2", "a513defac45c59e310ac42fcf2b8ae96f1f85746410f30b1ff2b710a4b6cd44b", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, + "ex_syslogger": {:hex, :ex_syslogger, "1.5.0", "bc936ee3fd13d9e592cb4c3a1e8a55fccd33b05e3aa7b185f211f3ed263ff8f0", [:mix], [{:poison, ">= 1.5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:syslog, "~> 1.0.5", [hex: :syslog, repo: "hexpm", optional: false]}], "hexpm", "f3b4b184dcdd5f356b7c26c6cd72ab0918ba9dfb4061ccfaf519e562942af87b"}, + "excoveralls": {:hex, :excoveralls, "0.12.2", "a513defac45c59e310ac42fcf2b8ae96f1f85746410f30b1ff2b710a4b6cd44b", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "151c476331d49b45601ffc45f43cb3a8beb396b02a34e3777fea0ad34ae57d89"}, "fast_html": {:hex, :fast_html, "1.0.3", "2cc0d4b68496266a1530e0c852cafeaede0bd10cfdee26fda50dc696c203162f", [:make, :mix], [], "hexpm", "ab3d782b639d3c4655fbaec0f9d032c91f8cab8dd791ac7469c2381bc7c32f85"}, "fast_sanitize": {:hex, :fast_sanitize, "0.1.7", "2a7cd8734c88a2de6de55022104f8a3b87f1fdbe8bbf131d9049764b53d50d0d", [:mix], [{:fast_html, "~> 1.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f39fe8ea08fbac17487c30bf09b7d9f3e12472e51fb07a88ffeb8fd17da8ab67"}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "31fc8090fde1acd267c07c36ea7365b8604055f897d3a53dd967658c691bd827"}, - "floki": {:hex, :floki, "0.25.0", "b1c9ddf5f32a3a90b43b76f3386ca054325dc2478af020e87b5111c19f2284ac", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"}, + "floki": {:hex, :floki, "0.25.0", "b1c9ddf5f32a3a90b43b76f3386ca054325dc2478af020e87b5111c19f2284ac", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "631f4e627c46d5ecd347df5a2accdaf0621c77c3693c5b75a8ad58e84c61f242"}, "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm", "29bd14a88030980849c7ed2447b8db6d6c9278a28b11a44cafe41b791205440f"}, "gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm"}, "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.4", "f13088e1ec10ce01665cf25f5ff779e7df3f2dc71b37084976cf89d1aa124d5c", [:mix], [], "hexpm", "3c75b5ea8288e2ee7ea503ff9e30dfe4d07ad3c054576a6e60040e79a801e14d"}, "gun": {:git, "https://github.com/ninenines/gun.git", "bd6425ab87428cf4c95f4d23e0a48fd065fbd714", [ref: "bd6425ab87428cf4c95f4d23e0a48fd065fbd714"]}, "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e0100f8ef7d1124222c11ad362c857d3df7cb5f4204054f9f0f4a728666591fc"}, - "html_entities": {:hex, :html_entities, "0.5.1", "1c9715058b42c35a2ab65edc5b36d0ea66dd083767bef6e3edb57870ef556549", [:mix], [], "hexpm"}, + "html_entities": {:hex, :html_entities, "0.5.1", "1c9715058b42c35a2ab65edc5b36d0ea66dd083767bef6e3edb57870ef556549", [:mix], [], "hexpm", "30efab070904eb897ff05cd52fa61c1025d7f8ef3a9ca250bc4e6513d16c32de"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "293d77bb6f4a67ac8bde1428735c3b42f22cbb30", [ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"]}, - "httpoison": {:hex, :httpoison, "1.6.2", "ace7c8d3a361cebccbed19c283c349b3d26991eff73a1eaaa8abae2e3c8089b6", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, + "httpoison": {:hex, :httpoison, "1.6.2", "ace7c8d3a361cebccbed19c283c349b3d26991eff73a1eaaa8abae2e3c8089b6", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "aa2c74bd271af34239a3948779612f87df2422c2fdcfdbcec28d9c105f0773fe"}, "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "4bdd305eb64e18b0273864920695cb18d7a2021f31a11b9c5fbcd9a253f936e2"}, "inet_cidr": {:hex, :inet_cidr, "1.0.4", "a05744ab7c221ca8e395c926c3919a821eb512e8f36547c062f62c4ca0cf3d6e", [:mix], [], "hexpm", "64a2d30189704ae41ca7dbdd587f5291db5d1dda1414e0774c29ffc81088c1bc"}, "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fdf843bca858203ae1de16da2ee206f53416bbda5dc8c9e78f43243de4bc3afe"}, - "joken": {:hex, :joken, "2.2.0", "2daa1b12be05184aff7b5ace1d43ca1f81345962285fff3f88db74927c954d3a", [:mix], [{:jose, "~> 1.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm"}, - "jose": {:hex, :jose, "1.10.1", "16d8e460dae7203c6d1efa3f277e25b5af8b659febfc2f2eb4bacf87f128b80a", [:mix, :rebar3], [], "hexpm"}, + "joken": {:hex, :joken, "2.2.0", "2daa1b12be05184aff7b5ace1d43ca1f81345962285fff3f88db74927c954d3a", [:mix], [{:jose, "~> 1.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "b4f92e30388206f869dd25d1af628a1d99d7586e5cf0672f64d4df84c4d2f5e9"}, + "jose": {:hex, :jose, "1.10.1", "16d8e460dae7203c6d1efa3f277e25b5af8b659febfc2f2eb4bacf87f128b80a", [:mix, :rebar3], [], "hexpm", "3c7ddc8a9394b92891db7c2771da94bf819834a1a4c92e30857b7d582e2f8257"}, "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"}, "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "a10c6eb62cca416019663129699769f0c2ccf39428b3bb3c0cb38c718a0c186d"}, "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "d4b316c7222a85bbaa2fd7c6e90e37e953257ad196dc229505137c5e505e9eff"}, @@ -69,38 +69,38 @@ "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm", "3bc928d817974fa10cc11e6c89b9a9361e37e96dbbf3d868c41094ec05745dcd"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm", "052346cf322311c49a0f22789f3698eea030eec09b8c47367f0686ef2634ae14"}, "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]}, - "nimble_parsec": {:hex, :nimble_parsec, "0.5.3", "def21c10a9ed70ce22754fdeea0810dafd53c2db3219a0cd54cf5526377af1c6", [:mix], [], "hexpm"}, + "nimble_parsec": {:hex, :nimble_parsec, "0.5.3", "def21c10a9ed70ce22754fdeea0810dafd53c2db3219a0cd54cf5526377af1c6", [:mix], [], "hexpm", "589b5af56f4afca65217a1f3eb3fee7e79b09c40c742fddc1c312b3ac0b3399f"}, "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, "oban": {:hex, :oban, "0.12.1", "695e9490c6e0edfca616d80639528e448bd29b3bff7b7dd10a56c79b00a5d7fb", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c1d58d69b8b5a86e7167abbb8cc92764a66f25f12f6172052595067fc6a30a17"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.4", "8dd29ed783f2e12195d7e0a4640effc0a7c37e6537da491f1db01839eee6d053", [:mix], [], "hexpm", "595d09db74cb093b1903381c9de423276a931a2480a46a1a5dc7f932a2a6375b"}, - "phoenix": {:hex, :phoenix, "1.4.13", "67271ad69b51f3719354604f4a3f968f83aa61c19199343656c9caee057ff3b8", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, - "phoenix_ecto": {:hex, :phoenix_ecto, "4.1.0", "a044d0756d0464c5a541b4a0bf4bcaf89bffcaf92468862408290682c73ae50d", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, - "phoenix_html": {:hex, :phoenix_html, "2.14.0", "d8c6bc28acc8e65f8ea0080ee05aa13d912c8758699283b8d3427b655aabe284", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, + "phoenix": {:hex, :phoenix, "1.4.13", "67271ad69b51f3719354604f4a3f968f83aa61c19199343656c9caee057ff3b8", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ab765a0feddb81fc62e2116c827b5f068df85159c162bee760745276ad7ddc1b"}, + "phoenix_ecto": {:hex, :phoenix_ecto, "4.1.0", "a044d0756d0464c5a541b4a0bf4bcaf89bffcaf92468862408290682c73ae50d", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "c5e666a341ff104d0399d8f0e4ff094559b2fde13a5985d4cb5023b2c2ac558b"}, + "phoenix_html": {:hex, :phoenix_html, "2.14.0", "d8c6bc28acc8e65f8ea0080ee05aa13d912c8758699283b8d3427b655aabe284", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "b0bb30eda478a06dbfbe96728061a93833db3861a49ccb516f839ecb08493fbb"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "1.1.2", "496c303bdf1b2e98a9d26e89af5bba3ab487ba3a3735f74bf1f4064d2a845a3e", [:mix], [], "hexpm", "1f13f9f0f3e769a667a6b6828d29dec37497a082d195cc52dbef401a9b69bf38"}, "phoenix_swoosh": {:hex, :phoenix_swoosh, "0.2.0", "a7e0b32077cd6d2323ae15198839b05d9caddfa20663fd85787479e81f89520e", [:mix], [{:phoenix, "~> 1.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.2", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:swoosh, "~> 0.1", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm", "ebf1bfa7b3c1c850c04929afe02e2e0d7ab135e0706332c865de03e761676b1f"}, "plug": {:hex, :plug, "1.9.0", "8d7c4e26962283ff9f8f3347bd73838e2413fbc38b7bb5467d5924f68f3a5a4a", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "9902eda2c52ada2a096434682e99a2493f5d06a94d6ac6bcfff9805f952350f1"}, - "plug_cowboy": {:hex, :plug_cowboy, "2.1.2", "8b0addb5908c5238fac38e442e81b6fcd32788eaa03246b4d55d147c47c5805e", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, - "plug_crypto": {:hex, :plug_crypto, "1.1.2", "bdd187572cc26dbd95b87136290425f2b580a116d3fb1f564216918c9730d227", [:mix], [], "hexpm"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.1.2", "8b0addb5908c5238fac38e442e81b6fcd32788eaa03246b4d55d147c47c5805e", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "7d722581ce865a237e14da6d946f92704101740a256bd13ec91e63c0b122fc70"}, + "plug_crypto": {:hex, :plug_crypto, "1.1.2", "bdd187572cc26dbd95b87136290425f2b580a116d3fb1f564216918c9730d227", [:mix], [], "hexpm", "6b8b608f895b6ffcfad49c37c7883e8df98ae19c6a28113b02aa1e9c5b22d6b5"}, "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, "postgrex": {:hex, :postgrex, "0.15.3", "5806baa8a19a68c4d07c7a624ccdb9b57e89cbc573f1b98099e3741214746ae4", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "4737ce62a31747b4c63c12b20c62307e51bb4fcd730ca0c32c280991e0606c90"}, - "prometheus": {:hex, :prometheus, "4.5.0", "8f4a2246fe0beb50af0f77c5e0a5bb78fe575c34a9655d7f8bc743aad1c6bf76", [:mix, :rebar3], [], "hexpm"}, + "prometheus": {:hex, :prometheus, "4.5.0", "8f4a2246fe0beb50af0f77c5e0a5bb78fe575c34a9655d7f8bc743aad1c6bf76", [:mix, :rebar3], [], "hexpm", "679b5215480fff612b8351f45c839d995a07ce403e42ff02f1c6b20960d41a4e"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.3", "3dd4da1812b8e0dbee81ea58bb3b62ed7588f2eae0c9e97e434c46807ff82311", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm", "8d66289f77f913b37eda81fd287340c17e61a447549deb28efc254532b2bed82"}, "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm", "9fd13404a48437e044b288b41f76e64acd9735fb8b0e3809f494811dfa66d0fb"}, "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.3.0", "c4b527e0b3a9ef1af26bdcfbfad3998f37795b9185d475ca610fe4388fdd3bb5", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm", "c4d1404ac4e9d3d963da601db2a7d8ea31194f0017057fabf0cfb9bf5a6c8c75"}, "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm", "0273a6483ccb936d79ca19b0ab629aef0dba958697c94782bb728b920dfc6a79"}, "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "d736bfa7444112eb840027bb887832a0e403a4a3437f48028c3b29a2dbbd2543"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"}, - "recon": {:hex, :recon, "2.5.0", "2f7fcbec2c35034bade2f9717f77059dc54eb4e929a3049ca7ba6775c0bd66cd", [:mix, :rebar3], [], "hexpm"}, + "recon": {:hex, :recon, "2.5.0", "2f7fcbec2c35034bade2f9717f77059dc54eb4e929a3049ca7ba6775c0bd66cd", [:mix, :rebar3], [], "hexpm", "72f3840fedd94f06315c523f6cecf5b4827233bed7ae3fe135b2a0ebeab5e196"}, "remote_ip": {:git, "https://git.pleroma.social/pleroma/remote_ip.git", "825dc00aaba5a1b7c4202a532b696b595dd3bcb3", [ref: "825dc00aaba5a1b7c4202a532b696b595dd3bcb3"]}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm", "13104d7897e38ed7f044c4de953a6c28597d1c952075eb2e328bc6d6f2bfc496"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm", "2e1ec458f892ffa81f9f8386e3f35a1af6db7a7a37748a64478f13163a1f3573"}, "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "e3928e1d2889a308aaf3e42755809ac21cffd77cb58eef01cbfdab4ce2fd1e21"}, - "syslog": {:hex, :syslog, "1.0.6", "995970c9aa7feb380ac493302138e308d6e04fd57da95b439a6df5bb3bf75076", [:rebar3], [], "hexpm"}, + "syslog": {:hex, :syslog, "1.0.6", "995970c9aa7feb380ac493302138e308d6e04fd57da95b439a6df5bb3bf75076", [:rebar3], [], "hexpm", "769ddfabd0d2a16f3f9c17eb7509951e0ca4f68363fb26f2ee51a8ec4a49881a"}, "telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm", "4738382e36a0a9a2b6e25d67c960e40e1a2c95560b9f936d8e29de8cd858480f"}, - "tesla": {:hex, :tesla, "1.3.2", "deb92c5c9ce35e747a395ba413ca78593a4f75bf0e1545630ee2e3d34264021e", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "922cc3db13b421763edbea76246b8ea61c38c6fa", [ref: "922cc3db13b421763edbea76246b8ea61c38c6fa"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "f354efb2400dd7a80fd9eb6c8419068c4f632da4ac47f3d8822d6e33f08bc852"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, "tzdata": {:hex, :tzdata, "0.5.22", "f2ba9105117ee0360eae2eca389783ef7db36d533899b2e84559404dbc77ebb8", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "cd66c8a1e6a9e121d1f538b01bef459334bb4029a1ffb4eeeb5e4eae0337e7b6"}, From a03c420b84d9901be70520d8c027ccb53449990d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 21 Feb 2020 12:32:42 +0300 Subject: [PATCH 07/73] by default don't use gun retries remove conn depends on retry setting from config --- config/config.exs | 2 +- lib/pleroma/pool/connections.ex | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/config/config.exs b/config/config.exs index 853a53fc9..7f3a4d1b6 100644 --- a/config/config.exs +++ b/config/config.exs @@ -599,7 +599,7 @@ config :pleroma, :connections_pool, receive_connection_timeout: 250, max_connections: 250, - retry: 5, + retry: 0, retry_timeout: 100, await_up_timeout: 5_000 diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index c7136e0e0..d20927580 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -5,6 +5,8 @@ defmodule Pleroma.Pool.Connections do use GenServer + alias Pleroma.Config + require Logger @type domain :: String.t() @@ -33,7 +35,7 @@ def checkin(url, name) def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name) def checkin(%URI{} = uri, name) do - timeout = Pleroma.Config.get([:connections_pool, :receive_connection_timeout], 250) + timeout = Config.get([:connections_pool, :receive_connection_timeout], 250) GenServer.call( name, @@ -47,7 +49,7 @@ def open_conn(url, name, opts \\ []) def open_conn(url, name, opts) when is_binary(url), do: open_conn(URI.parse(url), name, opts) def open_conn(%URI{} = uri, name, opts) do - pool_opts = Pleroma.Config.get([:connections_pool], []) + pool_opts = Config.get([:connections_pool], []) opts = opts @@ -193,12 +195,13 @@ def handle_info({:gun_up, conn_pid, _protocol}, state) do @impl true def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do + retries = Config.get([:connections_pool, :retry], 0) # we can't get info on this pid, because pid is dead state = with true <- Process.alive?(conn_pid), {key, conn} <- find_conn(state.conns, conn_pid) do - if conn.retries == 5 do - Logger.debug("closing conn if retries is eq 5 #{inspect(conn_pid)}") + if conn.retries == retries do + Logger.debug("closing conn if retries is eq #{inspect(conn_pid)}") :ok = API.close(conn.conn) put_in( From ad8f26c0a4a0a579e93547e78313d3e4ecef6ed5 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 21 Feb 2020 12:53:40 +0300 Subject: [PATCH 08/73] more info in Connections.checkin timout errors --- lib/pleroma/http/adapter/gun.ex | 13 +++++++++---- test/http_test.exs | 4 ++-- test/pool/connections_test.exs | 8 ++++++-- test/reverse_proxy/client/tesla_test.exs | 4 ++-- test/reverse_proxy/reverse_proxy_test.exs | 8 ++++---- 5 files changed, 23 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index f1018dd8d..fc40b324a 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -115,11 +115,16 @@ defp try_to_get_conn(uri, opts) do opts catch - :exit, {:timeout, _} -> + :exit, {:timeout, {_, operation, [_, {method, _}, _]}} -> + messages_len = + :gun_connections + |> Process.whereis() + |> Process.info(:message_queue_len) + Logger.warn( - "Gun connections pool checkin with timeout error #{uri.scheme}://#{ - Connections.compose_uri(uri) - }" + "Gun connections pool checkin with timeout error for #{operation} #{method} #{ + uri.scheme + }://#{Connections.compose_uri(uri)}. Messages length: #{messages_len}" ) opts diff --git a/test/http_test.exs b/test/http_test.exs index d80b96496..83c27f6e1 100644 --- a/test/http_test.exs +++ b/test/http_test.exs @@ -61,8 +61,8 @@ test "returns successfully result" do describe "connection pools" do @describetag :integration - clear_config([Pleroma.Gun.API]) do - Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun) + clear_config(Pleroma.Gun.API) do + Pleroma.Config.put(Pleroma.Gun.API, Pleroma.Gun) end test "gun" do diff --git a/test/pool/connections_test.exs b/test/pool/connections_test.exs index 6f0e041ae..d0d711c55 100644 --- a/test/pool/connections_test.exs +++ b/test/pool/connections_test.exs @@ -15,6 +15,10 @@ defmodule Pleroma.Pool.ConnectionsTest do :ok end + clear_config([:connections_pool, :retry]) do + Pleroma.Config.put([:connections_pool, :retry], 5) + end + setup do name = :test_connections adapter = Application.get_env(:tesla, :adapter) @@ -429,8 +433,8 @@ test "remove frequently used and idle", %{name: name} do describe "integration test" do @describetag :integration - clear_config([API]) do - Pleroma.Config.put([API], Pleroma.Gun) + clear_config(API) do + Pleroma.Config.put(API, Pleroma.Gun) end test "opens connection and reuse it on next request", %{name: name} do diff --git a/test/reverse_proxy/client/tesla_test.exs b/test/reverse_proxy/client/tesla_test.exs index 75a70988c..231271b0d 100644 --- a/test/reverse_proxy/client/tesla_test.exs +++ b/test/reverse_proxy/client/tesla_test.exs @@ -8,8 +8,8 @@ defmodule Pleroma.ReverseProxy.Client.TeslaTest do alias Pleroma.ReverseProxy.Client @moduletag :integration - clear_config_all([Pleroma.Gun.API]) do - Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun) + clear_config_all(Pleroma.Gun.API) do + Pleroma.Config.put(Pleroma.Gun.API, Pleroma.Gun) end setup do diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 1ab3cc4bb..f61fc02c5 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -345,12 +345,12 @@ test "with content-disposition header", %{conn: conn} do describe "tesla client using gun integration" do @describetag :integration - clear_config([Pleroma.ReverseProxy.Client]) do - Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) + clear_config(Pleroma.ReverseProxy.Client) do + Pleroma.Config.put(Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.Client.Tesla) end - clear_config([Pleroma.Gun.API]) do - Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun) + clear_config(Pleroma.Gun.API) do + Pleroma.Config.put(Pleroma.Gun.API, Pleroma.Gun) end setup do From 6806df80ddb1e52aef2b89b923d9a3e2844b5aeb Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 21 Feb 2020 14:28:16 +0300 Subject: [PATCH 09/73] don't log info ssl messages --- lib/pleroma/http/adapter/gun.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index fc40b324a..0a6872ad6 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -58,7 +58,8 @@ defp add_scheme_opts(opts, %URI{scheme: "https", host: host, port: port}) do depth: 20, reuse_sessions: false, verify_fun: - {&:ssl_verify_hostname.verify_fun/3, [check_hostname: Adapter.domain_or_fallback(host)]} + {&:ssl_verify_hostname.verify_fun/3, [check_hostname: Adapter.domain_or_fallback(host)]}, + log_level: :warning ] ] From f604f9e47061b9d47c1bb62cc7aaf44fabdf69b3 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 21 Feb 2020 14:33:55 +0300 Subject: [PATCH 10/73] hackney pool timeout --- lib/pleroma/http/connection.ex | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 85918341a..e2d7afbbd 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -33,8 +33,14 @@ def options(%URI{} = uri, opts \\ []) do end defp pool_timeout(opts) do - timeout = - Config.get([:pools, opts[:pool], :timeout]) || Config.get([:pools, :default, :timeout]) + {config_key, default} = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + {:pools, Config.get([:pools, :default, :timeout])} + else + {:hackney_pools, 10_000} + end + + timeout = Config.get([config_key, opts[:pool], :timeout], default) Keyword.merge(opts, timeout: timeout) end From d44f9e3b6cfd5a0dae07f6194bfd05360afd6560 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 21 Feb 2020 16:56:55 +0300 Subject: [PATCH 11/73] fix for timeout clause --- lib/pleroma/http/adapter/gun.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index 0a6872ad6..7b7e38d8c 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -117,7 +117,7 @@ defp try_to_get_conn(uri, opts) do opts catch :exit, {:timeout, {_, operation, [_, {method, _}, _]}} -> - messages_len = + {:message_queue_len, messages_len} = :gun_connections |> Process.whereis() |> Process.info(:message_queue_len) From 8efae966b1e87fe448a13d04eae0898c4a102c29 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 24 Feb 2020 19:56:27 +0300 Subject: [PATCH 12/73] open conn in separate task --- lib/mix/tasks/pleroma/benchmark.ex | 2 +- lib/pleroma/gun/api.ex | 7 +- lib/pleroma/gun/api/mock.ex | 5 +- lib/pleroma/gun/conn.ex | 146 +++++++++++++++ lib/pleroma/gun/gun.ex | 5 +- lib/pleroma/http/adapter/gun.ex | 21 +-- lib/pleroma/pool/connections.ex | 287 ++++++++++------------------- restarter/lib/pleroma.ex | 4 +- test/gun/gun_test.exs | 6 + test/http/adapter/gun_test.exs | 17 +- test/http/connection_test.exs | 2 +- test/pool/connections_test.exs | 186 ++++++++++--------- 12 files changed, 384 insertions(+), 304 deletions(-) diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index 01e079136..7a7430289 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -79,7 +79,7 @@ def run(["adapters"]) do start_pleroma() :ok = - Pleroma.Pool.Connections.open_conn( + Pleroma.Gun.Conn.open( "https://httpbin.org/stream-bytes/1500", :gun_connections ) diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex index a0c3c5415..f79c9f443 100644 --- a/lib/pleroma/gun/api.ex +++ b/lib/pleroma/gun/api.ex @@ -6,9 +6,10 @@ defmodule Pleroma.Gun.API do @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} @callback info(pid()) :: map() @callback close(pid()) :: :ok - @callback await_up(pid) :: {:ok, atom()} | {:error, atom()} + @callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()} @callback connect(pid(), map()) :: reference() @callback await(pid(), reference()) :: {:response, :fin, 200, []} + @callback set_owner(pid(), pid()) :: :ok def open(host, port, opts), do: api().open(host, port, opts) @@ -16,11 +17,13 @@ def info(pid), do: api().info(pid) def close(pid), do: api().close(pid) - def await_up(pid), do: api().await_up(pid) + def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout) def connect(pid, opts), do: api().connect(pid, opts) def await(pid, ref), do: api().await(pid, ref) + def set_owner(pid, owner), do: api().set_owner(pid, owner) + defp api, do: Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun) end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex index 0134b016e..6d24b0e69 100644 --- a/lib/pleroma/gun/api/mock.ex +++ b/lib/pleroma/gun/api/mock.ex @@ -118,7 +118,10 @@ def open('localhost', 9050, _) do end @impl API - def await_up(_pid), do: {:ok, :http} + def await_up(_pid, _timeout), do: {:ok, :http} + + @impl API + def set_owner(_pid, _owner), do: :ok @impl API def connect(pid, %{host: _, port: 80}) do diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 2474829d6..ddb9f30b0 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -6,6 +6,11 @@ defmodule Pleroma.Gun.Conn do @moduledoc """ Struct for gun connection data """ + alias Pleroma.Gun.API + alias Pleroma.Pool.Connections + + require Logger + @type gun_state :: :up | :down @type conn_state :: :active | :idle @@ -26,4 +31,145 @@ defmodule Pleroma.Gun.Conn do last_reference: 0, crf: 1, retries: 0 + + @spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil + def open(url, name, opts \\ []) + def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts) + + def open(%URI{} = uri, name, opts) do + pool_opts = Pleroma.Config.get([:connections_pool], []) + + opts = + opts + |> Enum.into(%{}) + |> Map.put_new(:retry, pool_opts[:retry] || 0) + |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100) + |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) + + key = "#{uri.scheme}:#{uri.host}:#{uri.port}" + + Logger.debug("opening new connection #{Connections.compose_uri_log(uri)}") + + conn_pid = + if Connections.count(name) < opts[:max_connection] do + do_open(uri, opts) + else + try_do_open(name, uri, opts) + end + + if is_pid(conn_pid) do + conn = %Pleroma.Gun.Conn{ + conn: conn_pid, + gun_state: :up, + conn_state: :active, + last_reference: :os.system_time(:second) + } + + :ok = API.set_owner(conn_pid, Process.whereis(name)) + Connections.add_conn(name, key, conn) + end + end + + defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do + connect_opts = + uri + |> destination_opts() + |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) + + with open_opts <- Map.delete(opts, :tls_opts), + {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), + {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]), + stream <- API.connect(conn, connect_opts), + {:response, :fin, 200, _} <- API.await(conn, stream) do + conn + else + error -> + Logger.warn( + "Received error on opening connection with http proxy #{ + Connections.compose_uri_log(uri) + } #{inspect(error)}" + ) + + nil + end + end + + defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do + version = + proxy_type + |> to_string() + |> String.last() + |> case do + "4" -> 4 + _ -> 5 + end + + socks_opts = + uri + |> destination_opts() + |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) + |> Map.put(:version, version) + + opts = + opts + |> Map.put(:protocols, [:socks]) + |> Map.put(:socks_opts, socks_opts) + + with {:ok, conn} <- API.open(proxy_host, proxy_port, opts), + {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do + conn + else + error -> + Logger.warn( + "Received error on opening connection with socks proxy #{ + Connections.compose_uri_log(uri) + } #{inspect(error)}" + ) + + nil + end + end + + defp do_open(%URI{host: host, port: port} = uri, opts) do + {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + + with {:ok, conn} <- API.open(host, port, opts), + {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do + conn + else + error -> + Logger.warn( + "Received error on opening connection #{Connections.compose_uri_log(uri)} #{ + inspect(error) + }" + ) + + nil + end + end + + defp destination_opts(%URI{host: host, port: port}) do + {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + %{host: host, port: port} + end + + defp add_http2_opts(opts, "https", tls_opts) do + Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts}) + end + + defp add_http2_opts(opts, _, _), do: opts + + defp try_do_open(name, uri, opts) do + Logger.debug("try to open conn #{Connections.compose_uri_log(uri)}") + + with [{close_key, least_used} | _conns] <- + Connections.get_unused_conns(name), + :ok <- Pleroma.Gun.API.close(least_used.conn) do + Connections.remove_conn(name, close_key) + + do_open(uri, opts) + else + [] -> nil + end + end end diff --git a/lib/pleroma/gun/gun.ex b/lib/pleroma/gun/gun.ex index 4a1bbc95f..da82983b1 100644 --- a/lib/pleroma/gun/gun.ex +++ b/lib/pleroma/gun/gun.ex @@ -32,7 +32,7 @@ def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun defdelegate close(pid), to: :gun @impl API - defdelegate await_up(pid), to: :gun + defdelegate await_up(pid, timeout \\ 5_000), to: :gun @impl API defdelegate connect(pid, opts), to: :gun @@ -42,4 +42,7 @@ def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun @spec flush(pid() | reference()) :: :ok defdelegate flush(pid), to: :gun + + @impl API + defdelegate set_owner(pid, owner), to: :gun end diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index 7b7e38d8c..908d71898 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -12,7 +12,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do alias Pleroma.Pool.Connections @defaults [ - connect_timeout: 20_000, + connect_timeout: 5_000, domain_lookup_timeout: 5_000, tls_handshake_timeout: 5_000, retry: 0, @@ -94,13 +94,11 @@ defp try_to_get_conn(uri, opts) do "Gun connections pool checkin was not successful. Trying to open conn for next request." ) - :ok = Connections.open_conn(uri, :gun_connections, opts) + Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end) opts conn when is_pid(conn) -> - Logger.debug( - "received conn #{inspect(conn)} #{uri.scheme}://#{Connections.compose_uri(uri)}" - ) + Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") opts |> Keyword.put(:conn, conn) @@ -109,13 +107,14 @@ defp try_to_get_conn(uri, opts) do rescue error -> Logger.warn( - "Gun connections pool checkin caused error #{uri.scheme}://#{ - Connections.compose_uri(uri) - } #{inspect(error)}" + "Gun connections pool checkin caused error #{Connections.compose_uri_log(uri)} #{ + inspect(error) + }" ) opts catch + # TODO: here must be no timeouts :exit, {:timeout, {_, operation, [_, {method, _}, _]}} -> {:message_queue_len, messages_len} = :gun_connections @@ -124,15 +123,15 @@ defp try_to_get_conn(uri, opts) do Logger.warn( "Gun connections pool checkin with timeout error for #{operation} #{method} #{ - uri.scheme - }://#{Connections.compose_uri(uri)}. Messages length: #{messages_len}" + Connections.compose_uri_log(uri) + }. Messages length: #{messages_len}" ) opts :exit, error -> Logger.warn( - "Gun pool checkin exited with error #{uri.scheme}://#{Connections.compose_uri(uri)} #{ + "Gun pool checkin exited with error #{Connections.compose_uri_log(uri)} #{ inspect(error) }" ) diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index d20927580..a444f822f 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -20,7 +20,6 @@ defmodule Pleroma.Pool.Connections do defstruct conns: %{}, opts: [] alias Pleroma.Gun.API - alias Pleroma.Gun.Conn @spec start_link({atom(), keyword()}) :: {:ok, pid()} def start_link({name, opts}) do @@ -44,23 +43,6 @@ def checkin(%URI{} = uri, name) do ) end - @spec open_conn(String.t() | URI.t(), atom(), keyword()) :: :ok - def open_conn(url, name, opts \\ []) - def open_conn(url, name, opts) when is_binary(url), do: open_conn(URI.parse(url), name, opts) - - def open_conn(%URI{} = uri, name, opts) do - pool_opts = Config.get([:connections_pool], []) - - opts = - opts - |> Enum.into(%{}) - |> Map.put_new(:retry, pool_opts[:retry] || 0) - |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100) - |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) - - GenServer.cast(name, {:open_conn, %{opts: opts, uri: uri}}) - end - @spec alive?(atom()) :: boolean() def alive?(name) do pid = Process.whereis(name) @@ -72,23 +54,37 @@ def get_state(name) do GenServer.call(name, :state) end + @spec count(atom()) :: pos_integer() + def count(name) do + GenServer.call(name, :count) + end + + @spec get_unused_conns(atom()) :: [{domain(), conn()}] + def get_unused_conns(name) do + GenServer.call(name, :unused_conns) + end + @spec checkout(pid(), pid(), atom()) :: :ok def checkout(conn, pid, name) do GenServer.cast(name, {:checkout, conn, pid}) end + @spec add_conn(atom(), String.t(), Pleroma.Gun.Conn.t()) :: :ok + def add_conn(name, key, conn) do + GenServer.cast(name, {:add_conn, key, conn}) + end + + @spec remove_conn(atom(), String.t()) :: :ok + def remove_conn(name, key) do + GenServer.cast(name, {:remove_conn, key}) + end + @impl true - def handle_cast({:open_conn, %{opts: opts, uri: uri}}, state) do - Logger.debug("opening new #{compose_uri(uri)}") - max_connections = state.opts[:max_connections] + def handle_cast({:add_conn, key, conn}, state) do + state = put_in(state.conns[key], conn) - key = compose_key(uri) - - if Enum.count(state.conns) < max_connections do - open_conn(key, uri, state, opts) - else - try_to_open_conn(key, uri, state, opts) - end + Process.monitor(conn.conn) + {:noreply, state} end @impl true @@ -120,14 +116,20 @@ def handle_cast({:checkout, conn_pid, pid}, state) do {:noreply, state} end + @impl true + def handle_cast({:remove_conn, key}, state) do + state = put_in(state.conns, Map.delete(state.conns, key)) + {:noreply, state} + end + @impl true def handle_call({:checkin, uri}, from, state) do - Logger.debug("checkin #{compose_uri(uri)}") - key = compose_key(uri) + key = "#{uri.scheme}:#{uri.host}:#{uri.port}" + Logger.debug("checkin #{key}") case state.conns[key] do %{conn: conn, gun_state: gun_state} = current_conn when gun_state == :up -> - Logger.debug("reusing conn #{compose_uri(uri)}") + Logger.debug("reusing conn #{key}") with time <- :os.system_time(:second), last_reference <- time - current_conn.last_reference, @@ -154,12 +156,31 @@ def handle_call({:checkin, uri}, from, state) do @impl true def handle_call(:state, _from, state), do: {:reply, state, state} + @impl true + def handle_call(:count, _from, state) do + {:reply, Enum.count(state.conns), state} + end + + @impl true + def handle_call(:unused_conns, _from, state) do + unused_conns = + state.conns + |> Enum.filter(fn {_k, v} -> + v.conn_state == :idle and v.used_by == [] + end) + |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> + x.crf <= y.crf and x.last_reference <= y.last_reference + end) + + {:reply, unused_conns, state} + end + @impl true def handle_info({:gun_up, conn_pid, _protocol}, state) do state = - with true <- Process.alive?(conn_pid), - conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid), + with conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid), {key, conn} <- find_conn(state.conns, conn_pid, conn_key), + {true, key} <- {Process.alive?(conn_pid), key}, time <- :os.system_time(:second), last_reference <- time - conn.last_reference, current_crf <- crf(last_reference, 100, conn.crf) do @@ -176,14 +197,16 @@ def handle_info({:gun_up, conn_pid, _protocol}, state) do Logger.debug(":gun.info caused error") state - false -> + {false, key} -> Logger.debug(":gun_up message for closed conn #{inspect(conn_pid)}") - state + + put_in( + state.conns, + Map.delete(state.conns, key) + ) nil -> - Logger.debug( - ":gun_up message for alive conn #{inspect(conn_pid)}, but deleted from state" - ) + Logger.debug(":gun_up message for conn which is not found in state") :ok = API.close(conn_pid) @@ -198,8 +221,8 @@ def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do retries = Config.get([:connections_pool, :retry], 0) # we can't get info on this pid, because pid is dead state = - with true <- Process.alive?(conn_pid), - {key, conn} <- find_conn(state.conns, conn_pid) do + with {key, conn} <- find_conn(state.conns, conn_pid), + {true, key} <- {Process.alive?(conn_pid), key} do if conn.retries == retries do Logger.debug("closing conn if retries is eq #{inspect(conn_pid)}") :ok = API.close(conn.conn) @@ -216,15 +239,17 @@ def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do }) end else - false -> + {false, key} -> # gun can send gun_down for closed conn, maybe connection is not closed yet Logger.debug(":gun_down message for closed conn #{inspect(conn_pid)}") - state + + put_in( + state.conns, + Map.delete(state.conns, key) + ) nil -> - Logger.debug( - ":gun_down message for alive conn #{inspect(conn_pid)}, but deleted from state" - ) + Logger.debug(":gun_down message for conn which is not found in state") :ok = API.close(conn_pid) @@ -234,7 +259,29 @@ def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do {:noreply, state} end - defp compose_key(%URI{scheme: scheme, host: host, port: port}), do: "#{scheme}:#{host}:#{port}" + @impl true + def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do + Logger.debug("received DOWM message for #{inspect(conn_pid)} reason -> #{inspect(reason)}") + + state = + with {key, conn} <- find_conn(state.conns, conn_pid) do + Enum.each(conn.used_by, fn {pid, _ref} -> + Process.exit(pid, reason) + end) + + put_in( + state.conns, + Map.delete(state.conns, key) + ) + else + nil -> + Logger.debug(":DOWN message for conn which is not found in state") + + state + end + + {:noreply, state} + end defp compose_key_gun_info(pid) do try do @@ -265,153 +312,11 @@ defp find_conn(conns, conn_pid, conn_key) do end) end - defp open_conn(key, uri, state, %{proxy: {proxy_host, proxy_port}} = opts) do - connect_opts = - uri - |> destination_opts() - |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) - - with open_opts <- Map.delete(opts, :tls_opts), - {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), - {:ok, _} <- API.await_up(conn), - stream <- API.connect(conn, connect_opts), - {:response, :fin, 200, _} <- API.await(conn, stream), - state <- - put_in(state.conns[key], %Conn{ - conn: conn, - gun_state: :up, - conn_state: :active, - last_reference: :os.system_time(:second) - }) do - {:noreply, state} - else - error -> - Logger.warn( - "Received error on opening connection with http proxy #{uri.scheme}://#{ - compose_uri(uri) - }: #{inspect(error)}" - ) - - {:noreply, state} - end - end - - defp open_conn(key, uri, state, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do - version = - proxy_type - |> to_string() - |> String.last() - |> case do - "4" -> 4 - _ -> 5 - end - - socks_opts = - uri - |> destination_opts() - |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) - |> Map.put(:version, version) - - opts = - opts - |> Map.put(:protocols, [:socks]) - |> Map.put(:socks_opts, socks_opts) - - with {:ok, conn} <- API.open(proxy_host, proxy_port, opts), - {:ok, _} <- API.await_up(conn), - state <- - put_in(state.conns[key], %Conn{ - conn: conn, - gun_state: :up, - conn_state: :active, - last_reference: :os.system_time(:second) - }) do - {:noreply, state} - else - error -> - Logger.warn( - "Received error on opening connection with socks proxy #{uri.scheme}://#{ - compose_uri(uri) - }: #{inspect(error)}" - ) - - {:noreply, state} - end - end - - defp open_conn(key, %URI{host: host, port: port} = uri, state, opts) do - Logger.debug("opening conn #{compose_uri(uri)}") - {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) - - with {:ok, conn} <- API.open(host, port, opts), - {:ok, _} <- API.await_up(conn), - state <- - put_in(state.conns[key], %Conn{ - conn: conn, - gun_state: :up, - conn_state: :active, - last_reference: :os.system_time(:second) - }) do - Logger.debug("new conn opened #{compose_uri(uri)}") - Logger.debug("replying to the call #{compose_uri(uri)}") - {:noreply, state} - else - error -> - Logger.warn( - "Received error on opening connection #{uri.scheme}://#{compose_uri(uri)}: #{ - inspect(error) - }" - ) - - {:noreply, state} - end - end - - defp destination_opts(%URI{host: host, port: port}) do - {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) - %{host: host, port: port} - end - - defp add_http2_opts(opts, "https", tls_opts) do - Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts}) - end - - defp add_http2_opts(opts, _, _), do: opts - - @spec get_unused_conns(map()) :: [{domain(), conn()}] - def get_unused_conns(conns) do - conns - |> Enum.filter(fn {_k, v} -> - v.conn_state == :idle and v.used_by == [] - end) - |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> - x.crf <= y.crf and x.last_reference <= y.last_reference - end) - end - - defp try_to_open_conn(key, uri, state, opts) do - Logger.debug("try to open conn #{compose_uri(uri)}") - - with [{close_key, least_used} | _conns] <- get_unused_conns(state.conns), - :ok <- API.close(least_used.conn), - state <- - put_in( - state.conns, - Map.delete(state.conns, close_key) - ) do - Logger.debug( - "least used conn found and closed #{inspect(least_used.conn)} #{compose_uri(uri)}" - ) - - open_conn(key, uri, state, opts) - else - [] -> {:noreply, state} - end - end - def crf(current, steps, crf) do 1 + :math.pow(0.5, current / steps) * crf end - def compose_uri(%URI{} = uri), do: "#{uri.host}#{uri.path}" + def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do + "#{scheme}://#{host}#{path}" + end end diff --git a/restarter/lib/pleroma.ex b/restarter/lib/pleroma.ex index d7817909d..4ade890f9 100644 --- a/restarter/lib/pleroma.ex +++ b/restarter/lib/pleroma.ex @@ -44,7 +44,7 @@ def handle_cast(:need_reboot, state) do end def handle_cast({:restart, :test, _}, state) do - Logger.warn("pleroma restarted") + Logger.warn("pleroma manually restarted") {:noreply, Map.put(state, :need_reboot?, false)} end @@ -57,7 +57,7 @@ def handle_cast({:restart, _, delay}, state) do def handle_cast({:after_boot, _}, %{after_boot: true} = state), do: {:noreply, state} def handle_cast({:after_boot, :test}, state) do - Logger.warn("pleroma restarted") + Logger.warn("pleroma restarted after boot") {:noreply, Map.put(state, :after_boot, true)} end diff --git a/test/gun/gun_test.exs b/test/gun/gun_test.exs index 7f185617c..9f3e0f938 100644 --- a/test/gun/gun_test.exs +++ b/test/gun/gun_test.exs @@ -19,6 +19,12 @@ test "opens connection and receive response" do assert json = receive_response(conn, ref) assert %{"args" => %{"a" => "b", "c" => "d"}} = Jason.decode!(json) + + {:ok, pid} = Task.start(fn -> Process.sleep(50) end) + + :ok = :gun.set_owner(conn, pid) + + assert :gun.info(conn).owner == pid end defp receive_response(conn, ref, acc \\ "") do diff --git a/test/http/adapter/gun_test.exs b/test/http/adapter/gun_test.exs index ef1b4a882..a8dcbae04 100644 --- a/test/http/adapter/gun_test.exs +++ b/test/http/adapter/gun_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.HTTP.Adapter.GunTest do use Pleroma.Tests.Helpers import ExUnit.CaptureLog alias Pleroma.Config + alias Pleroma.Gun.Conn alias Pleroma.HTTP.Adapter.Gun alias Pleroma.Pool.Connections @@ -72,7 +73,7 @@ test "https url with non standart port" do test "receive conn by default" do uri = URI.parse("http://another-domain.com") - :ok = Connections.open_conn(uri, :gun_connections) + :ok = Conn.open(uri, :gun_connections) received_opts = Gun.options(uri) assert received_opts[:close_conn] == false @@ -81,7 +82,7 @@ test "receive conn by default" do test "don't receive conn if receive_conn is false" do uri = URI.parse("http://another-domain2.com") - :ok = Connections.open_conn(uri, :gun_connections) + :ok = Conn.open(uri, :gun_connections) opts = [receive_conn: false] received_opts = Gun.options(opts, uri) @@ -118,7 +119,7 @@ test "merges with defaul http adapter config" do test "default ssl adapter opts with connection" do uri = URI.parse("https://some-domain.com") - :ok = Connections.open_conn(uri, :gun_connections) + :ok = Conn.open(uri, :gun_connections) opts = Gun.options(uri) @@ -167,7 +168,7 @@ test "passed opts have more weight than defaults" do describe "after_request/1" do test "body_as not chunks" do uri = URI.parse("http://some-domain.com") - :ok = Connections.open_conn(uri, :gun_connections) + :ok = Conn.open(uri, :gun_connections) opts = Gun.options(uri) :ok = Gun.after_request(opts) conn = opts[:conn] @@ -185,7 +186,7 @@ test "body_as not chunks" do test "body_as chunks" do uri = URI.parse("http://some-domain.com") - :ok = Connections.open_conn(uri, :gun_connections) + :ok = Conn.open(uri, :gun_connections) opts = Gun.options([body_as: :chunks], uri) :ok = Gun.after_request(opts) conn = opts[:conn] @@ -205,7 +206,7 @@ test "body_as chunks" do test "with no connection" do uri = URI.parse("http://uniq-domain.com") - :ok = Connections.open_conn(uri, :gun_connections) + :ok = Conn.open(uri, :gun_connections) opts = Gun.options([body_as: :chunks], uri) conn = opts[:conn] @@ -227,7 +228,7 @@ test "with no connection" do test "with ipv4" do uri = URI.parse("http://127.0.0.1") - :ok = Connections.open_conn(uri, :gun_connections) + :ok = Conn.open(uri, :gun_connections) opts = Gun.options(uri) send(:gun_connections, {:gun_up, opts[:conn], :http}) :ok = Gun.after_request(opts) @@ -246,7 +247,7 @@ test "with ipv4" do test "with ipv6" do uri = URI.parse("http://[2a03:2880:f10c:83:face:b00c:0:25de]") - :ok = Connections.open_conn(uri, :gun_connections) + :ok = Conn.open(uri, :gun_connections) opts = Gun.options(uri) send(:gun_connections, {:gun_up, opts[:conn], :http}) :ok = Gun.after_request(opts) diff --git a/test/http/connection_test.exs b/test/http/connection_test.exs index c1ff0cc21..53ccbc9cd 100644 --- a/test/http/connection_test.exs +++ b/test/http/connection_test.exs @@ -124,7 +124,7 @@ test "default ssl adapter opts with connection" do uri = URI.parse("https://some-domain.com") pid = Process.whereis(:federation) - :ok = Pleroma.Pool.Connections.open_conn(uri, :gun_connections, genserver_pid: pid) + :ok = Pleroma.Gun.Conn.open(uri, :gun_connections, genserver_pid: pid) opts = Connection.options(uri) diff --git a/test/pool/connections_test.exs b/test/pool/connections_test.exs index d0d711c55..f766e3b5f 100644 --- a/test/pool/connections_test.exs +++ b/test/pool/connections_test.exs @@ -45,7 +45,7 @@ test "opens connection and reuse it on next request", %{name: name} do url = "http://some-domain.com" key = "http:some-domain.com:80" refute Connections.checkin(url, name) - :ok = Connections.open_conn(url, name) + :ok = Conn.open(url, name) conn = Connections.checkin(url, name) assert is_pid(conn) @@ -110,7 +110,7 @@ test "reuse connection for idna domains", %{name: name} do url = "http://ですsome-domain.com" refute Connections.checkin(url, name) - :ok = Connections.open_conn(url, name) + :ok = Conn.open(url, name) conn = Connections.checkin(url, name) assert is_pid(conn) @@ -139,7 +139,7 @@ test "reuse for ipv4", %{name: name} do refute Connections.checkin(url, name) - :ok = Connections.open_conn(url, name) + :ok = Conn.open(url, name) conn = Connections.checkin(url, name) assert is_pid(conn) @@ -182,7 +182,7 @@ test "reuse for ipv6", %{name: name} do refute Connections.checkin(url, name) - :ok = Connections.open_conn(url, name) + :ok = Conn.open(url, name) conn = Connections.checkin(url, name) assert is_pid(conn) @@ -209,7 +209,7 @@ test "reuse for ipv6", %{name: name} do test "up and down ipv4", %{name: name} do self = self() url = "http://127.0.0.1" - :ok = Connections.open_conn(url, name) + :ok = Conn.open(url, name) conn = Connections.checkin(url, name) send(name, {:gun_down, conn, nil, nil, nil}) send(name, {:gun_up, conn, nil}) @@ -229,7 +229,7 @@ test "up and down ipv4", %{name: name} do test "up and down ipv6", %{name: name} do self = self() url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]" - :ok = Connections.open_conn(url, name) + :ok = Conn.open(url, name) conn = Connections.checkin(url, name) send(name, {:gun_down, conn, nil, nil, nil}) send(name, {:gun_up, conn, nil}) @@ -253,13 +253,13 @@ test "reuses connection based on protocol", %{name: name} do https_key = "https:some-domain.com:443" refute Connections.checkin(http_url, name) - :ok = Connections.open_conn(http_url, name) + :ok = Conn.open(http_url, name) conn = Connections.checkin(http_url, name) assert is_pid(conn) assert Process.alive?(conn) refute Connections.checkin(https_url, name) - :ok = Connections.open_conn(https_url, name) + :ok = Conn.open(https_url, name) https_conn = Connections.checkin(https_url, name) refute conn == https_conn @@ -288,17 +288,17 @@ test "connection can't get up", %{name: name} do url = "http://gun-not-up.com" assert capture_log(fn -> - :ok = Connections.open_conn(url, name) + refute Conn.open(url, name) refute Connections.checkin(url, name) end) =~ - "Received error on opening connection http://gun-not-up.com: {:error, :timeout}" + "Received error on opening connection http://gun-not-up.com {:error, :timeout}" end test "process gun_down message and then gun_up", %{name: name} do self = self() url = "http://gun-down-and-up.com" key = "http:gun-down-and-up.com:80" - :ok = Connections.open_conn(url, name) + :ok = Conn.open(url, name) conn = Connections.checkin(url, name) assert is_pid(conn) @@ -347,7 +347,7 @@ test "process gun_down message and then gun_up", %{name: name} do test "async processes get same conn for same domain", %{name: name} do url = "http://some-domain.com" - :ok = Connections.open_conn(url, name) + :ok = Conn.open(url, name) tasks = for _ <- 1..5 do @@ -381,8 +381,8 @@ test "remove frequently used and idle", %{name: name} do self = self() http_url = "http://some-domain.com" https_url = "https://some-domain.com" - :ok = Connections.open_conn(https_url, name) - :ok = Connections.open_conn(http_url, name) + :ok = Conn.open(https_url, name) + :ok = Conn.open(http_url, name) conn1 = Connections.checkin(https_url, name) @@ -413,7 +413,7 @@ test "remove frequently used and idle", %{name: name} do :ok = Connections.checkout(conn1, self, name) another_url = "http://another-domain.com" - :ok = Connections.open_conn(another_url, name) + :ok = Conn.open(another_url, name) conn = Connections.checkin(another_url, name) %Connections{ @@ -437,9 +437,19 @@ test "remove frequently used and idle", %{name: name} do Pleroma.Config.put(API, Pleroma.Gun) end + test "opens connection and change owner", %{name: name} do + url = "https://httpbin.org" + :ok = Conn.open(url, name) + conn = Connections.checkin(url, name) + + pid = Process.whereis(name) + + assert :gun.info(conn).owner == pid + end + test "opens connection and reuse it on next request", %{name: name} do url = "http://httpbin.org" - :ok = Connections.open_conn(url, name) + :ok = Conn.open(url, name) Process.sleep(250) conn = Connections.checkin(url, name) @@ -462,7 +472,7 @@ test "opens connection and reuse it on next request", %{name: name} do test "opens ssl connection and reuse it on next request", %{name: name} do url = "https://httpbin.org" - :ok = Connections.open_conn(url, name) + :ok = Conn.open(url, name) Process.sleep(1_000) conn = Connections.checkin(url, name) @@ -488,8 +498,8 @@ test "remove frequently used and idle", %{name: name} do https1 = "https://www.google.com" https2 = "https://httpbin.org" - :ok = Connections.open_conn(https1, name) - :ok = Connections.open_conn(https2, name) + :ok = Conn.open(https1, name) + :ok = Conn.open(https2, name) Process.sleep(1_500) conn = Connections.checkin(https1, name) @@ -513,7 +523,7 @@ test "remove frequently used and idle", %{name: name} do :ok = Connections.checkout(conn, self, name) http = "http://httpbin.org" Process.sleep(1_000) - :ok = Connections.open_conn(http, name) + :ok = Conn.open(http, name) conn = Connections.checkin(http, name) %Connections{ @@ -535,8 +545,8 @@ test "remove earlier used and idle", %{name: name} do https1 = "https://www.google.com" https2 = "https://httpbin.org" - :ok = Connections.open_conn(https1, name) - :ok = Connections.open_conn(https2, name) + :ok = Conn.open(https1, name) + :ok = Conn.open(https2, name) Process.sleep(1_500) Connections.checkin(https1, name) @@ -563,7 +573,7 @@ test "remove earlier used and idle", %{name: name} do :ok = Connections.checkout(conn, self, name) http = "http://httpbin.org" - :ok = Connections.open_conn(http, name) + :ok = Conn.open(http, name) Process.sleep(1_000) conn = Connections.checkin(http, name) @@ -587,8 +597,8 @@ test "doesn't open new conn on pool overflow", %{name: name} do https1 = "https://www.google.com" https2 = "https://httpbin.org" - :ok = Connections.open_conn(https1, name) - :ok = Connections.open_conn(https2, name) + :ok = Conn.open(https1, name) + :ok = Conn.open(https2, name) Process.sleep(1_000) Connections.checkin(https1, name) conn1 = Connections.checkin(https1, name) @@ -639,8 +649,8 @@ test "get idle connection with the smallest crf", %{ https1 = "https://www.google.com" https2 = "https://httpbin.org" - :ok = Connections.open_conn(https1, name) - :ok = Connections.open_conn(https2, name) + :ok = Conn.open(https1, name) + :ok = Conn.open(https2, name) Process.sleep(1_500) Connections.checkin(https1, name) Connections.checkin(https2, name) @@ -694,7 +704,7 @@ test "get idle connection with the smallest crf", %{ } = Connections.get_state(name) http = "http://httpbin.org" - :ok = Connections.open_conn(http, name) + :ok = Conn.open(http, name) Process.sleep(1_000) conn = Connections.checkin(http, name) @@ -725,7 +735,7 @@ test "get idle connection with the smallest crf", %{ test "as ip", %{name: name} do url = "http://proxy-string.com" key = "http:proxy-string.com:80" - :ok = Connections.open_conn(url, name, proxy: {{127, 0, 0, 1}, 8123}) + :ok = Conn.open(url, name, proxy: {{127, 0, 0, 1}, 8123}) conn = Connections.checkin(url, name) @@ -745,7 +755,7 @@ test "as ip", %{name: name} do test "as host", %{name: name} do url = "http://proxy-tuple-atom.com" - :ok = Connections.open_conn(url, name, proxy: {'localhost', 9050}) + :ok = Conn.open(url, name, proxy: {'localhost', 9050}) conn = Connections.checkin(url, name) %Connections{ @@ -765,7 +775,7 @@ test "as host", %{name: name} do test "as ip and ssl", %{name: name} do url = "https://proxy-string.com" - :ok = Connections.open_conn(url, name, proxy: {{127, 0, 0, 1}, 8123}) + :ok = Conn.open(url, name, proxy: {{127, 0, 0, 1}, 8123}) conn = Connections.checkin(url, name) %Connections{ @@ -784,7 +794,7 @@ test "as ip and ssl", %{name: name} do test "as host and ssl", %{name: name} do url = "https://proxy-tuple-atom.com" - :ok = Connections.open_conn(url, name, proxy: {'localhost', 9050}) + :ok = Conn.open(url, name, proxy: {'localhost', 9050}) conn = Connections.checkin(url, name) %Connections{ @@ -804,7 +814,7 @@ test "as host and ssl", %{name: name} do test "with socks type", %{name: name} do url = "http://proxy-socks.com" - :ok = Connections.open_conn(url, name, proxy: {:socks5, 'localhost', 1234}) + :ok = Conn.open(url, name, proxy: {:socks5, 'localhost', 1234}) conn = Connections.checkin(url, name) @@ -825,7 +835,7 @@ test "with socks type", %{name: name} do test "with socks4 type and ssl", %{name: name} do url = "https://proxy-socks.com" - :ok = Connections.open_conn(url, name, proxy: {:socks4, 'localhost', 1234}) + :ok = Conn.open(url, name, proxy: {:socks4, 'localhost', 1234}) conn = Connections.checkin(url, name) @@ -892,71 +902,75 @@ test "recently used will have higher crf", %{crf: crf} do end describe "get_unused_conns/1" do - test "crf is equalent, sorting by reference" do - conns = %{ - "1" => %Conn{ - conn_state: :idle, - last_reference: now() - 1 - }, - "2" => %Conn{ - conn_state: :idle, - last_reference: now() - } - } + test "crf is equalent, sorting by reference", %{name: name} do + Connections.add_conn(name, "1", %Conn{ + conn_state: :idle, + last_reference: now() - 1 + }) - assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(conns) + Connections.add_conn(name, "2", %Conn{ + conn_state: :idle, + last_reference: now() + }) + + assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name) end - test "reference is equalent, sorting by crf" do - conns = %{ - "1" => %Conn{ - conn_state: :idle, - crf: 1.999 - }, - "2" => %Conn{ - conn_state: :idle, - crf: 2 - } - } + test "reference is equalent, sorting by crf", %{name: name} do + Connections.add_conn(name, "1", %Conn{ + conn_state: :idle, + crf: 1.999 + }) - assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(conns) + Connections.add_conn(name, "2", %Conn{ + conn_state: :idle, + crf: 2 + }) + + assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name) end - test "higher crf and lower reference" do - conns = %{ - "1" => %Conn{ - conn_state: :idle, - crf: 3, - last_reference: now() - 1 - }, - "2" => %Conn{ - conn_state: :idle, - crf: 2, - last_reference: now() - } - } + test "higher crf and lower reference", %{name: name} do + Connections.add_conn(name, "1", %Conn{ + conn_state: :idle, + crf: 3, + last_reference: now() - 1 + }) - assert [{"2", _unused_conn} | _others] = Connections.get_unused_conns(conns) + Connections.add_conn(name, "2", %Conn{ + conn_state: :idle, + crf: 2, + last_reference: now() + }) + + assert [{"2", _unused_conn} | _others] = Connections.get_unused_conns(name) end - test "lower crf and lower reference" do - conns = %{ - "1" => %Conn{ - conn_state: :idle, - crf: 1.99, - last_reference: now() - 1 - }, - "2" => %Conn{ - conn_state: :idle, - crf: 2, - last_reference: now() - } - } + test "lower crf and lower reference", %{name: name} do + Connections.add_conn(name, "1", %Conn{ + conn_state: :idle, + crf: 1.99, + last_reference: now() - 1 + }) - assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(conns) + Connections.add_conn(name, "2", %Conn{ + conn_state: :idle, + crf: 2, + last_reference: now() + }) + + assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name) end end + test "count/1", %{name: name} do + assert Connections.count(name) == 0 + Connections.add_conn(name, "1", %Conn{conn: self()}) + assert Connections.count(name) == 1 + Connections.remove_conn(name, "1") + assert Connections.count(name) == 0 + end + defp now do :os.system_time(:second) end From 6b012ddd69aec0f85c22ad91dbb76e05f2edaf58 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 25 Feb 2020 19:01:29 +0300 Subject: [PATCH 13/73] some docs --- docs/configuration/cheatsheet.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index d99537a50..d5a978c5a 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -394,6 +394,8 @@ For each pool, the options are: Advanced settings for connections pool. Pool with opened connections. These connections can be reused in worker pools. +For big instances it's recommended to increase `max_connections` up to 500-1000. It will increase memory usage, but federation would work faster. + * `:receive_connection_timeout` - timeout to receive connection from pool. Default: 250ms. * `:max_connections` - maximum number of connections in the pool. Default: 250 connections. * `:retry` - number of retries, while `gun` will try to reconnect if connections goes down. Default: 5. From 2622cf1190fe8e6ec9145a8cd2538a56889aa7e2 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 2 Mar 2020 09:22:34 +0300 Subject: [PATCH 14/73] returning repo parameters --- config/config.exs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 159aa6398..82012dc10 100644 --- a/config/config.exs +++ b/config/config.exs @@ -49,8 +49,7 @@ config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes, telemetry_event: [Pleroma.Repo.Instrumenter], - migration_lock: nil, - parameters: [gin_fuzzy_search_limit: "500"] + migration_lock: nil config :pleroma, Pleroma.Captcha, enabled: true, @@ -603,6 +602,8 @@ config :pleroma, configurable_from_database: false +config :pleroma, Pleroma.Repo, parameters: [gin_fuzzy_search_limit: "500"] + config :pleroma, :connections_pool, receive_connection_timeout: 250, max_connections: 250, From 137c600cae9869e706d10b06dea04c9249e043da Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 2 Mar 2020 10:01:07 +0300 Subject: [PATCH 15/73] stop connections manually --- test/pool/connections_test.exs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/pool/connections_test.exs b/test/pool/connections_test.exs index f766e3b5f..0e7a118ab 100644 --- a/test/pool/connections_test.exs +++ b/test/pool/connections_test.exs @@ -23,11 +23,18 @@ defmodule Pleroma.Pool.ConnectionsTest do name = :test_connections adapter = Application.get_env(:tesla, :adapter) Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) - on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) - {:ok, _pid} = + {:ok, pid} = Connections.start_link({name, [max_connections: 2, receive_connection_timeout: 1_500]}) + on_exit(fn -> + Application.put_env(:tesla, :adapter, adapter) + + if Process.alive?(pid) do + GenServer.stop(name) + end + end) + {:ok, name: name} end From 85d571fc238c14bedbc0d9a0af2c7c0d76d62c4a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 2 Mar 2020 12:52:41 -0600 Subject: [PATCH 16/73] Move Tesla repo to our GitLab --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 017228b4c..5c1d89208 100644 --- a/mix.exs +++ b/mix.exs @@ -121,7 +121,7 @@ defp deps do {:poison, "~> 3.0", override: true}, # {:tesla, "~> 1.3", override: true}, {:tesla, - github: "alex-strizhakov/tesla", + git: "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git", ref: "922cc3db13b421763edbea76246b8ea61c38c6fa", override: true}, {:castore, "~> 0.1"}, diff --git a/mix.lock b/mix.lock index fecc959e0..8b5c61895 100644 --- a/mix.lock +++ b/mix.lock @@ -102,7 +102,7 @@ "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "e3928e1d2889a308aaf3e42755809ac21cffd77cb58eef01cbfdab4ce2fd1e21"}, "syslog": {:hex, :syslog, "1.0.6", "995970c9aa7feb380ac493302138e308d6e04fd57da95b439a6df5bb3bf75076", [:rebar3], [], "hexpm", "769ddfabd0d2a16f3f9c17eb7509951e0ca4f68363fb26f2ee51a8ec4a49881a"}, "telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm", "4738382e36a0a9a2b6e25d67c960e40e1a2c95560b9f936d8e29de8cd858480f"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "922cc3db13b421763edbea76246b8ea61c38c6fa", [ref: "922cc3db13b421763edbea76246b8ea61c38c6fa"]}, + "tesla": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git", "922cc3db13b421763edbea76246b8ea61c38c6fa", [ref: "922cc3db13b421763edbea76246b8ea61c38c6fa"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "f354efb2400dd7a80fd9eb6c8419068c4f632da4ac47f3d8822d6e33f08bc852"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, "tzdata": {:hex, :tzdata, "0.5.22", "f2ba9105117ee0360eae2eca389783ef7db36d533899b2e84559404dbc77ebb8", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "cd66c8a1e6a9e121d1f538b01bef459334bb4029a1ffb4eeeb5e4eae0337e7b6"}, From f987d83885eef7cd8d114feefe8870a8c5e841c6 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 2 Mar 2020 13:00:05 -0600 Subject: [PATCH 17/73] Clarify in docs how to control connections_pool for Gun. It could easily be confused with the Hackney settings. --- docs/configuration/cheatsheet.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 507f15b87..abb5a3c5f 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -395,7 +395,8 @@ For each pool, the options are: Advanced settings for connections pool. Pool with opened connections. These connections can be reused in worker pools. -For big instances it's recommended to increase `max_connections` up to 500-1000. It will increase memory usage, but federation would work faster. +For big instances it's recommended to increase `config :pleroma, :connections_pool, max_connections: 500` up to 500-1000. +It will increase memory usage, but federation would work faster. * `:receive_connection_timeout` - timeout to receive connection from pool. Default: 250ms. * `:max_connections` - maximum number of connections in the pool. Default: 250 connections. From 3ecdead31ae65f395104a5fd7fafc847a7b97eca Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 10:33:40 +0300 Subject: [PATCH 18/73] debug logs on pleroma restart --- restarter/lib/pleroma.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/restarter/lib/pleroma.ex b/restarter/lib/pleroma.ex index 4ade890f9..e48bc4d1d 100644 --- a/restarter/lib/pleroma.ex +++ b/restarter/lib/pleroma.ex @@ -44,7 +44,7 @@ def handle_cast(:need_reboot, state) do end def handle_cast({:restart, :test, _}, state) do - Logger.warn("pleroma manually restarted") + Logger.debug("pleroma manually restarted") {:noreply, Map.put(state, :need_reboot?, false)} end @@ -57,7 +57,7 @@ def handle_cast({:restart, _, delay}, state) do def handle_cast({:after_boot, _}, %{after_boot: true} = state), do: {:noreply, state} def handle_cast({:after_boot, :test}, state) do - Logger.warn("pleroma restarted after boot") + Logger.debug("pleroma restarted after boot") {:noreply, Map.put(state, :after_boot, true)} end From 4c8569d403f47957f7a5d698c595959007c8a95a Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 12:19:29 +0300 Subject: [PATCH 19/73] otp_version refactor --- lib/pleroma/application.ex | 35 +++++----- lib/pleroma/otp_version.ex | 68 +++++++++----------- test/fixtures/warnings/otp_version/error | 1 - test/fixtures/warnings/otp_version/undefined | 1 - test/otp_version_test.exs | 42 ++++-------- 5 files changed, 60 insertions(+), 87 deletions(-) delete mode 100644 test/fixtures/warnings/otp_version/error delete mode 100644 test/fixtures/warnings/otp_version/undefined diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 00e33d7ac..9b228d6b9 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -66,16 +66,23 @@ def start(_type, _args) do Pleroma.Gopher.Server ] - case Pleroma.OTPVersion.check_version() do - :ok -> :ok - {:error, version} -> raise " - !!!OTP VERSION WARNING!!! - You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. - " - :undefined -> raise " - !!!OTP VERSION WARNING!!! - To support correct handling of unordered certificates chains - OTP version must be > 22.2. - " + if adapter() == Tesla.Adapter.Gun do + case Pleroma.OTPVersion.check() do + :ok -> + :ok + + {:error, version} -> + raise " + !!!OTP VERSION WARNING!!! + You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. + " + + :undefined -> + raise " + !!!OTP VERSION WARNING!!! + To support correct handling of unordered certificates chains - OTP version must be > 22.2. + " + end end # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html @@ -202,11 +209,7 @@ defp http_pools_children(:test) do [hackney_pool, Pleroma.Pool.Supervisor] end - defp http_pools_children(_) do - :tesla - |> Application.get_env(:adapter) - |> http_pools() - end + defp http_pools_children(_), do: http_pools(adapter()) defp http_pools(Tesla.Adapter.Hackney) do pools = [:federation, :media] @@ -227,4 +230,6 @@ defp http_pools(Tesla.Adapter.Hackney) do defp http_pools(Tesla.Adapter.Gun), do: [Pleroma.Pool.Supervisor] defp http_pools(_), do: [] + + defp adapter, do: Application.get_env(:tesla, :adapter) end diff --git a/lib/pleroma/otp_version.ex b/lib/pleroma/otp_version.ex index 0be189304..54ceaff47 100644 --- a/lib/pleroma/otp_version.ex +++ b/lib/pleroma/otp_version.ex @@ -1,63 +1,53 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.OTPVersion do - @type check_status() :: :undefined | {:error, String.t()} | :ok + @type check_status() :: :ok | :undefined | {:error, String.t()} - require Logger - - @spec check_version() :: check_status() - def check_version do + @spec check() :: check_status() + def check do # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version - paths = [ + [ Path.join(:code.root_dir(), "OTP_VERSION"), Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"]) ] - - :tesla - |> Application.get_env(:adapter) - |> get_and_check_version(paths) + |> get_version_from_files() + |> do_check() end - @spec get_and_check_version(module(), [Path.t()]) :: check_status() - def get_and_check_version(Tesla.Adapter.Gun, paths) do + @spec check([Path.t()]) :: check_status() + def check(paths) do paths - |> check_files() - |> check_version() + |> get_version_from_files() + |> do_check() end - def get_and_check_version(_, _), do: :ok + defp get_version_from_files([]), do: nil - defp check_files([]), do: nil - - defp check_files([path | paths]) do + defp get_version_from_files([path | paths]) do if File.exists?(path) do File.read!(path) else - check_files(paths) + get_version_from_files(paths) end end - defp check_version(nil), do: :undefined + defp do_check(nil), do: :undefined - defp check_version(version) do - try do - version = String.replace(version, ~r/\r|\n|\s/, "") + defp do_check(version) do + version = String.replace(version, ~r/\r|\n|\s/, "") - formatted = - version - |> String.split(".") - |> Enum.map(&String.to_integer/1) - |> Enum.take(2) + [major, minor] = + version + |> String.split(".") + |> Enum.map(&String.to_integer/1) + |> Enum.take(2) - with [major, minor] when length(formatted) == 2 <- formatted, - true <- (major == 22 and minor >= 2) or major > 22 do - :ok - else - false -> {:error, version} - _ -> :undefined - end - rescue - _ -> :undefined - catch - _ -> :undefined + if (major == 22 and minor >= 2) or major > 22 do + :ok + else + {:error, version} end end end diff --git a/test/fixtures/warnings/otp_version/error b/test/fixtures/warnings/otp_version/error deleted file mode 100644 index 8fdd954df..000000000 --- a/test/fixtures/warnings/otp_version/error +++ /dev/null @@ -1 +0,0 @@ -22 \ No newline at end of file diff --git a/test/fixtures/warnings/otp_version/undefined b/test/fixtures/warnings/otp_version/undefined deleted file mode 100644 index 66dc9051d..000000000 --- a/test/fixtures/warnings/otp_version/undefined +++ /dev/null @@ -1 +0,0 @@ -undefined \ No newline at end of file diff --git a/test/otp_version_test.exs b/test/otp_version_test.exs index f26b90f61..af278cc72 100644 --- a/test/otp_version_test.exs +++ b/test/otp_version_test.exs @@ -1,58 +1,38 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.OTPVersionTest do use ExUnit.Case, async: true alias Pleroma.OTPVersion - describe "get_and_check_version/2" do + describe "check/1" do test "22.4" do - assert OTPVersion.get_and_check_version(Tesla.Adapter.Gun, [ - "test/fixtures/warnings/otp_version/22.4" - ]) == :ok + assert OTPVersion.check(["test/fixtures/warnings/otp_version/22.4"]) == :ok end test "22.1" do - assert OTPVersion.get_and_check_version(Tesla.Adapter.Gun, [ - "test/fixtures/warnings/otp_version/22.1" - ]) == {:error, "22.1"} + assert OTPVersion.check(["test/fixtures/warnings/otp_version/22.1"]) == {:error, "22.1"} end test "21.1" do - assert OTPVersion.get_and_check_version(Tesla.Adapter.Gun, [ - "test/fixtures/warnings/otp_version/21.1" - ]) == {:error, "21.1"} + assert OTPVersion.check(["test/fixtures/warnings/otp_version/21.1"]) == {:error, "21.1"} end test "23.0" do - assert OTPVersion.get_and_check_version(Tesla.Adapter.Gun, [ - "test/fixtures/warnings/otp_version/23.0" - ]) == :ok - end - - test "undefined" do - assert OTPVersion.get_and_check_version(Tesla.Adapter.Gun, [ - "test/fixtures/warnings/otp_version/undefined" - ]) == :undefined - end - - test "not parsable" do - assert OTPVersion.get_and_check_version(Tesla.Adapter.Gun, [ - "test/fixtures/warnings/otp_version/error" - ]) == :undefined + assert OTPVersion.check(["test/fixtures/warnings/otp_version/23.0"]) == :ok end test "with non existance file" do - assert OTPVersion.get_and_check_version(Tesla.Adapter.Gun, [ + assert OTPVersion.check([ "test/fixtures/warnings/otp_version/non-exising", "test/fixtures/warnings/otp_version/22.4" ]) == :ok end test "empty paths" do - assert OTPVersion.get_and_check_version(Tesla.Adapter.Gun, []) == :undefined - end - - test "another adapter" do - assert OTPVersion.get_and_check_version(Tesla.Adapter.Hackney, []) == :ok + assert OTPVersion.check([]) == :undefined end end end From 097ad10d02598fb6b77f305c10341a13fb57ceee Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:29:51 +0000 Subject: [PATCH 20/73] Apply suggestion to lib/pleroma/pool/connections.ex --- lib/pleroma/pool/connections.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index a444f822f..c5098cd86 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -128,7 +128,7 @@ def handle_call({:checkin, uri}, from, state) do Logger.debug("checkin #{key}") case state.conns[key] do - %{conn: conn, gun_state: gun_state} = current_conn when gun_state == :up -> + %{conn: conn, gun_state: :up} = current_conn -> Logger.debug("reusing conn #{key}") with time <- :os.system_time(:second), From 2c8d80dc0ad594cfe25ebadd9e7a187c95914b34 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:29:57 +0000 Subject: [PATCH 21/73] Apply suggestion to lib/pleroma/pool/connections.ex --- lib/pleroma/pool/connections.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index c5098cd86..c4c5fd66c 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -145,7 +145,7 @@ def handle_call({:checkin, uri}, from, state) do {:reply, conn, state} end - %{gun_state: gun_state} when gun_state == :down -> + %{gun_state: :down} -> {:reply, nil, state} nil -> From a3ad028973154dafad910d4d73d7d4d4822627c1 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:34:36 +0000 Subject: [PATCH 22/73] Apply suggestion to lib/pleroma/http/adapter.ex --- lib/pleroma/http/adapter.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/http/adapter.ex b/lib/pleroma/http/adapter.ex index 6166a3eb4..32046b1d3 100644 --- a/lib/pleroma/http/adapter.ex +++ b/lib/pleroma/http/adapter.ex @@ -57,7 +57,7 @@ def domain_or_ip(host) do {:error, :einval} -> {:domain, :idna.encode(charlist)} - {:ok, ip} when is_tuple(ip) and tuple_size(ip) in [4, 8] -> + {:ok, ip} -> {:ip, ip} end end From df3c59d9280b94cf99571cbbd1b10c334db8e44d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:45:18 +0000 Subject: [PATCH 23/73] Apply suggestion to docs/configuration/cheatsheet.md --- docs/configuration/cheatsheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index f735b19b8..65f37e846 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -416,7 +416,7 @@ It will increase memory usage, but federation would work faster. Advanced settings for workers pools. -There's four pools used: +There are four pools used: * `:federation` for the federation jobs. You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs. From d30ff35d94ff7d8bc07f0221323a75b07641ee8d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:46:53 +0000 Subject: [PATCH 24/73] Apply suggestion to lib/pleroma/http/request_builder.ex --- lib/pleroma/http/request_builder.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 491acd0f9..046741d99 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -35,7 +35,7 @@ def url(request, u), do: %{request | url: u} def headers(request, headers) do headers_list = if Pleroma.Config.get([:http, :send_user_agent]) do - headers ++ [{"user-agent", Pleroma.Application.user_agent()}] + [{"user-agent", Pleroma.Application.user_agent()} | headers] else headers end From 614e3934f9190ff199df087de34146ad5f34c660 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:50:42 +0000 Subject: [PATCH 25/73] Apply suggestion to lib/pleroma/http/http.ex --- lib/pleroma/http/http.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index ad47dc936..5fb468689 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -64,8 +64,8 @@ def request(method, url, body, headers, options) when is_binary(url) do client <- Tesla.client([Tesla.Middleware.FollowRedirects], tesla_adapter()), pid <- Process.whereis(adapter_opts[:pool]) do pool_alive? = - if tesla_adapter() == Tesla.Adapter.Gun do - if pid, do: Process.alive?(pid), else: false + if tesla_adapter() == Tesla.Adapter.Gun && pid do + Process.alive?(pid) else false end From a21a66972f8733de766bc538fe81f2e0ccb57925 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:52:01 +0000 Subject: [PATCH 26/73] Apply suggestion to lib/pleroma/http/http.ex --- lib/pleroma/http/http.ex | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 5fb468689..0235f89ea 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -76,12 +76,7 @@ def request(method, url, body, headers, options) when is_binary(url) do |> Map.put(:env, Pleroma.Config.get([:env])) |> Map.put(:pool_alive?, pool_alive?) - response = - request( - client, - request, - request_opts - ) + response = request(client, request, request_opts) Connection.after_request(adapter_opts) From 7eb65929924af50146d89192c2cf557e3bdbf07f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:53:31 +0000 Subject: [PATCH 27/73] Apply suggestion to lib/pleroma/pool/connections.ex --- lib/pleroma/pool/connections.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index c4c5fd66c..84617815f 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -180,10 +180,10 @@ def handle_info({:gun_up, conn_pid, _protocol}, state) do state = with conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid), {key, conn} <- find_conn(state.conns, conn_pid, conn_key), - {true, key} <- {Process.alive?(conn_pid), key}, - time <- :os.system_time(:second), - last_reference <- time - conn.last_reference, - current_crf <- crf(last_reference, 100, conn.crf) do + {true, key} <- {Process.alive?(conn_pid), key} do + time = :os.system_time(:second) + last_reference = time - conn.last_reference + current_crf = crf(last_reference, 100, conn.crf) put_in(state.conns[key], %{ conn | gun_state: :up, From 151dc4e387cfbb91b7cd85461ce0deb1e5f5fe30 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:53:37 +0000 Subject: [PATCH 28/73] Apply suggestion to lib/pleroma/reverse_proxy/client/tesla.ex --- lib/pleroma/reverse_proxy/client/tesla.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 55a11b4a8..498a905e1 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -16,7 +16,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @impl true def request(method, url, headers, body, opts \\ []) do - _adapter = check_adapter() + check_adapter() with opts <- Keyword.merge(opts, body_as: :chunks, mode: :passive), {:ok, response} <- From 28ed4b41d03c6a137d198b8c67fb081c7ebfbbc6 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 13:05:28 +0300 Subject: [PATCH 29/73] naming for checkin from pool timeout --- config/config.exs | 2 +- docs/configuration/cheatsheet.md | 2 +- lib/pleroma/pool/connections.ex | 3 ++- test/pool/connections_test.exs | 3 +-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/config.exs b/config/config.exs index 7c94a0f26..661dfad20 100644 --- a/config/config.exs +++ b/config/config.exs @@ -607,7 +607,7 @@ prepare: :unnamed config :pleroma, :connections_pool, - receive_connection_timeout: 250, + checkin_timeout: 250, max_connections: 250, retry: 0, retry_timeout: 100, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 65f37e846..ef3cc40e6 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -404,7 +404,7 @@ Advanced settings for connections pool. Pool with opened connections. These conn For big instances it's recommended to increase `config :pleroma, :connections_pool, max_connections: 500` up to 500-1000. It will increase memory usage, but federation would work faster. -* `:receive_connection_timeout` - timeout to receive connection from pool. Default: 250ms. +* `:checkin_timeout` - timeout to checkin connection from pool. Default: 250ms. * `:max_connections` - maximum number of connections in the pool. Default: 250 connections. * `:retry` - number of retries, while `gun` will try to reconnect if connections goes down. Default: 5. * `:retry_timeout` - timeout while `gun` will try to reconnect. Default: 100ms. diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 84617815f..05fa8f7ad 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -34,7 +34,7 @@ def checkin(url, name) def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name) def checkin(%URI{} = uri, name) do - timeout = Config.get([:connections_pool, :receive_connection_timeout], 250) + timeout = Config.get([:connections_pool, :checkin_timeout], 250) GenServer.call( name, @@ -184,6 +184,7 @@ def handle_info({:gun_up, conn_pid, _protocol}, state) do time = :os.system_time(:second) last_reference = time - conn.last_reference current_crf = crf(last_reference, 100, conn.crf) + put_in(state.conns[key], %{ conn | gun_state: :up, diff --git a/test/pool/connections_test.exs b/test/pool/connections_test.exs index 0e7a118ab..a084f31b9 100644 --- a/test/pool/connections_test.exs +++ b/test/pool/connections_test.exs @@ -24,8 +24,7 @@ defmodule Pleroma.Pool.ConnectionsTest do adapter = Application.get_env(:tesla, :adapter) Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) - {:ok, pid} = - Connections.start_link({name, [max_connections: 2, receive_connection_timeout: 1_500]}) + {:ok, pid} = Connections.start_link({name, [max_connections: 2, checkin_timeout: 1_500]}) on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) From 24d1ac125c6ae719b3d119f2ec0079dcd74eadc2 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 13:24:19 +0300 Subject: [PATCH 30/73] hiding raise error logic to otp_version module --- lib/pleroma/application.ex | 23 ++++------------------- lib/pleroma/otp_version.ex | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 9b228d6b9..d0b9c3c41 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -42,6 +42,10 @@ def start(_type, _args) do setup_instrumenters() load_custom_modules() + if adapter() == Tesla.Adapter.Gun do + Pleroma.OTPVersion.check!() + end + # Define workers and child supervisors to be supervised children = [ @@ -66,25 +70,6 @@ def start(_type, _args) do Pleroma.Gopher.Server ] - if adapter() == Tesla.Adapter.Gun do - case Pleroma.OTPVersion.check() do - :ok -> - :ok - - {:error, version} -> - raise " - !!!OTP VERSION WARNING!!! - You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. - " - - :undefined -> - raise " - !!!OTP VERSION WARNING!!! - To support correct handling of unordered certificates chains - OTP version must be > 22.2. - " - end - end - # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options opts = [strategy: :one_for_one, name: Pleroma.Supervisor] diff --git a/lib/pleroma/otp_version.ex b/lib/pleroma/otp_version.ex index 54ceaff47..9ced2d27d 100644 --- a/lib/pleroma/otp_version.ex +++ b/lib/pleroma/otp_version.ex @@ -5,6 +5,26 @@ defmodule Pleroma.OTPVersion do @type check_status() :: :ok | :undefined | {:error, String.t()} + @spec check!() :: :ok | no_return() + def check! do + case check() do + :ok -> + :ok + + {:error, version} -> + raise " + !!!OTP VERSION WARNING!!! + You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. + " + + :undefined -> + raise " + !!!OTP VERSION WARNING!!! + To support correct handling of unordered certificates chains - OTP version must be > 22.2. + " + end + end + @spec check() :: check_status() def check do # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version From d0e4d3ca3b9d8b8ed00d58e9e1c2a05ab561326c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 14:56:49 +0300 Subject: [PATCH 31/73] removing unnecessary with comment in tesla client impovement --- lib/pleroma/pool/connections.ex | 40 +++++++++++------------ lib/pleroma/reverse_proxy/client/tesla.ex | 8 +++-- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 05fa8f7ad..bde3ffd13 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -36,17 +36,16 @@ def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name) def checkin(%URI{} = uri, name) do timeout = Config.get([:connections_pool, :checkin_timeout], 250) - GenServer.call( - name, - {:checkin, uri}, - timeout - ) + GenServer.call(name, {:checkin, uri}, timeout) end @spec alive?(atom()) :: boolean() def alive?(name) do - pid = Process.whereis(name) - if pid, do: Process.alive?(pid), else: false + if pid = Process.whereis(name) do + Process.alive?(pid) + else + false + end end @spec get_state(atom()) :: t() @@ -131,19 +130,20 @@ def handle_call({:checkin, uri}, from, state) do %{conn: conn, gun_state: :up} = current_conn -> Logger.debug("reusing conn #{key}") - with time <- :os.system_time(:second), - last_reference <- time - current_conn.last_reference, - current_crf <- crf(last_reference, 100, current_conn.crf), - state <- - put_in(state.conns[key], %{ - current_conn - | last_reference: time, - crf: current_crf, - conn_state: :active, - used_by: [from | current_conn.used_by] - }) do - {:reply, conn, state} - end + time = :os.system_time(:second) + last_reference = time - current_conn.last_reference + current_crf = crf(last_reference, 100, current_conn.crf) + + state = + put_in(state.conns[key], %{ + current_conn + | last_reference: time, + crf: current_crf, + conn_state: :active, + used_by: [from | current_conn.used_by] + }) + + {:reply, conn, state} %{gun_state: :down} -> {:reply, nil, state} diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 498a905e1..80a0c8972 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -18,8 +18,9 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do def request(method, url, headers, body, opts \\ []) do check_adapter() - with opts <- Keyword.merge(opts, body_as: :chunks, mode: :passive), - {:ok, response} <- + opts = Keyword.merge(opts, body_as: :chunks) + + with {:ok, response} <- Pleroma.HTTP.request( method, url, @@ -40,7 +41,8 @@ def request(method, url, headers, body, opts \\ []) do @impl true @spec stream_body(map()) :: {:ok, binary(), map()} | {:error, atom() | String.t()} | :done def stream_body(%{pid: pid, opts: opts, fin: true}) do - # if connection was sended and there were redirects, we need to close new conn - pid manually + # if connection was reused, but in tesla were redirects, + # tesla returns new opened connection, which must be closed manually if opts[:old_conn], do: Tesla.Adapter.Gun.close(pid) # if there were redirects we need to checkout old conn conn = opts[:old_conn] || opts[:conn] From 05429730e46b8605544637feebd4c409a4e9ed18 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 15:11:48 +0300 Subject: [PATCH 32/73] unnecessary with --- lib/pleroma/http/http.ex | 51 ++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 0235f89ea..f7b0095d7 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -55,33 +55,36 @@ def post(url, body, headers \\ [], options \\ []), @spec request(atom(), Request.url(), String.t(), Request.headers(), keyword()) :: {:ok, Env.t()} | {:error, any()} def request(method, url, body, headers, options) when is_binary(url) do - with uri <- URI.parse(url), - received_adapter_opts <- Keyword.get(options, :adapter, []), - adapter_opts <- Connection.options(uri, received_adapter_opts), - options <- put_in(options[:adapter], adapter_opts), - params <- Keyword.get(options, :params, []), - request <- build_request(method, headers, options, url, body, params), - client <- Tesla.client([Tesla.Middleware.FollowRedirects], tesla_adapter()), - pid <- Process.whereis(adapter_opts[:pool]) do - pool_alive? = - if tesla_adapter() == Tesla.Adapter.Gun && pid do - Process.alive?(pid) - else - false - end + uri = URI.parse(url) + received_adapter_opts = Keyword.get(options, :adapter, []) + adapter_opts = Connection.options(uri, received_adapter_opts) + options = put_in(options[:adapter], adapter_opts) + params = Keyword.get(options, :params, []) + request = build_request(method, headers, options, url, body, params) - request_opts = - adapter_opts - |> Enum.into(%{}) - |> Map.put(:env, Pleroma.Config.get([:env])) - |> Map.put(:pool_alive?, pool_alive?) + adapter = Application.get_env(:tesla, :adapter) + client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter) - response = request(client, request, request_opts) + pid = Process.whereis(adapter_opts[:pool]) - Connection.after_request(adapter_opts) + pool_alive? = + if adapter == Tesla.Adapter.Gun && pid do + Process.alive?(pid) + else + false + end - response - end + request_opts = + adapter_opts + |> Enum.into(%{}) + |> Map.put(:env, Pleroma.Config.get([:env])) + |> Map.put(:pool_alive?, pool_alive?) + + response = request(client, request, request_opts) + + Connection.after_request(adapter_opts) + + response end @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()} @@ -138,6 +141,4 @@ defp build_request(method, headers, options, url, body, params) do |> Builder.add_param(:query, :query, params) |> Builder.convert_to_keyword() end - - defp tesla_adapter, do: Application.get_env(:tesla, :adapter) end From ee8071f0d5a8a53f6a9ae635d6ea57ce8576e21b Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 15:12:09 +0300 Subject: [PATCH 33/73] removing unused method --- lib/pleroma/http/request_builder.ex | 20 -------------------- test/http/request_builder_test.exs | 17 ----------------- 2 files changed, 37 deletions(-) diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 046741d99..5b92ce764 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -49,26 +49,6 @@ def headers(request, headers) do @spec opts(Request.t(), keyword()) :: Request.t() def opts(request, options), do: %{request | opts: options} - # NOTE: isn't used anywhere - @doc """ - Add optional parameters to the request - - """ - @spec add_optional_params(Request.t(), %{optional(atom) => atom}, keyword()) :: map() - def add_optional_params(request, _, []), do: request - - def add_optional_params(request, definitions, [{key, value} | tail]) do - case definitions do - %{^key => location} -> - request - |> add_param(location, key, value) - |> add_optional_params(definitions, tail) - - _ -> - add_optional_params(request, definitions, tail) - end - end - @doc """ Add optional parameters to the request """ diff --git a/test/http/request_builder_test.exs b/test/http/request_builder_test.exs index f87ca11d3..f6eeac6c0 100644 --- a/test/http/request_builder_test.exs +++ b/test/http/request_builder_test.exs @@ -36,23 +36,6 @@ test "send custom user agent" do end end - describe "add_optional_params/3" do - test "don't add if keyword is empty" do - assert RequestBuilder.add_optional_params(%{}, %{}, []) == %{} - end - - test "add query parameter" do - assert RequestBuilder.add_optional_params( - %Request{}, - %{query: :query, body: :body, another: :val}, - [ - {:query, "param1=val1¶m2=val2"}, - {:body, "some body"} - ] - ) == %Request{query: "param1=val1¶m2=val2", body: "some body"} - end - end - describe "add_param/4" do test "add file parameter" do %Request{ From e605e79df9761cef3d9f93c489dd4618c6b70eda Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 15:44:13 +0300 Subject: [PATCH 34/73] simplification of formatting host method case for format_proxy method --- lib/pleroma/gun/conn.ex | 6 ++--- lib/pleroma/http/adapter.ex | 29 +++--------------------- lib/pleroma/http/adapter/gun.ex | 20 +++++++++++++---- test/http/adapter/gun_test.exs | 21 ++++++++++++++++- test/http/adapter_test.exs | 40 +-------------------------------- 5 files changed, 43 insertions(+), 73 deletions(-) diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index ddb9f30b0..a33d75558 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun.Conn do @@ -131,7 +131,7 @@ defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do end defp do_open(%URI{host: host, port: port} = uri, opts) do - {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + host = Pleroma.HTTP.Connection.parse_host(host) with {:ok, conn} <- API.open(host, port, opts), {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do @@ -149,7 +149,7 @@ defp do_open(%URI{host: host, port: port} = uri, opts) do end defp destination_opts(%URI{host: host, port: port}) do - {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + host = Pleroma.HTTP.Connection.parse_host(host) %{host: host, port: port} end diff --git a/lib/pleroma/http/adapter.ex b/lib/pleroma/http/adapter.ex index 32046b1d3..a3b84d8f3 100644 --- a/lib/pleroma/http/adapter.ex +++ b/lib/pleroma/http/adapter.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.Adapter do @@ -8,7 +8,6 @@ defmodule Pleroma.HTTP.Adapter do @type proxy :: {Connection.host(), pos_integer()} | {Connection.proxy_type(), pos_integer()} - @type host_type :: :domain | :ip @callback options(keyword(), URI.t()) :: keyword() @callback after_request(keyword()) :: :ok @@ -29,9 +28,8 @@ def after_request(_opts), do: :ok def format_proxy(nil), do: nil def format_proxy(proxy_url) do - with {:ok, host, port} <- Connection.parse_proxy(proxy_url) do - {host, port} - else + case Connection.parse_proxy(proxy_url) do + {:ok, host, port} -> {host, port} {:ok, type, host, port} -> {type, host, port} _ -> nil end @@ -40,25 +38,4 @@ def format_proxy(proxy_url) do @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() def maybe_add_proxy(opts, nil), do: opts def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) - - @spec domain_or_fallback(String.t()) :: charlist() - def domain_or_fallback(host) do - case domain_or_ip(host) do - {:domain, domain} -> domain - {:ip, _ip} -> to_charlist(host) - end - end - - @spec domain_or_ip(String.t()) :: {host_type(), Connection.host()} - def domain_or_ip(host) do - charlist = to_charlist(host) - - case :inet.parse_address(charlist) do - {:error, :einval} -> - {:domain, :idna.encode(charlist)} - - {:ok, ip} -> - {:ip, ip} - end - end end diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index 908d71898..5e88786bd 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.Adapter.Gun do @@ -42,7 +42,7 @@ def after_request(opts) do end defp add_original(opts, %URI{host: host, port: port}) do - formatted_host = Adapter.domain_or_fallback(host) + formatted_host = format_host(host) Keyword.put(opts, :original, "#{formatted_host}:#{port}") end @@ -57,8 +57,7 @@ defp add_scheme_opts(opts, %URI{scheme: "https", host: host, port: port}) do cacertfile: CAStore.file_path(), depth: 20, reuse_sessions: false, - verify_fun: - {&:ssl_verify_hostname.verify_fun/3, [check_hostname: Adapter.domain_or_fallback(host)]}, + verify_fun: {&:ssl_verify_hostname.verify_fun/3, [check_hostname: format_host(host)]}, log_level: :warning ] ] @@ -139,4 +138,17 @@ defp try_to_get_conn(uri, opts) do opts end end + + @spec format_host(String.t()) :: charlist() + def format_host(host) do + host_charlist = to_charlist(host) + + case :inet.parse_address(host_charlist) do + {:error, :einval} -> + :idna.encode(host_charlist) + + {:ok, _ip} -> + host_charlist + end + end end diff --git a/test/http/adapter/gun_test.exs b/test/http/adapter/gun_test.exs index a8dcbae04..a05471ac6 100644 --- a/test/http/adapter/gun_test.exs +++ b/test/http/adapter/gun_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.Adapter.GunTest do @@ -264,4 +264,23 @@ test "with ipv6" do } = Connections.get_state(:gun_connections) end end + + describe "format_host/1" do + test "with domain" do + assert Gun.format_host("example.com") == 'example.com' + end + + test "with idna domain" do + assert Gun.format_host("ですexample.com") == 'xn--example-183fne.com' + end + + test "with ipv4" do + assert Gun.format_host("127.0.0.1") == '127.0.0.1' + end + + test "with ipv6" do + assert Gun.format_host("2a03:2880:f10c:83:face:b00c:0:25de") == + '2a03:2880:f10c:83:face:b00c:0:25de' + end + end end diff --git a/test/http/adapter_test.exs b/test/http/adapter_test.exs index 37e47dabe..4c805837c 100644 --- a/test/http/adapter_test.exs +++ b/test/http/adapter_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.AdapterTest do @@ -7,44 +7,6 @@ defmodule Pleroma.HTTP.AdapterTest do alias Pleroma.HTTP.Adapter - describe "domain_or_ip/1" do - test "with domain" do - assert Adapter.domain_or_ip("example.com") == {:domain, 'example.com'} - end - - test "with idna domain" do - assert Adapter.domain_or_ip("ですexample.com") == {:domain, 'xn--example-183fne.com'} - end - - test "with ipv4" do - assert Adapter.domain_or_ip("127.0.0.1") == {:ip, {127, 0, 0, 1}} - end - - test "with ipv6" do - assert Adapter.domain_or_ip("2a03:2880:f10c:83:face:b00c:0:25de") == - {:ip, {10_755, 10_368, 61_708, 131, 64_206, 45_068, 0, 9_694}} - end - end - - describe "domain_or_fallback/1" do - test "with domain" do - assert Adapter.domain_or_fallback("example.com") == 'example.com' - end - - test "with idna domain" do - assert Adapter.domain_or_fallback("ですexample.com") == 'xn--example-183fne.com' - end - - test "with ipv4" do - assert Adapter.domain_or_fallback("127.0.0.1") == '127.0.0.1' - end - - test "with ipv6" do - assert Adapter.domain_or_fallback("2a03:2880:f10c:83:face:b00c:0:25de") == - '2a03:2880:f10c:83:face:b00c:0:25de' - end - end - describe "format_proxy/1" do test "with nil" do assert Adapter.format_proxy(nil) == nil From 7d68924e4f7233590457aa7e32a21f082dd0584f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 16:08:21 +0300 Subject: [PATCH 35/73] naming --- lib/pleroma/gun/conn.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index a33d75558..a8b8c92c1 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -54,7 +54,7 @@ def open(%URI{} = uri, name, opts) do if Connections.count(name) < opts[:max_connection] do do_open(uri, opts) else - try_do_open(name, uri, opts) + close_least_used_and_do_open(name, uri, opts) end if is_pid(conn_pid) do @@ -159,7 +159,7 @@ defp add_http2_opts(opts, "https", tls_opts) do defp add_http2_opts(opts, _, _), do: opts - defp try_do_open(name, uri, opts) do + defp close_least_used_and_do_open(name, uri, opts) do Logger.debug("try to open conn #{Connections.compose_uri_log(uri)}") with [{close_key, least_used} | _conns] <- From 8fc00b7cbff86885ec99d01821c403a766202659 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 16:27:46 +0300 Subject: [PATCH 36/73] return error if connection failed to open --- lib/pleroma/gun/conn.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index a8b8c92c1..9ae419092 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -90,7 +90,7 @@ defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do } #{inspect(error)}" ) - nil + error end end @@ -126,7 +126,7 @@ defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do } #{inspect(error)}" ) - nil + error end end @@ -144,7 +144,7 @@ defp do_open(%URI{host: host, port: port} = uri, opts) do }" ) - nil + error end end @@ -169,7 +169,7 @@ defp close_least_used_and_do_open(name, uri, opts) do do_open(uri, opts) else - [] -> nil + [] -> {:error, :pool_overflowed} end end end From 7c0ed9302cb13ab44c1bf18017538315dcd0ce2e Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 16:46:20 +0300 Subject: [PATCH 37/73] unnecessary mock --- test/notification_test.exs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/notification_test.exs b/test/notification_test.exs index 1c60f6866..56a581810 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -649,13 +649,6 @@ test "notifications are deleted if a remote user is deleted" do "object" => remote_user.ap_id } - remote_user_url = remote_user.ap_id - - Tesla.Mock.mock(fn - %{method: :get, url: ^remote_user_url} -> - %Tesla.Env{status: 404, body: ""} - end) - {:ok, _delete_activity} = Transmogrifier.handle_incoming(delete_user_message) ObanHelpers.perform_all() From 6ebf389d6e6ca5f3e56f9b017531f5f7e301ed3c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 16:51:49 +0300 Subject: [PATCH 38/73] poolboy timeout fix --- lib/pleroma/http/http.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index f7b0095d7..4b774472e 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -102,8 +102,8 @@ def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do try do :poolboy.transaction( pool, - &Pleroma.Pool.Request.execute(&1, client, request, timeout + 500), - timeout + 1_000 + &Pleroma.Pool.Request.execute(&1, client, request, timeout), + timeout ) rescue e -> From aaa879ce75a62e69a458226e65bef31b0f2ed08c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 17:27:22 +0300 Subject: [PATCH 39/73] proxy parsing errors --- lib/pleroma/http/connection.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index e2d7afbbd..bdd062929 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -71,15 +71,15 @@ def parse_proxy(proxy) when is_binary(proxy) do else {_, _} -> Logger.warn("parsing port in proxy fail #{inspect(proxy)}") - {:error, :error_parsing_port_in_proxy} + {:error, :invalid_proxy_port} :error -> Logger.warn("parsing port in proxy fail #{inspect(proxy)}") - {:error, :error_parsing_port_in_proxy} + {:error, :invalid_proxy_port} _ -> Logger.warn("parsing proxy fail #{inspect(proxy)}") - {:error, :error_parsing_proxy} + {:error, :invalid_proxy} end end @@ -89,7 +89,7 @@ def parse_proxy(proxy) when is_tuple(proxy) do else _ -> Logger.warn("parsing proxy fail #{inspect(proxy)}") - {:error, :error_parsing_proxy} + {:error, :invalid_proxy} end end From 24bf5c4e89e6f97ed3d53157cead48c04015a51b Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 17:27:56 +0300 Subject: [PATCH 40/73] remove try block from pool request --- lib/pleroma/http/http.ex | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 4b774472e..cc0c39400 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -99,23 +99,11 @@ def request(%Client{} = client, request, %{pool_alive?: false}) do end def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do - try do - :poolboy.transaction( - pool, - &Pleroma.Pool.Request.execute(&1, client, request, timeout), - timeout - ) - rescue - e -> - {:error, e} - catch - :exit, {:timeout, _} -> - Logger.warn("Receive response from pool failed #{request[:url]}") - {:error, :recv_pool_timeout} - - :exit, e -> - {:error, e} - end + :poolboy.transaction( + pool, + &Pleroma.Pool.Request.execute(&1, client, request, timeout), + timeout + ) end @spec request_try(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()} From 3723d723652b747b00fc26054101c15e39a5af18 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 17:32:59 +0300 Subject: [PATCH 41/73] proxy parse tests fix --- test/http/connection_test.exs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/http/connection_test.exs b/test/http/connection_test.exs index 53ccbc9cd..37de11e7a 100644 --- a/test/http/connection_test.exs +++ b/test/http/connection_test.exs @@ -51,31 +51,31 @@ test "as tuple with string host" do describe "parse_proxy/1 errors" do test "ip without port" do capture_log(fn -> - assert Connection.parse_proxy("127.0.0.1") == {:error, :error_parsing_proxy} + assert Connection.parse_proxy("127.0.0.1") == {:error, :invalid_proxy} end) =~ "parsing proxy fail \"127.0.0.1\"" end test "host without port" do capture_log(fn -> - assert Connection.parse_proxy("localhost") == {:error, :error_parsing_proxy} + assert Connection.parse_proxy("localhost") == {:error, :invalid_proxy} end) =~ "parsing proxy fail \"localhost\"" end test "host with bad port" do capture_log(fn -> - assert Connection.parse_proxy("localhost:port") == {:error, :error_parsing_port_in_proxy} + assert Connection.parse_proxy("localhost:port") == {:error, :invalid_proxy_port} end) =~ "parsing port in proxy fail \"localhost:port\"" end test "ip with bad port" do capture_log(fn -> - assert Connection.parse_proxy("127.0.0.1:15.9") == {:error, :error_parsing_port_in_proxy} + assert Connection.parse_proxy("127.0.0.1:15.9") == {:error, :invalid_proxy_port} end) =~ "parsing port in proxy fail \"127.0.0.1:15.9\"" end test "as tuple without port" do capture_log(fn -> - assert Connection.parse_proxy({:socks5, :localhost}) == {:error, :error_parsing_proxy} + assert Connection.parse_proxy({:socks5, :localhost}) == {:error, :invalid_proxy} end) =~ "parsing proxy fail {:socks5, :localhost}" end From 1ad34bfdbaee7d98167dc7dc7be8b65fd5e6c5f1 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 17:44:04 +0300 Subject: [PATCH 42/73] no try block in checkout connection --- lib/pleroma/http/adapter/gun.ex | 53 ++++++--------------------------- 1 file changed, 9 insertions(+), 44 deletions(-) diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index 5e88786bd..30c5c3c16 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -86,56 +86,21 @@ defp maybe_get_conn(adapter_opts, uri, connection_opts) do end defp try_to_get_conn(uri, opts) do - try do - case Connections.checkin(uri, :gun_connections) do - nil -> - Logger.debug( - "Gun connections pool checkin was not successful. Trying to open conn for next request." - ) - - Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end) - opts - - conn when is_pid(conn) -> - Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") - - opts - |> Keyword.put(:conn, conn) - |> Keyword.put(:close_conn, false) - end - rescue - error -> - Logger.warn( - "Gun connections pool checkin caused error #{Connections.compose_uri_log(uri)} #{ - inspect(error) - }" - ) - - opts - catch - # TODO: here must be no timeouts - :exit, {:timeout, {_, operation, [_, {method, _}, _]}} -> - {:message_queue_len, messages_len} = - :gun_connections - |> Process.whereis() - |> Process.info(:message_queue_len) - - Logger.warn( - "Gun connections pool checkin with timeout error for #{operation} #{method} #{ - Connections.compose_uri_log(uri) - }. Messages length: #{messages_len}" + case Connections.checkin(uri, :gun_connections) do + nil -> + Logger.debug( + "Gun connections pool checkin was not successful. Trying to open conn for next request." ) + Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end) opts - :exit, error -> - Logger.warn( - "Gun pool checkin exited with error #{Connections.compose_uri_log(uri)} #{ - inspect(error) - }" - ) + conn when is_pid(conn) -> + Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") opts + |> Keyword.put(:conn, conn) + |> Keyword.put(:close_conn, false) end end From 8854770fc4e9079131a0897d5fb6c0ccccf98bc6 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 18:01:35 +0300 Subject: [PATCH 43/73] retry and retry_timeout settings default change --- config/config.exs | 4 ++-- docs/configuration/cheatsheet.md | 4 ++-- lib/pleroma/gun/conn.ex | 4 ++-- lib/pleroma/http/adapter/gun.ex | 3 ++- lib/pleroma/pool/connections.ex | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/config/config.exs b/config/config.exs index 661dfad20..f0dab24b5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -609,8 +609,8 @@ config :pleroma, :connections_pool, checkin_timeout: 250, max_connections: 250, - retry: 0, - retry_timeout: 100, + retry: 1, + retry_timeout: 1000, await_up_timeout: 5_000 config :pleroma, :pools, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index ef3cc40e6..a39a7436d 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -406,8 +406,8 @@ It will increase memory usage, but federation would work faster. * `:checkin_timeout` - timeout to checkin connection from pool. Default: 250ms. * `:max_connections` - maximum number of connections in the pool. Default: 250 connections. -* `:retry` - number of retries, while `gun` will try to reconnect if connections goes down. Default: 5. -* `:retry_timeout` - timeout while `gun` will try to reconnect. Default: 100ms. +* `:retry` - number of retries, while `gun` will try to reconnect if connections goes down. Default: 1. +* `:retry_timeout` - timeout while `gun` will try to reconnect. Default: 1000ms. * `:await_up_timeout` - timeout while `gun` will wait until connection is up. Default: 5000ms. ### :pools diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 9ae419092..d73bec360 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -42,8 +42,8 @@ def open(%URI{} = uri, name, opts) do opts = opts |> Enum.into(%{}) - |> Map.put_new(:retry, pool_opts[:retry] || 0) - |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100) + |> Map.put_new(:retry, pool_opts[:retry] || 1) + |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000) |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) key = "#{uri.scheme}:#{uri.host}:#{uri.port}" diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index 30c5c3c16..ecf9c5b62 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -15,7 +15,8 @@ defmodule Pleroma.HTTP.Adapter.Gun do connect_timeout: 5_000, domain_lookup_timeout: 5_000, tls_handshake_timeout: 5_000, - retry: 0, + retry: 1, + retry_timeout: 1000, await_up_timeout: 5_000 ] diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index bde3ffd13..0f7a1bfd8 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -219,7 +219,7 @@ def handle_info({:gun_up, conn_pid, _protocol}, state) do @impl true def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do - retries = Config.get([:connections_pool, :retry], 0) + retries = Config.get([:connections_pool, :retry], 1) # we can't get info on this pid, because pid is dead state = with {key, conn} <- find_conn(state.conns, conn_pid), From f98ee730f01de528797e38f27964b69a465662c4 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 18:53:44 +0300 Subject: [PATCH 44/73] adapter renaming to adapter_helper --- .../http/{adapter.ex => adapter_helper.ex} | 2 +- .../http/{adapter => adapter_helper}/gun.ex | 8 +++--- .../{adapter => adapter_helper}/hackney.ex | 6 ++-- lib/pleroma/http/connection.ex | 8 +++--- .../{adapter => adapter_helper}/gun_test.exs | 4 +-- .../hackney_test.exs | 4 +-- test/http/adapter_helper_test.exs | 28 +++++++++++++++++++ test/http/adapter_test.exs | 27 ------------------ 8 files changed, 44 insertions(+), 43 deletions(-) rename lib/pleroma/http/{adapter.ex => adapter_helper.ex} (96%) rename lib/pleroma/http/{adapter => adapter_helper}/gun.ex (94%) rename lib/pleroma/http/{adapter => adapter_helper}/hackney.ex (87%) rename test/http/{adapter => adapter_helper}/gun_test.exs (99%) rename test/http/{adapter => adapter_helper}/hackney_test.exs (93%) create mode 100644 test/http/adapter_helper_test.exs delete mode 100644 test/http/adapter_test.exs diff --git a/lib/pleroma/http/adapter.ex b/lib/pleroma/http/adapter_helper.ex similarity index 96% rename from lib/pleroma/http/adapter.ex rename to lib/pleroma/http/adapter_helper.ex index a3b84d8f3..2c13666ec 100644 --- a/lib/pleroma/http/adapter.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.HTTP.Adapter do +defmodule Pleroma.HTTP.AdapterHelper do alias Pleroma.HTTP.Connection @type proxy :: diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex similarity index 94% rename from lib/pleroma/http/adapter/gun.ex rename to lib/pleroma/http/adapter_helper/gun.ex index ecf9c5b62..b3298ec7f 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -2,10 +2,10 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.HTTP.Adapter.Gun do - @behaviour Pleroma.HTTP.Adapter +defmodule Pleroma.HTTP.AdapterHelper.Gun do + @behaviour Pleroma.HTTP.AdapterHelper - alias Pleroma.HTTP.Adapter + alias Pleroma.HTTP.AdapterHelper require Logger @@ -28,7 +28,7 @@ def options(connection_opts \\ [], %URI{} = uri) do |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) |> add_original(uri) |> add_scheme_opts(uri) - |> Adapter.maybe_add_proxy(Adapter.format_proxy(proxy)) + |> AdapterHelper.maybe_add_proxy(AdapterHelper.format_proxy(proxy)) |> maybe_get_conn(uri, connection_opts) end diff --git a/lib/pleroma/http/adapter/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex similarity index 87% rename from lib/pleroma/http/adapter/hackney.ex rename to lib/pleroma/http/adapter_helper/hackney.ex index 00db30083..a0e161eaa 100644 --- a/lib/pleroma/http/adapter/hackney.ex +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -1,5 +1,5 @@ -defmodule Pleroma.HTTP.Adapter.Hackney do - @behaviour Pleroma.HTTP.Adapter +defmodule Pleroma.HTTP.AdapterHelper.Hackney do + @behaviour Pleroma.HTTP.AdapterHelper @defaults [ connect_timeout: 10_000, @@ -17,7 +17,7 @@ def options(connection_opts \\ [], %URI{} = uri) do |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) |> Keyword.merge(connection_opts) |> add_scheme_opts(uri) - |> Pleroma.HTTP.Adapter.maybe_add_proxy(proxy) + |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy) end defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index bdd062929..dc2761182 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -18,7 +18,7 @@ defmodule Pleroma.HTTP.Connection do require Logger alias Pleroma.Config - alias Pleroma.HTTP.Adapter + alias Pleroma.HTTP.AdapterHelper @doc """ Merge default connection & adapter options with received ones. @@ -50,9 +50,9 @@ def after_request(opts), do: adapter().after_request(opts) defp adapter do case Application.get_env(:tesla, :adapter) do - Tesla.Adapter.Gun -> Adapter.Gun - Tesla.Adapter.Hackney -> Adapter.Hackney - _ -> Adapter + Tesla.Adapter.Gun -> AdapterHelper.Gun + Tesla.Adapter.Hackney -> AdapterHelper.Hackney + _ -> AdapterHelper end end diff --git a/test/http/adapter/gun_test.exs b/test/http/adapter_helper/gun_test.exs similarity index 99% rename from test/http/adapter/gun_test.exs rename to test/http/adapter_helper/gun_test.exs index a05471ac6..bc7e3f0e0 100644 --- a/test/http/adapter/gun_test.exs +++ b/test/http/adapter_helper/gun_test.exs @@ -2,13 +2,13 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.HTTP.Adapter.GunTest do +defmodule Pleroma.HTTP.AdapterHelper.GunTest do use ExUnit.Case, async: true use Pleroma.Tests.Helpers import ExUnit.CaptureLog alias Pleroma.Config alias Pleroma.Gun.Conn - alias Pleroma.HTTP.Adapter.Gun + alias Pleroma.HTTP.AdapterHelper.Gun alias Pleroma.Pool.Connections setup_all do diff --git a/test/http/adapter/hackney_test.exs b/test/http/adapter_helper/hackney_test.exs similarity index 93% rename from test/http/adapter/hackney_test.exs rename to test/http/adapter_helper/hackney_test.exs index 35cb58125..82f5a7883 100644 --- a/test/http/adapter/hackney_test.exs +++ b/test/http/adapter_helper/hackney_test.exs @@ -2,12 +2,12 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.HTTP.Adapter.HackneyTest do +defmodule Pleroma.HTTP.AdapterHelper.HackneyTest do use ExUnit.Case use Pleroma.Tests.Helpers alias Pleroma.Config - alias Pleroma.HTTP.Adapter.Hackney + alias Pleroma.HTTP.AdapterHelper.Hackney setup_all do uri = URI.parse("http://domain.com") diff --git a/test/http/adapter_helper_test.exs b/test/http/adapter_helper_test.exs new file mode 100644 index 000000000..24d501ad5 --- /dev/null +++ b/test/http/adapter_helper_test.exs @@ -0,0 +1,28 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.AdapterHelperTest do + use ExUnit.Case, async: true + + alias Pleroma.HTTP.AdapterHelper + + describe "format_proxy/1" do + test "with nil" do + assert AdapterHelper.format_proxy(nil) == nil + end + + test "with string" do + assert AdapterHelper.format_proxy("127.0.0.1:8123") == {{127, 0, 0, 1}, 8123} + end + + test "localhost with port" do + assert AdapterHelper.format_proxy("localhost:8123") == {'localhost', 8123} + end + + test "tuple" do + assert AdapterHelper.format_proxy({:socks4, :localhost, 9050}) == + {:socks4, 'localhost', 9050} + end + end +end diff --git a/test/http/adapter_test.exs b/test/http/adapter_test.exs deleted file mode 100644 index 4c805837c..000000000 --- a/test/http/adapter_test.exs +++ /dev/null @@ -1,27 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.AdapterTest do - use ExUnit.Case, async: true - - alias Pleroma.HTTP.Adapter - - describe "format_proxy/1" do - test "with nil" do - assert Adapter.format_proxy(nil) == nil - end - - test "with string" do - assert Adapter.format_proxy("127.0.0.1:8123") == {{127, 0, 0, 1}, 8123} - end - - test "localhost with port" do - assert Adapter.format_proxy("localhost:8123") == {'localhost', 8123} - end - - test "tuple" do - assert Adapter.format_proxy({:socks4, :localhost, 9050}) == {:socks4, 'localhost', 9050} - end - end -end From 23f407bf093723344e63eba6a63f5cd58aa7313e Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 18:57:16 +0300 Subject: [PATCH 45/73] don't test gun itself --- test/gun/gun_test.exs | 39 --------------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 test/gun/gun_test.exs diff --git a/test/gun/gun_test.exs b/test/gun/gun_test.exs deleted file mode 100644 index 9f3e0f938..000000000 --- a/test/gun/gun_test.exs +++ /dev/null @@ -1,39 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.GunTest do - use ExUnit.Case - alias Pleroma.Gun - - @moduletag :integration - - test "opens connection and receive response" do - {:ok, conn} = Gun.open('httpbin.org', 443) - assert is_pid(conn) - {:ok, _protocol} = Gun.await_up(conn) - ref = :gun.get(conn, '/get?a=b&c=d') - assert is_reference(ref) - - assert {:response, :nofin, 200, _} = Gun.await(conn, ref) - assert json = receive_response(conn, ref) - - assert %{"args" => %{"a" => "b", "c" => "d"}} = Jason.decode!(json) - - {:ok, pid} = Task.start(fn -> Process.sleep(50) end) - - :ok = :gun.set_owner(conn, pid) - - assert :gun.info(conn).owner == pid - end - - defp receive_response(conn, ref, acc \\ "") do - case Gun.await(conn, ref) do - {:data, :nofin, body} -> - receive_response(conn, ref, acc <> body) - - {:data, :fin, body} -> - acc <> body - end - end -end From 884d9710b209cc9981c7de61d4e95fd26cd83820 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 19:24:14 +0300 Subject: [PATCH 46/73] refactoring for gun api modules --- config/test.exs | 2 +- lib/pleroma/gun/api.ex | 46 ++++++++++----- lib/pleroma/gun/conn.ex | 22 +++---- lib/pleroma/gun/gun.ex | 49 +++++---------- lib/pleroma/pool/connections.ex | 10 ++-- test/http/adapter_helper/gun_test.exs | 2 +- test/http/connection_test.exs | 2 +- test/http_test.exs | 4 +- test/pool/connections_test.exs | 7 +-- test/reverse_proxy/client/tesla_test.exs | 4 +- test/reverse_proxy/reverse_proxy_test.exs | 4 +- .../api/mock.ex => test/support/gun_mock.ex | 59 ++++++++++--------- 12 files changed, 104 insertions(+), 107 deletions(-) rename lib/pleroma/gun/api/mock.ex => test/support/gun_mock.ex (79%) diff --git a/config/test.exs b/config/test.exs index 7cc669c19..bce9dd4aa 100644 --- a/config/test.exs +++ b/config/test.exs @@ -90,7 +90,7 @@ config :pleroma, :modules, runtime_dir: "test/fixtures/modules" -config :pleroma, Pleroma.Gun.API, Pleroma.Gun.API.Mock +config :pleroma, Pleroma.Gun, Pleroma.GunMock config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: true diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex index f79c9f443..76aac5874 100644 --- a/lib/pleroma/gun/api.ex +++ b/lib/pleroma/gun/api.ex @@ -3,27 +3,43 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun.API do - @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} - @callback info(pid()) :: map() - @callback close(pid()) :: :ok - @callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()} - @callback connect(pid(), map()) :: reference() - @callback await(pid(), reference()) :: {:response, :fin, 200, []} - @callback set_owner(pid(), pid()) :: :ok + @behaviour Pleroma.Gun - def open(host, port, opts), do: api().open(host, port, opts) + alias Pleroma.Gun - def info(pid), do: api().info(pid) + @gun_keys [ + :connect_timeout, + :http_opts, + :http2_opts, + :protocols, + :retry, + :retry_timeout, + :trace, + :transport, + :tls_opts, + :tcp_opts, + :socks_opts, + :ws_opts + ] - def close(pid), do: api().close(pid) + @impl Gun + def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun_keys)) - def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout) + @impl Gun + defdelegate info(pid), to: :gun - def connect(pid, opts), do: api().connect(pid, opts) + @impl Gun + defdelegate close(pid), to: :gun - def await(pid, ref), do: api().await(pid, ref) + @impl Gun + defdelegate await_up(pid, timeout \\ 5_000), to: :gun - def set_owner(pid, owner), do: api().set_owner(pid, owner) + @impl Gun + defdelegate connect(pid, opts), to: :gun - defp api, do: Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun) + @impl Gun + defdelegate await(pid, ref), to: :gun + + @impl Gun + defdelegate set_owner(pid, owner), to: :gun end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index d73bec360..319718690 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Gun.Conn do @moduledoc """ Struct for gun connection data """ - alias Pleroma.Gun.API + alias Pleroma.Gun alias Pleroma.Pool.Connections require Logger @@ -65,7 +65,7 @@ def open(%URI{} = uri, name, opts) do last_reference: :os.system_time(:second) } - :ok = API.set_owner(conn_pid, Process.whereis(name)) + :ok = Gun.set_owner(conn_pid, Process.whereis(name)) Connections.add_conn(name, key, conn) end end @@ -77,10 +77,10 @@ defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) with open_opts <- Map.delete(opts, :tls_opts), - {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), - {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]), - stream <- API.connect(conn, connect_opts), - {:response, :fin, 200, _} <- API.await(conn, stream) do + {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts), + {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]), + stream <- Gun.connect(conn, connect_opts), + {:response, :fin, 200, _} <- Gun.await(conn, stream) do conn else error -> @@ -115,8 +115,8 @@ defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do |> Map.put(:protocols, [:socks]) |> Map.put(:socks_opts, socks_opts) - with {:ok, conn} <- API.open(proxy_host, proxy_port, opts), - {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do + with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts), + {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do conn else error -> @@ -133,8 +133,8 @@ defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do defp do_open(%URI{host: host, port: port} = uri, opts) do host = Pleroma.HTTP.Connection.parse_host(host) - with {:ok, conn} <- API.open(host, port, opts), - {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do + with {:ok, conn} <- Gun.open(host, port, opts), + {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do conn else error -> @@ -164,7 +164,7 @@ defp close_least_used_and_do_open(name, uri, opts) do with [{close_key, least_used} | _conns] <- Connections.get_unused_conns(name), - :ok <- Pleroma.Gun.API.close(least_used.conn) do + :ok <- Gun.close(least_used.conn) do Connections.remove_conn(name, close_key) do_open(uri, opts) diff --git a/lib/pleroma/gun/gun.ex b/lib/pleroma/gun/gun.ex index da82983b1..35390bb11 100644 --- a/lib/pleroma/gun/gun.ex +++ b/lib/pleroma/gun/gun.ex @@ -3,46 +3,27 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun do - @behaviour Pleroma.Gun.API + @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} + @callback info(pid()) :: map() + @callback close(pid()) :: :ok + @callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()} + @callback connect(pid(), map()) :: reference() + @callback await(pid(), reference()) :: {:response, :fin, 200, []} + @callback set_owner(pid(), pid()) :: :ok - alias Pleroma.Gun.API + def open(host, port, opts), do: api().open(host, port, opts) - @gun_keys [ - :connect_timeout, - :http_opts, - :http2_opts, - :protocols, - :retry, - :retry_timeout, - :trace, - :transport, - :tls_opts, - :tcp_opts, - :socks_opts, - :ws_opts - ] + def info(pid), do: api().info(pid) - @impl API - def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun_keys)) + def close(pid), do: api().close(pid) - @impl API - defdelegate info(pid), to: :gun + def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout) - @impl API - defdelegate close(pid), to: :gun + def connect(pid, opts), do: api().connect(pid, opts) - @impl API - defdelegate await_up(pid, timeout \\ 5_000), to: :gun + def await(pid, ref), do: api().await(pid, ref) - @impl API - defdelegate connect(pid, opts), to: :gun + def set_owner(pid, owner), do: api().set_owner(pid, owner) - @impl API - defdelegate await(pid, ref), to: :gun - - @spec flush(pid() | reference()) :: :ok - defdelegate flush(pid), to: :gun - - @impl API - defdelegate set_owner(pid, owner), to: :gun + defp api, do: Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API) end diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 0f7a1bfd8..92179fbfc 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Pool.Connections do defstruct conns: %{}, opts: [] - alias Pleroma.Gun.API + alias Pleroma.Gun @spec start_link({atom(), keyword()}) :: {:ok, pid()} def start_link({name, opts}) do @@ -209,7 +209,7 @@ def handle_info({:gun_up, conn_pid, _protocol}, state) do nil -> Logger.debug(":gun_up message for conn which is not found in state") - :ok = API.close(conn_pid) + :ok = Gun.close(conn_pid) state end @@ -226,7 +226,7 @@ def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do {true, key} <- {Process.alive?(conn_pid), key} do if conn.retries == retries do Logger.debug("closing conn if retries is eq #{inspect(conn_pid)}") - :ok = API.close(conn.conn) + :ok = Gun.close(conn.conn) put_in( state.conns, @@ -252,7 +252,7 @@ def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do nil -> Logger.debug(":gun_down message for conn which is not found in state") - :ok = API.close(conn_pid) + :ok = Gun.close(conn_pid) state end @@ -287,7 +287,7 @@ def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do defp compose_key_gun_info(pid) do try do # sometimes :gun.info can raise MatchError, which lead to pool terminate - %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = API.info(pid) + %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) host = case :inet.ntoa(origin_host) do diff --git a/test/http/adapter_helper/gun_test.exs b/test/http/adapter_helper/gun_test.exs index bc7e3f0e0..66ca416d9 100644 --- a/test/http/adapter_helper/gun_test.exs +++ b/test/http/adapter_helper/gun_test.exs @@ -12,7 +12,7 @@ defmodule Pleroma.HTTP.AdapterHelper.GunTest do alias Pleroma.Pool.Connections setup_all do - {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.Gun.API.Mock) + {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.GunMock) :ok end diff --git a/test/http/connection_test.exs b/test/http/connection_test.exs index 37de11e7a..3f32898cb 100644 --- a/test/http/connection_test.exs +++ b/test/http/connection_test.exs @@ -10,7 +10,7 @@ defmodule Pleroma.HTTP.ConnectionTest do alias Pleroma.HTTP.Connection setup_all do - {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.Gun.API.Mock) + {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.GunMock) :ok end diff --git a/test/http_test.exs b/test/http_test.exs index 83c27f6e1..d45d34f32 100644 --- a/test/http_test.exs +++ b/test/http_test.exs @@ -61,8 +61,8 @@ test "returns successfully result" do describe "connection pools" do @describetag :integration - clear_config(Pleroma.Gun.API) do - Pleroma.Config.put(Pleroma.Gun.API, Pleroma.Gun) + clear_config(Pleroma.Gun) do + Pleroma.Config.put(Pleroma.Gun, Pleroma.Gun.API) end test "gun" do diff --git a/test/pool/connections_test.exs b/test/pool/connections_test.exs index a084f31b9..31dd5f6fa 100644 --- a/test/pool/connections_test.exs +++ b/test/pool/connections_test.exs @@ -6,12 +6,11 @@ defmodule Pleroma.Pool.ConnectionsTest do use ExUnit.Case use Pleroma.Tests.Helpers import ExUnit.CaptureLog - alias Pleroma.Gun.API alias Pleroma.Gun.Conn alias Pleroma.Pool.Connections setup_all do - {:ok, _} = Registry.start_link(keys: :unique, name: API.Mock) + {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.GunMock) :ok end @@ -439,8 +438,8 @@ test "remove frequently used and idle", %{name: name} do describe "integration test" do @describetag :integration - clear_config(API) do - Pleroma.Config.put(API, Pleroma.Gun) + clear_config(Pleroma.Gun) do + Pleroma.Config.put(Pleroma.Gun, Pleroma.Gun.API) end test "opens connection and change owner", %{name: name} do diff --git a/test/reverse_proxy/client/tesla_test.exs b/test/reverse_proxy/client/tesla_test.exs index 231271b0d..78bd31530 100644 --- a/test/reverse_proxy/client/tesla_test.exs +++ b/test/reverse_proxy/client/tesla_test.exs @@ -8,8 +8,8 @@ defmodule Pleroma.ReverseProxy.Client.TeslaTest do alias Pleroma.ReverseProxy.Client @moduletag :integration - clear_config_all(Pleroma.Gun.API) do - Pleroma.Config.put(Pleroma.Gun.API, Pleroma.Gun) + clear_config_all(Pleroma.Gun) do + Pleroma.Config.put(Pleroma.Gun, Pleroma.Gun.API) end setup do diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index f61fc02c5..8e72698ee 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -349,8 +349,8 @@ test "with content-disposition header", %{conn: conn} do Pleroma.Config.put(Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.Client.Tesla) end - clear_config(Pleroma.Gun.API) do - Pleroma.Config.put(Pleroma.Gun.API, Pleroma.Gun) + clear_config(Pleroma.Gun) do + Pleroma.Config.put(Pleroma.Gun, Pleroma.Gun.API) end setup do diff --git a/lib/pleroma/gun/api/mock.ex b/test/support/gun_mock.ex similarity index 79% rename from lib/pleroma/gun/api/mock.ex rename to test/support/gun_mock.ex index 6d24b0e69..e13afd08c 100644 --- a/lib/pleroma/gun/api/mock.ex +++ b/test/support/gun_mock.ex @@ -2,16 +2,17 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Gun.API.Mock do - @behaviour Pleroma.Gun.API +defmodule Pleroma.GunMock do + @behaviour Pleroma.Gun - alias Pleroma.Gun.API + alias Pleroma.Gun + alias Pleroma.GunMock - @impl API + @impl Gun def open('some-domain.com', 443, _) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - Registry.register(API.Mock, conn_pid, %{ + Registry.register(GunMock, conn_pid, %{ origin_scheme: "https", origin_host: 'some-domain.com', origin_port: 443 @@ -20,7 +21,7 @@ def open('some-domain.com', 443, _) do {:ok, conn_pid} end - @impl API + @impl Gun def open(ip, port, _) when ip in [{10_755, 10_368, 61_708, 131, 64_206, 45_068, 0, 9_694}, {127, 0, 0, 1}] and port in [80, 443] do @@ -28,7 +29,7 @@ def open(ip, port, _) scheme = if port == 443, do: "https", else: "http" - Registry.register(API.Mock, conn_pid, %{ + Registry.register(GunMock, conn_pid, %{ origin_scheme: scheme, origin_host: ip, origin_port: port @@ -37,7 +38,7 @@ def open(ip, port, _) {:ok, conn_pid} end - @impl API + @impl Gun def open('localhost', 1234, %{ protocols: [:socks], proxy: {:socks5, 'localhost', 1234}, @@ -45,7 +46,7 @@ def open('localhost', 1234, %{ }) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - Registry.register(API.Mock, conn_pid, %{ + Registry.register(GunMock, conn_pid, %{ origin_scheme: "http", origin_host: 'proxy-socks.com', origin_port: 80 @@ -54,7 +55,7 @@ def open('localhost', 1234, %{ {:ok, conn_pid} end - @impl API + @impl Gun def open('localhost', 1234, %{ protocols: [:socks], proxy: {:socks4, 'localhost', 1234}, @@ -69,7 +70,7 @@ def open('localhost', 1234, %{ }) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - Registry.register(API.Mock, conn_pid, %{ + Registry.register(GunMock, conn_pid, %{ origin_scheme: "https", origin_host: 'proxy-socks.com', origin_port: 443 @@ -78,14 +79,14 @@ def open('localhost', 1234, %{ {:ok, conn_pid} end - @impl API + @impl Gun def open('gun-not-up.com', 80, _opts), do: {:error, :timeout} - @impl API + @impl Gun def open('example.com', port, _) when port in [443, 115] do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - Registry.register(API.Mock, conn_pid, %{ + Registry.register(GunMock, conn_pid, %{ origin_scheme: "https", origin_host: 'example.com', origin_port: 443 @@ -94,11 +95,11 @@ def open('example.com', port, _) when port in [443, 115] do {:ok, conn_pid} end - @impl API + @impl Gun def open(domain, 80, _) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - Registry.register(API.Mock, conn_pid, %{ + Registry.register(GunMock, conn_pid, %{ origin_scheme: "http", origin_host: domain, origin_port: 80 @@ -107,48 +108,48 @@ def open(domain, 80, _) do {:ok, conn_pid} end - @impl API + @impl Gun def open({127, 0, 0, 1}, 8123, _) do Task.start_link(fn -> Process.sleep(1_000) end) end - @impl API + @impl Gun def open('localhost', 9050, _) do Task.start_link(fn -> Process.sleep(1_000) end) end - @impl API + @impl Gun def await_up(_pid, _timeout), do: {:ok, :http} - @impl API + @impl Gun def set_owner(_pid, _owner), do: :ok - @impl API + @impl Gun def connect(pid, %{host: _, port: 80}) do ref = make_ref() - Registry.register(API.Mock, ref, pid) + Registry.register(GunMock, ref, pid) ref end - @impl API + @impl Gun def connect(pid, %{host: _, port: 443, protocols: [:http2], transport: :tls}) do ref = make_ref() - Registry.register(API.Mock, ref, pid) + Registry.register(GunMock, ref, pid) ref end - @impl API + @impl Gun def await(pid, ref) do - [{_, ^pid}] = Registry.lookup(API.Mock, ref) + [{_, ^pid}] = Registry.lookup(GunMock, ref) {:response, :fin, 200, []} end - @impl API + @impl Gun def info(pid) do - [{_, info}] = Registry.lookup(API.Mock, pid) + [{_, info}] = Registry.lookup(GunMock, pid) info end - @impl API + @impl Gun def close(_pid), do: :ok end From d9c5ae7c09c7cbf3f4f66e01b7ed69a3d6388916 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 3 Mar 2020 17:16:24 -0600 Subject: [PATCH 47/73] Update Copyrights for gun related files --- lib/pleroma/gun/api.ex | 2 +- lib/pleroma/gun/gun.ex | 2 +- lib/pleroma/http/request.ex | 2 +- lib/pleroma/pool/connections.ex | 2 +- lib/pleroma/pool/pool.ex | 2 +- lib/pleroma/pool/request.ex | 2 +- lib/pleroma/pool/supervisor.ex | 2 +- lib/pleroma/reverse_proxy/client/hackney.ex | 2 +- lib/pleroma/reverse_proxy/client/tesla.ex | 2 +- test/http/adapter_helper/hackney_test.exs | 2 +- test/http/connection_test.exs | 2 +- test/pool/connections_test.exs | 2 +- test/reverse_proxy/client/tesla_test.exs | 2 +- test/support/gun_mock.ex | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex index 76aac5874..f51cd7db8 100644 --- a/lib/pleroma/gun/api.ex +++ b/lib/pleroma/gun/api.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun.API do diff --git a/lib/pleroma/gun/gun.ex b/lib/pleroma/gun/gun.ex index 35390bb11..81855e89e 100644 --- a/lib/pleroma/gun/gun.ex +++ b/lib/pleroma/gun/gun.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun do diff --git a/lib/pleroma/http/request.ex b/lib/pleroma/http/request.ex index 891d88d53..761bd6ccf 100644 --- a/lib/pleroma/http/request.ex +++ b/lib/pleroma/http/request.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.Request do diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 92179fbfc..f1fab2a24 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Pool.Connections do diff --git a/lib/pleroma/pool/pool.ex b/lib/pleroma/pool/pool.ex index a7ae64ce4..21a6fbbc5 100644 --- a/lib/pleroma/pool/pool.ex +++ b/lib/pleroma/pool/pool.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Pool do diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex index 2c3574561..cce309599 100644 --- a/lib/pleroma/pool/request.ex +++ b/lib/pleroma/pool/request.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Pool.Request do diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex index 32be2264d..f436849ac 100644 --- a/lib/pleroma/pool/supervisor.ex +++ b/lib/pleroma/pool/supervisor.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Pool.Supervisor do diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex index e41560ab0..e84118a90 100644 --- a/lib/pleroma/reverse_proxy/client/hackney.ex +++ b/lib/pleroma/reverse_proxy/client/hackney.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client.Hackney do diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 80a0c8972..dbc6b66a3 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client.Tesla do diff --git a/test/http/adapter_helper/hackney_test.exs b/test/http/adapter_helper/hackney_test.exs index 82f5a7883..3306616ef 100644 --- a/test/http/adapter_helper/hackney_test.exs +++ b/test/http/adapter_helper/hackney_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.AdapterHelper.HackneyTest do diff --git a/test/http/connection_test.exs b/test/http/connection_test.exs index 3f32898cb..5c1ecda0b 100644 --- a/test/http/connection_test.exs +++ b/test/http/connection_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.ConnectionTest do diff --git a/test/pool/connections_test.exs b/test/pool/connections_test.exs index 31dd5f6fa..963fae665 100644 --- a/test/pool/connections_test.exs +++ b/test/pool/connections_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Pool.ConnectionsTest do diff --git a/test/reverse_proxy/client/tesla_test.exs b/test/reverse_proxy/client/tesla_test.exs index 78bd31530..c8b0d5842 100644 --- a/test/reverse_proxy/client/tesla_test.exs +++ b/test/reverse_proxy/client/tesla_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client.TeslaTest do diff --git a/test/support/gun_mock.ex b/test/support/gun_mock.ex index e13afd08c..9d664e366 100644 --- a/test/support/gun_mock.ex +++ b/test/support/gun_mock.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.GunMock do From 8d9dee1ba951e81aaa08b4db64b431a7456dae56 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 4 Mar 2020 08:56:36 +0300 Subject: [PATCH 48/73] retry_timeout description change --- docs/configuration/cheatsheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index a39a7436d..85cc6170a 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -407,7 +407,7 @@ It will increase memory usage, but federation would work faster. * `:checkin_timeout` - timeout to checkin connection from pool. Default: 250ms. * `:max_connections` - maximum number of connections in the pool. Default: 250 connections. * `:retry` - number of retries, while `gun` will try to reconnect if connections goes down. Default: 1. -* `:retry_timeout` - timeout while `gun` will try to reconnect. Default: 1000ms. +* `:retry_timeout` - time between retries when gun will try to reconnect in milliseconds. Default: 1000ms. * `:await_up_timeout` - timeout while `gun` will wait until connection is up. Default: 5000ms. ### :pools From 6b2fb9160cd945cdd4b1265c793d1f85d559fccb Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 4 Mar 2020 09:23:42 +0300 Subject: [PATCH 49/73] otp version --- lib/pleroma/application.ex | 20 ++++++++++++- lib/pleroma/otp_version.ex | 61 +++++--------------------------------- test/otp_version_test.exs | 18 ++++++----- 3 files changed, 38 insertions(+), 61 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index d0b9c3c41..c8a0617a5 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -43,7 +43,25 @@ def start(_type, _args) do load_custom_modules() if adapter() == Tesla.Adapter.Gun do - Pleroma.OTPVersion.check!() + if version = Pleroma.OTPVersion.version() do + [major, minor] = + version + |> String.split(".") + |> Enum.map(&String.to_integer/1) + |> Enum.take(2) + + if (major == 22 and minor < 2) or major < 22 do + raise " + !!!OTP VERSION WARNING!!! + You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. + " + end + else + raise " + !!!OTP VERSION WARNING!!! + To support correct handling of unordered certificates chains - OTP version must be > 22.2. + " + end end # Define workers and child supervisors to be supervised diff --git a/lib/pleroma/otp_version.ex b/lib/pleroma/otp_version.ex index 9ced2d27d..114d0054f 100644 --- a/lib/pleroma/otp_version.ex +++ b/lib/pleroma/otp_version.ex @@ -3,71 +3,26 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.OTPVersion do - @type check_status() :: :ok | :undefined | {:error, String.t()} - - @spec check!() :: :ok | no_return() - def check! do - case check() do - :ok -> - :ok - - {:error, version} -> - raise " - !!!OTP VERSION WARNING!!! - You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. - " - - :undefined -> - raise " - !!!OTP VERSION WARNING!!! - To support correct handling of unordered certificates chains - OTP version must be > 22.2. - " - end - end - - @spec check() :: check_status() - def check do + @spec version() :: String.t() | nil + def version do # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version [ Path.join(:code.root_dir(), "OTP_VERSION"), Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"]) ] |> get_version_from_files() - |> do_check() end - @spec check([Path.t()]) :: check_status() - def check(paths) do - paths - |> get_version_from_files() - |> do_check() - end + @spec get_version_from_files([Path.t()]) :: String.t() | nil + def get_version_from_files([]), do: nil - defp get_version_from_files([]), do: nil - - defp get_version_from_files([path | paths]) do + def get_version_from_files([path | paths]) do if File.exists?(path) do - File.read!(path) + path + |> File.read!() + |> String.replace(~r/\r|\n|\s/, "") else get_version_from_files(paths) end end - - defp do_check(nil), do: :undefined - - defp do_check(version) do - version = String.replace(version, ~r/\r|\n|\s/, "") - - [major, minor] = - version - |> String.split(".") - |> Enum.map(&String.to_integer/1) - |> Enum.take(2) - - if (major == 22 and minor >= 2) or major > 22 do - :ok - else - {:error, version} - end - end end diff --git a/test/otp_version_test.exs b/test/otp_version_test.exs index af278cc72..7d2538ec8 100644 --- a/test/otp_version_test.exs +++ b/test/otp_version_test.exs @@ -9,30 +9,34 @@ defmodule Pleroma.OTPVersionTest do describe "check/1" do test "22.4" do - assert OTPVersion.check(["test/fixtures/warnings/otp_version/22.4"]) == :ok + assert OTPVersion.get_version_from_files(["test/fixtures/warnings/otp_version/22.4"]) == + "22.4" end test "22.1" do - assert OTPVersion.check(["test/fixtures/warnings/otp_version/22.1"]) == {:error, "22.1"} + assert OTPVersion.get_version_from_files(["test/fixtures/warnings/otp_version/22.1"]) == + "22.1" end test "21.1" do - assert OTPVersion.check(["test/fixtures/warnings/otp_version/21.1"]) == {:error, "21.1"} + assert OTPVersion.get_version_from_files(["test/fixtures/warnings/otp_version/21.1"]) == + "21.1" end test "23.0" do - assert OTPVersion.check(["test/fixtures/warnings/otp_version/23.0"]) == :ok + assert OTPVersion.get_version_from_files(["test/fixtures/warnings/otp_version/23.0"]) == + "23.0" end test "with non existance file" do - assert OTPVersion.check([ + assert OTPVersion.get_version_from_files([ "test/fixtures/warnings/otp_version/non-exising", "test/fixtures/warnings/otp_version/22.4" - ]) == :ok + ]) == "22.4" end test "empty paths" do - assert OTPVersion.check([]) == :undefined + assert OTPVersion.get_version_from_files([]) == nil end end end From 22d52f5691d985e7daaa955e97e0722f038f6fae Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 4 Mar 2020 09:41:23 +0300 Subject: [PATCH 50/73] same copyright date format --- lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex | 2 +- priv/repo/migrations/20190408123347_create_conversations.exs | 2 +- test/web/activity_pub/mrf/anti_followbot_policy_test.exs | 2 +- test/web/activity_pub/mrf/anti_link_spam_policy_test.exs | 2 +- test/web/activity_pub/mrf/ensure_re_prepended_test.exs | 2 +- test/web/activity_pub/mrf/no_placeholder_text_policy_test.exs | 2 +- test/web/activity_pub/mrf/normalize_markup_test.exs | 2 +- test/web/activity_pub/mrf/object_age_policy_test.exs | 2 +- test/web/activity_pub/mrf/reject_non_public_test.exs | 2 +- test/web/activity_pub/mrf/simple_policy_test.exs | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index b3547ecd4..0270b96ae 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index f67f48ab6..fc3475048 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do diff --git a/priv/repo/migrations/20190408123347_create_conversations.exs b/priv/repo/migrations/20190408123347_create_conversations.exs index d75459e82..3eaa6136c 100644 --- a/priv/repo/migrations/20190408123347_create_conversations.exs +++ b/priv/repo/migrations/20190408123347_create_conversations.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Repo.Migrations.CreateConversations do diff --git a/test/web/activity_pub/mrf/anti_followbot_policy_test.exs b/test/web/activity_pub/mrf/anti_followbot_policy_test.exs index 37a7bfcf7..fca0de7c6 100644 --- a/test/web/activity_pub/mrf/anti_followbot_policy_test.exs +++ b/test/web/activity_pub/mrf/anti_followbot_policy_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicyTest do diff --git a/test/web/activity_pub/mrf/anti_link_spam_policy_test.exs b/test/web/activity_pub/mrf/anti_link_spam_policy_test.exs index b524fdd23..fc0be6f91 100644 --- a/test/web/activity_pub/mrf/anti_link_spam_policy_test.exs +++ b/test/web/activity_pub/mrf/anti_link_spam_policy_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicyTest do diff --git a/test/web/activity_pub/mrf/ensure_re_prepended_test.exs b/test/web/activity_pub/mrf/ensure_re_prepended_test.exs index dbc8b9e80..38ddec5bb 100644 --- a/test/web/activity_pub/mrf/ensure_re_prepended_test.exs +++ b/test/web/activity_pub/mrf/ensure_re_prepended_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrependedTest do diff --git a/test/web/activity_pub/mrf/no_placeholder_text_policy_test.exs b/test/web/activity_pub/mrf/no_placeholder_text_policy_test.exs index 63ed71129..64ea61dd4 100644 --- a/test/web/activity_pub/mrf/no_placeholder_text_policy_test.exs +++ b/test/web/activity_pub/mrf/no_placeholder_text_policy_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicyTest do diff --git a/test/web/activity_pub/mrf/normalize_markup_test.exs b/test/web/activity_pub/mrf/normalize_markup_test.exs index 0207be56b..9b39c45bd 100644 --- a/test/web/activity_pub/mrf/normalize_markup_test.exs +++ b/test/web/activity_pub/mrf/normalize_markup_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkupTest do diff --git a/test/web/activity_pub/mrf/object_age_policy_test.exs b/test/web/activity_pub/mrf/object_age_policy_test.exs index 643609da4..e521fae44 100644 --- a/test/web/activity_pub/mrf/object_age_policy_test.exs +++ b/test/web/activity_pub/mrf/object_age_policy_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicyTest do diff --git a/test/web/activity_pub/mrf/reject_non_public_test.exs b/test/web/activity_pub/mrf/reject_non_public_test.exs index fc1d190bb..5cc68bca8 100644 --- a/test/web/activity_pub/mrf/reject_non_public_test.exs +++ b/test/web/activity_pub/mrf/reject_non_public_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublicTest do diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs index df0f223f8..e825a1514 100644 --- a/test/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/web/activity_pub/mrf/simple_policy_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do From d6bebd4f9c8086dd87c75f3637a5d392a05f2daf Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 4 Mar 2020 18:13:24 +0300 Subject: [PATCH 51/73] moving some logic to tesla adapter - checking original inside gun adapter - flushing streams on max_body error --- lib/pleroma/http/adapter_helper/gun.ex | 17 ++--------------- lib/pleroma/pool/request.ex | 10 ++-------- mix.exs | 2 +- mix.lock | 3 +-- test/http/adapter_helper/gun_test.exs | 7 ------- test/http/connection_test.exs | 1 - 6 files changed, 6 insertions(+), 34 deletions(-) diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index b3298ec7f..5d5870d90 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -26,7 +26,6 @@ def options(connection_opts \\ [], %URI{} = uri) do @defaults |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) - |> add_original(uri) |> add_scheme_opts(uri) |> AdapterHelper.maybe_add_proxy(AdapterHelper.format_proxy(proxy)) |> maybe_get_conn(uri, connection_opts) @@ -42,17 +41,12 @@ def after_request(opts) do :ok end - defp add_original(opts, %URI{host: host, port: port}) do - formatted_host = format_host(host) - - Keyword.put(opts, :original, "#{formatted_host}:#{port}") - end - defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts - defp add_scheme_opts(opts, %URI{scheme: "https", host: host, port: port}) do + defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do adapter_opts = [ certificates_verification: true, + transport: :tls, tls_opts: [ verify: :verify_peer, cacertfile: CAStore.file_path(), @@ -63,13 +57,6 @@ defp add_scheme_opts(opts, %URI{scheme: "https", host: host, port: port}) do ] ] - adapter_opts = - if port != 443 do - Keyword.put(adapter_opts, :transport, :tls) - else - adapter_opts - end - Keyword.merge(opts, adapter_opts) end diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex index cce309599..0f271b3d0 100644 --- a/lib/pleroma/pool/request.ex +++ b/lib/pleroma/pool/request.ex @@ -28,12 +28,7 @@ def handle_call({:execute, client, request}, _from, state) do end @impl true - def handle_info({:gun_data, _conn, stream, _, _}, state) do - # in some cases if we reuse conn and got {:error, :body_too_large} - # gun continues to send messages to this process, - # so we flush messages for this request - :ok = :gun.flush(stream) - + def handle_info({:gun_data, _conn, _stream, _, _}, state) do {:noreply, state} end @@ -49,8 +44,7 @@ def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do end @impl true - def handle_info({:gun_error, _conn, stream, _error}, state) do - :ok = :gun.flush(stream) + def handle_info({:gun_error, _conn, _stream, _error}, state) do {:noreply, state} end diff --git a/mix.exs b/mix.exs index 5c1d89208..43e7e6f63 100644 --- a/mix.exs +++ b/mix.exs @@ -122,7 +122,7 @@ defp deps do # {:tesla, "~> 1.3", override: true}, {:tesla, git: "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git", - ref: "922cc3db13b421763edbea76246b8ea61c38c6fa", + ref: "67436cf003d40370e944462649193706bb22ca35", override: true}, {:castore, "~> 0.1"}, {:cowlib, "~> 2.8", override: true}, diff --git a/mix.lock b/mix.lock index 255b4888b..b5daf50dc 100644 --- a/mix.lock +++ b/mix.lock @@ -102,7 +102,7 @@ "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "e3928e1d2889a308aaf3e42755809ac21cffd77cb58eef01cbfdab4ce2fd1e21"}, "syslog": {:hex, :syslog, "1.0.6", "995970c9aa7feb380ac493302138e308d6e04fd57da95b439a6df5bb3bf75076", [:rebar3], [], "hexpm", "769ddfabd0d2a16f3f9c17eb7509951e0ca4f68363fb26f2ee51a8ec4a49881a"}, "telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm", "4738382e36a0a9a2b6e25d67c960e40e1a2c95560b9f936d8e29de8cd858480f"}, - "tesla": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git", "922cc3db13b421763edbea76246b8ea61c38c6fa", [ref: "922cc3db13b421763edbea76246b8ea61c38c6fa"]}, + "tesla": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git", "67436cf003d40370e944462649193706bb22ca35", [ref: "67436cf003d40370e944462649193706bb22ca35"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "f354efb2400dd7a80fd9eb6c8419068c4f632da4ac47f3d8822d6e33f08bc852"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, "tzdata": {:hex, :tzdata, "0.5.22", "f2ba9105117ee0360eae2eca389783ef7db36d533899b2e84559404dbc77ebb8", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "cd66c8a1e6a9e121d1f538b01bef459334bb4029a1ffb4eeeb5e4eae0337e7b6"}, @@ -112,4 +112,3 @@ "web_push_encryption": {:hex, :web_push_encryption, "0.2.3", "a0ceab85a805a30852f143d22d71c434046fbdbafbc7292e7887cec500826a80", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm", "9315c8f37c108835cf3f8e9157d7a9b8f420a34f402d1b1620a31aed5b93ecdf"}, "websocket_client": {:git, "https://github.com/jeremyong/websocket_client.git", "9a6f65d05ebf2725d62fb19262b21f1805a59fbf", []}, } - diff --git a/test/http/adapter_helper/gun_test.exs b/test/http/adapter_helper/gun_test.exs index 66ca416d9..c1bf909a6 100644 --- a/test/http/adapter_helper/gun_test.exs +++ b/test/http/adapter_helper/gun_test.exs @@ -35,8 +35,6 @@ test "https url with default port" do {&:ssl_verify_hostname.verify_fun/3, [check_hostname: 'example.com']} assert File.exists?(tls_opts[:cacertfile]) - - assert opts[:original] == "example.com:443" end test "https ipv4 with default port" do @@ -46,8 +44,6 @@ test "https ipv4 with default port" do assert opts[:tls_opts][:verify_fun] == {&:ssl_verify_hostname.verify_fun/3, [check_hostname: '127.0.0.1']} - - assert opts[:original] == "127.0.0.1:443" end test "https ipv6 with default port" do @@ -58,8 +54,6 @@ test "https ipv6 with default port" do assert opts[:tls_opts][:verify_fun] == {&:ssl_verify_hostname.verify_fun/3, [check_hostname: '2a03:2880:f10c:83:face:b00c:0:25de']} - - assert opts[:original] == "2a03:2880:f10c:83:face:b00c:0:25de:443" end test "https url with non standart port" do @@ -129,7 +123,6 @@ test "default ssl adapter opts with connection" do assert tls_opts[:depth] == 20 assert tls_opts[:reuse_sessions] == false - assert opts[:original] == "some-domain.com:443" assert opts[:close_conn] == false assert is_pid(opts[:conn]) end diff --git a/test/http/connection_test.exs b/test/http/connection_test.exs index 5c1ecda0b..d4db3798c 100644 --- a/test/http/connection_test.exs +++ b/test/http/connection_test.exs @@ -134,7 +134,6 @@ test "default ssl adapter opts with connection" do assert tls_opts[:depth] == 20 assert tls_opts[:reuse_sessions] == false - assert opts[:original] == "some-domain.com:443" assert opts[:close_conn] == false assert is_pid(opts[:conn]) end From fe47bcde8c20d7c968a7fb20637b4bccc6389691 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 4 Mar 2020 19:44:03 +0300 Subject: [PATCH 52/73] updating tesla ref --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 43e7e6f63..3b1bbbaf2 100644 --- a/mix.exs +++ b/mix.exs @@ -122,7 +122,7 @@ defp deps do # {:tesla, "~> 1.3", override: true}, {:tesla, git: "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git", - ref: "67436cf003d40370e944462649193706bb22ca35", + ref: "61b7503cef33f00834f78ddfafe0d5d9dec2270b", override: true}, {:castore, "~> 0.1"}, {:cowlib, "~> 2.8", override: true}, diff --git a/mix.lock b/mix.lock index b5daf50dc..af53e5c0f 100644 --- a/mix.lock +++ b/mix.lock @@ -102,7 +102,7 @@ "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "e3928e1d2889a308aaf3e42755809ac21cffd77cb58eef01cbfdab4ce2fd1e21"}, "syslog": {:hex, :syslog, "1.0.6", "995970c9aa7feb380ac493302138e308d6e04fd57da95b439a6df5bb3bf75076", [:rebar3], [], "hexpm", "769ddfabd0d2a16f3f9c17eb7509951e0ca4f68363fb26f2ee51a8ec4a49881a"}, "telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm", "4738382e36a0a9a2b6e25d67c960e40e1a2c95560b9f936d8e29de8cd858480f"}, - "tesla": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git", "67436cf003d40370e944462649193706bb22ca35", [ref: "67436cf003d40370e944462649193706bb22ca35"]}, + "tesla": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git", "61b7503cef33f00834f78ddfafe0d5d9dec2270b", [ref: "61b7503cef33f00834f78ddfafe0d5d9dec2270b"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "f354efb2400dd7a80fd9eb6c8419068c4f632da4ac47f3d8822d6e33f08bc852"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, "tzdata": {:hex, :tzdata, "0.5.22", "f2ba9105117ee0360eae2eca389783ef7db36d533899b2e84559404dbc77ebb8", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "cd66c8a1e6a9e121d1f538b01bef459334bb4029a1ffb4eeeb5e4eae0337e7b6"}, From b34bc669b91903a4567f6f527ebe16f9cd7e0ccf Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 4 Mar 2020 20:09:18 +0300 Subject: [PATCH 53/73] adding descriptions --- config/description.exs | 213 +++++++++++++++++++++++++++++++ docs/configuration/cheatsheet.md | 4 +- 2 files changed, 215 insertions(+), 2 deletions(-) diff --git a/config/description.exs b/config/description.exs index 307f8b5bc..531d73145 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2966,5 +2966,218 @@ suggestions: [2] } ] + }, + %{ + group: :pleroma, + key: :connections_pool, + type: :group, + description: "Advanced settings for `gun` connections pool", + children: [ + %{ + key: :checkin_timeout, + type: :integer, + description: "Timeout to checkin connection from pool. Default: 250ms.", + suggestions: [250] + }, + %{ + key: :max_connections, + type: :integer, + description: "Maximum number of connections in the pool. Default: 250 connections.", + suggestions: [250] + }, + %{ + key: :retry, + type: :integer, + description: + "Number of retries, while `gun` will try to reconnect if connection goes down. Default: 1.", + suggestions: [1] + }, + %{ + key: :retry_timeout, + type: :integer, + description: + "Time between retries when `gun` will try to reconnect in milliseconds. Default: 1000ms.", + suggestions: [1000] + }, + %{ + key: :await_up_timeout, + type: :integer, + description: "Timeout while `gun` will wait until connection is up. Default: 5000ms.", + suggestions: [5000] + } + ] + }, + %{ + group: :pleroma, + key: :pools, + type: :group, + description: "Advanced settings for `gun` workers pools", + children: [ + %{ + key: :federation, + type: :keyword, + description: "Settings for federation pool.", + children: [ + %{ + key: :size, + type: :integer, + description: "Number workers in the pool.", + suggestions: [50] + }, + %{ + key: :max_overflow, + type: :integer, + description: "Number of additional workers if pool is under load.", + suggestions: [10] + }, + %{ + key: :timeout, + type: :integer, + description: "Timeout while `gun` will wait for response.", + suggestions: [150_000] + } + ] + }, + %{ + key: :media, + type: :keyword, + description: "Settings for media pool.", + children: [ + %{ + key: :size, + type: :integer, + description: "Number workers in the pool.", + suggestions: [50] + }, + %{ + key: :max_overflow, + type: :integer, + description: "Number of additional workers if pool is under load.", + suggestions: [10] + }, + %{ + key: :timeout, + type: :integer, + description: "Timeout while `gun` will wait for response.", + suggestions: [150_000] + } + ] + }, + %{ + key: :upload, + type: :keyword, + description: "Settings for upload pool.", + children: [ + %{ + key: :size, + type: :integer, + description: "Number workers in the pool.", + suggestions: [25] + }, + %{ + key: :max_overflow, + type: :integer, + description: "Number of additional workers if pool is under load.", + suggestions: [5] + }, + %{ + key: :timeout, + type: :integer, + description: "Timeout while `gun` will wait for response.", + suggestions: [300_000] + } + ] + }, + %{ + key: :default, + type: :keyword, + description: "Settings for default pool.", + children: [ + %{ + key: :size, + type: :integer, + description: "Number workers in the pool.", + suggestions: [10] + }, + %{ + key: :max_overflow, + type: :integer, + description: "Number of additional workers if pool is under load.", + suggestions: [2] + }, + %{ + key: :timeout, + type: :integer, + description: "Timeout while `gun` will wait for response.", + suggestions: [10_000] + } + ] + } + ] + }, + %{ + group: :pleroma, + key: :hackney_pools, + type: :group, + description: "Advanced settings for `hackney` connections pools", + children: [ + %{ + key: :federation, + type: :keyword, + description: "Settings for federation pool.", + children: [ + %{ + key: :max_connections, + type: :integer, + description: "Number workers in the pool.", + suggestions: [50] + }, + %{ + key: :timeout, + type: :integer, + description: "Timeout while `hackney` will wait for response.", + suggestions: [150_000] + } + ] + }, + %{ + key: :media, + type: :keyword, + description: "Settings for media pool.", + children: [ + %{ + key: :max_connections, + type: :integer, + description: "Number workers in the pool.", + suggestions: [50] + }, + %{ + key: :timeout, + type: :integer, + description: "Timeout while `hackney` will wait for response.", + suggestions: [150_000] + } + ] + }, + %{ + key: :upload, + type: :keyword, + description: "Settings for upload pool.", + children: [ + %{ + key: :max_connections, + type: :integer, + description: "Number workers in the pool.", + suggestions: [25] + }, + %{ + key: :timeout, + type: :integer, + description: "Timeout while `hackney` will wait for response.", + suggestions: [300_000] + } + ] + } + ] } ] diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 85cc6170a..833d243e8 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -406,8 +406,8 @@ It will increase memory usage, but federation would work faster. * `:checkin_timeout` - timeout to checkin connection from pool. Default: 250ms. * `:max_connections` - maximum number of connections in the pool. Default: 250 connections. -* `:retry` - number of retries, while `gun` will try to reconnect if connections goes down. Default: 1. -* `:retry_timeout` - time between retries when gun will try to reconnect in milliseconds. Default: 1000ms. +* `:retry` - number of retries, while `gun` will try to reconnect if connection goes down. Default: 1. +* `:retry_timeout` - time between retries when `gun` will try to reconnect in milliseconds. Default: 1000ms. * `:await_up_timeout` - timeout while `gun` will wait until connection is up. Default: 5000ms. ### :pools From eb324467d9c5c761a776ffc98347246c61ad02ae Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 5 Mar 2020 09:51:52 +0300 Subject: [PATCH 54/73] removing try block in getting gun info --- lib/pleroma/pool/connections.ex | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index f1fab2a24..f96c08f21 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -285,20 +285,15 @@ def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do end defp compose_key_gun_info(pid) do - try do - # sometimes :gun.info can raise MatchError, which lead to pool terminate - %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) + %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) - host = - case :inet.ntoa(origin_host) do - {:error, :einval} -> origin_host - ip -> ip - end + host = + case :inet.ntoa(origin_host) do + {:error, :einval} -> origin_host + ip -> ip + end - "#{scheme}:#{host}:#{port}" - rescue - _ -> :error_gun_info - end + "#{scheme}:#{host}:#{port}" end defp find_conn(conns, conn_pid) do From f0753eed0fdddd30e127213c89a118dd2e087dc9 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 5 Mar 2020 17:31:06 +0300 Subject: [PATCH 55/73] removing try block in tesla request added mocks for tests which fail with Tesla.Mock.Error --- lib/pleroma/http/http.ex | 24 +++-------- lib/pleroma/pool/request.ex | 2 +- lib/pleroma/web/push/impl.ex | 2 +- lib/pleroma/web/web_finger/web_finger.ex | 3 +- test/fixtures/users_mock/localhost.json | 41 +++++++++++++++++++ test/notification_test.exs | 20 +++++++++ .../mrf/anti_link_spam_policy_test.exs | 9 ++++ test/web/activity_pub/relay_test.exs | 5 +++ .../notification_controller_test.exs | 13 ++++++ .../views/notification_view_test.exs | 13 ++++++ .../mastodon_api/views/status_view_test.exs | 17 ++++++++ test/web/streamer/streamer_test.exs | 12 ++++++ 12 files changed, 139 insertions(+), 22 deletions(-) create mode 100644 test/fixtures/users_mock/localhost.json diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 7b7c79b64..466a94adc 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -88,15 +88,11 @@ def request(method, url, body, headers, options) when is_binary(url) do end @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()} - def request(%Client{} = client, request, %{env: :test}), do: request_try(client, request) + def request(%Client{} = client, request, %{env: :test}), do: request(client, request) - def request(%Client{} = client, request, %{body_as: :chunks}) do - request_try(client, request) - end + def request(%Client{} = client, request, %{body_as: :chunks}), do: request(client, request) - def request(%Client{} = client, request, %{pool_alive?: false}) do - request_try(client, request) - end + def request(%Client{} = client, request, %{pool_alive?: false}), do: request(client, request) def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do :poolboy.transaction( @@ -106,18 +102,8 @@ def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do ) end - @spec request_try(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()} - def request_try(client, request) do - try do - Tesla.request(client, request) - rescue - e -> - {:error, e} - catch - :exit, e -> - {:error, e} - end - end + @spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()} + def request(client, request), do: Tesla.request(client, request) defp build_request(method, headers, options, url, body, params) do Builder.new() diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex index 0f271b3d0..db7c10c01 100644 --- a/lib/pleroma/pool/request.ex +++ b/lib/pleroma/pool/request.ex @@ -22,7 +22,7 @@ def execute(pid, client, request, timeout) do @impl true def handle_call({:execute, client, request}, _from, state) do - response = Pleroma.HTTP.request_try(client, request) + response = Pleroma.HTTP.request(client, request) {:reply, response, state} end diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index afa510f08..233e55f21 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -32,7 +32,7 @@ def perform( type = Activity.mastodon_notification_type(notif.activity) gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key) avatar_url = User.avatar_url(actor) - object = Object.normalize(activity) + object = Object.normalize(activity) || activity user = User.get_cached_by_id(user_id) direct_conversation_id = Activity.direct_conversation_id(activity, user) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index db567a02e..7ffd0e51b 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -173,7 +173,8 @@ def find_lrdd_template(domain) do get_template_from_xml(body) else _ -> - with {:ok, %{body: body}} <- HTTP.get("https://#{domain}/.well-known/host-meta", []) do + with {:ok, %{body: body, status: status}} when status in 200..299 <- + HTTP.get("https://#{domain}/.well-known/host-meta", []) do get_template_from_xml(body) else e -> {:error, "Can't find LRDD template: #{inspect(e)}"} diff --git a/test/fixtures/users_mock/localhost.json b/test/fixtures/users_mock/localhost.json new file mode 100644 index 000000000..a49935db1 --- /dev/null +++ b/test/fixtures/users_mock/localhost.json @@ -0,0 +1,41 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "http://localhost:4001/schemas/litepub-0.1.jsonld", + { + "@language": "und" + } + ], + "attachment": [], + "endpoints": { + "oauthAuthorizationEndpoint": "http://localhost:4001/oauth/authorize", + "oauthRegistrationEndpoint": "http://localhost:4001/api/v1/apps", + "oauthTokenEndpoint": "http://localhost:4001/oauth/token", + "sharedInbox": "http://localhost:4001/inbox" + }, + "followers": "http://localhost:4001/users/{{nickname}}/followers", + "following": "http://localhost:4001/users/{{nickname}}/following", + "icon": { + "type": "Image", + "url": "http://localhost:4001/media/4e914f5b84e4a259a3f6c2d2edc9ab642f2ab05f3e3d9c52c81fc2d984b3d51e.jpg" + }, + "id": "http://localhost:4001/users/{{nickname}}", + "image": { + "type": "Image", + "url": "http://localhost:4001/media/f739efddefeee49c6e67e947c4811fdc911785c16ae43da4c3684051fbf8da6a.jpg?name=f739efddefeee49c6e67e947c4811fdc911785c16ae43da4c3684051fbf8da6a.jpg" + }, + "inbox": "http://localhost:4001/users/{{nickname}}/inbox", + "manuallyApprovesFollowers": false, + "name": "{{nickname}}", + "outbox": "http://localhost:4001/users/{{nickname}}/outbox", + "preferredUsername": "{{nickname}}", + "publicKey": { + "id": "http://localhost:4001/users/{{nickname}}#main-key", + "owner": "http://localhost:4001/users/{{nickname}}", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5DLtwGXNZElJyxFGfcVc\nXANhaMadj/iYYQwZjOJTV9QsbtiNBeIK54PJrYuU0/0YIdrvS1iqheX5IwXRhcwa\nhm3ZyLz7XeN9st7FBni4BmZMBtMpxAuYuu5p/jbWy13qAiYOhPreCx0wrWgm/lBD\n9mkgaxIxPooBE0S4ZWEJIDIV1Vft3AWcRUyWW1vIBK0uZzs6GYshbQZB952S0yo4\nFzI1hABGHncH8UvuFauh4EZ8tY7/X5I0pGRnDOcRN1dAht5w5yTA+6r5kebiFQjP\nIzN/eCO/a9Flrj9YGW7HDNtjSOH0A31PLRGlJtJO3yK57dnf5ppyCZGfL4emShQo\ncQIDAQAB\n-----END PUBLIC KEY-----\n\n" + }, + "summary": "your friendly neighborhood pleroma developer
I like cute things and distributed systems, and really hate delete and redrafts", + "tag": [], + "type": "Person", + "url": "http://localhost:4001/users/{{nickname}}" +} \ No newline at end of file diff --git a/test/notification_test.exs b/test/notification_test.exs index 56a581810..c71df4e07 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -649,12 +649,20 @@ test "notifications are deleted if a remote user is deleted" do "object" => remote_user.ap_id } + remote_user_url = remote_user.ap_id + + Tesla.Mock.mock(fn + %{method: :get, url: ^remote_user_url} -> + %Tesla.Env{status: 404, body: ""} + end) + {:ok, _delete_activity} = Transmogrifier.handle_incoming(delete_user_message) ObanHelpers.perform_all() assert Enum.empty?(Notification.for_user(local_user)) end + @tag capture_log: true test "move activity generates a notification" do %{ap_id: old_ap_id} = old_user = insert(:user) %{ap_id: new_ap_id} = new_user = insert(:user, also_known_as: [old_ap_id]) @@ -664,6 +672,18 @@ test "move activity generates a notification" do User.follow(follower, old_user) User.follow(other_follower, old_user) + old_user_url = old_user.ap_id + + body = + File.read!("test/fixtures/users_mock/localhost.json") + |> String.replace("{{nickname}}", old_user.nickname) + |> Jason.encode!() + + Tesla.Mock.mock(fn + %{method: :get, url: ^old_user_url} -> + %Tesla.Env{status: 200, body: body} + end) + Pleroma.Web.ActivityPub.ActivityPub.move(old_user, new_user) ObanHelpers.perform_all() diff --git a/test/web/activity_pub/mrf/anti_link_spam_policy_test.exs b/test/web/activity_pub/mrf/anti_link_spam_policy_test.exs index fc0be6f91..1a13699be 100644 --- a/test/web/activity_pub/mrf/anti_link_spam_policy_test.exs +++ b/test/web/activity_pub/mrf/anti_link_spam_policy_test.exs @@ -110,6 +110,15 @@ test "it allows posts with links" do end describe "with unknown actors" do + setup do + Tesla.Mock.mock(fn + %{method: :get, url: "http://invalid.actor"} -> + %Tesla.Env{status: 500, body: ""} + end) + + :ok + end + test "it rejects posts without links" do message = @linkless_message diff --git a/test/web/activity_pub/relay_test.exs b/test/web/activity_pub/relay_test.exs index e3115dcd8..12bf90d90 100644 --- a/test/web/activity_pub/relay_test.exs +++ b/test/web/activity_pub/relay_test.exs @@ -89,6 +89,11 @@ test "returns error when object is unknown" do } ) + Tesla.Mock.mock(fn + %{method: :get, url: "http://mastodon.example.org/eee/99541947525187367"} -> + %Tesla.Env{status: 500, body: ""} + end) + assert capture_log(fn -> assert Relay.publish(activity) == {:error, nil} end) =~ "[error] error: nil" diff --git a/test/web/mastodon_api/controllers/notification_controller_test.exs b/test/web/mastodon_api/controllers/notification_controller_test.exs index d452ddbdd..0f0a060d2 100644 --- a/test/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/web/mastodon_api/controllers/notification_controller_test.exs @@ -407,11 +407,24 @@ test "see notifications after muting user with notifications and with_muted para assert length(json_response(conn, 200)) == 1 end + @tag capture_log: true test "see move notifications with `with_move` parameter" do old_user = insert(:user) new_user = insert(:user, also_known_as: [old_user.ap_id]) %{user: follower, conn: conn} = oauth_access(["read:notifications"]) + old_user_url = old_user.ap_id + + body = + File.read!("test/fixtures/users_mock/localhost.json") + |> String.replace("{{nickname}}", old_user.nickname) + |> Jason.encode!() + + Tesla.Mock.mock(fn + %{method: :get, url: ^old_user_url} -> + %Tesla.Env{status: 200, body: body} + end) + User.follow(follower, old_user) Pleroma.Web.ActivityPub.ActivityPub.move(old_user, new_user) Pleroma.Tests.ObanHelpers.perform_all() diff --git a/test/web/mastodon_api/views/notification_view_test.exs b/test/web/mastodon_api/views/notification_view_test.exs index 4df9c3c03..57e4c8f1e 100644 --- a/test/web/mastodon_api/views/notification_view_test.exs +++ b/test/web/mastodon_api/views/notification_view_test.exs @@ -108,11 +108,24 @@ test "Follow notification" do NotificationView.render("index.json", %{notifications: [notification], for: followed}) end + @tag capture_log: true test "Move notification" do old_user = insert(:user) new_user = insert(:user, also_known_as: [old_user.ap_id]) follower = insert(:user) + old_user_url = old_user.ap_id + + body = + File.read!("test/fixtures/users_mock/localhost.json") + |> String.replace("{{nickname}}", old_user.nickname) + |> Jason.encode!() + + Tesla.Mock.mock(fn + %{method: :get, url: ^old_user_url} -> + %Tesla.Env{status: 200, body: body} + end) + User.follow(follower, old_user) Pleroma.Web.ActivityPub.ActivityPub.move(old_user, new_user) Pleroma.Tests.ObanHelpers.perform_all() diff --git a/test/web/mastodon_api/views/status_view_test.exs b/test/web/mastodon_api/views/status_view_test.exs index 191895c6f..7df72decb 100644 --- a/test/web/mastodon_api/views/status_view_test.exs +++ b/test/web/mastodon_api/views/status_view_test.exs @@ -92,6 +92,23 @@ test "returns a temporary ap_id based user for activities missing db users" do Repo.delete(user) Cachex.clear(:user_cache) + finger_url = + "https://localhost/.well-known/webfinger?resource=acct:#{user.nickname}@localhost" + + Tesla.Mock.mock_global(fn + %{method: :get, url: "http://localhost/.well-known/host-meta"} -> + %Tesla.Env{status: 404, body: ""} + + %{method: :get, url: "https://localhost/.well-known/host-meta"} -> + %Tesla.Env{status: 404, body: ""} + + %{ + method: :get, + url: ^finger_url + } -> + %Tesla.Env{status: 404, body: ""} + end) + %{account: ms_user} = StatusView.render("show.json", activity: activity) assert ms_user.acct == "erroruser@example.com" diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs index 339f99bbf..a04d70f21 100644 --- a/test/web/streamer/streamer_test.exs +++ b/test/web/streamer/streamer_test.exs @@ -122,6 +122,18 @@ test "it doesn't send notify to the 'user:notification' stream' when a domain is test "it sends follow activities to the 'user:notification' stream", %{ user: user } do + user_url = user.ap_id + + body = + File.read!("test/fixtures/users_mock/localhost.json") + |> String.replace("{{nickname}}", user.nickname) + |> Jason.encode!() + + Tesla.Mock.mock_global(fn + %{method: :get, url: ^user_url} -> + %Tesla.Env{status: 200, body: body} + end) + user2 = insert(:user) task = Task.async(fn -> assert_receive {:text, _}, @streamer_timeout end) From 058c9b01ac063f3cca22a653032663916a16a234 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 5 Mar 2020 18:28:04 +0300 Subject: [PATCH 56/73] returning, not needed --- lib/pleroma/web/push/impl.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index 233e55f21..afa510f08 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -32,7 +32,7 @@ def perform( type = Activity.mastodon_notification_type(notif.activity) gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key) avatar_url = User.avatar_url(actor) - object = Object.normalize(activity) || activity + object = Object.normalize(activity) user = User.get_cached_by_id(user_id) direct_conversation_id = Activity.direct_conversation_id(activity, user) From 931111fd5518cb79449cf79ffe29cb774c55d5ff Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 5 Mar 2020 18:57:45 +0300 Subject: [PATCH 57/73] removing integration tests --- test/http_test.exs | 25 -- test/pool/connections_test.exs | 301 ---------------------- test/reverse_proxy/client/tesla_test.exs | 93 ------- test/reverse_proxy/reverse_proxy_test.exs | 41 --- 4 files changed, 460 deletions(-) delete mode 100644 test/reverse_proxy/client/tesla_test.exs diff --git a/test/http_test.exs b/test/http_test.exs index 4aa08afcb..fd254b590 100644 --- a/test/http_test.exs +++ b/test/http_test.exs @@ -58,29 +58,4 @@ test "returns successfully result" do } end end - - describe "connection pools" do - @describetag :integration - clear_config(Pleroma.Gun) do - Pleroma.Config.put(Pleroma.Gun, Pleroma.Gun.API) - end - - test "gun" do - adapter = Application.get_env(:tesla, :adapter) - Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) - - on_exit(fn -> - Application.put_env(:tesla, :adapter, adapter) - end) - - options = [adapter: [pool: :federation]] - - assert {:ok, resp} = HTTP.get("https://httpbin.org/user-agent", [], options) - - assert resp.status == 200 - - state = Pleroma.Pool.Connections.get_state(:gun_connections) - assert state.conns["https:httpbin.org:443"] - end - end end diff --git a/test/pool/connections_test.exs b/test/pool/connections_test.exs index 963fae665..753fd8b0b 100644 --- a/test/pool/connections_test.exs +++ b/test/pool/connections_test.exs @@ -435,307 +435,6 @@ test "remove frequently used and idle", %{name: name} do } = Connections.get_state(name) end - describe "integration test" do - @describetag :integration - - clear_config(Pleroma.Gun) do - Pleroma.Config.put(Pleroma.Gun, Pleroma.Gun.API) - end - - test "opens connection and change owner", %{name: name} do - url = "https://httpbin.org" - :ok = Conn.open(url, name) - conn = Connections.checkin(url, name) - - pid = Process.whereis(name) - - assert :gun.info(conn).owner == pid - end - - test "opens connection and reuse it on next request", %{name: name} do - url = "http://httpbin.org" - :ok = Conn.open(url, name) - Process.sleep(250) - conn = Connections.checkin(url, name) - - assert is_pid(conn) - assert Process.alive?(conn) - - reused_conn = Connections.checkin(url, name) - - assert conn == reused_conn - - %Connections{ - conns: %{ - "http:httpbin.org:80" => %Conn{ - conn: ^conn, - gun_state: :up - } - } - } = Connections.get_state(name) - end - - test "opens ssl connection and reuse it on next request", %{name: name} do - url = "https://httpbin.org" - :ok = Conn.open(url, name) - Process.sleep(1_000) - conn = Connections.checkin(url, name) - - assert is_pid(conn) - assert Process.alive?(conn) - - reused_conn = Connections.checkin(url, name) - - assert conn == reused_conn - - %Connections{ - conns: %{ - "https:httpbin.org:443" => %Conn{ - conn: ^conn, - gun_state: :up - } - } - } = Connections.get_state(name) - end - - test "remove frequently used and idle", %{name: name} do - self = self() - https1 = "https://www.google.com" - https2 = "https://httpbin.org" - - :ok = Conn.open(https1, name) - :ok = Conn.open(https2, name) - Process.sleep(1_500) - conn = Connections.checkin(https1, name) - - for _ <- 1..4 do - Connections.checkin(https2, name) - end - - %Connections{ - conns: %{ - "https:httpbin.org:443" => %Conn{ - conn: _, - gun_state: :up - }, - "https:www.google.com:443" => %Conn{ - conn: _, - gun_state: :up - } - } - } = Connections.get_state(name) - - :ok = Connections.checkout(conn, self, name) - http = "http://httpbin.org" - Process.sleep(1_000) - :ok = Conn.open(http, name) - conn = Connections.checkin(http, name) - - %Connections{ - conns: %{ - "http:httpbin.org:80" => %Conn{ - conn: ^conn, - gun_state: :up - }, - "https:httpbin.org:443" => %Conn{ - conn: _, - gun_state: :up - } - } - } = Connections.get_state(name) - end - - test "remove earlier used and idle", %{name: name} do - self = self() - - https1 = "https://www.google.com" - https2 = "https://httpbin.org" - :ok = Conn.open(https1, name) - :ok = Conn.open(https2, name) - Process.sleep(1_500) - - Connections.checkin(https1, name) - conn = Connections.checkin(https1, name) - - Process.sleep(1_000) - Connections.checkin(https2, name) - Connections.checkin(https2, name) - - %Connections{ - conns: %{ - "https:httpbin.org:443" => %Conn{ - conn: _, - gun_state: :up - }, - "https:www.google.com:443" => %Conn{ - conn: ^conn, - gun_state: :up - } - } - } = Connections.get_state(name) - - :ok = Connections.checkout(conn, self, name) - :ok = Connections.checkout(conn, self, name) - - http = "http://httpbin.org" - :ok = Conn.open(http, name) - Process.sleep(1_000) - - conn = Connections.checkin(http, name) - - %Connections{ - conns: %{ - "http:httpbin.org:80" => %Conn{ - conn: ^conn, - gun_state: :up - }, - "https:httpbin.org:443" => %Conn{ - conn: _, - gun_state: :up - } - } - } = Connections.get_state(name) - end - - test "doesn't open new conn on pool overflow", %{name: name} do - self = self() - - https1 = "https://www.google.com" - https2 = "https://httpbin.org" - :ok = Conn.open(https1, name) - :ok = Conn.open(https2, name) - Process.sleep(1_000) - Connections.checkin(https1, name) - conn1 = Connections.checkin(https1, name) - conn2 = Connections.checkin(https2, name) - - %Connections{ - conns: %{ - "https:httpbin.org:443" => %Conn{ - conn: ^conn2, - gun_state: :up, - conn_state: :active, - used_by: [{^self, _}] - }, - "https:www.google.com:443" => %Conn{ - conn: ^conn1, - gun_state: :up, - conn_state: :active, - used_by: [{^self, _}, {^self, _}] - } - } - } = Connections.get_state(name) - - refute Connections.checkin("http://httpbin.org", name) - - %Connections{ - conns: %{ - "https:httpbin.org:443" => %Conn{ - conn: ^conn2, - gun_state: :up, - conn_state: :active, - used_by: [{^self, _}] - }, - "https:www.google.com:443" => %Conn{ - conn: ^conn1, - gun_state: :up, - conn_state: :active, - used_by: [{^self, _}, {^self, _}] - } - } - } = Connections.get_state(name) - end - - test "get idle connection with the smallest crf", %{ - name: name - } do - self = self() - - https1 = "https://www.google.com" - https2 = "https://httpbin.org" - - :ok = Conn.open(https1, name) - :ok = Conn.open(https2, name) - Process.sleep(1_500) - Connections.checkin(https1, name) - Connections.checkin(https2, name) - Connections.checkin(https1, name) - conn1 = Connections.checkin(https1, name) - conn2 = Connections.checkin(https2, name) - - %Connections{ - conns: %{ - "https:httpbin.org:443" => %Conn{ - conn: ^conn2, - gun_state: :up, - conn_state: :active, - used_by: [{^self, _}, {^self, _}], - crf: crf2 - }, - "https:www.google.com:443" => %Conn{ - conn: ^conn1, - gun_state: :up, - conn_state: :active, - used_by: [{^self, _}, {^self, _}, {^self, _}], - crf: crf1 - } - } - } = Connections.get_state(name) - - assert crf1 > crf2 - - :ok = Connections.checkout(conn1, self, name) - :ok = Connections.checkout(conn1, self, name) - :ok = Connections.checkout(conn1, self, name) - - :ok = Connections.checkout(conn2, self, name) - :ok = Connections.checkout(conn2, self, name) - - %Connections{ - conns: %{ - "https:httpbin.org:443" => %Conn{ - conn: ^conn2, - gun_state: :up, - conn_state: :idle, - used_by: [] - }, - "https:www.google.com:443" => %Conn{ - conn: ^conn1, - gun_state: :up, - conn_state: :idle, - used_by: [] - } - } - } = Connections.get_state(name) - - http = "http://httpbin.org" - :ok = Conn.open(http, name) - Process.sleep(1_000) - conn = Connections.checkin(http, name) - - %Connections{ - conns: %{ - "https:www.google.com:443" => %Conn{ - conn: ^conn1, - gun_state: :up, - conn_state: :idle, - used_by: [], - crf: crf1 - }, - "http:httpbin.org:80" => %Conn{ - conn: ^conn, - gun_state: :up, - conn_state: :active, - used_by: [{^self, _}], - crf: crf - } - } - } = Connections.get_state(name) - - assert crf1 > crf - end - end - describe "with proxy" do test "as ip", %{name: name} do url = "http://proxy-string.com" diff --git a/test/reverse_proxy/client/tesla_test.exs b/test/reverse_proxy/client/tesla_test.exs deleted file mode 100644 index c8b0d5842..000000000 --- a/test/reverse_proxy/client/tesla_test.exs +++ /dev/null @@ -1,93 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.ReverseProxy.Client.TeslaTest do - use ExUnit.Case - use Pleroma.Tests.Helpers - alias Pleroma.ReverseProxy.Client - @moduletag :integration - - clear_config_all(Pleroma.Gun) do - Pleroma.Config.put(Pleroma.Gun, Pleroma.Gun.API) - end - - setup do - Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) - - on_exit(fn -> - Application.put_env(:tesla, :adapter, Tesla.Mock) - end) - end - - test "get response body stream" do - {:ok, status, headers, ref} = - Client.Tesla.request( - :get, - "http://httpbin.org/stream-bytes/10", - [{"accept", "application/octet-stream"}], - "", - [] - ) - - assert status == 200 - assert headers != [] - - {:ok, response, ref} = Client.Tesla.stream_body(ref) - check_ref(ref) - assert is_binary(response) - assert byte_size(response) == 10 - - assert :done == Client.Tesla.stream_body(ref) - assert :ok = Client.Tesla.close(ref) - end - - test "head response" do - {:ok, status, headers} = Client.Tesla.request(:head, "https://httpbin.org/get", [], "") - - assert status == 200 - assert headers != [] - end - - test "get error response" do - {:ok, status, headers, _body} = - Client.Tesla.request( - :get, - "https://httpbin.org/status/500", - [], - "" - ) - - assert status == 500 - assert headers != [] - end - - describe "client error" do - setup do - adapter = Application.get_env(:tesla, :adapter) - Application.put_env(:tesla, :adapter, Tesla.Adapter.Hackney) - - on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) - :ok - end - - test "adapter doesn't support reading body in chunks" do - assert_raise RuntimeError, - "Elixir.Tesla.Adapter.Hackney doesn't support reading body in chunks", - fn -> - Client.Tesla.request( - :get, - "http://httpbin.org/stream-bytes/10", - [{"accept", "application/octet-stream"}], - "" - ) - end - end - end - - defp check_ref(%{pid: pid, stream: stream} = ref) do - assert is_pid(pid) - assert is_reference(stream) - assert ref[:fin] - end -end diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 18aae5a6b..c17ab0f89 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -341,45 +341,4 @@ test "with content-disposition header", %{conn: conn} do assert {"content-disposition", "attachment; filename=\"filename.jpg\""} in conn.resp_headers end end - - describe "tesla client using gun integration" do - @describetag :integration - - clear_config(Pleroma.ReverseProxy.Client) do - Pleroma.Config.put(Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.Client.Tesla) - end - - clear_config(Pleroma.Gun) do - Pleroma.Config.put(Pleroma.Gun, Pleroma.Gun.API) - end - - setup do - adapter = Application.get_env(:tesla, :adapter) - Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) - - on_exit(fn -> - Application.put_env(:tesla, :adapter, adapter) - end) - end - - test "common", %{conn: conn} do - conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") - assert byte_size(conn.resp_body) == 10 - assert conn.state == :chunked - assert conn.status == 200 - end - - test "ssl", %{conn: conn} do - conn = ReverseProxy.call(conn, "https://httpbin.org/stream-bytes/10") - assert byte_size(conn.resp_body) == 10 - assert conn.state == :chunked - assert conn.status == 200 - end - - test "follow redirects", %{conn: conn} do - conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/5") - assert conn.state == :chunked - assert conn.status == 200 - end - end end From 56ff02f2ef56465b14c9670b930d154911cc7470 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 6 Mar 2020 20:23:58 +0300 Subject: [PATCH 58/73] removing GunMock to use Mox --- test/http/adapter_helper/gun_test.exs | 88 ++++++------ test/http/adapter_helper/hackney_test.exs | 8 +- test/http/connection_test.exs | 25 ++-- test/pool/connections_test.exs | 127 ++++++++++++++---- test/support/gun_mock.ex | 155 ---------------------- test/test_helper.exs | 3 + 6 files changed, 172 insertions(+), 234 deletions(-) delete mode 100644 test/support/gun_mock.ex diff --git a/test/http/adapter_helper/gun_test.exs b/test/http/adapter_helper/gun_test.exs index c1bf909a6..b1b34858a 100644 --- a/test/http/adapter_helper/gun_test.exs +++ b/test/http/adapter_helper/gun_test.exs @@ -5,17 +5,29 @@ defmodule Pleroma.HTTP.AdapterHelper.GunTest do use ExUnit.Case, async: true use Pleroma.Tests.Helpers + import ExUnit.CaptureLog + import Mox + alias Pleroma.Config alias Pleroma.Gun.Conn alias Pleroma.HTTP.AdapterHelper.Gun alias Pleroma.Pool.Connections - setup_all do - {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.GunMock) + setup :verify_on_exit! + + defp gun_mock(_) do + gun_mock() :ok end + defp gun_mock do + Pleroma.GunMock + |> expect(:open, fn _, _, _ -> Task.start_link(fn -> Process.sleep(1000) end) end) + |> expect(:await_up, fn _, _ -> {:ok, :http} end) + |> expect(:set_owner, fn _, _ -> :ok end) + end + describe "options/1" do clear_config([:http, :adapter]) do Config.put([:http, :adapter], a: 1, b: 2) @@ -24,23 +36,20 @@ defmodule Pleroma.HTTP.AdapterHelper.GunTest do test "https url with default port" do uri = URI.parse("https://example.com") - opts = Gun.options(uri) + opts = Gun.options([receive_conn: false], uri) assert opts[:certificates_verification] - tls_opts = opts[:tls_opts] - assert tls_opts[:verify] == :verify_peer - assert tls_opts[:depth] == 20 - assert tls_opts[:reuse_sessions] == false + refute opts[:tls_opts] == [] - assert tls_opts[:verify_fun] == + assert opts[:tls_opts][:verify_fun] == {&:ssl_verify_hostname.verify_fun/3, [check_hostname: 'example.com']} - assert File.exists?(tls_opts[:cacertfile]) + assert File.exists?(opts[:tls_opts][:cacertfile]) end test "https ipv4 with default port" do uri = URI.parse("https://127.0.0.1") - opts = Gun.options(uri) + opts = Gun.options([receive_conn: false], uri) assert opts[:tls_opts][:verify_fun] == {&:ssl_verify_hostname.verify_fun/3, [check_hostname: '127.0.0.1']} @@ -49,7 +58,7 @@ test "https ipv4 with default port" do test "https ipv6 with default port" do uri = URI.parse("https://[2a03:2880:f10c:83:face:b00c:0:25de]") - opts = Gun.options(uri) + opts = Gun.options([receive_conn: false], uri) assert opts[:tls_opts][:verify_fun] == {&:ssl_verify_hostname.verify_fun/3, @@ -59,32 +68,14 @@ test "https ipv6 with default port" do test "https url with non standart port" do uri = URI.parse("https://example.com:115") - opts = Gun.options(uri) + opts = Gun.options([receive_conn: false], uri) assert opts[:certificates_verification] assert opts[:transport] == :tls end - test "receive conn by default" do - uri = URI.parse("http://another-domain.com") - :ok = Conn.open(uri, :gun_connections) - - received_opts = Gun.options(uri) - assert received_opts[:close_conn] == false - assert is_pid(received_opts[:conn]) - end - - test "don't receive conn if receive_conn is false" do - uri = URI.parse("http://another-domain2.com") - :ok = Conn.open(uri, :gun_connections) - - opts = [receive_conn: false] - received_opts = Gun.options(opts, uri) - assert received_opts[:close_conn] == nil - assert received_opts[:conn] == nil - end - test "get conn on next request" do + gun_mock() level = Application.get_env(:logger, :level) Logger.configure(level: :debug) on_exit(fn -> Logger.configure(level: level) end) @@ -105,12 +96,13 @@ test "get conn on next request" do end test "merges with defaul http adapter config" do - defaults = Gun.options(URI.parse("https://example.com")) + defaults = Gun.options([receive_conn: false], URI.parse("https://example.com")) assert Keyword.has_key?(defaults, :a) assert Keyword.has_key?(defaults, :b) end test "default ssl adapter opts with connection" do + gun_mock() uri = URI.parse("https://some-domain.com") :ok = Conn.open(uri, :gun_connections) @@ -118,10 +110,7 @@ test "default ssl adapter opts with connection" do opts = Gun.options(uri) assert opts[:certificates_verification] - tls_opts = opts[:tls_opts] - assert tls_opts[:verify] == :verify_peer - assert tls_opts[:depth] == 20 - assert tls_opts[:reuse_sessions] == false + refute opts[:tls_opts] == [] assert opts[:close_conn] == false assert is_pid(opts[:conn]) @@ -158,7 +147,32 @@ test "passed opts have more weight than defaults" do end end + describe "options/1 with receive_conn parameter" do + setup :gun_mock + + test "receive conn by default" do + uri = URI.parse("http://another-domain.com") + :ok = Conn.open(uri, :gun_connections) + + received_opts = Gun.options(uri) + assert received_opts[:close_conn] == false + assert is_pid(received_opts[:conn]) + end + + test "don't receive conn if receive_conn is false" do + uri = URI.parse("http://another-domain.com") + :ok = Conn.open(uri, :gun_connections) + + opts = [receive_conn: false] + received_opts = Gun.options(opts, uri) + assert received_opts[:close_conn] == nil + assert received_opts[:conn] == nil + end + end + describe "after_request/1" do + setup :gun_mock + test "body_as not chunks" do uri = URI.parse("http://some-domain.com") :ok = Conn.open(uri, :gun_connections) @@ -223,7 +237,6 @@ test "with ipv4" do uri = URI.parse("http://127.0.0.1") :ok = Conn.open(uri, :gun_connections) opts = Gun.options(uri) - send(:gun_connections, {:gun_up, opts[:conn], :http}) :ok = Gun.after_request(opts) conn = opts[:conn] @@ -242,7 +255,6 @@ test "with ipv6" do uri = URI.parse("http://[2a03:2880:f10c:83:face:b00c:0:25de]") :ok = Conn.open(uri, :gun_connections) opts = Gun.options(uri) - send(:gun_connections, {:gun_up, opts[:conn], :http}) :ok = Gun.after_request(opts) conn = opts[:conn] diff --git a/test/http/adapter_helper/hackney_test.exs b/test/http/adapter_helper/hackney_test.exs index 3306616ef..5fda075f6 100644 --- a/test/http/adapter_helper/hackney_test.exs +++ b/test/http/adapter_helper/hackney_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.AdapterHelper.HackneyTest do - use ExUnit.Case + use ExUnit.Case, async: true use Pleroma.Tests.Helpers alias Pleroma.Config @@ -20,11 +20,7 @@ defmodule Pleroma.HTTP.AdapterHelper.HackneyTest do end test "add proxy and opts from config", %{uri: uri} do - proxy = Config.get([:http, :proxy_url]) - Config.put([:http, :proxy_url], "localhost:8123") - on_exit(fn -> Config.put([:http, :proxy_url], proxy) end) - - opts = Hackney.options(uri) + opts = Hackney.options([proxy: "localhost:8123"], uri) assert opts[:a] == 1 assert opts[:b] == 2 diff --git a/test/http/connection_test.exs b/test/http/connection_test.exs index d4db3798c..a5ddfd435 100644 --- a/test/http/connection_test.exs +++ b/test/http/connection_test.exs @@ -3,16 +3,16 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.ConnectionTest do - use ExUnit.Case + use ExUnit.Case, async: true use Pleroma.Tests.Helpers + import ExUnit.CaptureLog + import Mox + alias Pleroma.Config alias Pleroma.HTTP.Connection - setup_all do - {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.GunMock) - :ok - end + setup :verify_on_exit! describe "parse_host/1" do test "as atom to charlist" do @@ -123,16 +123,19 @@ test "default ssl adapter opts with connection" do uri = URI.parse("https://some-domain.com") - pid = Process.whereis(:federation) - :ok = Pleroma.Gun.Conn.open(uri, :gun_connections, genserver_pid: pid) + Pleroma.GunMock + |> expect(:open, fn 'some-domain.com', 443, _ -> + Task.start_link(fn -> Process.sleep(1000) end) + end) + |> expect(:await_up, fn _, _ -> {:ok, :http2} end) + |> expect(:set_owner, fn _, _ -> :ok end) + + :ok = Pleroma.Gun.Conn.open(uri, :gun_connections) opts = Connection.options(uri) assert opts[:certificates_verification] - tls_opts = opts[:tls_opts] - assert tls_opts[:verify] == :verify_peer - assert tls_opts[:depth] == 20 - assert tls_opts[:reuse_sessions] == false + refute opts[:tls_opts] == [] assert opts[:close_conn] == false assert is_pid(opts[:conn]) diff --git a/test/pool/connections_test.exs b/test/pool/connections_test.exs index 753fd8b0b..06f32b74e 100644 --- a/test/pool/connections_test.exs +++ b/test/pool/connections_test.exs @@ -3,39 +3,83 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Pool.ConnectionsTest do - use ExUnit.Case + use ExUnit.Case, async: true use Pleroma.Tests.Helpers + import ExUnit.CaptureLog + import Mox + alias Pleroma.Gun.Conn + alias Pleroma.GunMock alias Pleroma.Pool.Connections + setup :verify_on_exit! + setup_all do - {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.GunMock) - :ok - end - - clear_config([:connections_pool, :retry]) do - Pleroma.Config.put([:connections_pool, :retry], 5) - end - - setup do name = :test_connections - adapter = Application.get_env(:tesla, :adapter) - Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) - - {:ok, pid} = Connections.start_link({name, [max_connections: 2, checkin_timeout: 1_500]}) + {:ok, pid} = Connections.start_link({name, [checkin_timeout: 150]}) + {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.GunMock) on_exit(fn -> - Application.put_env(:tesla, :adapter, adapter) - - if Process.alive?(pid) do - GenServer.stop(name) - end + if Process.alive?(pid), do: GenServer.stop(name) end) {:ok, name: name} end + defp open_mock(num \\ 1) do + GunMock + |> expect(:open, num, &start_and_register(&1, &2, &3)) + |> expect(:await_up, num, fn _, _ -> {:ok, :http} end) + |> expect(:set_owner, num, fn _, _ -> :ok end) + end + + defp connect_mock(mock) do + mock + |> expect(:connect, &connect(&1, &2)) + |> expect(:await, &await(&1, &2)) + end + + defp info_mock(mock), do: expect(mock, :info, &info(&1)) + + defp start_and_register('gun-not-up.com', _, _), do: {:error, :timeout} + + defp start_and_register(host, port, _) do + {:ok, pid} = Task.start_link(fn -> Process.sleep(1000) end) + + scheme = + case port do + 443 -> "https" + _ -> "http" + end + + Registry.register(GunMock, pid, %{ + origin_scheme: scheme, + origin_host: host, + origin_port: port + }) + + {:ok, pid} + end + + defp info(pid) do + [{_, info}] = Registry.lookup(GunMock, pid) + info + end + + defp connect(pid, _) do + ref = make_ref() + Registry.register(GunMock, ref, pid) + ref + end + + defp await(pid, ref) do + [{_, ^pid}] = Registry.lookup(GunMock, ref) + {:response, :fin, 200, []} + end + + defp now, do: :os.system_time(:second) + describe "alive?/2" do test "is alive", %{name: name} do assert Connections.alive?(name) @@ -47,6 +91,7 @@ test "returns false if not started" do end test "opens connection and reuse it on next request", %{name: name} do + open_mock() url = "http://some-domain.com" key = "http:some-domain.com:80" refute Connections.checkin(url, name) @@ -112,6 +157,7 @@ test "opens connection and reuse it on next request", %{name: name} do end test "reuse connection for idna domains", %{name: name} do + open_mock() url = "http://ですsome-domain.com" refute Connections.checkin(url, name) @@ -140,6 +186,7 @@ test "reuse connection for idna domains", %{name: name} do end test "reuse for ipv4", %{name: name} do + open_mock() url = "http://127.0.0.1" refute Connections.checkin(url, name) @@ -183,6 +230,7 @@ test "reuse for ipv4", %{name: name} do end test "reuse for ipv6", %{name: name} do + open_mock() url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]" refute Connections.checkin(url, name) @@ -212,6 +260,10 @@ test "reuse for ipv6", %{name: name} do end test "up and down ipv4", %{name: name} do + open_mock() + |> info_mock() + |> allow(self(), name) + self = self() url = "http://127.0.0.1" :ok = Conn.open(url, name) @@ -233,6 +285,11 @@ test "up and down ipv4", %{name: name} do test "up and down ipv6", %{name: name} do self = self() + + open_mock() + |> info_mock() + |> allow(self, name) + url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]" :ok = Conn.open(url, name) conn = Connections.checkin(url, name) @@ -252,6 +309,7 @@ test "up and down ipv6", %{name: name} do end test "reuses connection based on protocol", %{name: name} do + open_mock(2) http_url = "http://some-domain.com" http_key = "http:some-domain.com:80" https_url = "https://some-domain.com" @@ -290,6 +348,7 @@ test "reuses connection based on protocol", %{name: name} do end test "connection can't get up", %{name: name} do + expect(GunMock, :open, &start_and_register(&1, &2, &3)) url = "http://gun-not-up.com" assert capture_log(fn -> @@ -301,6 +360,11 @@ test "connection can't get up", %{name: name} do test "process gun_down message and then gun_up", %{name: name} do self = self() + + open_mock() + |> info_mock() + |> allow(self, name) + url = "http://gun-down-and-up.com" key = "http:gun-down-and-up.com:80" :ok = Conn.open(url, name) @@ -351,6 +415,7 @@ test "process gun_down message and then gun_up", %{name: name} do end test "async processes get same conn for same domain", %{name: name} do + open_mock() url = "http://some-domain.com" :ok = Conn.open(url, name) @@ -383,6 +448,7 @@ test "async processes get same conn for same domain", %{name: name} do end test "remove frequently used and idle", %{name: name} do + open_mock(3) self = self() http_url = "http://some-domain.com" https_url = "https://some-domain.com" @@ -437,6 +503,9 @@ test "remove frequently used and idle", %{name: name} do describe "with proxy" do test "as ip", %{name: name} do + open_mock() + |> connect_mock() + url = "http://proxy-string.com" key = "http:proxy-string.com:80" :ok = Conn.open(url, name, proxy: {{127, 0, 0, 1}, 8123}) @@ -458,6 +527,9 @@ test "as ip", %{name: name} do end test "as host", %{name: name} do + open_mock() + |> connect_mock() + url = "http://proxy-tuple-atom.com" :ok = Conn.open(url, name, proxy: {'localhost', 9050}) conn = Connections.checkin(url, name) @@ -477,6 +549,9 @@ test "as host", %{name: name} do end test "as ip and ssl", %{name: name} do + open_mock() + |> connect_mock() + url = "https://proxy-string.com" :ok = Conn.open(url, name, proxy: {{127, 0, 0, 1}, 8123}) @@ -497,6 +572,9 @@ test "as ip and ssl", %{name: name} do end test "as host and ssl", %{name: name} do + open_mock() + |> connect_mock() + url = "https://proxy-tuple-atom.com" :ok = Conn.open(url, name, proxy: {'localhost', 9050}) conn = Connections.checkin(url, name) @@ -516,6 +594,8 @@ test "as host and ssl", %{name: name} do end test "with socks type", %{name: name} do + open_mock() + url = "http://proxy-socks.com" :ok = Conn.open(url, name, proxy: {:socks5, 'localhost', 1234}) @@ -537,6 +617,7 @@ test "with socks type", %{name: name} do end test "with socks4 type and ssl", %{name: name} do + open_mock() url = "https://proxy-socks.com" :ok = Conn.open(url, name, proxy: {:socks4, 'localhost', 1234}) @@ -667,15 +748,13 @@ test "lower crf and lower reference", %{name: name} do end end - test "count/1", %{name: name} do + test "count/1" do + name = :test_count + {:ok, _} = Connections.start_link({name, [checkin_timeout: 150]}) assert Connections.count(name) == 0 Connections.add_conn(name, "1", %Conn{conn: self()}) assert Connections.count(name) == 1 Connections.remove_conn(name, "1") assert Connections.count(name) == 0 end - - defp now do - :os.system_time(:second) - end end diff --git a/test/support/gun_mock.ex b/test/support/gun_mock.ex deleted file mode 100644 index 9d664e366..000000000 --- a/test/support/gun_mock.ex +++ /dev/null @@ -1,155 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.GunMock do - @behaviour Pleroma.Gun - - alias Pleroma.Gun - alias Pleroma.GunMock - - @impl Gun - def open('some-domain.com', 443, _) do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - Registry.register(GunMock, conn_pid, %{ - origin_scheme: "https", - origin_host: 'some-domain.com', - origin_port: 443 - }) - - {:ok, conn_pid} - end - - @impl Gun - def open(ip, port, _) - when ip in [{10_755, 10_368, 61_708, 131, 64_206, 45_068, 0, 9_694}, {127, 0, 0, 1}] and - port in [80, 443] do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - scheme = if port == 443, do: "https", else: "http" - - Registry.register(GunMock, conn_pid, %{ - origin_scheme: scheme, - origin_host: ip, - origin_port: port - }) - - {:ok, conn_pid} - end - - @impl Gun - def open('localhost', 1234, %{ - protocols: [:socks], - proxy: {:socks5, 'localhost', 1234}, - socks_opts: %{host: 'proxy-socks.com', port: 80, version: 5} - }) do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - Registry.register(GunMock, conn_pid, %{ - origin_scheme: "http", - origin_host: 'proxy-socks.com', - origin_port: 80 - }) - - {:ok, conn_pid} - end - - @impl Gun - def open('localhost', 1234, %{ - protocols: [:socks], - proxy: {:socks4, 'localhost', 1234}, - socks_opts: %{ - host: 'proxy-socks.com', - port: 443, - protocols: [:http2], - tls_opts: [], - transport: :tls, - version: 4 - } - }) do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - Registry.register(GunMock, conn_pid, %{ - origin_scheme: "https", - origin_host: 'proxy-socks.com', - origin_port: 443 - }) - - {:ok, conn_pid} - end - - @impl Gun - def open('gun-not-up.com', 80, _opts), do: {:error, :timeout} - - @impl Gun - def open('example.com', port, _) when port in [443, 115] do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - Registry.register(GunMock, conn_pid, %{ - origin_scheme: "https", - origin_host: 'example.com', - origin_port: 443 - }) - - {:ok, conn_pid} - end - - @impl Gun - def open(domain, 80, _) do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - Registry.register(GunMock, conn_pid, %{ - origin_scheme: "http", - origin_host: domain, - origin_port: 80 - }) - - {:ok, conn_pid} - end - - @impl Gun - def open({127, 0, 0, 1}, 8123, _) do - Task.start_link(fn -> Process.sleep(1_000) end) - end - - @impl Gun - def open('localhost', 9050, _) do - Task.start_link(fn -> Process.sleep(1_000) end) - end - - @impl Gun - def await_up(_pid, _timeout), do: {:ok, :http} - - @impl Gun - def set_owner(_pid, _owner), do: :ok - - @impl Gun - def connect(pid, %{host: _, port: 80}) do - ref = make_ref() - Registry.register(GunMock, ref, pid) - ref - end - - @impl Gun - def connect(pid, %{host: _, port: 443, protocols: [:http2], transport: :tls}) do - ref = make_ref() - Registry.register(GunMock, ref, pid) - ref - end - - @impl Gun - def await(pid, ref) do - [{_, ^pid}] = Registry.lookup(GunMock, ref) - {:response, :fin, 200, []} - end - - @impl Gun - def info(pid) do - [{_, info}] = Registry.lookup(GunMock, pid) - info - end - - @impl Gun - def close(_pid), do: :ok -end diff --git a/test/test_helper.exs b/test/test_helper.exs index 6b91d2b46..ee880e226 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -6,7 +6,10 @@ ExUnit.start(exclude: [:federated | os_exclude]) Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, :manual) + Mox.defmock(Pleroma.ReverseProxy.ClientMock, for: Pleroma.ReverseProxy.Client) +Mox.defmock(Pleroma.GunMock, for: Pleroma.Gun) + {:ok, _} = Application.ensure_all_started(:ex_machina) ExUnit.after_suite(fn _results -> From c93c3096d5ffb2df1493f2b8e3f0627d9a8c5910 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 6 Mar 2020 21:04:18 +0300 Subject: [PATCH 59/73] little refactor --- lib/pleroma/gun/gun.ex | 6 ++++-- lib/pleroma/http/adapter_helper/gun.ex | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/gun/gun.ex b/lib/pleroma/gun/gun.ex index 81855e89e..4043e4880 100644 --- a/lib/pleroma/gun/gun.ex +++ b/lib/pleroma/gun/gun.ex @@ -11,6 +11,10 @@ defmodule Pleroma.Gun do @callback await(pid(), reference()) :: {:response, :fin, 200, []} @callback set_owner(pid(), pid()) :: :ok + @api Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API) + + defp api, do: @api + def open(host, port, opts), do: api().open(host, port, opts) def info(pid), do: api().info(pid) @@ -24,6 +28,4 @@ def connect(pid, opts), do: api().connect(pid, opts) def await(pid, ref), do: api().await(pid, ref) def set_owner(pid, owner), do: api().set_owner(pid, owner) - - defp api, do: Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API) end diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 5d5870d90..9b03f4653 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -5,10 +5,9 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @behaviour Pleroma.HTTP.AdapterHelper - alias Pleroma.HTTP.AdapterHelper - require Logger + alias Pleroma.HTTP.AdapterHelper alias Pleroma.Pool.Connections @defaults [ @@ -22,20 +21,23 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @spec options(keyword(), URI.t()) :: keyword() def options(connection_opts \\ [], %URI{} = uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) + formatted_proxy = + Pleroma.Config.get([:http, :proxy_url], nil) + |> AdapterHelper.format_proxy() + + config_opts = Pleroma.Config.get([:http, :adapter], []) @defaults - |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) + |> Keyword.merge(config_opts) |> add_scheme_opts(uri) - |> AdapterHelper.maybe_add_proxy(AdapterHelper.format_proxy(proxy)) + |> AdapterHelper.maybe_add_proxy(formatted_proxy) |> maybe_get_conn(uri, connection_opts) end @spec after_request(keyword()) :: :ok def after_request(opts) do - with conn when not is_nil(conn) <- opts[:conn], - body_as when body_as != :chunks <- opts[:body_as] do - Connections.checkout(conn, self(), :gun_connections) + if opts[:conn] && opts[:body_as] != :chunks do + Connections.checkout(opts[:conn], self(), :gun_connections) end :ok From 78282dc9839dbd17c4649cd3936bb8f4c8283745 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 6 Mar 2020 21:24:19 +0300 Subject: [PATCH 60/73] little polishing --- lib/pleroma/http/adapter_helper/gun.ex | 4 ++-- lib/pleroma/http/adapter_helper/hackney.ex | 4 +++- lib/pleroma/http/connection.ex | 15 ++++++++------- lib/pleroma/pool/connections.ex | 3 +-- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 9b03f4653..862e851c0 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -5,11 +5,11 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @behaviour Pleroma.HTTP.AdapterHelper - require Logger - alias Pleroma.HTTP.AdapterHelper alias Pleroma.Pool.Connections + require Logger + @defaults [ connect_timeout: 5_000, domain_lookup_timeout: 5_000, diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex index a0e161eaa..d08afae0c 100644 --- a/lib/pleroma/http/adapter_helper/hackney.ex +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -13,8 +13,10 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do def options(connection_opts \\ [], %URI{} = uri) do proxy = Pleroma.Config.get([:http, :proxy_url], nil) + config_opts = Pleroma.Config.get([:http, :adapter], []) + @defaults - |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) + |> Keyword.merge(config_opts) |> Keyword.merge(connection_opts) |> add_scheme_opts(uri) |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 97eec88c1..777e5d4c8 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -6,6 +6,14 @@ defmodule Pleroma.HTTP.Connection do @moduledoc """ Configure Tesla.Client with default and customized adapter options. """ + + alias Pleroma.Config + alias Pleroma.HTTP.AdapterHelper + + require Logger + + @defaults [pool: :federation] + @type ip_address :: ipv4_address() | ipv6_address() @type ipv4_address :: {0..255, 0..255, 0..255, 0..255} @type ipv6_address :: @@ -13,13 +21,6 @@ defmodule Pleroma.HTTP.Connection do @type proxy_type() :: :socks4 | :socks5 @type host() :: charlist() | ip_address() - @defaults [pool: :federation] - - require Logger - - alias Pleroma.Config - alias Pleroma.HTTP.AdapterHelper - @doc """ Merge default connection & adapter options with received ones. """ diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index f96c08f21..7529e9240 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Pool.Connections do use GenServer alias Pleroma.Config + alias Pleroma.Gun require Logger @@ -19,8 +20,6 @@ defmodule Pleroma.Pool.Connections do defstruct conns: %{}, opts: [] - alias Pleroma.Gun - @spec start_link({atom(), keyword()}) :: {:ok, pid()} def start_link({name, opts}) do GenServer.start_link(__MODULE__, opts, name: name) From 14678a7708fb43e60f2f3b610f15d5090616d85c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 7 Mar 2020 10:12:34 +0300 Subject: [PATCH 61/73] using `stub` instead `expect` --- test/http/adapter_helper/gun_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/http/adapter_helper/gun_test.exs b/test/http/adapter_helper/gun_test.exs index b1b34858a..c65b89786 100644 --- a/test/http/adapter_helper/gun_test.exs +++ b/test/http/adapter_helper/gun_test.exs @@ -23,7 +23,7 @@ defp gun_mock(_) do defp gun_mock do Pleroma.GunMock - |> expect(:open, fn _, _, _ -> Task.start_link(fn -> Process.sleep(1000) end) end) + |> stub(:open, fn _, _, _ -> Task.start_link(fn -> Process.sleep(1000) end) end) |> expect(:await_up, fn _, _ -> {:ok, :http} end) |> expect(:set_owner, fn _, _ -> :ok end) end From 9f884a263904c8b243507d35b29da712a31fb444 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 7 Mar 2020 11:01:37 +0300 Subject: [PATCH 62/73] tests changes --- test/http/adapter_helper/gun_test.exs | 4 +- test/http/connection_test.exs | 28 -------------- test/http_test.exs | 2 +- test/reverse_proxy/reverse_proxy_test.exs | 45 ++++++++++++----------- 4 files changed, 27 insertions(+), 52 deletions(-) diff --git a/test/http/adapter_helper/gun_test.exs b/test/http/adapter_helper/gun_test.exs index c65b89786..66622b605 100644 --- a/test/http/adapter_helper/gun_test.exs +++ b/test/http/adapter_helper/gun_test.exs @@ -24,8 +24,8 @@ defp gun_mock(_) do defp gun_mock do Pleroma.GunMock |> stub(:open, fn _, _, _ -> Task.start_link(fn -> Process.sleep(1000) end) end) - |> expect(:await_up, fn _, _ -> {:ok, :http} end) - |> expect(:set_owner, fn _, _ -> :ok end) + |> stub(:await_up, fn _, _ -> {:ok, :http} end) + |> stub(:set_owner, fn _, _ -> :ok end) end describe "options/1" do diff --git a/test/http/connection_test.exs b/test/http/connection_test.exs index a5ddfd435..25a2bac1c 100644 --- a/test/http/connection_test.exs +++ b/test/http/connection_test.exs @@ -7,13 +7,10 @@ defmodule Pleroma.HTTP.ConnectionTest do use Pleroma.Tests.Helpers import ExUnit.CaptureLog - import Mox alias Pleroma.Config alias Pleroma.HTTP.Connection - setup :verify_on_exit! - describe "parse_host/1" do test "as atom to charlist" do assert Connection.parse_host(:localhost) == 'localhost' @@ -115,30 +112,5 @@ test "passed opts have more weight than defaults" do assert opts[:proxy] == {'example.com', 4321} end - - test "default ssl adapter opts with connection" do - adapter = Application.get_env(:tesla, :adapter) - Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) - on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) - - uri = URI.parse("https://some-domain.com") - - Pleroma.GunMock - |> expect(:open, fn 'some-domain.com', 443, _ -> - Task.start_link(fn -> Process.sleep(1000) end) - end) - |> expect(:await_up, fn _, _ -> {:ok, :http2} end) - |> expect(:set_owner, fn _, _ -> :ok end) - - :ok = Pleroma.Gun.Conn.open(uri, :gun_connections) - - opts = Connection.options(uri) - - assert opts[:certificates_verification] - refute opts[:tls_opts] == [] - - assert opts[:close_conn] == false - assert is_pid(opts[:conn]) - end end end diff --git a/test/http_test.exs b/test/http_test.exs index fd254b590..618485b55 100644 --- a/test/http_test.exs +++ b/test/http_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTPTest do - use ExUnit.Case + use ExUnit.Case, async: true use Pleroma.Tests.Helpers import Tesla.Mock alias Pleroma.HTTP diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index c17ab0f89..abdfddcb7 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -3,14 +3,17 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxyTest do - use Pleroma.Web.ConnCase + use Pleroma.Web.ConnCase, async: true + import ExUnit.CaptureLog import Mox + alias Pleroma.ReverseProxy alias Pleroma.ReverseProxy.ClientMock + alias Plug.Conn setup_all do - {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.ReverseProxy.ClientMock) + {:ok, _} = Registry.start_link(keys: :unique, name: ClientMock) :ok end @@ -21,7 +24,7 @@ defp user_agent_mock(user_agent, invokes) do ClientMock |> expect(:request, fn :get, url, _, _, _ -> - Registry.register(Pleroma.ReverseProxy.ClientMock, url, 0) + Registry.register(ClientMock, url, 0) {:ok, 200, [ @@ -30,13 +33,13 @@ defp user_agent_mock(user_agent, invokes) do ], %{url: url}} end) |> expect(:stream_body, invokes, fn %{url: url} = client -> - case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do + case Registry.lookup(ClientMock, url) do [{_, 0}] -> - Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) + Registry.update_value(ClientMock, url, &(&1 + 1)) {:ok, json, client} [{_, 1}] -> - Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) + Registry.unregister(ClientMock, url) :done end end) @@ -81,7 +84,7 @@ test "closed connection", %{conn: conn} do defp stream_mock(invokes, with_close? \\ false) do ClientMock |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ -> - Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0) + Registry.register(ClientMock, "/stream-bytes/" <> length, 0) {:ok, 200, [{"content-type", "application/octet-stream"}], %{url: "/stream-bytes/" <> length}} @@ -89,10 +92,10 @@ defp stream_mock(invokes, with_close? \\ false) do |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} = client -> max = String.to_integer(length) - case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do + case Registry.lookup(ClientMock, "/stream-bytes/" <> length) do [{_, current}] when current < max -> Registry.update_value( - Pleroma.ReverseProxy.ClientMock, + ClientMock, "/stream-bytes/" <> length, &(&1 + 10) ) @@ -100,7 +103,7 @@ defp stream_mock(invokes, with_close? \\ false) do {:ok, "0123456789", client} [{_, ^max}] -> - Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) + Registry.unregister(ClientMock, "/stream-bytes/" <> length) :done end end) @@ -214,24 +217,24 @@ test "streaming", %{conn: conn} do conn = ReverseProxy.call(conn, "/stream-bytes/200") assert conn.state == :chunked assert byte_size(conn.resp_body) == 200 - assert Plug.Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"] + assert Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"] end defp headers_mock(_) do ClientMock |> expect(:request, fn :get, "/headers", headers, _, _ -> - Registry.register(Pleroma.ReverseProxy.ClientMock, "/headers", 0) + Registry.register(ClientMock, "/headers", 0) {:ok, 200, [{"content-type", "application/json"}], %{url: "/headers", headers: headers}} end) |> expect(:stream_body, 2, fn %{url: url, headers: headers} = client -> - case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do + case Registry.lookup(ClientMock, url) do [{_, 0}] -> - Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) + Registry.update_value(ClientMock, url, &(&1 + 1)) headers = for {k, v} <- headers, into: %{}, do: {String.capitalize(k), v} {:ok, Jason.encode!(%{headers: headers}), client} [{_, 1}] -> - Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) + Registry.unregister(ClientMock, url) :done end end) @@ -244,7 +247,7 @@ defp headers_mock(_) do test "header passes", %{conn: conn} do conn = - Plug.Conn.put_req_header( + Conn.put_req_header( conn, "accept", "text/html" @@ -257,7 +260,7 @@ test "header passes", %{conn: conn} do test "header is filtered", %{conn: conn} do conn = - Plug.Conn.put_req_header( + Conn.put_req_header( conn, "accept-language", "en-US" @@ -301,18 +304,18 @@ test "add cache-control", %{conn: conn} do defp disposition_headers_mock(headers) do ClientMock |> expect(:request, fn :get, "/disposition", _, _, _ -> - Registry.register(Pleroma.ReverseProxy.ClientMock, "/disposition", 0) + Registry.register(ClientMock, "/disposition", 0) {:ok, 200, headers, %{url: "/disposition"}} end) |> expect(:stream_body, 2, fn %{url: "/disposition"} = client -> - case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/disposition") do + case Registry.lookup(ClientMock, "/disposition") do [{_, 0}] -> - Registry.update_value(Pleroma.ReverseProxy.ClientMock, "/disposition", &(&1 + 1)) + Registry.update_value(ClientMock, "/disposition", &(&1 + 1)) {:ok, "", client} [{_, 1}] -> - Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/disposition") + Registry.unregister(ClientMock, "/disposition") :done end end) From 5f42ecc4c74172b1b17c126106fda9da24065b11 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 7 Mar 2020 12:24:39 +0300 Subject: [PATCH 63/73] start gun upload pool, if proxy_remote is enabled --- lib/pleroma/pool/supervisor.ex | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex index f436849ac..8dc5b64b7 100644 --- a/lib/pleroma/pool/supervisor.ex +++ b/lib/pleroma/pool/supervisor.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Pool.Supervisor do use Supervisor + alias Pleroma.Config alias Pleroma.Pool def start_link(args) do @@ -17,8 +18,7 @@ def init(_) do %{ id: Pool.Connections, start: - {Pool.Connections, :start_link, - [{:gun_connections, Pleroma.Config.get([:connections_pool])}]} + {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]} } ] ++ pools() @@ -26,7 +26,16 @@ def init(_) do end defp pools do - for {pool_name, pool_opts} <- Pleroma.Config.get([:pools]) do + pools = Config.get(:pools) + + pools = + if Config.get([Pleroma.Upload, :proxy_remote]) == false do + Keyword.delete(pools, :upload) + else + pools + end + + for {pool_name, pool_opts} <- pools do pool_opts |> Keyword.put(:id, {Pool, pool_name}) |> Keyword.put(:name, pool_name) From 426f5ee48a09dbf321c013db08cc849c8929d86d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 10 Mar 2020 15:31:44 +0300 Subject: [PATCH 64/73] tesla adapter can't be changed in adminFE --- lib/pleroma/config/transfer_task.ex | 58 +++++++++---------- .../admin_api/admin_api_controller_test.exs | 21 +------ 2 files changed, 31 insertions(+), 48 deletions(-) diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index bf1b943d8..4a4c022f0 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -20,8 +20,7 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, :markup}, {:pleroma, :streamer}, {:pleroma, :pools}, - {:pleroma, :connections_pool}, - {:tesla, :adapter} + {:pleroma, :connections_pool} ] @reboot_time_subkeys [ @@ -35,8 +34,6 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, :gopher, [:enabled]} ] - @reject [nil, :prometheus] - def start_link(_) do load_and_update_env() if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) @@ -45,35 +42,30 @@ def start_link(_) do @spec load_and_update_env([ConfigDB.t()]) :: :ok | false def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do - with {:configurable, true} <- - {:configurable, Pleroma.Config.get(:configurable_from_database)}, - true <- Ecto.Adapters.SQL.table_exists?(Repo, "config"), - started_applications <- Application.started_applications() do + with {_, true} <- {:configurable, Pleroma.Config.get(:configurable_from_database)} do # We need to restart applications for loaded settings take effect - in_db = Repo.all(ConfigDB) with_deleted = in_db ++ deleted - reject_for_restart = if restart_pleroma?, do: @reject, else: [:pleroma | @reject] + # TODO: some problem with prometheus after restart! + reject = [nil, :prometheus] - applications = - with_deleted - |> Enum.map(&merge_and_update(&1)) - |> Enum.uniq() - # TODO: some problem with prometheus after restart! - |> Enum.reject(&(&1 in reject_for_restart)) - - # to be ensured that pleroma will be restarted last - applications = - if :pleroma in applications do - List.delete(applications, :pleroma) ++ [:pleroma] + reject_for_restart = + if restart_pleroma? do + reject else - Restarter.Pleroma.rebooted() - applications + [:pleroma | reject] end - Enum.each(applications, &restart(started_applications, &1, Pleroma.Config.get(:env))) + started_applications = Application.started_applications() + + with_deleted + |> Enum.map(&merge_and_update(&1)) + |> Enum.uniq() + |> Enum.reject(&(&1 in reject_for_restart)) + |> maybe_set_pleroma_last() + |> Enum.each(&restart(started_applications, &1, Pleroma.Config.get(:env))) :ok else @@ -81,6 +73,18 @@ def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do end end + defp maybe_set_pleroma_last(apps) do + # to be ensured that pleroma will be restarted last + if :pleroma in apps do + apps + |> List.delete(:pleroma) + |> List.insert_at(-1, :pleroma) + else + Restarter.Pleroma.rebooted() + apps + end + end + defp group_for_restart(:logger, key, _, merged_value) do # change logger configuration in runtime, without restart if Keyword.keyword?(merged_value) and @@ -93,14 +97,10 @@ defp group_for_restart(:logger, key, _, merged_value) do nil end - defp group_for_restart(:tesla, _, _, _), do: :pleroma - defp group_for_restart(group, _, _, _) when group != :pleroma, do: group defp group_for_restart(group, key, value, _) do - if pleroma_need_restart?(group, key, value) do - group - end + if pleroma_need_restart?(group, key, value), do: group end defp merge_and_update(setting) do diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index d6b839948..76240e5bc 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -2513,8 +2513,7 @@ test "saving full setting if value is not keyword", %{conn: conn} do "value" => "Tesla.Adapter.Httpc", "db" => [":adapter"] } - ], - "need_reboot" => true + ] } end @@ -2586,9 +2585,6 @@ test "update config setting & delete with fallback to default value", %{ end test "common config example", %{conn: conn} do - adapter = Application.get_env(:tesla, :adapter) - on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) - conn = post(conn, "/api/pleroma/admin/config", %{ configs: [ @@ -2607,16 +2603,10 @@ test "common config example", %{conn: conn} do %{"tuple" => [":regex4", "~r/https:\/\/example.com/s"]}, %{"tuple" => [":name", "Pleroma"]} ] - }, - %{ - "group" => ":tesla", - "key" => ":adapter", - "value" => "Tesla.Adapter.Httpc" } ] }) - assert Application.get_env(:tesla, :adapter) == Tesla.Adapter.Httpc assert Config.get([Pleroma.Captcha.NotReal, :name]) == "Pleroma" assert json_response(conn, 200) == %{ @@ -2648,15 +2638,8 @@ test "common config example", %{conn: conn} do ":regex4", ":name" ] - }, - %{ - "group" => ":tesla", - "key" => ":adapter", - "value" => "Tesla.Adapter.Httpc", - "db" => [":adapter"] } - ], - "need_reboot" => true + ] } end From f39e1b9eff859c0795911212c59304f68fca92bc Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 10 Mar 2020 15:54:11 +0300 Subject: [PATCH 65/73] add verify tls_opts only when we open connection for other requests tesla will add tls_opts --- lib/pleroma/gun/conn.ex | 24 +++++++++++++++++ lib/pleroma/http/adapter_helper/gun.ex | 33 ++++------------------- lib/pleroma/http/connection.ex | 13 +++++++++ test/http/adapter_helper/gun_test.exs | 37 ++++---------------------- test/http/connection_test.exs | 19 +++++++++++++ 5 files changed, 66 insertions(+), 60 deletions(-) diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 319718690..57a847c30 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -45,6 +45,7 @@ def open(%URI{} = uri, name, opts) do |> Map.put_new(:retry, pool_opts[:retry] || 1) |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000) |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) + |> maybe_add_tls_opts(uri) key = "#{uri.scheme}:#{uri.host}:#{uri.port}" @@ -70,6 +71,29 @@ def open(%URI{} = uri, name, opts) do end end + defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts + + defp maybe_add_tls_opts(opts, %URI{scheme: "https", host: host}) do + tls_opts = [ + verify: :verify_peer, + cacertfile: CAStore.file_path(), + depth: 20, + reuse_sessions: false, + verify_fun: + {&:ssl_verify_hostname.verify_fun/3, + [check_hostname: Pleroma.HTTP.Connection.format_host(host)]} + ] + + tls_opts = + if Keyword.keyword?(opts[:tls_opts]) do + Keyword.merge(tls_opts, opts[:tls_opts]) + else + tls_opts + end + + Map.put(opts, :tls_opts, tls_opts) + end + defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do connect_opts = uri diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 862e851c0..55c2b192a 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -45,21 +45,11 @@ def after_request(opts) do defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts - defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do - adapter_opts = [ - certificates_verification: true, - transport: :tls, - tls_opts: [ - verify: :verify_peer, - cacertfile: CAStore.file_path(), - depth: 20, - reuse_sessions: false, - verify_fun: {&:ssl_verify_hostname.verify_fun/3, [check_hostname: format_host(host)]}, - log_level: :warning - ] - ] - - Keyword.merge(opts, adapter_opts) + defp add_scheme_opts(opts, %URI{scheme: "https"}) do + opts + |> Keyword.put(:certificates_verification, true) + |> Keyword.put(:transport, :tls) + |> Keyword.put(:tls_opts, log_level: :warning) end defp maybe_get_conn(adapter_opts, uri, connection_opts) do @@ -93,17 +83,4 @@ defp try_to_get_conn(uri, opts) do |> Keyword.put(:close_conn, false) end end - - @spec format_host(String.t()) :: charlist() - def format_host(host) do - host_charlist = to_charlist(host) - - case :inet.parse_address(host_charlist) do - {:error, :einval} -> - :idna.encode(host_charlist) - - {:ok, _ip} -> - host_charlist - end - end end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 777e5d4c8..0fc88f708 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -106,4 +106,17 @@ def parse_host(host) when is_binary(host) do {:ok, ip} -> ip end end + + @spec format_host(String.t()) :: charlist() + def format_host(host) do + host_charlist = to_charlist(host) + + case :inet.parse_address(host_charlist) do + {:error, :einval} -> + :idna.encode(host_charlist) + + {:ok, _ip} -> + host_charlist + end + end end diff --git a/test/http/adapter_helper/gun_test.exs b/test/http/adapter_helper/gun_test.exs index 66622b605..6af8be15d 100644 --- a/test/http/adapter_helper/gun_test.exs +++ b/test/http/adapter_helper/gun_test.exs @@ -38,31 +38,23 @@ test "https url with default port" do opts = Gun.options([receive_conn: false], uri) assert opts[:certificates_verification] - refute opts[:tls_opts] == [] - - assert opts[:tls_opts][:verify_fun] == - {&:ssl_verify_hostname.verify_fun/3, [check_hostname: 'example.com']} - - assert File.exists?(opts[:tls_opts][:cacertfile]) + assert opts[:tls_opts][:log_level] == :warning end test "https ipv4 with default port" do uri = URI.parse("https://127.0.0.1") opts = Gun.options([receive_conn: false], uri) - - assert opts[:tls_opts][:verify_fun] == - {&:ssl_verify_hostname.verify_fun/3, [check_hostname: '127.0.0.1']} + assert opts[:certificates_verification] + assert opts[:tls_opts][:log_level] == :warning end test "https ipv6 with default port" do uri = URI.parse("https://[2a03:2880:f10c:83:face:b00c:0:25de]") opts = Gun.options([receive_conn: false], uri) - - assert opts[:tls_opts][:verify_fun] == - {&:ssl_verify_hostname.verify_fun/3, - [check_hostname: '2a03:2880:f10c:83:face:b00c:0:25de']} + assert opts[:certificates_verification] + assert opts[:tls_opts][:log_level] == :warning end test "https url with non standart port" do @@ -269,23 +261,4 @@ test "with ipv6" do } = Connections.get_state(:gun_connections) end end - - describe "format_host/1" do - test "with domain" do - assert Gun.format_host("example.com") == 'example.com' - end - - test "with idna domain" do - assert Gun.format_host("ですexample.com") == 'xn--example-183fne.com' - end - - test "with ipv4" do - assert Gun.format_host("127.0.0.1") == '127.0.0.1' - end - - test "with ipv6" do - assert Gun.format_host("2a03:2880:f10c:83:face:b00c:0:25de") == - '2a03:2880:f10c:83:face:b00c:0:25de' - end - end end diff --git a/test/http/connection_test.exs b/test/http/connection_test.exs index 25a2bac1c..0f62eddd2 100644 --- a/test/http/connection_test.exs +++ b/test/http/connection_test.exs @@ -113,4 +113,23 @@ test "passed opts have more weight than defaults" do assert opts[:proxy] == {'example.com', 4321} end end + + describe "format_host/1" do + test "with domain" do + assert Connection.format_host("example.com") == 'example.com' + end + + test "with idna domain" do + assert Connection.format_host("ですexample.com") == 'xn--example-183fne.com' + end + + test "with ipv4" do + assert Connection.format_host("127.0.0.1") == '127.0.0.1' + end + + test "with ipv6" do + assert Connection.format_host("2a03:2880:f10c:83:face:b00c:0:25de") == + '2a03:2880:f10c:83:face:b00c:0:25de' + end + end end From 1306b92997dc6e76e5d617d529dbc229d5aee200 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 12 Mar 2020 18:28:54 +0300 Subject: [PATCH 66/73] clean up --- lib/pleroma/application.ex | 18 +++-- lib/pleroma/config/transfer_task.ex | 42 +++++------ lib/pleroma/gun/conn.ex | 31 ++++----- lib/pleroma/http/adapter_helper.ex | 2 +- lib/pleroma/http/adapter_helper/gun.ex | 33 ++++----- lib/pleroma/http/connection.ex | 8 +-- lib/pleroma/http/http.ex | 5 +- lib/pleroma/pool/connections.ex | 96 +++++++++----------------- test/http/adapter_helper/gun_test.exs | 12 ++-- test/pool/connections_test.exs | 2 +- 10 files changed, 94 insertions(+), 155 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index c8a0617a5..55b5be488 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -42,7 +42,9 @@ def start(_type, _args) do setup_instrumenters() load_custom_modules() - if adapter() == Tesla.Adapter.Gun do + adapter = Application.get_env(:tesla, :adapter) + + if adapter == Tesla.Adapter.Gun do if version = Pleroma.OTPVersion.version() do [major, minor] = version @@ -74,7 +76,7 @@ def start(_type, _args) do Pleroma.Plugs.RateLimiter.Supervisor ] ++ cachex_children() ++ - http_pools_children(Config.get(:env)) ++ + http_children(adapter, @env) ++ [ Pleroma.Stats, Pleroma.JobQueueMonitor, @@ -206,15 +208,13 @@ defp task_children(_) do end # start hackney and gun pools in tests - defp http_pools_children(:test) do + defp http_children(_, :test) do hackney_options = Config.get([:hackney_pools, :federation]) hackney_pool = :hackney_pool.child_spec(:federation, hackney_options) [hackney_pool, Pleroma.Pool.Supervisor] end - defp http_pools_children(_), do: http_pools(adapter()) - - defp http_pools(Tesla.Adapter.Hackney) do + defp http_children(Tesla.Adapter.Hackney, _) do pools = [:federation, :media] pools = @@ -230,9 +230,7 @@ defp http_pools(Tesla.Adapter.Hackney) do end end - defp http_pools(Tesla.Adapter.Gun), do: [Pleroma.Pool.Supervisor] + defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor] - defp http_pools(_), do: [] - - defp adapter, do: Application.get_env(:tesla, :adapter) + defp http_children(_, _), do: [] end diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 4a4c022f0..b6d80adb7 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Config.TransferTask do use Task + alias Pleroma.Config alias Pleroma.ConfigDB alias Pleroma.Repo @@ -36,36 +37,31 @@ defmodule Pleroma.Config.TransferTask do def start_link(_) do load_and_update_env() - if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) + if Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) :ignore end - @spec load_and_update_env([ConfigDB.t()]) :: :ok | false - def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do - with {_, true} <- {:configurable, Pleroma.Config.get(:configurable_from_database)} do + @spec load_and_update_env([ConfigDB.t()], boolean()) :: :ok + def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do + with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do # We need to restart applications for loaded settings take effect - in_db = Repo.all(ConfigDB) - - with_deleted = in_db ++ deleted # TODO: some problem with prometheus after restart! - reject = [nil, :prometheus] - - reject_for_restart = + reject_restart = if restart_pleroma? do - reject + [nil, :prometheus] else - [:pleroma | reject] + [:pleroma, nil, :prometheus] end started_applications = Application.started_applications() - with_deleted - |> Enum.map(&merge_and_update(&1)) + (Repo.all(ConfigDB) ++ deleted_settings) + |> Enum.map(&merge_and_update/1) |> Enum.uniq() - |> Enum.reject(&(&1 in reject_for_restart)) + |> Enum.reject(&(&1 in reject_restart)) |> maybe_set_pleroma_last() - |> Enum.each(&restart(started_applications, &1, Pleroma.Config.get(:env))) + |> Enum.each(&restart(started_applications, &1, Config.get(:env))) :ok else @@ -108,18 +104,14 @@ defp merge_and_update(setting) do key = ConfigDB.from_string(setting.key) group = ConfigDB.from_string(setting.group) - default = Pleroma.Config.Holder.config(group, key) + default = Config.Holder.config(group, key) value = ConfigDB.from_binary(setting.value) merged_value = - if Ecto.get_meta(setting, :state) == :deleted do - default - else - if can_be_merged?(default, value) do - ConfigDB.merge_group(group, key, default, value) - else - value - end + cond do + Ecto.get_meta(setting, :state) == :deleted -> default + can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value) + true -> value end :ok = update_env(group, key, merged_value) diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 57a847c30..20823a765 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -49,8 +49,6 @@ def open(%URI{} = uri, name, opts) do key = "#{uri.scheme}:#{uri.host}:#{uri.port}" - Logger.debug("opening new connection #{Connections.compose_uri_log(uri)}") - conn_pid = if Connections.count(name) < opts[:max_connection] do do_open(uri, opts) @@ -109,9 +107,9 @@ defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do else error -> Logger.warn( - "Received error on opening connection with http proxy #{ - Connections.compose_uri_log(uri) - } #{inspect(error)}" + "Opening proxied connection to #{compose_uri_log(uri)} failed with error #{ + inspect(error) + }" ) error @@ -145,9 +143,9 @@ defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do else error -> Logger.warn( - "Received error on opening connection with socks proxy #{ - Connections.compose_uri_log(uri) - } #{inspect(error)}" + "Opening socks proxied connection to #{compose_uri_log(uri)} failed with error #{ + inspect(error) + }" ) error @@ -163,9 +161,7 @@ defp do_open(%URI{host: host, port: port} = uri, opts) do else error -> Logger.warn( - "Received error on opening connection #{Connections.compose_uri_log(uri)} #{ - inspect(error) - }" + "Opening connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}" ) error @@ -184,16 +180,17 @@ defp add_http2_opts(opts, "https", tls_opts) do defp add_http2_opts(opts, _, _), do: opts defp close_least_used_and_do_open(name, uri, opts) do - Logger.debug("try to open conn #{Connections.compose_uri_log(uri)}") - - with [{close_key, least_used} | _conns] <- - Connections.get_unused_conns(name), - :ok <- Gun.close(least_used.conn) do - Connections.remove_conn(name, close_key) + with [{key, conn} | _conns] <- Connections.get_unused_conns(name), + :ok <- Gun.close(conn.conn) do + Connections.remove_conn(name, key) do_open(uri, opts) else [] -> {:error, :pool_overflowed} end end + + def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do + "#{scheme}://#{host}#{path}" + end end diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 2c13666ec..510722ff9 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -7,7 +7,7 @@ defmodule Pleroma.HTTP.AdapterHelper do @type proxy :: {Connection.host(), pos_integer()} - | {Connection.proxy_type(), pos_integer()} + | {Connection.proxy_type(), Connection.host(), pos_integer()} @callback options(keyword(), URI.t()) :: keyword() @callback after_request(keyword()) :: :ok diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 55c2b192a..f14b95c19 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -20,8 +20,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do ] @spec options(keyword(), URI.t()) :: keyword() - def options(connection_opts \\ [], %URI{} = uri) do - formatted_proxy = + def options(incoming_opts \\ [], %URI{} = uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) |> AdapterHelper.format_proxy() @@ -30,8 +30,8 @@ def options(connection_opts \\ [], %URI{} = uri) do @defaults |> Keyword.merge(config_opts) |> add_scheme_opts(uri) - |> AdapterHelper.maybe_add_proxy(formatted_proxy) - |> maybe_get_conn(uri, connection_opts) + |> AdapterHelper.maybe_add_proxy(proxy) + |> maybe_get_conn(uri, incoming_opts) end @spec after_request(keyword()) :: :ok @@ -43,44 +43,35 @@ def after_request(opts) do :ok end - defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts + defp add_scheme_opts(opts, %{scheme: "http"}), do: opts - defp add_scheme_opts(opts, %URI{scheme: "https"}) do + defp add_scheme_opts(opts, %{scheme: "https"}) do opts |> Keyword.put(:certificates_verification, true) - |> Keyword.put(:transport, :tls) |> Keyword.put(:tls_opts, log_level: :warning) end - defp maybe_get_conn(adapter_opts, uri, connection_opts) do + defp maybe_get_conn(adapter_opts, uri, incoming_opts) do {receive_conn?, opts} = adapter_opts - |> Keyword.merge(connection_opts) + |> Keyword.merge(incoming_opts) |> Keyword.pop(:receive_conn, true) if Connections.alive?(:gun_connections) and receive_conn? do - try_to_get_conn(uri, opts) + checkin_conn(uri, opts) else opts end end - defp try_to_get_conn(uri, opts) do + defp checkin_conn(uri, opts) do case Connections.checkin(uri, :gun_connections) do nil -> - Logger.debug( - "Gun connections pool checkin was not successful. Trying to open conn for next request." - ) - - Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end) + Task.start(Pleroma.Gun.Conn, :open, [uri, :gun_connections, opts]) opts conn when is_pid(conn) -> - Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") - - opts - |> Keyword.put(:conn, conn) - |> Keyword.put(:close_conn, false) + Keyword.merge(opts, conn: conn, close_conn: false) end end end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 0fc88f708..76de3fcfe 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -71,15 +71,15 @@ def parse_proxy(proxy) when is_binary(proxy) do {:ok, parse_host(host), port} else {_, _} -> - Logger.warn("parsing port in proxy fail #{inspect(proxy)}") + Logger.warn("Parsing port failed #{inspect(proxy)}") {:error, :invalid_proxy_port} :error -> - Logger.warn("parsing port in proxy fail #{inspect(proxy)}") + Logger.warn("Parsing port failed #{inspect(proxy)}") {:error, :invalid_proxy_port} _ -> - Logger.warn("parsing proxy fail #{inspect(proxy)}") + Logger.warn("Parsing proxy failed #{inspect(proxy)}") {:error, :invalid_proxy} end end @@ -89,7 +89,7 @@ def parse_proxy(proxy) when is_tuple(proxy) do {:ok, type, parse_host(host), port} else _ -> - Logger.warn("parsing proxy fail #{inspect(proxy)}") + Logger.warn("Parsing proxy failed #{inspect(proxy)}") {:error, :invalid_proxy} end end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 466a94adc..583b56484 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -56,10 +56,9 @@ def post(url, body, headers \\ [], options \\ []), {:ok, Env.t()} | {:error, any()} def request(method, url, body, headers, options) when is_binary(url) do uri = URI.parse(url) - received_adapter_opts = Keyword.get(options, :adapter, []) - adapter_opts = Connection.options(uri, received_adapter_opts) + adapter_opts = Connection.options(uri, options[:adapter] || []) options = put_in(options[:adapter], adapter_opts) - params = Keyword.get(options, :params, []) + params = options[:params] || [] request = build_request(method, headers, options, url, body, params) adapter = Application.get_env(:tesla, :adapter) diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 7529e9240..772833509 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -87,18 +87,11 @@ def handle_cast({:add_conn, key, conn}, state) do @impl true def handle_cast({:checkout, conn_pid, pid}, state) do - Logger.debug("checkout #{inspect(conn_pid)}") - state = with true <- Process.alive?(conn_pid), {key, conn} <- find_conn(state.conns, conn_pid), used_by <- List.keydelete(conn.used_by, pid, 0) do - conn_state = - if used_by == [] do - :idle - else - conn.conn_state - end + conn_state = if used_by == [], do: :idle, else: conn.conn_state put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by}) else @@ -123,26 +116,23 @@ def handle_cast({:remove_conn, key}, state) do @impl true def handle_call({:checkin, uri}, from, state) do key = "#{uri.scheme}:#{uri.host}:#{uri.port}" - Logger.debug("checkin #{key}") case state.conns[key] do - %{conn: conn, gun_state: :up} = current_conn -> - Logger.debug("reusing conn #{key}") - + %{conn: pid, gun_state: :up} = conn -> time = :os.system_time(:second) - last_reference = time - current_conn.last_reference - current_crf = crf(last_reference, 100, current_conn.crf) + last_reference = time - conn.last_reference + crf = crf(last_reference, 100, conn.crf) state = put_in(state.conns[key], %{ - current_conn + conn | last_reference: time, - crf: current_crf, + crf: crf, conn_state: :active, - used_by: [from | current_conn.used_by] + used_by: [from | conn.used_by] }) - {:reply, conn, state} + {:reply, pid, state} %{gun_state: :down} -> {:reply, nil, state} @@ -164,50 +154,48 @@ def handle_call(:count, _from, state) do def handle_call(:unused_conns, _from, state) do unused_conns = state.conns - |> Enum.filter(fn {_k, v} -> - v.conn_state == :idle and v.used_by == [] - end) - |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> - x.crf <= y.crf and x.last_reference <= y.last_reference - end) + |> Enum.filter(&filter_conns/1) + |> Enum.sort(&sort_conns/2) {:reply, unused_conns, state} end + defp filter_conns({_, %{conn_state: :idle, used_by: []}}), do: true + defp filter_conns(_), do: false + + defp sort_conns({_, c1}, {_, c2}) do + c1.crf <= c2.crf and c1.last_reference <= c2.last_reference + end + @impl true def handle_info({:gun_up, conn_pid, _protocol}, state) do - state = - with conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid), - {key, conn} <- find_conn(state.conns, conn_pid, conn_key), - {true, key} <- {Process.alive?(conn_pid), key} do - time = :os.system_time(:second) - last_reference = time - conn.last_reference - current_crf = crf(last_reference, 100, conn.crf) + %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) + host = + case :inet.ntoa(host) do + {:error, :einval} -> host + ip -> ip + end + + key = "#{scheme}:#{host}:#{port}" + + state = + with {_key, conn} <- find_conn(state.conns, conn_pid, key), + {true, key} <- {Process.alive?(conn_pid), key} do put_in(state.conns[key], %{ conn | gun_state: :up, - last_reference: time, - crf: current_crf, conn_state: :active, retries: 0 }) else - :error_gun_info -> - Logger.debug(":gun.info caused error") - state - {false, key} -> - Logger.debug(":gun_up message for closed conn #{inspect(conn_pid)}") - put_in( state.conns, Map.delete(state.conns, key) ) nil -> - Logger.debug(":gun_up message for conn which is not found in state") - :ok = Gun.close(conn_pid) state @@ -224,7 +212,6 @@ def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do with {key, conn} <- find_conn(state.conns, conn_pid), {true, key} <- {Process.alive?(conn_pid), key} do if conn.retries == retries do - Logger.debug("closing conn if retries is eq #{inspect(conn_pid)}") :ok = Gun.close(conn.conn) put_in( @@ -240,18 +227,13 @@ def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do end else {false, key} -> - # gun can send gun_down for closed conn, maybe connection is not closed yet - Logger.debug(":gun_down message for closed conn #{inspect(conn_pid)}") - put_in( state.conns, Map.delete(state.conns, key) ) nil -> - Logger.debug(":gun_down message for conn which is not found in state") - - :ok = Gun.close(conn_pid) + Logger.debug(":gun_down for conn which isn't found in state") state end @@ -275,7 +257,7 @@ def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do ) else nil -> - Logger.debug(":DOWN message for conn which is not found in state") + Logger.debug(":DOWN for conn which isn't found in state") state end @@ -283,18 +265,6 @@ def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do {:noreply, state} end - defp compose_key_gun_info(pid) do - %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) - - host = - case :inet.ntoa(origin_host) do - {:error, :einval} -> origin_host - ip -> ip - end - - "#{scheme}:#{host}:#{port}" - end - defp find_conn(conns, conn_pid) do Enum.find(conns, fn {_key, conn} -> conn.conn == conn_pid @@ -310,8 +280,4 @@ defp find_conn(conns, conn_pid, conn_key) do def crf(current, steps, crf) do 1 + :math.pow(0.5, current / steps) * crf end - - def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do - "#{scheme}://#{host}#{path}" - end end diff --git a/test/http/adapter_helper/gun_test.exs b/test/http/adapter_helper/gun_test.exs index 6af8be15d..18025b986 100644 --- a/test/http/adapter_helper/gun_test.exs +++ b/test/http/adapter_helper/gun_test.exs @@ -6,7 +6,6 @@ defmodule Pleroma.HTTP.AdapterHelper.GunTest do use ExUnit.Case, async: true use Pleroma.Tests.Helpers - import ExUnit.CaptureLog import Mox alias Pleroma.Config @@ -63,7 +62,6 @@ test "https url with non standart port" do opts = Gun.options([receive_conn: false], uri) assert opts[:certificates_verification] - assert opts[:transport] == :tls end test "get conn on next request" do @@ -73,14 +71,12 @@ test "get conn on next request" do on_exit(fn -> Logger.configure(level: level) end) uri = URI.parse("http://some-domain2.com") - assert capture_log(fn -> - opts = Gun.options(uri) + opts = Gun.options(uri) - assert opts[:conn] == nil - assert opts[:close_conn] == nil - end) =~ - "Gun connections pool checkin was not successful. Trying to open conn for next request." + assert opts[:conn] == nil + assert opts[:close_conn] == nil + Process.sleep(50) opts = Gun.options(uri) assert is_pid(opts[:conn]) diff --git a/test/pool/connections_test.exs b/test/pool/connections_test.exs index 06f32b74e..aeda54875 100644 --- a/test/pool/connections_test.exs +++ b/test/pool/connections_test.exs @@ -355,7 +355,7 @@ test "connection can't get up", %{name: name} do refute Conn.open(url, name) refute Connections.checkin(url, name) end) =~ - "Received error on opening connection http://gun-not-up.com {:error, :timeout}" + "Opening connection to http://gun-not-up.com failed with error {:error, :timeout}" end test "process gun_down message and then gun_up", %{name: name} do From 98ed0d1c4bd2db354154cc4a1d1e6530eb68f499 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 13 Mar 2020 09:37:57 +0300 Subject: [PATCH 67/73] more clean up --- lib/pleroma/http/adapter_helper/gun.ex | 2 +- lib/pleroma/http/adapter_helper/hackney.ex | 2 +- lib/pleroma/http/connection.ex | 12 +++++++----- lib/pleroma/pool/request.ex | 1 - lib/pleroma/pool/supervisor.ex | 15 ++++++--------- lib/pleroma/reverse_proxy/client/tesla.ex | 9 +++++---- lib/pleroma/reverse_proxy/reverse_proxy.ex | 2 +- 7 files changed, 21 insertions(+), 22 deletions(-) diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index f14b95c19..ead7cdc6b 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -22,7 +22,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @spec options(keyword(), URI.t()) :: keyword() def options(incoming_opts \\ [], %URI{} = uri) do proxy = - Pleroma.Config.get([:http, :proxy_url], nil) + Pleroma.Config.get([:http, :proxy_url]) |> AdapterHelper.format_proxy() config_opts = Pleroma.Config.get([:http, :adapter], []) diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex index d08afae0c..dcb4cac71 100644 --- a/lib/pleroma/http/adapter_helper/hackney.ex +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do @spec options(keyword(), URI.t()) :: keyword() def options(connection_opts \\ [], %URI{} = uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) + proxy = Pleroma.Config.get([:http, :proxy_url]) config_opts = Pleroma.Config.get([:http, :adapter], []) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 76de3fcfe..ebacf7902 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -30,12 +30,12 @@ def options(%URI{} = uri, opts \\ []) do @defaults |> pool_timeout() |> Keyword.merge(opts) - |> adapter().options(uri) + |> adapter_helper().options(uri) end defp pool_timeout(opts) do {config_key, default} = - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + if adapter() == Tesla.Adapter.Gun do {:pools, Config.get([:pools, :default, :timeout])} else {:hackney_pools, 10_000} @@ -47,10 +47,12 @@ defp pool_timeout(opts) do end @spec after_request(keyword()) :: :ok - def after_request(opts), do: adapter().after_request(opts) + def after_request(opts), do: adapter_helper().after_request(opts) - defp adapter do - case Application.get_env(:tesla, :adapter) do + defp adapter, do: Application.get_env(:tesla, :adapter) + + defp adapter_helper do + case adapter() do Tesla.Adapter.Gun -> AdapterHelper.Gun Tesla.Adapter.Hackney -> AdapterHelper.Hackney _ -> AdapterHelper diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex index db7c10c01..3fb930db7 100644 --- a/lib/pleroma/pool/request.ex +++ b/lib/pleroma/pool/request.ex @@ -39,7 +39,6 @@ def handle_info({:gun_up, _conn, _protocol}, state) do @impl true def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do - # don't flush messages here, because gun can reconnect {:noreply, state} end diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex index 8dc5b64b7..faf646cb2 100644 --- a/lib/pleroma/pool/supervisor.ex +++ b/lib/pleroma/pool/supervisor.ex @@ -13,16 +13,13 @@ def start_link(args) do end def init(_) do - children = - [ - %{ - id: Pool.Connections, - start: - {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]} - } - ] ++ pools() + conns_child = %{ + id: Pool.Connections, + start: + {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]} + } - Supervisor.init(children, strategy: :one_for_one) + Supervisor.init([conns_child | pools()], strategy: :one_for_one) end defp pools do diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index dbc6b66a3..e81ea8bde 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -3,11 +3,11 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client.Tesla do + @behaviour Pleroma.ReverseProxy.Client + @type headers() :: [{String.t(), String.t()}] @type status() :: pos_integer() - @behaviour Pleroma.ReverseProxy.Client - @spec request(atom(), String.t(), headers(), String.t(), keyword()) :: {:ok, status(), headers} | {:ok, status(), headers, map()} @@ -18,7 +18,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do def request(method, url, headers, body, opts \\ []) do check_adapter() - opts = Keyword.merge(opts, body_as: :chunks) + opts = Keyword.put(opts, :body_as, :chunks) with {:ok, response} <- Pleroma.HTTP.request( @@ -39,7 +39,8 @@ def request(method, url, headers, body, opts \\ []) do end @impl true - @spec stream_body(map()) :: {:ok, binary(), map()} | {:error, atom() | String.t()} | :done + @spec stream_body(map()) :: + {:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return() def stream_body(%{pid: pid, opts: opts, fin: true}) do # if connection was reused, but in tesla were redirects, # tesla returns new opened connection, which must be closed manually diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 8f1aa3200..35b973b56 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -59,7 +59,7 @@ defmodule Pleroma.ReverseProxy do * `req_headers`, `resp_headers` additional headers. - * `http`: options for [gun](https://github.com/ninenines/gun). + * `http`: options for [hackney](https://github.com/benoitc/hackney) or [gun](https://github.com/ninenines/gun). """ @default_options [pool: :media] From 35471205f862fa069c6d87aefc1d827c9fab6e08 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 16 Mar 2020 15:47:25 +0300 Subject: [PATCH 68/73] temp fix for `:gun.info` MatchError --- lib/pleroma/pool/connections.ex | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 772833509..16aa80548 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -169,19 +169,26 @@ defp sort_conns({_, c1}, {_, c2}) do @impl true def handle_info({:gun_up, conn_pid, _protocol}, state) do - %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) + # TODO: temp fix for gun MatchError https://github.com/ninenines/gun/issues/222 + # TODO: REMOVE LATER + {key, conn} = + try do + %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) - host = - case :inet.ntoa(host) do - {:error, :einval} -> host - ip -> ip + host = + case :inet.ntoa(host) do + {:error, :einval} -> host + ip -> ip + end + + key = "#{scheme}:#{host}:#{port}" + find_conn(state.conns, conn_pid, key) + rescue + MatcheError -> find_conn(state.conns, conn_pid) end - key = "#{scheme}:#{host}:#{port}" - state = - with {_key, conn} <- find_conn(state.conns, conn_pid, key), - {true, key} <- {Process.alive?(conn_pid), key} do + with {true, key} <- {Process.alive?(conn_pid), key} do put_in(state.conns[key], %{ conn | gun_state: :up, From bf474ca3c154544b54720ea23c06191e68f32522 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 16 Mar 2020 16:23:49 +0300 Subject: [PATCH 69/73] fix --- lib/pleroma/pool/connections.ex | 34 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 16aa80548..91102faf7 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -167,28 +167,30 @@ defp sort_conns({_, c1}, {_, c2}) do c1.crf <= c2.crf and c1.last_reference <= c2.last_reference end - @impl true - def handle_info({:gun_up, conn_pid, _protocol}, state) do + defp find_conn_from_gun_info(conns, pid) do # TODO: temp fix for gun MatchError https://github.com/ninenines/gun/issues/222 # TODO: REMOVE LATER - {key, conn} = - try do - %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) + try do + %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) - host = - case :inet.ntoa(host) do - {:error, :einval} -> host - ip -> ip - end + host = + case :inet.ntoa(host) do + {:error, :einval} -> host + ip -> ip + end - key = "#{scheme}:#{host}:#{port}" - find_conn(state.conns, conn_pid, key) - rescue - MatcheError -> find_conn(state.conns, conn_pid) - end + key = "#{scheme}:#{host}:#{port}" + find_conn(conns, pid, key) + rescue + MatcheError -> find_conn(conns, pid) + end + end + @impl true + def handle_info({:gun_up, conn_pid, _protocol}, state) do state = - with {true, key} <- {Process.alive?(conn_pid), key} do + with {key, conn} <- find_conn_from_gun_info(state.conns, conn_pid), + {true, key} <- {Process.alive?(conn_pid), key} do put_in(state.conns[key], %{ conn | gun_state: :up, From b17d8d305f5e9bf25644fd9b3457a965e3a5c001 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 16 Mar 2020 15:39:34 -0500 Subject: [PATCH 70/73] Enable Gun adapter by default We need devs to dogfood this before we merge it into the 2.1 release --- config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 3ec1868b2..154eda48a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -170,7 +170,7 @@ "application/ld+json" => ["activity+json"] } -config :tesla, adapter: Tesla.Adapter.Hackney +config :tesla, adapter: Tesla.Adapter.Gun # Configures http settings, upstream proxy etc. config :pleroma, :http, proxy_url: nil, From 7f9b5284fa7dd1d9100de730a6fe0c93739d1b30 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 20 Mar 2020 20:58:47 +0300 Subject: [PATCH 71/73] updating clear_config --- test/http/adapter_helper/gun_test.exs | 4 +--- test/http/adapter_helper/hackney_test.exs | 5 +---- test/http/connection_test.exs | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/test/http/adapter_helper/gun_test.exs b/test/http/adapter_helper/gun_test.exs index 18025b986..2e961826e 100644 --- a/test/http/adapter_helper/gun_test.exs +++ b/test/http/adapter_helper/gun_test.exs @@ -28,9 +28,7 @@ defp gun_mock do end describe "options/1" do - clear_config([:http, :adapter]) do - Config.put([:http, :adapter], a: 1, b: 2) - end + setup do: clear_config([:http, :adapter], a: 1, b: 2) test "https url with default port" do uri = URI.parse("https://example.com") diff --git a/test/http/adapter_helper/hackney_test.exs b/test/http/adapter_helper/hackney_test.exs index 5fda075f6..3f7e708e0 100644 --- a/test/http/adapter_helper/hackney_test.exs +++ b/test/http/adapter_helper/hackney_test.exs @@ -6,7 +6,6 @@ defmodule Pleroma.HTTP.AdapterHelper.HackneyTest do use ExUnit.Case, async: true use Pleroma.Tests.Helpers - alias Pleroma.Config alias Pleroma.HTTP.AdapterHelper.Hackney setup_all do @@ -15,9 +14,7 @@ defmodule Pleroma.HTTP.AdapterHelper.HackneyTest do end describe "options/2" do - clear_config([:http, :adapter]) do - Config.put([:http, :adapter], a: 1, b: 2) - end + setup do: clear_config([:http, :adapter], a: 1, b: 2) test "add proxy and opts from config", %{uri: uri} do opts = Hackney.options([proxy: "localhost:8123"], uri) diff --git a/test/http/connection_test.exs b/test/http/connection_test.exs index 0f62eddd2..5cc78ad5b 100644 --- a/test/http/connection_test.exs +++ b/test/http/connection_test.exs @@ -82,7 +82,7 @@ test "with nil" do end describe "options/3" do - clear_config([:http, :proxy_url]) + setup do: clear_config([:http, :proxy_url]) test "without proxy_url in config" do Config.delete([:http, :proxy_url]) From 2afc7a9112fc11bc51abc2b65aea03d6d5045695 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 30 Mar 2020 12:16:45 +0300 Subject: [PATCH 72/73] changelog fix --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f393ea8eb..52e6c33f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - NodeInfo: `pleroma:api/v1/notifications:include_types_filter` to the `features` list. - Configuration: `:restrict_unauthenticated` setting, restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses. +- New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires minimum OTP version of 22.2 otherwise Pleroma won’t start. For hackney OTP update is not required.
API Changes - Mastodon API: Support for `include_types` in `/api/v1/notifications`. @@ -97,7 +98,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - User settings: Add _This account is a_ option. - A new users admin digest email - OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`). -- New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires minimum OTP version of 22.2 otherwise Pleroma won’t start. For hackney OTP update is not required. - Add an option `authorized_fetch_mode` to require HTTP signatures for AP fetches. - ActivityPub: support for `replies` collection (output for outgoing federation & fetching on incoming federation). - Mix task to refresh counter cache (`mix pleroma.refresh_counter_cache`) From 1fcdcb12a717fa3dbd54a5c3778bd216df6449ad Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 30 Mar 2020 12:47:12 +0300 Subject: [PATCH 73/73] updating gun with bug fix https://github.com/ninenines/gun/issues/222 --- lib/pleroma/pool/connections.ex | 31 +++++++++++-------------------- mix.exs | 2 +- mix.lock | 2 +- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 91102faf7..4d4ba913c 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -167,29 +167,20 @@ defp sort_conns({_, c1}, {_, c2}) do c1.crf <= c2.crf and c1.last_reference <= c2.last_reference end - defp find_conn_from_gun_info(conns, pid) do - # TODO: temp fix for gun MatchError https://github.com/ninenines/gun/issues/222 - # TODO: REMOVE LATER - try do - %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) - - host = - case :inet.ntoa(host) do - {:error, :einval} -> host - ip -> ip - end - - key = "#{scheme}:#{host}:#{port}" - find_conn(conns, pid, key) - rescue - MatcheError -> find_conn(conns, pid) - end - end - @impl true def handle_info({:gun_up, conn_pid, _protocol}, state) do + %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) + + host = + case :inet.ntoa(host) do + {:error, :einval} -> host + ip -> ip + end + + key = "#{scheme}:#{host}:#{port}" + state = - with {key, conn} <- find_conn_from_gun_info(state.conns, conn_pid), + with {key, conn} <- find_conn(state.conns, conn_pid, key), {true, key} <- {Process.alive?(conn_pid), key} do put_in(state.conns[key], %{ conn diff --git a/mix.exs b/mix.exs index 77d043d37..87c025d89 100644 --- a/mix.exs +++ b/mix.exs @@ -127,7 +127,7 @@ defp deps do {:castore, "~> 0.1"}, {:cowlib, "~> 2.8", override: true}, {:gun, - github: "ninenines/gun", ref: "bd6425ab87428cf4c95f4d23e0a48fd065fbd714", override: true}, + github: "ninenines/gun", ref: "e1a69b36b180a574c0ac314ced9613fdd52312cc", override: true}, {:jason, "~> 1.0"}, {:mogrify, "~> 0.6.1"}, {:ex_aws, "~> 2.1"}, diff --git a/mix.lock b/mix.lock index b791dccc4..6cca578d6 100644 --- a/mix.lock +++ b/mix.lock @@ -47,7 +47,7 @@ "gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm"}, "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.4", "f13088e1ec10ce01665cf25f5ff779e7df3f2dc71b37084976cf89d1aa124d5c", [:mix], [], "hexpm", "3c75b5ea8288e2ee7ea503ff9e30dfe4d07ad3c054576a6e60040e79a801e14d"}, - "gun": {:git, "https://github.com/ninenines/gun.git", "bd6425ab87428cf4c95f4d23e0a48fd065fbd714", [ref: "bd6425ab87428cf4c95f4d23e0a48fd065fbd714"]}, + "gun": {:git, "https://github.com/ninenines/gun.git", "e1a69b36b180a574c0ac314ced9613fdd52312cc", [ref: "e1a69b36b180a574c0ac314ced9613fdd52312cc"]}, "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e0100f8ef7d1124222c11ad362c857d3df7cb5f4204054f9f0f4a728666591fc"}, "html_entities": {:hex, :html_entities, "0.5.1", "1c9715058b42c35a2ab65edc5b36d0ea66dd083767bef6e3edb57870ef556549", [:mix], [], "hexpm", "30efab070904eb897ff05cd52fa61c1025d7f8ef3a9ca250bc4e6513d16c32de"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"},