From 8daacc911498d827fd68ea3d34eb1be9ae4a1ffe Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 23 Jun 2020 14:17:23 -0500 Subject: [PATCH 01/18] AutoLinker --> Linkify, update to latest version https://git.pleroma.social/pleroma/elixir-libraries/linkify --- CHANGELOG.md | 1 + config/config.exs | 18 ++++++-------- config/description.exs | 26 +++++++++++++------- docs/configuration/cheatsheet.md | 35 +++++++++++++-------------- lib/pleroma/config/config_db.ex | 2 +- lib/pleroma/formatter.ex | 26 +++++++++++--------- lib/pleroma/web/rich_media/helpers.ex | 4 +-- mix.exs | 4 +-- mix.lock | 2 +- 9 files changed, 63 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71963d206..4d3bda99e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MFR policy to set global expiration for all local Create activities - OGP rich media parser merged with TwitterCard - Configuration: `:instance, rewrite_policy` moved to `:mrf, policies`, `:instance, :mrf_transparency` moved to `:mrf, :transparency`, `:instance, :mrf_transparency_exclusions` moved to `:mrf, :transparency_exclusions`. Old config namespace is deprecated. +- **Breaking:** Configuration: `:auto_linker, :opts` moved to `:pleroma, Pleroma.Formatter`. Old config namespace is deprecated.
API Changes diff --git a/config/config.exs b/config/config.exs index 5aad26e95..a74a6a5ba 100644 --- a/config/config.exs +++ b/config/config.exs @@ -520,16 +520,14 @@ federator_outgoing: 5 ] -config :auto_linker, - opts: [ - extra: true, - # TODO: Set to :no_scheme when it works properly - validate_tld: true, - class: false, - strip_prefix: false, - new_window: false, - rel: "ugc" - ] +config :pleroma, Pleroma.Formatter, + class: false, + rel: "ugc", + new_window: false, + truncate: false, + strip_prefix: false, + extra: true, + validate_tld: :no_scheme config :pleroma, :ldap, enabled: System.get_env("LDAP_ENABLED") == "true", diff --git a/config/description.exs b/config/description.exs index f54ac2a2a..204de8324 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2157,44 +2157,52 @@ ] }, %{ - group: :auto_linker, - key: :opts, + group: :pleroma, + key: Pleroma.Formatter, type: :group, - description: "Configuration for the auto_linker library", + description: + "Configuration for Pleroma's link formatter which parses mentions, hashtags, and URLs.", children: [ %{ key: :class, type: [:string, false], - description: "Specify the class to be added to the generated link. Disable to clear", + description: "Specify the class to be added to the generated link. Disable to clear.", suggestions: ["auto-linker", false] }, %{ key: :rel, type: [:string, false], - description: "Override the rel attribute. Disable to clear", + description: "Override the rel attribute. Disable to clear.", suggestions: ["ugc", "noopener noreferrer", false] }, %{ key: :new_window, type: :boolean, - description: "Link urls will open in new window/tab" + description: "Link URLs will open in new window/tab." }, %{ key: :truncate, type: [:integer, false], description: - "Set to a number to truncate urls longer then the number. Truncated urls will end in `..`", + "Set to a number to truncate URLs longer then the number. Truncated URLs will end in `...`", suggestions: [15, false] }, %{ key: :strip_prefix, type: :boolean, - description: "Strip the scheme prefix" + description: "Strip the scheme prefix." }, %{ key: :extra, type: :boolean, - description: "Link urls with rarely used schemes (magnet, ipfs, irc, etc.)" + description: "Link URLs with rarely used schemes (magnet, ipfs, irc, etc.)" + }, + %{ + key: :validate_tld, + type: [:atom, :boolean], + description: + "Set to false to disable TLD validation for URLs/emails. Can be set to :no_scheme to validate TLDs only for URLs without a scheme (e.g `example.com` will be validated, but `http://example.loki` won't)", + suggestions: [:no_scheme, true] } ] }, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 6759d5e93..22b28d423 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -908,30 +908,29 @@ Configure OAuth 2 provider capabilities: ### :uri_schemes * `valid_schemes`: List of the scheme part that is considered valid to be an URL. -### :auto_linker +### Pleroma.Formatter -Configuration for the `auto_linker` library: +Configuration for Pleroma's link formatter which parses mentions, hashtags, and URLs. -* `class: "auto-linker"` - specify the class to be added to the generated link. false to clear. -* `rel: "noopener noreferrer"` - override the rel attribute. false to clear. -* `new_window: true` - set to false to remove `target='_blank'` attribute. -* `scheme: false` - Set to true to link urls with schema `http://google.com`. -* `truncate: false` - Set to a number to truncate urls longer then the number. Truncated urls will end in `..`. -* `strip_prefix: true` - Strip the scheme prefix. -* `extra: false` - link urls with rarely used schemes (magnet, ipfs, irc, etc.). +* `class` - specify the class to be added to the generated link (default: `false`) +* `rel` - specify the rel attribute (default: `ugc`) +* `new_window` - adds `target="_blank"` attribute (default: `false`) +* `truncate` - Set to a number to truncate URLs longer then the number. Truncated URLs will end in `...` (default: `false`) +* `strip_prefix` - Strip the scheme prefix (default: `false`) +* `extra` - link URLs with rarely used schemes (magnet, ipfs, irc, etc.) (default: `true`) +* `validate_tld` - Set to false to disable TLD validation for URLs/emails. Can be set to :no_scheme to validate TLDs only for urls without a scheme (e.g `example.com` will be validated, but `http://example.loki` won't) (default: `:no_scheme`) Example: ```elixir -config :auto_linker, - opts: [ - scheme: true, - extra: true, - class: false, - strip_prefix: false, - new_window: false, - rel: "ugc" - ] +config :pleroma, Pleroma.Formatter, + class: false, + rel: "ugc", + new_window: false, + truncate: false, + strip_prefix: false, + extra: true, + validate_tld: :no_scheme ``` ## Custom Runtime Modules (`:modules`) diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex index 1a89d8895..f8141ced8 100644 --- a/lib/pleroma/config/config_db.ex +++ b/lib/pleroma/config/config_db.ex @@ -156,7 +156,7 @@ defp only_full_update?(%ConfigDB{group: group, key: key}) do {:quack, :meta}, {:mime, :types}, {:cors_plug, [:max_age, :methods, :expose, :headers]}, - {:auto_linker, :opts}, + {:linkify, :opts}, {:swarm, :node_blacklist}, {:logger, :backends} ] diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 02a93a8dc..0c450eae4 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -10,11 +10,15 @@ defmodule Pleroma.Formatter do @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/ - @auto_linker_config hashtag: true, - hashtag_handler: &Pleroma.Formatter.hashtag_handler/4, - mention: true, - mention_handler: &Pleroma.Formatter.mention_handler/4, - scheme: true + defp linkify_opts do + Pleroma.Config.get(Pleroma.Formatter) ++ + [ + hashtag: true, + hashtag_handler: &Pleroma.Formatter.hashtag_handler/4, + mention: true, + mention_handler: &Pleroma.Formatter.mention_handler/4 + ] + end def escape_mention_handler("@" <> nickname = mention, buffer, _, _) do case User.get_cached_by_nickname(nickname) do @@ -80,19 +84,19 @@ def hashtag_handler("#" <> tag = tag_text, _buffer, _opts, acc) do @spec linkify(String.t(), keyword()) :: {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]} def linkify(text, options \\ []) do - options = options ++ @auto_linker_config + options = linkify_opts() ++ options if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text) acc = %{mentions: MapSet.new(), tags: MapSet.new()} - {text_mentions, %{mentions: mentions}} = AutoLinker.link_map(mentions, acc, options) - {text_rest, %{tags: tags}} = AutoLinker.link_map(rest, acc, options) + {text_mentions, %{mentions: mentions}} = Linkify.link_map(mentions, acc, options) + {text_rest, %{tags: tags}} = Linkify.link_map(rest, acc, options) {text_mentions <> text_rest, MapSet.to_list(mentions), MapSet.to_list(tags)} else acc = %{mentions: MapSet.new(), tags: MapSet.new()} - {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options) + {text, %{mentions: mentions, tags: tags}} = Linkify.link_map(text, acc, options) {text, MapSet.to_list(mentions), MapSet.to_list(tags)} end @@ -111,9 +115,9 @@ def mentions_escape(text, options \\ []) do if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text) - AutoLinker.link(mentions, options) <> AutoLinker.link(rest, options) + Linkify.link(mentions, options) <> Linkify.link(rest, options) else - AutoLinker.link(text, options) + Linkify.link(text, options) end end diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 1729141e9..747f2dc6b 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -11,10 +11,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do @spec validate_page_url(URI.t() | binary()) :: :ok | :error defp validate_page_url(page_url) when is_binary(page_url) do - validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld] + validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld]) page_url - |> AutoLinker.Parser.url?(scheme: true, validate_tld: validate_tld) + |> Linkify.Parser.url?(validate_tld: validate_tld) |> parse_uri(page_url) end diff --git a/mix.exs b/mix.exs index b638be541..c773a3162 100644 --- a/mix.exs +++ b/mix.exs @@ -167,9 +167,7 @@ defp deps do {:floki, "~> 0.25"}, {:timex, "~> 3.5"}, {:ueberauth, "~> 0.4"}, - {:auto_linker, - git: "https://git.pleroma.social/pleroma/auto_linker.git", - ref: "95e8188490e97505c56636c1379ffdf036c1fdde"}, + {:linkify, "~> 0.1.0"}, {:http_signatures, git: "https://git.pleroma.social/pleroma/http_signatures.git", ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"}, diff --git a/mix.lock b/mix.lock index 5ad49391d..458cda6cf 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,5 @@ %{ "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm", "11b18c220bcc2eab63b5470c038ef10eb6783bcb1fcdb11aa4137defa5ac1bb8"}, - "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "95e8188490e97505c56636c1379ffdf036c1fdde", [ref: "95e8188490e97505c56636c1379ffdf036c1fdde"]}, "base62": {:hex, :base62, "1.2.1", "4866763e08555a7b3917064e9eef9194c41667276c51b59de2bc42c6ea65f806", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "3b29948de2013d3f93aa898c884a9dff847e7aec75d9d6d8c1dc4c61c2716c42"}, "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"}, "bbcode": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/bbcode.git", "f2d267675e9a7e1ad1ea9beb4cc23382762b66c2", [ref: "v0.2.0"]}, @@ -62,6 +61,7 @@ "jose": {:hex, :jose, "1.10.1", "16d8e460dae7203c6d1efa3f277e25b5af8b659febfc2f2eb4bacf87f128b80a", [:mix, :rebar3], [], "hexpm", "3c7ddc8a9394b92891db7c2771da94bf819834a1a4c92e30857b7d582e2f8257"}, "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"}, "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"}, + "linkify": {:hex, :linkify, "0.1.0", "a2d35de64271c7fbbc7d8773adb9f595592b7fbaa581271c7733f39d3058bfa4", [:mix], [], "hexpm", "d3140ef8dbdcc53ef93a6a5374c11fffe0189f00d132161e9d020a417780bee7"}, "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "a10c6eb62cca416019663129699769f0c2ccf39428b3bb3c0cb38c718a0c186d"}, "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "d4b316c7222a85bbaa2fd7c6e90e37e953257ad196dc229505137c5e505e9eff"}, "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"}, From 3be64556dbe5618de3429a481f41eff917053ce8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 16 Jul 2020 13:11:03 -0500 Subject: [PATCH 02/18] Improve TOTP token and recovery input fields in OAuth login --- lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex | 2 +- lib/pleroma/web/templates/o_auth/mfa/totp.html.eex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex index 750f65386..5ab59b57b 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex @@ -10,7 +10,7 @@ <%= form_for @conn, mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %>
<%= label f, :code, "Recovery code" %> - <%= text_input f, :code %> + <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, spellcheck: false] %> <%= hidden_input f, :mfa_token, value: @mfa_token %> <%= hidden_input f, :state, value: @state %> <%= hidden_input f, :redirect_uri, value: @redirect_uri %> diff --git a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex index af6e546b0..8323ff8a1 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex @@ -10,7 +10,7 @@ <%= form_for @conn, mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %>
<%= label f, :code, "Authentication code" %> - <%= text_input f, :code %> + <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, maxlength: 6, pattern: "[0-9]{6}", spellcheck: false] %> <%= hidden_input f, :mfa_token, value: @mfa_token %> <%= hidden_input f, :state, value: @state %> <%= hidden_input f, :redirect_uri, value: @redirect_uri %> From 6fdaee7caed16a083d751b63c3dcfd119da57b21 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 16 Jul 2020 14:52:48 -0500 Subject: [PATCH 03/18] description.exs typofixes --- config/description.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/description.exs b/config/description.exs index a936b7abf..b97b0a7ec 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2238,13 +2238,13 @@ %{ key: :new_window, type: :boolean, - description: "Link URLs will open in new window/tab." + description: "Link URLs will open in a new window/tab." }, %{ key: :truncate, type: [:integer, false], description: - "Set to a number to truncate URLs longer then the number. Truncated URLs will end in `...`", + "Set to a number to truncate URLs longer than the number. Truncated URLs will end in `...`", suggestions: [15, false] }, %{ From 5701840d30f8f70721598115039519e3fe747186 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 16 Jul 2020 14:54:20 -0500 Subject: [PATCH 04/18] Use updated Linkify from git --- mix.exs | 4 +++- mix.lock | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 9886de666..727cdb71f 100644 --- a/mix.exs +++ b/mix.exs @@ -166,7 +166,9 @@ defp deps do {:floki, "~> 0.25"}, {:timex, "~> 3.5"}, {:ueberauth, "~> 0.4"}, - {:linkify, "~> 0.1.0"}, + {:linkify, + git: "https://git.pleroma.social/pleroma/elixir-libraries/linkify.git", + ref: "a08513aa7e879f056c44c5b8aea8c0fd073be5c8"}, {:http_signatures, git: "https://git.pleroma.social/pleroma/http_signatures.git", ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"}, diff --git a/mix.lock b/mix.lock index 150d14875..2025a965d 100644 --- a/mix.lock +++ b/mix.lock @@ -62,7 +62,7 @@ "jose": {:hex, :jose, "1.10.1", "16d8e460dae7203c6d1efa3f277e25b5af8b659febfc2f2eb4bacf87f128b80a", [:mix, :rebar3], [], "hexpm", "3c7ddc8a9394b92891db7c2771da94bf819834a1a4c92e30857b7d582e2f8257"}, "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"}, "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"}, - "linkify": {:hex, :linkify, "0.1.0", "a2d35de64271c7fbbc7d8773adb9f595592b7fbaa581271c7733f39d3058bfa4", [:mix], [], "hexpm", "d3140ef8dbdcc53ef93a6a5374c11fffe0189f00d132161e9d020a417780bee7"}, + "linkify": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/linkify.git", "a08513aa7e879f056c44c5b8aea8c0fd073be5c8", [ref: "a08513aa7e879f056c44c5b8aea8c0fd073be5c8"]}, "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "a10c6eb62cca416019663129699769f0c2ccf39428b3bb3c0cb38c718a0c186d"}, "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "d4b316c7222a85bbaa2fd7c6e90e37e953257ad196dc229505137c5e505e9eff"}, "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"}, From 880301985b49dd0e38a748a9c834eec4d550ea1b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 1 Jun 2020 19:51:41 -0500 Subject: [PATCH 05/18] Formatter: Test link with local mention --- test/formatter_test.exs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/formatter_test.exs b/test/formatter_test.exs index bef5a2c28..ae0d7b377 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -255,6 +255,16 @@ test "it can parse mentions and return the relevant users" do assert {_text, ^expected_mentions, []} = Formatter.linkify(text) end + + test "it parses URL containing local mention" do + _user = insert(:user, %{nickname: "lain"}) + + text = "https://example.com/@lain" + + expected = ~S(https://example.com/@lain) + + assert {^expected, [], []} = Formatter.linkify(text) + end end describe ".parse_tags" do From 613e096389d6945016e78499505c3ec5786d0ab0 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 16 Jul 2020 16:35:03 -0500 Subject: [PATCH 06/18] Migrate :auto_linker --> Pleroma.Formatter in ConfigDB --- .../20200716195806_autolinker_to_linkify.exs | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 priv/repo/migrations/20200716195806_autolinker_to_linkify.exs diff --git a/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs b/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs new file mode 100644 index 000000000..9ec4203eb --- /dev/null +++ b/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs @@ -0,0 +1,37 @@ +defmodule Pleroma.Repo.Migrations.AutolinkerToLinkify do + use Ecto.Migration + + alias Pleroma.Repo + alias Pleroma.ConfigDB + + @autolinker_path %{group: :auto_linker, key: :opts} + @linkify_path %{group: :pleroma, key: Pleroma.Formatter} + + @compat_opts [:class, :rel, :new_window, :truncate, :strip_prefix, :extra] + + def change do + with {:ok, {old, new}} <- maybe_get_params() do + move_config(old, new) + end + end + + defp move_config(%{} = old, %{} = new) do + {:ok, _} = ConfigDB.update_or_create(new) + {:ok, _} = ConfigDB.delete(old) + :ok + end + + defp maybe_get_params() do + with %ConfigDB{value: opts} <- ConfigDB.get_by_params(@autolinker_path), + %{} = opts <- transform_opts(opts), + %{} = linkify_params <- Map.put(@linkify_path, :value, opts) do + {:ok, {@autolinker_path, linkify_params}} + end + end + + defp transform_opts(opts) when is_list(opts) do + opts + |> Enum.into(%{}) + |> Map.take(@compat_opts) + end +end From 2a3bb23091fcc857ff8e678f81107186e0aeb3fe Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 17 Jul 2020 11:03:47 +0300 Subject: [PATCH 07/18] Bug issue template: remove choice in "Installation type" It made gitlab display "1 of 2 tasks completed" when one is chosen, which is totally not what this was used for. --- .gitlab/issue_templates/Bug.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab/issue_templates/Bug.md b/.gitlab/issue_templates/Bug.md index 9ce9b6918..dd0d6eb24 100644 --- a/.gitlab/issue_templates/Bug.md +++ b/.gitlab/issue_templates/Bug.md @@ -8,9 +8,7 @@ ### Environment -* Installation type: - - [ ] OTP - - [ ] From source +* Installation type (OTP or From Source): * Pleroma version (could be found in the "Version" tab of settings in Pleroma-FE): * Elixir version (`elixir -v` for from source installations, N/A for OTP): * Operating system: From 62438530e24d9553b8c1240ad7a39ea0906832b9 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 17 Jul 2020 08:19:49 -0500 Subject: [PATCH 08/18] TOTP length is configurable, so we can't hardcode this here. --- lib/pleroma/web/templates/o_auth/mfa/totp.html.eex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex index 8323ff8a1..af85777eb 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex @@ -10,7 +10,7 @@ <%= form_for @conn, mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %>
<%= label f, :code, "Authentication code" %> - <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, maxlength: 6, pattern: "[0-9]{6}", spellcheck: false] %> + <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, pattern: "[0-9]*", spellcheck: false] %> <%= hidden_input f, :mfa_token, value: @mfa_token %> <%= hidden_input f, :state, value: @state %> <%= hidden_input f, :redirect_uri, value: @redirect_uri %> From af376cbffbae3ae594e594813873719dfd69664e Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 17 Jul 2020 18:06:05 +0300 Subject: [PATCH 09/18] using atom keys in search params --- lib/pleroma/gopher/server.ex | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/gopher/server.ex b/lib/pleroma/gopher/server.ex index 3d56d50a9..e9f54c4c0 100644 --- a/lib/pleroma/gopher/server.ex +++ b/lib/pleroma/gopher/server.ex @@ -96,16 +96,18 @@ def response("") do def response("/main/public") do posts = - ActivityPub.fetch_public_activities(%{"type" => ["Create"], "local_only" => true}) - |> render_activities + %{type: ["Create"], local_only: true} + |> ActivityPub.fetch_public_activities() + |> render_activities() info("Welcome to the Public Timeline!") <> posts <> ".\r\n" end def response("/main/all") do posts = - ActivityPub.fetch_public_activities(%{"type" => ["Create"]}) - |> render_activities + %{type: ["Create"]} + |> ActivityPub.fetch_public_activities() + |> render_activities() info("Welcome to the Federated Timeline!") <> posts <> ".\r\n" end @@ -130,13 +132,14 @@ def response("/notices/" <> id) do def response("/users/" <> nickname) do with %User{} = user <- User.get_cached_by_nickname(nickname) do params = %{ - "type" => ["Create"], - "actor_id" => user.ap_id + type: ["Create"], + actor_id: user.ap_id } activities = - ActivityPub.fetch_public_activities(params) - |> render_activities + params + |> ActivityPub.fetch_public_activities() + |> render_activities() info("Posts by #{user.nickname}") <> activities <> ".\r\n" else From 20a496d2cbea18c563694c7026c0e951e99cfc3b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 17 Jul 2020 10:45:41 -0500 Subject: [PATCH 10/18] Expose the post formats in /api/v1/instance --- CHANGELOG.md | 1 + docs/API/differences_in_mastoapi_responses.md | 1 + lib/pleroma/web/mastodon_api/views/instance_view.ex | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a02f28241..75488f026 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). has been simplified down to `block_from_strangers`. - **Breaking:** Notification Settings API option for hiding push notification contents has been renamed to `hide_notification_contents` +- Mastodon API: Added `pleroma.metadata.post_formats` to /api/v1/instance
diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md index c4a9c6dad..38865dc68 100644 --- a/docs/API/differences_in_mastoapi_responses.md +++ b/docs/API/differences_in_mastoapi_responses.md @@ -236,6 +236,7 @@ Has theses additional parameters (which are the same as in Pleroma-API): - `pleroma.metadata.features`: A list of supported features - `pleroma.metadata.federation`: The federation restrictions of this instance - `pleroma.metadata.fields_limits`: A list of values detailing the length and count limitation for various instance-configurable fields. +- `pleroma.metadata.post_formats`: A list of the allowed post format types - `vapid_public_key`: The public key needed for push messages ## Markers diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 5deb0d7ed..cd3bc7f00 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -41,7 +41,8 @@ def render("show.json", _) do account_activation_required: Keyword.get(instance, :account_activation_required), features: features(), federation: federation(), - fields_limits: fields_limits() + fields_limits: fields_limits(), + post_formats: Config.get([:instance, :allowed_post_formats]) }, vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) } From 48f8b26c92880c0898daac3d691c61be0b891d0b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 16 Jul 2020 21:39:10 -0500 Subject: [PATCH 11/18] OpenAPI: Add :id to follower/following endpoints, fixes #1958 --- .../api_spec/operations/account_operation.ex | 2 ++ test/pagination_test.exs | 14 ++++++++++++++ .../controllers/account_controller_test.exs | 19 +++++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 952d9347b..50c8e0242 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -159,6 +159,7 @@ def followers_operation do "Accounts which follow the given account, if network is not hidden by the account owner.", parameters: [ %Reference{"$ref": "#/components/parameters/accountIdOrNickname"}, + Operation.parameter(:id, :query, :string, "ID of the resource owner"), with_relationships_param() | pagination_params() ], responses: %{ @@ -177,6 +178,7 @@ def following_operation do "Accounts which the given account is following, if network is not hidden by the account owner.", parameters: [ %Reference{"$ref": "#/components/parameters/accountIdOrNickname"}, + Operation.parameter(:id, :query, :string, "ID of the resource owner"), with_relationships_param() | pagination_params() ], responses: %{200 => Operation.response("Accounts", "application/json", array_of_accounts())} diff --git a/test/pagination_test.exs b/test/pagination_test.exs index 9165427ae..e526f23e8 100644 --- a/test/pagination_test.exs +++ b/test/pagination_test.exs @@ -54,6 +54,20 @@ test "paginates by min_id & limit", %{notes: notes} do assert length(paginated) == 1 end + + test "handles id gracefully", %{notes: notes} do + id = Enum.at(notes, 1).id |> Integer.to_string() + + paginated = + Pagination.fetch_paginated(Object, %{ + id: "9s99Hq44Cnv8PKBwWG", + max_id: id, + limit: 20, + offset: 0 + }) + + assert length(paginated) == 1 + end end describe "offset" do diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs index 9c7b5e9b2..c304487ea 100644 --- a/test/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/web/mastodon_api/controllers/account_controller_test.exs @@ -583,6 +583,15 @@ test "getting followers, pagination", %{user: user, conn: conn} do |> get("/api/v1/accounts/#{user.id}/followers?max_id=#{follower3_id}") |> json_response_and_validate_schema(200) + assert [%{"id" => ^follower2_id}, %{"id" => ^follower1_id}] = + conn + |> get( + "/api/v1/accounts/#{user.id}/followers?id=#{user.id}&limit=20&max_id=#{ + follower3_id + }" + ) + |> json_response_and_validate_schema(200) + res_conn = get(conn, "/api/v1/accounts/#{user.id}/followers?limit=1&max_id=#{follower3_id}") assert [%{"id" => ^follower2_id}] = json_response_and_validate_schema(res_conn, 200) @@ -654,6 +663,16 @@ test "getting following, pagination", %{user: user, conn: conn} do assert id2 == following2.id assert id1 == following1.id + res_conn = + get( + conn, + "/api/v1/accounts/#{user.id}/following?id=#{user.id}&limit=20&max_id=#{following3.id}" + ) + + assert [%{"id" => id2}, %{"id" => id1}] = json_response_and_validate_schema(res_conn, 200) + assert id2 == following2.id + assert id1 == following1.id + res_conn = get(conn, "/api/v1/accounts/#{user.id}/following?limit=1&max_id=#{following3.id}") From 4bac25e6f5d332b06e481d25b80efb62026c6a1e Mon Sep 17 00:00:00 2001 From: href Date: Sat, 18 Jul 2020 13:17:38 +0200 Subject: [PATCH 12/18] Don't enable Pleroma.HTTP.Middleware.FollowRedirects unless Gun is used --- lib/pleroma/http/http.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 6128bc4cf..b37b3fa89 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -69,7 +69,8 @@ def request(method, url, body, headers, options) when is_binary(url) do request = build_request(method, headers, options, url, body, params) adapter = Application.get_env(:tesla, :adapter) - client = Tesla.client([Pleroma.HTTP.Middleware.FollowRedirects], adapter) + + client = Tesla.client(adapter_middlewares(adapter), adapter) maybe_limit( fn -> @@ -107,4 +108,10 @@ defp maybe_limit(fun, Tesla.Adapter.Gun, opts) do defp maybe_limit(fun, _, _) do fun.() end + + defp adapter_middlewares(Tesla.Adapter.Gun) do + [Pleroma.HTTP.Middleware.FollowRedirects] + end + + defp adapter_middlewares(_), do: [] end From ae74c52e222c6e230ac04f484306c0a16aa270a5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 Jul 2020 15:10:48 -0500 Subject: [PATCH 13/18] Test angry face in formatter D:< #1968 --- test/formatter_test.exs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/formatter_test.exs b/test/formatter_test.exs index ae0d7b377..8713ab9c2 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -265,6 +265,26 @@ test "it parses URL containing local mention" do assert {^expected, [], []} = Formatter.linkify(text) end + + test "it correctly parses angry face D:< with mention" do + lain = + insert(:user, %{ + nickname: "lain@lain.com", + ap_id: "https://lain.com/users/lain", + id: "9qrWmR0cKniB0YU0TA" + }) + + text = "@lain@lain.com D:<" + + expected_text = + ~S(@lain D:<) + + expected_mentions = [ + {"@lain@lain.com", lain} + ] + + assert {^expected_text, ^expected_mentions, []} = Formatter.linkify(text) + end end describe ".parse_tags" do From 531c3ab9f34a41812f82e9e7dd3a604fbc11405d Mon Sep 17 00:00:00 2001 From: Dym Sohin Date: Mon, 20 Jul 2020 11:41:43 +0000 Subject: [PATCH 14/18] fix markdown rendering withing
; typo parent**s**_visible --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75488f026..080270073 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,7 +67,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
API Changes -- Mastodon API: Add pleroma.parents_visible field to statuses. + +- Mastodon API: Add pleroma.parent_visible field to statuses. - Mastodon API: Extended `/api/v1/instance`. - Mastodon API: Support for `include_types` in `/api/v1/notifications`. - Mastodon API: Added `/api/v1/notifications/:id/dismiss` endpoint. @@ -121,6 +122,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Follow request notifications
API Changes + - Admin API: `GET /api/pleroma/admin/need_reboot`.
@@ -188,6 +190,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking**: Using third party engines for user recommendation
API Changes + - **Breaking**: AdminAPI: migrate_from_db endpoint
From 5d263dfdb314f1ed6eca9e5c183149efcc58c367 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 20 Jul 2020 09:29:03 -0500 Subject: [PATCH 15/18] Update linkify to latest release --- mix.exs | 4 +--- mix.lock | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/mix.exs b/mix.exs index 727cdb71f..f44d7a887 100644 --- a/mix.exs +++ b/mix.exs @@ -166,9 +166,7 @@ defp deps do {:floki, "~> 0.25"}, {:timex, "~> 3.5"}, {:ueberauth, "~> 0.4"}, - {:linkify, - git: "https://git.pleroma.social/pleroma/elixir-libraries/linkify.git", - ref: "a08513aa7e879f056c44c5b8aea8c0fd073be5c8"}, + {:linkify, "~> 0.2.0"}, {:http_signatures, git: "https://git.pleroma.social/pleroma/http_signatures.git", ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"}, diff --git a/mix.lock b/mix.lock index 2025a965d..6430ddd19 100644 --- a/mix.lock +++ b/mix.lock @@ -62,7 +62,7 @@ "jose": {:hex, :jose, "1.10.1", "16d8e460dae7203c6d1efa3f277e25b5af8b659febfc2f2eb4bacf87f128b80a", [:mix, :rebar3], [], "hexpm", "3c7ddc8a9394b92891db7c2771da94bf819834a1a4c92e30857b7d582e2f8257"}, "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"}, "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"}, - "linkify": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/linkify.git", "a08513aa7e879f056c44c5b8aea8c0fd073be5c8", [ref: "a08513aa7e879f056c44c5b8aea8c0fd073be5c8"]}, + "linkify": {:hex, :linkify, "0.2.0", "2518bbbea21d2caa9d372424e1ad845b640c6630e2d016f1bd1f518f9ebcca28", [:mix], [], "hexpm", "b8ca8a68b79e30b7938d6c996085f3db14939f29538a59ca5101988bb7f917f6"}, "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "a10c6eb62cca416019663129699769f0c2ccf39428b3bb3c0cb38c718a0c186d"}, "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "d4b316c7222a85bbaa2fd7c6e90e37e953257ad196dc229505137c5e505e9eff"}, "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"}, From 204dddcfaaa5ff1113ef2f772ce5d6fcbbaaec6e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 20 Jul 2020 13:45:05 -0500 Subject: [PATCH 16/18] Pleroma.Formatter can have partial updates --- lib/pleroma/config/config_db.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex index f8141ced8..e5b7811aa 100644 --- a/lib/pleroma/config/config_db.ex +++ b/lib/pleroma/config/config_db.ex @@ -156,7 +156,6 @@ defp only_full_update?(%ConfigDB{group: group, key: key}) do {:quack, :meta}, {:mime, :types}, {:cors_plug, [:max_age, :methods, :expose, :headers]}, - {:linkify, :opts}, {:swarm, :node_blacklist}, {:logger, :backends} ] From 6afc6717d642060b086d01c2bfff5ead0aad1273 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 21 Jul 2020 10:31:58 +0300 Subject: [PATCH 17/18] copy tmp file if test depends on it --- test/upload/filter/anonymize_filename_test.exs | 2 ++ test/uploaders/local_test.exs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/upload/filter/anonymize_filename_test.exs b/test/upload/filter/anonymize_filename_test.exs index 2d5c580f1..adff70f57 100644 --- a/test/upload/filter/anonymize_filename_test.exs +++ b/test/upload/filter/anonymize_filename_test.exs @@ -9,6 +9,8 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilenameTest do alias Pleroma.Upload setup do + File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") + upload_file = %Upload{ name: "an… image.jpg", content_type: "image/jpg", diff --git a/test/uploaders/local_test.exs b/test/uploaders/local_test.exs index ae2cfef94..18122ff6c 100644 --- a/test/uploaders/local_test.exs +++ b/test/uploaders/local_test.exs @@ -14,6 +14,7 @@ test "it returns path to local folder for files" do describe "put_file/1" do test "put file to local folder" do + File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") file_path = "local_upload/files/image.jpg" file = %Pleroma.Upload{ @@ -32,6 +33,7 @@ test "put file to local folder" do describe "delete_file/1" do test "deletes local file" do + File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") file_path = "local_upload/files/image.jpg" file = %Pleroma.Upload{ From 696c13ce54aff25737f8f753a94747d79b9c54b0 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 21 Jul 2020 22:17:34 +0000 Subject: [PATCH 18/18] Revert "Merge branch 'linkify' into 'develop'" This reverts merge request !2677 --- CHANGELOG.md | 1 - config/config.exs | 18 +++++---- config/description.exs | 20 +++------- docs/configuration/cheatsheet.md | 35 +++++++++--------- lib/pleroma/config/config_db.ex | 1 + lib/pleroma/formatter.ex | 26 ++++++------- lib/pleroma/web/rich_media/helpers.ex | 4 +- mix.exs | 4 +- mix.lock | 2 +- .../20200716195806_autolinker_to_linkify.exs | 37 ------------------- test/formatter_test.exs | 30 --------------- 11 files changed, 52 insertions(+), 126 deletions(-) delete mode 100644 priv/repo/migrations/20200716195806_autolinker_to_linkify.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index f4397ec3c..080270073 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - **Breaking:** Elixir >=1.9 is now required (was >= 1.8) -- **Breaking:** Configuration: `:auto_linker, :opts` moved to `:pleroma, Pleroma.Formatter`. Old config namespace is deprecated. - In Conversations, return only direct messages as `last_status` - Using the `only_media` filter on timelines will now exclude reblog media - MFR policy to set global expiration for all local Create activities diff --git a/config/config.exs b/config/config.exs index 406bf2a9b..2d3f35e70 100644 --- a/config/config.exs +++ b/config/config.exs @@ -527,14 +527,16 @@ federator_outgoing: 5 ] -config :pleroma, Pleroma.Formatter, - class: false, - rel: "ugc", - new_window: false, - truncate: false, - strip_prefix: false, - extra: true, - validate_tld: :no_scheme +config :auto_linker, + opts: [ + extra: true, + # TODO: Set to :no_scheme when it works properly + validate_tld: true, + class: false, + strip_prefix: false, + new_window: false, + rel: "ugc" + ] config :pleroma, :ldap, enabled: System.get_env("LDAP_ENABLED") == "true", diff --git a/config/description.exs b/config/description.exs index b97b0a7ec..f1c6773f1 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2216,12 +2216,11 @@ ] }, %{ - group: :pleroma, - key: Pleroma.Formatter, + group: :auto_linker, + key: :opts, label: "Auto Linker", type: :group, - description: - "Configuration for Pleroma's link formatter which parses mentions, hashtags, and URLs.", + description: "Configuration for the auto_linker library", children: [ %{ key: :class, @@ -2238,31 +2237,24 @@ %{ key: :new_window, type: :boolean, - description: "Link URLs will open in a new window/tab." + description: "Link URLs will open in new window/tab" }, %{ key: :truncate, type: [:integer, false], description: - "Set to a number to truncate URLs longer than the number. Truncated URLs will end in `...`", + "Set to a number to truncate URLs longer then the number. Truncated URLs will end in `..`", suggestions: [15, false] }, %{ key: :strip_prefix, type: :boolean, - description: "Strip the scheme prefix." + description: "Strip the scheme prefix" }, %{ key: :extra, type: :boolean, description: "Link URLs with rarely used schemes (magnet, ipfs, irc, etc.)" - }, - %{ - key: :validate_tld, - type: [:atom, :boolean], - description: - "Set to false to disable TLD validation for URLs/emails. Can be set to :no_scheme to validate TLDs only for URLs without a scheme (e.g `example.com` will be validated, but `http://example.loki` won't)", - suggestions: [:no_scheme, true] } ] }, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 042ad30c9..6c1babba3 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -934,29 +934,30 @@ Configure OAuth 2 provider capabilities: ### :uri_schemes * `valid_schemes`: List of the scheme part that is considered valid to be an URL. -### Pleroma.Formatter +### :auto_linker -Configuration for Pleroma's link formatter which parses mentions, hashtags, and URLs. +Configuration for the `auto_linker` library: -* `class` - specify the class to be added to the generated link (default: `false`) -* `rel` - specify the rel attribute (default: `ugc`) -* `new_window` - adds `target="_blank"` attribute (default: `false`) -* `truncate` - Set to a number to truncate URLs longer then the number. Truncated URLs will end in `...` (default: `false`) -* `strip_prefix` - Strip the scheme prefix (default: `false`) -* `extra` - link URLs with rarely used schemes (magnet, ipfs, irc, etc.) (default: `true`) -* `validate_tld` - Set to false to disable TLD validation for URLs/emails. Can be set to :no_scheme to validate TLDs only for urls without a scheme (e.g `example.com` will be validated, but `http://example.loki` won't) (default: `:no_scheme`) +* `class: "auto-linker"` - specify the class to be added to the generated link. false to clear. +* `rel: "noopener noreferrer"` - override the rel attribute. false to clear. +* `new_window: true` - set to false to remove `target='_blank'` attribute. +* `scheme: false` - Set to true to link urls with schema `http://google.com`. +* `truncate: false` - Set to a number to truncate urls longer then the number. Truncated urls will end in `..`. +* `strip_prefix: true` - Strip the scheme prefix. +* `extra: false` - link urls with rarely used schemes (magnet, ipfs, irc, etc.). Example: ```elixir -config :pleroma, Pleroma.Formatter, - class: false, - rel: "ugc", - new_window: false, - truncate: false, - strip_prefix: false, - extra: true, - validate_tld: :no_scheme +config :auto_linker, + opts: [ + scheme: true, + extra: true, + class: false, + strip_prefix: false, + new_window: false, + rel: "ugc" + ] ``` ## Custom Runtime Modules (`:modules`) diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex index e5b7811aa..1a89d8895 100644 --- a/lib/pleroma/config/config_db.ex +++ b/lib/pleroma/config/config_db.ex @@ -156,6 +156,7 @@ defp only_full_update?(%ConfigDB{group: group, key: key}) do {:quack, :meta}, {:mime, :types}, {:cors_plug, [:max_age, :methods, :expose, :headers]}, + {:auto_linker, :opts}, {:swarm, :node_blacklist}, {:logger, :backends} ] diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 0c450eae4..02a93a8dc 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -10,15 +10,11 @@ defmodule Pleroma.Formatter do @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/ - defp linkify_opts do - Pleroma.Config.get(Pleroma.Formatter) ++ - [ - hashtag: true, - hashtag_handler: &Pleroma.Formatter.hashtag_handler/4, - mention: true, - mention_handler: &Pleroma.Formatter.mention_handler/4 - ] - end + @auto_linker_config hashtag: true, + hashtag_handler: &Pleroma.Formatter.hashtag_handler/4, + mention: true, + mention_handler: &Pleroma.Formatter.mention_handler/4, + scheme: true def escape_mention_handler("@" <> nickname = mention, buffer, _, _) do case User.get_cached_by_nickname(nickname) do @@ -84,19 +80,19 @@ def hashtag_handler("#" <> tag = tag_text, _buffer, _opts, acc) do @spec linkify(String.t(), keyword()) :: {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]} def linkify(text, options \\ []) do - options = linkify_opts() ++ options + options = options ++ @auto_linker_config if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text) acc = %{mentions: MapSet.new(), tags: MapSet.new()} - {text_mentions, %{mentions: mentions}} = Linkify.link_map(mentions, acc, options) - {text_rest, %{tags: tags}} = Linkify.link_map(rest, acc, options) + {text_mentions, %{mentions: mentions}} = AutoLinker.link_map(mentions, acc, options) + {text_rest, %{tags: tags}} = AutoLinker.link_map(rest, acc, options) {text_mentions <> text_rest, MapSet.to_list(mentions), MapSet.to_list(tags)} else acc = %{mentions: MapSet.new(), tags: MapSet.new()} - {text, %{mentions: mentions, tags: tags}} = Linkify.link_map(text, acc, options) + {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options) {text, MapSet.to_list(mentions), MapSet.to_list(tags)} end @@ -115,9 +111,9 @@ def mentions_escape(text, options \\ []) do if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text) - Linkify.link(mentions, options) <> Linkify.link(rest, options) + AutoLinker.link(mentions, options) <> AutoLinker.link(rest, options) else - Linkify.link(text, options) + AutoLinker.link(text, options) end end diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 747f2dc6b..1729141e9 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -11,10 +11,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do @spec validate_page_url(URI.t() | binary()) :: :ok | :error defp validate_page_url(page_url) when is_binary(page_url) do - validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld]) + validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld] page_url - |> Linkify.Parser.url?(validate_tld: validate_tld) + |> AutoLinker.Parser.url?(scheme: true, validate_tld: validate_tld) |> parse_uri(page_url) end diff --git a/mix.exs b/mix.exs index f44d7a887..52b4cf268 100644 --- a/mix.exs +++ b/mix.exs @@ -166,7 +166,9 @@ defp deps do {:floki, "~> 0.25"}, {:timex, "~> 3.5"}, {:ueberauth, "~> 0.4"}, - {:linkify, "~> 0.2.0"}, + {:auto_linker, + git: "https://git.pleroma.social/pleroma/auto_linker.git", + ref: "95e8188490e97505c56636c1379ffdf036c1fdde"}, {:http_signatures, git: "https://git.pleroma.social/pleroma/http_signatures.git", ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"}, diff --git a/mix.lock b/mix.lock index 6430ddd19..8dd37a40f 100644 --- a/mix.lock +++ b/mix.lock @@ -1,5 +1,6 @@ %{ "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm", "11b18c220bcc2eab63b5470c038ef10eb6783bcb1fcdb11aa4137defa5ac1bb8"}, + "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "95e8188490e97505c56636c1379ffdf036c1fdde", [ref: "95e8188490e97505c56636c1379ffdf036c1fdde"]}, "base62": {:hex, :base62, "1.2.1", "4866763e08555a7b3917064e9eef9194c41667276c51b59de2bc42c6ea65f806", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "3b29948de2013d3f93aa898c884a9dff847e7aec75d9d6d8c1dc4c61c2716c42"}, "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"}, "bbcode": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/bbcode.git", "f2d267675e9a7e1ad1ea9beb4cc23382762b66c2", [ref: "v0.2.0"]}, @@ -62,7 +63,6 @@ "jose": {:hex, :jose, "1.10.1", "16d8e460dae7203c6d1efa3f277e25b5af8b659febfc2f2eb4bacf87f128b80a", [:mix, :rebar3], [], "hexpm", "3c7ddc8a9394b92891db7c2771da94bf819834a1a4c92e30857b7d582e2f8257"}, "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"}, "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"}, - "linkify": {:hex, :linkify, "0.2.0", "2518bbbea21d2caa9d372424e1ad845b640c6630e2d016f1bd1f518f9ebcca28", [:mix], [], "hexpm", "b8ca8a68b79e30b7938d6c996085f3db14939f29538a59ca5101988bb7f917f6"}, "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "a10c6eb62cca416019663129699769f0c2ccf39428b3bb3c0cb38c718a0c186d"}, "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "d4b316c7222a85bbaa2fd7c6e90e37e953257ad196dc229505137c5e505e9eff"}, "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"}, diff --git a/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs b/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs deleted file mode 100644 index 9ec4203eb..000000000 --- a/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs +++ /dev/null @@ -1,37 +0,0 @@ -defmodule Pleroma.Repo.Migrations.AutolinkerToLinkify do - use Ecto.Migration - - alias Pleroma.Repo - alias Pleroma.ConfigDB - - @autolinker_path %{group: :auto_linker, key: :opts} - @linkify_path %{group: :pleroma, key: Pleroma.Formatter} - - @compat_opts [:class, :rel, :new_window, :truncate, :strip_prefix, :extra] - - def change do - with {:ok, {old, new}} <- maybe_get_params() do - move_config(old, new) - end - end - - defp move_config(%{} = old, %{} = new) do - {:ok, _} = ConfigDB.update_or_create(new) - {:ok, _} = ConfigDB.delete(old) - :ok - end - - defp maybe_get_params() do - with %ConfigDB{value: opts} <- ConfigDB.get_by_params(@autolinker_path), - %{} = opts <- transform_opts(opts), - %{} = linkify_params <- Map.put(@linkify_path, :value, opts) do - {:ok, {@autolinker_path, linkify_params}} - end - end - - defp transform_opts(opts) when is_list(opts) do - opts - |> Enum.into(%{}) - |> Map.take(@compat_opts) - end -end diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 8713ab9c2..bef5a2c28 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -255,36 +255,6 @@ test "it can parse mentions and return the relevant users" do assert {_text, ^expected_mentions, []} = Formatter.linkify(text) end - - test "it parses URL containing local mention" do - _user = insert(:user, %{nickname: "lain"}) - - text = "https://example.com/@lain" - - expected = ~S(https://example.com/@lain) - - assert {^expected, [], []} = Formatter.linkify(text) - end - - test "it correctly parses angry face D:< with mention" do - lain = - insert(:user, %{ - nickname: "lain@lain.com", - ap_id: "https://lain.com/users/lain", - id: "9qrWmR0cKniB0YU0TA" - }) - - text = "@lain@lain.com D:<" - - expected_text = - ~S(@lain D:<) - - expected_mentions = [ - {"@lain@lain.com", lain} - ] - - assert {^expected_text, ^expected_mentions, []} = Formatter.linkify(text) - end end describe ".parse_tags" do