From 869e5ccfe1defb4b1a711729f75b7893f6f82573 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 24 May 2017 13:53:20 +0200 Subject: [PATCH 1/7] Fix factory. --- test/support/factory.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/support/factory.ex b/test/support/factory.ex index 3cf35e3bc..bb3bdb02c 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -22,7 +22,7 @@ def note_factory do "id" => Pleroma.Web.ActivityPub.Utils.generate_object_id, "actor" => user.ap_id, "to" => ["https://www.w3.org/ns/activitystreams#Public"], - "published_at" => DateTime.utc_now() |> DateTime.to_iso8601, + "published" => DateTime.utc_now() |> DateTime.to_iso8601, "likes" => [], "like_count" => 0, "context" => "2hu", @@ -42,7 +42,7 @@ def note_activity_factory do "actor" => note.data["actor"], "to" => note.data["to"], "object" => note.data, - "published_at" => DateTime.utc_now() |> DateTime.to_iso8601, + "published" => DateTime.utc_now() |> DateTime.to_iso8601, "context" => note.data["context"] } From 94e980d6b4ab91ee37c30b266eb91e748e23861c Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 24 May 2017 13:53:31 +0200 Subject: [PATCH 2/7] Output proper published data in ostatus. --- TODO.txt | 5 +- .../web/ostatus/activity_representer.ex | 30 ++++------ .../web/ostatus/activity_representer_test.exs | 57 ++++--------------- 3 files changed, 26 insertions(+), 66 deletions(-) diff --git a/TODO.txt b/TODO.txt index 304e95e77..3138d0848 100644 --- a/TODO.txt +++ b/TODO.txt @@ -5,5 +5,8 @@ Unliking: WEBSUB: - Add unsubscription -- Add periodical renewal + +OSTATUS: + +- Save and output 'updated' diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index ccf71218c..aa13cd62b 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -28,10 +28,8 @@ def to_simple_form(activity, user, with_author \\ false) def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do h = fn(str) -> [to_charlist(str)] end - updated_at = activity.updated_at - |> NaiveDateTime.to_iso8601 - inserted_at = activity.inserted_at - |> NaiveDateTime.to_iso8601 + updated_at = activity.data["published"] + inserted_at = activity.data["published"] attachments = Enum.map(activity.data["object"]["attachment"] || [], fn(attachment) -> url = hd(attachment["url"]) @@ -62,10 +60,8 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, def to_simple_form(%{data: %{"type" => "Like"}} = activity, user, with_author) do h = fn(str) -> [to_charlist(str)] end - updated_at = activity.updated_at - |> NaiveDateTime.to_iso8601 - inserted_at = activity.inserted_at - |> NaiveDateTime.to_iso8601 + updated_at = activity.data["published"] + inserted_at = activity.data["published"] in_reply_to = get_in_reply_to(activity.data) author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] @@ -92,10 +88,8 @@ def to_simple_form(%{data: %{"type" => "Like"}} = activity, user, with_author) d def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_author) do h = fn(str) -> [to_charlist(str)] end - updated_at = activity.updated_at - |> NaiveDateTime.to_iso8601 - inserted_at = activity.inserted_at - |> NaiveDateTime.to_iso8601 + updated_at = activity.data["published"] + inserted_at = activity.data["published"] in_reply_to = get_in_reply_to(activity.data) author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] @@ -124,10 +118,8 @@ def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_autho def to_simple_form(%{data: %{"type" => "Follow"}} = activity, user, with_author) do h = fn(str) -> [to_charlist(str)] end - updated_at = activity.updated_at - |> NaiveDateTime.to_iso8601 - inserted_at = activity.inserted_at - |> NaiveDateTime.to_iso8601 + updated_at = activity.data["published"] + inserted_at = activity.data["published"] author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] @@ -153,10 +145,8 @@ def to_simple_form(%{data: %{"type" => "Follow"}} = activity, user, with_author) def to_simple_form(%{data: %{"type" => "Undo"}} = activity, user, with_author) do h = fn(str) -> [to_charlist(str)] end - updated_at = activity.updated_at - |> NaiveDateTime.to_iso8601 - inserted_at = activity.inserted_at - |> NaiveDateTime.to_iso8601 + updated_at = activity.data["published"] + inserted_at = activity.data["published"] author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] follow_activity = Activity.get_by_ap_id(activity.data["object"]) diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 0f011f31c..b23334d55 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -9,10 +9,6 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenterTest do test "a note activity" do note_activity = insert(:note_activity) - updated_at = note_activity.updated_at - |> NaiveDateTime.to_iso8601 - inserted_at = note_activity.inserted_at - |> NaiveDateTime.to_iso8601 user = User.get_cached_by_ap_id(note_activity.data["actor"]) @@ -22,8 +18,8 @@ test "a note activity" do #{note_activity.data["object"]["id"]} New note by #{user.nickname} #{note_activity.data["object"]["content"]} - #{inserted_at} - #{updated_at} + #{note_activity.data["published"]} + #{note_activity.data["published"]} #{note_activity.data["context"]} @@ -47,11 +43,6 @@ test "a reply note" do data = %{answer.data | "object" => object} answer = %{answer | data: data} - updated_at = answer.updated_at - |> NaiveDateTime.to_iso8601 - inserted_at = answer.inserted_at - |> NaiveDateTime.to_iso8601 - user = User.get_cached_by_ap_id(answer.data["actor"]) expected = """ @@ -60,8 +51,8 @@ test "a reply note" do #{answer.data["object"]["id"]} New note by #{user.nickname} #{answer.data["object"]["content"]} - #{inserted_at} - #{updated_at} + #{answer.data["published"]} + #{answer.data["published"]} #{answer.data["context"]} @@ -92,19 +83,14 @@ test "an announce activity" do |> :xmerl.export_simple_content(:xmerl_xml) |> to_string - updated_at = announce.updated_at - |> NaiveDateTime.to_iso8601 - inserted_at = announce.inserted_at - |> NaiveDateTime.to_iso8601 - expected = """ http://activitystrea.ms/schema/1.0/activity http://activitystrea.ms/schema/1.0/share #{announce.data["id"]} #{user.nickname} repeated a notice RT #{note.data["object"]["content"]} - #{inserted_at} - #{updated_at} + #{announce.data["published"]} + #{announce.data["published"]} #{announce.data["context"]} @@ -126,12 +112,6 @@ test "a like activity" do user = insert(:user) {:ok, like, _note} = ActivityPub.like(user, note) - # TODO: Are these the correct dates? - updated_at = like.updated_at - |> NaiveDateTime.to_iso8601 - inserted_at = like.inserted_at - |> NaiveDateTime.to_iso8601 - tuple = ActivityRepresenter.to_simple_form(like, user) refute is_nil(tuple) @@ -142,8 +122,8 @@ test "a like activity" do #{like.data["id"]} New favorite by #{user.nickname} #{user.nickname} favorited something - #{inserted_at} - #{updated_at} + #{like.data["published"]} + #{like.data["published"]} http://activitystrea.ms/schema/1.0/note #{note.data["id"]} @@ -168,13 +148,6 @@ test "a follow activity" do "to" => [followed.ap_id] }) - - # TODO: Are these the correct dates? - updated_at = activity.updated_at - |> NaiveDateTime.to_iso8601 - inserted_at = activity.inserted_at - |> NaiveDateTime.to_iso8601 - tuple = ActivityRepresenter.to_simple_form(activity, follower) refute is_nil(tuple) @@ -187,8 +160,8 @@ test "a follow activity" do #{activity.data["id"]} #{follower.nickname} started following #{activity.data["object"]} #{follower.nickname} started following #{activity.data["object"]} - #{inserted_at} - #{updated_at} + #{activity.data["published"]} + #{activity.data["published"]} http://activitystrea.ms/schema/1.0/person #{activity.data["object"]} @@ -207,12 +180,6 @@ test "an unfollow activity" do {:ok, _activity} = ActivityPub.follow(follower, followed) {:ok, activity} = ActivityPub.unfollow(follower, followed) - # TODO: Are these the correct dates? - updated_at = activity.updated_at - |> NaiveDateTime.to_iso8601 - inserted_at = activity.inserted_at - |> NaiveDateTime.to_iso8601 - tuple = ActivityRepresenter.to_simple_form(activity, follower) refute is_nil(tuple) @@ -225,8 +192,8 @@ test "an unfollow activity" do #{activity.data["id"]} #{follower.nickname} stopped following #{followed.ap_id} #{follower.nickname} stopped following #{followed.ap_id} - #{inserted_at} - #{updated_at} + #{activity.data["published"]} + #{activity.data["published"]} http://activitystrea.ms/schema/1.0/person #{followed.ap_id} From bdcf42180fa67e43f13584d5e19f26d751199d03 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 24 May 2017 17:34:38 +0200 Subject: [PATCH 3/7] Save remote user bio and update if we see new data. --- lib/pleroma/web/ostatus/ostatus.ex | 31 ++++++++++++++++++++++-------- lib/pleroma/web/websub/websub.ex | 4 +++- test/fixtures/lambadalambda.atom | 2 +- test/web/ostatus/ostatus_test.exs | 9 ++++++++- test/web/websub/websub_test.exs | 1 + 5 files changed, 36 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 02fc273cf..9300de16e 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -228,16 +228,30 @@ def handle_note(entry, doc \\ nil) do end end + def maybe_update(doc, user) do + old_data = %{ + avatar: user.avatar, + bio: user.bio, + name: user.name + } + + with false <- user.local, + avatar <- make_avatar_object(doc), + bio when not is_nil(bio) <- string_from_xpath("//author[1]/summary", doc), + name when not is_nil(name) <- string_from_xpath("//author[1]/poco:displayName", doc), + new_data <- %{avatar: avatar, name: name, bio: bio}, + false <- new_data == old_data do + change = Ecto.Changeset.change(user, new_data) + Repo.update(change) + else e -> + {:ok, user} + end + end + def find_make_or_update_user(doc) do uri = string_from_xpath("//author/uri[1]", doc) with {:ok, user} <- find_or_make_user(uri) do - avatar = make_avatar_object(doc) - if !user.local && user.avatar != avatar do - change = Ecto.Changeset.change(user, %{avatar: avatar}) - Repo.update(change) - else - {:ok, user} - end + maybe_update(doc, user) end end @@ -261,7 +275,8 @@ def make_user(uri) do nickname: info["nickname"] <> "@" <> info["host"], ap_id: info["uri"], info: info, - avatar: info["avatar"] + avatar: info["avatar"], + bio: info["bio"] } with %User{} = user <- User.get_by_ap_id(data.ap_id) do {:ok, user} diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index e46e0a2ce..a683f6da4 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -156,6 +156,7 @@ def gather_feed_data(topic, getter \\ &@httpoison.get/1) do preferredUsername = XML.string_from_xpath("/feed/author[1]/poco:preferredUsername", doc) displayName = XML.string_from_xpath("/feed/author[1]/poco:displayName", doc) avatar = OStatus.make_avatar_object(doc) + bio = XML.string_from_xpath("/feed/author[1]/summary", doc) {:ok, %{ "uri" => uri, @@ -163,7 +164,8 @@ def gather_feed_data(topic, getter \\ &@httpoison.get/1) do "nickname" => preferredUsername || name, "name" => displayName || name, "host" => URI.parse(uri).host, - "avatar" => avatar + "avatar" => avatar, + "bio" => bio }} else e -> {:error, e} diff --git a/test/fixtures/lambadalambda.atom b/test/fixtures/lambadalambda.atom index 35e506420..964a416f7 100644 --- a/test/fixtures/lambadalambda.atom +++ b/test/fixtures/lambadalambda.atom @@ -11,7 +11,7 @@ https://mastodon.social/users/lambadalambda lambadalambda lambadalambda@mastodon.social - + a cool dude. diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 6599b00b3..e40fae78c 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -233,6 +233,7 @@ test "tries to use the information in poco fields" do assert user.local == false assert user.info["uri"] == uri assert user.ap_id == uri + assert user.bio == "Call me Deacon Blues." assert user.avatar["type"] == "Image" {:ok, user_again} = OStatus.find_or_make_user(uri) @@ -244,7 +245,9 @@ test "find_make_or_update_user takes an author element and returns an updated us uri = "https://social.heldscal.la/user/23211" {:ok, user} = OStatus.find_or_make_user(uri) - change = Ecto.Changeset.change(user, %{avatar: nil}) + old_name = user.name + old_bio = user.bio + change = Ecto.Changeset.change(user, %{avatar: nil, bio: nil, old_name: nil}) {:ok, user} = Repo.update(change) refute user.avatar @@ -253,6 +256,8 @@ test "find_make_or_update_user takes an author element and returns an updated us [author] = :xmerl_xpath.string('//author[1]', doc) {:ok, user} = OStatus.find_make_or_update_user(author) assert user.avatar["type"] == "Image" + assert user.name == old_name + assert user.bio == old_bio {:ok, user_again} = OStatus.find_make_or_update_user(author) assert user_again == user @@ -277,6 +282,7 @@ test "it returns user info in a hash" do "uri" => "https://social.heldscal.la/user/29191", "host" => "social.heldscal.la", "fqn" => user, + "bio" => "cofe", "avatar" => %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} } assert data == expected @@ -299,6 +305,7 @@ test "it works with the uri" do "uri" => "https://social.heldscal.la/user/29191", "host" => "social.heldscal.la", "fqn" => user, + "bio" => "cofe", "avatar" => %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} } assert data == expected diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index aaed9226e..1ca573d66 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -120,6 +120,7 @@ test "discovers the hub and canonical url" do "nickname" => "lambadalambda", "name" => "Critical Value", "host" => "mastodon.social", + "bio" => "a cool dude.", "avatar" => %{"type" => "Image", "url" => [%{"href" => "https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif?1492379244", "mediaType" => "image/gif", "type" => "Link"}]} } From fb787731122477c80dab6ab0ca9e3fdcc3461ea2 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 29 May 2017 11:33:40 +0200 Subject: [PATCH 4/7] Make longer bios possible. --- priv/repo/migrations/20170529093232_longer_bios.exs | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 priv/repo/migrations/20170529093232_longer_bios.exs diff --git a/priv/repo/migrations/20170529093232_longer_bios.exs b/priv/repo/migrations/20170529093232_longer_bios.exs new file mode 100644 index 000000000..f8c14b117 --- /dev/null +++ b/priv/repo/migrations/20170529093232_longer_bios.exs @@ -0,0 +1,9 @@ +defmodule Pleroma.Repo.Migrations.LongerBios do + use Ecto.Migration + + def change do + alter table(:users) do + modify :bio, :text + end + end +end From c6aa60c829abb7eeff3e0aec0b8b5eb33b8a4947 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 30 May 2017 14:26:31 +0200 Subject: [PATCH 5/7] Shorten names in links. --- lib/pleroma/web/twitter_api/utils.ex | 3 ++- test/web/twitter_api/twitter_api_test.exs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/twitter_api/utils.ex b/lib/pleroma/web/twitter_api/utils.ex index 82e3620f2..91dfa191f 100644 --- a/lib/pleroma/web/twitter_api/utils.ex +++ b/lib/pleroma/web/twitter_api/utils.ex @@ -38,7 +38,8 @@ def add_user_links(text, mentions) do end) Enum.reduce(mentions, step_one, fn ({match, %User{ap_id: ap_id}, uuid}, text) -> - String.replace(text, uuid, "#{match}") + short_match = String.split(match, "@") |> tl() |> hd() + String.replace(text, uuid, "@#{short_match}") end) end diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index 4df1aba9e..df1022d6b 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -263,7 +263,7 @@ test "it adds user links to an existing text" do archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"}) mentions = TwitterAPI.parse_mentions(text) - expected_text = "@gsimg According to @archaeme, that is @daggsy. Also hello @archaeme@archae.me" + expected_text = "@gsimg According to @archaeme, that is @daggsy. Also hello @archaeme" assert Utils.add_user_links(text, mentions) == expected_text end From 80705c7a204be461a5c7a4abb8c675f4482a9cd2 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 31 May 2017 17:48:22 +0200 Subject: [PATCH 6/7] Redirect to frontend on html notice path. --- lib/pleroma/web/ostatus/ostatus_controller.ex | 10 ++++++++-- lib/pleroma/web/router.ex | 3 +-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index fd8dcdf52..948996d01 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -47,7 +47,10 @@ def object(conn, %{"uuid" => uuid}) do with id <- o_status_url(conn, :object, uuid), %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id), %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do - represent_activity(conn, activity, user) + case get_format(conn) do + "html" -> redirect(conn, to: "/notice/#{activity.id}") + _ -> represent_activity(conn, activity, user) + end end end @@ -55,7 +58,10 @@ def activity(conn, %{"uuid" => uuid}) do with id <- o_status_url(conn, :activity, uuid), %Activity{} = activity <- Activity.get_by_ap_id(id), %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do - represent_activity(conn, activity, user) + case get_format(conn) do + "html" -> redirect(conn, to: "/notice/#{activity.id}") + _ -> represent_activity(conn, activity, user) + end end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 12159cffc..07c1f8028 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -70,7 +70,7 @@ def user_fetcher(username) do end pipeline :ostatus do - plug :accepts, ["xml", "atom"] + plug :accepts, ["xml", "atom", "html"] end scope "/", Pleroma.Web do @@ -97,7 +97,6 @@ def user_fetcher(username) do scope "/", Fallback do get "/*path", RedirectController, :redirector end - end defmodule Fallback.RedirectController do From 4cfe1e0c9becebbc5665eb28b187451f96c73555 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 31 May 2017 18:24:50 +0200 Subject: [PATCH 7/7] Make timeouts longer, some servers are very slow. --- lib/pleroma/web/ostatus/ostatus.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 9300de16e..29291b7a5 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -341,9 +341,9 @@ def get_atom_url(body) do def fetch_activity_from_html_url(url) do Logger.debug("Trying to fetch #{url}") - with {:ok, %{body: body}} <- @httpoison.get(url, [], follow_redirect: true), + with {:ok, %{body: body}} <- @httpoison.get(url, [], follow_redirect: true, timeout: 10000, recv_timeout: 20000), {:ok, atom_url} <- get_atom_url(body), - {:ok, %{status_code: code, body: body}} when code in 200..299 <- @httpoison.get(atom_url, [], follow_redirect: true) do + {:ok, %{status_code: code, body: body}} when code in 200..299 <- @httpoison.get(atom_url, [], follow_redirect: true, timeout: 10000, recv_timeout: 20000) do Logger.debug("Got document from #{url}, handling...") handle_incoming(body) else e -> Logger.debug("Couldn't get #{url}: #{inspect(e)}")