From 43d7a4b2cfe686c15b68f6599ce16446fa1dfab0 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 24 Apr 2017 08:48:52 +0200 Subject: [PATCH 01/88] Add basic fields to support remote users. --- lib/pleroma/user.ex | 2 ++ .../migrations/20170423154511_add_fields_to_users.exs | 10 ++++++++++ 2 files changed, 12 insertions(+) create mode 100644 priv/repo/migrations/20170423154511_add_fields_to_users.exs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3ce07d510..160acbdb9 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -15,6 +15,8 @@ defmodule Pleroma.User do field :following, { :array, :string }, default: [] field :ap_id, :string field :avatar, :map + field :local, :boolean, default: true + field :info, :map timestamps() end diff --git a/priv/repo/migrations/20170423154511_add_fields_to_users.exs b/priv/repo/migrations/20170423154511_add_fields_to_users.exs new file mode 100644 index 000000000..84de74bc4 --- /dev/null +++ b/priv/repo/migrations/20170423154511_add_fields_to_users.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.AddFieldsToUsers do + use Ecto.Migration + + def change do + alter table(:users) do + add :local, :boolean, default: true + add :info, :map + end + end +end From 34d3aea92f1bce7ba51a44ef1fdc68e47822c3a4 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 24 Apr 2017 18:46:02 +0200 Subject: [PATCH 02/88] Add incoming xml fixtures. --- test/fixtures/incoming_note_activity.xml | 40 ++++++++++++++++++++++++ test/fixtures/user_full.xml | 10 ++++++ test/fixtures/user_name_only.xml | 5 +++ 3 files changed, 55 insertions(+) create mode 100644 test/fixtures/incoming_note_activity.xml create mode 100644 test/fixtures/user_full.xml create mode 100644 test/fixtures/user_name_only.xml diff --git a/test/fixtures/incoming_note_activity.xml b/test/fixtures/incoming_note_activity.xml new file mode 100644 index 000000000..e54b25e39 --- /dev/null +++ b/test/fixtures/incoming_note_activity.xml @@ -0,0 +1,40 @@ + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note + New note by lambda + @<a href="http://pleroma.example.org:4000/users/lain3" class="h-card mention">lain3</a> + + + http://activitystrea.ms/schema/1.0/post + 2017-04-23T14:51:03+00:00 + 2017-04-23T14:51:03+00:00 + + http://activitystrea.ms/schema/1.0/person + http://gs.example.org:4040/index.php/user/1 + lambda + + + + + lambda + lambda + + + + + tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b + + + + http://gs.example.org:4040/index.php/api/statuses/user_timeline/1.atom + lambda + + + + http://gs.example.org:4040/theme/neo-gnu/default-avatar-profile.png + 2017-04-23T14:51:03+00:00 + + + + + diff --git a/test/fixtures/user_full.xml b/test/fixtures/user_full.xml new file mode 100644 index 000000000..8eee8c686 --- /dev/null +++ b/test/fixtures/user_full.xml @@ -0,0 +1,10 @@ + + http://activitystrea.ms/schema/1.0/person + http://gs.example.org:4040/index.php/user/1 + lambda + + + + Constance Variable + lambadalambda + diff --git a/test/fixtures/user_name_only.xml b/test/fixtures/user_name_only.xml new file mode 100644 index 000000000..6d895d5c2 --- /dev/null +++ b/test/fixtures/user_name_only.xml @@ -0,0 +1,5 @@ + + http://activitystrea.ms/schema/1.0/person + http://gs.example.org:4040/index.php/user/1 + lambda + From ab0114fbaabd28d1e1a6961f6bfbd683f3e7fbbc Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 24 Apr 2017 18:46:34 +0200 Subject: [PATCH 03/88] Return salmon path for users, basic incoming salmon handling. --- lib/pleroma/web/activity_pub/activity_pub.ex | 42 ++++++ lib/pleroma/web/ostatus/feed_representer.ex | 1 + lib/pleroma/web/ostatus/ostatus.ex | 134 +++++++++++++++++- lib/pleroma/web/ostatus/ostatus_controller.ex | 11 +- lib/pleroma/web/router.ex | 1 + lib/pleroma/web/twitter_api/twitter_api.ex | 65 ++++----- lib/pleroma/web/web_finger/web_finger.ex | 3 +- test/web/ostatus/feed_representer_test.exs | 1 + test/web/ostatus/ostatus_test.exs | 53 +++++++ 9 files changed, 272 insertions(+), 39 deletions(-) create mode 100644 test/web/ostatus/ostatus_test.exs diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index e9f0dcd32..7264123d8 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -19,6 +19,48 @@ def insert(map) when is_map(map) do Repo.insert(%Activity{data: map}) end + def create(to, actor, context, object, additional \\ %{}, published \\ nil) do + published = published || make_date() + + activity = %{ + "type" => "Create", + "to" => to, + "actor" => actor.ap_id, + "object" => object, + "published" => published, + "context" => context + } + |> Map.merge(additional) + + with {:ok, activity} <- insert(activity) do + {:ok, activity} = add_conversation_id(activity) + + if actor.local do + Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity) + end + + {:ok, activity} + end + end + + defp add_conversation_id(activity) do + if is_integer(activity.data["statusnetConversationId"]) do + {:ok, activity} + else + data = activity.data + |> put_in(["object", "statusnetConversationId"], activity.id) + |> put_in(["statusnetConversationId"], activity.id) + + object = Object.get_by_ap_id(activity.data["object"]["id"]) + + changeset = Ecto.Changeset.change(object, data: data["object"]) + Repo.update(changeset) + + changeset = Ecto.Changeset.change(activity, data: data) + Repo.update(changeset) + end + end + def like(%User{ap_id: ap_id} = user, %Object{data: %{ "id" => id}} = object) do cond do # There's already a like here, so return the original activity. diff --git a/lib/pleroma/web/ostatus/feed_representer.ex b/lib/pleroma/web/ostatus/feed_representer.ex index 14ac3ebf4..2cc0da9ba 100644 --- a/lib/pleroma/web/ostatus/feed_representer.ex +++ b/lib/pleroma/web/ostatus/feed_representer.ex @@ -23,6 +23,7 @@ def to_simple_form(user, activities, users) do {:title, ['#{user.nickname}\'s timeline']}, {:updated, h.(most_recent_update)}, {:link, [rel: 'hub', href: h.(OStatus.pubsub_path(user))], []}, + {:link, [rel: 'salmon', href: h.(OStatus.salmon_path(user))], []}, {:link, [rel: 'self', href: h.(OStatus.feed_path(user))], []}, {:author, UserRepresenter.to_simple_form(user)}, ] ++ entries diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index d21b9078f..4fd649c92 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -1,5 +1,9 @@ defmodule Pleroma.Web.OStatus do - alias Pleroma.Web + import Ecto.Query + require Logger + + alias Pleroma.{Repo, User, Web} + alias Pleroma.Web.ActivityPub.ActivityPub def feed_path(user) do "#{user.ap_id}/feed.atom" @@ -9,6 +13,132 @@ def pubsub_path(user) do "#{Web.base_url}/push/hub/#{user.nickname}" end - def user_path(user) do + def salmon_path(user) do + "#{user.ap_id}/salmon" + end + + def handle_incoming(xml_string) do + {doc, _rest} = :xmerl_scan.string(to_charlist(xml_string)) + + {:xmlObj, :string, object_type } = :xmerl_xpath.string('string(/entry/activity:object-type[1])', doc) + + case object_type do + 'http://activitystrea.ms/schema/1.0/note' -> + handle_note(doc) + _ -> + Logger.error("Couldn't parse incoming document") + end + end + + # TODO + # Parse mention + # wire up replies + # Set correct context + # Set correct statusnet ids. + def handle_note(doc) do + content_html = string_from_xpath("/entry/content[1]", doc) + + [author] = :xmerl_xpath.string('/entry/author[1]', doc) + {:ok, actor} = find_or_make_user(author) + + context = ActivityPub.generate_context_id + + to = [ + "https://www.w3.org/ns/activitystreams#Public" + ] + + date = string_from_xpath("/entry/published", doc) + + object = %{ + "type" => "Note", + "to" => to, + "content" => content_html, + "published" => date, + "context" => context, + "actor" => actor.ap_id + } + + ActivityPub.create(to, actor, context, object, %{}, date) + end + + def find_or_make(author, doc) do + query = from user in User, + where: user.local == false and fragment("? @> ?", user.info, ^%{ostatus_uri: author}) + + user = Repo.one(query) + + if is_nil(user) do + make_user(doc) + else + {:ok, user} + end + end + + def find_or_make_user(author_doc) do + {:xmlObj, :string, uri } = :xmerl_xpath.string('string(/author[1]/uri)', author_doc) + + query = from user in User, + where: user.local == false and fragment("? @> ?", user.info, ^%{ostatus_uri: to_string(uri)}) + + user = Repo.one(query) + + if is_nil(user) do + make_user(author_doc) + else + {:ok, user} + end + end + + defp string_from_xpath(xpath, doc) do + {:xmlObj, :string, res} = :xmerl_xpath.string('string(#{xpath})', doc) + + res = res + |> to_string + |> String.trim + + if res == "", do: nil, else: res + end + + def make_user(author_doc) do + author = string_from_xpath("/author[1]/uri", author_doc) + name = string_from_xpath("/author[1]/name", author_doc) + preferredUsername = string_from_xpath("/author[1]/poco:preferredUsername", author_doc) + displayName = string_from_xpath("/author[1]/poco:displayName", author_doc) + avatar = make_avatar_object(author_doc) + + data = %{ + local: false, + name: preferredUsername || name, + nickname: displayName || name, + ap_id: author, + info: %{ + "ostatus_uri" => author, + "host" => URI.parse(author).host, + "system" => "ostatus" + }, + avatar: avatar + } + + Repo.insert(Ecto.Changeset.change(%User{}, data)) + end + + # TODO: Just takes the first one for now. + defp make_avatar_object(author_doc) do + href = string_from_xpath("/author[1]/link[@rel=\"avatar\"]/@href", author_doc) + type = string_from_xpath("/author[1]/link[@rel=\"avatar\"]/@type", author_doc) + + if href do + %{ + "type" => "Image", + "url" => + [%{ + "type" => "Link", + "mediaType" => type, + "href" => href + }] + } + else + nil + end end end diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 3c8d8c0f1..4174db786 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -25,7 +25,14 @@ def feed(conn, %{"nickname" => nickname}) do |> send_resp(200, response) end - def temp(conn, params) do - IO.inspect(params) + def salmon_incoming(conn, params) do + {:ok, body, _conn} = read_body(conn) + magic_key = Pleroma.Web.Salmon.fetch_magic_key(body) + {:ok, doc} = Pleroma.Web.Salmon.decode_and_validate(magic_key, body) + + Pleroma.Web.OStatus.handle_incoming(doc) + + conn + |> send_resp(200, "") end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a4f13c879..c98eac688 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -74,6 +74,7 @@ def user_fetcher(username) do pipe_through :ostatus get "/users/:nickname/feed", OStatus.OStatusController, :feed + post "/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming post "/push/hub/:nickname", Websub.WebsubController, :websub_subscription_request end diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 0f84cffbd..9049b4efc 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -28,11 +28,33 @@ def create_status(user = %User{}, data = %{}) do date = make_date() - activity = %{ - "type" => "Create", - "to" => to, - "actor" => user.ap_id, - "object" => %{ + # Wire up reply info. + [to, context, object, additional] = + with inReplyToId when not is_nil(inReplyToId) <- data["in_reply_to_status_id"], + inReplyTo <- Repo.get(Activity, inReplyToId), + context <- inReplyTo.data["context"] + do + to = to ++ [inReplyTo.data["actor"]] + + object = %{ + "type" => "Note", + "to" => to, + "content" => content_html, + "published" => date, + "context" => context, + "attachment" => attachments, + "actor" => user.ap_id, + "inReplyTo" => inReplyTo.data["object"]["id"], + "inReplyToStatusId" => inReplyToId, + "statusnetConversationId" => inReplyTo.data["statusnetConversationId"] + } + additional = %{ + "statusnetConversationId" => inReplyTo.data["statusnetConversationId"] + } + + [to, context, object, additional] + else _e -> + object = %{ "type" => "Note", "to" => to, "content" => content_html, @@ -40,36 +62,11 @@ def create_status(user = %User{}, data = %{}) do "context" => context, "attachment" => attachments, "actor" => user.ap_id - }, - "published" => date, - "context" => context - } - - # Wire up reply info. - activity = with inReplyToId when not is_nil(inReplyToId) <- data["in_reply_to_status_id"], - inReplyTo <- Repo.get(Activity, inReplyToId), - context <- inReplyTo.data["context"] - do - - to = activity["to"] ++ [inReplyTo.data["actor"]] - - activity - |> put_in(["to"], to) - |> put_in(["context"], context) - |> put_in(["object", "context"], context) - |> put_in(["object", "inReplyTo"], inReplyTo.data["object"]["id"]) - |> put_in(["object", "inReplyToStatusId"], inReplyToId) - |> put_in(["statusnetConversationId"], inReplyTo.data["statusnetConversationId"]) - |> put_in(["object", "statusnetConversationId"], inReplyTo.data["statusnetConversationId"]) - else _e -> - activity - end - - with {:ok, activity} <- ActivityPub.insert(activity) do - {:ok, activity} = add_conversation_id(activity) - Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(user), user, activity) - {:ok, activity} + } + [to, context, object, %{}] end + + ActivityPub.create(to, user, context, object, additional, data) end def fetch_friend_statuses(user, opts \\ %{}) do diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index eb540e92a..18459e8f0 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -31,7 +31,8 @@ def represent_user(user) do [ {:Subject, "acct:#{user.nickname}@#{Pleroma.Web.host}"}, {:Alias, user.ap_id}, - {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}} + {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}}, + {:Link, %{rel: "salmon", href: OStatus.salmon_path(user)}} ] } |> XmlBuilder.to_doc diff --git a/test/web/ostatus/feed_representer_test.exs b/test/web/ostatus/feed_representer_test.exs index 9a02d8c16..13cdeb79d 100644 --- a/test/web/ostatus/feed_representer_test.exs +++ b/test/web/ostatus/feed_representer_test.exs @@ -27,6 +27,7 @@ test "returns a feed of the last 20 items of the user" do #{user.nickname}'s timeline #{most_recent_update} + #{user_xml} diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs new file mode 100644 index 000000000..8ee605494 --- /dev/null +++ b/test/web/ostatus/ostatus_test.exs @@ -0,0 +1,53 @@ +defmodule Pleroma.Web.OStatusTest do + use Pleroma.DataCase + alias Pleroma.Web.OStatus + + test "handle incoming notes" do + incoming = File.read!("test/fixtures/incoming_note_activity.xml") + {:ok, activity} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Create" + assert activity.data["object"]["type"] == "Note" + assert activity.data["published"] == "2017-04-23T14:51:03+00:00" + end + + describe "new remote user creation" do + test "make new user or find them based on an 'author' xml doc" do + incoming = File.read!("test/fixtures/user_name_only.xml") + {doc, _rest} = :xmerl_scan.string(to_charlist(incoming)) + + {:ok, user} = OStatus.find_or_make_user(doc) + + assert user.name == "lambda" + assert user.nickname == "lambda" + assert user.local == false + assert user.info["ostatus_uri"] == "http://gs.example.org:4040/index.php/user/1" + assert user.info["system"] == "ostatus" + assert user.ap_id == "http://gs.example.org:4040/index.php/user/1" + + {:ok, user_again} = OStatus.find_or_make_user(doc) + + assert user == user_again + end + + test "tries to use the information in poco fields" do + incoming = File.read!("test/fixtures/user_full.xml") + {doc, _rest} = :xmerl_scan.string(to_charlist(incoming)) + + {:ok, user} = OStatus.find_or_make_user(doc) + + assert user.name == "Constance Variable" + assert user.nickname == "lambadalambda" + assert user.local == false + assert user.info["ostatus_uri"] == "http://gs.example.org:4040/index.php/user/1" + assert user.info["system"] == "ostatus" + assert user.ap_id == "http://gs.example.org:4040/index.php/user/1" + + assert List.first(user.avatar["url"])["href"] == "http://gs.example.org:4040/theme/neo-gnu/default-avatar-profile.png" + + {:ok, user_again} = OStatus.find_or_make_user(doc) + + assert user == user_again + end + end +end From ef4190b3abbc581e9405f2a32fe7579345a3d155 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 25 Apr 2017 17:26:05 +0200 Subject: [PATCH 04/88] Clean up status create method. --- lib/pleroma/web/twitter_api/twitter_api.ex | 33 +++++++++++++--------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 9049b4efc..ad73e82ce 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -5,26 +5,33 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do import Ecto.Query - def create_status(user = %User{}, data = %{}) do - attachments = Enum.map(data["media_ids"] || [], fn (media_id) -> - Repo.get(Object, media_id).data - end) - - context = ActivityPub.generate_context_id - - content = HtmlSanitizeEx.strip_tags(data["status"]) - |> String.replace("\n", "
") - - mentions = parse_mentions(content) - + def to_for_user_and_mentions(user, mentions) do default_to = [ User.ap_followers(user), "https://www.w3.org/ns/activitystreams#Public" ] to = default_to ++ Enum.map(mentions, fn ({_, %{ap_id: ap_id}}) -> ap_id end) + end - content_html = add_user_links(content, mentions) + def format_input(text, mentions) do + content = HtmlSanitizeEx.strip_tags(text) + |> String.replace("\n", "
") + |> add_user_links(mentions) + end + + def attachments_from_ids(ids) do + Enum.map(ids || [], fn (media_id) -> + Repo.get(Object, media_id).data + end) + end + + def create_status(user = %User{}, data = %{"status" => status}) do + attachments = attachments_from_ids(data["media_ids"]) + context = ActivityPub.generate_context_id + mentions = parse_mentions(status) + content_html = format_input(status, mentions) + to = to_for_user_and_mentions(user, mentions) date = make_date() From 4771962a5d0749746b7bb921074b97e13348dedf Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 25 Apr 2017 17:32:36 +0200 Subject: [PATCH 05/88] More refactoring. --- lib/pleroma/web/twitter_api/twitter_api.ex | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index ad73e82ce..ad4bf7153 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -26,21 +26,26 @@ def attachments_from_ids(ids) do end) end + def get_replied_to_activity(id) when not is_nil(id) do + Repo.get(Activity, id) + end + + def get_replied_to_activity(_), do: nil + def create_status(user = %User{}, data = %{"status" => status}) do attachments = attachments_from_ids(data["media_ids"]) context = ActivityPub.generate_context_id mentions = parse_mentions(status) content_html = format_input(status, mentions) to = to_for_user_and_mentions(user, mentions) - date = make_date() + inReplyTo = get_replied_to_activity(data["in_reply_to_status_id"]) + # Wire up reply info. [to, context, object, additional] = - with inReplyToId when not is_nil(inReplyToId) <- data["in_reply_to_status_id"], - inReplyTo <- Repo.get(Activity, inReplyToId), - context <- inReplyTo.data["context"] - do + if inReplyTo do + context = inReplyTo.data["context"] to = to ++ [inReplyTo.data["actor"]] object = %{ @@ -52,7 +57,7 @@ def create_status(user = %User{}, data = %{"status" => status}) do "attachment" => attachments, "actor" => user.ap_id, "inReplyTo" => inReplyTo.data["object"]["id"], - "inReplyToStatusId" => inReplyToId, + "inReplyToStatusId" => inReplyTo.id, "statusnetConversationId" => inReplyTo.data["statusnetConversationId"] } additional = %{ @@ -60,7 +65,7 @@ def create_status(user = %User{}, data = %{"status" => status}) do } [to, context, object, additional] - else _e -> + else object = %{ "type" => "Note", "to" => to, From 6c5f5e18ec1103a9d10d88081a27c2ae9dba4f41 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 25 Apr 2017 17:35:21 +0200 Subject: [PATCH 06/88] Even more refactoring. --- lib/pleroma/web/twitter_api/twitter_api.ex | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index ad4bf7153..cb48c7f5f 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -11,11 +11,11 @@ def to_for_user_and_mentions(user, mentions) do "https://www.w3.org/ns/activitystreams#Public" ] - to = default_to ++ Enum.map(mentions, fn ({_, %{ap_id: ap_id}}) -> ap_id end) + default_to ++ Enum.map(mentions, fn ({_, %{ap_id: ap_id}}) -> ap_id end) end def format_input(text, mentions) do - content = HtmlSanitizeEx.strip_tags(text) + HtmlSanitizeEx.strip_tags(text) |> String.replace("\n", "
") |> add_user_links(mentions) end @@ -235,24 +235,6 @@ def add_user_links(text, mentions) do Enum.reduce(mentions, text, fn ({match, %User{ap_id: ap_id}}, text) -> String.replace(text, match, "#{match}") end) end - defp add_conversation_id(activity) do - if is_integer(activity.data["statusnetConversationId"]) do - {:ok, activity} - else - data = activity.data - |> put_in(["object", "statusnetConversationId"], activity.id) - |> put_in(["statusnetConversationId"], activity.id) - - object = Object.get_by_ap_id(activity.data["object"]["id"]) - - changeset = Ecto.Changeset.change(object, data: data["object"]) - Repo.update(changeset) - - changeset = Ecto.Changeset.change(activity, data: data) - Repo.update(changeset) - end - end - def register_user(params) do params = %{ nickname: params["nickname"], From b438ea24ee936ae10efdcd3c9079e3b45ae521f4 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 25 Apr 2017 17:45:34 +0200 Subject: [PATCH 07/88] Add ostatus conversation as context. --- lib/pleroma/web/ostatus/ostatus.ex | 7 ++++++- lib/pleroma/web/twitter_api/twitter_api.ex | 2 +- test/web/ostatus/ostatus_test.exs | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 4fd649c92..8c31ce5aa 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -41,7 +41,12 @@ def handle_note(doc) do [author] = :xmerl_xpath.string('/entry/author[1]', doc) {:ok, actor} = find_or_make_user(author) - context = ActivityPub.generate_context_id + context = string_from_xpath("/entry/ostatus:conversation[1]", doc) |> String.trim + context = if String.length(context) > 0 do + context + else + ActivityPub.generate_context_id + end to = [ "https://www.w3.org/ns/activitystreams#Public" diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index cb48c7f5f..e4e26df15 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -253,7 +253,7 @@ def register_user(params) do {:error, changeset} -> errors = Ecto.Changeset.traverse_errors(changeset, fn {msg, _opts} -> msg end) |> Poison.encode! - {:error, %{error: errors}} + {:error, %{error: errors}} end end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 8ee605494..61dca5446 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -9,6 +9,7 @@ test "handle incoming notes" do assert activity.data["type"] == "Create" assert activity.data["object"]["type"] == "Note" assert activity.data["published"] == "2017-04-23T14:51:03+00:00" + assert activity.data["context"] == "tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b" end describe "new remote user creation" do From f980f6778b1447b808299fa9274854bb25f9823b Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 25 Apr 2017 18:03:14 +0200 Subject: [PATCH 08/88] Wire up mentions. --- lib/pleroma/web/ostatus/ostatus.ex | 21 +++++---------------- test/web/ostatus/ostatus_test.exs | 1 + 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 8c31ce5aa..65141f826 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -31,10 +31,7 @@ def handle_incoming(xml_string) do end # TODO - # Parse mention # wire up replies - # Set correct context - # Set correct statusnet ids. def handle_note(doc) do content_html = string_from_xpath("/entry/content[1]", doc) @@ -52,6 +49,11 @@ def handle_note(doc) do "https://www.w3.org/ns/activitystreams#Public" ] + mentions = :xmerl_xpath.string('/entry/link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]', doc) + |> Enum.map(fn(person) -> string_from_xpath("@href", person) end) + + to = to ++ mentions + date = string_from_xpath("/entry/published", doc) object = %{ @@ -66,19 +68,6 @@ def handle_note(doc) do ActivityPub.create(to, actor, context, object, %{}, date) end - def find_or_make(author, doc) do - query = from user in User, - where: user.local == false and fragment("? @> ?", user.info, ^%{ostatus_uri: author}) - - user = Repo.one(query) - - if is_nil(user) do - make_user(doc) - else - {:ok, user} - end - end - def find_or_make_user(author_doc) do {:xmlObj, :string, uri } = :xmerl_xpath.string('string(/author[1]/uri)', author_doc) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 61dca5446..dffebf5a7 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -10,6 +10,7 @@ test "handle incoming notes" do assert activity.data["object"]["type"] == "Note" assert activity.data["published"] == "2017-04-23T14:51:03+00:00" assert activity.data["context"] == "tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b" + assert "http://pleroma.example.org:4000/users/lain3" in activity.data["to"] end describe "new remote user creation" do From b91ccef2371fb0bbc23638b174e815dd7189482e Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 26 Apr 2017 08:47:22 +0200 Subject: [PATCH 09/88] Output conversation id. --- lib/pleroma/web/ostatus/activity_representer.ex | 4 +++- lib/pleroma/web/ostatus/feed_representer.ex | 3 ++- test/support/factory.ex | 6 ++++-- test/web/ostatus/activity_representer_test.exs | 2 ++ test/web/ostatus/feed_representer_test.exs | 2 +- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 590abc8bb..367212fe1 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -19,7 +19,9 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) {:title, ['New note by #{user.nickname}']}, {:content, [type: 'html'], h.(activity.data["object"]["content"])}, {:published, h.(inserted_at)}, - {:updated, h.(updated_at)} + {:updated, h.(updated_at)}, + {:"ostatus:conversation", [], h.(activity.data["context"])}, + {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []} ] ++ attachments end diff --git a/lib/pleroma/web/ostatus/feed_representer.ex b/lib/pleroma/web/ostatus/feed_representer.ex index 2cc0da9ba..10a1ffb25 100644 --- a/lib/pleroma/web/ostatus/feed_representer.ex +++ b/lib/pleroma/web/ostatus/feed_representer.ex @@ -17,7 +17,8 @@ def to_simple_form(user, activities, users) do :feed, [ xmlns: 'http://www.w3.org/2005/Atom', "xmlns:activity": 'http://activitystrea.ms/spec/1.0/', - "xmlns:poco": 'http://portablecontacts.net/spec/1.0' + "xmlns:poco": 'http://portablecontacts.net/spec/1.0', + "xmlns:ostatus": 'http://ostatus.org/schema/1.0' ], [ {:id, h.(OStatus.feed_path(user))}, {:title, ['#{user.nickname}\'s timeline']}, diff --git a/test/support/factory.ex b/test/support/factory.ex index d7c16f0e0..d037be4a6 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -24,7 +24,8 @@ def note_factory do "to" => ["https://www.w3.org/ns/activitystreams#Public"], "published_at" => DateTime.utc_now() |> DateTime.to_iso8601, "likes" => [], - "like_count" => 0 + "like_count" => 0, + "context" => "2hu" } %Pleroma.Object{ @@ -40,7 +41,8 @@ def note_activity_factory do "actor" => note.data["actor"], "to" => note.data["to"], "object" => note.data, - "published_at" => DateTime.utc_now() |> DateTime.to_iso8601 + "published_at" => DateTime.utc_now() |> DateTime.to_iso8601, + "context" => note.data["context"] } %Pleroma.Activity{ diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 61df41a1d..10f9a9d0b 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -23,6 +23,8 @@ test "a note activity" do #{note_activity.data["object"]["content"]} #{inserted_at} #{updated_at} + #{note_activity.data["context"]} + """ tuple = ActivityRepresenter.to_simple_form(note_activity, user) diff --git a/test/web/ostatus/feed_representer_test.exs b/test/web/ostatus/feed_representer_test.exs index 13cdeb79d..ef0f4d5ff 100644 --- a/test/web/ostatus/feed_representer_test.exs +++ b/test/web/ostatus/feed_representer_test.exs @@ -22,7 +22,7 @@ test "returns a feed of the last 20 items of the user" do |> :xmerl.export_simple_content(:xmerl_xml) expected = """ - + #{OStatus.feed_path(user)} #{user.nickname}'s timeline #{most_recent_update} From d9ebd785ab7d9b371ba5accdc6ca5d72af7b509d Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 26 Apr 2017 10:08:13 +0200 Subject: [PATCH 10/88] Ostatus doesn't distinguish between activities / objects on create. --- lib/pleroma/web/ostatus/activity_representer.ex | 2 +- test/web/ostatus/activity_representer_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 367212fe1..30e695bcc 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -15,7 +15,7 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) [ {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']}, {:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']}, - {:id, h.(activity.data["object"]["id"])}, + {:id, h.(activity.data["id"])}, {:title, ['New note by #{user.nickname}']}, {:content, [type: 'html'], h.(activity.data["object"]["content"])}, {:published, h.(inserted_at)}, diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 10f9a9d0b..6cea9cff0 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -18,7 +18,7 @@ test "a note activity" do expected = """ http://activitystrea.ms/schema/1.0/note http://activitystrea.ms/schema/1.0/post - #{note_activity.data["object"]["id"]} + #{note_activity.data["id"]} New note by #{user.nickname} #{note_activity.data["object"]["content"]} #{inserted_at} From f1ebf812eede5b77931d2315757a7ad8e0ea5a7e Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 26 Apr 2017 10:22:51 +0200 Subject: [PATCH 11/88] Add inReplyTo to incoming messages. --- lib/pleroma/web/ostatus/ostatus.ex | 8 ++++ .../incoming_note_activity_answer.xml | 42 +++++++++++++++++++ test/web/ostatus/ostatus_test.exs | 10 +++++ 3 files changed, 60 insertions(+) create mode 100644 test/fixtures/incoming_note_activity_answer.xml diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 65141f826..5b68f057e 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -65,6 +65,14 @@ def handle_note(doc) do "actor" => actor.ap_id } + inReplyTo = string_from_xpath("/entry/thr:in-reply-to[1]/@href", doc) + + object = if inReplyTo do + Map.put(object, "inReplyTo", inReplyTo) + else + object + end + ActivityPub.create(to, actor, context, object, %{}, date) end diff --git a/test/fixtures/incoming_note_activity_answer.xml b/test/fixtures/incoming_note_activity_answer.xml new file mode 100644 index 000000000..b1244faa6 --- /dev/null +++ b/test/fixtures/incoming_note_activity_answer.xml @@ -0,0 +1,42 @@ + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note + New note by lambda + hey. + + + http://activitystrea.ms/schema/1.0/post + 2017-04-25T18:16:13+00:00 + 2017-04-25T18:16:13+00:00 + + http://activitystrea.ms/schema/1.0/person + http://gs.example.org:4040/index.php/user/1 + lambda + + + + + lambda + lambda + + + + + + + http://pleroma.example.org:4000/contexts/8f6f45d4-8e4d-4e1a-a2de-09f27367d2d0 + + + + http://gs.example.org:4040/index.php/api/statuses/user_timeline/1.atom + lambda + + + + http://gs.example.org:4040/theme/neo-gnu/default-avatar-profile.png + 2017-04-25T18:16:13+00:00 + + + + + diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index dffebf5a7..96f2cb4f3 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -13,6 +13,16 @@ test "handle incoming notes" do assert "http://pleroma.example.org:4000/users/lain3" in activity.data["to"] end + test "handle incoming replies" do + incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") + {:ok, activity} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Create" + assert activity.data["object"]["type"] == "Note" + assert activity.data["object"]["inReplyTo"] == "http://pleroma.example.org:4000/objects/55bce8fc-b423-46b1-af71-3759ab4670bc" + assert "http://pleroma.example.org:4000/users/lain5" in activity.data["to"] + end + describe "new remote user creation" do test "make new user or find them based on an 'author' xml doc" do incoming = File.read!("test/fixtures/user_name_only.xml") From 57bd59e4071adf847f94229479e5ffa0951721fd Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 26 Apr 2017 14:25:44 +0200 Subject: [PATCH 12/88] Salmon creation. --- lib/pleroma/web/salmon/salmon.ex | 56 +++++++++++++++++++++++++++++++- test/fixtures/private_key.pem | 27 +++++++++++++++ test/web/salmon/salmon_test.exs | 33 +++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/private_key.pem diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 3881f2758..24b5eb0d9 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -57,7 +57,7 @@ def decode_and_validate(magickey, salmon) do end end - defp decode_key("RSA." <> magickey) do + def decode_key("RSA." <> magickey) do make_integer = fn(bin) -> list = :erlang.binary_to_list(bin) Enum.reduce(list, 0, fn (el, acc) -> (acc <<< 8) ||| el end) @@ -70,4 +70,58 @@ defp decode_key("RSA." <> magickey) do {:RSAPublicKey, modulus, exponent} end + + def encode_key({:RSAPublicKey, modulus, exponent}) do + modulus_enc = :binary.encode_unsigned(modulus) |> Base.url_encode64 + exponent_enc = :binary.encode_unsigned(exponent) |> Base.url_encode64 + + "RSA.#{modulus_enc}.#{exponent_enc}" + end + + def generate_rsa_pem do + port = Port.open({:spawn, "openssl genrsa"}, [:binary]) + {:ok, pem} = receive do + {^port, {:data, pem}} -> {:ok, pem} + end + Port.close(port) + if Regex.match?(~r/RSA PRIVATE KEY/, pem) do + {:ok, pem} + else + :error + end + end + + def keys_from_pem(pem) do + [private_key_code] = :public_key.pem_decode(pem) + private_key = :public_key.pem_entry_decode(private_key_code) + {:RSAPrivateKey, _, modulus, exponent, _, _, _, _, _, _, _} = private_key + public_key = {:RSAPublicKey, modulus, exponent} + {:ok, private_key, public_key} + end + + def encode(private_key, doc) do + type = "application/atom+xml" + encoding = "base64url" + alg = "RSA-SHA256" + + signed_text = [doc, type, encoding, alg] + |> Enum.map(&Base.url_encode64/1) + |> Enum.join(".") + + signature = :public_key.sign(signed_text, :sha256, private_key) |> to_string |> Base.url_encode64 + doc_base64= doc |> Base.url_encode64 + + # Don't need proper xml building, these strings are safe to leave unescaped + salmon = """ + + + #{doc_base64} + #{encoding} + #{alg} + #{signature} + + """ + + {:ok, salmon} + end end diff --git a/test/fixtures/private_key.pem b/test/fixtures/private_key.pem new file mode 100644 index 000000000..7a4b14654 --- /dev/null +++ b/test/fixtures/private_key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAqnWeDtrqWasCKNXiuSq1tSCLI5H7BSvIROy5YfuGsXHrIlCq +LdIm9QlIUUmIi9QyzgiGEDsPCCkA1UguCVgF/UrJ1+FvHcHsTELkkBu/yCl9mrgt +WzTckhb6KjOhqtxi/TKgRaJ2Rlwz2bvH5sbCP9qffthitdxfh14KC5V0gqDt1xCy +WgZo79vbYMcVkcQoh5uLtG64ksYFBMfgnLaSj7xg5i2qCDiIY7bqBujo5HllDqeo +w3LXmsztt1cT8heXEjW0SYJvAHJK00OsG1kp4cqhfKzxLCHNGQJVHQxLOXy97I7o +HOeuhbxPhjpGSBMgw7YFm3ODXviqf557eqFcaQIDAQABAoIBAC6f+VnK22sncXHF +/zvyyL0AZ86U8XpanW7s6VA5wn/qzwwV0Fa0Mt+3aEaDvIuywSrF/hWWcegjfwzX +r2/y2cCMomUgTopvLrk1WttoG68eWjLlydI2xVZYXpkIgmH/4juri1dAtuVL9wrJ +aEZhe2SH4jSJ74Ya/y5BtLGycaoA9FHyIzHPTx52Ix2jWKWtKimW8J+aERi2uHdN +7yTnLT2APhs5fnvNnn0tg85CI3Ny2GNiqmAail14yVfRz8Sf6qDIepH5Jfz9oll4 +I+GYUOLs6eTgkHXBn8LGhtHTE/9UJmb42OyWrW8X+nc/Mjz5xh0u/g1Gdp36oUMz +OotfneECgYEA3cGfQxmxjEqSbXt9jbxiCukU7PmkDDQqBu97URC4N8qEcMF1wW7X +AddU7Kq/UJU+oqjD/7UQHoS2ZThPtto6SpVdXQzsnrnPWQcrv5b1DV/TpXfwGoZ3 +svUIAcx4vGzhhmHDJCBsdY6n8xWBYtSqfLFXgN5UkdafLGy3EkCEtmUCgYEAxMgl +7eU2QkWkzgJxOj6xjG2yqM3jxOvvoiRnD0rIQaBS70P/1N94ZkMXzOwddddZ5OW+ +55h/a8TmFKP/+NW4PHRYra/dazGI4IBlw6Yeq6uq/4jbuSqtBbaNn/Dz5kdHBTqM +PtbBvc9Fztd2zb3InyyLbb4c+WjMqi0AooN027UCgYB4Tax7GJtLwsEBiDcrB4Ig +7SYfEae/vyT1skIyTmHCUqnbCfk6QUl/hDRcWJ2FuBHM6MW8GZxvEgxpiU0lo+pv +v+xwqKxNx/wHDm7bd6fl45DMee7WVRDnEyuO3kC56E/JOYxGMxjkBcpzg703wqvj +Dcqs7PDwVYDw9uGykzHsSQKBgEQnNcvA+RvW1w9qlSChGgkS7S+9r0dCl8pGZVNM +iTMBfffUS0TE6QQx9IpKtKFdpoq6b3XywR7oIO/BJSRfkOGPQi9Vm5BGpatrjNNI +M5Mtb5n1InRtLWOvKDnez/pPcW+EKZKR+qPsp7bNtR3ovxUx7lBh6dMP0uKVl4Sx +lsWJAoGBAIeek9eG+S3m2jaJRHasfKo5mJ2JrrmnjQXUOGUP8/CgO8sW1VmG2WAk +Av7+BRI2mP2f+3SswG/AoRGmRXXw65ly63ws8ixrhK0MG3MgqDkWc69SbTaaMJ+u +BQFYMsB1vZdUV3CaRqySkjY68QWGcJ4Z5JKHuTXzKv/GeFmw0V9R +-----END RSA PRIVATE KEY----- diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs index 4ebb32081..6fbabd19f 100644 --- a/test/web/salmon/salmon_test.exs +++ b/test/web/salmon/salmon_test.exs @@ -16,4 +16,37 @@ test "errors on wrong magic key" do {:ok, salmon} = File.read("test/fixtures/salmon.xml") assert Salmon.decode_and_validate(@wrong_magickey, salmon) == :error end + + test "generates an RSA private key pem" do + {:ok, key} = Salmon.generate_rsa_pem + assert is_binary(key) + assert Regex.match?(~r/RSA/, key) + end + + test "it encodes a magic key from a public key" do + key = Salmon.decode_key(@magickey) + magic_key = Salmon.encode_key(key) + + assert @magickey == magic_key + end + + test "returns a public and private key from a pem" do + pem = File.read!("test/fixtures/private_key.pem") + {:ok, private, public} = Salmon.keys_from_pem(pem) + + assert elem(private, 0) == :RSAPrivateKey + assert elem(public, 0) == :RSAPublicKey + end + + test "encodes an xml payload with a private key" do + doc = File.read!("test/fixtures/incoming_note_activity.xml") + pem = File.read!("test/fixtures/private_key.pem") + {:ok, private, public} = Salmon.keys_from_pem(pem) + + # Let's try a roundtrip. + {:ok, salmon} = Salmon.encode(private, doc) + {:ok, decoded_doc} = Salmon.decode_and_validate(Salmon.encode_key(public), salmon) + + assert doc == decoded_doc + end end From c5fa682c317717c64168bf2d77b28d805ffff450 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 26 Apr 2017 18:33:10 +0200 Subject: [PATCH 13/88] Refactor, add beginnings of websub client subscriptions. --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/federator/federator.ex | 32 +++++++++++++++++++ lib/pleroma/web/websub/websub.ex | 26 ++++++++++++--- .../web/websub/websub_client_subscription.ex | 13 ++++++++ ...4155_create_websub_client_subscription.exs | 15 +++++++++ test/web/websub/websub_test.exs | 12 ++++++- 6 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 lib/pleroma/web/federator/federator.ex create mode 100644 lib/pleroma/web/websub/websub_client_subscription.ex create mode 100644 priv/repo/migrations/20170426154155_create_websub_client_subscription.exs diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 7264123d8..82f9fcc1c 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -36,7 +36,7 @@ def create(to, actor, context, object, additional \\ %{}, published \\ nil) do {:ok, activity} = add_conversation_id(activity) if actor.local do - Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity) + Pleroma.Web.Federator.enqueue(:publish, activity) end {:ok, activity} diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex new file mode 100644 index 000000000..f489ed837 --- /dev/null +++ b/lib/pleroma/web/federator/federator.ex @@ -0,0 +1,32 @@ +defmodule Pleroma.Web.Federator do + alias Pleroma.User + require Logger + + @websub_verifier Application.get_env(:pleroma, :websub_verifier) + + def handle(:publish, activity) do + Logger.debug("Running publish for #{activity.data["id"]}") + with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do + Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity) + end + end + + def handle(:verify_websub, websub) do + Logger.debug("Running websub verification for #{websub.id} (#{websub.topic}, #{websub.callback})") + @websub_verifier.verify(websub) + end + + def handle(type, payload) do + Logger.debug("Unknown task: #{type}") + {:error, "Don't know what do do with this"} + end + + def enqueue(type, payload) do + # for now, just run immediately in a new process. + if Mix.env == :test do + handle(type, payload) + else + spawn(fn -> handle(type, payload) end) + end + end +end diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index cc66b52dd..03b0aec8f 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -1,13 +1,11 @@ defmodule Pleroma.Web.Websub do alias Pleroma.Repo - alias Pleroma.Web.Websub.WebsubServerSubscription + alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription} alias Pleroma.Web.OStatus.FeedRepresenter alias Pleroma.Web.OStatus import Ecto.Query - @websub_verifier Application.get_env(:pleroma, :websub_verifier) - def verify(subscription, getter \\ &HTTPoison.get/3 ) do challenge = Base.encode16(:crypto.strong_rand_bytes(8)) lease_seconds = NaiveDateTime.diff(subscription.valid_until, subscription.updated_at) |> to_string @@ -71,8 +69,7 @@ def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) d change = Ecto.Changeset.change(websub, %{valid_until: NaiveDateTime.add(websub.updated_at, lease_time)}) websub = Repo.update!(change) - # Just spawn that for now, maybe pool later. - spawn(fn -> @websub_verifier.verify(websub) end) + Pleroma.Web.Federator.enqueue(:verify_websub, websub) {:ok, websub} else {:error, reason} -> @@ -99,4 +96,23 @@ defp valid_topic(%{"hub.topic" => topic}, user) do {:error, "Wrong topic requested, expected #{OStatus.feed_path(user)}, got #{topic}"} end end + + def subscribe(user, topic) do + # Race condition, use transactions + {:ok, subscription} = with subscription when not is_nil(subscription) <- Repo.get_by(WebsubClientSubscription, topic: topic) do + subscribers = [user.ap_id, subscription.subcribers] |> Enum.uniq + change = Ecto.Changeset.change(subscription, %{subscribers: subscribers}) + Repo.update(change) + else _e -> + subscription = %WebsubClientSubscription{ + topic: topic, + subscribers: [user.ap_id], + state: "requested", + secret: :crypto.strong_rand_bytes(8) |> Base.url_encode64 + } + Repo.insert(subscription) + end + + {:ok, subscription} + end end diff --git a/lib/pleroma/web/websub/websub_client_subscription.ex b/lib/pleroma/web/websub/websub_client_subscription.ex new file mode 100644 index 000000000..341e27c51 --- /dev/null +++ b/lib/pleroma/web/websub/websub_client_subscription.ex @@ -0,0 +1,13 @@ +defmodule Pleroma.Web.Websub.WebsubClientSubscription do + use Ecto.Schema + + schema "websub_client_subscriptions" do + field :topic, :string + field :secret, :string + field :valid_until, :naive_datetime + field :state, :string + field :subscribers, {:array, :string}, default: [] + + timestamps() + end +end diff --git a/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs b/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs new file mode 100644 index 000000000..f42782840 --- /dev/null +++ b/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs @@ -0,0 +1,15 @@ +defmodule Pleroma.Repo.Migrations.CreateWebsubClientSubscription do + use Ecto.Migration + + def change do + create table(:websub_client_subscriptions) do + add :topic, :string + add :secret, :string + add :valid_until, :naive_datetime + add :state, :string + add :subscribers, :map + + timestamps() + end + end +end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 334ba03fc..7b77e696b 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -58,7 +58,6 @@ test "an incoming subscription request" do "hub.lease_seconds" => "100" } - {:ok, subscription } = Websub.incoming_subscription_request(user, data) assert subscription.topic == Pleroma.Web.OStatus.feed_path(user) assert subscription.state == "requested" @@ -87,4 +86,15 @@ test "an incoming subscription request for an existing subscription" do assert length(Repo.all(WebsubServerSubscription)) == 1 assert subscription.id == sub.id end + + test "initiate a subscription for a given user and topic" do + user = insert(:user) + topic = "http://example.org/some-topic.atom" + + {:ok, websub} = Websub.subscribe(user, topic) + assert websub.subscribers == [user.ap_id] + assert websub.topic == topic + assert is_binary(websub.secret) + assert websub.state == "accepted" + end end From d1dce56a85e041f78e1d50900a0c9591610de2b9 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 27 Apr 2017 09:43:58 +0200 Subject: [PATCH 14/88] Refactor XML parsing. --- lib/pleroma/web/ostatus/ostatus.ex | 13 ++----------- lib/pleroma/web/salmon/salmon.ex | 8 +++++--- lib/pleroma/web/websub/websub.ex | 1 + lib/pleroma/web/xml/xml.ex | 19 +++++++++++++++++++ test/web/ostatus/ostatus_test.exs | 5 +++-- 5 files changed, 30 insertions(+), 16 deletions(-) create mode 100644 lib/pleroma/web/xml/xml.ex diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 5b68f057e..89b482592 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -1,5 +1,6 @@ defmodule Pleroma.Web.OStatus do import Ecto.Query + import Pleroma.Web.XML require Logger alias Pleroma.{Repo, User, Web} @@ -18,7 +19,7 @@ def salmon_path(user) do end def handle_incoming(xml_string) do - {doc, _rest} = :xmerl_scan.string(to_charlist(xml_string)) + doc = parse_document(xml_string) {:xmlObj, :string, object_type } = :xmerl_xpath.string('string(/entry/activity:object-type[1])', doc) @@ -91,16 +92,6 @@ def find_or_make_user(author_doc) do end end - defp string_from_xpath(xpath, doc) do - {:xmlObj, :string, res} = :xmerl_xpath.string('string(#{xpath})', doc) - - res = res - |> to_string - |> String.trim - - if res == "", do: nil, else: res - end - def make_user(author_doc) do author = string_from_xpath("/author[1]/uri", author_doc) name = string_from_xpath("/author[1]/name", author_doc) diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 24b5eb0d9..99cca1f55 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -1,8 +1,9 @@ defmodule Pleroma.Web.Salmon do use Bitwise + alias Pleroma.Web.XML def decode(salmon) do - {doc, _rest} = :xmerl_scan.string(to_charlist(salmon)) + doc = XML.parse_document(salmon) {:xmlObj, :string, data} = :xmerl_xpath.string('string(//me:data[1])', doc) {:xmlObj, :string, sig} = :xmerl_xpath.string('string(//me:sig[1])', doc) @@ -22,16 +23,17 @@ def decode(salmon) do def fetch_magic_key(salmon) do [data, _, _, _, _] = decode(salmon) - {doc, _rest} = :xmerl_scan.string(to_charlist(data)) + doc = XML.parse_document(data) {:xmlObj, :string, uri} = :xmerl_xpath.string('string(//author[1]/uri)', doc) uri = to_string(uri) base = URI.parse(uri).host # TODO: Find out if this endpoint is mandated by the standard. + # At least diaspora does it differently {:ok, response} = HTTPoison.get(base <> "/.well-known/webfinger", ["Accept": "application/xrd+xml"], [params: [resource: uri]]) - {doc, _rest} = :xmerl_scan.string(to_charlist(response.body)) + doc = XML.parse_document(response.body) {:xmlObj, :string, magickey} = :xmerl_xpath.string('string(//Link[@rel="magic-public-key"]/@href)', doc) "data:application/magic-public-key," <> magickey = to_string(magickey) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 03b0aec8f..5372416e6 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -3,6 +3,7 @@ defmodule Pleroma.Web.Websub do alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription} alias Pleroma.Web.OStatus.FeedRepresenter alias Pleroma.Web.OStatus + alias Pleroma.Web.XML import Ecto.Query diff --git a/lib/pleroma/web/xml/xml.ex b/lib/pleroma/web/xml/xml.ex new file mode 100644 index 000000000..22faf72df --- /dev/null +++ b/lib/pleroma/web/xml/xml.ex @@ -0,0 +1,19 @@ +defmodule Pleroma.Web.XML do + def string_from_xpath(xpath, doc) do + {:xmlObj, :string, res} = :xmerl_xpath.string('string(#{xpath})', doc) + + res = res + |> to_string + |> String.trim + + if res == "", do: nil, else: res + end + + def parse_document(text) do + {doc, _rest} = text + |> :binary.bin_to_list + |> :xmerl_scan.string + + doc + end +end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 96f2cb4f3..140b32f36 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -1,6 +1,7 @@ defmodule Pleroma.Web.OStatusTest do use Pleroma.DataCase alias Pleroma.Web.OStatus + alias Pleroma.Web.XML test "handle incoming notes" do incoming = File.read!("test/fixtures/incoming_note_activity.xml") @@ -26,7 +27,7 @@ test "handle incoming replies" do describe "new remote user creation" do test "make new user or find them based on an 'author' xml doc" do incoming = File.read!("test/fixtures/user_name_only.xml") - {doc, _rest} = :xmerl_scan.string(to_charlist(incoming)) + doc = XML.parse_document(incoming) {:ok, user} = OStatus.find_or_make_user(doc) @@ -44,7 +45,7 @@ test "make new user or find them based on an 'author' xml doc" do test "tries to use the information in poco fields" do incoming = File.read!("test/fixtures/user_full.xml") - {doc, _rest} = :xmerl_scan.string(to_charlist(incoming)) + doc = XML.parse_document(incoming) {:ok, user} = OStatus.find_or_make_user(doc) From e8a311ecffe1fb19a3d194b1c5628853263909a7 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 27 Apr 2017 09:44:20 +0200 Subject: [PATCH 15/88] Add user and hub to websub client subscriptions. --- lib/pleroma/web/websub/websub_client_subscription.ex | 3 +++ .../migrations/20170427054757_add_user_and_hub.exs | 10 ++++++++++ 2 files changed, 13 insertions(+) create mode 100644 priv/repo/migrations/20170427054757_add_user_and_hub.exs diff --git a/lib/pleroma/web/websub/websub_client_subscription.ex b/lib/pleroma/web/websub/websub_client_subscription.ex index 341e27c51..c7a25ea22 100644 --- a/lib/pleroma/web/websub/websub_client_subscription.ex +++ b/lib/pleroma/web/websub/websub_client_subscription.ex @@ -1,5 +1,6 @@ defmodule Pleroma.Web.Websub.WebsubClientSubscription do use Ecto.Schema + alias Pleroma.User schema "websub_client_subscriptions" do field :topic, :string @@ -7,6 +8,8 @@ defmodule Pleroma.Web.Websub.WebsubClientSubscription do field :valid_until, :naive_datetime field :state, :string field :subscribers, {:array, :string}, default: [] + field :hub, :string + belongs_to :user, User timestamps() end diff --git a/priv/repo/migrations/20170427054757_add_user_and_hub.exs b/priv/repo/migrations/20170427054757_add_user_and_hub.exs new file mode 100644 index 000000000..4f9a520bd --- /dev/null +++ b/priv/repo/migrations/20170427054757_add_user_and_hub.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.AddUserAndHub do + use Ecto.Migration + + def change do + alter table(:websub_client_subscriptions) do + add :hub, :string + add :user_id, references(:users) + end + end +end From 1ea4325fecaed981b2f949b00d3b37171013012c Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 27 Apr 2017 09:46:04 +0200 Subject: [PATCH 16/88] Add user feed fixture. --- test/fixtures/lambadalambda.atom | 479 +++++++++++++++++++++++++++++++ 1 file changed, 479 insertions(+) create mode 100644 test/fixtures/lambadalambda.atom diff --git a/test/fixtures/lambadalambda.atom b/test/fixtures/lambadalambda.atom new file mode 100644 index 000000000..35e506420 --- /dev/null +++ b/test/fixtures/lambadalambda.atom @@ -0,0 +1,479 @@ + + + https://mastodon.social/users/lambadalambda.atom + Critical Value + + 2017-04-16T21:47:25Z + https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif?1492379244 + + https://mastodon.social/users/lambadalambda + http://activitystrea.ms/schema/1.0/person + https://mastodon.social/users/lambadalambda + lambadalambda + lambadalambda@mastodon.social + + + + + lambadalambda + Critical Value + public + + + + + + + + tag:mastodon.social,2017-04-07:objectId=1874242:objectType=Status + 2017-04-07T11:02:56Z + 2017-04-07T11:02:56Z + lambadalambda shared a status by 0xroy@social.wxcafe.net + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + tag:social.wxcafe.net,2017-04-07:objectId=72554:objectType=Status + 2017-04-07T11:01:59Z + 2017-04-07T11:02:00Z + New status by 0xroy@social.wxcafe.net + + https://social.wxcafe.net/users/0xroy + http://activitystrea.ms/schema/1.0/person + https://social.wxcafe.net/users/0xroy + 0xroy + 0xroy@social.wxcafe.net + ta caution weeb | discussions privées : <a href="https://💌.0xroy.me" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="">💌.0xroy.me</span><span class="invisible"></span></a> + + + + 0xroy + 「R O Y 🍵 B O S」 + ta caution weeb | discussions privées : <a href="https://%F0%9F%92%8C.0xroy.me" rel="nofollow noopener"><span class="invisible">https://</span><span class="">💌.0xroy.me</span><span class="invisible"></span></a> + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p>someone pls eli5 matrix (protocol) and riot</p> + + public + + + <p>someone pls eli5 matrix (protocol) and riot</p> + + public + + + + + tag:mastodon.social,2017-04-06:objectId=1768247:objectType=Status + 2017-04-06T11:10:19Z + 2017-04-06T11:10:19Z + lambadalambda shared a status by areyoutoo@mastodon.xyz + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + tag:mastodon.xyz,2017-04-05:objectId=133327:objectType=Status + 2017-04-05T17:36:41Z + 2017-04-05T18:12:14Z + New status by areyoutoo@mastodon.xyz + + https://mastodon.xyz/users/areyoutoo + http://activitystrea.ms/schema/1.0/person + https://mastodon.xyz/users/areyoutoo + areyoutoo + areyoutoo@mastodon.xyz + devops | retired gamedev | always boost puppy pics + + + + areyoutoo + Raw Butter + devops | retired gamedev | always boost puppy pics + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p>Some UX thoughts for <a href="https://mastodon.xyz/tags/mastodev" class="mention hashtag">#<span>mastodev</span></a>:</p><p>- Would be nice if I could work on multiple draft toots? Clicking to reply to someone seems to erase any draft I had been working on.</p><p>- Kinda risky to click on the Federated Timeline if it loads new toots and scrolls 10ms before I click on something.</p><p>I probably don't know enough web frontend to help, but it might be fun to try.</p> + + + public + + + <p>Some UX thoughts for <a href="https://mastodon.xyz/tags/mastodev" class="mention hashtag">#<span>mastodev</span></a>:</p><p>- Would be nice if I could work on multiple draft toots? Clicking to reply to someone seems to erase any draft I had been working on.</p><p>- Kinda risky to click on the Federated Timeline if it loads new toots and scrolls 10ms before I click on something.</p><p>I probably don't know enough web frontend to help, but it might be fun to try.</p> + + public + + + + + tag:mastodon.social,2017-04-06:objectId=1764509:objectType=Status + 2017-04-06T10:15:38Z + 2017-04-06T10:15:38Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + This is a test for cw federation + <p>This is a test for cw federation body text.</p> + + public + + + + + tag:mastodon.social,2017-04-05:objectId=1645208:objectType=Status + 2017-04-05T07:14:53Z + 2017-04-05T07:14:53Z + lambadalambda shared a status by lambadalambda@social.heldscal.la + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + tag:social.heldscal.la,2017-04-05:noticeId=1502088:objectType=note + 2017-04-05T06:12:09Z + 2017-04-05T07:12:47Z + New status by lambadalambda@social.heldscal.la + + https://social.heldscal.la/user/23211 + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + lambadalambda@social.heldscal.la + Call me Deacon Blues. + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + Federation 101: <a href="https://www.youtube.com/watch?v=t1lYU5CA40o" rel="nofollow external noreferrer" class="attachment thumbnail">https://www.youtube.com/watch?v=t1lYU5CA40o</a> + + public + + + Federation 101: <a href="https://www.youtube.com/watch?v=t1lYU5CA40o" rel="nofollow external noreferrer" class="attachment thumbnail">https://www.youtube.com/watch?v=t1lYU5CA40o</a> + + public + + + + + tag:mastodon.social,2017-04-05:objectId=1641750:objectType=Status + 2017-04-05T05:44:48Z + 2017-04-05T05:44:48Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> just a test.</p> + + + public + + + + + tag:mastodon.social,2017-04-04:objectId=1540149:objectType=Status + 2017-04-04T06:31:09Z + 2017-04-04T06:31:09Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p>Looks like you still can&apos;t delete your account here (PRIVACY!), but I won&apos;t be posting here anymore, my main account is <span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span></p> + + + public + + + + + tag:mastodon.social,2017-04-04:objectId=1539608:objectType=Status + 2017-04-04T06:18:16Z + 2017-04-04T06:18:16Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.social/@ghostbar" class="u-url mention">@<span>ghostbar</span></a></span> Remember to rewrite it in Rust once you&apos;re done.</p> + + + public + + + + + + tag:mastodon.social,2017-04-03:objectId=1504813:objectType=Status + 2017-04-03T18:01:20Z + 2017-04-03T18:01:20Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.xyz/@Azurolu" class="u-url mention">@<span>Azurolu</span></a></span> You mean gs.smuglo.li?</p> + + + public + + + + + + tag:mastodon.social,2017-04-03:objectId=1504805:objectType=Status + 2017-04-03T18:01:05Z + 2017-04-03T18:01:05Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p>There&apos;s nothing wrong with having several alt accounts all across the fediverse. Try out another mastodon instance (<a href="https://icosahedron.website" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="">icosahedron.website</span><span class="invisible"></span></a>) or a GNU Social instance (like <a href="https://shitposter.club" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="">shitposter.club</span><span class="invisible"></span></a> or <a href="https://freezepeach.xyz" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="">freezepeach.xyz</span><span class="invisible"></span></a>), or friendica. They are all on the same network, so you can still follow all your friends!</p> + + public + + + + + tag:mastodon.social,2017-04-03:objectId=1503965:objectType=Status + 2017-04-03T17:31:30Z + 2017-04-03T17:31:30Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.social/@20Hz" class="u-url mention">@<span>20Hz</span></a></span> you could also try out a GS instance, which are on the same network :)</p> + + + public + + + + + + tag:mastodon.social,2017-04-03:objectId=1503955:objectType=Status + 2017-04-03T17:31:08Z + 2017-04-03T17:31:08Z + lambadalambda shared a status by shpuld@shitposter.club + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + tag:shitposter.club,2017-04-03:noticeId=2251717:objectType=note + 2017-04-03T17:06:43Z + 2017-04-03T17:12:06Z + New status by shpuld@shitposter.club + + https://shitposter.club/user/5381 + http://activitystrea.ms/schema/1.0/person + https://shitposter.club/user/5381 + shpuld + shpuld@shitposter.club + + + + + shpuld + shp + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + reposting the classic <a href="https://shitposter.club/file/89c5fe483526caf3a46cfc5cdd4ae68061054350e767397731af658d54786e31.jpg" class="attachment" rel="nofollow external">https://shitposter.club/attachment/219846</a> + + + public + + + reposting the classic <a href="https://shitposter.club/file/89c5fe483526caf3a46cfc5cdd4ae68061054350e767397731af658d54786e31.jpg" class="attachment" rel="nofollow external">https://shitposter.club/attachment/219846</a> + + public + + + + + tag:mastodon.social,2017-04-03:objectId=1503929:objectType=Status + 2017-04-03T17:30:43Z + 2017-04-03T17:30:43Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.social/@ghostbar" class="u-url mention">@<span>ghostbar</span></a></span> Normally you shouldn&apos;t be running tens of thousands of users on one instance... That&apos;s one of the reasons for federation.</p> + + + public + + + + + + tag:mastodon.social,2017-04-03:objectId=1477255:objectType=Status + 2017-04-03T08:24:39Z + 2017-04-03T08:24:39Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.social/@dot_tiff" class="u-url mention">@<span>dot_tiff</span></a></span> it&apos;s the vaporwave mode.</p> + + + public + + + + + + tag:mastodon.social,2017-04-03:objectId=1476210:objectType=Status + 2017-04-03T07:45:42Z + 2017-04-03T07:45:42Z + lambadalambda shared a status by lambadalambda@social.heldscal.la + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + tag:social.heldscal.la,2017-04-03:noticeId=1475727:objectType=note + 2017-04-03T07:44:43Z + 2017-04-03T07:44:48Z + New status by lambadalambda@social.heldscal.la + + https://social.heldscal.la/user/23211 + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + lambadalambda@social.heldscal.la + Call me Deacon Blues. + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + Here's a song by the original anti-idol, Togawa Jun: <a href="https://www.youtube.com/watch?v=kNI_NK2YY-s" rel="nofollow external noreferrer" class="attachment">https://www.youtube.com/watch?v=kNI_NK2YY-s</a> + + public + + + Here's a song by the original anti-idol, Togawa Jun: <a href="https://www.youtube.com/watch?v=kNI_NK2YY-s" rel="nofollow external noreferrer" class="attachment">https://www.youtube.com/watch?v=kNI_NK2YY-s</a> + + public + + + + + tag:mastodon.social,2017-04-03:objectId=1476047:objectType=Status + 2017-04-03T07:39:14Z + 2017-04-03T07:39:14Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.social/@amrrr" class="u-url mention">@<span>amrrr</span></a></span> tumblr/10, but pretty good!</p> + + + public + + + + + + tag:mastodon.social,2017-04-03:objectId=1475949:objectType=Status + 2017-04-03T07:35:45Z + 2017-04-03T07:35:45Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.social/@Shookaite" class="u-url mention">@<span>Shookaite</span></a></span> Oh, you mean like userstyles?</p> + + + public + + + + + + tag:mastodon.social,2017-04-03:objectId=1475581:objectType=Status + 2017-04-03T07:20:03Z + 2017-04-03T07:20:03Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.social/@Shookaite" class="u-url mention">@<span>Shookaite</span></a></span> Would be nice if someone helped port Pleroma to Mastodon, that has a theme switcher (click on the cog in the upper right): <a href="https://pleroma.heldscal.la/main/all" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="">pleroma.heldscal.la/main/all</span><span class="invisible"></span></a></p> + + + public + + + + + + tag:mastodon.social,2017-04-02:objectId=1457325:objectType=Status + 2017-04-02T21:57:43Z + 2017-04-02T21:57:43Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.social/@rhosyn" class="u-url mention">@<span>rhosyn</span></a></span> <span class="h-card"><a href="https://mastodon.social/@Meaningness" class="u-url mention">@<span>Meaningness</span></a></span> you could take a look at those listed at social.guhnoo.org</p> + + + + public + + + + + + tag:mastodon.social,2017-04-02:objectId=1447926:objectType=Status + 2017-04-02T18:31:52Z + 2017-04-02T18:31:52Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p>My main account is <span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> , btw.</p> + + + public + + + + + tag:mastodon.social,2017-04-02:objectId=1447878:objectType=Status + 2017-04-02T18:30:37Z + 2017-04-02T18:30:37Z + lambadalambda shared a status by Firstaide@awoo.space + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + tag:awoo.space,2017-04-02:objectId=135324:objectType=Status + 2017-04-02T18:29:32Z + 2017-04-02T18:29:32Z + New status by Firstaide@awoo.space + + https://awoo.space/users/Firstaide + http://activitystrea.ms/schema/1.0/person + https://awoo.space/users/Firstaide + Firstaide + Firstaide@awoo.space + A smol awoo account, for a smol autistic 💙 +They/them please! +NB/white/ace + + + + Firstaide + Miff🚑✨ + A smol awoo account, for a smol autistic 💙 +They/them please! +NB/white/ace + public + + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><a href="https://mastodon.social/users/lambadalambda" class="h-card u-url p-nickname mention">@<span>lambadalambda</span></a> yeah, I think that's p much the big issue here? <br>When I first heard of Masto, I thought it was just like twitter at first, I had no idea federation was even a thing?, and I actually joined p early on? :-o </p><p>idk I think more stuff needs to be done about federation promotion, but honestly its gotta come from the get go when people get here to make an account I feel :-o</p> + + + public + + + + <p><a href="https://mastodon.social/users/lambadalambda" class="h-card u-url p-nickname mention">@<span>lambadalambda</span></a> yeah, I think that's p much the big issue here? <br>When I first heard of Masto, I thought it was just like twitter at first, I had no idea federation was even a thing?, and I actually joined p early on? :-o </p><p>idk I think more stuff needs to be done about federation promotion, but honestly its gotta come from the get go when people get here to make an account I feel :-o</p> + + public + + + + From 90da25505f9cfbd16a9088e20714b24c2c6fa215 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 27 Apr 2017 09:46:45 +0200 Subject: [PATCH 17/88] Add discovery and subscription requests to websub. --- lib/pleroma/web/websub/websub.ex | 58 +++++++++++++++++++++++++++--- test/support/factory.ex | 10 ++++++ test/web/websub/websub_test.exs | 61 ++++++++++++++++++++++++++++++-- 3 files changed, 123 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 5372416e6..4a35ca8fc 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.Websub do alias Pleroma.Web.OStatus.FeedRepresenter alias Pleroma.Web.OStatus alias Pleroma.Web.XML + require Logger import Ecto.Query @@ -98,8 +99,8 @@ defp valid_topic(%{"hub.topic" => topic}, user) do end end - def subscribe(user, topic) do - # Race condition, use transactions + def subscribe(user, topic, requester \\ &request_subscription/1) do + # FIXME: Race condition, use transactions {:ok, subscription} = with subscription when not is_nil(subscription) <- Repo.get_by(WebsubClientSubscription, topic: topic) do subscribers = [user.ap_id, subscription.subcribers] |> Enum.uniq change = Ecto.Changeset.change(subscription, %{subscribers: subscribers}) @@ -109,11 +110,60 @@ def subscribe(user, topic) do topic: topic, subscribers: [user.ap_id], state: "requested", - secret: :crypto.strong_rand_bytes(8) |> Base.url_encode64 + secret: :crypto.strong_rand_bytes(8) |> Base.url_encode64, + user: user } Repo.insert(subscription) end + requester.(subscription) + end - {:ok, subscription} + def discover(topic, getter \\ &HTTPoison.get/1) do + with {:ok, response} <- getter.(topic), + status_code when status_code in 200..299 <- response.status_code, + body <- response.body, + doc <- XML.parse_document(body), + url when not is_nil(url) <- XML.string_from_xpath(~S{/feed/link[@rel="self"]/@href}, doc), + hub when not is_nil(hub) <- XML.string_from_xpath(~S{/feed/link[@rel="hub"]/@href}, doc) do + {:ok, %{url: url, hub: hub}} + else e -> + {:error, e} + end + end + + def request_subscription(websub, poster \\ &HTTPoison.post/3, timeout \\ 10_000) do + data = [ + "hub.mode": "subscribe", + "hub.topic": websub.topic, + "hub.secret": websub.secret, + "hub.callback": "https://social.heldscal.la/callback" + ] + + # This checks once a second if we are confirmed yet + websub_checker = fn -> + helper = fn (helper) -> + :timer.sleep(1000) + websub = Repo.get_by(WebsubClientSubscription, id: websub.id, state: "accepted") + if websub, do: websub, else: helper.(helper) + end + helper.(helper) + end + + task = Task.async(websub_checker) + + with {:ok, %{status_code: 202}} <- poster.(websub.hub, {:form, data}, ["Content-type": "application/x-www-form-urlencoded"]), + {:ok, websub} <- Task.yield(task, timeout) do + {:ok, websub} + else e -> + Task.shutdown(task) + + change = Ecto.Changeset.change(websub, %{state: "rejected"}) + {:ok, websub} = Repo.update(change) + + Logger.debug("Couldn't confirm subscription: #{inspect(websub)}") + Logger.debug("error: #{inspect(e)}") + + {:error, websub} + end end end diff --git a/test/support/factory.ex b/test/support/factory.ex index d037be4a6..ac276567a 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -76,4 +76,14 @@ def websub_subscription_factory do state: "requested" } end + + def websub_client_subscription_factory do + %Pleroma.Web.Websub.WebsubClientSubscription{ + topic: "http://example.org", + secret: "here's a secret", + valid_until: nil, + state: "requested", + subscribers: [] + } + end end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 7b77e696b..bf243ac91 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -77,7 +77,6 @@ test "an incoming subscription request for an existing subscription" do "hub.lease_seconds" => "100" } - {:ok, subscription } = Websub.incoming_subscription_request(user, data) assert subscription.topic == Pleroma.Web.OStatus.feed_path(user) assert subscription.state == sub.state @@ -87,14 +86,72 @@ test "an incoming subscription request for an existing subscription" do assert subscription.id == sub.id end + def accepting_verifier(subscription) do + {:ok, %{ subscription | state: "accepted" }} + end + test "initiate a subscription for a given user and topic" do user = insert(:user) topic = "http://example.org/some-topic.atom" - {:ok, websub} = Websub.subscribe(user, topic) + {:ok, websub} = Websub.subscribe(user, topic, &accepting_verifier/1) assert websub.subscribers == [user.ap_id] assert websub.topic == topic assert is_binary(websub.secret) + assert websub.user == user assert websub.state == "accepted" end + + test "discovers the hub and canonical url" do + topic = "https://mastodon.social/users/lambadalambda.atom" + + getter = fn(^topic) -> + doc = File.read!("test/fixtures/lambadalambda.atom") + {:ok, %{status_code: 200, body: doc}} + end + + {:ok, discovered} = Websub.discover(topic, getter) + assert %{hub: "https://mastodon.social/api/push", url: topic} == discovered + end + + test "calls the hub, requests topic" do + hub = "https://social.heldscal.la/main/push/hub" + topic = "https://social.heldscal.la/api/statuses/user_timeline/23211.atom" + websub = insert(:websub_client_subscription, %{hub: hub, topic: topic}) + + poster = fn (^hub, {:form, data}, _headers) -> + assert Keyword.get(data, :"hub.mode") == "subscribe" + {:ok, %{status_code: 202}} + end + + task = Task.async(fn -> Websub.request_subscription(websub, poster) end) + + change = Ecto.Changeset.change(websub, %{state: "accepted"}) + {:ok, _} = Repo.update(change) + + {:ok, websub} = Task.await(task) + + assert websub.state == "accepted" + end + + test "rejects the subscription if it can't be accepted" do + hub = "https://social.heldscal.la/main/push/hub" + topic = "https://social.heldscal.la/api/statuses/user_timeline/23211.atom" + websub = insert(:websub_client_subscription, %{hub: hub, topic: topic}) + + poster = fn (^hub, {:form, _data}, _headers) -> + {:ok, %{status_code: 202}} + end + + {:error, websub} = Websub.request_subscription(websub, poster, 1000) + assert websub.state == "rejected" + + websub = insert(:websub_client_subscription, %{hub: hub, topic: topic}) + poster = fn (^hub, {:form, _data}, _headers) -> + {:ok, %{status_code: 400}} + end + + {:error, websub} = Websub.request_subscription(websub, poster, 1000) + assert websub.state == "rejected" + end end From 451d18af63fcf97f0d9621e5bfe296e1f18a0312 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Fri, 28 Apr 2017 09:51:47 +0200 Subject: [PATCH 18/88] Add proper callback route for websub confirmation. --- lib/pleroma/web/router.ex | 1 + lib/pleroma/web/websub/websub.ex | 6 +++--- lib/pleroma/web/websub/websub_controller.ex | 5 +++++ test/web/websub/websub_test.exs | 3 +++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index c98eac688..bff981f9f 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -75,6 +75,7 @@ def user_fetcher(username) do get "/users/:nickname/feed", OStatus.OStatusController, :feed post "/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming + post "/push/subscriptions/:id", Websub.WebsubController, :websub_subscription_confirmation post "/push/hub/:nickname", Websub.WebsubController, :websub_subscription_request end diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 4a35ca8fc..ad352ee26 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -2,8 +2,8 @@ defmodule Pleroma.Web.Websub do alias Pleroma.Repo alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription} alias Pleroma.Web.OStatus.FeedRepresenter - alias Pleroma.Web.OStatus - alias Pleroma.Web.XML + alias Pleroma.Web.{XML, Endpoint, OStatus} + alias Pleroma.Web.Router.Helpers require Logger import Ecto.Query @@ -136,7 +136,7 @@ def request_subscription(websub, poster \\ &HTTPoison.post/3, timeout \\ 10_000) "hub.mode": "subscribe", "hub.topic": websub.topic, "hub.secret": websub.secret, - "hub.callback": "https://social.heldscal.la/callback" + "hub.callback": Helpers.websub_url(Endpoint, :websub_subscription_confirmation, websub.id) ] # This checks once a second if we are confirmed yet diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex index 5d54c6ef5..c6b15c0c2 100644 --- a/lib/pleroma/web/websub/websub_controller.ex +++ b/lib/pleroma/web/websub/websub_controller.ex @@ -15,4 +15,9 @@ def websub_subscription_request(conn, %{"nickname" => nickname} = params) do |> send_resp(500, reason) end end + + def websub_subscription_confirmation(conn, params) do + IO.inspect(params) + conn + end end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index bf243ac91..ca04a55cd 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -3,11 +3,13 @@ def verify(sub) do {:ok, sub} end end + defmodule Pleroma.Web.WebsubTest do use Pleroma.DataCase alias Pleroma.Web.Websub alias Pleroma.Web.Websub.WebsubServerSubscription import Pleroma.Factory + alias Pleroma.Web.Router.Helpers test "a verification of a request that is accepted" do sub = insert(:websub_subscription) @@ -121,6 +123,7 @@ test "calls the hub, requests topic" do poster = fn (^hub, {:form, data}, _headers) -> assert Keyword.get(data, :"hub.mode") == "subscribe" + assert Keyword.get(data, :"hub.callback") == Helpers.websub_url(Pleroma.Web.Endpoint, :websub_subscription_confirmation, websub.id) {:ok, %{status_code: 202}} end From 1422e7aa84a897c6026e9dcd26b7d5955050687a Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Fri, 28 Apr 2017 15:45:10 +0200 Subject: [PATCH 19/88] Handle incoming websub subscriptions. --- config/config.exs | 3 +- config/test.exs | 3 +- lib/pleroma/web/federator/federator.ex | 4 +- lib/pleroma/web/router.ex | 3 +- lib/pleroma/web/websub/websub.ex | 7 ++- lib/pleroma/web/websub/websub_controller.ex | 34 ++++++++++-- test/web/websub/websub_controller_test.exs | 59 +++++++++++++++++++++ 7 files changed, 102 insertions(+), 11 deletions(-) diff --git a/config/config.exs b/config/config.exs index 3826dddff..a5df31b5a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -30,7 +30,8 @@ "application/xrd+xml" => ["xrd+xml"] } -config :pleroma, :websub_verifier, Pleroma.Web.Websub +config :pleroma, :websub, Pleroma.Web.Websub +config :pleroma, :ostatus, Pleroma.Web.OStatus # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/config/test.exs b/config/test.exs index 5d91279a2..85b6ad26b 100644 --- a/config/test.exs +++ b/config/test.exs @@ -25,4 +25,5 @@ # Reduce hash rounds for testing config :comeonin, :pbkdf2_rounds, 1 -config :pleroma, :websub_verifier, Pleroma.Web.WebsubMock +config :pleroma, :websub, Pleroma.Web.WebsubMock +config :pleroma, :ostatus, Pleroma.Web.OStatusMock diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index f489ed837..38df13540 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -2,7 +2,7 @@ defmodule Pleroma.Web.Federator do alias Pleroma.User require Logger - @websub_verifier Application.get_env(:pleroma, :websub_verifier) + @websub Application.get_env(:pleroma, :websub) def handle(:publish, activity) do Logger.debug("Running publish for #{activity.data["id"]}") @@ -13,7 +13,7 @@ def handle(:publish, activity) do def handle(:verify_websub, websub) do Logger.debug("Running websub verification for #{websub.id} (#{websub.topic}, #{websub.callback})") - @websub_verifier.verify(websub) + @websub.verify(websub) end def handle(type, payload) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index bff981f9f..2ff75ec5d 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -75,8 +75,9 @@ def user_fetcher(username) do get "/users/:nickname/feed", OStatus.OStatusController, :feed post "/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming - post "/push/subscriptions/:id", Websub.WebsubController, :websub_subscription_confirmation post "/push/hub/:nickname", Websub.WebsubController, :websub_subscription_request + get "/push/subscriptions/:id", Websub.WebsubController, :websub_subscription_confirmation + post "/push/subscriptions/:id", Websub.WebsubController, :websub_incoming end scope "/.well-known", Pleroma.Web do diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index ad352ee26..ad9e47b46 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -42,8 +42,7 @@ def publish(topic, user, activity) do response = FeedRepresenter.to_simple_form(user, [activity], [user]) |> :xmerl.export_simple(:xmerl_xml) - signature = :crypto.hmac(:sha, sub.secret, response) |> Base.encode16 - + signature = sign(sub.secret, response) HTTPoison.post(sub.callback, response, [ {"Content-Type", "application/atom+xml"}, {"X-Hub-Signature", "sha1=#{signature}"} @@ -51,6 +50,10 @@ def publish(topic, user, activity) do end) end + def sign(secret, doc) do + :crypto.hmac(:sha, secret, doc) |> Base.encode16 + end + def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) do with {:ok, topic} <- valid_topic(params, user), {:ok, lease_time} <- lease_time(params), diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex index c6b15c0c2..cd59a70a3 100644 --- a/lib/pleroma/web/websub/websub_controller.ex +++ b/lib/pleroma/web/websub/websub_controller.ex @@ -1,7 +1,11 @@ defmodule Pleroma.Web.Websub.WebsubController do use Pleroma.Web, :controller - alias Pleroma.User + alias Pleroma.{Repo, User} alias Pleroma.Web.Websub + alias Pleroma.Web.Websub.WebsubClientSubscription + require Logger + + @ostatus Application.get_env(:pleroma, :ostatus) def websub_subscription_request(conn, %{"nickname" => nickname} = params) do user = User.get_cached_by_nickname(nickname) @@ -16,8 +20,30 @@ def websub_subscription_request(conn, %{"nickname" => nickname} = params) do end end - def websub_subscription_confirmation(conn, params) do - IO.inspect(params) - conn + def websub_subscription_confirmation(conn, %{"id" => id, "hub.mode" => "subscribe", "hub.challenge" => challenge, "hub.topic" => topic}) do + with %WebsubClientSubscription{} = websub <- Repo.get_by(WebsubClientSubscription, id: id, topic: topic) do + change = Ecto.Changeset.change(websub, %{state: "accepted"}) + {:ok, _websub} = Repo.update(change) + conn + |> send_resp(200, challenge) + else _e -> + conn + |> send_resp(500, "Error") + end + end + + def websub_incoming(conn, %{"id" => id}) do + with "sha1=" <> signature <- hd(get_req_header(conn, "x-hub-signature")), + %WebsubClientSubscription{} = websub <- Repo.get(WebsubClientSubscription, id), + {:ok, body, _conn} = read_body(conn), + ^signature <- Websub.sign(websub.secret, body) do + @ostatus.handle_incoming(body) + conn + |> send_resp(200, "OK") + else _e -> + Logger.debug("Can't handle incoming subscription post") + conn + |> send_resp(500, "Error") + end end end diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs index 8368cafea..521bbb9aa 100644 --- a/test/web/websub/websub_controller_test.exs +++ b/test/web/websub/websub_controller_test.exs @@ -1,6 +1,9 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory + alias Pleroma.Web.Websub.WebsubClientSubscription + alias Pleroma.{Repo, Activity} + alias Pleroma.Web.Websub test "websub subscription request", %{conn: conn} do user = insert(:user) @@ -20,4 +23,60 @@ test "websub subscription request", %{conn: conn} do assert response(conn, 202) == "Accepted" end + + test "websub subscription confirmation", %{conn: conn} do + websub = insert(:websub_client_subscription) + + params = %{ + "hub.mode" => "subscribe", + "hub.topic" => websub.topic, + "hub.challenge" => "some challenge", + "hub.lease_seconds" => 100 + } + + conn = conn + |> get("/push/subscriptions/#{websub.id}", params) + + websub = Repo.get(WebsubClientSubscription, websub.id) + + assert response(conn, 200) == "some challenge" + assert websub.state == "accepted" + end + + test "handles incoming feed updates", %{conn: conn} do + websub = insert(:websub_client_subscription) + doc = "some stuff" + signature = Websub.sign(websub.secret, doc) + + conn = conn + |> put_req_header("x-hub-signature", "sha1=" <> signature) + |> put_req_header("content-type", "application/atom+xml") + |> post("/push/subscriptions/#{websub.id}", doc) + + assert response(conn, 200) == "OK" + + assert length(Repo.all(Activity)) == 1 + end + + test "rejects incoming feed updates with the wrong signature", %{conn: conn} do + websub = insert(:websub_client_subscription) + doc = "some stuff" + signature = Websub.sign("wrong secret", doc) + + conn = conn + |> put_req_header("x-hub-signature", "sha1=" <> signature) + |> put_req_header("content-type", "application/atom+xml") + |> post("/push/subscriptions/#{websub.id}", doc) + + assert response(conn, 500) == "Error" + + assert length(Repo.all(Activity)) == 0 + end +end + +defmodule Pleroma.Web.OStatusMock do + import Pleroma.Factory + def handle_incoming(_doc) do + insert(:note_activity) + end end From 59d4cc60364fd1abf5cbc881e88757b378456b64 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Fri, 28 Apr 2017 15:53:45 +0200 Subject: [PATCH 20/88] normalize hex number. --- lib/pleroma/web/websub/websub_controller.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex index cd59a70a3..e5ecf6523 100644 --- a/lib/pleroma/web/websub/websub_controller.ex +++ b/lib/pleroma/web/websub/websub_controller.ex @@ -34,6 +34,7 @@ def websub_subscription_confirmation(conn, %{"id" => id, "hub.mode" => "subscrib def websub_incoming(conn, %{"id" => id}) do with "sha1=" <> signature <- hd(get_req_header(conn, "x-hub-signature")), + signature <- String.upcase(signature), %WebsubClientSubscription{} = websub <- Repo.get(WebsubClientSubscription, id), {:ok, body, _conn} = read_body(conn), ^signature <- Websub.sign(websub.secret, body) do From ca40dda04c114c32ca9ecfe5f998a083448a189c Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Fri, 28 Apr 2017 17:41:12 +0200 Subject: [PATCH 21/88] Add some basic webfingering. --- lib/pleroma/web/web_finger/web_finger.ex | 37 ++++++++++++++++++++++++ test/fixtures/webfinger.xml | 20 +++++++++++++ test/web/web_finger/web_finger_test.exs | 20 ++++++++++++- 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/webfinger.xml diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 18459e8f0..08ff6d278 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -2,6 +2,8 @@ defmodule Pleroma.Web.WebFinger do alias Pleroma.XmlBuilder alias Pleroma.User alias Pleroma.Web.OStatus + alias Pleroma.Web.XML + require Logger def host_meta() do base_url = Pleroma.Web.base_url @@ -37,4 +39,39 @@ def represent_user(user) do } |> XmlBuilder.to_doc end + + # FIXME: Make this call the host-meta to find the actual address. + defp webfinger_address(domain) do + "https://#{domain}/.well-known/webfinger" + end + + defp webfinger_from_xml(doc) do + magic_key = XML.string_from_xpath(~s{//Link[@rel="magic-public-key"]/@href}, doc) + "data:application/magic-public-key," <> magic_key = magic_key + topic = XML.string_from_xpath(~s{//Link[@rel="http://schemas.google.com/g/2010#updates-from"]/@href}, doc) + subject = XML.string_from_xpath("//Subject", doc) + salmon = XML.string_from_xpath(~s{//Link[@rel="salmon"]/@href}, doc) + data = %{ + magic_key: magic_key, + topic: topic, + subject: subject, + salmon: salmon + } + {:ok, data} + end + + def finger(account, getter \\ &HTTPoison.get/3) do + [name, domain] = String.split(account, "@") + address = webfinger_address(domain) + with {:ok, %{status_code: status_code, body: body}} when status_code in 200..299 <- getter.(address, ["Accept": "application/xrd+xml"], [params: [resource: account]]), + doc <- XML.parse_document(body), + {:ok, data} <- webfinger_from_xml(doc) do + {:ok, data} + else + e -> + Logger.debug("Couldn't finger #{account}.") + Logger.debug(e) + {:error, e} + end + end end diff --git a/test/fixtures/webfinger.xml b/test/fixtures/webfinger.xml new file mode 100644 index 000000000..4cde42e3f --- /dev/null +++ b/test/fixtures/webfinger.xml @@ -0,0 +1,20 @@ + + + acct:shp@social.heldscal.la + https://social.heldscal.la/user/29191 + https://social.heldscal.la/shp + https://social.heldscal.la/index.php/user/29191 + https://social.heldscal.la/index.php/shp + + + + + + + + + + + + + diff --git a/test/web/web_finger/web_finger_test.exs b/test/web/web_finger/web_finger_test.exs index 8a3007ff9..e5347a2b0 100644 --- a/test/web/web_finger/web_finger_test.exs +++ b/test/web/web_finger/web_finger_test.exs @@ -1,11 +1,29 @@ defmodule Pleroma.Web.WebFingerTest do use Pleroma.DataCase + alias Pleroma.Web.WebFinger describe "host meta" do test "returns a link to the xml lrdd" do - host_info = Pleroma.Web.WebFinger.host_meta + host_info = WebFinger.host_meta() assert String.contains?(host_info, Pleroma.Web.base_url) end end + + describe "fingering" do + test "returns the info for a user" do + user = "shp@social.heldscal.la" + + getter = fn(_url, _headers, [params: [resource: ^user]]) -> + {:ok, %{status_code: 200, body: File.read!("test/fixtures/webfinger.xml")}} + end + + {:ok, data} = WebFinger.finger(user, getter) + + assert data.magic_key == "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB" + assert data.topic == "https://social.heldscal.la/api/statuses/user_timeline/29191.atom" + assert data.subject == "acct:shp@social.heldscal.la" + assert data.salmon == "https://social.heldscal.la/main/salmon/user/29191" + end + end end From 69922bc724736fb07bf36beaef42d944158d9269 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sat, 29 Apr 2017 17:51:59 +0200 Subject: [PATCH 22/88] Add user info gathering. --- lib/pleroma/web/ostatus/ostatus.ex | 11 +++++++++++ lib/pleroma/web/web_finger/web_finger.ex | 4 ++-- lib/pleroma/web/websub/websub.ex | 16 +++++++++++++--- test/web/ostatus/ostatus_test.exs | 22 ++++++++++++++++++++++ test/web/websub/websub_test.exs | 11 +++++++++-- 5 files changed, 57 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 89b482592..90be86755 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.OStatus do alias Pleroma.{Repo, User, Web} alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.{WebFinger, Websub} def feed_path(user) do "#{user.ap_id}/feed.atom" @@ -134,4 +135,14 @@ defp make_avatar_object(author_doc) do nil end end + + def gather_user_info(username) do + with {:ok, webfinger_data} <- WebFinger.finger(username), + {:ok, feed_data} <- Websub.gather_feed_data(webfinger_data.topic) do + {:ok, Map.merge(webfinger_data, feed_data) |> Map.put(:fqn, username)} + else e -> + Logger.debug("Couldn't gather info for #{username}") + {:error, e} + end + end end diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 08ff6d278..1d8c4d0c8 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -18,7 +18,7 @@ def host_meta() do def webfinger(resource) do host = Pleroma.Web.host - regex = ~r/acct:(?\w+)@#{host}/ + regex = ~r/(acct:)?(?\w+)@#{host}/ case Regex.named_captures(regex, resource) do %{"username" => username} -> user = User.get_cached_by_nickname(username) @@ -70,7 +70,7 @@ def finger(account, getter \\ &HTTPoison.get/3) do else e -> Logger.debug("Couldn't finger #{account}.") - Logger.debug(e) + Logger.debug(inspect(e)) {:error, e} end end diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index ad9e47b46..c1d48ad7a 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -121,14 +121,24 @@ def subscribe(user, topic, requester \\ &request_subscription/1) do requester.(subscription) end - def discover(topic, getter \\ &HTTPoison.get/1) do + def gather_feed_data(topic, getter \\ &HTTPoison.get/1) do with {:ok, response} <- getter.(topic), status_code when status_code in 200..299 <- response.status_code, body <- response.body, doc <- XML.parse_document(body), - url when not is_nil(url) <- XML.string_from_xpath(~S{/feed/link[@rel="self"]/@href}, doc), + uri when not is_nil(uri) <- XML.string_from_xpath("/feed/author[1]/uri", doc), hub when not is_nil(hub) <- XML.string_from_xpath(~S{/feed/link[@rel="hub"]/@href}, doc) do - {:ok, %{url: url, hub: hub}} + + name = XML.string_from_xpath("/feed/author[1]/name", doc) + preferredUsername = XML.string_from_xpath("/feed/author[1]/poco:preferredUsername", doc) + displayName = XML.string_from_xpath("/feed/author[1]/poco:displayName", doc) + + {:ok, %{ + uri: uri, + hub: hub, + nickname: preferredUsername || name, + name: displayName || name + }} else e -> {:error, e} end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 140b32f36..2a5156b31 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -63,4 +63,26 @@ test "tries to use the information in poco fields" do assert user == user_again end end + + describe "gathering user info from a user id" do + test "it returns user info in a hash" do + user = "shp@social.heldscal.la" + + # TODO: make test local + {:ok, data} = OStatus.gather_user_info(user) + + expected = %{ + hub: "https://social.heldscal.la/main/push/hub", + magic_key: "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", + name: "shp", + nickname: "shp", + salmon: "https://social.heldscal.la/main/salmon/user/29191", + subject: "acct:shp@social.heldscal.la", + topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", + uri: "https://social.heldscal.la/user/29191", + fqn: user + } + assert data == expected + end + end end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index ca04a55cd..1b1ef3fa6 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -112,8 +112,15 @@ test "discovers the hub and canonical url" do {:ok, %{status_code: 200, body: doc}} end - {:ok, discovered} = Websub.discover(topic, getter) - assert %{hub: "https://mastodon.social/api/push", url: topic} == discovered + {:ok, discovered} = Websub.gather_feed_data(topic, getter) + expected = %{ + hub: "https://mastodon.social/api/push", + uri: "https://mastodon.social/users/lambadalambda", + nickname: "lambadalambda", + name: "Critical Value" + } + + assert expected == discovered end test "calls the hub, requests topic" do From 427bac0966c551eb16eaa6595d99fc5361a32ea9 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sat, 29 Apr 2017 19:06:01 +0200 Subject: [PATCH 23/88] Rework remote user subscription. --- lib/pleroma/web/ostatus/ostatus.ex | 44 +++++++------------ lib/pleroma/web/web_finger/web_finger.ex | 18 ++++++-- lib/pleroma/web/websub/websub.ex | 10 +++-- test/web/ostatus/ostatus_test.exs | 54 ++++++++++++------------ test/web/websub/websub_test.exs | 11 ++--- 5 files changed, 70 insertions(+), 67 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 90be86755..3e239179e 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -37,8 +37,8 @@ def handle_incoming(xml_string) do def handle_note(doc) do content_html = string_from_xpath("/entry/content[1]", doc) - [author] = :xmerl_xpath.string('/entry/author[1]', doc) - {:ok, actor} = find_or_make_user(author) + uri = string_from_xpath("/entry/author/uri[1]", doc) + {:ok, actor} = find_or_make_user(uri) context = string_from_xpath("/entry/ostatus:conversation[1]", doc) |> String.trim context = if String.length(context) > 0 do @@ -78,42 +78,30 @@ def handle_note(doc) do ActivityPub.create(to, actor, context, object, %{}, date) end - def find_or_make_user(author_doc) do - {:xmlObj, :string, uri } = :xmerl_xpath.string('string(/author[1]/uri)', author_doc) - + def find_or_make_user(uri) do query = from user in User, - where: user.local == false and fragment("? @> ?", user.info, ^%{ostatus_uri: to_string(uri)}) + where: user.local == false and fragment("? @> ?", user.info, ^%{uri: uri}) user = Repo.one(query) if is_nil(user) do - make_user(author_doc) + make_user(uri) else {:ok, user} end end - def make_user(author_doc) do - author = string_from_xpath("/author[1]/uri", author_doc) - name = string_from_xpath("/author[1]/name", author_doc) - preferredUsername = string_from_xpath("/author[1]/poco:preferredUsername", author_doc) - displayName = string_from_xpath("/author[1]/poco:displayName", author_doc) - avatar = make_avatar_object(author_doc) - - data = %{ - local: false, - name: preferredUsername || name, - nickname: displayName || name, - ap_id: author, - info: %{ - "ostatus_uri" => author, - "host" => URI.parse(author).host, - "system" => "ostatus" - }, - avatar: avatar - } - - Repo.insert(Ecto.Changeset.change(%User{}, data)) + def make_user(uri) do + with {:ok, info} <- gather_user_info(uri) do + data = %{ + local: false, + name: info.name, + nickname: info.nickname, + ap_id: info.uri, + info: info + } + Repo.insert(Ecto.Changeset.change(%User{}, data)) + end end # TODO: Just takes the first one for now. diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 1d8c4d0c8..49796dab8 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -42,7 +42,7 @@ def represent_user(user) do # FIXME: Make this call the host-meta to find the actual address. defp webfinger_address(domain) do - "https://#{domain}/.well-known/webfinger" + "//#{domain}/.well-known/webfinger" end defp webfinger_from_xml(doc) do @@ -61,9 +61,21 @@ defp webfinger_from_xml(doc) do end def finger(account, getter \\ &HTTPoison.get/3) do - [name, domain] = String.split(account, "@") + domain = with [_name, domain] <- String.split(account, "@") do + domain + else _e -> + URI.parse(account).host + end address = webfinger_address(domain) - with {:ok, %{status_code: status_code, body: body}} when status_code in 200..299 <- getter.(address, ["Accept": "application/xrd+xml"], [params: [resource: account]]), + + # try https first + response = with {:ok, result} <- getter.("https:" <> address, ["Accept": "application/xrd+xml"], [params: [resource: account]]) do + {:ok, result} + else _ -> + getter.("http:" <> address, ["Accept": "application/xrd+xml"], [params: [resource: account]]) + end + + with {:ok, %{status_code: status_code, body: body}} when status_code in 200..299 <- response, doc <- XML.parse_document(body), {:ok, data} <- webfinger_from_xml(doc) do {:ok, data} diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index c1d48ad7a..8e3e0a54e 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -102,19 +102,21 @@ defp valid_topic(%{"hub.topic" => topic}, user) do end end - def subscribe(user, topic, requester \\ &request_subscription/1) do + def subscribe(subscriber, subscribed, requester \\ &request_subscription/1) do + topic = subscribed.info["topic"] # FIXME: Race condition, use transactions {:ok, subscription} = with subscription when not is_nil(subscription) <- Repo.get_by(WebsubClientSubscription, topic: topic) do - subscribers = [user.ap_id, subscription.subcribers] |> Enum.uniq + subscribers = [subscriber.ap_id, subscription.subscribers] |> Enum.uniq change = Ecto.Changeset.change(subscription, %{subscribers: subscribers}) Repo.update(change) else _e -> subscription = %WebsubClientSubscription{ topic: topic, - subscribers: [user.ap_id], + hub: subscribed.info["hub"], + subscribers: [subscriber.ap_id], state: "requested", secret: :crypto.strong_rand_bytes(8) |> Base.url_encode64, - user: user + user: subscribed } Repo.insert(subscription) end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 2a5156b31..4f396d940 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -25,40 +25,20 @@ test "handle incoming replies" do end describe "new remote user creation" do - test "make new user or find them based on an 'author' xml doc" do - incoming = File.read!("test/fixtures/user_name_only.xml") - doc = XML.parse_document(incoming) - - {:ok, user} = OStatus.find_or_make_user(doc) - - assert user.name == "lambda" - assert user.nickname == "lambda" - assert user.local == false - assert user.info["ostatus_uri"] == "http://gs.example.org:4040/index.php/user/1" - assert user.info["system"] == "ostatus" - assert user.ap_id == "http://gs.example.org:4040/index.php/user/1" - - {:ok, user_again} = OStatus.find_or_make_user(doc) - - assert user == user_again - end - test "tries to use the information in poco fields" do - incoming = File.read!("test/fixtures/user_full.xml") - doc = XML.parse_document(incoming) + # TODO make test local + uri = "https://social.heldscal.la/user/23211" - {:ok, user} = OStatus.find_or_make_user(doc) + {:ok, user} = OStatus.find_or_make_user(uri) + user = Repo.get(Pleroma.User, user.id) assert user.name == "Constance Variable" assert user.nickname == "lambadalambda" assert user.local == false - assert user.info["ostatus_uri"] == "http://gs.example.org:4040/index.php/user/1" - assert user.info["system"] == "ostatus" - assert user.ap_id == "http://gs.example.org:4040/index.php/user/1" + assert user.info["uri"] == uri + assert user.ap_id == uri - assert List.first(user.avatar["url"])["href"] == "http://gs.example.org:4040/theme/neo-gnu/default-avatar-profile.png" - - {:ok, user_again} = OStatus.find_or_make_user(doc) + {:ok, user_again} = OStatus.find_or_make_user(uri) assert user == user_again end @@ -84,5 +64,25 @@ test "it returns user info in a hash" do } assert data == expected end + + test "it works with the uri" do + user = "https://social.heldscal.la/user/29191" + + # TODO: make test local + {:ok, data} = OStatus.gather_user_info(user) + + expected = %{ + hub: "https://social.heldscal.la/main/push/hub", + magic_key: "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", + name: "shp", + nickname: "shp", + salmon: "https://social.heldscal.la/main/salmon/user/29191", + subject: "https://social.heldscal.la/user/29191", + topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", + uri: "https://social.heldscal.la/user/29191", + fqn: user + } + assert data == expected + end end end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 1b1ef3fa6..25c2b8baa 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -93,12 +93,13 @@ def accepting_verifier(subscription) do end test "initiate a subscription for a given user and topic" do - user = insert(:user) - topic = "http://example.org/some-topic.atom" + subscriber = insert(:user) + user = insert(:user, %{info: %{ "topic" => "some_topic", "hub" => "some_hub"}}) - {:ok, websub} = Websub.subscribe(user, topic, &accepting_verifier/1) - assert websub.subscribers == [user.ap_id] - assert websub.topic == topic + {:ok, websub} = Websub.subscribe(subscriber, user, &accepting_verifier/1) + assert websub.subscribers == [subscriber.ap_id] + assert websub.topic == "some_topic" + assert websub.hub == "some_hub" assert is_binary(websub.secret) assert websub.user == user assert websub.state == "accepted" From ba1ea770012893ea818f248e9a0a2ee3ab854676 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sat, 29 Apr 2017 19:47:56 +0200 Subject: [PATCH 24/88] Make key fetching use ostatus fetching. --- lib/pleroma/web/salmon/salmon.ex | 18 +++++------------- test/fixtures/salmon2.xml | 2 ++ test/web/salmon/salmon_test.exs | 8 ++++++++ 3 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 test/fixtures/salmon2.xml diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 99cca1f55..777898cfa 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -21,24 +21,16 @@ def decode(salmon) do [data, type, encoding, alg, sig] end + # TODO rewrite in with-stile + # Make it fetch the key from the saved user if there is one def fetch_magic_key(salmon) do [data, _, _, _, _] = decode(salmon) doc = XML.parse_document(data) - {:xmlObj, :string, uri} = :xmerl_xpath.string('string(//author[1]/uri)', doc) + uri = XML.string_from_xpath("/entry/author[1]/uri", doc) - uri = to_string(uri) - base = URI.parse(uri).host + {:ok, info} = Pleroma.Web.OStatus.gather_user_info(uri) - # TODO: Find out if this endpoint is mandated by the standard. - # At least diaspora does it differently - {:ok, response} = HTTPoison.get(base <> "/.well-known/webfinger", ["Accept": "application/xrd+xml"], [params: [resource: uri]]) - - doc = XML.parse_document(response.body) - - {:xmlObj, :string, magickey} = :xmerl_xpath.string('string(//Link[@rel="magic-public-key"]/@href)', doc) - "data:application/magic-public-key," <> magickey = to_string(magickey) - - magickey + info.magic_key end def decode_and_validate(magickey, salmon) do diff --git a/test/fixtures/salmon2.xml b/test/fixtures/salmon2.xml new file mode 100644 index 000000000..d8ecbc17e --- /dev/null +++ b/test/fixtures/salmon2.xml @@ -0,0 +1,2 @@ + +PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiID8-PGVudHJ5IHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDA1L0F0b20iIHhtbG5zOnRocj0iaHR0cDovL3B1cmwub3JnL3N5bmRpY2F0aW9uL3RocmVhZC8xLjAiIHhtbG5zOmFjdGl2aXR5PSJodHRwOi8vYWN0aXZpdHlzdHJlYS5tcy9zcGVjLzEuMC8iIHhtbG5zOmdlb3Jzcz0iaHR0cDovL3d3dy5nZW9yc3Mub3JnL2dlb3JzcyIgeG1sbnM6b3N0YXR1cz0iaHR0cDovL29zdGF0dXMub3JnL3NjaGVtYS8xLjAiIHhtbG5zOnBvY289Imh0dHA6Ly9wb3J0YWJsZWNvbnRhY3RzLm5ldC9zcGVjLzEuMCIgeG1sbnM6bWVkaWE9Imh0dHA6Ly9wdXJsLm9yZy9zeW5kaWNhdGlvbi9hdG9tbWVkaWEiIHhtbG5zOnN0YXR1c25ldD0iaHR0cDovL3N0YXR1cy5uZXQvc2NoZW1hL2FwaS8xLyI-CiA8YWN0aXZpdHk6b2JqZWN0LXR5cGU-aHR0cDovL2FjdGl2aXR5c3RyZWEubXMvc2NoZW1hLzEuMC9ub3RlPC9hY3Rpdml0eTpvYmplY3QtdHlwZT4KIDxpZD50YWc6c29jaWFsLmhlbGRzY2FsLmxhLDIwMTctMDQtMjk6bm90aWNlSWQ9MTk2NzEwNjpvYmplY3RUeXBlPW5vdGU8L2lkPgogPHRpdGxlPk5ldyBub3RlIGJ5IGxhbWJhZGFsYW1iZGE8L3RpdGxlPgogPGNvbnRlbnQgdHlwZT0iaHRtbCI-dGVzdCBAJmx0O2EgaHJlZj0mcXVvdDtodHRwczovL3BsZXJvbWEuc295a2FmLmNvbS91c2Vycy9sYWluJnF1b3Q7IGNsYXNzPSZxdW90O2gtY2FyZCB1LXVybCBwLW5pY2tuYW1lIG1lbnRpb24mcXVvdDsmZ3Q7bGFpbiZsdDsvYSZndDs8L2NvbnRlbnQ-CiA8bGluayByZWw9ImFsdGVybmF0ZSIgdHlwZT0idGV4dC9odG1sIiBocmVmPSJodHRwczovL3NvY2lhbC5oZWxkc2NhbC5sYS9ub3RpY2UvMTk2NzEwNiIvPgogPHN0YXR1c19uZXQgbm90aWNlX2lkPSIxOTY3MTA2Ij48L3N0YXR1c19uZXQ-CiA8YWN0aXZpdHk6dmVyYj5odHRwOi8vYWN0aXZpdHlzdHJlYS5tcy9zY2hlbWEvMS4wL3Bvc3Q8L2FjdGl2aXR5OnZlcmI-CiA8cHVibGlzaGVkPjIwMTctMDQtMjlUMTc6Mjg6MjErMDA6MDA8L3B1Ymxpc2hlZD4KIDx1cGRhdGVkPjIwMTctMDQtMjlUMTc6Mjg6MjErMDA6MDA8L3VwZGF0ZWQ-CiA8YXV0aG9yPgogIDxhY3Rpdml0eTpvYmplY3QtdHlwZT5odHRwOi8vYWN0aXZpdHlzdHJlYS5tcy9zY2hlbWEvMS4wL3BlcnNvbjwvYWN0aXZpdHk6b2JqZWN0LXR5cGU-CiAgPHVyaT5odHRwczovL3NvY2lhbC5oZWxkc2NhbC5sYS91c2VyLzIzMjExPC91cmk-CiAgPG5hbWU-bGFtYmFkYWxhbWJkYTwvbmFtZT4KICA8c3VtbWFyeT5DYWxsIG1lIERlYWNvbiBCbHVlcy48L3N1bW1hcnk-CiAgPGxpbmsgcmVsPSJhbHRlcm5hdGUiIHR5cGU9InRleHQvaHRtbCIgaHJlZj0iaHR0cHM6Ly9zb2NpYWwuaGVsZHNjYWwubGEvbGFtYmFkYWxhbWJkYSIvPgogIDxsaW5rIHJlbD0iYXZhdGFyIiB0eXBlPSJpbWFnZS9qcGVnIiBtZWRpYTp3aWR0aD0iMjM2IiBtZWRpYTpoZWlnaHQ9IjIzNiIgaHJlZj0iaHR0cHM6Ly9zb2NpYWwuaGVsZHNjYWwubGEvYXZhdGFyLzIzMjExLW9yaWdpbmFsLTIwMTcwNDE2MTE0MjU1LmpwZWciLz4KICA8bGluayByZWw9ImF2YXRhciIgdHlwZT0iaW1hZ2UvanBlZyIgbWVkaWE6d2lkdGg9Ijk2IiBtZWRpYTpoZWlnaHQ9Ijk2IiBocmVmPSJodHRwczovL3NvY2lhbC5oZWxkc2NhbC5sYS9hdmF0YXIvMjMyMTEtOTYtMjAxNzA0MTYxMTQyNTUuanBlZyIvPgogIDxsaW5rIHJlbD0iYXZhdGFyIiB0eXBlPSJpbWFnZS9qcGVnIiBtZWRpYTp3aWR0aD0iNDgiIG1lZGlhOmhlaWdodD0iNDgiIGhyZWY9Imh0dHBzOi8vc29jaWFsLmhlbGRzY2FsLmxhL2F2YXRhci8yMzIxMS00OC0yMDE3MDQxNjExNDI1NS5qcGVnIi8-CiAgPGxpbmsgcmVsPSJhdmF0YXIiIHR5cGU9ImltYWdlL2pwZWciIG1lZGlhOndpZHRoPSIyNCIgbWVkaWE6aGVpZ2h0PSIyNCIgaHJlZj0iaHR0cHM6Ly9zb2NpYWwuaGVsZHNjYWwubGEvYXZhdGFyLzIzMjExLTI0LTIwMTcwNDE2MTE0MjU3LmpwZWciLz4KICA8cG9jbzpwcmVmZXJyZWRVc2VybmFtZT5sYW1iYWRhbGFtYmRhPC9wb2NvOnByZWZlcnJlZFVzZXJuYW1lPgogIDxwb2NvOmRpc3BsYXlOYW1lPkNvbnN0YW5jZSBWYXJpYWJsZTwvcG9jbzpkaXNwbGF5TmFtZT4KICA8cG9jbzpub3RlPkNhbGwgbWUgRGVhY29uIEJsdWVzLjwvcG9jbzpub3RlPgogIDxwb2NvOmFkZHJlc3M-CiAgIDxwb2NvOmZvcm1hdHRlZD5CZXJsaW48L3BvY286Zm9ybWF0dGVkPgogIDwvcG9jbzphZGRyZXNzPgogIDxwb2NvOnVybHM-CiAgIDxwb2NvOnR5cGU-aG9tZXBhZ2U8L3BvY286dHlwZT4KICAgPHBvY286dmFsdWU-aHR0cHM6Ly9oZWxkc2NhbC5sYTwvcG9jbzp2YWx1ZT4KICAgPHBvY286cHJpbWFyeT50cnVlPC9wb2NvOnByaW1hcnk-CiAgPC9wb2NvOnVybHM-CiAgPGZvbGxvd2VycyB1cmw9Imh0dHBzOi8vc29jaWFsLmhlbGRzY2FsLmxhL2xhbWJhZGFsYW1iZGEvc3Vic2NyaWJlcnMiPjwvZm9sbG93ZXJzPgogIDxzdGF0dXNuZXQ6cHJvZmlsZV9pbmZvIGxvY2FsX2lkPSIyMzIxMSI-PC9zdGF0dXNuZXQ6cHJvZmlsZV9pbmZvPgogPC9hdXRob3I-CiA8bGluayByZWw9Im9zdGF0dXM6Y29udmVyc2F0aW9uIiBocmVmPSJodHRwczovL3NvY2lhbC5oZWxkc2NhbC5sYS9jb252ZXJzYXRpb24vMTAwNzQ5NiIvPgogPG9zdGF0dXM6Y29udmVyc2F0aW9uIGhyZWY9Imh0dHBzOi8vc29jaWFsLmhlbGRzY2FsLmxhL2NvbnZlcnNhdGlvbi8xMDA3NDk2IiBsb2NhbF9pZD0iMTAwNzQ5NiIgcmVmPSJ0YWc6c29jaWFsLmhlbGRzY2FsLmxhLDIwMTctMDQtMjk6b2JqZWN0VHlwZT10aHJlYWQ6bm9uY2U9NDU5ZGYyMjM2NDFiMDNkZSI-dGFnOnNvY2lhbC5oZWxkc2NhbC5sYSwyMDE3LTA0LTI5Om9iamVjdFR5cGU9dGhyZWFkOm5vbmNlPTQ1OWRmMjIzNjQxYjAzZGU8L29zdGF0dXM6Y29udmVyc2F0aW9uPgogPGxpbmsgcmVsPSJtZW50aW9uZWQiIG9zdGF0dXM6b2JqZWN0LXR5cGU9Imh0dHA6Ly9hY3Rpdml0eXN0cmVhLm1zL3NjaGVtYS8xLjAvcGVyc29uIiBocmVmPSJodHRwczovL3BsZXJvbWEuc295a2FmLmNvbS91c2Vycy9sYWluIi8-CiA8bGluayByZWw9Im1lbnRpb25lZCIgb3N0YXR1czpvYmplY3QtdHlwZT0iaHR0cDovL2FjdGl2aXR5c3RyZWEubXMvc2NoZW1hLzEuMC9jb2xsZWN0aW9uIiBocmVmPSJodHRwOi8vYWN0aXZpdHlzY2hlbWEub3JnL2NvbGxlY3Rpb24vcHVibGljIi8-CiA8c291cmNlPgogIDxpZD5odHRwczovL3NvY2lhbC5oZWxkc2NhbC5sYS9hcGkvc3RhdHVzZXMvdXNlcl90aW1lbGluZS8yMzIxMS5hdG9tPC9pZD4KICA8dGl0bGU-Q29uc3RhbmNlIFZhcmlhYmxlPC90aXRsZT4KICA8bGluayByZWw9ImFsdGVybmF0ZSIgdHlwZT0idGV4dC9odG1sIiBocmVmPSJodHRwczovL3NvY2lhbC5oZWxkc2NhbC5sYS9sYW1iYWRhbGFtYmRhIi8-CiAgPGxpbmsgcmVsPSJzZWxmIiB0eXBlPSJhcHBsaWNhdGlvbi9hdG9tK3htbCIgaHJlZj0iaHR0cHM6Ly9zb2NpYWwuaGVsZHNjYWwubGEvYXBpL3N0YXR1c2VzL3VzZXJfdGltZWxpbmUvMjMyMTEuYXRvbSIvPgogIDxsaW5rIHJlbD0ibGljZW5zZSIgaHJlZj0iaHR0cHM6Ly9jcmVhdGl2ZWNvbW1vbnMub3JnL2xpY2Vuc2VzL2J5LzMuMC8iLz4KICA8aWNvbj5odHRwczovL3NvY2lhbC5oZWxkc2NhbC5sYS9hdmF0YXIvMjMyMTEtOTYtMjAxNzA0MTYxMTQyNTUuanBlZzwvaWNvbj4KICA8dXBkYXRlZD4yMDE3LTA0LTI5VDE3OjI4OjIxKzAwOjAwPC91cGRhdGVkPgogPC9zb3VyY2U-CiA8bGluayByZWw9InNlbGYiIHR5cGU9ImFwcGxpY2F0aW9uL2F0b20reG1sIiBocmVmPSJodHRwczovL3NvY2lhbC5oZWxkc2NhbC5sYS9hcGkvc3RhdHVzZXMvc2hvdy8xOTY3MTA2LmF0b20iLz4KIDxsaW5rIHJlbD0iZWRpdCIgdHlwZT0iYXBwbGljYXRpb24vYXRvbSt4bWwiIGhyZWY9Imh0dHBzOi8vc29jaWFsLmhlbGRzY2FsLmxhL2FwaS9zdGF0dXNlcy9zaG93LzE5NjcxMDYuYXRvbSIvPgogPHN0YXR1c25ldDpub3RpY2VfaW5mbyBsb2NhbF9pZD0iMTk2NzEwNiIgc291cmNlPSJQbGVyb21hIEZFIj48L3N0YXR1c25ldDpub3RpY2VfaW5mbz4KPC9lbnRyeT4Kbase64urlRSA-SHA256CJ3wiWW9Io6Y24To3PFBF8cGuvJG8ps5zEwu1k1kSAlSX7WcysvS4ZoPKICFrD4brJxMLpW3AQCLNPIa246-Y0noGiNdpj0w0_TWgWXukWo50pD7cWVugr15YCMUtC-v00iDYfZTlmrTVM6kSCcpAmGMbZPTaXVmKZryjTDoXSI= \ No newline at end of file diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs index 6fbabd19f..aa77659d0 100644 --- a/test/web/salmon/salmon_test.exs +++ b/test/web/salmon/salmon_test.exs @@ -49,4 +49,12 @@ test "encodes an xml payload with a private key" do assert doc == decoded_doc end + + test "it gets a magic key" do + # TODO: Make test local + salmon = File.read!("test/fixtures/salmon2.xml") + key = Salmon.fetch_magic_key(salmon) + + assert key == "RSA.uzg6r1peZU0vXGADWxGJ0PE34WvmhjUmydbX5YYdOiXfODVLwCMi1umGoqUDm-mRu4vNEdFBVJU1CpFA7dKzWgIsqsa501i2XqElmEveXRLvNRWFB6nG03Q5OUY2as8eE54BJm0p20GkMfIJGwP6TSFb-ICp3QjzbatuSPJ6xCE=.AQAB" + end end From 20015b4b67cf0dfab6bdb658c9eb0e1ae04febdc Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sat, 29 Apr 2017 20:08:45 +0200 Subject: [PATCH 25/88] Save remote users with fqn as nickname. --- lib/pleroma/web/ostatus/ostatus.ex | 4 +++- lib/pleroma/web/websub/websub.ex | 3 ++- test/web/ostatus/ostatus_test.exs | 4 +++- test/web/websub/websub_test.exs | 3 ++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 3e239179e..59c5d8e9e 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -96,10 +96,12 @@ def make_user(uri) do data = %{ local: false, name: info.name, - nickname: info.nickname, + nickname: info.nickname <> "@" <> info.host, ap_id: info.uri, info: info } + # TODO: Make remote user changeset + # SHould enforce fqn nickname Repo.insert(Ecto.Changeset.change(%User{}, data)) end end diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 8e3e0a54e..3fd779fba 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -139,7 +139,8 @@ def gather_feed_data(topic, getter \\ &HTTPoison.get/1) do uri: uri, hub: hub, nickname: preferredUsername || name, - name: displayName || name + name: displayName || name, + host: URI.parse(uri).host }} else e -> {:error, e} diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 4f396d940..cc0975bb5 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -33,7 +33,7 @@ test "tries to use the information in poco fields" do user = Repo.get(Pleroma.User, user.id) assert user.name == "Constance Variable" - assert user.nickname == "lambadalambda" + assert user.nickname == "lambadalambda@social.heldscal.la" assert user.local == false assert user.info["uri"] == uri assert user.ap_id == uri @@ -60,6 +60,7 @@ test "it returns user info in a hash" do subject: "acct:shp@social.heldscal.la", topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", uri: "https://social.heldscal.la/user/29191", + host: "social.heldscal.la", fqn: user } assert data == expected @@ -80,6 +81,7 @@ test "it works with the uri" do subject: "https://social.heldscal.la/user/29191", topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", uri: "https://social.heldscal.la/user/29191", + host: "social.heldscal.la", fqn: user } assert data == expected diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 25c2b8baa..e0d71e16d 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -118,7 +118,8 @@ test "discovers the hub and canonical url" do hub: "https://mastodon.social/api/push", uri: "https://mastodon.social/users/lambadalambda", nickname: "lambadalambda", - name: "Critical Value" + name: "Critical Value", + host: "mastodon.social" } assert expected == discovered From a16da387d251edc4d1bae949146c807d217cee1f Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sat, 29 Apr 2017 21:13:21 +0200 Subject: [PATCH 26/88] Handle full incoming feeds. --- lib/pleroma/web/ostatus/ostatus.ex | 34 +++++++------ test/fixtures/ostatus_incoming_post.xml | 57 ++++++++++++++++++++++ test/web/ostatus/ostatus_test.exs | 17 +++++-- test/web/websub/websub_controller_test.exs | 2 + 4 files changed, 92 insertions(+), 18 deletions(-) create mode 100644 test/fixtures/ostatus_incoming_post.xml diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 59c5d8e9e..9f85d971a 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -21,26 +21,32 @@ def salmon_path(user) do def handle_incoming(xml_string) do doc = parse_document(xml_string) + entries = :xmerl_xpath.string('//entry', doc) - {:xmlObj, :string, object_type } = :xmerl_xpath.string('string(/entry/activity:object-type[1])', doc) + activities = Enum.map(entries, fn (entry) -> + {:xmlObj, :string, object_type } = :xmerl_xpath.string('string(/entry/activity:object-type[1])', entry) - case object_type do - 'http://activitystrea.ms/schema/1.0/note' -> - handle_note(doc) - _ -> - Logger.error("Couldn't parse incoming document") - end + case object_type do + 'http://activitystrea.ms/schema/1.0/note' -> + {:ok, activity} = handle_note(entry, doc) + activity + _ -> + Logger.error("Couldn't parse incoming document") + nil + end + end) + {:ok, activities} end # TODO # wire up replies - def handle_note(doc) do - content_html = string_from_xpath("/entry/content[1]", doc) + def handle_note(entry, doc \\ nil) do + content_html = string_from_xpath("/entry/content[1]", entry) - uri = string_from_xpath("/entry/author/uri[1]", doc) + uri = string_from_xpath("/entry/author/uri[1]", entry) || string_from_xpath("/feed/author/uri[1]", doc) {:ok, actor} = find_or_make_user(uri) - context = string_from_xpath("/entry/ostatus:conversation[1]", doc) |> String.trim + context = string_from_xpath("/entry/ostatus:conversation[1]", entry) |> String.trim context = if String.length(context) > 0 do context else @@ -51,12 +57,12 @@ def handle_note(doc) do "https://www.w3.org/ns/activitystreams#Public" ] - mentions = :xmerl_xpath.string('/entry/link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]', doc) + mentions = :xmerl_xpath.string('/entry/link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]', entry) |> Enum.map(fn(person) -> string_from_xpath("@href", person) end) to = to ++ mentions - date = string_from_xpath("/entry/published", doc) + date = string_from_xpath("/entry/published", entry) object = %{ "type" => "Note", @@ -67,7 +73,7 @@ def handle_note(doc) do "actor" => actor.ap_id } - inReplyTo = string_from_xpath("/entry/thr:in-reply-to[1]/@href", doc) + inReplyTo = string_from_xpath("/entry/thr:in-reply-to[1]/@href", entry) object = if inReplyTo do Map.put(object, "inReplyTo", inReplyTo) diff --git a/test/fixtures/ostatus_incoming_post.xml b/test/fixtures/ostatus_incoming_post.xml new file mode 100644 index 000000000..7967e1b32 --- /dev/null +++ b/test/fixtures/ostatus_incoming_post.xml @@ -0,0 +1,57 @@ + + + GNU social + https://social.heldscal.la/api/statuses/user_timeline/23211.atom + lambadalambda timeline + Updates from lambadalambda on social.heldscal.la! + https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg + 2017-04-29T18:25:38+00:00 + + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + Call me Deacon Blues. + + + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + + Berlin + + + homepage + https://heldscal.la + true + + + + + + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:social.heldscal.la,2017-04-29:noticeId=1967725:objectType=note + New note by lambadalambda + Will it blend? + + + http://activitystrea.ms/schema/1.0/post + 2017-04-29T18:25:38+00:00 + 2017-04-29T18:25:38+00:00 + + tag:social.heldscal.la,2017-04-29:objectType=thread:nonce=3f3a9dd83acc4e35 + + + + + + diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index cc0975bb5..1e747c728 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -1,11 +1,10 @@ defmodule Pleroma.Web.OStatusTest do use Pleroma.DataCase alias Pleroma.Web.OStatus - alias Pleroma.Web.XML - test "handle incoming notes" do + test "handle incoming note - GS, Salmon" do incoming = File.read!("test/fixtures/incoming_note_activity.xml") - {:ok, activity} = OStatus.handle_incoming(incoming) + {:ok, [activity]} = OStatus.handle_incoming(incoming) assert activity.data["type"] == "Create" assert activity.data["object"]["type"] == "Note" @@ -14,9 +13,19 @@ test "handle incoming notes" do assert "http://pleroma.example.org:4000/users/lain3" in activity.data["to"] end + test "handle incoming notes - GS, subscription" do + incoming = File.read!("test/fixtures/ostatus_incoming_post.xml") + {:ok, [activity]} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Create" + assert activity.data["object"]["type"] == "Note" + assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" + assert activity.data["object"]["content"] == "Will it blend?" + end + test "handle incoming replies" do incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") - {:ok, activity} = OStatus.handle_incoming(incoming) + {:ok, [activity]} = OStatus.handle_incoming(incoming) assert activity.data["type"] == "Create" assert activity.data["object"]["type"] == "Note" diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs index 521bbb9aa..8f68248a4 100644 --- a/test/web/websub/websub_controller_test.exs +++ b/test/web/websub/websub_controller_test.exs @@ -41,6 +41,8 @@ test "websub subscription confirmation", %{conn: conn} do assert response(conn, 200) == "some challenge" assert websub.state == "accepted" + + # TODO valid_until end test "handles incoming feed updates", %{conn: conn} do From 8a0d2b33d8c9a1cef347c5daf5589a2245eb01b0 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 09:25:46 +0200 Subject: [PATCH 27/88] Keep ostatus id as activity id. --- lib/pleroma/web/ostatus/ostatus.ex | 3 ++- test/web/ostatus/ostatus_test.exs | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 9f85d971a..f8e33bc7e 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -63,6 +63,7 @@ def handle_note(entry, doc \\ nil) do to = to ++ mentions date = string_from_xpath("/entry/published", entry) + id = string_from_xpath("/entry/id", entry) object = %{ "type" => "Note", @@ -81,7 +82,7 @@ def handle_note(entry, doc \\ nil) do object end - ActivityPub.create(to, actor, context, object, %{}, date) + ActivityPub.create(to, actor, context, object, %{"id" => id}, date) end def find_or_make_user(uri) do diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 1e747c728..a53e0ebde 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -7,6 +7,7 @@ test "handle incoming note - GS, Salmon" do {:ok, [activity]} = OStatus.handle_incoming(incoming) assert activity.data["type"] == "Create" + assert activity.data["id"] == "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note" assert activity.data["object"]["type"] == "Note" assert activity.data["published"] == "2017-04-23T14:51:03+00:00" assert activity.data["context"] == "tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b" From ffc604a2c2d963b63e6cd13d0ee7cc9024f632a4 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 10:04:54 +0200 Subject: [PATCH 28/88] Use cache for user info data. Later these should be persisted in the user. --- lib/pleroma/user.ex | 5 +++++ lib/pleroma/web/twitter_api/representers/user_representer.ex | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 9b7912c5b..cd6104680 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -122,4 +122,9 @@ def get_cached_by_nickname(nickname) do key = "nickname:#{nickname}" Cachex.get!(:user_cache, key, fallback: fn(_) -> Repo.get_by(User, nickname: nickname) end) end + + def get_cached_user_info(user) do + key = "user_info:#{user.id}" + Cachex.get!(:user_cache, key, fallback: fn(_) -> user_info(user) end) + end end diff --git a/lib/pleroma/web/twitter_api/representers/user_representer.ex b/lib/pleroma/web/twitter_api/representers/user_representer.ex index ab7d6d353..29c7451f4 100644 --- a/lib/pleroma/web/twitter_api/representers/user_representer.ex +++ b/lib/pleroma/web/twitter_api/representers/user_representer.ex @@ -11,7 +11,7 @@ def to_map(user, opts) do false end - user_info = User.user_info(user) + user_info = User.get_cached_user_info(user) map = %{ "id" => user.id, From 11ea08649d5a5e5d2ac9ee29406f53240be77ec4 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 10:06:57 +0200 Subject: [PATCH 29/88] Make cache bigger and longer lived. --- lib/pleroma/application.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 86b6c0c1e..6267d0695 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -15,9 +15,9 @@ def start(_type, _args) do # Start your own worker by calling: Pleroma.Worker.start_link(arg1, arg2, arg3) # worker(Pleroma.Worker, [arg1, arg2, arg3]), worker(Cachex, [:user_cache, [ - default_ttl: 5000, + default_ttl: 25000, ttl_interval: 1000, - limit: 500 + limit: 2500 ]]) ] From 9d7c3190cc346bf2a5576b6b93c26723059ae9a1 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 11:16:41 +0200 Subject: [PATCH 30/88] Get create activity from created object id. This is useful for Ostatus federation because ostatus doesn't have different ids for objects and activities... --- lib/pleroma/activity.ex | 5 +++++ test/activity_test.exs | 7 +++++++ test/web/ostatus/feed_representer_test.exs | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 46568bb13..80d96d0f2 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -18,4 +18,9 @@ def all_by_object_ap_id(ap_id) do Repo.all(from activity in Activity, where: fragment("? @> ?", activity.data, ^%{object: %{id: ap_id}})) end + + def get_create_activity_by_object_ap_id(ap_id) do + Repo.one(from activity in Activity, + where: fragment("? @> ?", activity.data, ^%{type: "Create", object: %{id: ap_id}})) + end end diff --git a/test/activity_test.exs b/test/activity_test.exs index ce6eb1545..366a2f957 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -15,4 +15,11 @@ test "returns activities by it's objects AP ids" do assert activity == found_activity end + + test "returns the activity that created an object" do + activity = insert(:note_activity) + found_activity = Pleroma.Activity.get_create_activity_by_object_ap_id(activity.data["object"]["id"]) + + assert activity == found_activity + end end diff --git a/test/web/ostatus/feed_representer_test.exs b/test/web/ostatus/feed_representer_test.exs index ef0f4d5ff..7bbfae49a 100644 --- a/test/web/ostatus/feed_representer_test.exs +++ b/test/web/ostatus/feed_representer_test.exs @@ -22,7 +22,7 @@ test "returns a feed of the last 20 items of the user" do |> :xmerl.export_simple_content(:xmerl_xml) expected = """ - + #{OStatus.feed_path(user)} #{user.nickname}'s timeline #{most_recent_update} From d937a8e69567ace33a72d5248c046860305076d7 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 11:17:34 +0200 Subject: [PATCH 31/88] Add thr:in-reply-to to ostatus representer. --- .../web/ostatus/activity_representer.ex | 19 +++++++++- lib/pleroma/web/ostatus/feed_representer.ex | 1 + .../web/ostatus/activity_representer_test.exs | 36 +++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 30e695bcc..07b9033b9 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -1,4 +1,19 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do + alias Pleroma.Activity + require Logger + + defp get_in_reply_to(%{"object" => %{ "inReplyTo" => in_reply_to}}) do + with %Activity{data: %{"id" => id}} <- Activity.get_create_activity_by_object_ap_id(in_reply_to) do + [{:"thr:in-reply-to", [ref: to_charlist(id)], []}] + else _e -> + Logger.debug("Couldn't find replied-to activity:") + Logger.debug(in_reply_to) + [] + end + end + + defp get_in_reply_to(_), do: [] + def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) do h = fn(str) -> [to_charlist(str)] end @@ -12,6 +27,8 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) {:link, [rel: 'enclosure', href: to_charlist(url["href"]), type: to_charlist(url["mediaType"])], []} end) + in_reply_to = get_in_reply_to(activity.data) + [ {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']}, {:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']}, @@ -22,7 +39,7 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) {:updated, h.(updated_at)}, {:"ostatus:conversation", [], h.(activity.data["context"])}, {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []} - ] ++ attachments + ] ++ attachments ++ in_reply_to end def to_simple_form(_,_), do: nil diff --git a/lib/pleroma/web/ostatus/feed_representer.ex b/lib/pleroma/web/ostatus/feed_representer.ex index 10a1ffb25..db7b685f3 100644 --- a/lib/pleroma/web/ostatus/feed_representer.ex +++ b/lib/pleroma/web/ostatus/feed_representer.ex @@ -16,6 +16,7 @@ def to_simple_form(user, activities, users) do [{ :feed, [ xmlns: 'http://www.w3.org/2005/Atom', + "xmlns:thr": 'http://purl.org/syndication/thread/1.0', "xmlns:activity": 'http://activitystrea.ms/spec/1.0/', "xmlns:poco": 'http://portablecontacts.net/spec/1.0', "xmlns:ostatus": 'http://ostatus.org/schema/1.0' diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 6cea9cff0..fd1b1598c 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -34,6 +34,42 @@ test "a note activity" do assert clean(res) == clean(expected) end + test "a reply note" do + note = insert(:note_activity) + answer = insert(:note_activity) + object = answer.data["object"] + object = Map.put(object, "inReplyTo", note.data["object"]["id"]) + + data = %{answer.data | "object" => object} + answer = %{answer | data: data} + + updated_at = answer.updated_at + |> NaiveDateTime.to_iso8601 + inserted_at = answer.inserted_at + |> NaiveDateTime.to_iso8601 + + user = User.get_cached_by_ap_id(answer.data["actor"]) + + expected = """ + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + #{answer.data["id"]} + New note by #{user.nickname} + #{answer.data["object"]["content"]} + #{inserted_at} + #{updated_at} + #{answer.data["context"]} + + + """ + + tuple = ActivityRepresenter.to_simple_form(answer, user) + + res = :xmerl.export_simple_content(tuple, :xmerl_xml) |> IO.iodata_to_binary + + assert clean(res) == clean(expected) + end + test "an unknown activity" do tuple = ActivityRepresenter.to_simple_form(%Activity{}, nil) assert is_nil(tuple) From 84027ff00b7fc63934f12129f84b5c7ee1d39248 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 11:39:27 +0200 Subject: [PATCH 32/88] Handle comments. --- lib/pleroma/web/ostatus/ostatus.ex | 5 +- test/fixtures/ostatus_incoming_reply.xml | 60 ++++++++++++++++++++++++ test/web/ostatus/ostatus_test.exs | 11 +++++ 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/ostatus_incoming_reply.xml diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index f8e33bc7e..cd471f860 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -30,6 +30,9 @@ def handle_incoming(xml_string) do 'http://activitystrea.ms/schema/1.0/note' -> {:ok, activity} = handle_note(entry, doc) activity + 'http://activitystrea.ms/schema/1.0/comment' -> + {:ok, activity} = handle_note(entry, doc) + activity _ -> Logger.error("Couldn't parse incoming document") nil @@ -74,7 +77,7 @@ def handle_note(entry, doc \\ nil) do "actor" => actor.ap_id } - inReplyTo = string_from_xpath("/entry/thr:in-reply-to[1]/@href", entry) + inReplyTo = string_from_xpath("/entry/thr:in-reply-to[1]/@ref", entry) object = if inReplyTo do Map.put(object, "inReplyTo", inReplyTo) diff --git a/test/fixtures/ostatus_incoming_reply.xml b/test/fixtures/ostatus_incoming_reply.xml new file mode 100644 index 000000000..83a427a68 --- /dev/null +++ b/test/fixtures/ostatus_incoming_reply.xml @@ -0,0 +1,60 @@ + + + GNU social + https://social.heldscal.la/api/statuses/user_timeline/23211.atom + lambadalambda timeline + Updates from lambadalambda on social.heldscal.la! + https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg + 2017-04-30T09:30:32+00:00 + + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + Call me Deacon Blues. + + + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + + Berlin + + + homepage + https://heldscal.la + true + + + + + + + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:social.heldscal.la,2017-04-30:noticeId=1978790:objectType=comment + New comment by lambadalambda + @<a href="https://gs.archae.me/user/4687" class="h-card u-url p-nickname mention" title="shpbot">shpbot</a> why not indeed. + + + http://activitystrea.ms/schema/1.0/post + 2017-04-30T09:30:32+00:00 + 2017-04-30T09:30:32+00:00 + + + + https://gs.archae.me/conversation/327120 + + + + + + + diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index a53e0ebde..5452e5888 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -24,6 +24,17 @@ test "handle incoming notes - GS, subscription" do assert activity.data["object"]["content"] == "Will it blend?" end + test "handle incoming notes - GS, subscription, reply" do + incoming = File.read!("test/fixtures/ostatus_incoming_reply.xml") + {:ok, [activity]} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Create" + assert activity.data["object"]["type"] == "Note" + assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" + assert activity.data["object"]["content"] == "@shpbot why not indeed." + assert activity.data["object"]["inReplyTo"] == "tag:gs.archae.me,2017-04-30:noticeId=778260:objectType=note" + end + test "handle incoming replies" do incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) From 62607f37dcf3ab149baa09fe144959a25322be69 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 11:55:19 +0200 Subject: [PATCH 33/88] Federate object id for posts in ostatus. This is because ostatus doens't have an id for the activities. --- lib/pleroma/web/ostatus/activity_representer.ex | 10 ++-------- lib/pleroma/web/ostatus/ostatus.ex | 3 ++- test/web/ostatus/activity_representer_test.exs | 6 +++--- test/web/ostatus/ostatus_test.exs | 2 +- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 07b9033b9..274111ac9 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -3,13 +3,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do require Logger defp get_in_reply_to(%{"object" => %{ "inReplyTo" => in_reply_to}}) do - with %Activity{data: %{"id" => id}} <- Activity.get_create_activity_by_object_ap_id(in_reply_to) do - [{:"thr:in-reply-to", [ref: to_charlist(id)], []}] - else _e -> - Logger.debug("Couldn't find replied-to activity:") - Logger.debug(in_reply_to) - [] - end + [{:"thr:in-reply-to", [ref: to_charlist(in_reply_to)], []}] end defp get_in_reply_to(_), do: [] @@ -32,7 +26,7 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) [ {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']}, {:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']}, - {:id, h.(activity.data["id"])}, + {:id, h.(activity.data["object"]["id"])}, # For notes, federate the object id. {:title, ['New note by #{user.nickname}']}, {:content, [type: 'html'], h.(activity.data["object"]["content"])}, {:published, h.(inserted_at)}, diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index cd471f860..6f169af73 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -69,6 +69,7 @@ def handle_note(entry, doc \\ nil) do id = string_from_xpath("/entry/id", entry) object = %{ + "id" => id, "type" => "Note", "to" => to, "content" => content_html, @@ -85,7 +86,7 @@ def handle_note(entry, doc \\ nil) do object end - ActivityPub.create(to, actor, context, object, %{"id" => id}, date) + ActivityPub.create(to, actor, context, object, %{}, date) end def find_or_make_user(uri) do diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index fd1b1598c..6344889b1 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -18,7 +18,7 @@ test "a note activity" do expected = """ http://activitystrea.ms/schema/1.0/note http://activitystrea.ms/schema/1.0/post - #{note_activity.data["id"]} + #{note_activity.data["object"]["id"]} New note by #{user.nickname} #{note_activity.data["object"]["content"]} #{inserted_at} @@ -53,14 +53,14 @@ test "a reply note" do expected = """ http://activitystrea.ms/schema/1.0/note http://activitystrea.ms/schema/1.0/post - #{answer.data["id"]} + #{answer.data["object"]["id"]} New note by #{user.nickname} #{answer.data["object"]["content"]} #{inserted_at} #{updated_at} #{answer.data["context"]} - + """ tuple = ActivityRepresenter.to_simple_form(answer, user) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 5452e5888..3edd39911 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -7,8 +7,8 @@ test "handle incoming note - GS, Salmon" do {:ok, [activity]} = OStatus.handle_incoming(incoming) assert activity.data["type"] == "Create" - assert activity.data["id"] == "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note" assert activity.data["object"]["type"] == "Note" + assert activity.data["object"]["id"] == "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note" assert activity.data["published"] == "2017-04-23T14:51:03+00:00" assert activity.data["context"] == "tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b" assert "http://pleroma.example.org:4000/users/lain3" in activity.data["to"] From 18edc299b262974d3acb9d6f9c3758629b2c0968 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 12:36:47 +0200 Subject: [PATCH 34/88] Handle duplicates. --- lib/pleroma/web/ostatus/ostatus.ex | 15 +++++++++------ test/web/ostatus/ostatus_test.exs | 6 ++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 6f169af73..16b6ac421 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -3,7 +3,7 @@ defmodule Pleroma.Web.OStatus do import Pleroma.Web.XML require Logger - alias Pleroma.{Repo, User, Web} + alias Pleroma.{Repo, User, Web, Object} alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.{WebFinger, Websub} @@ -28,11 +28,9 @@ def handle_incoming(xml_string) do case object_type do 'http://activitystrea.ms/schema/1.0/note' -> - {:ok, activity} = handle_note(entry, doc) - activity + with {:ok, activity} <- handle_note(entry, doc), do: activity 'http://activitystrea.ms/schema/1.0/comment' -> - {:ok, activity} = handle_note(entry, doc) - activity + with {:ok, activity} <- handle_note(entry, doc), do: activity _ -> Logger.error("Couldn't parse incoming document") nil @@ -86,7 +84,12 @@ def handle_note(entry, doc \\ nil) do object end - ActivityPub.create(to, actor, context, object, %{}, date) + # TODO: Bail out sooner and use transaction. + if Object.get_by_ap_id(id) do + {:error, "duplicate activity"} + else + ActivityPub.create(to, actor, context, object, %{}, date) + end end def find_or_make_user(uri) do diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 3edd39911..07073a40d 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -2,6 +2,12 @@ defmodule Pleroma.Web.OStatusTest do use Pleroma.DataCase alias Pleroma.Web.OStatus + test "don't insert create notes twice" do + incoming = File.read!("test/fixtures/incoming_note_activity.xml") + {:ok, [_activity]} = OStatus.handle_incoming(incoming) + assert {:ok, [{:error, "duplicate activity"}]} == OStatus.handle_incoming(incoming) + end + test "handle incoming note - GS, Salmon" do incoming = File.read!("test/fixtures/incoming_note_activity.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) From f9912599c4688a8609bd3500e0548eb2bf06c4a9 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 12:53:49 +0200 Subject: [PATCH 35/88] Pull in remote avatar on federation. --- lib/pleroma/web/ostatus/ostatus.ex | 11 +++++------ lib/pleroma/web/websub/websub.ex | 4 +++- test/web/ostatus/ostatus_test.exs | 7 +++++-- test/web/websub/websub_test.exs | 3 ++- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 16b6ac421..01d6745ef 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -39,8 +39,6 @@ def handle_incoming(xml_string) do {:ok, activities} end - # TODO - # wire up replies def handle_note(entry, doc \\ nil) do content_html = string_from_xpath("/entry/content[1]", entry) @@ -112,7 +110,8 @@ def make_user(uri) do name: info.name, nickname: info.nickname <> "@" <> info.host, ap_id: info.uri, - info: info + info: info, + avatar: info.avatar } # TODO: Make remote user changeset # SHould enforce fqn nickname @@ -121,9 +120,9 @@ def make_user(uri) do end # TODO: Just takes the first one for now. - defp make_avatar_object(author_doc) do - href = string_from_xpath("/author[1]/link[@rel=\"avatar\"]/@href", author_doc) - type = string_from_xpath("/author[1]/link[@rel=\"avatar\"]/@type", author_doc) + def make_avatar_object(author_doc) do + href = string_from_xpath("/feed/author[1]/link[@rel=\"avatar\"]/@href", author_doc) + type = string_from_xpath("/feed/author[1]/link[@rel=\"avatar\"]/@type", author_doc) if href do %{ diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 3fd779fba..63a91055a 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -134,13 +134,15 @@ def gather_feed_data(topic, getter \\ &HTTPoison.get/1) do name = XML.string_from_xpath("/feed/author[1]/name", doc) preferredUsername = XML.string_from_xpath("/feed/author[1]/poco:preferredUsername", doc) displayName = XML.string_from_xpath("/feed/author[1]/poco:displayName", doc) + avatar = OStatus.make_avatar_object(doc) {:ok, %{ uri: uri, hub: hub, nickname: preferredUsername || name, name: displayName || name, - host: URI.parse(uri).host + host: URI.parse(uri).host, + avatar: avatar }} else e -> {:error, e} diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 07073a40d..4e7e401cd 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -64,6 +64,7 @@ test "tries to use the information in poco fields" do assert user.local == false assert user.info["uri"] == uri assert user.ap_id == uri + assert user.avatar["type"] == "Image" {:ok, user_again} = OStatus.find_or_make_user(uri) @@ -88,7 +89,8 @@ test "it returns user info in a hash" do topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", uri: "https://social.heldscal.la/user/29191", host: "social.heldscal.la", - fqn: user + fqn: user, + avatar: %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} } assert data == expected end @@ -109,7 +111,8 @@ test "it works with the uri" do topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", uri: "https://social.heldscal.la/user/29191", host: "social.heldscal.la", - fqn: user + fqn: user, + avatar: %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} } assert data == expected end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index e0d71e16d..ad312cd25 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -119,7 +119,8 @@ test "discovers the hub and canonical url" do uri: "https://mastodon.social/users/lambadalambda", nickname: "lambadalambda", name: "Critical Value", - host: "mastodon.social" + host: "mastodon.social", + avatar: %{"type" => "Image", "url" => [%{"href" => "https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif?1492379244", "mediaType" => "image/gif", "type" => "Link"}]} } assert expected == discovered From 4c8111c3342aa57cf38accf64f0aa06be6958704 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 13:53:26 +0200 Subject: [PATCH 36/88] Use conversation mapping objects to get / retrieve context from TwAPI. --- lib/pleroma/object.ex | 4 +++ .../representers/activity_representer.ex | 10 ++++-- lib/pleroma/web/twitter_api/twitter_api.ex | 25 +++++++++++---- .../activity_representer_test.exs | 11 ++++--- .../twitter_api_controller_test.exs | 7 ++-- test/web/twitter_api/twitter_api_test.exs | 32 +++++++++++++++++-- 6 files changed, 71 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index f932034d7..a924c3199 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -13,4 +13,8 @@ def get_by_ap_id(ap_id) do Repo.one(from object in Object, where: fragment("? @> ?", object.data, ^%{id: ap_id})) end + + def context_mapping(context) do + %Object{data: %{"id" => context}} + end end diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex index f2bf93abb..bfaabb4e4 100644 --- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex +++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex @@ -1,9 +1,9 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do use Pleroma.Web.TwitterAPI.Representers.BaseRepresenter alias Pleroma.Web.TwitterAPI.Representers.{UserRepresenter, ObjectRepresenter} + alias Pleroma.Web.TwitterAPI.TwitterAPI alias Pleroma.Activity - defp user_by_ap_id(user_list, ap_id) do Enum.find(user_list, fn (%{ap_id: user_id}) -> ap_id == user_id end) end @@ -82,6 +82,12 @@ def to_map(%Activity{} = activity, %{user: user} = opts) do |> Enum.filter(&(&1)) |> Enum.map(fn (user) -> UserRepresenter.to_map(user, opts) end) + + conversation_id = with context when not is_nil(context) <- activity.data["context"] do + TwitterAPI.context_to_conversation_id(context) + else _e -> nil + end + %{ "id" => activity.id, "user" => UserRepresenter.to_map(user, opts), @@ -92,7 +98,7 @@ def to_map(%Activity{} = activity, %{user: user} = opts) do "is_post_verb" => true, "created_at" => created_at, "in_reply_to_status_id" => activity.data["object"]["inReplyToStatusId"], - "statusnet_conversation_id" => activity.data["object"]["statusnetConversationId"], + "statusnet_conversation_id" => conversation_id, "attachments" => (activity.data["object"]["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts), "attentions" => attentions, "fave_num" => like_count, diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 1c3396d27..b2fb72a81 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -102,12 +102,7 @@ def fetch_mentions(user, opts \\ %{}) do end def fetch_conversation(user, id) do - query = from activity in Activity, - where: fragment("? @> ?", activity.data, ^%{ statusnetConversationId: id}), - limit: 1 - - with %Activity{} = activity <- Repo.one(query), - context <- activity.data["context"], + with context when is_binary(context) <- conversation_id_to_context(id), activities <- ActivityPub.fetch_activities_for_context(context), statuses <- activities |> activities_to_statuses(%{for: user}) do @@ -322,4 +317,22 @@ defp activity_to_status(activity, opts) do defp make_date do DateTime.utc_now() |> DateTime.to_iso8601 end + + def context_to_conversation_id(context) do + with %Object{id: id} <- Object.get_by_ap_id(context) do + id + else _e -> + changeset = Object.context_mapping(context) + {:ok, %{id: id}} = Repo.insert(changeset) + id + end + end + + def conversation_id_to_context(id) do + with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do + context + else _e -> + {:error, "No such conversation"} + end + end end diff --git a/test/web/twitter_api/representers/activity_representer_test.exs b/test/web/twitter_api/representers/activity_representer_test.exs index d0cccb149..64e7f0641 100644 --- a/test/web/twitter_api/representers/activity_representer_test.exs +++ b/test/web/twitter_api/representers/activity_representer_test.exs @@ -69,6 +69,8 @@ test "an activity" do content = HtmlSanitizeEx.strip_tags(content_html) date = DateTime.from_naive!(~N[2016-05-24 13:26:08.003], "Etc/UTC") |> DateTime.to_iso8601 + {:ok, convo_object} = Object.context_mapping("2hu") |> Repo.insert + activity = %Activity{ id: 1, data: %{ @@ -84,14 +86,15 @@ test "an activity" do "type" => "Note", "content" => content_html, "inReplyToStatusId" => 213123, - "statusnetConversationId" => 4711, "attachment" => [ object ], "like_count" => 5, - "announcement_count" => 3 + "announcement_count" => 3, + "context" => "2hu" }, - "published" => date + "published" => date, + "context" => "2hu" } } @@ -106,7 +109,7 @@ test "an activity" do "is_post_verb" => true, "created_at" => "Tue May 24 13:26:08 +0000 2016", "in_reply_to_status_id" => 213123, - "statusnet_conversation_id" => 4711, + "statusnet_conversation_id" => convo_object.id, "attachments" => [ ObjectRepresenter.to_map(object) ], diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index 6c249be7d..05cd084b4 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -84,12 +84,13 @@ test "returns one status", %{conn: conn} do describe "GET /statusnet/conversation/:id.json" do test "returns the statuses in the conversation", %{conn: conn} do {:ok, _user} = UserBuilder.insert - {:ok, _activity} = ActivityBuilder.insert(%{"statusnetConversationId" => 1, "context" => "2hu"}) - {:ok, _activity_two} = ActivityBuilder.insert(%{"statusnetConversationId" => 1,"context" => "2hu"}) + {:ok, _activity} = ActivityBuilder.insert(%{"context" => "2hu"}) + {:ok, _activity_two} = ActivityBuilder.insert(%{"context" => "2hu"}) {:ok, _activity_three} = ActivityBuilder.insert(%{"context" => "3hu"}) + {:ok, object} = Object.context_mapping("2hu") |> Repo.insert conn = conn - |> get("/api/statusnet/conversation/1.json") + |> get("/api/statusnet/conversation/#{object.id}.json") response = json_response(conn, 200) diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index 590428423..720011257 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -201,11 +201,13 @@ test "Unfollow another user using screen_name" do test "fetch statuses in a context using the conversation id" do {:ok, user} = UserBuilder.insert() - {:ok, activity} = ActivityBuilder.insert(%{"statusnetConversationId" => 1, "context" => "2hu"}) - {:ok, activity_two} = ActivityBuilder.insert(%{"statusnetConversationId" => 1,"context" => "2hu"}) + {:ok, activity} = ActivityBuilder.insert(%{"context" => "2hu"}) + {:ok, activity_two} = ActivityBuilder.insert(%{"context" => "2hu"}) {:ok, _activity_three} = ActivityBuilder.insert(%{"context" => "3hu"}) - statuses = TwitterAPI.fetch_conversation(user, 1) + {:ok, object} = Object.context_mapping("2hu") |> Repo.insert + + statuses = TwitterAPI.fetch_conversation(user, object.id) assert length(statuses) == 2 assert Enum.at(statuses, 0)["id"] == activity.id @@ -314,9 +316,33 @@ test "it returns the error on registration problems" do refute Repo.get_by(User, nickname: "lain") end + test "it assigns an integer conversation_id" do + note_activity = insert(:note_activity) + user = User.get_cached_by_ap_id(note_activity.data["actor"]) + status = ActivityRepresenter.to_map(note_activity, %{user: user}) + + assert is_number(status["statusnet_conversation_id"]) + end + setup do Supervisor.terminate_child(Pleroma.Supervisor, Cachex) Supervisor.restart_child(Pleroma.Supervisor, Cachex) :ok end + + describe "context_to_conversation_id" do + test "creates a mapping object" do + conversation_id = TwitterAPI.context_to_conversation_id("random context") + object = Object.get_by_ap_id("random context") + + assert conversation_id == object.id + end + + test "returns an existing mapping for an existing object" do + {:ok, object} = Object.context_mapping("random context") |> Repo.insert + conversation_id = TwitterAPI.context_to_conversation_id("random context") + + assert conversation_id == object.id + end + end end From 379caca01d818613ba7e013e8f0bebba160c6871 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 13:58:40 +0200 Subject: [PATCH 37/88] Wrap context creation in transaction. --- lib/pleroma/web/twitter_api/twitter_api.ex | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index b2fb72a81..13dc3bd49 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -319,13 +319,16 @@ defp make_date do end def context_to_conversation_id(context) do - with %Object{id: id} <- Object.get_by_ap_id(context) do - id - else _e -> - changeset = Object.context_mapping(context) - {:ok, %{id: id}} = Repo.insert(changeset) - id - end + {:ok, id} = Repo.transaction(fn -> + with %Object{id: id} <- Object.get_by_ap_id(context) do + id + else _e -> + changeset = Object.context_mapping(context) + {:ok, %{id: id}} = Repo.insert(changeset) + id + end + end) + id end def conversation_id_to_context(id) do From 009fcd2acfdc3ae3ba4b706eb71c50015227de50 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 14:02:04 +0200 Subject: [PATCH 38/88] Stop adding statusnetConversationIds. --- lib/pleroma/web/activity_pub/activity_pub.ex | 20 -------------------- test/web/twitter_api/twitter_api_test.exs | 4 +--- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 82f9fcc1c..9441a37ab 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -33,8 +33,6 @@ def create(to, actor, context, object, additional \\ %{}, published \\ nil) do |> Map.merge(additional) with {:ok, activity} <- insert(activity) do - {:ok, activity} = add_conversation_id(activity) - if actor.local do Pleroma.Web.Federator.enqueue(:publish, activity) end @@ -43,24 +41,6 @@ def create(to, actor, context, object, additional \\ %{}, published \\ nil) do end end - defp add_conversation_id(activity) do - if is_integer(activity.data["statusnetConversationId"]) do - {:ok, activity} - else - data = activity.data - |> put_in(["object", "statusnetConversationId"], activity.id) - |> put_in(["statusnetConversationId"], activity.id) - - object = Object.get_by_ap_id(activity.data["object"]["id"]) - - changeset = Ecto.Changeset.change(object, data: data["object"]) - Repo.update(changeset) - - changeset = Ecto.Changeset.change(activity, data: data) - Repo.update(changeset) - end - end - def like(%User{ap_id: ap_id} = user, %Object{data: %{ "id" => id}} = object) do cond do # There's already a like here, so return the original activity. diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index 720011257..eb061d334 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -41,11 +41,9 @@ test "create a status" do assert Enum.member?(get_in(activity.data, ["to"]), "https://www.w3.org/ns/activitystreams#Public") assert Enum.member?(get_in(activity.data, ["to"]), "shp") - # Add a context + 'statusnet_conversation_id' + # Add a context assert is_binary(get_in(activity.data, ["context"])) assert is_binary(get_in(activity.data, ["object", "context"])) - assert get_in(activity.data, ["object", "statusnetConversationId"]) == activity.id - assert get_in(activity.data, ["statusnetConversationId"]) == activity.id assert is_list(activity.data["object"]["attachment"]) From 09f7ed421497e12797f30ae34e0f2346f1b6a428 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 14:26:29 +0200 Subject: [PATCH 39/88] Don't set statusnetConversationIds on replies anymore. --- lib/pleroma/web/twitter_api/twitter_api.ex | 5 +---- test/web/twitter_api/twitter_api_test.exs | 2 -- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 13dc3bd49..85fac9146 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -58,11 +58,8 @@ def create_status(user = %User{}, data = %{"status" => status}) do "actor" => user.ap_id, "inReplyTo" => inReplyTo.data["object"]["id"], "inReplyToStatusId" => inReplyTo.id, - "statusnetConversationId" => inReplyTo.data["statusnetConversationId"] - } - additional = %{ - "statusnetConversationId" => inReplyTo.data["statusnetConversationId"] } + additional = %{} [to, context, object, additional] else diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index eb061d334..207d9d12a 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -67,8 +67,6 @@ test "create a status that is a reply" do assert get_in(reply.data, ["context"]) == get_in(activity.data, ["context"]) assert get_in(reply.data, ["object", "context"]) == get_in(activity.data, ["object", "context"]) - assert get_in(reply.data, ["statusnetConversationId"]) == get_in(activity.data, ["statusnetConversationId"]) - assert get_in(reply.data, ["object", "statusnetConversationId"]) == get_in(activity.data, ["object", "statusnetConversationId"]) assert get_in(reply.data, ["object", "inReplyTo"]) == get_in(activity.data, ["object", "id"]) assert get_in(reply.data, ["object", "inReplyToStatusId"]) == activity.id assert Enum.member?(get_in(reply.data, ["to"]), "some_cool_id") From bb1d08a47c34c70d42f6c3afa08232765a24884d Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 15:00:04 +0200 Subject: [PATCH 40/88] Return keys in webfinger. --- lib/pleroma/user.ex | 2 +- lib/pleroma/web/web_finger/web_finger.ex | 22 ++++++++++++++++++---- test/web/web_finger/web_finger_test.exs | 16 ++++++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index cd6104680..49ba9b22e 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -16,7 +16,7 @@ defmodule Pleroma.User do field :ap_id, :string field :avatar, :map field :local, :boolean, default: true - field :info, :map + field :info, :map, default: %{} timestamps() end diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 49796dab8..13e3baad6 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -1,8 +1,7 @@ defmodule Pleroma.Web.WebFinger do alias Pleroma.XmlBuilder - alias Pleroma.User - alias Pleroma.Web.OStatus - alias Pleroma.Web.XML + alias Pleroma.{Repo, User} + alias Pleroma.Web.{XML, Salmon, OStatus} require Logger def host_meta() do @@ -28,18 +27,33 @@ def webfinger(resource) do end def represent_user(user) do + {:ok, user} = ensure_keys_present(user) + {:ok, _private, public} = Salmon.keys_from_pem(user.info["keys"]) + magic_key = Salmon.encode_key(public) { :XRD, %{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"}, [ {:Subject, "acct:#{user.nickname}@#{Pleroma.Web.host}"}, {:Alias, user.ap_id}, {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}}, - {:Link, %{rel: "salmon", href: OStatus.salmon_path(user)}} + {:Link, %{rel: "salmon", href: OStatus.salmon_path(user)}}, + {:Link, %{rel: "magic-public-key", href: "data:application/magic-public-key,#{magic_key}"}} ] } |> XmlBuilder.to_doc end + def ensure_keys_present(user) do + info = user.info || %{} + if info["keys"] do + {:ok, user} + else + {:ok, pem} = Salmon.generate_rsa_pem + info = Map.put(info, "keys", pem) + Repo.update(Ecto.Changeset.change(user, info: info)) + end + end + # FIXME: Make this call the host-meta to find the actual address. defp webfinger_address(domain) do "//#{domain}/.well-known/webfinger" diff --git a/test/web/web_finger/web_finger_test.exs b/test/web/web_finger/web_finger_test.exs index e5347a2b0..303abe529 100644 --- a/test/web/web_finger/web_finger_test.exs +++ b/test/web/web_finger/web_finger_test.exs @@ -1,6 +1,7 @@ defmodule Pleroma.Web.WebFingerTest do use Pleroma.DataCase alias Pleroma.Web.WebFinger + import Pleroma.Factory describe "host meta" do test "returns a link to the xml lrdd" do @@ -26,4 +27,19 @@ test "returns the info for a user" do assert data.salmon == "https://social.heldscal.la/main/salmon/user/29191" end end + + describe "ensure_keys_present" do + test "it creates keys for a user and stores them in info" do + user = insert(:user) + refute is_binary(user.info["keys"]) + {:ok, user} = WebFinger.ensure_keys_present(user) + assert is_binary(user.info["keys"]) + end + + test "it doesn't create keys if there already are some" do + user = insert(:user, %{info: %{"keys" => "xxx"}}) + {:ok, user} = WebFinger.ensure_keys_present(user) + assert user.info["keys"] == "xxx" + end + end end From a173fb9e417cbb4fc7694672dd31bce90a3f9099 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 15:05:16 +0200 Subject: [PATCH 41/88] Get users fresh, might so we don't make new keys all the time. --- lib/pleroma/user.ex | 4 ++++ lib/pleroma/web/web_finger/web_finger.ex | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 49ba9b22e..2c297433a 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -123,6 +123,10 @@ def get_cached_by_nickname(nickname) do Cachex.get!(:user_cache, key, fallback: fn(_) -> Repo.get_by(User, nickname: nickname) end) end + def get_cached_by_nickname(nickname) do + Repo.get_by(User, nickname: nickname) + end + def get_cached_user_info(user) do key = "user_info:#{user.id}" Cachex.get!(:user_cache, key, fallback: fn(_) -> user_info(user) end) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 13e3baad6..7ceca042b 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -20,7 +20,7 @@ def webfinger(resource) do regex = ~r/(acct:)?(?\w+)@#{host}/ case Regex.named_captures(regex, resource) do %{"username" => username} -> - user = User.get_cached_by_nickname(username) + user = User.get_by_nickname(username) {:ok, represent_user(user)} _ -> nil end From eb12a89d22c09bccad7cb13780e0313de8be8e93 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 15:06:22 +0200 Subject: [PATCH 42/88] Rename wrongly-named function. --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 2c297433a..58f89a915 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -123,7 +123,7 @@ def get_cached_by_nickname(nickname) do Cachex.get!(:user_cache, key, fallback: fn(_) -> Repo.get_by(User, nickname: nickname) end) end - def get_cached_by_nickname(nickname) do + def get_by_nickname(nickname) do Repo.get_by(User, nickname: nickname) end From bed0b398139897ebe9f839d1263acf6934c4a42f Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 18:48:48 +0200 Subject: [PATCH 43/88] Add function to fetch users from fqn. --- lib/pleroma/user.ex | 12 ++++++++++++ test/user_test.exs | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 58f89a915..c264d7e90 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -3,6 +3,7 @@ defmodule Pleroma.User do import Ecto.Changeset import Ecto.Query alias Pleroma.{Repo, User, Activity, Object} + alias Pleroma.Web.OStatus schema "users" do field :bio, :string @@ -131,4 +132,15 @@ def get_cached_user_info(user) do key = "user_info:#{user.id}" Cachex.get!(:user_cache, key, fallback: fn(_) -> user_info(user) end) end + + def get_or_fetch_by_nickname(nickname) do + with %User{} = user <- get_by_nickname(nickname) do + user + else _e -> + with {:ok, user} <- OStatus.make_user(nickname) do + user + else _e -> nil + end + end + end end diff --git a/test/user_test.exs b/test/user_test.exs index d711adb9d..6684aa434 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -86,4 +86,24 @@ test "it sets the password_hash, ap_id and following fields" do assert changeset.changes[:following] == [User.ap_followers(%User{nickname: @full_user_data.nickname})] end end + + describe "fetching a user from nickname or trying to build one" do + test "gets an existing user" do + user = insert(:user) + fetched_user = User.get_or_fetch_by_nickname(user.nickname) + + assert user == fetched_user + end + + test "fetches an external user via ostatus if no user exists" do + fetched_user = User.get_or_fetch_by_nickname("shp@social.heldscal.la") + assert fetched_user.nickname == "shp@social.heldscal.la" + end + + test "returns nil if no user could be fetched" do + fetched_user = User.get_or_fetch_by_nickname("nonexistant@social.heldscal.la") + assert fetched_user == nil + end + end end + From 6843755834192c671aebece505a1ab9322e57eee Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 13:14:58 +0200 Subject: [PATCH 44/88] Make outgoing salmons work. --- TODO.txt | 8 +++-- lib/pleroma/user.ex | 5 +-- lib/pleroma/web/federator/federator.ex | 4 +++ .../web/ostatus/activity_representer.ex | 31 ++++++++++++++-- lib/pleroma/web/salmon/salmon.ex | 36 +++++++++++++++++++ test/user_test.exs | 6 ++++ .../web/ostatus/activity_representer_test.exs | 2 ++ test/web/salmon/salmon_test.exs | 32 +++++++++++++++++ 8 files changed, 117 insertions(+), 7 deletions(-) diff --git a/TODO.txt b/TODO.txt index dd85c5239..304e95e77 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,5 +1,9 @@ -- Add cache for user fetching / representing. (mostly in TwitterAPI.activity_to_status) - Unliking: - Add a proper undo activity, find out how to ignore those in twitter api. + +WEBSUB: + +- Add unsubscription +- Add periodical renewal + diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index c264d7e90..01cbfe796 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -121,7 +121,7 @@ def get_cached_by_ap_id(ap_id) do def get_cached_by_nickname(nickname) do key = "nickname:#{nickname}" - Cachex.get!(:user_cache, key, fallback: fn(_) -> Repo.get_by(User, nickname: nickname) end) + Cachex.get!(:user_cache, key, fallback: fn(_) -> get_or_fetch_by_nickname(nickname) end) end def get_by_nickname(nickname) do @@ -137,7 +137,8 @@ def get_or_fetch_by_nickname(nickname) do with %User{} = user <- get_by_nickname(nickname) do user else _e -> - with {:ok, user} <- OStatus.make_user(nickname) do + with [nick, domain] <- String.split(nickname, "@"), + {:ok, user} <- OStatus.make_user(nickname) do user else _e -> nil end diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 38df13540..5293507b5 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -7,7 +7,11 @@ defmodule Pleroma.Web.Federator do def handle(:publish, activity) do Logger.debug("Running publish for #{activity.data["id"]}") with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do + Logger.debug("Sending #{activity.data["id"]} out via websub") Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity) + + Logger.debug("Sending #{activity.data["id"]} out via salmon") + Pleroma.Web.Salmon.publish(actor, activity) end end diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 274111ac9..c64bb3a3b 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -1,5 +1,6 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do alias Pleroma.Activity + alias Pleroma.Web.OStatus.UserRepresenter require Logger defp get_in_reply_to(%{"object" => %{ "inReplyTo" => in_reply_to}}) do @@ -8,7 +9,17 @@ defp get_in_reply_to(%{"object" => %{ "inReplyTo" => in_reply_to}}) do defp get_in_reply_to(_), do: [] - def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) do + defp get_mentions(to) do + Enum.map(to, fn + ("https://www.w3.org/ns/activitystreams#Public") -> + {:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/collection", href: "http://activityschema.org/collection/public"], []} + (id) -> + {:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/person", href: id], []} + end) + end + + def to_simple_form(activity, user, with_author \\ false) + def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do h = fn(str) -> [to_charlist(str)] end updated_at = activity.updated_at @@ -22,6 +33,8 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) end) in_reply_to = get_in_reply_to(activity.data) + author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] + mentions = activity.data["to"] |> get_mentions [ {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']}, @@ -33,8 +46,20 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) {:updated, h.(updated_at)}, {:"ostatus:conversation", [], h.(activity.data["context"])}, {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []} - ] ++ attachments ++ in_reply_to + ] ++ attachments ++ in_reply_to ++ author ++ mentions end - def to_simple_form(_,_), do: nil + def wrap_with_entry(simple_form) do + [{ + :entry, [ + xmlns: 'http://www.w3.org/2005/Atom', + "xmlns:thr": 'http://purl.org/syndication/thread/1.0', + "xmlns:activity": 'http://activitystrea.ms/spec/1.0/', + "xmlns:poco": 'http://portablecontacts.net/spec/1.0', + "xmlns:ostatus": 'http://ostatus.org/schema/1.0' + ], simple_form + }] + end + + def to_simple_form(_,_,_), do: nil end diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 777898cfa..b4f214d46 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -1,6 +1,9 @@ defmodule Pleroma.Web.Salmon do use Bitwise alias Pleroma.Web.XML + alias Pleroma.Web.OStatus.ActivityRepresenter + alias Pleroma.User + require Logger def decode(salmon) do doc = XML.parse_document(salmon) @@ -118,4 +121,37 @@ def encode(private_key, doc) do {:ok, salmon} end + + def remote_users(%{data: %{"to" => to}}) do + to + |> Enum.map(fn(id) -> User.get_cached_by_ap_id(id) end) + |> Enum.filter(fn(user) -> user && !user.local end) + end + + defp send_to_user(%{info: %{"salmon" => salmon}}, feed, poster) do + poster.(salmon, feed, [{"Content-Type", "application/magic-envelope+xml"}]) + end + + defp send_to_user(_,_,_), do: nil + + def publish(user, activity, poster \\ &HTTPoison.post/3) + def publish(%{info: %{"keys" => keys}} = user, activity, poster) do + feed = ActivityRepresenter.to_simple_form(activity, user, true) + |> ActivityRepresenter.wrap_with_entry + |> :xmerl.export_simple(:xmerl_xml) + |> to_string + + if feed do + {:ok, private, _} = keys_from_pem(keys) + {:ok, feed} = encode(private, feed) + + remote_users(activity) + |> Enum.each(fn(remote_user) -> + Logger.debug("sending salmon to #{remote_user.ap_id}") + send_to_user(remote_user, feed, poster) + end) + end + end + + def publish(%{id: id}, _, _), do: Logger.debug("Keys missing for user #{id}") end diff --git a/test/user_test.exs b/test/user_test.exs index 6684aa434..1331ac971 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -95,6 +95,7 @@ test "gets an existing user" do assert user == fetched_user end + # TODO: Make the test local. test "fetches an external user via ostatus if no user exists" do fetched_user = User.get_or_fetch_by_nickname("shp@social.heldscal.la") assert fetched_user.nickname == "shp@social.heldscal.la" @@ -104,6 +105,11 @@ test "returns nil if no user could be fetched" do fetched_user = User.get_or_fetch_by_nickname("nonexistant@social.heldscal.la") assert fetched_user == nil end + + test "returns nil for nonexistant local user" do + fetched_user = User.get_or_fetch_by_nickname("nonexistant") + assert fetched_user == nil + end end end diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 6344889b1..439c733d7 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -25,6 +25,7 @@ test "a note activity" do #{updated_at} #{note_activity.data["context"]} + """ tuple = ActivityRepresenter.to_simple_form(note_activity, user) @@ -61,6 +62,7 @@ test "a reply note" do #{answer.data["context"]} + """ tuple = ActivityRepresenter.to_simple_form(answer, user) diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs index aa77659d0..77dacc1c0 100644 --- a/test/web/salmon/salmon_test.exs +++ b/test/web/salmon/salmon_test.exs @@ -1,6 +1,8 @@ defmodule Pleroma.Web.Salmon.SalmonTest do use Pleroma.DataCase alias Pleroma.Web.Salmon + alias Pleroma.{Repo, Activity, User} + import Pleroma.Factory @magickey "RSA.pu0s-halox4tu7wmES1FVSx6u-4wc0YrUFXcqWXZG4-27UmbCOpMQftRCldNRfyA-qLbz-eqiwQhh-1EwUvjsD4cYbAHNGHwTvDOyx5AKthQUP44ykPv7kjKGh3DWKySJvcs9tlUG87hlo7AvnMo9pwRS_Zz2CacQ-MKaXyDepk=.AQAB" @@ -57,4 +59,34 @@ test "it gets a magic key" do assert key == "RSA.uzg6r1peZU0vXGADWxGJ0PE34WvmhjUmydbX5YYdOiXfODVLwCMi1umGoqUDm-mRu4vNEdFBVJU1CpFA7dKzWgIsqsa501i2XqElmEveXRLvNRWFB6nG03Q5OUY2as8eE54BJm0p20GkMfIJGwP6TSFb-ICp3QjzbatuSPJ6xCE=.AQAB" end + + test "it pushes an activity to remote accounts it's addressed to" do + user_data = %{ + info: %{ + "salmon" => "http://example.org/salmon" + }, + local: false + } + + mentioned_user = insert(:user, user_data) + note = insert(:note) + activity_data = %{ + "id" => Pleroma.Web.ActivityPub.ActivityPub.generate_activity_id, + "type" => "Create", + "actor" => note.data["actor"], + "to" => note.data["to"] ++ [mentioned_user.ap_id], + "object" => note.data, + "published_at" => DateTime.utc_now() |> DateTime.to_iso8601, + "context" => note.data["context"] + } + + {:ok, activity} = Repo.insert(%Activity{data: activity_data}) + user = Repo.get_by(User, ap_id: activity.data["actor"]) + {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + + poster = fn (url, data, headers) -> + assert url == "http://example.org/salmon" + end + Salmon.publish(user, activity, poster) + end end From e54e592d6c1d0ff5de5a029ae1fccee447f97149 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 13:51:17 +0200 Subject: [PATCH 45/88] Return webfinger for ap_ids. --- lib/pleroma/web/web_finger/web_finger.ex | 12 ++++++++---- test/web/web_finger/web_finger_test.exs | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 7ceca042b..f8f4d5e42 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -18,11 +18,15 @@ def host_meta() do def webfinger(resource) do host = Pleroma.Web.host regex = ~r/(acct:)?(?\w+)@#{host}/ - case Regex.named_captures(regex, resource) do - %{"username" => username} -> - user = User.get_by_nickname(username) + with %{"username" => username} <- Regex.named_captures(regex, resource) do + user = User.get_by_nickname(username) + {:ok, represent_user(user)} + else _e -> + with user when not is_nil(user) <- User.get_cached_by_ap_id(resource) do {:ok, represent_user(user)} - _ -> nil + else _e -> + {:error, "Couldn't find user"} + end end end diff --git a/test/web/web_finger/web_finger_test.exs b/test/web/web_finger/web_finger_test.exs index 303abe529..b48fdd0aa 100644 --- a/test/web/web_finger/web_finger_test.exs +++ b/test/web/web_finger/web_finger_test.exs @@ -11,6 +11,22 @@ test "returns a link to the xml lrdd" do end end + describe "incoming webfinger request" do + test "works for fqns" do + user = insert(:user) + + {:ok, result} = WebFinger.webfinger("#{user.nickname}@#{Pleroma.Web.host}") + assert is_binary(result) + end + + test "works for ap_ids" do + user = insert(:user) + + {:ok, result} = WebFinger.webfinger(user.ap_id) + assert is_binary(result) + end + end + describe "fingering" do test "returns the info for a user" do user = "shp@social.heldscal.la" From 35938656ab4186912ee6593cc09754ef945e17fc Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 14:07:29 +0200 Subject: [PATCH 46/88] Make user keys on usage. --- lib/pleroma/web/federator/federator.ex | 2 ++ lib/pleroma/web/web_finger/web_finger.ex | 1 + 2 files changed, 3 insertions(+) diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 5293507b5..675e804a2 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -1,5 +1,6 @@ defmodule Pleroma.Web.Federator do alias Pleroma.User + alias Pleroma.Web.WebFinger require Logger @websub Application.get_env(:pleroma, :websub) @@ -10,6 +11,7 @@ def handle(:publish, activity) do Logger.debug("Sending #{activity.data["id"]} out via websub") Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity) + {:ok, actor} = WebFinger.ensure_keys_present(actor) Logger.debug("Sending #{activity.data["id"]} out via salmon") Pleroma.Web.Salmon.publish(actor, activity) end diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index f8f4d5e42..ff10173ef 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -47,6 +47,7 @@ def represent_user(user) do |> XmlBuilder.to_doc end + # This seems a better fit in Salmon def ensure_keys_present(user) do info = user.info || %{} if info["keys"] do From d187a4965fbba93149621478e5257fcca2cea4f9 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 14:07:41 +0200 Subject: [PATCH 47/88] Return feed for xml requests of the user. --- lib/pleroma/web/router.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 2ff75ec5d..e875839df 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -74,6 +74,7 @@ def user_fetcher(username) do pipe_through :ostatus get "/users/:nickname/feed", OStatus.OStatusController, :feed + get "/users/:nickname", OStatus.OStatusController, :feed post "/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming post "/push/hub/:nickname", Websub.WebsubController, :websub_subscription_request get "/push/subscriptions/:id", Websub.WebsubController, :websub_subscription_confirmation From 2f093db051efb2252342e3490eea3a8ae67e06d3 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 14:54:58 +0200 Subject: [PATCH 48/88] Ensure we have no duplicate ap ids. --- ...124823_add_id_contraints_to_activities_and_objects.exs | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 priv/repo/migrations/20170501124823_add_id_contraints_to_activities_and_objects.exs diff --git a/priv/repo/migrations/20170501124823_add_id_contraints_to_activities_and_objects.exs b/priv/repo/migrations/20170501124823_add_id_contraints_to_activities_and_objects.exs new file mode 100644 index 000000000..21534adc7 --- /dev/null +++ b/priv/repo/migrations/20170501124823_add_id_contraints_to_activities_and_objects.exs @@ -0,0 +1,8 @@ +defmodule Pleroma.Repo.Migrations.AddIdContraintsToActivitiesAndObjects do + use Ecto.Migration + + def change do + create index(:objects, ["(data->>\"id\")"], name: :objects_unique_apid_index) + create index(:activities, ["(data->>\"id\")"], name: :activities_unique_apid_index) + end +end From b9d1fc05b22e29b15208cd6fdcb5d40d34d2a83e Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 15:42:05 +0200 Subject: [PATCH 49/88] Actually make index unique. --- ...d_contraints_to_activities_and_objects_part_two.exs | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs diff --git a/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs b/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs new file mode 100644 index 000000000..12eea1369 --- /dev/null +++ b/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.AddIdContraintsToActivitiesAndObjectsPartTwo do + use Ecto.Migration + + def change do + drop index(:objects, ["(data->>\"id\")"], name: :objects_unique_apid_index) + drop index(:activities, ["(data->>\"id\")"], name: :activities_unique_apid_index) + create unique_index(:objects, ["(data->>'id')"], name: :objects_unique_apid_index) + create unique_index(:activities, ["(data->>'id')"], name: :activities_unique_apid_index) + end +end From f169de34544a12c174c454da59781a694b8c2387 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 16:12:20 +0200 Subject: [PATCH 50/88] Cache objects in dev and prod. --- lib/pleroma/object.ex | 9 +++++++++ lib/pleroma/web/twitter_api/twitter_api.ex | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index a924c3199..168843bd9 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -14,6 +14,15 @@ def get_by_ap_id(ap_id) do where: fragment("? @> ?", object.data, ^%{id: ap_id})) end + def get_cached_by_ap_id(ap_id) do + if Mix.env == :test do + get_by_ap_id(ap_id) + else + key = "object:#{ap_id}" + Cachex.get!(:user_cache, key, fallback: fn(_) -> get_by_ap_id(ap_id) end) + end + end + def context_mapping(context) do %Object{data: %{"id" => context}} end diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 85fac9146..941bacaa9 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -317,7 +317,7 @@ defp make_date do def context_to_conversation_id(context) do {:ok, id} = Repo.transaction(fn -> - with %Object{id: id} <- Object.get_by_ap_id(context) do + with %Object{id: id} <- Object.get_cached_by_ap_id(context) do id else _e -> changeset = Object.context_mapping(context) From 3cb518270ab8c41f73ed449f0c12127c3625c6ca Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 16:15:21 +0200 Subject: [PATCH 51/88] Remove superfluous transaction. --- lib/pleroma/web/twitter_api/twitter_api.ex | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 941bacaa9..e6f5fc906 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -316,16 +316,13 @@ defp make_date do end def context_to_conversation_id(context) do - {:ok, id} = Repo.transaction(fn -> - with %Object{id: id} <- Object.get_cached_by_ap_id(context) do - id - else _e -> - changeset = Object.context_mapping(context) - {:ok, %{id: id}} = Repo.insert(changeset) - id - end - end) - id + with %Object{id: id} <- Object.get_cached_by_ap_id(context) do + id + else _e -> + changeset = Object.context_mapping(context) + {:ok, %{id: id}} = Repo.insert(changeset) + id + end end def conversation_id_to_context(id) do From 108573265aaf237a37937544c3f416b85f57e0fb Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 16:28:40 +0200 Subject: [PATCH 52/88] Don't commit nil values in object cache. --- lib/pleroma/object.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 168843bd9..949ccb0f6 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -19,7 +19,14 @@ def get_cached_by_ap_id(ap_id) do get_by_ap_id(ap_id) else key = "object:#{ap_id}" - Cachex.get!(:user_cache, key, fallback: fn(_) -> get_by_ap_id(ap_id) end) + Cachex.get!(:user_cache, key, fallback: fn(_) -> + object = get_by_ap_id(ap_id) + if object do + {:commit, object} + else + {:ignore, object} + end + end) end end From 1854842b093524c25e08ece0b33150172036f53c Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 17:28:49 +0200 Subject: [PATCH 53/88] Log subscription error. --- lib/pleroma/web/websub/websub.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 63a91055a..67055a116 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -78,6 +78,9 @@ def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) d {:ok, websub} else {:error, reason} -> + Logger.debug("Couldn't create subscription.") + Logger.debug(inspect(reason)) + {:error, reason} end end From 92a8944dfe043444af6b4b422789129c04bd34a0 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 18:05:02 +0200 Subject: [PATCH 54/88] Redirect to user feed instead of directly serving it. --- lib/pleroma/web/ostatus/ostatus_controller.ex | 6 ++++++ lib/pleroma/web/router.ex | 2 +- lib/pleroma/web/websub/websub.ex | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 4174db786..1c609f6f2 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -4,8 +4,14 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Pleroma.{User, Activity} alias Pleroma.Web.OStatus.FeedRepresenter alias Pleroma.Repo + alias Pleroma.Web.OStatus import Ecto.Query + def feed_redirect(conn, %{"nickname" => nickname}) do + user = User.get_cached_by_nickname(nickname) + redirect conn, external: OStatus.feed_path(user) + end + def feed(conn, %{"nickname" => nickname}) do user = User.get_cached_by_nickname(nickname) query = from activity in Activity, diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e875839df..e1475a03e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -74,7 +74,7 @@ def user_fetcher(username) do pipe_through :ostatus get "/users/:nickname/feed", OStatus.OStatusController, :feed - get "/users/:nickname", OStatus.OStatusController, :feed + get "/users/:nickname", OStatus.OStatusController, :feed_redirect post "/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming post "/push/hub/:nickname", Websub.WebsubController, :websub_subscription_request get "/push/subscriptions/:id", Websub.WebsubController, :websub_subscription_confirmation diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 67055a116..b279a5060 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -99,7 +99,7 @@ defp lease_time(_) do defp valid_topic(%{"hub.topic" => topic}, user) do if topic == OStatus.feed_path(user) do - {:ok, topic} + {:ok, OStatus.feed_path(user)} else {:error, "Wrong topic requested, expected #{OStatus.feed_path(user)}, got #{topic}"} end From 97d11dec0ee78d24fea398f23d123baf0111362a Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 18:07:50 +0200 Subject: [PATCH 55/88] Also accept user id as feed topic. --- lib/pleroma/web/websub/websub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index b279a5060..fc253b930 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -98,7 +98,7 @@ defp lease_time(_) do end defp valid_topic(%{"hub.topic" => topic}, user) do - if topic == OStatus.feed_path(user) do + if topic == OStatus.feed_path(user) || topic == Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :feed_redirect, user.nickname) do {:ok, OStatus.feed_path(user)} else {:error, "Wrong topic requested, expected #{OStatus.feed_path(user)}, got #{topic}"} From e88062494e04c257b1dea33965764a51e04cbdf7 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 18:34:15 +0200 Subject: [PATCH 56/88] Revert "Also accept user id as feed topic." This reverts commit 97d11dec0ee78d24fea398f23d123baf0111362a. --- lib/pleroma/web/websub/websub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index fc253b930..b279a5060 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -98,7 +98,7 @@ defp lease_time(_) do end defp valid_topic(%{"hub.topic" => topic}, user) do - if topic == OStatus.feed_path(user) || topic == Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :feed_redirect, user.nickname) do + if topic == OStatus.feed_path(user) do {:ok, OStatus.feed_path(user)} else {:error, "Wrong topic requested, expected #{OStatus.feed_path(user)}, got #{topic}"} From ceb2f68432e2861f09f7ba34b98bef259be9158a Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 18:40:36 +0200 Subject: [PATCH 57/88] Add type to rel=self link in feed. --- lib/pleroma/web/ostatus/feed_representer.ex | 2 +- test/web/ostatus/feed_representer_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/feed_representer.ex b/lib/pleroma/web/ostatus/feed_representer.ex index db7b685f3..7f9d6a46b 100644 --- a/lib/pleroma/web/ostatus/feed_representer.ex +++ b/lib/pleroma/web/ostatus/feed_representer.ex @@ -26,7 +26,7 @@ def to_simple_form(user, activities, users) do {:updated, h.(most_recent_update)}, {:link, [rel: 'hub', href: h.(OStatus.pubsub_path(user))], []}, {:link, [rel: 'salmon', href: h.(OStatus.salmon_path(user))], []}, - {:link, [rel: 'self', href: h.(OStatus.feed_path(user))], []}, + {:link, [rel: 'self', href: h.(OStatus.feed_path(user)), type: 'application/atom+xml'], []}, {:author, UserRepresenter.to_simple_form(user)}, ] ++ entries }] diff --git a/test/web/ostatus/feed_representer_test.exs b/test/web/ostatus/feed_representer_test.exs index 7bbfae49a..df5a964e2 100644 --- a/test/web/ostatus/feed_representer_test.exs +++ b/test/web/ostatus/feed_representer_test.exs @@ -28,7 +28,7 @@ test "returns a feed of the last 20 items of the user" do #{most_recent_update} - + #{user_xml} From 76e653b0d80279491a4b57278aec7a83efa003d0 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 20:02:32 +0200 Subject: [PATCH 58/88] Add user profile page link. --- lib/pleroma/web/web_finger/web_finger.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index ff10173ef..217b09dc5 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -40,6 +40,7 @@ def represent_user(user) do {:Subject, "acct:#{user.nickname}@#{Pleroma.Web.host}"}, {:Alias, user.ap_id}, {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}}, + {:Link, %{rel: "ttp://webfinger.net/rel/profile-page", type: "text/html", href: user.ap_id}}, {:Link, %{rel: "salmon", href: OStatus.salmon_path(user)}}, {:Link, %{rel: "magic-public-key", href: "data:application/magic-public-key,#{magic_key}"}} ] From 703d9f36281e90ef049bfe0a0d579e4e07b38bb6 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 20:04:32 +0200 Subject: [PATCH 59/88] Not enough h. --- lib/pleroma/web/web_finger/web_finger.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 217b09dc5..402184d3f 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -40,7 +40,7 @@ def represent_user(user) do {:Subject, "acct:#{user.nickname}@#{Pleroma.Web.host}"}, {:Alias, user.ap_id}, {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}}, - {:Link, %{rel: "ttp://webfinger.net/rel/profile-page", type: "text/html", href: user.ap_id}}, + {:Link, %{rel: "http://webfinger.net/rel/profile-page", type: "text/html", href: user.ap_id}}, {:Link, %{rel: "salmon", href: OStatus.salmon_path(user)}}, {:Link, %{rel: "magic-public-key", href: "data:application/magic-public-key,#{magic_key}"}} ] From aa209414164cf098376d8aefb3f2af16111bd220 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 20:09:00 +0200 Subject: [PATCH 60/88] Some servers send empty lease_seconds requests... --- lib/pleroma/web/websub/websub.ex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index b279a5060..905c237a0 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -89,6 +89,11 @@ defp get_subscription(topic, callback) do Repo.get_by(WebsubServerSubscription, topic: topic, callback: callback) || %WebsubServerSubscription{} end + # Temp hack for mastodon. + defp lease_time(%{"hub.lease_seconds" => ""}) do + {:ok, 60 * 60 * 24 * 3} # three days + end + defp lease_time(%{"hub.lease_seconds" => lease_seconds}) do {:ok, String.to_integer(lease_seconds)} end From 8ae13d94dc69e4fcb7f454c2eb7665955c8e37fb Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 20:38:01 +0200 Subject: [PATCH 61/88] Use empty context id if we get none Thanks mastodon. --- lib/pleroma/web/ostatus/ostatus.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 01d6745ef..6a6f43acf 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -45,7 +45,7 @@ def handle_note(entry, doc \\ nil) do uri = string_from_xpath("/entry/author/uri[1]", entry) || string_from_xpath("/feed/author/uri[1]", doc) {:ok, actor} = find_or_make_user(uri) - context = string_from_xpath("/entry/ostatus:conversation[1]", entry) |> String.trim + context = (string_from_xpath("/entry/ostatus:conversation[1]", entry) || "") |> String.trim context = if String.length(context) > 0 do context else From 89c1e90eb2a5da0a6f635a6158fe880076518a38 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 22:02:07 +0200 Subject: [PATCH 62/88] Don't crypt raw iolists. --- lib/pleroma/web/websub/websub.ex | 3 ++- test/web/websub/websub_test.exs | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 905c237a0..546bfb5a4 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -41,6 +41,7 @@ def publish(topic, user, activity) do Enum.each(subscriptions, fn(sub) -> response = FeedRepresenter.to_simple_form(user, [activity], [user]) |> :xmerl.export_simple(:xmerl_xml) + |> to_string signature = sign(sub.secret, response) HTTPoison.post(sub.callback, response, [ @@ -51,7 +52,7 @@ def publish(topic, user, activity) do end def sign(secret, doc) do - :crypto.hmac(:sha, secret, doc) |> Base.encode16 + :crypto.hmac(:sha, secret, to_string(doc)) |> Base.encode16 end def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) do diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index ad312cd25..63acb3c43 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -167,4 +167,11 @@ test "rejects the subscription if it can't be accepted" do {:error, websub} = Websub.request_subscription(websub, poster, 1000) assert websub.state == "rejected" end + + test "sign a text" do + signed = Websub.sign("secret", "text") + assert signed == "B8392C23690CCF871F37EC270BE1582DEC57A503" + + signed = Websub.sign("secret", [["て"], ['す']]) + end end From 56bacc90d1f401f8867e4ca7a052f7d15e18a304 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 10:43:35 +0200 Subject: [PATCH 63/88] Fix specs, add local marker to actitivies. --- lib/pleroma/activity.ex | 1 + .../20170502083023_add_local_field_to_activities.exs | 11 +++++++++++ test/support/builders/activity_builder.ex | 4 ++-- test/web/twitter_api/twitter_api_test.exs | 1 + 4 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 priv/repo/migrations/20170502083023_add_local_field_to_activities.exs diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 80d96d0f2..d77c88997 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Activity do schema "activities" do field :data, :map + field :local, :boolean, default: true timestamps() end diff --git a/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs b/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs new file mode 100644 index 000000000..088d68f67 --- /dev/null +++ b/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs @@ -0,0 +1,11 @@ +defmodule Pleroma.Repo.Migrations.AddLocalFieldToActivities do + use Ecto.Migration + + def change do + alter table(:activities) do + add :local, :boolean, default: true + end + + create index(:activities, [:local]) + end +end diff --git a/test/support/builders/activity_builder.ex b/test/support/builders/activity_builder.ex index 0f9cd0d15..16011edbf 100644 --- a/test/support/builders/activity_builder.ex +++ b/test/support/builders/activity_builder.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Builders.ActivityBuilder do def build(data \\ %{}, opts \\ %{}) do user = opts[:user] || Pleroma.Factory.insert(:user) activity = %{ - "id" => 1, + "id" => Pleroma.Web.ActivityPub.ActivityPub.generate_object_id, "actor" => user.ap_id, "to" => ["https://www.w3.org/ns/activitystreams#Public"], "object" => %{ @@ -23,7 +23,7 @@ def insert(data \\ %{}, opts \\ %{}) do def insert_list(times, data \\ %{}, opts \\ %{}) do Enum.map(1..times, fn (n) -> - {:ok, activity} = insert(Map.merge(data, %{"id" => n})) + {:ok, activity} = insert(data) activity end) end diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index 207d9d12a..57dcddd4c 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -40,6 +40,7 @@ test "create a status" do assert Enum.member?(get_in(activity.data, ["to"]), User.ap_followers(user)) assert Enum.member?(get_in(activity.data, ["to"]), "https://www.w3.org/ns/activitystreams#Public") assert Enum.member?(get_in(activity.data, ["to"]), "shp") + assert activity.local == true # Add a context assert is_binary(get_in(activity.data, ["context"])) From 6dd8335477ff3adc2dda5fe4e45b0e1b38dc5b9b Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 10:47:04 +0200 Subject: [PATCH 64/88] Mark incoming activties as non-local. --- lib/pleroma/web/activity_pub/activity_pub.ex | 8 ++++---- lib/pleroma/web/ostatus/ostatus.ex | 2 +- test/web/ostatus/ostatus_test.exs | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 9441a37ab..4eab2e2d0 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -3,7 +3,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.{Activity, Object, Upload, User} import Ecto.Query - def insert(map) when is_map(map) do + def insert(map, local \\ true) when is_map(map) do map = map |> Map.put_new_lazy("id", &generate_activity_id/0) |> Map.put_new_lazy("published", &make_date/0) @@ -16,10 +16,10 @@ def insert(map) when is_map(map) do map end - Repo.insert(%Activity{data: map}) + Repo.insert(%Activity{data: map, local: local}) end - def create(to, actor, context, object, additional \\ %{}, published \\ nil) do + def create(to, actor, context, object, additional \\ %{}, published \\ nil, local \\ true) do published = published || make_date() activity = %{ @@ -32,7 +32,7 @@ def create(to, actor, context, object, additional \\ %{}, published \\ nil) do } |> Map.merge(additional) - with {:ok, activity} <- insert(activity) do + with {:ok, activity} <- insert(activity, local) do if actor.local do Pleroma.Web.Federator.enqueue(:publish, activity) end diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 6a6f43acf..db32d2c35 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -86,7 +86,7 @@ def handle_note(entry, doc \\ nil) do if Object.get_by_ap_id(id) do {:error, "duplicate activity"} else - ActivityPub.create(to, actor, context, object, %{}, date) + ActivityPub.create(to, actor, context, object, %{}, date, false) end end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 4e7e401cd..3951dbc9c 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -18,6 +18,7 @@ test "handle incoming note - GS, Salmon" do assert activity.data["published"] == "2017-04-23T14:51:03+00:00" assert activity.data["context"] == "tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b" assert "http://pleroma.example.org:4000/users/lain3" in activity.data["to"] + assert activity.local == false end test "handle incoming notes - GS, subscription" do From 32a95d73daf94a1186ccdbcdc9ce0f91b559119c Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 14:12:43 +0200 Subject: [PATCH 65/88] Add twkn timeline. --- lib/pleroma/web/activity_pub/activity_pub.ex | 6 ++++++ lib/pleroma/web/router.ex | 2 +- lib/pleroma/web/twitter_api/twitter_api.ex | 6 ++++++ .../web/twitter_api/twitter_api_controller.ex | 8 ++++++++ test/web/twitter_api/twitter_api_test.exs | 15 ++++++++++++++- 5 files changed, 35 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 4eab2e2d0..0fb8db520 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -149,6 +149,12 @@ def fetch_activities(recipients, opts \\ %{}) do query = from activity in query, where: activity.id > ^since_id + query = if opts["local_only"] do + from activity in query, where: activity.local == true + else + query + end + query = if opts["max_id"] do from activity in query, where: activity.id < ^opts["max_id"] else diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e1475a03e..b0c1dcd91 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -30,7 +30,7 @@ def user_fetcher(username) do get "/statusnet/config", TwitterAPI.Controller, :config get "/statuses/public_timeline", TwitterAPI.Controller, :public_timeline - get "/statuses/public_and_external_timeline", TwitterAPI.Controller, :public_timeline + get "/statuses/public_and_external_timeline", TwitterAPI.Controller, :public_and_external_timeline get "/statuses/user_timeline", TwitterAPI.Controller, :user_timeline get "/statuses/show/:id", TwitterAPI.Controller, :fetch_status diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index e6f5fc906..b1759a6f0 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -84,6 +84,12 @@ def fetch_friend_statuses(user, opts \\ %{}) do end def fetch_public_statuses(user, opts \\ %{}) do + opts = Map.put(opts, "local_only", true) + ActivityPub.fetch_public_activities(opts) + |> activities_to_statuses(%{for: user}) + end + + def fetch_public_and_external_statuses(user, opts \\ %{}) do ActivityPub.fetch_public_activities(opts) |> activities_to_statuses(%{for: user}) end diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index b5b829ca0..4b329a21f 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -41,6 +41,14 @@ defp extract_media_ids(status_data) do end end + def public_and_external_timeline(%{assigns: %{user: user}} = conn, params) do + statuses = TwitterAPI.fetch_public_and_external_statuses(user, params) + {:ok, json} = Poison.encode(statuses) + + conn + |> json_reply(200, json) + end + def public_timeline(%{assigns: %{user: user}} = conn, params) do statuses = TwitterAPI.fetch_public_statuses(user, params) {:ok, json} = Poison.encode(statuses) diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index 57dcddd4c..4e17f3298 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -73,8 +73,9 @@ test "create a status that is a reply" do assert Enum.member?(get_in(reply.data, ["to"]), "some_cool_id") end - test "fetch public statuses" do + test "fetch public statuses, excluding remote ones." do %{ public: activity, user: user } = ActivityBuilder.public_and_non_public + insert(:note_activity, %{local: false}) follower = insert(:user, following: [User.ap_followers(user)]) @@ -84,6 +85,18 @@ test "fetch public statuses" do assert Enum.at(statuses, 0) == ActivityRepresenter.to_map(activity, %{user: user, for: follower}) end + test "fetch whole known network statuses" do + %{ public: activity, user: user } = ActivityBuilder.public_and_non_public + insert(:note_activity, %{local: false}) + + follower = insert(:user, following: [User.ap_followers(user)]) + + statuses = TwitterAPI.fetch_public_and_external_statuses(follower) + + assert length(statuses) == 2 + assert Enum.at(statuses, 0) == ActivityRepresenter.to_map(activity, %{user: user, for: follower}) + end + test "fetch friends' statuses" do user = insert(:user, %{following: ["someguy/followers"]}) {:ok, activity} = ActivityBuilder.insert(%{"to" => ["someguy/followers"]}) From 16f8406eb60562b961536ecfabecde8e15160aa6 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 14:36:04 +0200 Subject: [PATCH 66/88] Add statusnet_profile_url to the TwAPI. --- .../web/twitter_api/representers/user_representer.ex | 3 ++- test/web/twitter_api/representers/user_representer_test.exs | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/twitter_api/representers/user_representer.ex b/lib/pleroma/web/twitter_api/representers/user_representer.ex index 29c7451f4..493077413 100644 --- a/lib/pleroma/web/twitter_api/representers/user_representer.ex +++ b/lib/pleroma/web/twitter_api/representers/user_representer.ex @@ -28,7 +28,8 @@ def to_map(user, opts) do "profile_image_url_https" => image, "profile_image_url_profile_size" => image, "profile_image_url_original" => image, - "rights" => %{} + "rights" => %{}, + "statusnet_profile_url" => user.ap_id } map diff --git a/test/web/twitter_api/representers/user_representer_test.exs b/test/web/twitter_api/representers/user_representer_test.exs index 1e92c5190..77f065948 100644 --- a/test/web/twitter_api/representers/user_representer_test.exs +++ b/test/web/twitter_api/representers/user_representer_test.exs @@ -48,7 +48,8 @@ test "A user" do "profile_image_url_profile_size" => image, "profile_image_url_original" => image, "following" => false, - "rights" => %{} + "rights" => %{}, + "statusnet_profile_url" => user.ap_id } assert represented == UserRepresenter.to_map(user) @@ -72,7 +73,8 @@ test "A user for a given other follower", %{user: user} do "profile_image_url_profile_size" => image, "profile_image_url_original" => image, "following" => true, - "rights" => %{} + "rights" => %{}, + "statusnet_profile_url" => user.ap_id } assert represented == UserRepresenter.to_map(user, %{for: follower}) From a3e82c5c246a4852d7bfaa5f6e216145b89fe0d8 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 15:54:14 +0200 Subject: [PATCH 67/88] Save context in likes / announces. --- lib/pleroma/web/activity_pub/activity_pub.ex | 6 ++++-- test/web/activity_pub/activity_pub_test.exs | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 0fb8db520..e9de3573e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -55,7 +55,8 @@ def like(%User{ap_id: ap_id} = user, %Object{data: %{ "id" => id}} = object) do "type" => "Like", "actor" => ap_id, "object" => id, - "to" => [User.ap_followers(user), object.data["actor"]] + "to" => [User.ap_followers(user), object.data["actor"]], + "context" => object.data["context"] } {:ok, activity} = insert(data) @@ -177,7 +178,8 @@ def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) "type" => "Announce", "actor" => ap_id, "object" => id, - "to" => [User.ap_followers(user), object.data["actor"]] + "to" => [User.ap_followers(user), object.data["actor"]], + "context" => object.data["context"] } {:ok, activity} = insert(data) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 744021c8c..6e42fbda2 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -125,6 +125,7 @@ test "adds a like activity to the db" do assert like_activity.data["type"] == "Like" assert like_activity.data["object"] == object.data["id"] assert like_activity.data["to"] == [User.ap_followers(user), note_activity.data["actor"]] + assert like_activity.data["context"] == object.data["context"] assert object.data["like_count"] == 1 assert object.data["likes"] == [user.ap_id] @@ -174,6 +175,7 @@ test "adds an announce activity to the db" do assert announce_activity.data["to"] == [User.ap_followers(user), note_activity.data["actor"]] assert announce_activity.data["object"] == object.data["id"] assert announce_activity.data["actor"] == user.ap_id + assert announce_activity.data["context"] == object.data["context"] end end From 93de6039667b9fe6f3b9019c4c2297d4f23b3a1a Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 16:35:53 +0200 Subject: [PATCH 68/88] Add an ostatus representer for like activities. --- .../web/ostatus/activity_representer.ex | 45 ++++++++++++++++--- .../web/ostatus/activity_representer_test.exs | 36 +++++++++++++++ 2 files changed, 76 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index c64bb3a3b..cf6aae727 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -10,11 +10,17 @@ defp get_in_reply_to(%{"object" => %{ "inReplyTo" => in_reply_to}}) do defp get_in_reply_to(_), do: [] defp get_mentions(to) do - Enum.map(to, fn - ("https://www.w3.org/ns/activitystreams#Public") -> - {:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/collection", href: "http://activityschema.org/collection/public"], []} - (id) -> - {:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/person", href: id], []} + Enum.map(to, fn (id) -> + cond do + # Special handling for the AP/Ostatus public collections + "https://www.w3.org/ns/activitystreams#Public" == id -> + {:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/collection", href: "http://activityschema.org/collection/public"], []} + # Ostatus doesn't handle follower collections, ignore these. + Regex.match?(~r/^#{Pleroma.Web.base_url}.+followers$/, id) -> + [] + true -> + {:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/person", href: id], []} + end end) end @@ -49,6 +55,35 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, ] ++ attachments ++ in_reply_to ++ author ++ mentions end + def to_simple_form(%{data: %{"type" => "Like"}} = activity, user, with_author) do + h = fn(str) -> [to_charlist(str)] end + + updated_at = activity.updated_at + |> NaiveDateTime.to_iso8601 + inserted_at = activity.inserted_at + |> NaiveDateTime.to_iso8601 + + in_reply_to = get_in_reply_to(activity.data) + author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] + mentions = activity.data["to"] |> get_mentions + + [ + {:"activity:verb", ['http://activitystrea.ms/schema/1.0/favorite']}, + {:id, h.(activity.data["id"])}, + {:title, ['New favorite by #{user.nickname}']}, + {:content, [type: 'html'], ['#{user.nickname} favorited something']}, + {:published, h.(inserted_at)}, + {:updated, h.(updated_at)}, + {:"activity:object", [ + {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']}, + {:id, h.(activity.data["object"])}, # For notes, federate the object id. + ]}, + {:"ostatus:conversation", [], h.(activity.data["context"])}, + {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}, + {:"thr:in-reply-to", [ref: to_charlist(activity.data["object"])], []} + ] ++ author ++ mentions + end + def wrap_with_entry(simple_form) do [{ :entry, [ diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 439c733d7..4cf73427b 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -3,6 +3,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenterTest do alias Pleroma.Web.OStatus.ActivityRepresenter alias Pleroma.{User, Activity} + alias Pleroma.Web.ActivityPub.ActivityPub import Pleroma.Factory @@ -72,6 +73,41 @@ test "a reply note" do assert clean(res) == clean(expected) end + test "a like activity" do + note = insert(:note) + user = insert(:user) + {:ok, like, _note} = ActivityPub.like(user, note) + + updated_at = like.updated_at + |> NaiveDateTime.to_iso8601 + inserted_at = like.inserted_at + |> NaiveDateTime.to_iso8601 + + tuple = ActivityRepresenter.to_simple_form(like, user) + refute is_nil(tuple) + + res = :xmerl.export_simple_content(tuple, :xmerl_xml) |> IO.iodata_to_binary + + expected = """ + http://activitystrea.ms/schema/1.0/favorite + #{like.data["id"]} + New favorite by #{user.nickname} + #{user.nickname} favorited something + #{inserted_at} + #{updated_at} + + http://activitystrea.ms/schema/1.0/note + #{note.data["id"]} + + #{like.data["context"]} + + + + """ + + assert clean(res) == clean(expected) + end + test "an unknown activity" do tuple = ActivityRepresenter.to_simple_form(%Activity{}, nil) assert is_nil(tuple) From 945b4b55e651341ae9452c9799f432ec2de11787 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 16:45:54 +0200 Subject: [PATCH 69/88] Federate likes. --- lib/pleroma/web/activity_pub/activity_pub.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index e9de3573e..12d6912df 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -72,6 +72,10 @@ def like(%User{ap_id: ap_id} = user, %Object{data: %{ "id" => id}} = object) do update_object_in_activities(object) + if user.local do + Pleroma.Web.Federator.enqueue(:publish, activity) + end + {:ok, activity, object} end end From 102455bf296165a88578a04f0ded259c32349d7f Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 17:13:41 +0200 Subject: [PATCH 70/88] Add avatar updating from incoming messages. --- lib/pleroma/web/ostatus/ostatus.ex | 17 +- test/fixtures/23211.atom | 508 +++++++++++++++++++++++++++++ test/web/ostatus/ostatus_test.exs | 20 ++ 3 files changed, 543 insertions(+), 2 deletions(-) create mode 100644 test/fixtures/23211.atom diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index db32d2c35..4c72e9cd1 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -90,6 +90,19 @@ def handle_note(entry, doc \\ nil) do end end + def find_make_or_update_user(doc) do + uri = string_from_xpath("//author/uri[1]", doc) + with {:ok, user} <- find_or_make_user(uri) do + avatar = make_avatar_object(doc) + if user.avatar != avatar do + change = Ecto.Changeset.change(user, %{avatar: avatar}) + Repo.update(change) + else + {:ok, user} + end + end + end + def find_or_make_user(uri) do query = from user in User, where: user.local == false and fragment("? @> ?", user.info, ^%{uri: uri}) @@ -121,8 +134,8 @@ def make_user(uri) do # TODO: Just takes the first one for now. def make_avatar_object(author_doc) do - href = string_from_xpath("/feed/author[1]/link[@rel=\"avatar\"]/@href", author_doc) - type = string_from_xpath("/feed/author[1]/link[@rel=\"avatar\"]/@type", author_doc) + href = string_from_xpath("//author[1]/link[@rel=\"avatar\"]/@href", author_doc) + type = string_from_xpath("//author[1]/link[@rel=\"avatar\"]/@type", author_doc) if href do %{ diff --git a/test/fixtures/23211.atom b/test/fixtures/23211.atom new file mode 100644 index 000000000..d5d111baa --- /dev/null +++ b/test/fixtures/23211.atom @@ -0,0 +1,508 @@ + + + GNU social + https://social.heldscal.la/api/statuses/user_timeline/23211.atom + lambadalambda timeline + Updates from lambadalambda on social.heldscal.la! + https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg + 2017-05-02T14:59:30+00:00 + + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + Call me Deacon Blues. + + + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + + Berlin + + + homepage + https://heldscal.la + true + + + + + + + + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2015260:2017-05-02T14:45:47+00:00 + Favorite + lambadalambda favorited something by godemperorofdune: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> It's because your instance decided to be trap! lol.</p> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T14:45:47+00:00 + 2017-05-02T14:45:47+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:pawoo.net,2017-05-02:objectId=7397439:objectType=Status + New comment by godemperorofdune + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> It's because your instance decided to be trap! lol.</p> + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=136e244b26cdf1e9 + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:social.heldscal.la,2017-05-02:noticeId=2015221:objectType=note + New note by lambadalambda + Some script thinks I'm a mastodon server.<br /> <br /> [info] GET /api/v1/timelines/public<br /> [debug] Processing with Fallback.RedirectController.redirector/2<br /> Parameters: %{&quot;limit&quot; =&gt; &quot;40&quot;, &quot;path&quot; =&gt; [&quot;api&quot;, &quot;v1&quot;, &quot;timelines&quot;, &quot;public&quot;]}<br /> Pipelines: [] + + + http://activitystrea.ms/schema/1.0/post + 2017-05-02T14:40:50+00:00 + 2017-05-02T14:40:50+00:00 + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=136e244b26cdf1e9 + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:social.heldscal.la,2017-05-02:noticeId=2014759:objectType=comment + New comment by lambadalambda + @<a href="https://mstdn.io/users/mattskala" class="h-card u-url p-nickname mention" title="Matthew Skala">mattskala</a> You and @<a href="https://mastodon.social/users/kevinmarks" class="h-card u-url p-nickname mention" title="Kevin Marks">kevinmarks</a> are not wrong, but my comment was a suggestion to users and admins: Don't use big instances, don't run big instances. Also, it's a secondary advice to devs: Don't add features that encourage big instances. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-02T14:11:54+00:00 + 2017-05-02T14:11:54+00:00 + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=58e32e013ab6487d + + + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:social.heldscal.la,2017-05-02:noticeId=2014684:objectType=comment + New comment by lambadalambda + @<a href="https://mastodon.social/users/Ronkjeffries" class="h-card u-url p-nickname mention" title="Ron K Jeffries social">ronkjeffries</a> @<a href="https://xoxo.zone/users/KevinMarks" class="h-card u-url p-nickname mention" title="Kevin Marks ">kevinmarks</a> Usually people who run their own private instance just look at the timelines of other servers, follow a seed population and then go from there. This is of course hard on Mastodon, because it doesn't have a publicly visible timeline. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-02T14:07:00+00:00 + 2017-05-02T14:07:00+00:00 + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=58e32e013ab6487d + + + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2014584:2017-05-02T14:05:32+00:00 + Favorite + lambadalambda favorited something by mattskala: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> It's reasonable to expect that instance sizes will obey a power-law distribution because that's what such things in nature nearly always do. If so, there'll necessarily be a few instances much larger than the others; even if most are small, the network both socially and technically has to be able to deal with the existence of the few large ones.</p> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T14:05:32+00:00 + 2017-05-02T14:05:32+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:mstdn.io,2017-05-02:objectId=1316931:objectType=Status + New comment by mattskala + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> It's reasonable to expect that instance sizes will obey a power-law distribution because that's what such things in nature nearly always do. If so, there'll necessarily be a few instances much larger than the others; even if most are small, the network both socially and technically has to be able to deal with the existence of the few large ones.</p> + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=58e32e013ab6487d + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2013568:2017-05-02T14:05:29+00:00 + Favorite + lambadalambda favorited something by kevinmarks: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> except instance populations will be power law distributed, and the problems for the tummlers are worse at scale</p> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T14:05:29+00:00 + 2017-05-02T14:05:29+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:xoxo.zone,2017-05-02:objectId=89478:objectType=Status + New comment by kevinmarks + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> except instance populations will be power law distributed, and the problems for the tummlers are worse at scale</p> + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=58e32e013ab6487d + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2014060:2017-05-02T13:34:32+00:00 + Favorite + lambadalambda favorited something by gcarregues: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> Oh purée ! Ma vie en images !</p> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T13:34:32+00:00 + 2017-05-02T13:34:32+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:mastodon.etalab.gouv.fr,2017-05-02:objectId=55287:objectType=Status + New comment by gcarregues + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> Oh purée ! Ma vie en images !</p> + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:note:2013573:2017-05-02T13:03:33+00:00 + Favorite + lambadalambda favorited something by phildobangnz: also @<a href="https://sealion.club/user/579" class="h-card mention" title="Sim Bot">sim</a> reminder you are awesome; don't even trip- u kewler than Tutankhamen's cucumber, fam. Okay, good night. + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T13:03:33+00:00 + 2017-05-02T13:03:33+00:00 + + http://activitystrea.ms/schema/1.0/note + tag:sealion.club,2017-05-02:noticeId=3060818:objectType=note + New note by phildobangnz + also @<a href="https://sealion.club/user/579" class="h-card mention" title="Sim Bot">sim</a> reminder you are awesome; don't even trip- u kewler than Tutankhamen's cucumber, fam. Okay, good night. + + + + + + + https://sealion.club/conversation/1633267 + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:social.heldscal.la,2017-05-02:noticeId=2013586:objectType=comment + New comment by lambadalambda + @<a href="https://xoxo.zone/users/KevinMarks" class="h-card u-url p-nickname mention" title="Kevin Marks ">kevinmarks</a> People can stay in their giant unmoderatable instances with meaningless public and federated timelines and experience constant federation drama if they want. I'll stay here with my 5 friends. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-02T12:54:59+00:00 + 2017-05-02T12:54:59+00:00 + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=58e32e013ab6487d + + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:note:2013486:2017-05-02T12:46:48+00:00 + Favorite + lambadalambda favorited something by fortune: There once was a dentist named Stone<br /> Who saw all his patients alone.<br /> In a fit of depravity<br /> He filled the wrong cavity,<br /> And my, how his practice has grown! + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T12:46:48+00:00 + 2017-05-02T12:46:48+00:00 + + http://activitystrea.ms/schema/1.0/note + tag:gs.kawa-kun.com,2017-05-02:noticeId=1655658:objectType=note + New note by fortune + There once was a dentist named Stone<br /> Who saw all his patients alone.<br /> In a fit of depravity<br /> He filled the wrong cavity,<br /> And my, how his practice has grown! + + + + + + + https://gs.kawa-kun.com/conversation/714072 + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:note:2013365:2017-05-02T12:37:55+00:00 + Favorite + lambadalambda favorited something by xj9: <p>&gt; rollerblading to work</p> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T12:37:55+00:00 + 2017-05-02T12:37:55+00:00 + + http://activitystrea.ms/schema/1.0/note + tag:sunshinegardens.org,2017-05-02:objectId=61020:objectType=Status + New note by xj9 + <p>&gt; rollerblading to work</p> + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=5a0e98612f634218 + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2013259:2017-05-02T12:29:03+00:00 + Favorite + lambadalambda favorited something by cereal: @<a href="https://gs.smuglo.li/user/28250" class="h-card mention" title="Bricky">thatbrickster</a> @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> But why? + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T12:29:03+00:00 + 2017-05-02T12:29:03+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:sealion.club,2017-05-02:noticeId=3059985:objectType=comment + New comment by cereal + @<a href="https://gs.smuglo.li/user/28250" class="h-card mention" title="Bricky">thatbrickster</a> @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> But why? + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2013227:2017-05-02T12:24:27+00:00 + Favorite + lambadalambda favorited something by thatbrickster: @<a href="https://social.heldscal.la/user/23211" class="h-card u-url p-nickname mention" title="Constance Variable">lambadalambda</a> install gentoo + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T12:24:27+00:00 + 2017-05-02T12:24:27+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:gs.smuglo.li,2017-05-02:noticeId=2144296:objectType=comment + New comment by thatbrickster + @<a href="https://social.heldscal.la/user/23211" class="h-card u-url p-nickname mention" title="Constance Variable">lambadalambda</a> install gentoo + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2013213:2017-05-02T12:22:53+00:00 + Favorite + lambadalambda favorited something by dwmatiz: @<a href="https://social.heldscal.la/user/23211" class="h-card mention">lambadalambda</a> *unzips dick* + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T12:22:53+00:00 + 2017-05-02T12:22:53+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:sealion.club,2017-05-02:noticeId=3059800:objectType=comment + New comment by dwmatiz + @<a href="https://social.heldscal.la/user/23211" class="h-card mention">lambadalambda</a> *unzips dick* + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2013199:2017-05-02T12:22:03+00:00 + Favorite + lambadalambda favorited something by shpuld: @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> get #<span class="tag"><a href="https://shitposter.club/tag/cofe" rel="tag">cofe</a></span> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T12:22:03+00:00 + 2017-05-02T12:22:03+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-02:noticeId=2783524:objectType=comment + New comment by shpuld + @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> get #<span class="tag"><a href="https://shitposter.club/tag/cofe" rel="tag">cofe</a></span> + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:social.heldscal.la,2017-05-02:noticeId=2013185:objectType=note + New note by lambadalambda + What now? <a href="https://social.heldscal.la/file/e4822d95de677757ff50d49672a4046c83218b76c04a0ad5e5f1f0a9a9eb1a74.gif" title="https://social.heldscal.la/file/e4822d95de677757ff50d49672a4046c83218b76c04a0ad5e5f1f0a9a9eb1a74.gif" rel="nofollow external noreferrer" class="attachment" id="attachment-422572">https://social.heldscal.la/attachment/422572</a> + + + http://activitystrea.ms/schema/1.0/post + 2017-05-02T12:21:04+00:00 + 2017-05-02T12:21:04+00:00 + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc + + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:note:2012929:2017-05-02T12:01:25+00:00 + Favorite + lambadalambda favorited something by drkmttr: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> I checked out No Agenda because I saw you mention it several time. Sadly, I wasn't impressed. I'm all about varying perspectives but Adam and John basically just sound like resentful curmudgeons. It seems like their shtick is basically playing devil's advocate to everything to arouse some discontent. Just my two cents. 😉</p> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T12:01:25+00:00 + 2017-05-02T12:01:25+00:00 + + http://activitystrea.ms/schema/1.0/note + tag:mstdn.io,2017-05-02:objectId=1310093:objectType=Status + New note by drkmttr + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> I checked out No Agenda because I saw you mention it several time. Sadly, I wasn't impressed. I'm all about varying perspectives but Adam and John basically just sound like resentful curmudgeons. It seems like their shtick is basically playing devil's advocate to everything to arouse some discontent. Just my two cents. 😉</p> + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2f329b4eb20e83e2 + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2012336:2017-05-02T11:06:42+00:00 + Favorite + lambadalambda favorited something by clacke: @<a href="https://mastodon.org.uk/users/dick_turpin" class="h-card u-url p-nickname mention" title="dick_turpin">dickturpin</a> @<a href="http://quitter.se/user/113503" class="h-card u-url p-nickname mention" title="Luke">luke</a> Oh no, I miss being irritated by you, it helps me understand myself and others. Also it builds character. :-)<br /> <br /> So if this is not federation because you can't follow all of online mankind, what should we call it? Proto-federated? Pre-federated?<br /> <br /> The term has been used decades ago for just one Microsoft Active Directory domain cross-certifying the root of another, by mutual agreement. I don't see how it's any less relevant to opportunistic federation between open servers on an open internet.<br /> <br /> I'm not saying we should be satisfied, I'm just saying that "federate" is a useful word and to build a big system we need to start with a small one. And focus on the things we *can* change, like helping the OStatus network grow and making the tools more useful.<br /> <br /> Saying that the network's ideals have failed because other networks aren't joining is doing neither of that. + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T11:06:42+00:00 + 2017-05-02T11:06:42+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:social.heldscal.la,2017-05-02:noticeId=2012336:objectType=comment + New comment by clacke + @<a href="https://mastodon.org.uk/users/dick_turpin" class="h-card u-url p-nickname mention" title="dick_turpin">dickturpin</a> @<a href="http://quitter.se/user/113503" class="h-card u-url p-nickname mention" title="Luke">luke</a> Oh no, I miss being irritated by you, it helps me understand myself and others. Also it builds character. :-)<br /> <br /> So if this is not federation because you can't follow all of online mankind, what should we call it? Proto-federated? Pre-federated?<br /> <br /> The term has been used decades ago for just one Microsoft Active Directory domain cross-certifying the root of another, by mutual agreement. I don't see how it's any less relevant to opportunistic federation between open servers on an open internet.<br /> <br /> I'm not saying we should be satisfied, I'm just saying that &quot;federate&quot; is a useful word and to build a big system we need to start with a small one. And focus on the things we *can* change, like helping the OStatus network grow and making the tools more useful.<br /> <br /> Saying that the network's ideals have failed because other networks aren't joining is doing neither of that. + + + + + + + https://s.wefamlee.be/conversation/16478 + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2011332:2017-05-02T10:37:40+00:00 + Favorite + lambadalambda favorited something by moonman: @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> <a href="https://www.youtube.com/watch?v=mKLizztikRk" title="https://www.youtube.com/watch?v=mKLizztikRk" class="attachment" rel="nofollow">https://www.youtube.com/watch?v=mKLizztikRk</a> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T10:37:40+00:00 + 2017-05-02T10:37:40+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-02:noticeId=2781833:objectType=comment + New comment by moonman + @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> <a href="https://www.youtube.com/watch?v=mKLizztikRk" title="https://www.youtube.com/watch?v=mKLizztikRk" class="attachment" rel="nofollow">https://www.youtube.com/watch?v=mKLizztikRk</a> + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=11d8b8c27d9513ec + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:social.heldscal.la,2017-05-02:noticeId=2012145:objectType=comment + New comment by lambadalambda + @<a href="https://sealion.club/user/186" class="h-card u-url p-nickname mention" title="I'M CEREAL U GUISE">cereal</a> ? No, you don't even need the identity servers for federation. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-02T10:37:33+00:00 + 2017-05-02T10:37:33+00:00 + + + + https://sealion.club/conversation/1629037 + + + + + + + diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 3951dbc9c..1674edbd5 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -1,6 +1,7 @@ defmodule Pleroma.Web.OStatusTest do use Pleroma.DataCase alias Pleroma.Web.OStatus + alias Pleroma.Web.XML test "don't insert create notes twice" do incoming = File.read!("test/fixtures/incoming_note_activity.xml") @@ -71,6 +72,25 @@ test "tries to use the information in poco fields" do assert user == user_again end + + test "find_make_or_update_user takes an author element and returns an updated user" do + # TODO make test local + uri = "https://social.heldscal.la/user/23211" + + {:ok, user} = OStatus.find_or_make_user(uri) + change = Ecto.Changeset.change(user, %{avatar: nil}) + + {:ok, user} = Repo.update(change) + refute user.avatar + + doc = XML.parse_document(File.read!("test/fixtures/23211.atom")) + [author] = :xmerl_xpath.string('//author[1]', doc) + {:ok, user} = OStatus.find_make_or_update_user(author) + assert user.avatar["type"] == "Image" + + {:ok, user_again} = OStatus.find_make_or_update_user(author) + assert user_again == user + end end describe "gathering user info from a user id" do From 96014f8e0b7bc6b28170f06914ef646f3f22ecfc Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 17:16:01 +0200 Subject: [PATCH 71/88] Update incoming new avatars. --- lib/pleroma/web/ostatus/ostatus.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 4c72e9cd1..340228dcf 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -42,8 +42,8 @@ def handle_incoming(xml_string) do def handle_note(entry, doc \\ nil) do content_html = string_from_xpath("/entry/content[1]", entry) - uri = string_from_xpath("/entry/author/uri[1]", entry) || string_from_xpath("/feed/author/uri[1]", doc) - {:ok, actor} = find_or_make_user(uri) + [author] = :xmerl_xpath.string('//author[1]', doc) + {:ok, actor} = find_make_or_update_user(author) context = (string_from_xpath("/entry/ostatus:conversation[1]", entry) || "") |> String.trim context = if String.length(context) > 0 do From b104348fa52c6ea51b9a159b145a48ca74a22332 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 17:44:55 +0200 Subject: [PATCH 72/88] Follow webfinger redirects. --- lib/pleroma/web/web_finger/web_finger.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 402184d3f..d16bdd982 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -92,7 +92,7 @@ def finger(account, getter \\ &HTTPoison.get/3) do response = with {:ok, result} <- getter.("https:" <> address, ["Accept": "application/xrd+xml"], [params: [resource: account]]) do {:ok, result} else _ -> - getter.("http:" <> address, ["Accept": "application/xrd+xml"], [params: [resource: account]]) + getter.("http:" <> address, ["Accept": "application/xrd+xml"], [params: [resource: account], follow_redirect: true]) end with {:ok, %{status_code: status_code, body: body}} when status_code in 200..299 <- response, From 33c803d6da91e0253a100cbd480d253706c44964 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 18:25:39 +0200 Subject: [PATCH 73/88] Add attachment link to posts. --- lib/pleroma/web/twitter_api/twitter_api.ex | 14 +++++++++++++- test/web/twitter_api/twitter_api_test.exs | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index b1759a6f0..7656d4d33 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -32,11 +32,23 @@ def get_replied_to_activity(id) when not is_nil(id) do def get_replied_to_activity(_), do: nil + def add_attachments(text, attachments) do + attachment_text = Enum.map(attachments, fn + (%{"url" => [%{"href" => href} | _]}) -> + "#{href}" + _ -> "" + end) + Enum.join([text | attachment_text], "
") + end + def create_status(user = %User{}, data = %{"status" => status}) do attachments = attachments_from_ids(data["media_ids"]) context = ActivityPub.generate_context_id mentions = parse_mentions(status) - content_html = format_input(status, mentions) + content_html = status + |> format_input(mentions) + |> add_attachments(attachments) + to = to_for_user_and_mentions(user, mentions) date = make_date() diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index 4e17f3298..a92440f32 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -33,7 +33,7 @@ test "create a status" do { :ok, activity = %Activity{} } = TwitterAPI.create_status(user, input) - assert get_in(activity.data, ["object", "content"]) == "Hello again, @shp.
This is on another line." + assert get_in(activity.data, ["object", "content"]) == "Hello again, @shp.
This is on another line.
http://example.org/image.jpg" assert get_in(activity.data, ["object", "type"]) == "Note" assert get_in(activity.data, ["object", "actor"]) == user.ap_id assert get_in(activity.data, ["actor"]) == user.ap_id From 018a1a390fdb72652c615c28ac36f1b9a6a84d82 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 21:31:01 +0200 Subject: [PATCH 74/88] Use inReplyTo to find context. --- lib/pleroma/web/ostatus/ostatus.ex | 18 ++++++++------ test/fixtures/incoming_reply_mastodon.xml | 29 +++++++++++++++++++++++ test/web/ostatus/ostatus_test.exs | 17 +++++++++++++ 3 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 test/fixtures/incoming_reply_mastodon.xml diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 340228dcf..7aa1ac4ac 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -44,13 +44,19 @@ def handle_note(entry, doc \\ nil) do [author] = :xmerl_xpath.string('//author[1]', doc) {:ok, actor} = find_make_or_update_user(author) + inReplyTo = string_from_xpath("/entry/thr:in-reply-to[1]/@ref", entry) context = (string_from_xpath("/entry/ostatus:conversation[1]", entry) || "") |> String.trim - context = if String.length(context) > 0 do - context - else - ActivityPub.generate_context_id - end + + context = with %{data: %{"context" => context}} <- Object.get_cached_by_ap_id(inReplyTo) do + context + else _e -> + if String.length(context) > 0 do + context + else + ActivityPub.generate_context_id + end + end to = [ "https://www.w3.org/ns/activitystreams#Public" @@ -74,8 +80,6 @@ def handle_note(entry, doc \\ nil) do "actor" => actor.ap_id } - inReplyTo = string_from_xpath("/entry/thr:in-reply-to[1]/@ref", entry) - object = if inReplyTo do Map.put(object, "inReplyTo", inReplyTo) else diff --git a/test/fixtures/incoming_reply_mastodon.xml b/test/fixtures/incoming_reply_mastodon.xml new file mode 100644 index 000000000..8ee1186cc --- /dev/null +++ b/test/fixtures/incoming_reply_mastodon.xml @@ -0,0 +1,29 @@ + + + tag:mastodon.social,2017-05-02:objectId=4901603:objectType=Status + 2017-05-02T18:33:06Z + 2017-05-02T18:33:06Z + New status by lambadalambda + + https://mastodon.social/users/lambadalambda + http://activitystrea.ms/schema/1.0/person + https://mastodon.social/users/lambadalambda + lambadalambda + lambadalambda@mastodon.social + + + + lambadalambda + Critical Value + public + + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> hey</p> + + + public + + + + diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 1674edbd5..e39952807 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -2,6 +2,7 @@ defmodule Pleroma.Web.OStatusTest do use Pleroma.DataCase alias Pleroma.Web.OStatus alias Pleroma.Web.XML + alias Pleroma.{Object, Repo} test "don't insert create notes twice" do incoming = File.read!("test/fixtures/incoming_note_activity.xml") @@ -32,6 +33,22 @@ test "handle incoming notes - GS, subscription" do assert activity.data["object"]["content"] == "Will it blend?" end + test "handle incoming notes - Mastodon, salmon, reply" do + # It uses the context of the replied to object + Repo.insert!(%Object{ + data: %{ + "id" => "https://pleroma.soykaf.com/objects/c237d966-ac75-4fe3-a87a-d89d71a3a7a4", + "context" => "2hu" + }}) + incoming = File.read!("test/fixtures/incoming_reply_mastodon.xml") + {:ok, [activity]} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Create" + assert activity.data["object"]["type"] == "Note" + assert activity.data["object"]["actor"] == "https://mastodon.social/users/lambadalambda" + assert activity.data["context"] == "2hu" + end + test "handle incoming notes - GS, subscription, reply" do incoming = File.read!("test/fixtures/ostatus_incoming_reply.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) From 9c42453e068b683517f6a72602c08527222f8fea Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 09:54:17 +0200 Subject: [PATCH 75/88] Return note objects as ostatus post activities. --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/ostatus/ostatus_controller.ex | 15 +++++++++++++++ lib/pleroma/web/router.ex | 4 +++- test/web/ostatus/ostatus_controller_test.exs | 12 ++++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 12d6912df..194a5ec3d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -126,7 +126,7 @@ def generate_context_id do end def generate_object_id do - generate_id("objects") + Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :object, Ecto.UUID.generate) end def generate_id(type) do diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 1c609f6f2..6a4199846 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -41,4 +41,19 @@ def salmon_incoming(conn, params) do conn |> send_resp(200, "") end + + def object(conn, %{"uuid" => uuid}) do + IO.inspect(uuid) + id = o_status_url(conn, :object, uuid) + activity = Activity.get_create_activity_by_object_ap_id(id) + user = User.get_cached_by_ap_id(activity.data["actor"]) + + response = FeedRepresenter.to_simple_form(user, [activity], [user]) + |> :xmerl.export_simple(:xmerl_xml) + |> to_string + + conn + |> put_resp_content_type("application/atom+xml") + |> send_resp(200, response) + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index b0c1dcd91..ac9d97e0f 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -73,6 +73,8 @@ def user_fetcher(username) do scope "/", Pleroma.Web do pipe_through :ostatus + get "/objects/:uuid", OStatus.OStatusController, :object + get "/users/:nickname/feed", OStatus.OStatusController, :feed get "/users/:nickname", OStatus.OStatusController, :feed_redirect post "/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming @@ -96,5 +98,5 @@ def user_fetcher(username) do defmodule Fallback.RedirectController do use Pleroma.Web, :controller - def redirector(conn, _params), do: send_file(conn, 200, "priv/static/index.html") + def redirector(conn, _params), do: (if Mix.env != :test, do: send_file(conn, 200, "priv/static/index.html")) end diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index 229cd9b1e..f07698747 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -12,4 +12,16 @@ test "gets a feed", %{conn: conn} do assert response(conn, 200) end + + test "gets an object", %{conn: conn} do + note_activity = insert(:note_activity) + [_, uuid] = hd Regex.scan(~r/.+\/([\w-]+)$/, note_activity.data["object"]["id"]) + url = "/objects/#{uuid}" + |> IO.inspect + + conn = conn + |> get(url) + + assert response(conn, 200) + end end From 16afea399d330c28de05c77649fe0540598ee8ec Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 10:01:26 +0200 Subject: [PATCH 76/88] Just give out the entry, not the whole feed. --- lib/pleroma/web/ostatus/ostatus_controller.ex | 6 +++--- test/web/ostatus/ostatus_controller_test.exs | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 6a4199846..c6700ae78 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -2,7 +2,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do use Pleroma.Web, :controller alias Pleroma.{User, Activity} - alias Pleroma.Web.OStatus.FeedRepresenter + alias Pleroma.Web.OStatus.{FeedRepresenter, ActivityRepresenter} alias Pleroma.Repo alias Pleroma.Web.OStatus import Ecto.Query @@ -43,12 +43,12 @@ def salmon_incoming(conn, params) do end def object(conn, %{"uuid" => uuid}) do - IO.inspect(uuid) id = o_status_url(conn, :object, uuid) activity = Activity.get_create_activity_by_object_ap_id(id) user = User.get_cached_by_ap_id(activity.data["actor"]) - response = FeedRepresenter.to_simple_form(user, [activity], [user]) + response = ActivityRepresenter.to_simple_form(activity, user, true) + |> ActivityRepresenter.wrap_with_entry |> :xmerl.export_simple(:xmerl_xml) |> to_string diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index f07698747..8b7ca4d89 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -17,7 +17,6 @@ test "gets an object", %{conn: conn} do note_activity = insert(:note_activity) [_, uuid] = hd Regex.scan(~r/.+\/([\w-]+)$/, note_activity.data["object"]["id"]) url = "/objects/#{uuid}" - |> IO.inspect conn = conn |> get(url) From 8141024259ee4bebd58d6ecd963f181aad420846 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 14:26:49 +0200 Subject: [PATCH 77/88] Attachment parsing, better magic key fetching. --- lib/pleroma/web/ostatus/ostatus.ex | 35 ++++++++--- lib/pleroma/web/ostatus/ostatus_controller.ex | 2 +- lib/pleroma/web/salmon/salmon.ex | 15 ++--- lib/pleroma/web/web.ex | 22 +------ lib/pleroma/web/web_finger/web_finger.ex | 12 ++-- lib/pleroma/web/websub/websub.ex | 12 ++-- .../incoming_websub_gnusocial_attachments.xml | 59 +++++++++++++++++++ test/user_test.exs | 10 ++++ test/web/ostatus/ostatus_test.exs | 54 ++++++++++------- test/web/salmon/salmon_test.exs | 2 +- test/web/web_finger/web_finger_test.exs | 10 ++-- test/web/websub/websub_test.exs | 12 ++-- 12 files changed, 161 insertions(+), 84 deletions(-) create mode 100644 test/fixtures/incoming_websub_gnusocial_attachments.xml diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 7aa1ac4ac..f81751a25 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -39,6 +39,24 @@ def handle_incoming(xml_string) do {:ok, activities} end + def get_attachments(entry) do + :xmerl_xpath.string('/entry/link[@rel="enclosure"]', entry) + |> Enum.map(fn (enclosure) -> + with href when not is_nil(href) <- string_from_xpath("/link/@href", enclosure), + type when not is_nil(type) <- string_from_xpath("/link/@type", enclosure) do + %{ + "type" => "Attachment", + "url" => [%{ + "type" => "Link", + "mediaType" => type, + "href" => href + }] + } + end + end) + |> Enum.filter(&(&1)) + end + def handle_note(entry, doc \\ nil) do content_html = string_from_xpath("/entry/content[1]", entry) @@ -48,6 +66,8 @@ def handle_note(entry, doc \\ nil) do context = (string_from_xpath("/entry/ostatus:conversation[1]", entry) || "") |> String.trim + attachments = get_attachments(entry) + context = with %{data: %{"context" => context}} <- Object.get_cached_by_ap_id(inReplyTo) do context else _e -> @@ -77,7 +97,8 @@ def handle_note(entry, doc \\ nil) do "content" => content_html, "published" => date, "context" => context, - "actor" => actor.ap_id + "actor" => actor.ap_id, + "attachment" => attachments } object = if inReplyTo do @@ -124,11 +145,11 @@ def make_user(uri) do with {:ok, info} <- gather_user_info(uri) do data = %{ local: false, - name: info.name, - nickname: info.nickname <> "@" <> info.host, - ap_id: info.uri, + name: info["name"], + nickname: info["nickname"] <> "@" <> info["host"], + ap_id: info["uri"], info: info, - avatar: info.avatar + avatar: info["avatar"] } # TODO: Make remote user changeset # SHould enforce fqn nickname @@ -158,8 +179,8 @@ def make_avatar_object(author_doc) do def gather_user_info(username) do with {:ok, webfinger_data} <- WebFinger.finger(username), - {:ok, feed_data} <- Websub.gather_feed_data(webfinger_data.topic) do - {:ok, Map.merge(webfinger_data, feed_data) |> Map.put(:fqn, username)} + {:ok, feed_data} <- Websub.gather_feed_data(webfinger_data["topic"]) do + {:ok, Map.merge(webfinger_data, feed_data) |> Map.put("fqn", username)} else e -> Logger.debug("Couldn't gather info for #{username}") {:error, e} diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index c6700ae78..1f2dedd30 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -33,7 +33,7 @@ def feed(conn, %{"nickname" => nickname}) do def salmon_incoming(conn, params) do {:ok, body, _conn} = read_body(conn) - magic_key = Pleroma.Web.Salmon.fetch_magic_key(body) + {:ok, magic_key} = Pleroma.Web.Salmon.fetch_magic_key(body) {:ok, doc} = Pleroma.Web.Salmon.decode_and_validate(magic_key, body) Pleroma.Web.OStatus.handle_incoming(doc) diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index b4f214d46..f02cb11dc 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -24,16 +24,13 @@ def decode(salmon) do [data, type, encoding, alg, sig] end - # TODO rewrite in with-stile - # Make it fetch the key from the saved user if there is one def fetch_magic_key(salmon) do - [data, _, _, _, _] = decode(salmon) - doc = XML.parse_document(data) - uri = XML.string_from_xpath("/entry/author[1]/uri", doc) - - {:ok, info} = Pleroma.Web.OStatus.gather_user_info(uri) - - info.magic_key + with [data, _, _, _, _] <- decode(salmon), + doc <- XML.parse_document(data), + uri when not is_nil(uri) <- XML.string_from_xpath("/entry/author[1]/uri", doc), + {:ok, %{info: %{"magic_key" => magic_key}}} <- Pleroma.Web.OStatus.find_or_make_user(uri) do + {:ok, magic_key} + end end def decode_and_validate(magickey, salmon) do diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex index a81e3e6e1..ee7ee78e9 100644 --- a/lib/pleroma/web/web.ex +++ b/lib/pleroma/web/web.ex @@ -61,27 +61,7 @@ defmacro __using__(which) when is_atom(which) do apply(__MODULE__, which, []) end - def host do - settings = Application.get_env(:pleroma, Pleroma.Web.Endpoint) - settings - |> Keyword.fetch!(:url) - |> Keyword.fetch!(:host) - end - def base_url do - settings = Application.get_env(:pleroma, Pleroma.Web.Endpoint) - - host = host() - - protocol = settings |> Keyword.fetch!(:protocol) - - port_fragment = with {:ok, protocol_info} <- settings |> Keyword.fetch(String.to_atom(protocol)), - {:ok, port} <- protocol_info |> Keyword.fetch(:port) - do - ":#{port}" - else _e -> - "" - end - "#{protocol}://#{host}#{port_fragment}" + Pleroma.Web.Endpoint.url end end diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index d16bdd982..5fa69c2c8 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -16,7 +16,7 @@ def host_meta() do end def webfinger(resource) do - host = Pleroma.Web.host + host = Pleroma.Web.Endpoint.host regex = ~r/(acct:)?(?\w+)@#{host}/ with %{"username" => username} <- Regex.named_captures(regex, resource) do user = User.get_by_nickname(username) @@ -37,7 +37,7 @@ def represent_user(user) do { :XRD, %{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"}, [ - {:Subject, "acct:#{user.nickname}@#{Pleroma.Web.host}"}, + {:Subject, "acct:#{user.nickname}@#{Pleroma.Web.Endpoint.host}"}, {:Alias, user.ap_id}, {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}}, {:Link, %{rel: "http://webfinger.net/rel/profile-page", type: "text/html", href: user.ap_id}}, @@ -72,10 +72,10 @@ defp webfinger_from_xml(doc) do subject = XML.string_from_xpath("//Subject", doc) salmon = XML.string_from_xpath(~s{//Link[@rel="salmon"]/@href}, doc) data = %{ - magic_key: magic_key, - topic: topic, - subject: subject, - salmon: salmon + "magic_key" => magic_key, + "topic" => topic, + "subject" => subject, + "salmon" => salmon } {:ok, data} end diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 546bfb5a4..e32fc8817 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -146,12 +146,12 @@ def gather_feed_data(topic, getter \\ &HTTPoison.get/1) do avatar = OStatus.make_avatar_object(doc) {:ok, %{ - uri: uri, - hub: hub, - nickname: preferredUsername || name, - name: displayName || name, - host: URI.parse(uri).host, - avatar: avatar + "uri" => uri, + "hub" => hub, + "nickname" => preferredUsername || name, + "name" => displayName || name, + "host" => URI.parse(uri).host, + "avatar" => avatar }} else e -> {:error, e} diff --git a/test/fixtures/incoming_websub_gnusocial_attachments.xml b/test/fixtures/incoming_websub_gnusocial_attachments.xml new file mode 100644 index 000000000..9d331ef32 --- /dev/null +++ b/test/fixtures/incoming_websub_gnusocial_attachments.xml @@ -0,0 +1,59 @@ + + + GNU social + https://social.heldscal.la/api/statuses/user_timeline/23211.atom + lambadalambda timeline + Updates from lambadalambda on social.heldscal.la! + https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg + 2017-05-02T20:29:35+00:00 + + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + Call me Deacon Blues. + + + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + + Berlin + + + homepage + https://heldscal.la + true + + + + + + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:social.heldscal.la,2017-05-02:noticeId=2020923:objectType=note + New note by lambadalambda + Okay gonna stream some cool games!! <a href="https://social.heldscal.la/file/7ed5ee508e6376a6e3dd581e17e7ed0b7b638147c7e86784bf83abc2641ee3d4.gif" title="https://social.heldscal.la/file/7ed5ee508e6376a6e3dd581e17e7ed0b7b638147c7e86784bf83abc2641ee3d4.gif" rel="nofollow external noreferrer" class="attachment" id="attachment-423842">https://social.heldscal.la/attachment/423842</a> <a href="https://social.heldscal.la/file/4c209099cadfc5afd3e27a334aa0db96b3a7510dde1603305d68a2707e59a11f.png" title="https://social.heldscal.la/file/4c209099cadfc5afd3e27a334aa0db96b3a7510dde1603305d68a2707e59a11f.png" rel="nofollow external noreferrer" class="attachment" id="attachment-423843">https://social.heldscal.la/attachment/423843</a> + + + http://activitystrea.ms/schema/1.0/post + 2017-05-02T20:29:35+00:00 + 2017-05-02T20:29:35+00:00 + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=26c7afdcbcf4ebd4 + + + + + + + + diff --git a/test/user_test.exs b/test/user_test.exs index 1331ac971..7435e30e0 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -111,5 +111,15 @@ test "returns nil for nonexistant local user" do assert fetched_user == nil end end + + test "returns an ap_id for a user" do + user = insert(:user) + assert User.ap_id(user) == Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :feed_redirect, user.nickname) + end + + test "returns an ap_followers link for a user" do + user = insert(:user) + assert User.ap_followers(user) == Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :feed_redirect, user.nickname) <> "/followers" + end end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index e39952807..94a735337 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -33,6 +33,16 @@ test "handle incoming notes - GS, subscription" do assert activity.data["object"]["content"] == "Will it blend?" end + test "handle incoming notes with attachments - GS, subscription" do + incoming = File.read!("test/fixtures/incoming_websub_gnusocial_attachments.xml") + {:ok, [activity]} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Create" + assert activity.data["object"]["type"] == "Note" + assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" + assert activity.data["object"]["attachment"] |> length == 2 + end + test "handle incoming notes - Mastodon, salmon, reply" do # It uses the context of the replied to object Repo.insert!(%Object{ @@ -118,17 +128,17 @@ test "it returns user info in a hash" do {:ok, data} = OStatus.gather_user_info(user) expected = %{ - hub: "https://social.heldscal.la/main/push/hub", - magic_key: "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", - name: "shp", - nickname: "shp", - salmon: "https://social.heldscal.la/main/salmon/user/29191", - subject: "acct:shp@social.heldscal.la", - topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", - uri: "https://social.heldscal.la/user/29191", - host: "social.heldscal.la", - fqn: user, - avatar: %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} + "hub" => "https://social.heldscal.la/main/push/hub", + "magic_key" => "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", + "name" => "shp", + "nickname" => "shp", + "salmon" => "https://social.heldscal.la/main/salmon/user/29191", + "subject" => "acct:shp@social.heldscal.la", + "topic" => "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", + "uri" => "https://social.heldscal.la/user/29191", + "host" => "social.heldscal.la", + "fqn" => user, + "avatar" => %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} } assert data == expected end @@ -140,17 +150,17 @@ test "it works with the uri" do {:ok, data} = OStatus.gather_user_info(user) expected = %{ - hub: "https://social.heldscal.la/main/push/hub", - magic_key: "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", - name: "shp", - nickname: "shp", - salmon: "https://social.heldscal.la/main/salmon/user/29191", - subject: "https://social.heldscal.la/user/29191", - topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", - uri: "https://social.heldscal.la/user/29191", - host: "social.heldscal.la", - fqn: user, - avatar: %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} + "hub" => "https://social.heldscal.la/main/push/hub", + "magic_key" => "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", + "name" => "shp", + "nickname" => "shp", + "salmon" => "https://social.heldscal.la/main/salmon/user/29191", + "subject" => "https://social.heldscal.la/user/29191", + "topic" => "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", + "uri" => "https://social.heldscal.la/user/29191", + "host" => "social.heldscal.la", + "fqn" => user, + "avatar" => %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} } assert data == expected end diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs index 77dacc1c0..ed26ccf83 100644 --- a/test/web/salmon/salmon_test.exs +++ b/test/web/salmon/salmon_test.exs @@ -55,7 +55,7 @@ test "encodes an xml payload with a private key" do test "it gets a magic key" do # TODO: Make test local salmon = File.read!("test/fixtures/salmon2.xml") - key = Salmon.fetch_magic_key(salmon) + {:ok, key} = Salmon.fetch_magic_key(salmon) assert key == "RSA.uzg6r1peZU0vXGADWxGJ0PE34WvmhjUmydbX5YYdOiXfODVLwCMi1umGoqUDm-mRu4vNEdFBVJU1CpFA7dKzWgIsqsa501i2XqElmEveXRLvNRWFB6nG03Q5OUY2as8eE54BJm0p20GkMfIJGwP6TSFb-ICp3QjzbatuSPJ6xCE=.AQAB" end diff --git a/test/web/web_finger/web_finger_test.exs b/test/web/web_finger/web_finger_test.exs index b48fdd0aa..495d3d50b 100644 --- a/test/web/web_finger/web_finger_test.exs +++ b/test/web/web_finger/web_finger_test.exs @@ -15,7 +15,7 @@ test "returns a link to the xml lrdd" do test "works for fqns" do user = insert(:user) - {:ok, result} = WebFinger.webfinger("#{user.nickname}@#{Pleroma.Web.host}") + {:ok, result} = WebFinger.webfinger("#{user.nickname}@#{Pleroma.Web.Endpoint.host}") assert is_binary(result) end @@ -37,10 +37,10 @@ test "returns the info for a user" do {:ok, data} = WebFinger.finger(user, getter) - assert data.magic_key == "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB" - assert data.topic == "https://social.heldscal.la/api/statuses/user_timeline/29191.atom" - assert data.subject == "acct:shp@social.heldscal.la" - assert data.salmon == "https://social.heldscal.la/main/salmon/user/29191" + assert data["magic_key"] == "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB" + assert data["topic"] == "https://social.heldscal.la/api/statuses/user_timeline/29191.atom" + assert data["subject"] == "acct:shp@social.heldscal.la" + assert data["salmon"] == "https://social.heldscal.la/main/salmon/user/29191" end end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 63acb3c43..065fb250a 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -115,12 +115,12 @@ test "discovers the hub and canonical url" do {:ok, discovered} = Websub.gather_feed_data(topic, getter) expected = %{ - hub: "https://mastodon.social/api/push", - uri: "https://mastodon.social/users/lambadalambda", - nickname: "lambadalambda", - name: "Critical Value", - host: "mastodon.social", - avatar: %{"type" => "Image", "url" => [%{"href" => "https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif?1492379244", "mediaType" => "image/gif", "type" => "Link"}]} + "hub" => "https://mastodon.social/api/push", + "uri" => "https://mastodon.social/users/lambadalambda", + "nickname" => "lambadalambda", + "name" => "Critical Value", + "host" => "mastodon.social", + "avatar" => %{"type" => "Image", "url" => [%{"href" => "https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif?1492379244", "mediaType" => "image/gif", "type" => "Link"}]} } assert expected == discovered From df71c142cfadaae8866303768bca00c343b8bed1 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 16:08:24 +0200 Subject: [PATCH 78/88] Remove doubled 'to' recipients. --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- test/web/activity_pub/activity_pub_test.exs | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 194a5ec3d..f18f3df2e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -24,7 +24,7 @@ def create(to, actor, context, object, additional \\ %{}, published \\ nil, loca activity = %{ "type" => "Create", - "to" => to, + "to" => to |> Enum.uniq, "actor" => actor.ap_id, "object" => object, "published" => published, diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 6e42fbda2..dfa73b775 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -40,6 +40,13 @@ test "adds an id to a given object if it lacks one and inserts it to the object end end + describe "create activities" do + test "removes doubled 'to' recipients" do + {:ok, activity} = ActivityPub.create(["user1", "user1", "user2"], %User{ap_id: "1"}, "", %{}) + assert activity.data["to"] == ["user1", "user2"] + end + end + describe "fetch activities for recipients" do test "retrieve the activities for certain recipients" do {:ok, activity_one} = ActivityBuilder.insert(%{"to" => ["someone"]}) From 138641589dffc6ba69710ec15c690b50769f07b4 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 17:39:12 +0200 Subject: [PATCH 79/88] OStatus announce representer. --- .../web/ostatus/activity_representer.ex | 33 +++++++++++++- .../web/ostatus/activity_representer_test.exs | 45 ++++++++++++++++++- 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index cf6aae727..9e3a9abcb 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -1,5 +1,5 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do - alias Pleroma.Activity + alias Pleroma.{Activity, User} alias Pleroma.Web.OStatus.UserRepresenter require Logger @@ -84,6 +84,37 @@ def to_simple_form(%{data: %{"type" => "Like"}} = activity, user, with_author) d ] ++ author ++ mentions end + def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_author) do + h = fn(str) -> [to_charlist(str)] end + + updated_at = activity.updated_at + |> NaiveDateTime.to_iso8601 + inserted_at = activity.inserted_at + |> NaiveDateTime.to_iso8601 + + in_reply_to = get_in_reply_to(activity.data) + author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] + + retweeted_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"]) + retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"]) + + retweeted_xml = to_simple_form(retweeted_activity, retweeted_user) + + mentions = activity.data["to"] |> get_mentions + [ + {:"activity:verb", ['http://activitystrea.ms/schema/1.0/share']}, + {:id, h.(activity.data["id"])}, + {:title, ['#{user.nickname} repeated a notice']}, + {:content, [type: 'html'], ['RT #{retweeted_activity.data["object"]["content"]}']}, + {:published, h.(inserted_at)}, + {:updated, h.(updated_at)}, + {:"ostatus:conversation", [], h.(activity.data["context"])}, + {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}, + {:"thr:in-reply-to", [ref: to_charlist(activity.data["object"])], []}, + {:"activity:object", retweeted_xml} + ] ++ mentions ++ author + end + def wrap_with_entry(simple_form) do [{ :entry, [ diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 4cf73427b..d3c32e938 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenterTest do use Pleroma.DataCase alias Pleroma.Web.OStatus.ActivityRepresenter - alias Pleroma.{User, Activity} + alias Pleroma.{User, Activity, Object} alias Pleroma.Web.ActivityPub.ActivityPub import Pleroma.Factory @@ -73,6 +73,49 @@ test "a reply note" do assert clean(res) == clean(expected) end + test "an announce activity" do + note = insert(:note_activity) + user = insert(:user) + object = Object.get_cached_by_ap_id(note.data["object"]["id"]) + + {:ok, announce, object} = ActivityPub.announce(user, object) + + announce = Repo.get(Activity, announce.id) + + note_user = User.get_cached_by_ap_id(note.data["actor"]) + note = Repo.get(Activity, note.id) + note_xml = ActivityRepresenter.to_simple_form(note, note_user) + |> :xmerl.export_simple_content(:xmerl_xml) + |> IO.iodata_to_binary + + updated_at = announce.updated_at + |> NaiveDateTime.to_iso8601 + inserted_at = announce.inserted_at + |> NaiveDateTime.to_iso8601 + + expected = """ + http://activitystrea.ms/schema/1.0/share + #{announce.data["id"]} + #{user.nickname} repeated a notice + RT #{note.data["object"]["content"]} + #{inserted_at} + #{updated_at} + #{announce.data["context"]} + + + + #{note_xml} + + + """ + + announce_xml = ActivityRepresenter.to_simple_form(announce, user) + |> :xmerl.export_simple_content(:xmerl_xml) + |> IO.iodata_to_binary + + assert clean(expected) == clean(announce_xml) + end + test "a like activity" do note = insert(:note) user = insert(:user) From 861a294cdae313c4c2edfc9840bf1083da0acd6e Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 17:41:55 +0200 Subject: [PATCH 80/88] Add announce federation. --- lib/pleroma/web/activity_pub/activity_pub.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index f18f3df2e..5583a1f41 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -199,6 +199,10 @@ def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) update_object_in_activities(object) + if user.local do + Pleroma.Web.Federator.enqueue(:publish, activity) + end + {:ok, activity, object} end From b34b046f16a44172ac96709dd0b6f5bced96d0b5 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 17:51:36 +0200 Subject: [PATCH 81/88] Add user to announced status. --- lib/pleroma/web/ostatus/activity_representer.ex | 2 +- test/web/ostatus/activity_representer_test.exs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 9e3a9abcb..d064b09ee 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -98,7 +98,7 @@ def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_autho retweeted_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"]) retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"]) - retweeted_xml = to_simple_form(retweeted_activity, retweeted_user) + retweeted_xml = to_simple_form(retweeted_activity, retweeted_user, true) mentions = activity.data["to"] |> get_mentions [ diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index d3c32e938..7f003226b 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -84,9 +84,9 @@ test "an announce activity" do note_user = User.get_cached_by_ap_id(note.data["actor"]) note = Repo.get(Activity, note.id) - note_xml = ActivityRepresenter.to_simple_form(note, note_user) + note_xml = ActivityRepresenter.to_simple_form(note, note_user, true) |> :xmerl.export_simple_content(:xmerl_xml) - |> IO.iodata_to_binary + |> to_string updated_at = announce.updated_at |> NaiveDateTime.to_iso8601 @@ -111,7 +111,7 @@ test "an announce activity" do announce_xml = ActivityRepresenter.to_simple_form(announce, user) |> :xmerl.export_simple_content(:xmerl_xml) - |> IO.iodata_to_binary + |> to_string assert clean(expected) == clean(announce_xml) end From 5d7831ee3e1ff62c2e54fe47aa1a6cf3474e8578 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 18:10:19 +0200 Subject: [PATCH 82/88] Add self links to federated statuses. --- lib/pleroma/web/ostatus/activity_representer.ex | 6 +++++- test/web/ostatus/activity_representer_test.exs | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index d064b09ee..dc7526598 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -51,7 +51,8 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, {:published, h.(inserted_at)}, {:updated, h.(updated_at)}, {:"ostatus:conversation", [], h.(activity.data["context"])}, - {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []} + {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}, + {:link, [type: ['application/atom+xml'], href: h.(activity.data["object"]["id"]), rel: 'self'], []} ] ++ attachments ++ in_reply_to ++ author ++ mentions end @@ -80,6 +81,7 @@ def to_simple_form(%{data: %{"type" => "Like"}} = activity, user, with_author) d ]}, {:"ostatus:conversation", [], h.(activity.data["context"])}, {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}, + {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []}, {:"thr:in-reply-to", [ref: to_charlist(activity.data["object"])], []} ] ++ author ++ mentions end @@ -102,6 +104,7 @@ def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_autho mentions = activity.data["to"] |> get_mentions [ + {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']}, {:"activity:verb", ['http://activitystrea.ms/schema/1.0/share']}, {:id, h.(activity.data["id"])}, {:title, ['#{user.nickname} repeated a notice']}, @@ -110,6 +113,7 @@ def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_autho {:updated, h.(updated_at)}, {:"ostatus:conversation", [], h.(activity.data["context"])}, {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}, + {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []}, {:"thr:in-reply-to", [ref: to_charlist(activity.data["object"])], []}, {:"activity:object", retweeted_xml} ] ++ mentions ++ author diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 7f003226b..03b3e248f 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -26,6 +26,7 @@ test "a note activity" do #{updated_at} #{note_activity.data["context"]} + """ @@ -62,6 +63,7 @@ test "a reply note" do #{updated_at} #{answer.data["context"]} + """ @@ -94,6 +96,7 @@ test "an announce activity" do |> NaiveDateTime.to_iso8601 expected = """ + http://activitystrea.ms/schema/1.0/activity http://activitystrea.ms/schema/1.0/share #{announce.data["id"]} #{user.nickname} repeated a notice @@ -102,6 +105,7 @@ test "an announce activity" do #{updated_at} #{announce.data["context"]} + #{note_xml} @@ -144,6 +148,7 @@ test "a like activity" do #{like.data["context"]} + """ From 53d05af5b61771782af3946181cc3139f3897cca Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 19:23:12 +0200 Subject: [PATCH 83/88] Fix Mastodon signature bug. --- lib/pleroma/web/websub/websub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index e32fc8817..0d0d19c88 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -52,7 +52,7 @@ def publish(topic, user, activity) do end def sign(secret, doc) do - :crypto.hmac(:sha, secret, to_string(doc)) |> Base.encode16 + :crypto.hmac(:sha, secret, to_string(doc)) |> Base.encode16 |> String.downcase end def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) do From 1077c5c58d13325cd61893c609cad6505ad1d32e Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 20:06:00 +0200 Subject: [PATCH 84/88] Remove reply-to for shares, mastodon gets confused. --- lib/pleroma/web/ostatus/activity_representer.ex | 1 - test/web/ostatus/activity_representer_test.exs | 1 - 2 files changed, 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index dc7526598..41a42b7cb 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -114,7 +114,6 @@ def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_autho {:"ostatus:conversation", [], h.(activity.data["context"])}, {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}, {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []}, - {:"thr:in-reply-to", [ref: to_charlist(activity.data["object"])], []}, {:"activity:object", retweeted_xml} ] ++ mentions ++ author end diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 03b3e248f..12c9bbaa2 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -106,7 +106,6 @@ test "an announce activity" do #{announce.data["context"]} - #{note_xml} From 97257c692c5786b370d8f0769533d11c1d00334e Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 20:06:20 +0200 Subject: [PATCH 85/88] Fix specs. --- lib/pleroma/web/websub/websub_controller.ex | 2 +- test/user_test.exs | 2 +- test/web/websub/websub_test.exs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex index e5ecf6523..e860ec9e5 100644 --- a/lib/pleroma/web/websub/websub_controller.ex +++ b/lib/pleroma/web/websub/websub_controller.ex @@ -34,7 +34,7 @@ def websub_subscription_confirmation(conn, %{"id" => id, "hub.mode" => "subscrib def websub_incoming(conn, %{"id" => id}) do with "sha1=" <> signature <- hd(get_req_header(conn, "x-hub-signature")), - signature <- String.upcase(signature), + signature <- String.downcase(signature), %WebsubClientSubscription{} = websub <- Repo.get(WebsubClientSubscription, id), {:ok, body, _conn} = read_body(conn), ^signature <- Websub.sign(websub.secret, body) do diff --git a/test/user_test.exs b/test/user_test.exs index 7435e30e0..036e70dff 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -13,7 +13,7 @@ test "ap_id returns the activity pub id for the user" do user = UserBuilder.build - expected_ap_id = "https://#{host}/users/#{user.nickname}" + expected_ap_id = "#{Pleroma.Web.base_url}/users/#{user.nickname}" assert expected_ap_id == User.ap_id(user) end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 065fb250a..48774dc69 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -170,7 +170,7 @@ test "rejects the subscription if it can't be accepted" do test "sign a text" do signed = Websub.sign("secret", "text") - assert signed == "B8392C23690CCF871F37EC270BE1582DEC57A503" + assert signed == "B8392C23690CCF871F37EC270BE1582DEC57A503" |> String.downcase signed = Websub.sign("secret", [["て"], ['す']]) end From 151da344beca98b2c007397cb0f8e47510bf747a Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 4 May 2017 09:54:22 +0200 Subject: [PATCH 86/88] Add debugging logs. --- lib/pleroma/web/websub/websub.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 0d0d19c88..c1532b6ce 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -44,6 +44,7 @@ def publish(topic, user, activity) do |> to_string signature = sign(sub.secret, response) + Logger.debug("Pushing to #{sub.callback}") HTTPoison.post(sub.callback, response, [ {"Content-Type", "application/atom+xml"}, {"X-Hub-Signature", "sha1=#{signature}"} From 5d9f3df714fa986367e105c2267324c8478ccf9c Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 4 May 2017 09:57:11 +0200 Subject: [PATCH 87/88] Just sign with an empty string if needed. --- lib/pleroma/web/websub/websub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index c1532b6ce..ba86db50e 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -43,7 +43,7 @@ def publish(topic, user, activity) do |> :xmerl.export_simple(:xmerl_xml) |> to_string - signature = sign(sub.secret, response) + signature = sign(sub.secret || "", response) Logger.debug("Pushing to #{sub.callback}") HTTPoison.post(sub.callback, response, [ {"Content-Type", "application/atom+xml"}, From c85998ab8a21f042ab57345a7baa9e1e27c308d1 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 4 May 2017 18:42:29 +0200 Subject: [PATCH 88/88] Parse incoming retweets. --- lib/pleroma/web/activity_pub/activity_pub.ex | 4 +- lib/pleroma/web/ostatus/ostatus.ex | 50 +++++++--- test/fixtures/share-gs.xml | 99 ++++++++++++++++++++ test/fixtures/share.xml | 54 +++++++++++ test/web/ostatus/ostatus_test.exs | 26 +++++ 5 files changed, 218 insertions(+), 15 deletions(-) create mode 100644 test/fixtures/share-gs.xml create mode 100644 test/fixtures/share.xml diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 5583a1f41..1816b2e66 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -177,7 +177,7 @@ def fetch_activities(recipients, opts \\ %{}) do |> Enum.reverse end - def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) do + def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object, local \\ true) do data = %{ "type" => "Announce", "actor" => ap_id, @@ -186,7 +186,7 @@ def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) "context" => object.data["context"] } - {:ok, activity} = insert(data) + {:ok, activity} = insert(data, local) announcements = [ap_id | (object.data["announcements"] || [])] |> Enum.uniq diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index f81751a25..2fab67663 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -25,20 +25,44 @@ def handle_incoming(xml_string) do activities = Enum.map(entries, fn (entry) -> {:xmlObj, :string, object_type } = :xmerl_xpath.string('string(/entry/activity:object-type[1])', entry) + {:xmlObj, :string, verb } = :xmerl_xpath.string('string(/entry/activity:verb[1])', entry) - case object_type do - 'http://activitystrea.ms/schema/1.0/note' -> - with {:ok, activity} <- handle_note(entry, doc), do: activity - 'http://activitystrea.ms/schema/1.0/comment' -> - with {:ok, activity} <- handle_note(entry, doc), do: activity + case verb do + 'http://activitystrea.ms/schema/1.0/share' -> + with {:ok, activity, retweeted_activity} <- handle_share(entry, doc), do: [activity, retweeted_activity] _ -> - Logger.error("Couldn't parse incoming document") - nil + case object_type do + 'http://activitystrea.ms/schema/1.0/note' -> + with {:ok, activity} <- handle_note(entry, doc), do: activity + 'http://activitystrea.ms/schema/1.0/comment' -> + with {:ok, activity} <- handle_note(entry, doc), do: activity + _ -> + Logger.error("Couldn't parse incoming document") + nil + end end end) {:ok, activities} end + def make_share(entry, doc, retweeted_activity) do + with {:ok, actor} <- find_make_or_update_user(doc), + %Object{} = object <- Object.get_cached_by_ap_id(retweeted_activity.data["object"]["id"]), + {:ok, activity, object} = ActivityPub.announce(actor, object, false) do + {:ok, activity} + end + end + + def handle_share(entry, doc) do + with [object] <- :xmerl_xpath.string('/entry/activity:object', entry), + {:ok, retweeted_activity} <- handle_note(object, object), + {:ok, activity} <- make_share(entry, doc, retweeted_activity) do + {:ok, activity, retweeted_activity} + else + e -> {:error, e} + end + end + def get_attachments(entry) do :xmerl_xpath.string('/entry/link[@rel="enclosure"]', entry) |> Enum.map(fn (enclosure) -> @@ -58,13 +82,13 @@ def get_attachments(entry) do end def handle_note(entry, doc \\ nil) do - content_html = string_from_xpath("/entry/content[1]", entry) + content_html = string_from_xpath("//content[1]", entry) [author] = :xmerl_xpath.string('//author[1]', doc) {:ok, actor} = find_make_or_update_user(author) - inReplyTo = string_from_xpath("/entry/thr:in-reply-to[1]/@ref", entry) + inReplyTo = string_from_xpath("//thr:in-reply-to[1]/@ref", entry) - context = (string_from_xpath("/entry/ostatus:conversation[1]", entry) || "") |> String.trim + context = (string_from_xpath("//ostatus:conversation[1]", entry) || "") |> String.trim attachments = get_attachments(entry) @@ -82,13 +106,13 @@ def handle_note(entry, doc \\ nil) do "https://www.w3.org/ns/activitystreams#Public" ] - mentions = :xmerl_xpath.string('/entry/link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]', entry) + mentions = :xmerl_xpath.string('//link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]', entry) |> Enum.map(fn(person) -> string_from_xpath("@href", person) end) to = to ++ mentions - date = string_from_xpath("/entry/published", entry) - id = string_from_xpath("/entry/id", entry) + date = string_from_xpath("//published", entry) + id = string_from_xpath("//id", entry) object = %{ "id" => id, diff --git a/test/fixtures/share-gs.xml b/test/fixtures/share-gs.xml new file mode 100644 index 000000000..ab5e488bd --- /dev/null +++ b/test/fixtures/share-gs.xml @@ -0,0 +1,99 @@ + + + GNU social + https://social.heldscal.la/api/statuses/user_timeline/23211.atom + lambadalambda timeline + Updates from lambadalambda on social.heldscal.la! + https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg + 2017-05-03T08:05:41+00:00 + + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + Call me Deacon Blues. + + + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + + Berlin + + + homepage + https://heldscal.la + true + + + + + + + + + + + + + tag:social.heldscal.la,2017-05-03:noticeId=2028428:objectType=note + lambadalambda repeated a notice by lain + RT @<a href="https://pleroma.soykaf.com/users/lain" class="h-card u-url p-nickname mention" title="Lain Iwakura">lain</a> Added returning the entries as xml... let's see if the mastodon hammering stops now. + + http://activitystrea.ms/schema/1.0/share + 2017-05-03T08:05:41+00:00 + 2017-05-03T08:05:41+00:00 + + http://activitystrea.ms/schema/1.0/activity + https://pleroma.soykaf.com/objects/4c1bda26-902e-4525-9fcd-b9fd44925193 + + Added returning the entries as xml... let's see if the mastodon hammering stops now. + + http://activitystrea.ms/schema/1.0/post + 2017-05-03T08:04:44+00:00 + 2017-05-03T08:04:44+00:00 + + http://activitystrea.ms/schema/1.0/person + https://pleroma.soykaf.com/users/lain + lain + Test account + + + + + + lain + Lain Iwakura + Test account + + + + http://activitystrea.ms/schema/1.0/note + https://pleroma.soykaf.com/objects/4c1bda26-902e-4525-9fcd-b9fd44925193 + New note by lain + Added returning the entries as xml... let's see if the mastodon hammering stops now. + + + + + https://pleroma.soykaf.com/contexts/ede39a2b-7cf3-4fa4-8ccd-cb97431bcc22 + + + https://pleroma.soykaf.com/users/lain/feed.atom + Lain Iwakura + + + https://social.heldscal.la/avatar/43188-96-20170429172422.jpeg + 2017-05-03T08:04:44+00:00 + + + + https://pleroma.soykaf.com/contexts/ede39a2b-7cf3-4fa4-8ccd-cb97431bcc22 + + + + + + diff --git a/test/fixtures/share.xml b/test/fixtures/share.xml new file mode 100644 index 000000000..e07b88680 --- /dev/null +++ b/test/fixtures/share.xml @@ -0,0 +1,54 @@ + + + tag:mastodon.social,2017-05-03:objectId=4934452:objectType=Status + 2017-05-03T08:21:09Z + 2017-05-03T08:21:09Z + lambadalambda shared a status by lain@pleroma.soykaf.com + + https://mastodon.social/users/lambadalambda + http://activitystrea.ms/schema/1.0/person + https://mastodon.social/users/lambadalambda + lambadalambda + lambadalambda@mastodon.social + + + + lambadalambda + Critical Value + public + + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + https://pleroma.soykaf.com/objects/4c1bda26-902e-4525-9fcd-b9fd44925193 + 2017-05-03T08:04:44Z + 2017-05-03T08:05:52Z + New status by lain@pleroma.soykaf.com + + https://pleroma.soykaf.com/users/lain + http://activitystrea.ms/schema/1.0/person + https://pleroma.soykaf.com/users/lain + lain + lain@pleroma.soykaf.com + Test account + + + + lain + Lain Iwakura + Test account + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + Added returning the entries as xml... let's see if the mastodon hammering stops now. + + public + + + Added returning the entries as xml... let's see if the mastodon hammering stops now. + + public + + + diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 94a735337..e85d7677c 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -70,6 +70,32 @@ test "handle incoming notes - GS, subscription, reply" do assert activity.data["object"]["inReplyTo"] == "tag:gs.archae.me,2017-04-30:noticeId=778260:objectType=note" end + test "handle incoming retweets - GS, subscription" do + incoming = File.read!("test/fixtures/share-gs.xml") + {:ok, [[activity, retweeted_activity]]} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Announce" + assert activity.data["actor"] == "https://social.heldscal.la/user/23211" + assert activity.data["object"] == retweeted_activity.data["object"]["id"] + refute activity.local + assert retweeted_activity.data["type"] == "Create" + assert retweeted_activity.data["actor"] == "https://pleroma.soykaf.com/users/lain" + refute retweeted_activity.local + end + + test "handle incoming retweets - Mastodon, salmon" do + incoming = File.read!("test/fixtures/share.xml") + {:ok, [[activity, retweeted_activity]]} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Announce" + assert activity.data["actor"] == "https://mastodon.social/users/lambadalambda" + assert activity.data["object"] == retweeted_activity.data["object"]["id"] + refute activity.local + assert retweeted_activity.data["type"] == "Create" + assert retweeted_activity.data["actor"] == "https://pleroma.soykaf.com/users/lain" + refute retweeted_activity.local + end + test "handle incoming replies" do incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming)