Merge branch 'develop' of ssh.gitgud.io:lambadalambda/pleroma into develop

TODO.txt

@@ -5,5 +5,8 @@ Unliking:
 WEBSUB:
 
 - Add unsubscription
-- Add periodical renewal
+
+OSTATUS:
+
+- Save and output 'updated'
 

lib/pleroma/web/ostatus/activity_representer.ex

@@ -28,10 +28,8 @@ def to_simple_form(activity, user, with_author \\ false)
   def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do
     h = fn(str) -> [to_charlist(str)] end
 
-    updated_at = activity.updated_at
+    updated_at = activity.data["published"]
-    |> NaiveDateTime.to_iso8601
+    inserted_at = activity.data["published"]
-    inserted_at = activity.inserted_at
-    |> NaiveDateTime.to_iso8601
 
     attachments = Enum.map(activity.data["object"]["attachment"] || [], fn(attachment) ->
       url = hd(attachment["url"])
@@ -62,10 +60,8 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user,
   def to_simple_form(%{data: %{"type" => "Like"}} = activity, user, with_author) do
     h = fn(str) -> [to_charlist(str)] end
 
-    updated_at = activity.updated_at
+    updated_at = activity.data["published"]
-    |> NaiveDateTime.to_iso8601
+    inserted_at = activity.data["published"]
-    inserted_at = activity.inserted_at
-    |> NaiveDateTime.to_iso8601
 
     in_reply_to = get_in_reply_to(activity.data)
     author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
@@ -92,10 +88,8 @@ def to_simple_form(%{data: %{"type" => "Like"}} = activity, user, with_author) d
   def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_author) do
     h = fn(str) -> [to_charlist(str)] end
 
-    updated_at = activity.updated_at
+    updated_at = activity.data["published"]
-    |> NaiveDateTime.to_iso8601
+    inserted_at = activity.data["published"]
-    inserted_at = activity.inserted_at
-    |> NaiveDateTime.to_iso8601
 
     in_reply_to = get_in_reply_to(activity.data)
     author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
@@ -124,10 +118,8 @@ def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_autho
   def to_simple_form(%{data: %{"type" => "Follow"}} = activity, user, with_author) do
     h = fn(str) -> [to_charlist(str)] end
 
-    updated_at = activity.updated_at
+    updated_at = activity.data["published"]
-    |> NaiveDateTime.to_iso8601
+    inserted_at = activity.data["published"]
-    inserted_at = activity.inserted_at
-    |> NaiveDateTime.to_iso8601
 
     author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
 
@@ -153,10 +145,8 @@ def to_simple_form(%{data: %{"type" => "Follow"}} = activity, user, with_author)
   def to_simple_form(%{data: %{"type" => "Undo"}} = activity, user, with_author) do
     h = fn(str) -> [to_charlist(str)] end
 
-    updated_at = activity.updated_at
+    updated_at = activity.data["published"]
-    |> NaiveDateTime.to_iso8601
+    inserted_at = activity.data["published"]
-    inserted_at = activity.inserted_at
-    |> NaiveDateTime.to_iso8601
 
     author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
     follow_activity = Activity.get_by_ap_id(activity.data["object"])

lib/pleroma/web/ostatus/ostatus.ex

@@ -152,16 +152,30 @@ def get_tags(entry) do
     |> Enum.map(fn (category) -> string_from_xpath("/category/@term", category) end)
   end
 
+  def maybe_update(doc, user) do
+    old_data = %{
+      avatar: user.avatar,
+      bio: user.bio,
+      name: user.name
+    }
+
+    with false <- user.local,
+         avatar <- make_avatar_object(doc),
+         bio when not is_nil(bio) <- string_from_xpath("//author[1]/summary", doc),
+         name when not is_nil(name) <- string_from_xpath("//author[1]/poco:displayName", doc),
+         new_data <- %{avatar: avatar, name: name, bio: bio},
+         false <- new_data == old_data do
+      change = Ecto.Changeset.change(user, new_data)
+      Repo.update(change)
+    else e ->
+      {:ok, user}
+    end
+  end
+
   def find_make_or_update_user(doc) do
     uri = string_from_xpath("//author/uri[1]", doc)
     with {:ok, user} <- find_or_make_user(uri) do
-      avatar = make_avatar_object(doc)
+      maybe_update(doc, user)
-      if !user.local && user.avatar != avatar do
-        change = Ecto.Changeset.change(user, %{avatar: avatar})
-        Repo.update(change)
-      else
-        {:ok, user}
-      end
     end
   end
 
@@ -185,7 +199,8 @@ def make_user(uri) do
         nickname: info["nickname"] <> "@" <> info["host"],
         ap_id: info["uri"],
         info: info,
-        avatar: info["avatar"]
+        avatar: info["avatar"],
+        bio: info["bio"]
       }
       with %User{} = user <- User.get_by_ap_id(data.ap_id) do
         {:ok, user}
@@ -250,9 +265,9 @@ def get_atom_url(body) do
 
   def fetch_activity_from_html_url(url) do
     Logger.debug("Trying to fetch #{url}")
-    with {:ok, %{body: body}} <- @httpoison.get(url, [], follow_redirect: true),
+    with {:ok, %{body: body}} <- @httpoison.get(url, [], follow_redirect: true, timeout: 10000, recv_timeout: 20000),
          {:ok, atom_url} <- get_atom_url(body),
-         {:ok, %{status_code: code, body: body}} when code in 200..299 <- @httpoison.get(atom_url, [], follow_redirect: true) do
+         {:ok, %{status_code: code, body: body}} when code in 200..299 <- @httpoison.get(atom_url, [], follow_redirect: true, timeout: 10000, recv_timeout: 20000) do
       Logger.debug("Got document from #{url}, handling...")
       handle_incoming(body)
     else e -> Logger.debug("Couldn't get #{url}: #{inspect(e)}")

lib/pleroma/web/ostatus/ostatus_controller.ex

@@ -47,7 +47,10 @@ def object(conn, %{"uuid" => uuid}) do
     with id <- o_status_url(conn, :object, uuid),
          %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id),
          %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
-      represent_activity(conn, activity, user)
+      case get_format(conn) do
+        "html" -> redirect(conn, to: "/notice/#{activity.id}")
+        _ -> represent_activity(conn, activity, user)
+      end
     end
   end
 
@@ -55,7 +58,10 @@ def activity(conn, %{"uuid" => uuid}) do
     with id <- o_status_url(conn, :activity, uuid),
          %Activity{} = activity <- Activity.get_by_ap_id(id),
          %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
-      represent_activity(conn, activity, user)
+      case get_format(conn) do
+        "html" -> redirect(conn, to: "/notice/#{activity.id}")
+        _ -> represent_activity(conn, activity, user)
+      end
     end
   end
 

lib/pleroma/web/router.ex

@@ -70,7 +70,7 @@ def user_fetcher(username) do
   end
 
   pipeline :ostatus do
-    plug :accepts, ["xml", "atom"]
+    plug :accepts, ["xml", "atom", "html"]
   end
 
   scope "/", Pleroma.Web do
@@ -97,7 +97,6 @@ def user_fetcher(username) do
   scope "/", Fallback do
     get "/*path", RedirectController, :redirector
   end
-
 end
 
 defmodule Fallback.RedirectController do

lib/pleroma/web/twitter_api/utils.ex

@@ -38,7 +38,8 @@ def add_user_links(text, mentions) do
     end)
 
     Enum.reduce(mentions, step_one, fn ({match, %User{ap_id: ap_id}, uuid}, text) ->
-      String.replace(text, uuid, "<a href='#{ap_id}'>#{match}</a>")
+      short_match = String.split(match, "@") |> tl() |> hd()
+      String.replace(text, uuid, "<a href='#{ap_id}'>@#{short_match}</a>")
     end)
   end
 

lib/pleroma/web/websub/websub.ex

@@ -156,6 +156,7 @@ def gather_feed_data(topic, getter \\ &@httpoison.get/1) do
       preferredUsername = XML.string_from_xpath("/feed/author[1]/poco:preferredUsername", doc)
       displayName = XML.string_from_xpath("/feed/author[1]/poco:displayName", doc)
       avatar = OStatus.make_avatar_object(doc)
+      bio = XML.string_from_xpath("/feed/author[1]/summary", doc)
 
       {:ok, %{
         "uri" => uri,
@@ -163,7 +164,8 @@ def gather_feed_data(topic, getter \\ &@httpoison.get/1) do
         "nickname" => preferredUsername || name,
         "name" => displayName || name,
         "host" => URI.parse(uri).host,
-        "avatar" => avatar
+        "avatar" => avatar,
+        "bio" => bio
       }}
     else e ->
       {:error, e}

priv/repo/migrations/20170529093232_longer_bios.exs

@@ -0,0 +1,9 @@
+defmodule Pleroma.Repo.Migrations.LongerBios do
+  use Ecto.Migration
+
+  def change do
+    alter table(:users) do
+      modify :bio, :text
+    end
+  end
+end

test/fixtures/lambadalambda.atom

@@ -11,7 +11,7 @@
     <uri>https://mastodon.social/users/lambadalambda</uri>
     <name>lambadalambda</name>
     <email>lambadalambda@mastodon.social</email>
-    <summary></summary>
+    <summary>a cool dude.</summary>
     <link rel="alternate" type="text/html" href="https://mastodon.social/@lambadalambda"/>
     <link rel="avatar" type="image/gif" media:width="120" media:height="120" href="https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif?1492379244"/>
     <link rel="header" type="" media:width="700" media:height="335" href="/headers/original/missing.png"/>

test/support/factory.ex

@@ -22,7 +22,7 @@ def note_factory do
       "id" => Pleroma.Web.ActivityPub.Utils.generate_object_id,
       "actor" => user.ap_id,
       "to" => ["https://www.w3.org/ns/activitystreams#Public"],
-      "published_at" => DateTime.utc_now() |> DateTime.to_iso8601,
+      "published" => DateTime.utc_now() |> DateTime.to_iso8601,
       "likes" => [],
       "like_count" => 0,
       "context" => "2hu",
@@ -42,7 +42,7 @@ def note_activity_factory do
       "actor" => note.data["actor"],
       "to" => note.data["to"],
       "object" => note.data,
-      "published_at" => DateTime.utc_now() |> DateTime.to_iso8601,
+      "published" => DateTime.utc_now() |> DateTime.to_iso8601,
       "context" => note.data["context"]
     }
 

test/web/ostatus/activity_representer_test.exs

@@ -9,10 +9,6 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenterTest do
 
   test "a note activity" do
     note_activity = insert(:note_activity)
-    updated_at = note_activity.updated_at
-    |> NaiveDateTime.to_iso8601
-    inserted_at = note_activity.inserted_at
-    |> NaiveDateTime.to_iso8601
 
     user = User.get_cached_by_ap_id(note_activity.data["actor"])
 
@@ -22,8 +18,8 @@ test "a note activity" do
     <id>#{note_activity.data["object"]["id"]}</id>
     <title>New note by #{user.nickname}</title>
     <content type="html">#{note_activity.data["object"]["content"]}</content>
-    <published>#{inserted_at}</published>
+    <published>#{note_activity.data["published"]}</published>
-    <updated>#{updated_at}</updated>
+    <updated>#{note_activity.data["published"]}</updated>
     <ostatus:conversation>#{note_activity.data["context"]}</ostatus:conversation>
     <link ref="#{note_activity.data["context"]}" rel="ostatus:conversation" />
     <link type="application/atom+xml" href="#{note_activity.data["object"]["id"]}" rel="self" />
@@ -47,11 +43,6 @@ test "a reply note" do
     data = %{answer.data | "object" => object}
     answer = %{answer | data: data}
 
-    updated_at = answer.updated_at
-    |> NaiveDateTime.to_iso8601
-    inserted_at = answer.inserted_at
-    |> NaiveDateTime.to_iso8601
-
     user = User.get_cached_by_ap_id(answer.data["actor"])
 
     expected = """
@@ -60,8 +51,8 @@ test "a reply note" do
     <id>#{answer.data["object"]["id"]}</id>
     <title>New note by #{user.nickname}</title>
     <content type="html">#{answer.data["object"]["content"]}</content>
-    <published>#{inserted_at}</published>
+    <published>#{answer.data["published"]}</published>
-    <updated>#{updated_at}</updated>
+    <updated>#{answer.data["published"]}</updated>
     <ostatus:conversation>#{answer.data["context"]}</ostatus:conversation>
     <link ref="#{answer.data["context"]}" rel="ostatus:conversation" />
     <link type="application/atom+xml" href="#{answer.data["object"]["id"]}" rel="self" />
@@ -92,19 +83,14 @@ test "an announce activity" do
     |> :xmerl.export_simple_content(:xmerl_xml)
     |> to_string
 
-    updated_at = announce.updated_at
-    |> NaiveDateTime.to_iso8601
-    inserted_at = announce.inserted_at
-    |> NaiveDateTime.to_iso8601
-
     expected = """
     <activity:object-type>http://activitystrea.ms/schema/1.0/activity</activity:object-type>
     <activity:verb>http://activitystrea.ms/schema/1.0/share</activity:verb>
     <id>#{announce.data["id"]}</id>
     <title>#{user.nickname} repeated a notice</title>
     <content type="html">RT #{note.data["object"]["content"]}</content>
-    <published>#{inserted_at}</published>
+    <published>#{announce.data["published"]}</published>
-    <updated>#{updated_at}</updated>
+    <updated>#{announce.data["published"]}</updated>
     <ostatus:conversation>#{announce.data["context"]}</ostatus:conversation>
     <link ref="#{announce.data["context"]}" rel="ostatus:conversation" />
     <link rel="self" type="application/atom+xml" href="#{announce.data["id"]}"/>
@@ -126,12 +112,6 @@ test "a like activity" do
     user = insert(:user)
     {:ok, like, _note} = ActivityPub.like(user, note)
 
-    # TODO: Are these the correct dates?
-    updated_at = like.updated_at
-    |> NaiveDateTime.to_iso8601
-    inserted_at = like.inserted_at
-    |> NaiveDateTime.to_iso8601
-
     tuple = ActivityRepresenter.to_simple_form(like, user)
     refute is_nil(tuple)
 
@@ -142,8 +122,8 @@ test "a like activity" do
     <id>#{like.data["id"]}</id>
     <title>New favorite by #{user.nickname}</title>
     <content type="html">#{user.nickname} favorited something</content>
-    <published>#{inserted_at}</published>
+    <published>#{like.data["published"]}</published>
-    <updated>#{updated_at}</updated>
+    <updated>#{like.data["published"]}</updated>
     <activity:object>
       <activity:object-type>http://activitystrea.ms/schema/1.0/note</activity:object-type>
       <id>#{note.data["id"]}</id>
@@ -168,13 +148,6 @@ test "a follow activity" do
           "to" => [followed.ap_id]
     })
 
-
-    # TODO: Are these the correct dates?
-    updated_at = activity.updated_at
-    |> NaiveDateTime.to_iso8601
-    inserted_at = activity.inserted_at
-    |> NaiveDateTime.to_iso8601
-
     tuple = ActivityRepresenter.to_simple_form(activity, follower)
 
     refute is_nil(tuple)
@@ -187,8 +160,8 @@ test "a follow activity" do
     <id>#{activity.data["id"]}</id>
     <title>#{follower.nickname} started following #{activity.data["object"]}</title>
     <content type="html"> #{follower.nickname} started following #{activity.data["object"]}</content>
-    <published>#{inserted_at}</published>
+    <published>#{activity.data["published"]}</published>
-    <updated>#{updated_at}</updated>
+    <updated>#{activity.data["published"]}</updated>
     <activity:object>
       <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
       <id>#{activity.data["object"]}</id>
@@ -207,12 +180,6 @@ test "an unfollow activity" do
     {:ok, _activity} = ActivityPub.follow(follower, followed)
     {:ok, activity} = ActivityPub.unfollow(follower, followed)
 
-    # TODO: Are these the correct dates?
-    updated_at = activity.updated_at
-    |> NaiveDateTime.to_iso8601
-    inserted_at = activity.inserted_at
-    |> NaiveDateTime.to_iso8601
-
     tuple = ActivityRepresenter.to_simple_form(activity, follower)
 
     refute is_nil(tuple)
@@ -225,8 +192,8 @@ test "an unfollow activity" do
     <id>#{activity.data["id"]}</id>
     <title>#{follower.nickname} stopped following #{followed.ap_id}</title>
     <content type="html"> #{follower.nickname} stopped following #{followed.ap_id}</content>
-    <published>#{inserted_at}</published>
+    <published>#{activity.data["published"]}</published>
-    <updated>#{updated_at}</updated>
+    <updated>#{activity.data["published"]}</updated>
     <activity:object>
       <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
       <id>#{followed.ap_id}</id>

test/web/ostatus/ostatus_test.exs

@@ -233,6 +233,7 @@ test "tries to use the information in poco fields" do
       assert user.local == false
       assert user.info["uri"] == uri
       assert user.ap_id == uri
+      assert user.bio == "Call me Deacon Blues."
       assert user.avatar["type"] == "Image"
 
       {:ok, user_again} = OStatus.find_or_make_user(uri)
@@ -244,7 +245,9 @@ test "find_make_or_update_user takes an author element and returns an updated us
       uri = "https://social.heldscal.la/user/23211"
 
       {:ok, user} = OStatus.find_or_make_user(uri)
-      change = Ecto.Changeset.change(user, %{avatar: nil})
+      old_name = user.name
+      old_bio = user.bio
+      change = Ecto.Changeset.change(user, %{avatar: nil, bio: nil, old_name: nil})
 
       {:ok, user} = Repo.update(change)
       refute user.avatar
@@ -253,6 +256,8 @@ test "find_make_or_update_user takes an author element and returns an updated us
       [author] = :xmerl_xpath.string('//author[1]', doc)
       {:ok, user} = OStatus.find_make_or_update_user(author)
       assert user.avatar["type"] == "Image"
+      assert user.name == old_name
+      assert user.bio == old_bio
 
       {:ok, user_again} = OStatus.find_make_or_update_user(author)
       assert user_again == user
@@ -277,6 +282,7 @@ test "it returns user info in a hash" do
         "uri" => "https://social.heldscal.la/user/29191",
         "host" => "social.heldscal.la",
         "fqn" => user,
+        "bio" => "cofe",
         "avatar" => %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]}
       }
       assert data == expected
@@ -299,6 +305,7 @@ test "it works with the uri" do
         "uri" => "https://social.heldscal.la/user/29191",
         "host" => "social.heldscal.la",
         "fqn" => user,
+        "bio" => "cofe",
         "avatar" => %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]}
       }
       assert data == expected

test/web/twitter_api/twitter_api_test.exs

@@ -263,7 +263,7 @@ test "it adds user links to an existing text" do
     archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"})
 
     mentions = TwitterAPI.parse_mentions(text)
-    expected_text = "<a href='#{gsimg.ap_id}'>@gsimg</a> According to <a href='#{archaeme.ap_id}'>@archaeme</a>, that is @daggsy. Also hello <a href='#{archaeme_remote.ap_id}'>@archaeme@archae.me</a>"
+    expected_text = "<a href='#{gsimg.ap_id}'>@gsimg</a> According to <a href='#{archaeme.ap_id}'>@archaeme</a>, that is @daggsy. Also hello <a href='#{archaeme_remote.ap_id}'>@archaeme</a>"
 
     assert Utils.add_user_links(text, mentions) == expected_text
   end

test/web/websub/websub_test.exs

@@ -120,6 +120,7 @@ test "discovers the hub and canonical url" do
       "nickname" => "lambadalambda",
       "name" => "Critical Value",
       "host" => "mastodon.social",
+      "bio" => "a cool dude.",
       "avatar" => %{"type" => "Image", "url" => [%{"href" => "https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif?1492379244", "mediaType" => "image/gif", "type" => "Link"}]}
     }