Merge branch 'revert-b122b6ff' into 'develop'

Revert "Merge branch 'features/hashtag-column' into 'develop'"

See merge request pleroma/pleroma!3222

CHANGELOG.md

@@ -8,7 +8,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Changed
 
-- **Breaking:** Changed storage of hashtags in plain-text to `object->hashtags`, run [`pleroma.database fill_old_hashtags` mix task](docs/administration/CLI_tasks/database.md) for old objects (works while pleroma is running).
 - Polls now always return a `voters_count`, even if they are single-choice.
 - Admin Emails: The ap id is used as the user link in emails now.
 - Improved registration workflow for email confirmation and account approval modes.
@@ -444,6 +443,7 @@ switched to a new configuration mechanism, however it was not officially removed
 - Static-FE: Fix remote posts not being sanitized
 
 ### Fixed
+=======
 - Rate limiter crashes when there is no explicitly specified ip in the config
 - 500 errors when no `Accept` header is present if Static-FE is enabled
 - Instance panel not being updated immediately due to wrong `Cache-Control` headers

docs/administration/CLI_tasks/database.md

@@ -91,18 +91,6 @@ Can be safely re-run
     mix pleroma.database fix_likes_collections
     ```
 
-## Fill hashtags for old objects
-
-Migrate hashags fields for old objects, from now to `months_limit` months.
-
-```sh tab="OTP"
-./bin/pleroma_ctl database fill_old_hashtags <months_limit>
-```
-
-```sh tab="From Source"
-mix pleroma.database fill_old_hashtags <months_limit>
-```
-
 ## Vacuum the database
 
 ### Analyze

lib/mix/tasks/pleroma/database.ex

@@ -128,57 +128,6 @@ def run(["fix_likes_collections"]) do
     |> Stream.run()
   end
 
-  def run(["fill_old_hashtags", month_limit]) do
-    import Ecto.Query
-
-    start_pleroma()
-
-    month_limit = String.to_integer(month_limit)
-
-    if month_limit < 1 do
-      shell_error("Invalid `month_limit` argument, needs to be greater than 1")
-    else
-      time_limit = DateTime.utc_now() |> Timex.shift(months: -month_limit)
-
-      from(
-        o in Object,
-        where: fragment("(?)->>'hashtags' is null", o.data),
-        where: fragment("(?)->>'tag' != '[]'", o.data),
-        where: o.inserted_at < ^time_limit,
-        select: %{id: o.id, tag: fragment("(?)->>'tag'", o.data)}
-      )
-      |> Pleroma.Repo.chunk_stream(200, :batches)
-      |> Stream.each(fn objects ->
-        Repo.transaction(fn ->
-          objects_first = objects |> List.first()
-          objects_last = objects |> List.last()
-
-          Logger.info(
-            "fill_old_hashtags: #{objects_first.id} (#{objects_first.inserted_at}) -- #{
-              objects_last.id
-            } (#{objects_last.inserted_at})"
-          )
-
-          objects
-          |> Enum.map(fn object ->
-            tags =
-              object.tag
-              |> Jason.decode!()
-              |> Enum.filter(&is_bitstring(&1))
-
-            Object
-            |> where([o], o.id == ^object.id)
-            |> update([o],
-              set: [data: fragment("safe_jsonb_set(?, '{hashtags}', ?, true)", o.data, ^tags)]
-            )
-            |> Repo.update_all([], timeout: :infinity)
-          end)
-        end)
-      end)
-      |> Stream.run()
-    end
-  end
-
   def run(["vacuum", args]) do
     start_pleroma()
 

lib/pleroma/activity/ir/topics.ex

@@ -48,12 +48,14 @@ defp item_creation_tags(tags, _, _) do
     tags
   end
 
-  defp hashtags_to_topics(object) do
+  defp hashtags_to_topics(%{data: %{"tag" => tags}}) do
-    object
+    tags
-    |> Object.hashtags()
+    |> Enum.filter(&is_bitstring(&1))
     |> Enum.map(fn tag -> "hashtag:" <> tag end)
   end
 
+  defp hashtags_to_topics(_), do: []
+
   defp remote_topics(%{local: true}), do: []
 
   defp remote_topics(%{actor: actor}) when is_binary(actor),

lib/pleroma/constants.ex

@@ -18,8 +18,7 @@ defmodule Pleroma.Constants do
       "emoji",
       "context_id",
       "deleted_activity_id",
-      "pleroma_internal",
+      "pleroma_internal"
-      "hashtags"
     ]
   )
 

lib/pleroma/object.ex

@@ -346,8 +346,4 @@ def replies(object, opts \\ []) do
 
   def self_replies(object, opts \\ []),
     do: replies(object, Keyword.put(opts, :self_only, true))
-
-  def hashtags(%Object{data: %{"hashtags" => hashtags}}), do: hashtags || []
-  def hashtags(%Object{data: %{"tag" => tags}}), do: Enum.filter(tags, &is_bitstring(&1))
-  def hashtags(_), do: []
 end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -669,7 +669,7 @@ defp restrict_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true})
   defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do
     from(
       [_activity, object] in query,
-      where: fragment("not (?)->'hashtags' \\?| (?)", object.data, ^tag_reject)
+      where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
     )
   end
 
@@ -682,7 +682,7 @@ defp restrict_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do
   defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
     from(
       [_activity, object] in query,
-      where: fragment("(?)->'hashtags' \\?& (?)", object.data, ^tag_all)
+      where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
     )
   end
 
@@ -695,14 +695,14 @@ defp restrict_tag(_query, %{tag: _tag, skip_preload: true}) do
   defp restrict_tag(query, %{tag: tag}) when is_list(tag) do
     from(
       [_activity, object] in query,
-      where: fragment("(?)->'hashtags' \\?| (?)", object.data, ^tag)
+      where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag)
     )
   end
 
   defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do
     from(
       [_activity, object] in query,
-      where: fragment("(?)->'hashtags' \\? (?)", object.data, ^tag)
+      where: fragment("(?)->'tag' \\? (?)", object.data, ^tag)
     )
   end
 

lib/pleroma/web/activity_pub/mrf/simple_policy.ex

@@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
 
   alias Pleroma.Config
   alias Pleroma.FollowingRelationship
-  alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.MRF
 
@@ -75,11 +74,9 @@ defp check_media_nsfw(
 
     object =
       if MRF.subdomain_match?(media_nsfw, actor_host) do
-        child_object =
+        tags = (child_object["tag"] || []) ++ ["nsfw"]
-          child_object
+        child_object = Map.put(child_object, "tag", tags)
-          |> Map.put("hashtags", Object.hashtags(%Object{data: child_object}) ++ ["nsfw"])
+        child_object = Map.put(child_object, "sensitive", true)
-          |> Map.put("sensitive", true)
-
         Map.put(object, "object", child_object)
       else
         object

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -312,15 +312,16 @@ def fix_emoji(%{"tag" => %{"type" => "Emoji"} = tag} = object) do
   def fix_emoji(object), do: object
 
   def fix_tag(%{"tag" => tag} = object) when is_list(tag) do
-    hashtags =
+    tags =
       tag
       |> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end)
-      |> Enum.map(fn
+      |> Enum.map(fn %{"name" => name} ->
-        %{"name" => "#" <> hashtag} -> String.downcase(hashtag)
+        name
-        %{"name" => hashtag} -> String.downcase(hashtag)
+        |> String.slice(1..-1)
+        |> String.downcase()
       end)
 
-    Map.put(object, "hashtags", hashtags)
+    Map.put(object, "tag", tag ++ tags)
   end
 
   def fix_tag(%{"tag" => %{} = tag} = object) do
@@ -863,18 +864,23 @@ def maybe_fix_object_url(%{"object" => object} = data) when is_binary(object) do
   def maybe_fix_object_url(data), do: data
 
   def add_hashtags(object) do
-    hashtags =
+    tags =
-      %Object{data: object}
+      (object["tag"] || [])
-      |> Object.hashtags()
+      |> Enum.map(fn
-      |> Enum.map(fn tag ->
+        # Expand internal representation tags into AS2 tags.
-        %{
+        tag when is_binary(tag) ->
-          "href" => Pleroma.Web.Endpoint.url() <> "/tags/#{tag}",
+          %{
-          "name" => "##{tag}",
+            "href" => Pleroma.Web.Endpoint.url() <> "/tags/#{tag}",
-          "type" => "Hashtag"
+            "name" => "##{tag}",
-        }
+            "type" => "Hashtag"
+          }
+
+        # Do not process tags which are already AS2 tag objects.
+        tag when is_map(tag) ->
+          tag
       end)
 
-    Map.put(object, "tag", hashtags ++ (object["tag"] || []))
+    Map.put(object, "tag", tags)
   end
 
   # TODO These should be added on our side on insertion, it doesn't make much
@@ -930,7 +936,7 @@ def set_sensitive(%{"sensitive" => _} = object) do
   end
 
   def set_sensitive(object) do
-    tags = object["hashtags"] || object["tag"] || []
+    tags = object["tag"] || []
     Map.put(object, "sensitive", "nsfw" in tags)
   end
 

lib/pleroma/web/common_api/utils.ex

@@ -310,16 +310,7 @@ def make_note_data(%ActivityDraft{} = draft) do
       "context" => draft.context,
       "attachment" => draft.attachments,
       "actor" => draft.user.ap_id,
-      "tag" => Enum.filter(draft.tags, &is_map(&1)) |> Enum.uniq(),
+      "tag" => Keyword.values(draft.tags) |> Enum.uniq()
-      "hashtags" =>
-        draft.tags
-        |> Enum.reduce([], fn
-          # Why so many formats
-          {:name, x}, acc -> if is_bitstring(x), do: [x | acc], else: acc
-          {"#" <> _, x}, acc -> if is_bitstring(x), do: [x | acc], else: acc
-          x, acc -> if is_bitstring(x), do: [x | acc], else: acc
-        end)
-        |> Enum.uniq()
     }
     |> add_in_reply_to(draft.in_reply_to)
     |> Map.merge(draft.extra)

lib/pleroma/web/feed/feed_view.ex

@@ -32,7 +32,6 @@ def prepare_activity(activity, opts \\ []) do
 
     %{
       activity: activity,
-      object: object,
       data: Map.get(object, :data),
       actor: actor
     }

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -347,7 +347,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
       media_attachments: attachments,
       poll: render(PollView, "show.json", object: object, for: opts[:for]),
       mentions: mentions,
-      tags: build_tags(Object.hashtags(object)),
+      tags: build_tags(tags),
       application: %{
         name: "Web",
         website: nil

lib/pleroma/web/templates/feed/feed/_activity.atom.eex

@@ -22,8 +22,8 @@
     <link type="text/html" href='<%= @data["external_url"] %>' rel="alternate"/>
   <% end %>
 
-  <%= for hashtag <- Object.hashtags(@object) do %>
+  <%= for tag <- @data["tag"] || [] do %>
-    <category term="<%= hashtag %>"></category>
+    <category term="<%= tag %>"></category>
   <% end %>
 
   <%= for attachment <- @data["attachment"] || [] do %>

lib/pleroma/web/templates/feed/feed/_activity.rss.eex

@@ -21,8 +21,8 @@
     <link><%= @data["external_url"] %></link>
   <% end %>
 
-  <%= for hashtag <- Object.hashtags(@object) do %>
+  <%= for tag <- @data["tag"] || [] do %>
-    <category term="<%= hashtag %>"></category>
+    <category term="<%= tag %>"></category>
   <% end %>
 
   <%= for attachment <- @data["attachment"] || [] do %>

lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex

@@ -41,8 +41,8 @@
       <% end %>
     <% end %>
 
-    <%= for hashtag <- Object.hashtags(@object) do %>
+    <%= for tag <- @data["tag"] || [] do %>
-      <category term="<%= hashtag %>"></category>
+      <category term="<%= tag %>"></category>
     <% end %>
 
     <%= for {emoji, file} <- @data["emoji"] || %{} do %>

priv/repo/migrations/20200731165800_add_hashtags_index_to_objects.exs

@@ -1,11 +0,0 @@
-defmodule Pleroma.Repo.Migrations.AddHashtagsIndexToObjects do
-  use Ecto.Migration
-
-  def change do
-    drop_if_exists(index(:objects, ["(data->'tag')"], using: :gin, name: :objects_tags))
-
-    create_if_not_exists(
-      index(:objects, ["(data->'hashtags')"], using: :gin, name: :objects_hashtags)
-    )
-  end
-end

test/pleroma/activity/ir/topics_test.exs

@@ -78,7 +78,7 @@ test "with no attachments doesn't produce public:media topics", %{activity: acti
     end
 
     test "converts tags to hash tags", %{activity: %{object: %{data: data} = object} = activity} do
-      tagged_data = Map.put(data, "hashtags", ["foo", "bar"])
+      tagged_data = Map.put(data, "tag", ["foo", "bar"])
       activity = %{activity | object: %{object | data: tagged_data}}
 
       topics = Topics.get_activity_topics(activity)

test/pleroma/web/activity_pub/mrf/simple_policy_test.exs

@@ -78,7 +78,7 @@ test "has a matching host" do
       assert SimplePolicy.filter(media_message) ==
                {:ok,
                 media_message
-                |> put_in(["object", "hashtags"], ["foo", "nsfw"])
+                |> put_in(["object", "tag"], ["foo", "nsfw"])
                 |> put_in(["object", "sensitive"], true)}
 
       assert SimplePolicy.filter(local_message) == {:ok, local_message}
@@ -92,7 +92,7 @@ test "match with wildcard domain" do
       assert SimplePolicy.filter(media_message) ==
                {:ok,
                 media_message
-                |> put_in(["object", "hashtags"], ["foo", "nsfw"])
+                |> put_in(["object", "tag"], ["foo", "nsfw"])
                 |> put_in(["object", "sensitive"], true)}
 
       assert SimplePolicy.filter(local_message) == {:ok, local_message}
@@ -105,7 +105,7 @@ defp build_media_message do
       "type" => "Create",
       "object" => %{
         "attachment" => [%{}],
-        "hashtags" => ["foo"],
+        "tag" => ["foo"],
         "sensitive" => false
       }
     }

test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs

@@ -39,7 +39,7 @@ test "it works for incoming notices with tag not being an array (kroeg)" do
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
       object = Object.normalize(data["object"])
 
-      assert ["test"] == object.data["hashtags"]
+      assert "test" in object.data["tag"]
     end
 
     test "it cleans up incoming notices which are not really DMs" do
@@ -220,7 +220,7 @@ test "it works for incoming notices with hashtags" do
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
       object = Object.normalize(data["object"])
 
-      assert object.data["hashtags"] == ["moo"]
+      assert Enum.at(object.data["tag"], 2) == "moo"
     end
 
     test "it works for incoming notices with contentMap" do

test/pleroma/web/activity_pub/transmogrifier_test.exs

@@ -204,37 +204,30 @@ test "it strips internal fields" do
 
       {:ok, activity} = CommonAPI.post(user, %{status: "#2hu :firefox:"})
 
-      {:ok, %{"object" => modified_object}} = Transmogrifier.prepare_outgoing(activity.data)
+      {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
 
-      assert [
+      assert length(modified["object"]["tag"]) == 2
-               %{"name" => "#2hu", "type" => "Hashtag"},
-               %{"name" => ":firefox:", "type" => "Emoji"}
-             ] = modified_object["tag"]
 
-      refute Map.has_key?(modified_object, "hashtags")
+      assert is_nil(modified["object"]["emoji"])
-      refute Map.has_key?(modified_object, "emoji")
+      assert is_nil(modified["object"]["like_count"])
-      refute Map.has_key?(modified_object, "like_count")
+      assert is_nil(modified["object"]["announcements"])
-      refute Map.has_key?(modified_object, "announcements")
+      assert is_nil(modified["object"]["announcement_count"])
-      refute Map.has_key?(modified_object, "announcement_count")
+      assert is_nil(modified["object"]["context_id"])
-      refute Map.has_key?(modified_object, "context_id")
     end
 
     test "it strips internal fields of article" do
       activity = insert(:article_activity)
 
-      {:ok, %{"object" => modified_object}} = Transmogrifier.prepare_outgoing(activity.data)
+      {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
 
-      assert [
+      assert length(modified["object"]["tag"]) == 2
-               %{"name" => "#2hu", "type" => "Hashtag"},
-               %{"name" => ":2hu:", "type" => "Emoji"}
-             ] = modified_object["tag"]
 
-      refute Map.has_key?(modified_object, "hashtags")
+      assert is_nil(modified["object"]["emoji"])
-      refute Map.has_key?(modified_object, "emoji")
+      assert is_nil(modified["object"]["like_count"])
-      refute Map.has_key?(modified_object, "like_count")
+      assert is_nil(modified["object"]["announcements"])
-      refute Map.has_key?(modified_object, "announcements")
+      assert is_nil(modified["object"]["announcement_count"])
-      refute Map.has_key?(modified_object, "announcement_count")
+      assert is_nil(modified["object"]["context_id"])
-      refute Map.has_key?(modified_object, "context_id")
+      assert is_nil(modified["object"]["likes"])
     end
 
     test "the directMessage flag is present" do

test/pleroma/web/common_api/utils_test.exs

@@ -591,8 +591,7 @@ test "returns note data" do
                "context" => "2hu",
                "sensitive" => false,
                "summary" => "test summary",
-               "hashtags" => ["jimm"],
+               "tag" => ["jimm"],
-               "tag" => [],
                "to" => [user2.ap_id],
                "type" => "Note",
                "custom_tag" => "test"

test/pleroma/web/common_api_test.exs

@@ -493,8 +493,7 @@ test "it de-duplicates tags" do
 
     object = Object.normalize(activity)
 
-    assert object.data["tag"] == []
+    assert object.data["tag"] == ["2hu"]
-    assert object.data["hashtags"] == ["2hu"]
   end
 
   test "it adds emoji in the object" do

test/pleroma/web/mastodon_api/views/status_view_test.exs

@@ -262,8 +262,8 @@ test "a note activity" do
       mentions: [],
       tags: [
         %{
-          name: "2hu",
+          name: "#{object_data["tag"]}",
-          url: "/tag/2hu"
+          url: "/tag/#{object_data["tag"]}"
         }
       ],
       application: %{

test/support/factory.ex

@@ -93,7 +93,7 @@ def note_factory(attrs \\ %{}) do
       "like_count" => 0,
       "context" => "2hu",
       "summary" => "2hu",
-      "hashtags" => ["2hu"],
+      "tag" => ["2hu"],
       "emoji" => %{
         "2hu" => "corndog.png"
       }