From 20b8b8774345d07537687ad5dd2fdb6f809e684f Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 9 Nov 2017 12:33:38 +0100 Subject: [PATCH 1/6] Update in chunks. --- .../20171109091239_add_actor_to_activity.exs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/priv/repo/migrations/20171109091239_add_actor_to_activity.exs b/priv/repo/migrations/20171109091239_add_actor_to_activity.exs index bac53972c..c04922c76 100644 --- a/priv/repo/migrations/20171109091239_add_actor_to_activity.exs +++ b/priv/repo/migrations/20171109091239_add_actor_to_activity.exs @@ -1,6 +1,8 @@ defmodule Pleroma.Repo.Migrations.AddActorToActivity do use Ecto.Migration + alias Pleroma.{Repo, Activity} + @disable_ddl_transaction true def up do @@ -8,9 +10,18 @@ def up do add :actor, :string end - execute """ - update activities set actor = data->>'actor'; - """ + max = Repo.aggregate(Activity, :max, :id) + IO.puts("#{max} activities") + chunks = 0..(round(max / 10_000)) + + Enum.each(chunks, fn (i) -> + min = i * 10_000 + max = min + 10_000 + IO.puts("Updating #{min}") + execute """ + update activities set actor = data->>'actor' where id > #{min} and id <= #{max}; + """ + end) create index(:activities, [:actor, "id DESC NULLS LAST"], concurrently: true) end From 6bf261589f736c8bfd9eb10b230e56d857cbaa3c Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 9 Nov 2017 13:32:53 +0100 Subject: [PATCH 2/6] Update activities in own migration. --- .../20171109091239_add_actor_to_activity.exs | 15 ----------- .../20171109114020_fill_actor_field.exs | 25 +++++++++++++++++++ 2 files changed, 25 insertions(+), 15 deletions(-) create mode 100644 priv/repo/migrations/20171109114020_fill_actor_field.exs diff --git a/priv/repo/migrations/20171109091239_add_actor_to_activity.exs b/priv/repo/migrations/20171109091239_add_actor_to_activity.exs index c04922c76..2d8b60a91 100644 --- a/priv/repo/migrations/20171109091239_add_actor_to_activity.exs +++ b/priv/repo/migrations/20171109091239_add_actor_to_activity.exs @@ -1,8 +1,6 @@ defmodule Pleroma.Repo.Migrations.AddActorToActivity do use Ecto.Migration - alias Pleroma.{Repo, Activity} - @disable_ddl_transaction true def up do @@ -10,19 +8,6 @@ def up do add :actor, :string end - max = Repo.aggregate(Activity, :max, :id) - IO.puts("#{max} activities") - chunks = 0..(round(max / 10_000)) - - Enum.each(chunks, fn (i) -> - min = i * 10_000 - max = min + 10_000 - IO.puts("Updating #{min}") - execute """ - update activities set actor = data->>'actor' where id > #{min} and id <= #{max}; - """ - end) - create index(:activities, [:actor, "id DESC NULLS LAST"], concurrently: true) end diff --git a/priv/repo/migrations/20171109114020_fill_actor_field.exs b/priv/repo/migrations/20171109114020_fill_actor_field.exs new file mode 100644 index 000000000..d4ac601da --- /dev/null +++ b/priv/repo/migrations/20171109114020_fill_actor_field.exs @@ -0,0 +1,25 @@ +defmodule Pleroma.Repo.Migrations.FillActorField do + use Ecto.Migration + + alias Pleroma.{Repo, Activity} + + def up do + max = Repo.aggregate(Activity, :max, :id) + IO.puts("#{max} activities") + chunks = 0..(round(max / 10_000)) + + Enum.each(chunks, fn (i) -> + min = i * 10_000 + max = min + 10_000 + IO.puts("Updating #{min}") + execute(""" + update activities set actor = data->>'actor' where id > #{min} and id <= #{max}; + """) + |> IO.inspect + end) + end + + def down do + end +end + From 9e60de695db74bbe66c2be411ec85420b0ff54a7 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 9 Nov 2017 13:45:17 +0100 Subject: [PATCH 3/6] Fix migration for empty db. --- .../20171109114020_fill_actor_field.exs | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/priv/repo/migrations/20171109114020_fill_actor_field.exs b/priv/repo/migrations/20171109114020_fill_actor_field.exs index d4ac601da..255ca46d5 100644 --- a/priv/repo/migrations/20171109114020_fill_actor_field.exs +++ b/priv/repo/migrations/20171109114020_fill_actor_field.exs @@ -5,18 +5,19 @@ defmodule Pleroma.Repo.Migrations.FillActorField do def up do max = Repo.aggregate(Activity, :max, :id) - IO.puts("#{max} activities") - chunks = 0..(round(max / 10_000)) + if max do + IO.puts("#{max} activities") + chunks = 0..(round(max / 10_000)) - Enum.each(chunks, fn (i) -> - min = i * 10_000 - max = min + 10_000 - IO.puts("Updating #{min}") - execute(""" + Enum.each(chunks, fn (i) -> + min = i * 10_000 + max = min + 10_000 + execute(""" update activities set actor = data->>'actor' where id > #{min} and id <= #{max}; - """) - |> IO.inspect - end) + """) + |> IO.inspect + end) + end end def down do From e942e1e5528f587d2d5411f235d17599965f0f85 Mon Sep 17 00:00:00 2001 From: eal Date: Thu, 9 Nov 2017 14:11:37 +0200 Subject: [PATCH 4/6] Correct mimetype on bad uploads. --- lib/pleroma/upload.ex | 10 +++++++++- test/upload_test.exs | 12 ++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 2717377a3..d5723f5de 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -8,11 +8,19 @@ def store(%Plug.Upload{} = file) do result_file = Path.join(upload_folder, file.filename) File.cp!(file.path, result_file) + # fix content type on some image uploads + matches = Regex.named_captures(~r/\.(?(jpg|jpeg|png|gif))$/i, file.filename) + content_type = if file.content_type == "application/octet-stream" and matches do + if matches["ext"] == "jpg", do: "image/jpeg", else: "image/#{matches["ext"]}" + else + file.content_type + end + %{ "type" => "Image", "url" => [%{ "type" => "Link", - "mediaType" => file.content_type, + "mediaType" => content_type, "href" => url_for(Path.join(uuid, :cow_uri.urlencode(file.filename))) }], "name" => file.filename, diff --git a/test/upload_test.exs b/test/upload_test.exs index 71041e83c..f90c4d713 100644 --- a/test/upload_test.exs +++ b/test/upload_test.exs @@ -9,5 +9,17 @@ test "copies the file to the configured folder" do assert data["name"] == "an [image.jpg" assert List.first(data["url"])["href"] == "http://localhost:4001/media/#{data["uuid"]}/an%20%5Bimage.jpg" end + + test "fixes an incorrect content type" do + file = %Plug.Upload{content_type: "application/octet-stream", path: Path.absname("test/fixtures/image.jpg"), filename: "an [image.jpg"} + data = Upload.store(file) + assert hd(data["url"])["mediaType"] == "image/jpeg" + end + + test "does not modify a valid content type" do + file = %Plug.Upload{content_type: "image/png", path: Path.absname("test/fixtures/image.jpg"), filename: "an [image.jpg"} + data = Upload.store(file) + assert hd(data["url"])["mediaType"] == "image/png" + end end end From ccde03285debe8def5d89d49b9afdcc48a76d7a6 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 9 Nov 2017 15:16:10 +0100 Subject: [PATCH 5/6] add sort index for activities --- .../20171109141309_add_sort_index_to_activities.exs | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 priv/repo/migrations/20171109141309_add_sort_index_to_activities.exs diff --git a/priv/repo/migrations/20171109141309_add_sort_index_to_activities.exs b/priv/repo/migrations/20171109141309_add_sort_index_to_activities.exs new file mode 100644 index 000000000..2d21c56ca --- /dev/null +++ b/priv/repo/migrations/20171109141309_add_sort_index_to_activities.exs @@ -0,0 +1,8 @@ +defmodule Pleroma.Repo.Migrations.AddSortIndexToActivities do + use Ecto.Migration + @disable_ddl_transaction true + + def change do + create index(:activities, ["id desc nulls last"], concurrently: true) + end +end From 266d9c008d2a85395bb7ab773d5d548c89e7ed97 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 9 Nov 2017 16:48:45 +0100 Subject: [PATCH 6/6] MastoAPI: Fetch statuses in search. --- .../web/mastodon_api/mastodon_api_controller.ex | 12 ++++++++++-- .../mastodon_api/mastodon_api_controller_test.exs | 9 +++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index feaf9a900..c28e20ed1 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Pleroma.Web.MastodonAPI.{StatusView, AccountView} alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.TwitterAPI.TwitterAPI - alias Pleroma.Web.CommonAPI + alias Pleroma.Web.{CommonAPI, OStatus} import Ecto.Query import Logger @@ -361,11 +361,19 @@ def blocks(%{assigns: %{user: user}} = conn, _) do def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do accounts = User.search(query, params["resolve"] == "true") + fetched = if Regex.match?(~r/https?:/, query) do + with {:ok, activities} <- OStatus.fetch_activity_from_url(query) do + activities + else + _e -> [] + end + end || [] + q = from a in Activity, where: fragment("?->>'type' = 'Create'", a.data), where: fragment("to_tsvector('english', ?->'object'->>'content') @@ plainto_tsquery('english', ?)", a.data, ^query), limit: 20 - statuses = Repo.all(q) + statuses = Repo.all(q) ++ fetched res = %{ "accounts" => AccountView.render("accounts.json", users: accounts, for: user, as: :user), diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index d118026eb..8bbdecaa8 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -386,6 +386,15 @@ test "search", %{conn: conn} do assert status["id"] == to_string(activity.id) end + test "search fetches remote statuses", %{conn: conn} do + conn = conn + |> get("/api/v1/search", %{"q" => "https://shitposter.club/notice/2827873"}) + assert results = json_response(conn, 200) + + [status] = results["statuses"] + assert status["uri"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" + end + test "search fetches remote accounts", %{conn: conn} do conn = conn |> get("/api/v1/search", %{"q" => "shp@social.heldscal.la", "resolve" => "true"})