From 2ee6754095f5b81807efe97c73ada42e2c990ede Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 14 Jan 2020 17:24:26 +0100 Subject: [PATCH 1/3] Mix Tasks: Add pleroma.benchmarks.tags --- benchmarks/load_testing/generator.ex | 44 +++++++++++++- lib/mix/tasks/pleroma/benchmarks/tags.ex | 57 +++++++++++++++++++ .../controllers/timeline_controller.ex | 14 +++-- 3 files changed, 109 insertions(+), 6 deletions(-) create mode 100644 lib/mix/tasks/pleroma/benchmarks/tags.ex diff --git a/benchmarks/load_testing/generator.ex b/benchmarks/load_testing/generator.ex index a957e0ffb..3f88fefd7 100644 --- a/benchmarks/load_testing/generator.ex +++ b/benchmarks/load_testing/generator.ex @@ -9,7 +9,7 @@ def generate_like_activities(user, posts) do {time, _} = :timer.tc(fn -> Task.async_stream( - Enum.take_random(posts, count_likes), + Enum.take_random(posts, count_likes), fn post -> {:ok, _, _} = CommonAPI.favorite(post.id, user) end, max_concurrency: 10, timeout: 30_000 @@ -142,6 +142,48 @@ defp do_generate_activity(users) do CommonAPI.post(Enum.random(users), post) end + def generate_power_intervals(opts \\ []) do + count = Keyword.get(opts, :count, 20) + power = Keyword.get(opts, :power, 2) + IO.puts("Generating #{count} intervals for a power #{power} series...") + counts = Enum.map(1..count, fn n -> :math.pow(n, power) end) + sum = Enum.sum(counts) + + densities = + Enum.map(counts, fn c -> + c / sum + end) + + densities + |> Enum.reduce(0, fn density, acc -> + if acc == 0 do + [{0, density}] + else + [{_, lower} | _] = acc + [{lower, lower + density} | acc] + end + end) + |> Enum.reverse() + end + + def generate_tagged_activities(opts \\ []) do + tag_count = Keyword.get(opts, :tag_count, 20) + users = Keyword.get(opts, :users, Repo.all(User)) + activity_count = Keyword.get(opts, :count, 200_000) + + intervals = generate_power_intervals(count: tag_count) + + IO.puts( + "Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0" + ) + + Enum.each(1..activity_count, fn _ -> + random = :rand.uniform() + i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end) + CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"}) + end) + end + defp do_generate_activity_with_mention(user, users) do mentions_cnt = Enum.random([2, 3, 4, 5]) with_user = Enum.random([true, false]) diff --git a/lib/mix/tasks/pleroma/benchmarks/tags.ex b/lib/mix/tasks/pleroma/benchmarks/tags.ex new file mode 100644 index 000000000..73796b5f9 --- /dev/null +++ b/lib/mix/tasks/pleroma/benchmarks/tags.ex @@ -0,0 +1,57 @@ +defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do + use Mix.Task + alias Pleroma.Repo + alias Pleroma.LoadTesting.Generator + import Ecto.Query + + def run(_args) do + Mix.Pleroma.start_pleroma() + activities_count = Repo.aggregate(from(a in Pleroma.Activity), :count, :id) + + if activities_count == 0 do + IO.puts("Did not find any activities, cleaning and generating") + clean_tables() + Generator.generate_users(users_max: 10) + Generator.generate_tagged_activities() + else + IO.puts("Found #{activities_count} activities, won't generate new ones") + end + + tags = Enum.map(0..20, fn i -> {"For #tag_#{i}", "tag_#{i}"} end) + + Enum.each(tags, fn {_, tag} -> + query = + from(o in Pleroma.Object, + where: fragment("(?)->'tag' \\? (?)", o.data, ^tag) + ) + + count = Repo.aggregate(query, :count, :id) + IO.puts("Database contains #{count} posts tagged with #{tag}") + end) + + user = Repo.all(Pleroma.User) |> List.first() + + Benchee.run( + %{ + "Hashtag fetching" => fn tag -> + Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching( + %{ + "tag" => tag + }, + user, + false + ) + end + }, + inputs: tags, + time: 5 + ) + end + + defp clean_tables do + IO.puts("Deleting old data...\n") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;") + end +end diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex index 384159336..29964a1d4 100644 --- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex @@ -77,10 +77,7 @@ def public(%{assigns: %{user: user}} = conn, params) do |> render("index.json", activities: activities, for: user, as: :activity) end - # GET /api/v1/timelines/tag/:tag - def hashtag(%{assigns: %{user: user}} = conn, params) do - local_only = truthy_param?(params["local"]) - + def hashtag_fetching(params, user, local_only) do tags = [params["tag"], params["any"]] |> List.flatten() @@ -98,7 +95,7 @@ def hashtag(%{assigns: %{user: user}} = conn, params) do |> Map.get("none", []) |> Enum.map(&String.downcase(&1)) - activities = + _activities = params |> Map.put("type", "Create") |> Map.put("local_only", local_only) @@ -109,6 +106,13 @@ def hashtag(%{assigns: %{user: user}} = conn, params) do |> Map.put("tag_all", tag_all) |> Map.put("tag_reject", tag_reject) |> ActivityPub.fetch_public_activities() + end + + # GET /api/v1/timelines/tag/:tag + def hashtag(%{assigns: %{user: user}} = conn, params) do + local_only = truthy_param?(params["local"]) + + activities = hashtag_fetching(params, user, local_only) conn |> add_link_headers(activities, %{"local" => local_only}) From 167e9c45eccf5ddb89077c979b1d587318f78cc0 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 15 Jan 2020 12:37:50 +0100 Subject: [PATCH 2/3] Benchmarks: Move to correct folder --- {lib => benchmarks}/mix/tasks/pleroma/benchmarks/tags.ex | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {lib => benchmarks}/mix/tasks/pleroma/benchmarks/tags.ex (100%) diff --git a/lib/mix/tasks/pleroma/benchmarks/tags.ex b/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex similarity index 100% rename from lib/mix/tasks/pleroma/benchmarks/tags.ex rename to benchmarks/mix/tasks/pleroma/benchmarks/tags.ex From 271afcd940b743b84fae2ee40245d1e0c53cc714 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 20 Jan 2020 12:19:15 +0100 Subject: [PATCH 3/3] Add benchmark of any/all tag options --- .../mix/tasks/pleroma/benchmarks/tags.ex | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex b/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex index 73796b5f9..fd1506907 100644 --- a/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex +++ b/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex @@ -31,6 +31,36 @@ def run(_args) do user = Repo.all(Pleroma.User) |> List.first() + Benchee.run( + %{ + "Hashtag fetching, any" => fn tags -> + Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching( + %{ + "any" => tags + }, + user, + false + ) + end, + # Will always return zero results because no overlapping hashtags are generated. + "Hashtag fetching, all" => fn tags -> + Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching( + %{ + "all" => tags + }, + user, + false + ) + end + }, + inputs: + tags + |> Enum.map(fn {_, v} -> v end) + |> Enum.chunk_every(2) + |> Enum.map(fn tags -> {"For #{inspect(tags)}", tags} end), + time: 5 + ) + Benchee.run( %{ "Hashtag fetching" => fn tag ->