[#477] User: FTS and trigram search results mixing (to handle misspelled requests).

This commit is contained in:
Ivan Tashkinov 2019-01-18 10:35:45 +03:00
parent 0bc6d30f7d
commit ed8f55ab8e
3 changed files with 87 additions and 59 deletions

View File

@ -679,13 +679,35 @@ def get_recipients_from_activity(%Activity{recipients: to}) do
end end
def search(query, resolve \\ false, for_user \\ nil) do def search(query, resolve \\ false, for_user \\ nil) do
# strip the beginning @ off if there is a query # Strip the beginning @ off if there is a query
query = String.trim_leading(query, "@") query = String.trim_leading(query, "@")
if resolve do if resolve, do: User.get_or_fetch_by_nickname(query)
User.get_or_fetch_by_nickname(query)
end
fts_results = do_search(fts_search_subquery(query), for_user)
trigram_results = do_search(trigram_search_subquery(query), for_user)
Enum.uniq_by(fts_results ++ trigram_results, & &1.id)
end
defp do_search(subquery, for_user, options \\ []) do
q =
from(
s in subquery(subquery),
order_by: [desc: s.search_rank],
limit: ^(options[:limit] || 20)
)
results =
q
|> Repo.all()
|> Enum.filter(&(&1.search_rank > 0))
boost_search_results(results, for_user)
end
defp fts_search_subquery(query) do
processed_query = processed_query =
query query
|> String.replace(~r/\W+/, " ") |> String.replace(~r/\W+/, " ")
@ -694,69 +716,69 @@ def search(query, resolve \\ false, for_user \\ nil) do
|> Enum.map(&(&1 <> ":*")) |> Enum.map(&(&1 <> ":*"))
|> Enum.join(" | ") |> Enum.join(" | ")
inner = from(
from( u in User,
u in User, select_merge: %{
select_merge: %{ search_rank:
search_rank: fragment(
fragment( """
""" ts_rank_cd(
ts_rank_cd( setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') || setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'),
setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'), to_tsquery('simple', ?),
to_tsquery('simple', ?), 32
32
)
""",
u.nickname,
u.name,
^processed_query
) )
}, """,
where: not is_nil(u.nickname) u.nickname,
) u.name,
^processed_query
)
},
where: not is_nil(u.nickname)
)
end
q = defp trigram_search_subquery(query) do
from( from(
s in subquery(inner), u in User,
order_by: [desc: s.search_rank], select_merge: %{
limit: 20 search_rank:
) fragment(
"similarity(?, ? || ' ' || coalesce(?, ''))",
^query,
u.nickname,
u.name
)
},
where: not is_nil(u.nickname)
)
end
results = defp boost_search_results(results, nil), do: results
q
|> Repo.all()
|> Enum.filter(&(&1.search_rank > 0))
weighted_results = defp boost_search_results(results, for_user) do
if for_user do friends_ids = get_friends_ids(for_user)
friends_ids = get_friends_ids(for_user) followers_ids = get_followers_ids(for_user)
followers_ids = get_followers_ids(for_user)
Enum.map( Enum.map(
results, results,
fn u -> fn u ->
search_rank_coef = search_rank_coef =
cond do cond do
u.id in friends_ids -> u.id in friends_ids ->
1.2 1.2
u.id in followers_ids -> u.id in followers_ids ->
1.1 1.1
true -> true ->
1 1
end
Map.put(u, :search_rank, u.search_rank * search_rank_coef)
end end
)
|> Enum.sort_by(&(-&1.search_rank))
else
results
end
weighted_results Map.put(u, :search_rank, u.search_rank * search_rank_coef)
end
)
|> Enum.sort_by(&(-&1.search_rank))
end end
def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do

View File

@ -814,6 +814,12 @@ test "finds users, ranking by similarity" do
assert [u4.id, u3.id, u1.id] == Enum.map(User.search("lain@ple"), & &1.id) assert [u4.id, u3.id, u1.id] == Enum.map(User.search("lain@ple"), & &1.id)
end end
test "finds users, handling misspelled requests" do
u1 = insert(:user, %{name: "lain"})
assert [u1.id] == Enum.map(User.search("laiin"), & &1.id)
end
test "finds users, boosting ranks of friends and followers" do test "finds users, boosting ranks of friends and followers" do
u1 = insert(:user) u1 = insert(:user)
u2 = insert(:user, %{name: "Doe"}) u2 = insert(:user, %{name: "Doe"})

View File

@ -1656,7 +1656,7 @@ test "it denies a friend request" do
test "it returns users, ordered by similarity", %{conn: conn} do test "it returns users, ordered by similarity", %{conn: conn} do
user = insert(:user, %{name: "eal"}) user = insert(:user, %{name: "eal"})
user_two = insert(:user, %{name: "eal me"}) user_two = insert(:user, %{name: "eal me"})
_user_three = insert(:user, %{name: "ebn"}) _user_three = insert(:user, %{name: "zzz"})
resp = resp =
conn conn