alyssa
/
pleroma
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'admin-config' into 'develop' 

Admin config

See merge request pleroma/pleroma!1328
This commit is contained in:
kaniini 2019-06-23 05:42:37 +00:00
parent 72f365df07 b37ede5df3
commit 8b7860f12f
13 changed files with 404 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
docs/api/admin_api.md
View File

@ -568,8 +568,9 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
{
configs: [
{
"group": string,
"key": string,
"value": string or {} or []
"value": string or {} or [] or {"tuple": []}
}
]
}
@ -580,6 +581,8 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
Module name can be passed as string, which starts with `Pleroma`, e.g. `"Pleroma.Upload"`.
Atom or boolean value can be passed with `:` in the beginning, e.g. `":true"`, `":upload"`.
Integer with `i:`, e.g. `"i:150"`.
Tuple with more than 2 values with `{"tuple": ["first_val", Pleroma.Module, []]}`.
`{"tuple": ["some_string", "Pleroma.Some.Module", []]}` will be converted to `{"some_string", Pleroma.Some.Module, []}`.
Compile time settings (need instance reboot):
- all settings by this keys:
@ -595,8 +598,9 @@ Compile time settings (need instance reboot):
- Method `POST`
- Params:
- `configs` => [
- `group` (string)
- `key` (string)
- `value` (string, [], {})
- `value` (string, [], {} or {"tuple": []})
- `delete` = true (optional, if parameter must be deleted)
]
@ -606,6 +610,7 @@ Compile time settings (need instance reboot):
{
configs: [
{
"group": "pleroma",
"key": "Pleroma.Upload",
"value": {
"uploader": "Pleroma.Uploaders.Local",
@ -619,6 +624,9 @@ Compile time settings (need instance reboot):
"follow_redirect": ":true",
"pool": ":upload"
}
},
"dispatch": {
"tuple": ["/api/v1/streaming", "Pleroma.Web.MastodonAPI.WebsocketHandler", []]
}
}
}
@ -631,8 +639,9 @@ Compile time settings (need instance reboot):
{
configs: [
{
"group": string,
"key": string,
"value": string or {} or []
"value": string or {} or [] or {"tuple": []}
}
]
}

6
lib/mix/tasks/pleroma/config.ex
View File

@ -24,7 +24,7 @@ def run(["migrate_to_db"]) do
|> Enum.reject(fn {k, _v} -> k in [Pleroma.Repo, :env] end)
|> Enum.each(fn {k, v} ->
key = to_string(k) |> String.replace("Elixir.", "")
{:ok, _} = Config.update_or_create(%{key: key, value: v})
{:ok, _} = Config.update_or_create(%{group: "pleroma", key: key, value: v})
Mix.shell().info("#{key} is migrated.")
end)
@ -51,7 +51,9 @@ def run(["migrate_from_db", env]) do
IO.write(
file,
"config :pleroma, #{config.key}#{mark} #{inspect(Config.from_binary(config.value))}\r\n"
"config :#{config.group}, #{config.key}#{mark} #{
inspect(Config.from_binary(config.value))
}\r\n"
)
{:ok, _} = Repo.delete(config)

17
lib/pleroma/config/transfer_task.ex
View File

@ -11,8 +11,17 @@ def start_link do
def load_and_update_env do
if Pleroma.Config.get([:instance, :dynamic_configuration]) and
Ecto.Adapters.SQL.table_exists?(Pleroma.Repo, "config") do
for_restart =
Pleroma.Repo.all(Config)
|> Enum.each(&update_env(&1))
|> Enum.map(&update_env(&1))
# We need to restart applications for loaded settings take effect
for_restart
|> Enum.reject(&(&1 in [:pleroma, :ok]))
|> Enum.each(fn app ->
Application.stop(app)
:ok = Application.start(app)
end)
end
end
@ -25,11 +34,15 @@ defp update_env(setting) do
setting.key
end
group = String.to_existing_atom(setting.group)
Application.put_env(
:pleroma,
group,
String.to_existing_atom(key),
Config.from_binary(setting.value)
)
group
rescue
e ->
require Logger

8
lib/pleroma/web/admin_api/admin_api_controller.ex
View File

@ -377,12 +377,12 @@ def config_update(conn, %{"configs" => configs}) do
if Pleroma.Config.get([:instance, :dynamic_configuration]) do
updated =
Enum.map(configs, fn
%{"key" => key, "value" => value} ->
{:ok, config} = Config.update_or_create(%{key: key, value: value})
%{"group" => group, "key" => key, "value" => value} ->
{:ok, config} = Config.update_or_create(%{group: group, key: key, value: value})
config
%{"key" => key, "delete" => "true"} ->
{:ok, _} = Config.delete(key)
%{"group" => group, "key" => key, "delete" => "true"} ->
{:ok, _} = Config.delete(%{group: group, key: key})
nil
end)
|> Enum.reject(&is_nil(&1))

72
lib/pleroma/web/admin_api/config.ex
View File

@ -12,26 +12,27 @@ defmodule Pleroma.Web.AdminAPI.Config do
schema "config" do
field(:key, :string)
field(:group, :string)
field(:value, :binary)
timestamps()
end
@spec get_by_key(String.t()) :: Config.t() | nil
def get_by_key(key), do: Repo.get_by(Config, key: key)
@spec get_by_params(map()) :: Config.t() | nil
def get_by_params(params), do: Repo.get_by(Config, params)
@spec changeset(Config.t(), map()) :: Changeset.t()
def changeset(config, params \\ %{}) do
config
|> cast(params, [:key, :value])
|> validate_required([:key, :value])
|> unique_constraint(:key)
|> cast(params, [:key, :group, :value])
|> validate_required([:key, :group, :value])
|> unique_constraint(:key, name: :config_group_key_index)
end
@spec create(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
def create(%{key: key, value: value}) do
def create(params) do
%Config{}
|> changeset(%{key: key, value: transform(value)})
|> changeset(Map.put(params, :value, transform(params[:value])))
|> Repo.insert()
end
@ -43,20 +44,20 @@ def update(%Config{} = config, %{value: value}) do
end
@spec update_or_create(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
def update_or_create(%{key: key} = params) do
with %Config{} = config <- Config.get_by_key(key) do
def update_or_create(params) do
with %Config{} = config <- Config.get_by_params(Map.take(params, [:group, :key])) do
Config.update(config, params)
else
nil -> Config.create(params)
end
end
@spec delete(String.t()) :: {:ok, Config.t()} | {:error, Changeset.t()}
def delete(key) do
with %Config{} = config <- Config.get_by_key(key) do
@spec delete(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
def delete(params) do
with %Config{} = config <- Config.get_by_params(params) do
Repo.delete(config)
else
nil -> {:error, "Config with key #{key} not found"}
nil -> {:error, "Config with params #{inspect(params)} not found"}
end
end
@ -77,10 +78,21 @@ defp do_convert(values) when is_list(values), do: for(val <- values, do: do_conv
defp do_convert({k, v} = value) when is_tuple(value),
do: %{k => do_convert(v)}
defp do_convert(value) when is_binary(value) or is_atom(value) or is_map(value),
do: value
defp do_convert(value) when is_tuple(value), do: %{"tuple" => do_convert(Tuple.to_list(value))}
defp do_convert(value) when is_binary(value) or is_map(value) or is_number(value), do: value
defp do_convert(value) when is_atom(value) do
string = to_string(value)
if String.starts_with?(string, "Elixir."),
do: String.trim_leading(string, "Elixir."),
else: value
end
@spec transform(any()) :: binary()
def transform(%{"tuple" => _} = entity), do: :erlang.term_to_binary(do_transform(entity))
def transform(entity) when is_map(entity) do
tuples =
for {k, v} <- entity,
@ -101,11 +113,16 @@ def transform(entity), do: :erlang.term_to_binary(entity)
defp do_transform(%Regex{} = value) when is_map(value), do: value
defp do_transform(%{"tuple" => [k, values] = entity}) when length(entity) == 2 do
{do_transform(k), do_transform(values)}
end
defp do_transform(%{"tuple" => values}) do
Enum.reduce(values, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end)
end
defp do_transform(value) when is_map(value) do
values =
for {key, val} <- value,
into: [],
do: {String.to_atom(key), do_transform(val)}
values = for {key, val} <- value, into: [], do: {String.to_atom(key), do_transform(val)}
Enum.sort(values)
end
@ -117,15 +134,17 @@ defp do_transform(value) when is_list(value) do
defp do_transform(entity) when is_list(entity) and length(entity) == 1, do: hd(entity)
defp do_transform(value) when is_binary(value) do
value = String.trim(value)
String.trim(value)
|> do_transform_string()
end
case String.length(value) do
0 ->
nil
defp do_transform(value), do: value
_ ->
defp do_transform_string(value) when byte_size(value) == 0, do: nil
defp do_transform_string(value) do
cond do
String.starts_with?(value, "Pleroma") ->
String.starts_with?(value, "Pleroma") or String.starts_with?(value, "Phoenix") ->
String.to_existing_atom("Elixir." <> value)
String.starts_with?(value, ":") ->
@ -138,7 +157,4 @@ defp do_transform(value) when is_binary(value) do
value
end
end
end
defp do_transform(value), do: value
end

1
lib/pleroma/web/admin_api/views/config_view.ex
View File

@ -10,6 +10,7 @@ def render("index.json", %{configs: configs}) do
def render("show.json", %{config: config}) do
%{
key: config.key,
group: config.group,
value: Pleroma.Web.AdminAPI.Config.from_binary_to_map(config.value)
}
end

12
priv/repo/migrations/20190622151019_add_group_key_to_config.exs Normal file
View File

@ -0,0 +1,12 @@
defmodule Pleroma.Repo.Migrations.AddGroupKeyToConfig do
use Ecto.Migration
def change do
alter table("config") do
add(:group, :string)
end
drop(unique_index("config", :key))
create(unique_index("config", [:group, :key]))
end
end

22
test/config/transfer_task_test.exs
View File

@ -13,19 +13,37 @@ defmodule Pleroma.Config.TransferTaskTest do
test "transfer config values from db to env" do
refute Application.get_env(:pleroma, :test_key)
Pleroma.Web.AdminAPI.Config.create(%{key: "test_key", value: [live: 2, com: 3]})
refute Application.get_env(:idna, :test_key)
Pleroma.Web.AdminAPI.Config.create(%{
group: "pleroma",
key: "test_key",
value: [live: 2, com: 3]
})
Pleroma.Web.AdminAPI.Config.create(%{
group: "idna",
key: "test_key",
value: [live: 15, com: 35]
})
Pleroma.Config.TransferTask.start_link()
assert Application.get_env(:pleroma, :test_key) == [live: 2, com: 3]
assert Application.get_env(:idna, :test_key) == [live: 15, com: 35]
on_exit(fn ->
Application.delete_env(:pleroma, :test_key)
Application.delete_env(:idna, :test_key)
end)
end
test "non existing atom" do
Pleroma.Web.AdminAPI.Config.create(%{key: "undefined_atom_key", value: [live: 2, com: 3]})
Pleroma.Web.AdminAPI.Config.create(%{
group: "pleroma",
key: "undefined_atom_key",
value: [live: 2, com: 3]
})
assert ExUnit.CaptureLog.capture_log(fn ->
Pleroma.Config.TransferTask.start_link()

1
test/support/factory.ex
View File

@ -314,6 +314,7 @@ def registration_factory do
def config_factory do
%Pleroma.Web.AdminAPI.Config{
key: sequence(:key, &"some_key_#{&1}"),
group: "pleroma",
value:
sequence(
:value,

19
test/tasks/config_test.exs
View File

@ -30,17 +30,26 @@ test "settings are migrated to db" do
Mix.Tasks.Pleroma.Config.run(["migrate_to_db"])
first_db = Config.get_by_key("first_setting")
second_db = Config.get_by_key("second_setting")
refute Config.get_by_key("Pleroma.Repo")
first_db = Config.get_by_params(%{group: "pleroma", key: "first_setting"})
second_db = Config.get_by_params(%{group: "pleroma", key: "second_setting"})
refute Config.get_by_params(%{group: "pleroma", key: "Pleroma.Repo"})
assert Config.from_binary(first_db.value) == [key: "value", key2: [Pleroma.Repo]]
assert Config.from_binary(second_db.value) == [key: "value2", key2: [Pleroma.Activity]]
end
test "settings are migrated to file and deleted from db", %{temp_file: temp_file} do
Config.create(%{key: "setting_first", value: [key: "value", key2: [Pleroma.Activity]]})
Config.create(%{key: "setting_second", value: [key: "valu2", key2: [Pleroma.Repo]]})
Config.create(%{
group: "pleroma",
key: "setting_first",
value: [key: "value", key2: [Pleroma.Activity]]
})
Config.create(%{
group: "pleroma",
key: "setting_second",
value: [key: "valu2", key2: [Pleroma.Repo]]
})
Mix.Tasks.Pleroma.Config.run(["migrate_from_db", "temp"])

5
test/web/activity_pub/mrf/anti_link_spam_policy_test.exs
View File

@ -5,6 +5,7 @@
defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicyTest do
use Pleroma.DataCase
import Pleroma.Factory
import ExUnit.CaptureLog
alias Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy
@ -114,7 +115,9 @@ test "it rejects posts without links" do
@linkless_message
|> Map.put("actor", "http://invalid.actor")
assert capture_log(fn ->
{:reject, _} = AntiLinkSpamPolicy.filter(message)
end) =~ "[error] Could not decode user at fetch http://invalid.actor"
end
test "it rejects posts with links" do
@ -122,7 +125,9 @@ test "it rejects posts with links" do
@linkful_message
|> Map.put("actor", "http://invalid.actor")
assert capture_log(fn ->
{:reject, _} = AntiLinkSpamPolicy.filter(message)
end) =~ "[error] Could not decode user at fetch http://invalid.actor"
end
end

175
test/web/admin_api/admin_api_controller_test.exs
View File

@ -1343,6 +1343,8 @@ test "with settings in db", %{conn: conn} do
Application.delete_env(:pleroma, :key4)
Application.delete_env(:pleroma, :keyaa1)
Application.delete_env(:pleroma, :keyaa2)
Application.delete_env(:pleroma, Pleroma.Web.Endpoint.NotReal)
Application.delete_env(:pleroma, Pleroma.Captcha.NotReal)
:ok = File.rm(temp_file)
end)
@ -1361,8 +1363,9 @@ test "create new config setting in db", %{conn: conn} do
conn =
post(conn, "/api/pleroma/admin/config", %{
configs: [
%{key: "key1", value: "value1"},
%{group: "pleroma", key: "key1", value: "value1"},
%{
group: "pleroma",
key: "key2",
value: %{
"nested_1" => "nested_value1",
@ -1373,6 +1376,7 @@ test "create new config setting in db", %{conn: conn} do
}
},
%{
group: "pleroma",
key: "key3",
value: [
%{"nested_3" => ":nested_3", "nested_33" => "nested_33"},
@ -1380,8 +1384,14 @@ test "create new config setting in db", %{conn: conn} do
]
},
%{
group: "pleroma",
key: "key4",
value: %{"nested_5" => ":upload", "endpoint" => "https://example.com"}
},
%{
group: "idna",
key: "key5",
value: %{"tuple" => ["string", "Pleroma.Captcha.NotReal", []]}
}
]
})
@ -1389,10 +1399,12 @@ test "create new config setting in db", %{conn: conn} do
assert json_response(conn, 200) == %{
"configs" => [
%{
"group" => "pleroma",
"key" => "key1",
"value" => "value1"
},
%{
"group" => "pleroma",
"key" => "key2",
"value" => [
%{"nested_1" => "nested_value1"},
@ -1405,6 +1417,7 @@ test "create new config setting in db", %{conn: conn} do
]
},
%{
"group" => "pleroma",
"key" => "key3",
"value" => [
[%{"nested_3" => "nested_3"}, %{"nested_33" => "nested_33"}],
@ -1412,8 +1425,14 @@ test "create new config setting in db", %{conn: conn} do
]
},
%{
"group" => "pleroma",
"key" => "key4",
"value" => [%{"endpoint" => "https://example.com"}, %{"nested_5" => "upload"}]
},
%{
"group" => "idna",
"key" => "key5",
"value" => %{"tuple" => ["string", "Pleroma.Captcha.NotReal", []]}
}
]
}
@ -1437,6 +1456,8 @@ test "create new config setting in db", %{conn: conn} do
endpoint: "https://example.com",
nested_5: :upload
]
assert Application.get_env(:idna, :key5) == {"string", Pleroma.Captcha.NotReal, []}
end
test "update config setting & delete", %{conn: conn} do
@ -1446,14 +1467,15 @@ test "update config setting & delete", %{conn: conn} do
conn =
post(conn, "/api/pleroma/admin/config", %{
configs: [
%{key: config1.key, value: "another_value"},
%{key: config2.key, delete: "true"}
%{group: config1.group, key: config1.key, value: "another_value"},
%{group: config2.group, key: config2.key, delete: "true"}
]
})
assert json_response(conn, 200) == %{
"configs" => [
%{
"group" => "pleroma",
"key" => config1.key,
"value" => "another_value"
}
@ -1463,5 +1485,152 @@ test "update config setting & delete", %{conn: conn} do
assert Application.get_env(:pleroma, :keyaa1) == "another_value"
refute Application.get_env(:pleroma, :keyaa2)
end
test "common config example", %{conn: conn} do
conn =
post(conn, "/api/pleroma/admin/config", %{
configs: [
%{
"group" => "pleroma",
"key" => "Pleroma.Captcha.NotReal",
"value" => %{
"enabled" => ":false",
"method" => "Pleroma.Captcha.Kocaptcha",
"seconds_valid" => "i:60"
}
}
]
})
assert json_response(conn, 200) == %{
"configs" => [
%{
"group" => "pleroma",
"key" => "Pleroma.Captcha.NotReal",
"value" => [
%{"enabled" => false},
%{"method" => "Pleroma.Captcha.Kocaptcha"},
%{"seconds_valid" => 60}
]
}
]
}
end
test "tuples with more than two values", %{conn: conn} do
conn =
post(conn, "/api/pleroma/admin/config", %{
configs: [
%{
"group" => "pleroma",
"key" => "Pleroma.Web.Endpoint.NotReal",
"value" => [
%{
"http" => %{
"dispatch" => [
%{
"tuple" => [
":_",
[
%{
"tuple" => [
"/api/v1/streaming",
"Pleroma.Web.MastodonAPI.WebsocketHandler",
[]
]
},
%{
"tuple" => [
"/websocket",
"Phoenix.Endpoint.CowboyWebSocket",
%{
"tuple" => [
"Phoenix.Transports.WebSocket",
%{
"tuple" => [
"Pleroma.Web.Endpoint",
"Pleroma.Web.UserSocket",
[]
]
}
]
}
]
},
%{
"tuple" => [
":_",
"Phoenix.Endpoint.Cowboy2Handler",
%{
"tuple" => ["Pleroma.Web.Endpoint", []]
}
]
}
]
]
}
]
}
}
]
}
]
})
assert json_response(conn, 200) == %{
"configs" => [
%{
"group" => "pleroma",
"key" => "Pleroma.Web.Endpoint.NotReal",
"value" => [
%{
"http" => %{
"dispatch" => %{
"_" => [
%{
"tuple" => [
"/api/v1/streaming",
"Pleroma.Web.MastodonAPI.WebsocketHandler",
[]
]
},
%{
"tuple" => [
"/websocket",
"Phoenix.Endpoint.CowboyWebSocket",
%{
"Elixir.Phoenix.Transports.WebSocket" => %{
"tuple" => [
"Pleroma.Web.Endpoint",
"Pleroma.Web.UserSocket",
[]
]
}
}
]
},
%{
"tuple" => [
"_",
"Phoenix.Endpoint.Cowboy2Handler",
%{"Elixir.Pleroma.Web.Endpoint" => []}
]
}
]
}
}
}
]
}
]
}
end
end
end
# Needed for testing
defmodule Pleroma.Web.Endpoint.NotReal do
end
defmodule Pleroma.Captcha.NotReal do
end

95
test/web/admin_api/config_test.exs
View File

@ -7,18 +7,18 @@ test "get_by_key/1" do
config = insert(:config)
insert(:config)
assert config == Config.get_by_key(config.key)
assert config == Config.get_by_params(%{group: config.group, key: config.key})
end
test "create/1" do
{:ok, config} = Config.create(%{key: "some_key", value: "some_value"})
assert config == Config.get_by_key("some_key")
{:ok, config} = Config.create(%{group: "pleroma", key: "some_key", value: "some_value"})
assert config == Config.get_by_params(%{group: "pleroma", key: "some_key"})
end
test "update/1" do
config = insert(:config)
{:ok, updated} = Config.update(config, %{value: "some_value"})
loaded = Config.get_by_key(config.key)
loaded = Config.get_by_params(%{group: config.group, key: config.key})
assert loaded == updated
end
@ -27,8 +27,8 @@ test "update_or_create/1" do
key2 = "another_key"
params = [
%{key: key2, value: "another_value"},
%{key: config.key, value: "new_value"}
%{group: "pleroma", key: key2, value: "another_value"},
%{group: config.group, key: config.key, value: "new_value"}
]
assert Repo.all(Config) |> length() == 1
@ -37,8 +37,8 @@ test "update_or_create/1" do
assert Repo.all(Config) |> length() == 2
config1 = Config.get_by_key(config.key)
config2 = Config.get_by_key(key2)
config1 = Config.get_by_params(%{group: config.group, key: config.key})
config2 = Config.get_by_params(%{group: "pleroma", key: key2})
assert config1.value == Config.transform("new_value")
assert config2.value == Config.transform("another_value")
@ -46,8 +46,8 @@ test "update_or_create/1" do
test "delete/1" do
config = insert(:config)
{:ok, _} = Config.delete(config.key)
refute Config.get_by_key(config.key)
{:ok, _} = Config.delete(%{key: config.key, group: config.group})
refute Config.get_by_params(%{key: config.key, group: config.group})
end
describe "transform/1" do
@ -179,5 +179,80 @@ test "complex map with sigil" do
assert Config.from_binary(binary) ==
[federated_timeline_removal: [], reject: [~r/comp[lL][aA][iI][nN]er/], replace: []]
end
test "complex map with tuples with more than 2 values" do
binary =
Config.transform(%{
"http" => %{
"dispatch" => [
%{
"tuple" => [
":_",
[
%{
"tuple" => [
"/api/v1/streaming",
"Pleroma.Web.MastodonAPI.WebsocketHandler",
[]
]
},
%{
"tuple" => [
"/websocket",
"Phoenix.Endpoint.CowboyWebSocket",
%{
"tuple" => [
"Phoenix.Transports.WebSocket",
%{"tuple" => ["Pleroma.Web.Endpoint", "Pleroma.Web.UserSocket", []]}
]
}
]
},
%{
"tuple" => [
":_",
"Phoenix.Endpoint.Cowboy2Handler",
%{
"tuple" => ["Pleroma.Web.Endpoint", []]
}
]
}
]
]
}
]
}
})
assert binary ==
:erlang.term_to_binary(
http: [
dispatch: [
_: [
{"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []},
{"/websocket", Phoenix.Endpoint.CowboyWebSocket,
{Phoenix.Transports.WebSocket,
{Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, []}}},
{:_, Phoenix.Endpoint.Cowboy2Handler, {Pleroma.Web.Endpoint, []}}
]
]
]
)
assert Config.from_binary(binary) == [
http: [
dispatch: [
{:_,
[
{"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []},
{"/websocket", Phoenix.Endpoint.CowboyWebSocket,
{Phoenix.Transports.WebSocket,
{Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, []}}},
{:_, Phoenix.Endpoint.Cowboy2Handler, {Pleroma.Web.Endpoint, []}}
]}
]
]
]
end
end
end