saving to DB only added by user settings

This commit is contained in:
Alexander Strizhakov 2020-01-17 11:45:44 +03:00
parent 29155137fd
commit 60ba2339a2
No known key found for this signature in database
GPG Key ID: 022896A53AEF1381
11 changed files with 280 additions and 44 deletions

View File

@ -4,42 +4,25 @@
defmodule Mix.Tasks.Pleroma.Config do defmodule Mix.Tasks.Pleroma.Config do
use Mix.Task use Mix.Task
import Mix.Pleroma import Mix.Pleroma
alias Pleroma.ConfigDB alias Pleroma.ConfigDB
alias Pleroma.Repo alias Pleroma.Repo
require Logger
@shortdoc "Manages the location of the config" @shortdoc "Manages the location of the config"
@moduledoc File.read!("docs/administration/CLI_tasks/config.md") @moduledoc File.read!("docs/administration/CLI_tasks/config.md")
@groups [
:pleroma,
:logger,
:quack,
:mime,
:tesla,
:phoenix,
:cors_plug,
:auto_linker,
:esshd,
:ueberauth,
:http_signatures,
:web_push_encryption,
:joken
]
def run(["migrate_to_db"]) do def run(["migrate_to_db"]) do
# we want to save original logger level # we want to save original logger level
start_pleroma(false) start_pleroma(false)
migrate_to_db()
if Pleroma.Config.get([:configurable_from_database]) do
Enum.each(@groups, &load_and_create(&1))
else
Mix.shell().info(
"Migration is not allowed by config. You can change this behavior in instance settings."
)
end
end end
def run(["migrate_from_db" | options]) do def run(["migrate_from_db" | options]) do
# TODO: add support for releases
start_pleroma() start_pleroma()
{opts, _} = {opts, _} =
@ -72,10 +55,36 @@ def run(["migrate_from_db" | options]) do
end end
end end
defp load_and_create(group) do @spec migrate_to_db(Path.t() | nil) :: any()
group def migrate_to_db(file_path \\ nil) do
|> Application.get_all_env() if Pleroma.Config.get([:configurable_from_database]) do
|> Enum.reject(fn {k, _v} -> # TODO: add support for releases
config_file = file_path || "config/#{Pleroma.Config.get(:env)}.secret.exs"
do_migrate_to_db(config_file)
else
Mix.shell().info(
"migration is not allowed by config. You can change this behavior in instance settings."
)
end
end
defp do_migrate_to_db(config_file) do
if File.exists?(config_file) do
{custom_config, _paths} =
if Code.ensure_loaded?(Config.Reader),
do: Config.Reader.read_imports!(config_file),
else: Mix.Config.eval!(config_file)
custom_config
|> Keyword.keys()
|> Enum.each(&create(&1, custom_config[&1]))
else
Logger.warn("to migrate settings, you must define custom settings in #{config_file}")
end
end
defp create(group, settings) do
Enum.reject(settings, fn {k, _v} ->
k in [Pleroma.Repo, Pleroma.Web.Endpoint, :env, :configurable_from_database] or k in [Pleroma.Repo, Pleroma.Web.Endpoint, :env, :configurable_from_database] or
(group == :phoenix and k == :serve_endpoints) (group == :phoenix and k == :serve_endpoints)
end) end)

View File

@ -5,6 +5,7 @@
defmodule Pleroma.ConfigDB do defmodule Pleroma.ConfigDB do
use Ecto.Schema use Ecto.Schema
import Ecto.Changeset import Ecto.Changeset
import Ecto.Query
import Pleroma.Web.Gettext import Pleroma.Web.Gettext
alias __MODULE__ alias __MODULE__
alias Pleroma.Repo alias Pleroma.Repo
@ -19,6 +20,20 @@ defmodule Pleroma.ConfigDB do
timestamps() timestamps()
end end
@spec get_all_as_keyword() :: keyword()
def get_all_as_keyword do
ConfigDB
|> select([c], {c.group, c.key, c.value})
|> Repo.all()
|> Enum.reduce([], fn {group, key, value}, acc ->
group = ConfigDB.from_string(group)
key = ConfigDB.from_string(key)
value = from_binary(value)
Keyword.update(acc, group, [{key, value}], &Keyword.merge(&1, [{key, value}]))
end)
end
@spec get_by_params(map()) :: ConfigDB.t() | nil @spec get_by_params(map()) :: ConfigDB.t() | nil
def get_by_params(params), do: Repo.get_by(ConfigDB, params) def get_by_params(params), do: Repo.get_by(ConfigDB, params)
@ -137,6 +152,9 @@ def from_string(entity) when is_binary(entity) do
end end
end end
@spec convert(any()) :: any()
def convert(entity), do: do_convert(entity)
defp do_convert(entity) when is_list(entity) do defp do_convert(entity) when is_list(entity) do
for v <- entity, into: [], do: do_convert(v) for v <- entity, into: [], do: do_convert(v)
end end

View File

@ -0,0 +1,51 @@
defmodule Pleroma.Config.Loader do
# TODO: add support for releases
if Code.ensure_loaded?(Config.Reader) do
@spec load() :: map()
def load do
config = Config.Reader.read!("config/config.exs")
env_config = Config.Reader.read!("config/#{Mix.env()}.exs")
Config.Reader.merge(config, env_config)
end
else
# support for Elixir less than 1.9
@spec load() :: map()
def load do
config = Mix.Config.eval!("config/config.exs")
env_config = Mix.Config.eval!("config/#{Mix.env()}.exs")
Mix.Config.merge(config, env_config)
end
end
end
defmodule Pleroma.Config.Holder do
@config Pleroma.Config.Loader.load()
@spec config() :: keyword()
def config do
@config
|> Keyword.keys()
|> Enum.map(&filter(&1, config(&1)))
|> List.flatten()
end
@spec config(atom()) :: any()
def config(group), do: @config[group]
@spec config(atom(), atom()) :: any()
def config(group, key), do: @config[group][key]
defp filter(group, settings) when group not in [:swarm] do
filtered =
Enum.reject(settings, fn {k, _v} ->
k in [Pleroma.Repo, Pleroma.Web.Endpoint, :env, :configurable_from_database] or
(group == :phoenix and k == :serve_endpoints)
end)
{group, filtered}
end
defp filter(_, _), do: []
end

View File

@ -38,10 +38,18 @@ defp update_env(setting) do
value = ConfigDB.from_binary(setting.value) value = ConfigDB.from_binary(setting.value)
if group != :phoenix and key != :serve_endpoints do if group != :phoenix and key != :serve_endpoints do
:ok = Application.put_env(group, key, value) default = Pleroma.Config.Holder.config(group, key)
merged_value =
if can_be_merged?(default, value) do
DeepMerge.deep_merge(default, value)
else
value
end end
:ok = Application.put_env(group, key, merged_value)
group group
end
rescue rescue
e -> e ->
Logger.warn( Logger.warn(
@ -61,4 +69,12 @@ defp restart(started_applications, app) do
error -> Logger.warn(inspect(error)) error -> Logger.warn(inspect(error))
end end
end end
defp can_be_merged?(val1, val2) when is_map(val1) and is_map(val2), do: true
defp can_be_merged?(val1, val2) when is_list(val1) and is_list(val2) do
Keyword.keyword?(val1) and Keyword.keyword?(val2)
end
defp can_be_merged?(_val1, _val2), do: false
end end

View File

@ -797,7 +797,7 @@ def migrate_from_db(conn, _params) do
end end
end end
def config_show(conn, _params) do def config_show(conn, %{"only_db" => true}) do
with :ok <- configurable_from_database(conn) do with :ok <- configurable_from_database(conn) do
configs = Pleroma.Repo.all(ConfigDB) configs = Pleroma.Repo.all(ConfigDB)
@ -814,6 +814,46 @@ def config_show(conn, _params) do
end end
end end
def config_show(conn, _params) do
with :ok <- configurable_from_database(conn) do
configs = ConfigDB.get_all_as_keyword()
if configs == [] do
errors(
conn,
{:error, "To use configuration from database migrate your settings to database."}
)
else
merged =
Pleroma.Config.Holder.config()
|> DeepMerge.deep_merge(configs)
|> Enum.map(fn {group, value} ->
Enum.map(value, fn {key, value} ->
db =
if configs[group][key] do
if Keyword.keyword?(value) do
Keyword.keys(value) |> Enum.map(fn key -> ConfigDB.convert(key) end)
else
ConfigDB.convert(key)
end
end
setting = %{
group: ConfigDB.convert(group),
key: ConfigDB.convert(key),
value: ConfigDB.convert(value)
}
if db, do: Map.put(setting, :db, db), else: setting
end)
end)
|> List.flatten()
json(conn, %{configs: merged})
end
end
end
def config_update(conn, %{"configs" => configs}) do def config_update(conn, %{"configs" => configs}) do
with :ok <- configurable_from_database(conn) do with :ok <- configurable_from_database(conn) do
updated = updated =

View File

@ -196,7 +196,6 @@ defmodule Pleroma.Web.Router do
get("/config", AdminAPIController, :config_show) get("/config", AdminAPIController, :config_show)
post("/config", AdminAPIController, :config_update) post("/config", AdminAPIController, :config_update)
get("/config/descriptions", AdminAPIController, :config_descriptions) get("/config/descriptions", AdminAPIController, :config_descriptions)
get("/config/migrate_to_db", AdminAPIController, :migrate_to_db)
get("/config/migrate_from_db", AdminAPIController, :migrate_from_db) get("/config/migrate_from_db", AdminAPIController, :migrate_from_db)
get("/moderation_log", AdminAPIController, :list_log) get("/moderation_log", AdminAPIController, :list_log)

View File

@ -15,8 +15,8 @@ test "get_by_key/1" do
end end
test "create/1" do test "create/1" do
{:ok, config} = ConfigDB.create(%{group: "pleroma", key: "some_key", value: "some_value"}) {:ok, config} = ConfigDB.create(%{group: ":pleroma", key: ":some_key", value: "some_value"})
assert config == ConfigDB.get_by_params(%{group: "pleroma", key: "some_key"}) assert config == ConfigDB.get_by_params(%{group: ":pleroma", key: ":some_key"})
end end
test "update/1" do test "update/1" do
@ -26,6 +26,27 @@ test "update/1" do
assert loaded == updated assert loaded == updated
end end
test "get_all_as_keyword/0" do
insert(:config)
insert(:config, group: ":quack", key: ":level", value: ConfigDB.to_binary(:info))
insert(:config, group: ":quack", key: ":meta", value: ConfigDB.to_binary([:none]))
insert(:config,
group: ":quack",
key: ":webhook_url",
value: ConfigDB.to_binary("https://hooks.slack.com/services/KEY/some_val")
)
assert [
pleroma: [{_, %{another: _, another_key: _}}],
quack: [
level: :info,
meta: [:none],
webhook_url: "https://hooks.slack.com/services/KEY/some_val"
]
] = ConfigDB.get_all_as_keyword()
end
describe "update_or_create/1" do describe "update_or_create/1" do
test "common" do test "common" do
config = insert(:config) config = insert(:config)

View File

@ -47,6 +47,35 @@ test "transfer config values from db to env" do
end) end)
end end
test "transfer config values for 1 group and some keys" do
level = Application.get_env(:quack, :level)
meta = Application.get_env(:quack, :meta)
ConfigDB.create(%{
group: ":quack",
key: ":level",
value: :info
})
ConfigDB.create(%{
group: ":quack",
key: ":meta",
value: [:none]
})
Pleroma.Config.TransferTask.start_link([])
assert Application.get_env(:quack, :level) == :info
assert Application.get_env(:quack, :meta) == [:none]
default = Pleroma.Config.Holder.config(:quack, :webhook_url)
assert Application.get_env(:quack, :webhook_url) == default
on_exit(fn ->
Application.put_env(:quack, :level, level)
Application.put_env(:quack, :meta, meta)
end)
end
test "non existing atom" do test "non existing atom" do
ConfigDB.create(%{ ConfigDB.create(%{
group: ":pleroma", group: ":pleroma",

9
test/fixtures/config/temp.secret.exs vendored Normal file
View File

@ -0,0 +1,9 @@
use Mix.Config
config :pleroma, :first_setting, key: "value", key2: [Pleroma.Repo]
config :pleroma, :second_setting, key: "value2", key2: ["Activity"]
config :quack, level: :info
config :pleroma, Pleroma.Repo, pool: Ecto.Adapters.SQL.Sandbox

View File

@ -4,6 +4,9 @@
defmodule Mix.Tasks.Pleroma.ConfigTest do defmodule Mix.Tasks.Pleroma.ConfigTest do
use Pleroma.DataCase use Pleroma.DataCase
import ExUnit.CaptureLog
alias Pleroma.ConfigDB alias Pleroma.ConfigDB
alias Pleroma.Repo alias Pleroma.Repo
@ -23,16 +26,17 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do
Pleroma.Config.put(:configurable_from_database, true) Pleroma.Config.put(:configurable_from_database, true)
end end
test "warning if file with custom settings doesn't exist" do
assert capture_log(fn -> Mix.Tasks.Pleroma.Config.run(["migrate_to_db"]) end) =~
"to migrate settings, you must define custom settings in config/test.secret.exs"
end
test "settings are migrated to db" do test "settings are migrated to db" do
initial = Application.get_env(:quack, :level) initial = Application.get_env(:quack, :level)
on_exit(fn -> Application.put_env(:quack, :level, initial) end) on_exit(fn -> Application.put_env(:quack, :level, initial) end)
assert Repo.all(ConfigDB) == [] assert Repo.all(ConfigDB) == []
Application.put_env(:pleroma, :first_setting, key: "value", key2: [Repo]) Mix.Tasks.Pleroma.Config.migrate_to_db("test/fixtures/config/temp.secret.exs")
Application.put_env(:pleroma, :second_setting, key: "value2", key2: ["Activity"])
Application.put_env(:quack, :level, :info)
Mix.Tasks.Pleroma.Config.run(["migrate_to_db"])
config1 = ConfigDB.get_by_params(%{group: ":pleroma", key: ":first_setting"}) config1 = ConfigDB.get_by_params(%{group: ":pleroma", key: ":first_setting"})
config2 = ConfigDB.get_by_params(%{group: ":pleroma", key: ":second_setting"}) config2 = ConfigDB.get_by_params(%{group: ":pleroma", key: ":second_setting"})

View File

@ -7,6 +7,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
use Oban.Testing, repo: Pleroma.Repo use Oban.Testing, repo: Pleroma.Repo
alias Pleroma.Activity alias Pleroma.Activity
alias Pleroma.ConfigDB
alias Pleroma.HTML alias Pleroma.HTML
alias Pleroma.ModerationLog alias Pleroma.ModerationLog
alias Pleroma.Repo alias Pleroma.Repo
@ -1881,11 +1882,11 @@ test "without any settings in db", %{conn: conn} do
"To use configuration from database migrate your settings to database." "To use configuration from database migrate your settings to database."
end end
test "with settings in db", %{conn: conn} do test "with settings only in db", %{conn: conn} do
config1 = insert(:config) config1 = insert(:config)
config2 = insert(:config) config2 = insert(:config)
conn = get(conn, "/api/pleroma/admin/config") conn = get(conn, "/api/pleroma/admin/config", %{"only_db" => true})
%{ %{
"configs" => [ "configs" => [
@ -1895,6 +1896,7 @@ test "with settings in db", %{conn: conn} do
"value" => _ "value" => _
}, },
%{ %{
"group" => ":pleroma",
"key" => key2, "key" => key2,
"value" => _ "value" => _
} }
@ -1904,6 +1906,45 @@ test "with settings in db", %{conn: conn} do
assert key1 == config1.key assert key1 == config1.key
assert key2 == config2.key assert key2 == config2.key
end end
test "merged default setting with db settings", %{conn: conn} do
config1 = insert(:config)
config2 = insert(:config)
config3 =
insert(:config,
value: ConfigDB.to_binary(k1: :v1, k2: :v2)
)
conn = get(conn, "/api/pleroma/admin/config")
%{"configs" => configs} = json_response(conn, 200)
assert length(configs) > 3
received_configs =
Enum.filter(configs, fn %{"group" => group, "key" => key} ->
group == ":pleroma" and key in [config1.key, config2.key, config3.key]
end)
assert length(received_configs) == 3
db_keys =
config3.value
|> ConfigDB.from_binary()
|> Keyword.keys()
|> ConfigDB.convert()
Enum.each(received_configs, fn %{"value" => value, "db" => db} ->
assert db in [config1.key, config2.key, db_keys]
assert value in [
ConfigDB.from_binary_with_convert(config1.value),
ConfigDB.from_binary_with_convert(config2.value),
ConfigDB.from_binary_with_convert(config3.value)
]
end)
end
end end
test "POST /api/pleroma/admin/config error", %{conn: conn} do test "POST /api/pleroma/admin/config error", %{conn: conn} do
@ -2831,9 +2872,8 @@ test "proxy tuple ip", %{conn: conn} do
end end
test "transfer settings to DB and to file", %{conn: conn} do test "transfer settings to DB and to file", %{conn: conn} do
on_exit(fn -> :ok = File.rm("config/test.exported_from_db.secret.exs") end)
assert Repo.all(Pleroma.ConfigDB) == [] assert Repo.all(Pleroma.ConfigDB) == []
Mix.Tasks.Pleroma.Config.run(["migrate_to_db"]) Mix.Tasks.Pleroma.Config.migrate_to_db("test/fixtures/config/temp.secret.exs")
assert Repo.aggregate(Pleroma.ConfigDB, :count, :id) > 0 assert Repo.aggregate(Pleroma.ConfigDB, :count, :id) > 0
conn = get(conn, "/api/pleroma/admin/config/migrate_from_db") conn = get(conn, "/api/pleroma/admin/config/migrate_from_db")