./ct_report/coverage/mongoose_config_parser_toml.COVER.html

1 %% @doc Config parsing and processing for the TOML format
2 -module(mongoose_config_parser_toml).
3
4 -behaviour(mongoose_config_parser).
5
6 -export([parse_file/1]).
7
8 -ifdef(TEST).
9 -export([process/1,
10 extract_errors/1]).
11 -endif.
12
13 -include("mongoose_config_spec.hrl").
14
15 %% Input: TOML parsed by tomerl
16 -type toml_key() :: binary().
17 -type toml_value() :: tomerl:value().
18 -type toml_section() :: tomerl:section().
19
20 %% Output: list of config records, containing key-value pairs
21 -type option_value() :: atom() | binary() | string() | float(). % parsed leaf value
22 -type config_part() :: term(). % any part of a top-level option value, may contain config errors
23 -type top_level_config() :: {mongoose_config:key(), mongoose_config:value()}.
24 -type config_error() :: #{class := error, what := atom(), text := string(), any() => any()}.
25 -type config() :: top_level_config() | config_error().
26
27 -type list_processor() :: fun((path(), [config_part()]) -> config_part())
28 | fun(([config_part()]) -> config_part()).
29
30 -type processor() :: fun((path(), config_part()) -> config_part())
31 | fun((config_part()) -> config_part()).
32
33 -type step() ::
34 parse % Recursive processing (section/list) or type conversion (leaf option)
35
36 | validate % Value check with one of the predefined validators
37
38 | format_items % Optional formatting of section/list items as a map
39
40 | process % Optional processing of the value with a custom function
41
42 | wrap. % Wrapping the value into a list, which will be concatenated
43 % with other items of the parent node.
44 % In case of a KV pair the key is also added here.
45
46 %% Path from the currently processed config node to the root
47 %% - toml_key(): key in a toml_section()
48 %% - item: item in a list
49 %% - {host, Host}: item in the list of hosts in host_config
50 -type path() :: [toml_key() | item | {host, jid:server()}].
51
52 -export_type([toml_key/0, toml_value/0, toml_section/0,
53 option_value/0, config/0, config_error/0, config_part/0,
54 list_processor/0, processor/0]).
55
56 -spec parse_file(FileName :: string()) -> mongoose_config_parser:state().
57 parse_file(FileName) ->
58 80 case tomerl:read_file(FileName) of
59 {ok, Content} ->
60 80 process(Content);
61 {error, Error} ->
62 
:-(
 error(config_error([#{what => toml_parsing_failed, text => Error}]))
63 end.
64
65 -spec process(toml_section()) -> mongoose_config_parser:state().
66 process(Content) ->
67 80 Config = parse(Content),
68 80 Hosts = proplists:get_value(hosts, Config, []),
69 80 HostTypes = proplists:get_value(host_types, Config, []),
70 80 case extract_errors(Config) of
71 [] ->
72 80 build_state(Hosts, HostTypes, Config);
73 Errors ->
74 
:-(
 error(config_error(Errors))
75 end.
76
77 config_error(Errors) ->
78 
:-(
 {config_error, "Could not read the TOML configuration file", Errors}.
79
80 -spec parse(toml_section()) -> [config()].
81 parse(Content) ->
82 80 handle([], Content, mongoose_config_spec:root()).
83
84 %% TODO replace with binary_to_existing_atom where possible, prevent atom leak
85 52715 b2a(B) -> binary_to_atom(B, utf8).
86
87 -spec ensure_keys([toml_key()], toml_section()) -> any().
88 ensure_keys(Keys, Section) ->
89 9668 case lists:filter(fun(Key) -> not maps:is_key(Key, Section) end, Keys) of
90 9668 [] -> ok;
91 
:-(
 MissingKeys -> error(#{what => missing_mandatory_keys, missing_keys => MissingKeys})
92 end.
93
94 -spec parse_section(path(), toml_section(), mongoose_config_spec:config_section()) ->
95 [config_part()].
96 parse_section(Path, M, #section{items = Items, defaults = Defaults}) ->
97 9668 FilteredDefaults = maps:filter(fun(K, _V) -> not maps:is_key(K, M) end, Defaults),
98 9668 M1 = maps:merge(get_always_included(Items), M),
99 9668 ProcessedConfig = maps:map(fun(K, V) -> handle([K|Path], V, get_spec_for_key(K, Items)) end, M1),
100 9668 ProcessedDefaults = maps:map(fun(K, V) -> handle_default([K|Path], V, maps:get(K, Items)) end,
101 FilteredDefaults),
102 9668 lists:flatmap(fun({_K, ConfigParts}) -> ConfigParts end,
103 lists:keysort(1, maps:to_list(maps:merge(ProcessedDefaults, ProcessedConfig)))).
104
105 -spec get_spec_for_key(toml_key(), map()) -> mongoose_config_spec:config_node().
106 get_spec_for_key(Key, Items) ->
107 24256 case maps:is_key(Key, Items) of
108 true ->
109 21186 maps:get(Key, Items);
110 false ->
111 3070 case maps:find(default, Items) of
112 3070 {ok, Spec} -> Spec;
113 
:-(
 error -> error(#{what => unexpected_key, key => Key})
114 end
115 end.
116
117 get_always_included(Items) ->
118 9668 maps:from_list([{K, #{}} || {K, #section{include = always}} <- maps:to_list(Items)]).
119
120 -spec parse_list(path(), [toml_value()], mongoose_config_spec:config_list()) -> [config_part()].
121 parse_list(Path, L, #list{items = ItemSpec}) ->
122 4050 lists:flatmap(fun(Elem) ->
123 6265 Key = item_key(Path, Elem),
124 6265 handle([Key|Path], Elem, ItemSpec)
125 end, L).
126
127 -spec handle(path(), toml_value(), mongoose_config_spec:config_node()) -> [config_part()].
128 handle(Path, Value, Spec) ->
129 30601 handle(Path, Value, Spec, [parse, validate, format_items, process, wrap]).
130
131 -spec handle_default(path(), toml_value(), mongoose_config_spec:config_node()) -> [config_part()].
132 handle_default(Path, Value, Spec) ->
133 2241 handle(Path, Value, Spec, [wrap]).
134
135 -spec handle(path(), toml_value(), mongoose_config_spec:config_node(), [step()]) -> [config_part()].
136 handle(Path, Value, Spec, Steps) ->
137 32842 lists:foldl(fun(_, [#{what := _, class := error}|_] = Errors) ->
138 
:-(
 Errors;
139 (Step, Acc) ->
140 155246 try_step(Step, Path, Value, Acc, Spec)
141 end, Value, Steps).
142
143 -spec handle_step(step(), path(), toml_value(), mongoose_config_spec:config_node()) ->
144 config_part().
145 handle_step(parse, Path, Value, Spec) ->
146 30601 ParsedValue = case Spec of
147 #section{} when is_map(Value) ->
148 9668 check_required_keys(Spec, Value),
149 9668 validate_keys(Spec, Value),
150 9668 parse_section(Path, Value, Spec);
151 #list{} when is_list(Value) ->
152 4050 parse_list(Path, Value, Spec);
153 #option{type = Type} when not is_list(Value), not is_map(Value) ->
154 16883 convert(Value, Type)
155 end,
156 30601 case extract_errors(ParsedValue) of
157 30601 [] -> ParsedValue;
158 
:-(
 Errors -> Errors
159 end;
160 handle_step(format_items, _Path, Items, Spec) ->
161 30601 format_items(Items, format_items_spec(Spec));
162 handle_step(validate, _Path, ParsedValue, Spec) ->
163 30601 validate(ParsedValue, Spec),
164 30601 ParsedValue;
165 handle_step(process, Path, ParsedValue, Spec) ->
166 30601 process(Path, ParsedValue, process_spec(Spec));
167 handle_step(wrap, Path, ProcessedValue, Spec) ->
168 32842 wrap(Path, ProcessedValue, wrap_spec(Spec)).
169
170 -spec check_required_keys(mongoose_config_spec:config_section(), toml_section()) -> any().
171 check_required_keys(#section{required = all, items = Items}, Section) ->
172 2480 ensure_keys(maps:keys(Items), Section);
173 check_required_keys(#section{required = Required}, Section) ->
174 7188 ensure_keys(Required, Section).
175
176 -spec validate_keys(mongoose_config_spec:config_section(), toml_section()) -> any().
177 validate_keys(#section{validate_keys = Validator}, Section) ->
178 9668 lists:foreach(fun(Key) ->
179 24075 mongoose_config_validator:validate(b2a(Key), atom, Validator)
180 end, maps:keys(Section)).
181
182 -spec format_items_spec(mongoose_config_spec:config_node()) -> mongoose_config_spec:format_items().
183 9668 format_items_spec(#section{format_items = FormatItems}) -> FormatItems;
184 4050 format_items_spec(#list{format_items = FormatItems}) -> FormatItems;
185 16883 format_items_spec(#option{}) -> none.
186
187 -spec format_items(config_part(), mongoose_config_spec:format_items()) -> config_part().
188 format_items(KVs, map) ->
189 4307 Keys = lists:map(fun({K, _}) -> K end, KVs),
190 4307 mongoose_config_validator:validate_list(Keys, unique),
191 4307 maps:from_list(KVs);
192 format_items(Value, none) ->
193 26294 Value.
194
195 -spec validate(config_part(), mongoose_config_spec:config_node()) -> any().
196 validate(Value, #section{validate = Validator}) ->
197 9668 mongoose_config_validator:validate_section(Value, Validator);
198 validate(Value, #list{validate = Validator}) ->
199 4050 mongoose_config_validator:validate_list(Value, Validator);
200 validate(Value, #option{type = Type, validate = Validator}) ->
201 16883 mongoose_config_validator:validate(Value, Type, Validator).
202
203 -spec process_spec(mongoose_config_spec:config_section() |
204 mongoose_config_spec:config_list()) -> undefined | list_processor();
205 (mongoose_config_spec:config_option()) -> undefined | processor().
206 9668 process_spec(#section{process = Process}) -> Process;
207 4050 process_spec(#list{process = Process}) -> Process;
208 16883 process_spec(#option{process = Process}) -> Process.
209
210 -spec process(path(), config_part(), undefined | processor()) -> config_part().
211 26244 process(_Path, V, undefined) -> V;
212 1826 process(_Path, V, F) when is_function(F, 1) -> F(V);
213 2531 process(Path, V, F) when is_function(F, 2) -> F(Path, V).
214
215 -spec convert(toml_value(), mongoose_config_spec:option_type()) -> option_value().
216 390 convert(V, boolean) when is_boolean(V) -> V;
217 1102 convert(V, binary) when is_binary(V) -> V;
218 5309 convert(V, string) -> binary_to_list(V);
219 4975 convert(V, atom) -> b2a(V);
220 80 convert(<<"infinity">>, int_or_infinity) -> infinity; %% TODO maybe use TOML '+inf'
221 590 convert(V, int_or_infinity) when is_integer(V) -> V;
222 240 convert(V, int_or_atom) when is_integer(V) -> V;
223 1520 convert(V, int_or_atom) -> b2a(V);
224 2677 convert(V, integer) when is_integer(V) -> V;
225 
:-(
 convert(V, float) when is_float(V) -> V.
226
227 -spec wrap_spec(mongoose_config_spec:config_node()) -> mongoose_config_spec:wrapper().
228 9748 wrap_spec(#section{wrap = Wrap}) -> Wrap;
229 4516 wrap_spec(#list{wrap = Wrap}) -> Wrap;
230 18578 wrap_spec(#option{wrap = Wrap}) -> Wrap.
231
232 -spec wrap(path(), config_part(), mongoose_config_spec:wrapper()) -> [config_part()].
233 wrap([Key|_] = Path, V, host_config) ->
234 632 wrap(Path, V, {host_config, b2a(Key)});
235 wrap([Key|_] = Path, V, global_config) ->
236 1360 wrap(Path, V, {global_config, b2a(Key)});
237 wrap(Path, V, {host_config, Key}) ->
238 712 [{{Key, get_host(Path)}, V}];
239 wrap(Path, V, {global_config, Key}) ->
240 1721 global = get_host(Path),
241 1721 [{Key, V}];
242 wrap([item|_] = Path, V, default) ->
243 6134 wrap(Path, V, item);
244 wrap([Key|_] = Path, V, default) ->
245 20153 wrap(Path, V, {kv, b2a(Key)});
246 wrap(_Path, V, {kv, Key}) ->
247 21629 [{Key, V}];
248 wrap(_Path, V, item) ->
249 6214 [V];
250 wrap(_Path, _V, remove) ->
251 131 [];
252 wrap([Key|_], V, prepend_key) ->
253 
:-(
 L = [b2a(Key) | tuple_to_list(V)],
254 
:-(
 [list_to_tuple(L)];
255 wrap(_Path, V, none) when is_list(V) ->
256 2435 V.
257
258 -spec get_host(path()) -> jid:server() | global.
259 get_host(Path) ->
260 2433 case lists:reverse(Path) of
261 232 [<<"host_config">>, {host, Host} | _] -> Host;
262 2201 _ -> global
263 end.
264
265 -spec try_step(step(), path(), toml_value(), term(),
266 mongoose_config_spec:config_node()) -> config_part().
267 try_step(Step, Path, Value, Acc, Spec) ->
268 155246 try
269 155246 handle_step(Step, Path, Acc, Spec)
270 catch error:Reason:Stacktrace ->
271 
:-(
 BasicFields = #{what => toml_processing_failed,
272 class => error,
273 stacktrace => Stacktrace,
274 text => "TOML configuration error: " ++ error_text(Step),
275 toml_path => path_to_string(Path),
276 toml_value => Value},
277 
:-(
 ErrorFields = error_fields(Reason),
278 
:-(
 [maps:merge(BasicFields, ErrorFields)]
279 end.
280
281 -spec error_text(step()) -> string().
282 
:-(
 error_text(parse) -> "Malformed node";
283 
:-(
 error_text(validate) -> "Invalid node value";
284 
:-(
 error_text(format_items) -> "List or section has invalid key-value pairs";
285 
:-(
 error_text(process) -> "Node could not be processed";
286 
:-(
 error_text(wrap) -> "Node could not be wrapped in a config option".
287
288 -spec error_fields(any()) -> map().
289 
:-(
 error_fields(#{what := Reason} = M) -> maps:remove(what, M#{reason => Reason});
290 
:-(
 error_fields(Reason) -> #{reason => Reason}.
291
292 -spec path_to_string(path()) -> string().
293 path_to_string(Path) ->
294 
:-(
 Items = lists:flatmap(fun node_to_string/1, lists:reverse(Path)),
295 
:-(
 string:join(Items, ".").
296
297 
:-(
 node_to_string(item) -> [];
298 
:-(
 node_to_string({host, _}) -> [];
299 
:-(
 node_to_string(Node) -> [binary_to_list(Node)].
300
301 -spec item_key(path(), toml_value()) -> {host, jid:server()} | item.
302 101 item_key([<<"host_config">>], #{<<"host_type">> := Host}) -> {host, Host};
303 30 item_key([<<"host_config">>], #{<<"host">> := Host}) -> {host, Host};
304 6134 item_key(_, _) -> item.
305
306 %% Processing of the parsed options
307
308 -spec build_state([jid:server()], [jid:server()], [top_level_config()]) ->
309 mongoose_config_parser:state().
310 build_state(Hosts, HostTypes, Opts) ->
311 80 lists:foldl(fun(F, StateIn) -> F(StateIn) end,
312 mongoose_config_parser:new_state(),
313 80 [fun(S) -> mongoose_config_parser:set_hosts(Hosts, S) end,
314 80 fun(S) -> mongoose_config_parser:set_host_types(HostTypes, S) end,
315 80 fun(S) -> mongoose_config_parser:set_opts(Opts, S) end,
316 fun mongoose_config_parser:unfold_globals/1,
317 fun mongoose_config_parser:post_process_modules/1]).
318
319 %% Any nested config_part() may be a config_error() - this function extracts them all recursively
320 -spec extract_errors([config()]) -> [config_error()].
321 extract_errors(Config) ->
322 30681 extract(fun(#{what := _, class := error}) -> true;
323 904247 (_) -> false
324 end, Config).
325
326 -spec extract(fun((config_part()) -> boolean()), config_part()) -> [config_part()].
327 extract(Pred, Data) ->
328 904247 case Pred(Data) of
329 
:-(
 true -> [Data];
330 904247 false -> extract_items(Pred, Data)
331 end.
332
333 -spec extract_items(fun((config_part()) -> boolean()), config_part()) -> [config_part()].
334 236275 extract_items(Pred, L) when is_list(L) -> lists:flatmap(fun(El) -> extract(Pred, El) end, L);
335 129036 extract_items(Pred, T) when is_tuple(T) -> extract_items(Pred, tuple_to_list(T));
336 16922 extract_items(Pred, M) when is_map(M) -> extract_items(Pred, maps:to_list(M));
337 667972 extract_items(_, _) -> [].
Line Hits Source