./ct_report/coverage/mongoose_config_parser_toml.COVER.html

1 %% @doc Config parsing and processing for the TOML format
2 -module(mongoose_config_parser_toml).
3
4 -behaviour(mongoose_config_parser).
5
6 -export([parse_file/1]).
7
8 %% Utilities for section manipulation
9 -export([process/3]).
10
11 -ifdef(TEST).
12 -export([process/1,
13 extract_errors/1]).
14 -endif.
15
16 -include("mongoose_config_spec.hrl").
17
18 %% Input: TOML parsed by tomerl
19 -type toml_key() :: binary().
20 -type toml_value() :: tomerl:value().
21 -type toml_section() :: tomerl:section().
22
23 %% Output: list of config records, containing key-value pairs
24 -type option_value() :: atom() | binary() | string() | float(). % parsed leaf value
25 -type config_part() :: term(). % any part of a top-level option value, may contain config errors
26 -type top_level_config() :: {mongoose_config:key(), mongoose_config:value()}.
27 -type config_error() :: #{class := error, what := atom(), text := string(), any() => any()}.
28 -type config() :: top_level_config() | config_error().
29
30 -type list_processor() :: fun((path(), [config_part()]) -> config_part())
31 | fun(([config_part()]) -> config_part()).
32
33 -type processor() :: fun((path(), config_part()) -> config_part())
34 | fun((config_part()) -> config_part()).
35
36 -type step() ::
37 parse % Recursive processing (section/list) or type conversion (leaf option)
38
39 | validate % Value check with one of the predefined validators
40
41 | format_items % Optional formatting of section/list items as a map
42
43 | process % Optional processing of the value with a custom function
44
45 | wrap. % Wrapping the value into a list, which will be concatenated
46 % with other items of the parent node.
47 % In case of a KV pair the key is also added here.
48
49 %% Path from the currently processed config node to the root
50 %% - toml_key(): key in a toml_section()
51 %% - item: item in a list
52 %% - {host, Host}: item in the list of hosts in host_config
53 -type path() :: [toml_key() | item | {host, jid:server()}].
54
55 -export_type([toml_key/0, toml_value/0, toml_section/0,
56 option_value/0, config/0, config_error/0, config_part/0,
57 list_processor/0, processor/0]).
58
59 -spec parse_file(FileName :: string()) -> mongoose_config_parser:state().
60 parse_file(FileName) ->
61 101 case tomerl:read_file(FileName) of
62 {ok, Content} ->
63 101 process(Content);
64 {error, Error} ->
65 
:-(
 error(config_error([#{what => toml_parsing_failed, text => Error}]))
66 end.
67
68 -spec process(toml_section()) -> mongoose_config_parser:state().
69 process(Content) ->
70 101 Config = parse(Content),
71 101 Hosts = proplists:get_value(hosts, Config, []),
72 101 HostTypes = proplists:get_value(host_types, Config, []),
73 101 case extract_errors(Config) of
74 [] ->
75 101 mongoose_config_parser:build_state(Hosts, HostTypes, Config);
76 Errors ->
77 
:-(
 error(config_error(Errors))
78 end.
79
80 config_error(Errors) ->
81 
:-(
 {config_error, "Could not read the TOML configuration file", Errors}.
82
83 -spec parse(toml_section()) -> [config()].
84 parse(Content) ->
85 101 handle([], Content, mongoose_config_spec:root()).
86
87 %% TODO replace with binary_to_existing_atom where possible, prevent atom leak
88 85227 b2a(B) -> binary_to_atom(B, utf8).
89
90 -spec ensure_keys([toml_key()], toml_section()) -> any().
91 ensure_keys(Keys, Section) ->
92 14238 case lists:filter(fun(Key) -> not maps:is_key(Key, Section) end, Keys) of
93 14238 [] -> ok;
94 
:-(
 MissingKeys -> error(#{what => missing_mandatory_keys, missing_keys => MissingKeys})
95 end.
96
97 -spec parse_section(path(), toml_section(), mongoose_config_spec:config_section()) ->
98 [config_part()].
99 parse_section(Path, M, #section{items = Items, defaults = Defaults}) ->
100 14238 FilteredDefaults = maps:filter(fun(K, _V) -> not maps:is_key(K, M) end, Defaults),
101 14238 M1 = maps:merge(get_always_included(Items), M),
102 14238 ProcessedConfig = maps:map(fun(K, V) -> handle([K|Path], V, get_spec_for_key(K, Items)) end, M1),
103 14238 ProcessedDefaults = maps:map(fun(K, V) -> handle_default([K|Path], V, maps:get(K, Items)) end,
104 FilteredDefaults),
105 14238 lists:flatmap(fun({_K, ConfigParts}) -> ConfigParts end,
106 lists:keysort(1, maps:to_list(maps:merge(ProcessedDefaults, ProcessedConfig)))).
107
108 -spec get_spec_for_key(toml_key(), map()) -> mongoose_config_spec:config_node().
109 get_spec_for_key(Key, Items) ->
110 31908 case maps:is_key(Key, Items) of
111 true ->
112 28594 maps:get(Key, Items);
113 false ->
114 3314 case maps:find(default, Items) of
115 3314 {ok, Spec} -> Spec;
116 
:-(
 error -> error(#{what => unexpected_key, key => Key, items => Items})
117 end
118 end.
119
120 get_always_included(Items) ->
121 14238 maps:from_list([{K, #{}} || {K, #section{include = always}} <- maps:to_list(Items)]).
122
123 -spec parse_list(path(), [toml_value()], mongoose_config_spec:config_list()) -> [config_part()].
124 parse_list(Path, L, #list{items = ItemSpec}) ->
125 4218 lists:flatmap(fun(Elem) ->
126 6201 Key = item_key(Path, Elem),
127 6201 handle([Key|Path], Elem, ItemSpec)
128 end, L).
129
130 -spec handle(path(), toml_value(), mongoose_config_spec:config_node()) -> [config_part()].
131 handle(Path, Value, Spec = #option{}) ->
132 19754 handle(Path, Value, Spec, [parse, validate, process, wrap]);
133 handle(Path, Value, Spec) ->
134 18456 handle(Path, Value, Spec, [parse, validate, format_items, process, wrap]).
135
136 -spec handle_default(path(), toml_value(), mongoose_config_spec:config_node()) -> [config_part()].
137 handle_default(Path, Value, Spec) ->
138 17870 handle(Path, Value, Spec, [wrap]).
139
140 -spec handle(path(), toml_value(), mongoose_config_spec:config_node(), [step()]) -> [config_part()].
141 handle(Path, Value, Spec, Steps) ->
142 56080 lists:foldl(fun(_, [#{what := _, class := error}|_] = Errors) ->
143 
:-(
 Errors;
144 (Step, Acc) ->
145 189166 try_step(Step, Path, Value, Acc, Spec)
146 end, Value, Steps).
147
148 -spec handle_step(step(), path(), toml_value(), mongoose_config_spec:config_node()) ->
149 config_part().
150 handle_step(parse, Path, Value, Spec) ->
151 38210 ParsedValue = case Spec of
152 #section{} when is_map(Value) ->
153 14238 check_required_keys(Spec, Value),
154 14238 validate_keys(Spec, Value),
155 14238 parse_section(Path, Value, Spec);
156 #list{} when is_list(Value) ->
157 4218 parse_list(Path, Value, Spec);
158 #option{type = Type} when not is_list(Value), not is_map(Value) ->
159 19754 convert(Value, Type)
160 end,
161 38210 case extract_errors(ParsedValue) of
162 38210 [] -> ParsedValue;
163 
:-(
 Errors -> Errors
164 end;
165 handle_step(format_items, _Path, Items, Spec) ->
166 18456 format_items(Items, format_items_spec(Spec));
167 handle_step(validate, _Path, ParsedValue, Spec) ->
168 38210 validate(ParsedValue, Spec),
169 38210 ParsedValue;
170 handle_step(process, Path, ParsedValue, Spec) ->
171 38210 process(Path, ParsedValue, process_spec(Spec));
172 handle_step(wrap, Path, ProcessedValue, Spec) ->
173 56080 wrap(Path, ProcessedValue, wrap_spec(Spec)).
174
175 -spec check_required_keys(mongoose_config_spec:config_section(), toml_section()) -> any().
176 check_required_keys(#section{required = all, items = Items}, Section) ->
177 2828 ensure_keys(maps:keys(Items), Section);
178 check_required_keys(#section{required = Required}, Section) ->
179 11410 ensure_keys(Required, Section).
180
181 -spec validate_keys(mongoose_config_spec:config_section(), toml_section()) -> any().
182 validate_keys(#section{validate_keys = Validator}, Section) ->
183 14238 lists:foreach(fun(Key) ->
184 30397 mongoose_config_validator:validate(b2a(Key), atom, Validator)
185 end, maps:keys(Section)).
186
187 -spec format_items_spec(mongoose_config_spec:config_node()) -> mongoose_config_spec:format_items().
188 14238 format_items_spec(#section{format_items = FormatItems}) -> FormatItems;
189 4218 format_items_spec(#list{format_items = FormatItems}) -> FormatItems.
190
191 -spec format_items(config_part(), mongoose_config_spec:format_items()) -> config_part().
192 format_items(KVs, map) ->
193 12685 Keys = lists:map(fun({K, _}) -> K end, KVs),
194 12685 mongoose_config_validator:validate_list(Keys, unique),
195 12685 maps:from_list(KVs);
196 format_items(Value, list) when is_list(Value) ->
197 5771 Value.
198
199 -spec validate(config_part(), mongoose_config_spec:config_node()) -> any().
200 validate(Value, #section{validate = Validator}) ->
201 14238 mongoose_config_validator:validate_section(Value, Validator);
202 validate(Value, #list{validate = Validator}) ->
203 4218 mongoose_config_validator:validate_list(Value, Validator);
204 validate(Value, #option{type = Type, validate = Validator}) ->
205 19754 mongoose_config_validator:validate(Value, Type, Validator).
206
207 -spec process_spec(mongoose_config_spec:config_section() |
208 mongoose_config_spec:config_list()) -> undefined | list_processor();
209 (mongoose_config_spec:config_option()) -> undefined | processor().
210 14238 process_spec(#section{process = Process}) -> Process;
211 4218 process_spec(#list{process = Process}) -> Process;
212 19754 process_spec(#option{process = Process}) -> Process.
213
214 -spec process(path(), config_part(), undefined | processor()) -> config_part().
215 36547 process(_Path, V, undefined) -> V;
216 4961 process(_Path, V, F) when is_function(F, 1) -> F(V);
217 8512 process(Path, V, F) when is_function(F, 2) -> F(Path, V).
218
219 -spec convert(toml_value(), mongoose_config_spec:option_type()) -> option_value().
220 448 convert(V, boolean) when is_boolean(V) -> V;
221 1861 convert(V, binary) when is_binary(V) -> V;
222 5199 convert(V, string) -> binary_to_list(V);
223 5492 convert(V, atom) -> b2a(V);
224 101 convert(<<"infinity">>, int_or_infinity) -> infinity; %% TODO maybe use TOML '+inf'
225 1050 convert(V, int_or_infinity) when is_integer(V) -> V;
226 303 convert(V, int_or_atom) when is_integer(V) -> V;
227 1919 convert(V, int_or_atom) -> b2a(V);
228 3381 convert(V, integer) when is_integer(V) -> V;
229 
:-(
 convert(V, float) when is_float(V) -> V.
230
231 -spec wrap_spec(mongoose_config_spec:config_node()) -> mongoose_config_spec:wrapper().
232 15115 wrap_spec(#section{wrap = Wrap}) -> Wrap;
233 6363 wrap_spec(#list{wrap = Wrap}) -> Wrap;
234 34602 wrap_spec(#option{wrap = Wrap}) -> Wrap.
235
236 -spec wrap(path(), config_part(), mongoose_config_spec:wrapper()) -> [config_part()].
237 wrap([Key|_] = Path, V, host_config) ->
238 987 [{{b2a(Key), get_host(Path)}, V}];
239 wrap([Key|_] = Path, V, global_config) ->
240 2077 global = get_host(Path),
241 2077 [{b2a(Key), V}];
242 wrap([item|_], V, default) ->
243 5961 [V];
244 wrap([Key|_], V, default) ->
245 44355 [{b2a(Key), V}];
246 wrap(_Path, V, item) ->
247 303 [V];
248 wrap(_Path, _V, remove) ->
249 240 [];
250 wrap(_Path, V, none) when is_list(V) ->
251 2157 V.
252
253 -spec get_host(path()) -> jid:server() | global.
254 get_host(Path) ->
255 3064 case lists:reverse(Path) of
256 381 [<<"host_config">>, {host, Host} | _] -> Host;
257 2683 _ -> global
258 end.
259
260 -spec try_step(step(), path(), toml_value(), term(),
261 mongoose_config_spec:config_node()) -> config_part().
262 try_step(Step, Path, Value, Acc, Spec) ->
263 189166 try
264 189166 handle_step(Step, Path, Acc, Spec)
265 catch error:Reason:Stacktrace ->
266 
:-(
 BasicFields = #{what => toml_processing_failed,
267 class => error,
268 stacktrace => Stacktrace,
269 text => "TOML configuration error: " ++ error_text(Step),
270 toml_path => path_to_string(Path),
271 toml_value => Value},
272 
:-(
 ErrorFields = error_fields(Reason),
273 
:-(
 [maps:merge(BasicFields, ErrorFields)]
274 end.
275
276 -spec error_text(step()) -> string().
277 
:-(
 error_text(parse) -> "Malformed node";
278 
:-(
 error_text(validate) -> "Invalid node value";
279 
:-(
 error_text(format_items) -> "List or section has invalid key-value pairs";
280 
:-(
 error_text(process) -> "Node could not be processed";
281 
:-(
 error_text(wrap) -> "Node could not be wrapped in a config option".
282
283 -spec error_fields(any()) -> map().
284 
:-(
 error_fields(#{what := Reason} = M) -> maps:remove(what, M#{reason => Reason});
285 
:-(
 error_fields(Reason) -> #{reason => Reason}.
286
287 -spec path_to_string(path()) -> string().
288 path_to_string(Path) ->
289 
:-(
 Items = lists:flatmap(fun node_to_string/1, lists:reverse(Path)),
290 
:-(
 string:join(Items, ".").
291
292 
:-(
 node_to_string(item) -> [];
293 
:-(
 node_to_string({host, _}) -> [];
294 
:-(
 node_to_string(Node) -> [binary_to_list(Node)].
295
296 -spec item_key(path(), toml_value()) -> {host, jid:server()} | item.
297 141 item_key([<<"host_config">>], #{<<"host_type">> := Host}) -> {host, Host};
298 99 item_key([<<"host_config">>], #{<<"host">> := Host}) -> {host, Host};
299 5961 item_key(_, _) -> item.
300
301 %% Processing of the parsed options
302
303 %% Any nested config_part() may be a config_error() - this function extracts them all recursively
304 -spec extract_errors([config()]) -> [config_error()].
305 extract_errors(Config) ->
306 38311 extract(fun(#{what := _, class := error}) -> true;
307 1587605 (_) -> false
308 end, Config).
309
310 -spec extract(fun((config_part()) -> boolean()), config_part()) -> [config_part()].
311 extract(Pred, Data) ->
312 1587605 case Pred(Data) of
313 
:-(
 true -> [Data];
314 1587605 false -> extract_items(Pred, Data)
315 end.
316
317 -spec extract_items(fun((config_part()) -> boolean()), config_part()) -> [config_part()].
318 460815 extract_items(Pred, L) when is_list(L) -> lists:flatmap(fun(El) -> extract(Pred, El) end, L);
319 306724 extract_items(Pred, T) when is_tuple(T) -> extract_items(Pred, tuple_to_list(T));
320 53898 extract_items(Pred, M) when is_map(M) -> extract_items(Pred, maps:to_list(M));
321 1126790 extract_items(_, _) -> [].
Line Hits Source