Current section
Files
Jump to
Current section
Files
lib/mavlink/parser.ex
defmodule XMAVLink.Parser do
@moduledoc """
Parse a mavlink xml file into an idiomatic Elixir representation:
%{
version: 2,
dialect: 0,
enums: [
%{
name: :mav_autopilot,
description: "Micro air vehicle...",
entries: [
%{
value: 0,
name: :mav_autopilot_generic, (use atoms for identifiers)
description: "Generic autopilot..."
params: [ (only used by commands)
%{
index: 0,
description: ""
},
... more entry params
]
},
... more enum entries
]
},
... more enums
],
messages: [
%{
id: 0,
name: "optical_flow",
description: "Optical flow...",
fields: [
%{
type: "uint16_t",
ordinality: 1,
name: "flow_x",
units: "dpixels", (note: string not atom)
description: "Flow in pixels..."
},
... more message fields
]
},
... more messages
]
}
"""
import Enum, only: [empty?: 1, reduce: 3, reverse: 1, map: 2, sort_by: 2, into: 3, filter: 2]
import Record, only: [defrecord: 2, extract: 2]
import Regex, only: [replace: 3]
import String, only: [to_integer: 1, downcase: 1, to_atom: 1, split: 3]
@identifier_regex ~r/\A[A-Za-z][A-Za-z0-9_]*\z/
@unit_regex ~r/\A[A-Za-z0-9_%@\/\*\^\.\-]+\z/
@scalar_field_types ~w(char uint8_t int8_t uint16_t int16_t uint32_t int32_t uint64_t int64_t float double)
@valid_display_values ~w(bitmask)
@default_limits %{
max_xml_file_bytes: 2_000_000,
max_include_depth: 32,
max_include_files: 128
}
@xmerl_header "xmerl/include/xmerl.hrl"
defrecord :xmlElement, extract(:xmlElement, from_lib: @xmerl_header)
defrecord :xmlAttribute, extract(:xmlAttribute, from_lib: @xmerl_header)
defrecord :xmlText, extract(:xmlText, from_lib: @xmerl_header)
@type mavlink_definition :: %{
version: String.t(),
dialect: String.t(),
enums: [enum_description],
messages: [message_description]
}
@type parser_limit :: pos_integer | :infinity
@type parse_option ::
{:max_xml_file_bytes, parser_limit}
| {:max_include_depth, parser_limit}
| {:max_include_files, parser_limit}
@type parse_options :: [parse_option]
@spec parse_mavlink_xml(String.t()) ::
mavlink_definition | {:error, String.t()}
def parse_mavlink_xml(path) do
parse_mavlink_xml(path, [])
end
@spec parse_mavlink_xml(String.t(), parse_options) ::
mavlink_definition | {:error, String.t()}
def parse_mavlink_xml(path, opts) when is_list(opts) do
with {:ok, limits} <- parse_limits(opts) do
path
|> parse_mavlink_xml_file(initial_acc(limits))
|> combined_definition()
end
end
@doc false
def parse_mavlink_xml(path, paths) when is_map(paths) do
acc =
if Map.has_key?(paths, :seen) and Map.has_key?(paths, :definitions) do
normalize_acc(paths)
else
normalize_acc(%{
seen: MapSet.new(Map.keys(paths)),
definitions: Map.values(paths)
})
end
parse_mavlink_xml_file(path, acc)
end
defp combined_definition({:error, _message} = error), do: error
defp combined_definition(%{definitions: definitions}) do
definitions
|> reverse()
|> combine_definitions()
|> validate_definition()
end
defp initial_acc(limits) do
%{
seen: MapSet.new(),
definitions: [],
stack: [],
file_count: 0,
limits: limits
}
end
defp normalize_acc(acc) do
seen =
acc
|> Map.get(:seen, MapSet.new())
|> MapSet.to_list()
|> map(&Path.expand/1)
|> MapSet.new()
Map.merge(acc, %{
seen: seen,
definitions: Map.get(acc, :definitions, []),
stack: Map.get(acc, :stack, []),
file_count: Map.get(acc, :file_count, 0),
limits: Map.get(acc, :limits, @default_limits)
})
end
defp parse_limits(opts) do
reduce(opts, {:ok, @default_limits}, fn
_option, {:error, _message} = error ->
error
{key, value}, {:ok, limits} when is_map_key(@default_limits, key) ->
if valid_limit?(value) do
{:ok, Map.put(limits, key, value)}
else
{:error, "Invalid MAVLink XML parser limit #{inspect(key)}: #{inspect(value)}"}
end
option, {:ok, _limits} ->
{:error, "Invalid MAVLink XML parser option #{inspect(option)}"}
end)
end
defp valid_limit?(:infinity), do: true
defp valid_limit?(value), do: is_integer(value) and value > 0
defp parse_mavlink_xml_file(path, acc) do
path_key = Path.expand(path)
cond do
path_key in acc.stack ->
{:error,
"Cyclic MAVLink XML include detected: #{format_path_chain(acc.stack ++ [path_key])}"}
MapSet.member?(acc.seen, path_key) ->
# Don't include a file twice
acc
exceeds_limit?(length(acc.stack) + 1, acc.limits.max_include_depth) ->
{:error,
"MAVLink XML include depth for '#{path_key}' exceeds max_include_depth limit of #{format_limit(acc.limits.max_include_depth)} in #{format_path_chain(acc.stack ++ [path_key])}"}
exceeds_limit?(acc.file_count + 1, acc.limits.max_include_files) ->
{:error,
"MAVLink XML include graph exceeds max_include_files limit of #{format_limit(acc.limits.max_include_files)} while parsing '#{path_key}'"}
true ->
parse_new_mavlink_xml_file(path, path_key, acc)
end
end
defp parse_new_mavlink_xml_file(path, path_key, acc) do
parent_stack = acc.stack
acc = %{
acc
| seen: MapSet.put(acc.seen, path_key),
stack: parent_stack ++ [path_key],
file_count: acc.file_count + 1
}
case scan_file(path, acc.limits) do
{defs, []} ->
with {:ok, acc} <- parse_includes(defs, path_key, acc),
{:ok, definition} <- parse_definition(defs) do
%{acc | definitions: [definition | acc.definitions], stack: parent_stack}
end
{:error, :enoent} ->
{:error, "File '#{path}' does not exist"}
{:error, message} when is_binary(message) ->
{:error, message}
{:error, message} ->
{:error, "Failed to parse MAVLink XML file '#{path}': #{inspect(message)}"}
end
end
defp parse_includes(defs, path, acc) do
with {:ok, includes} <- include_paths(defs, path),
:ok <- validate_include_conflicts(includes, path) do
reduce(includes, {:ok, acc}, fn
_include, {:error, _message} = error ->
error
{_include, include_path}, {:ok, acc} ->
case parse_mavlink_xml_file(include_path, acc) do
{:error, _message} = error -> error
acc -> {:ok, acc}
end
end)
end
end
defp include_paths(defs, path) do
:xmerl_xpath.string(~c"/mavlink/include/text()", defs)
|> map(&extract_text/1)
|> reduce({:ok, []}, fn
_include, {:error, _message} = error ->
error
include, {:ok, _acc} when include in [nil, ""] ->
{:error, "Empty MAVLink XML include in '#{path}'"}
include, {:ok, acc} ->
{:ok, [{include, Path.expand(include, Path.dirname(path))} | acc]}
end)
|> case do
{:ok, includes} -> {:ok, reverse(includes)}
{:error, _message} = error -> error
end
end
defp validate_include_conflicts(includes, path) do
includes
|> Enum.group_by(fn {_include, include_path} -> include_path end, fn {include, _path} ->
include
end)
|> Enum.find(fn {_include_path, include_names} ->
include_names
|> MapSet.new()
|> MapSet.size()
|> Kernel.>(1)
end)
|> case do
nil ->
:ok
{include_path, include_names} ->
names =
include_names
|> Enum.uniq()
|> map(&inspect/1)
|> Enum.join(", ")
{:error,
"Conflicting MAVLink XML includes in '#{path}' resolve to '#{include_path}': #{names}"}
end
end
defp parse_definition(defs) do
with {:ok, enums} <-
parse_elements(:xmerl_xpath.string(~c"/mavlink/enums/enum", defs), &parse_enum/1) do
version =
:xmerl_xpath.string(~c"/mavlink/version/text()", defs)
|> extract_text
|> nil_to_zero_string
case parse_elements(
:xmerl_xpath.string(~c"/mavlink/messages/message", defs),
&parse_message(&1, version)
) do
{:ok, messages} ->
{:ok,
%{
version: version,
dialect:
:xmerl_xpath.string(~c"/mavlink/dialect/text()", defs)
|> extract_text
|> nil_to_zero_string,
enums: enums,
messages: messages
}}
{:error, _message} = error ->
error
end
end
end
defp scan_file(path, limits) do
with :ok <- validate_file_size(path, limits) do
try do
:xmerl_scan.file(path)
catch
kind, reason ->
{:error, "Failed to parse MAVLink XML file '#{path}': #{inspect({kind, reason})}"}
end
end
end
defp validate_file_size(path, limits) do
case File.stat(path) do
{:ok, %{type: :regular, size: size}} ->
if exceeds_limit?(size, limits.max_xml_file_bytes) do
{:error,
"MAVLink XML file '#{path}' is #{size} bytes, exceeding max_xml_file_bytes limit of #{format_limit(limits.max_xml_file_bytes)}"}
else
:ok
end
{:ok, %{type: type}} ->
{:error, "MAVLink XML path '#{path}' is not a regular file: #{inspect(type)}"}
{:error, :enoent} ->
{:error, :enoent}
{:error, reason} ->
{:error, "Failed to stat MAVLink XML file '#{path}': #{inspect(reason)}"}
end
end
defp exceeds_limit?(_value, :infinity), do: false
defp exceeds_limit?(value, limit), do: value > limit
defp format_limit(limit) do
case limit do
:infinity -> "infinity"
limit -> Integer.to_string(limit)
end
end
defp format_path_chain(paths) do
paths
|> map(&Path.basename/1)
|> Enum.join(" -> ")
end
def combine_definitions([single_def]) do
single_def
end
def combine_definitions([
%{
version: v1,
dialect: d1,
enums: e1,
messages: m1
},
%{
version: v2,
dialect: d2,
enums: e2,
messages: m2
}
| more_definitions
]) do
combine_definitions([
%{
# strings > nil
version: max(v1, v2),
dialect: max(d1, d2),
enums: merge_enums(e1, e2),
messages: sort_by(m1 ++ m2, & &1.id)
}
| more_definitions
])
end
def merge_enums(as, bs) do
a_index = into(as, %{}, fn enum -> {enum.name, enum} end)
b_index = into(bs, %{}, fn enum -> {enum.name, enum} end)
only_in_a =
for name <- filter(Map.keys(a_index), &(!Map.has_key?(b_index, &1))), do: a_index[name]
only_in_b =
for name <- filter(Map.keys(b_index), &(!Map.has_key?(a_index, &1))), do: b_index[name]
in_a_and_b =
for name <- filter(Map.keys(a_index), &Map.has_key?(b_index, &1)) do
%{
a_index[name]
| description:
preferred_description(a_index[name].description, b_index[name].description),
bitmask: a_index[name].bitmask or b_index[name].bitmask,
entries: sort_by(a_index[name].entries ++ b_index[name].entries, & &1.value)
}
end
sort_by(only_in_a ++ in_a_and_b ++ only_in_b, & &1.name)
end
@type enum_description :: %{
name: atom,
bitmask: boolean,
description: String.t(),
entries: [entry_description]
}
@spec parse_enum(tuple) :: {:ok, enum_description} | {:error, String.t()}
defp parse_enum(element) do
raw_name = :xmerl_xpath.string(~c"@name", element) |> extract_text
with {:ok, enum_name} <- required_identifier(raw_name, "enum name", "enum"),
{:ok, entries} <-
parse_elements(
:xmerl_xpath.string(~c"/enum/entry", element),
&parse_entry(&1, enum_name)
) do
{:ok,
%{
name: enum_name |> downcase |> to_atom,
bitmask: :xmerl_xpath.string(~c"@bitmask", element) |> extract_text |> true_string?,
description:
:xmerl_xpath.string(~c"/enum/description/text()", element)
|> extract_text
|> nil_to_empty_string,
entries: entries
}}
end
end
@type entry_description :: %{
value: integer | nil,
name: atom,
description: String.t(),
params: [param_description]
}
@spec parse_entry(tuple, String.t()) :: {:ok, entry_description} | {:error, String.t()}
defp parse_entry(element, enum_name) do
# Apparently optional in common.xml?
value_attr = :xmerl_xpath.string(~c"@value", element)
raw_name = :xmerl_xpath.string(~c"@name", element) |> extract_text
context = "enum #{enum_name}"
with {:ok, entry_name} <- required_identifier(raw_name, "enum entry name", context),
{:ok, value} <-
optional_integer(value_attr, "enum entry value", "#{context} entry #{entry_name}"),
{:ok, params} <-
parse_elements(
:xmerl_xpath.string(~c"/entry/param", element),
&parse_param(&1, entry_name)
) do
{:ok,
%{
value: value,
name: entry_name |> downcase |> to_atom,
description:
:xmerl_xpath.string(~c"/entry/description/text()", element)
|> extract_text
|> nil_to_empty_string,
params: params
}}
end
end
@type param_description :: %{
index: integer,
description: String.t()
}
@spec parse_param(tuple, String.t()) :: {:ok, param_description} | {:error, String.t()}
defp parse_param(element, entry_name) do
with {:ok, index} <-
required_integer(
:xmerl_xpath.string(~c"@index", element) |> extract_text,
"param index",
"entry #{entry_name}"
) do
{:ok,
%{
index: index,
description:
:xmerl_xpath.string(~c"/param/text()", element)
|> extract_text
|> nil_to_empty_string
}}
end
end
@type message_description :: %{
id: integer,
name: String.t(),
description: String.t(),
has_ext_fields: boolean,
fields: [field_description]
}
@spec parse_message(tuple, String.t()) :: {:ok, message_description} | {:error, String.t()}
defp parse_message(element, version) do
raw_name = :xmerl_xpath.string(~c"@name", element) |> extract_text
with {:ok, message_id} <-
required_integer(
:xmerl_xpath.string(~c"@id", element) |> extract_text,
"message id",
"message"
),
{:ok, message_name} <-
required_identifier(raw_name, "message name", "message id #{message_id}") do
message_description =
reduce(
xmlElement(element, :content),
%{
id: message_id,
name: message_name,
description:
:xmerl_xpath.string(~c"/message/description/text()", element)
|> extract_text
|> nil_to_empty_string,
has_ext_fields: false,
fields: []
},
fn
_next_child, {:error, _message} = error ->
error
next_child, acc ->
case xmlElement(next_child, :name) do
:field ->
case parse_field(next_child, version, acc.has_ext_fields, message_name) do
{:ok, field} -> %{acc | fields: [field | acc.fields]}
{:error, _message} = error -> error
end
:extensions ->
%{acc | has_ext_fields: true}
_ ->
acc
end
end
)
case message_description do
{:error, _message} = error ->
error
message_description ->
{:ok, %{message_description | fields: reverse(message_description.fields)}}
end
end
end
@type field_description :: %{
type: String.t(),
ordinality: integer,
omit_arg: boolean,
is_extension: boolean,
constant_val: String.t() | nil,
name: String.t(),
enum: String.t(),
display: :bitmask | nil,
print_format: String.t() | nil,
units: atom | nil,
description: String.t()
}
@spec parse_field(tuple, binary(), boolean, String.t()) ::
{:ok, field_description} | {:error, String.t()}
defp parse_field(element, version, is_extension_field, message_name) do
context = "message #{message_name}"
with {:ok, {type, ordinality, omit_arg, constant_val}} <-
:xmerl_xpath.string(~c"@type", element)
|> extract_text
|> parse_type_ordinality_omit_arg_constant_val(version, context),
{:ok, field_name} <-
:xmerl_xpath.string(~c"@name", element)
|> extract_text
|> required_identifier("field name", context),
{:ok, enum} <-
:xmerl_xpath.string(~c"@enum", element)
|> extract_text
|> optional_identifier("field enum", "#{context} field #{field_name}"),
{:ok, display} <-
:xmerl_xpath.string(~c"@display", element)
|> extract_text
|> optional_display("#{context} field #{field_name}"),
{:ok, units} <-
:xmerl_xpath.string(~c"@units", element)
|> extract_text
|> optional_unit("#{context} field #{field_name}") do
{:ok,
%{
type: type,
ordinality: ordinality,
omit_arg: omit_arg,
is_extension: is_extension_field,
constant_val: constant_val,
# You can't downcase this, wrecks crc_extra calc for POWER_STATUS
name: field_name,
enum: enum |> nil_to_empty_string |> downcase,
display: display,
print_format: :xmerl_xpath.string(~c"@print_format", element) |> extract_text,
units: units,
description:
:xmerl_xpath.string(~c"/field/text()", element) |> extract_text |> nil_to_empty_string
}}
end
end
@spec parse_type_ordinality_omit_arg_constant_val(String.t(), String.t(), String.t()) ::
{:ok, {String.t(), integer, boolean, String.t() | nil}} | {:error, String.t()}
defp parse_type_ordinality_omit_arg_constant_val(nil, _version, context) do
{:error, "Missing field type in #{context}"}
end
defp parse_type_ordinality_omit_arg_constant_val(type_string, version, context) do
[type | ordinality] =
type_string
|> split(["[", "]"], trim: true)
case type do
"uint8_t_mavlink_version" ->
{:ok, {"uint8_t", 1, true, version}}
type when type in @scalar_field_types ->
with {:ok, ordinality} <- parse_ordinality(ordinality, context) do
{:ok, {type, ordinality, false, nil}}
end
_ ->
{:error, "Invalid field type #{inspect(type_string)} in #{context}"}
end
end
defp parse_ordinality([], _context), do: {:ok, 1}
defp parse_ordinality([raw_ordinality], context) do
case required_integer(raw_ordinality, "field array length", context) do
{:ok, ordinality} when ordinality > 0 ->
{:ok, ordinality}
{:ok, _ordinality} ->
{:error, "Invalid field array length #{inspect(raw_ordinality)} in #{context}"}
{:error, _message} = error ->
error
end
end
defp parse_ordinality(_ordinality, context),
do: {:error, "Invalid field type array declaration in #{context}"}
defp parse_elements(elements, parser) do
elements
|> reduce({:ok, []}, fn
_element, {:error, _message} = error ->
error
element, {:ok, acc} ->
case parser.(element) do
{:ok, parsed} -> {:ok, [parsed | acc]}
{:error, _message} = error -> error
end
end)
|> case do
{:ok, parsed} -> {:ok, reverse(parsed)}
{:error, _message} = error -> error
end
end
defp validate_definition(definition) do
with :ok <- validate_unique_message_ids(definition.messages),
:ok <- validate_unique_message_modules(definition.messages),
:ok <- validate_unique_enums(definition.enums),
:ok <- validate_enum_entries(definition.enums),
:ok <- validate_message_fields(definition.messages) do
definition
end
end
defp validate_unique_message_ids(messages) do
case duplicate_by(messages, & &1.id) do
nil ->
:ok
{id, duplicate_messages} ->
names = duplicate_messages |> map(& &1.name) |> Enum.join(", ")
{:error, "Duplicate message id #{id} for #{names}"}
end
end
defp validate_unique_message_modules(messages) do
case duplicate_by(messages, &generated_message_module_name(&1.name)) do
nil ->
:ok
{module_name, duplicate_messages} ->
names = duplicate_messages |> map(& &1.name) |> Enum.join(", ")
{:error, "Duplicate generated message module #{module_name} for #{names}"}
end
end
defp validate_unique_enums(enums) do
case duplicate_by(enums, & &1.name) do
nil ->
:ok
{name, _duplicate_enums} ->
{:error, "Duplicate enum #{inspect(name)}"}
end
end
defp validate_enum_entries(enums) do
reduce(enums, :ok, fn
_enum, {:error, _message} = error ->
error
enum, :ok ->
with :ok <- validate_unique_enum_entry_names(enum),
:ok <- validate_unique_enum_entry_values(enum) do
:ok
end
end)
end
defp validate_unique_enum_entry_names(enum) do
case duplicate_by(enum.entries, & &1.name) do
nil ->
:ok
{name, _duplicate_entries} ->
{:error, "Duplicate enum entry #{inspect(name)} in enum #{inspect(enum.name)}"}
end
end
defp validate_unique_enum_entry_values(enum) do
resolved_entries = resolve_enum_entry_values(enum.entries)
case duplicate_by(resolved_entries, & &1.value) do
nil ->
:ok
{value, duplicate_entries} ->
names = duplicate_entries |> map(&inspect(&1.name)) |> Enum.join(", ")
{:error, "Duplicate enum value #{value} in enum #{inspect(enum.name)} for #{names}"}
end
end
defp validate_message_fields(messages) do
reduce(messages, :ok, fn
_message, {:error, _reason} = error ->
error
message, :ok ->
case duplicate_by(message.fields, &(&1.name |> downcase)) do
nil ->
:ok
{name, _duplicate_fields} ->
{:error, "Duplicate field #{inspect(name)} in message #{message.name}"}
end
end)
end
defp resolve_enum_entry_values(entries) do
entries
|> Enum.map_reduce(0, fn entry, next_value ->
value = entry.value || next_value
{%{entry | value: value}, value + 1}
end)
|> elem(0)
end
defp duplicate_by(values, fun) do
values
|> Enum.group_by(fun)
|> Enum.find(fn {_key, grouped_values} -> length(grouped_values) > 1 end)
end
defp preferred_description(_a, b) when b not in [nil, ""], do: b
defp preferred_description(a, _b), do: a
defp generated_message_module_name(message_name) do
message_name
|> String.split("_")
|> map(&String.capitalize/1)
|> Enum.join("")
end
defp required_identifier(nil, kind, context), do: {:error, "Missing #{kind} in #{context}"}
defp required_identifier("", kind, context), do: {:error, "Missing #{kind} in #{context}"}
defp required_identifier(value, kind, context) when is_binary(value) do
if Regex.match?(@identifier_regex, value) do
{:ok, value}
else
{:error, "Invalid #{kind} #{inspect(value)} in #{context}"}
end
end
defp optional_identifier(nil, _kind, _context), do: {:ok, nil}
defp optional_identifier("", _kind, _context), do: {:ok, nil}
defp optional_identifier(value, kind, context), do: required_identifier(value, kind, context)
defp optional_display(nil, _context), do: {:ok, nil}
defp optional_display("", _context), do: {:ok, nil}
defp optional_display(value, _context) when value in @valid_display_values do
{:ok, to_atom(value)}
end
defp optional_display(value, context),
do: {:error, "Invalid field display #{inspect(value)} in #{context}"}
defp optional_unit(nil, _context), do: {:ok, nil}
defp optional_unit("", _context), do: {:ok, nil}
defp optional_unit(value, context) do
if Regex.match?(@unit_regex, value) do
{:ok, to_atom(value)}
else
{:error, "Invalid field unit #{inspect(value)} in #{context}"}
end
end
defp optional_integer([], _kind, _context), do: {:ok, nil}
defp optional_integer(value_attr, kind, context) do
if empty?(value_attr) do
{:ok, nil}
else
value_attr
|> extract_text
|> required_integer(kind, context)
end
end
defp required_integer(nil, kind, context), do: {:error, "Missing #{kind} in #{context}"}
defp required_integer("", kind, context), do: {:error, "Missing #{kind} in #{context}"}
defp required_integer(value, kind, context) when is_binary(value) do
{:ok, to_integer(value)}
rescue
ArgumentError -> {:error, "Invalid #{kind} #{inspect(value)} in #{context}"}
end
# TODO Can't spec this without causing dialyzer "nil can't match binary" - Erlang types?
defp extract_text([xml]), do: extract_text(xml)
defp extract_text(xmlText(value: value)), do: clean_string(value)
defp extract_text(xmlAttribute(value: value)), do: clean_string(value)
defp extract_text(_), do: nil
@spec clean_string([char] | binary) :: String.t()
defp clean_string(s) do
trimmed = s |> List.to_string() |> String.trim()
replace(~r/\s+/, trimmed, " ")
end
@spec nil_to_empty_string(String.t() | nil) :: String.t()
defp nil_to_empty_string(nil), do: ""
defp nil_to_empty_string(value) when is_binary(value), do: value
@spec nil_to_zero_string(String.t() | nil) :: String.t()
defp nil_to_zero_string(nil), do: "0"
defp nil_to_zero_string(value) when is_binary(value), do: value
defp true_string?("true"), do: true
defp true_string?(_), do: false
end