Packages
beancount_ex
0.6.0
An idiomatic Elixir interface to Beancount that serves as the long-term behavioral oracle for a future native Elixir General Ledger.
Current section
Files
Jump to
Current section
Files
lib/beancount/parser/grammar.ex
defmodule Beancount.Parser.Grammar do
@moduledoc false
alias Beancount.Directives.{
Balance,
Close,
Commodity,
Custom,
Document,
Event,
Include,
Note,
Open,
Option,
Pad,
Plugin,
PopTag,
Price,
PushTag,
Query,
Transaction
}
alias Beancount.Parser.{Error, Lexer, Metadata, Posting}
alias Beancount.Value
@date_regex ~r/^(\d{4}-\d{2}-\d{2})\s+/
@spec parse(binary()) :: {:ok, [Beancount.Directive.t()]} | {:error, Error.t()}
def parse(text) do
lines = split_lines(text)
case parse_lines(lines, 1, []) do
{:ok, directives} -> {:ok, Enum.reverse(directives)}
{:error, _} = error -> error
end
end
defp split_lines(text) do
text
|> String.replace("\r\n", "\n")
|> String.split("\n")
end
defp parse_lines([], _line_no, acc), do: {:ok, acc}
defp parse_lines([line | rest], line_no, acc) do
line = String.trim_trailing(line)
cond do
line == "" or String.starts_with?(line, ";") ->
parse_lines(rest, line_no + 1, acc)
org_header?(line) ->
parse_lines(rest, line_no + 1, acc)
undated_directive?(line) ->
with {:ok, directive, rest, next_line} <- parse_undated(line, rest, line_no) do
parse_lines(rest, next_line, [directive | acc])
end
Regex.match?(@date_regex, line) ->
with {:ok, directive, rest, next_line} <- parse_dated(line, rest, line_no) do
parse_lines(rest, next_line, [directive | acc])
end
true ->
{:error, error("expected directive", line_no, 1)}
end
end
defp org_header?(line), do: Regex.match?(~r/^\*+\s/, line)
defp undated_directive?(line) do
String.starts_with?(line, "include ") or
String.starts_with?(line, "option ") or
String.starts_with?(line, "plugin ") or
String.starts_with?(line, "pushtag ") or
String.starts_with?(line, "poptag ")
end
defp parse_undated(line, rest, line_no) do
opts = [line: line_no]
cond do
String.starts_with?(line, "include ") ->
with {:ok, path} <- parse_quoted_rest(String.slice(line, 8..-1//1), opts) do
{:ok, %Include{path: path}, rest, line_no + 1}
end
String.starts_with?(line, "option ") ->
with {:ok, name, value} <- parse_option(String.slice(line, 7..-1//1), opts) do
{:ok, %Option{name: name, value: value}, rest, line_no + 1}
end
String.starts_with?(line, "plugin ") ->
with {:ok, plugin} <- parse_plugin(String.slice(line, 7..-1//1), opts) do
{:ok, plugin, rest, line_no + 1}
end
String.starts_with?(line, "pushtag ") ->
with {:ok, tag} <- parse_tag(String.slice(line, 8..-1//1), opts) do
{:ok, %PushTag{tag: tag}, rest, line_no + 1}
end
String.starts_with?(line, "poptag ") ->
with {:ok, tag} <- parse_tag(String.slice(line, 7..-1//1), opts) do
{:ok, %PopTag{tag: tag}, rest, line_no + 1}
end
end
end
defp parse_dated(line, rest, line_no) do
case Regex.run(@date_regex, line) do
[_, date_text] ->
case Date.from_iso8601(date_text) do
{:ok, date} ->
remainder = line |> String.replace_prefix(date_text, "") |> String.trim_leading()
parse_dated_body(date, remainder, rest, line_no)
{:error, _reason} ->
{:error, error("invalid date #{date_text}", line_no, 1)}
end
_ ->
{:error, error("expected dated directive", line_no, 1)}
end
end
defp parse_dated_body(date, remainder, rest, line_no) do
opts = [line: line_no]
case String.split(remainder, " ", parts: 2) do
["commodity", currency] ->
parse_simple_metadata(
date,
currency,
rest,
line_no,
fn date, currency, metadata ->
%Commodity{date: date, currency: currency, metadata: metadata}
end,
opts
)
[kind, arg] ->
parse_dated_directive(kind, arg, date, rest, line_no, opts)
_ ->
{:error, error("unknown dated directive", line_no, 1)}
end
end
defp parse_dated_directive("open", rest_line, date, rest, line_no, opts),
do: parse_open(date, rest_line, rest, line_no, opts)
defp parse_dated_directive("close", account, date, rest, line_no, opts),
do: parse_close(date, account, rest, line_no, opts)
defp parse_dated_directive("balance", rest_line, date, rest, line_no, opts),
do: parse_balance(date, rest_line, rest, line_no, opts)
defp parse_dated_directive("price", rest_line, date, rest, line_no, opts),
do: parse_price(date, rest_line, rest, line_no, opts)
defp parse_dated_directive("note", rest_line, date, rest, line_no, opts),
do: parse_note(date, rest_line, rest, line_no, opts)
defp parse_dated_directive("document", rest_line, date, rest, line_no, opts),
do: parse_document(date, rest_line, rest, line_no, opts)
defp parse_dated_directive("event", rest_line, date, rest, line_no, opts),
do: parse_event(date, rest_line, rest, line_no, opts)
defp parse_dated_directive("custom", rest_line, date, rest, line_no, opts),
do: parse_custom(date, rest_line, rest, line_no, opts)
defp parse_dated_directive("pad", rest_line, date, rest, line_no, opts),
do: parse_pad(date, rest_line, rest, line_no, opts)
defp parse_dated_directive("query", rest_line, date, rest, line_no, opts),
do: parse_query(date, rest_line, rest, line_no, opts)
defp parse_dated_directive(flag, rest_line, date, rest, line_no, opts),
do: parse_transaction(date, flag, rest_line, rest, line_no, opts)
defp parse_open(date, rest_line, rest, line_no, opts) do
case Lexer.split_tokens(rest_line) do
[account | tail] ->
{metadata_lines, rest, next_line} = collect_metadata_lines(rest, line_no)
with {:ok, account} <- parse_account_token(account, opts),
{:ok, currencies, booking} <- parse_open_tail(tail, opts),
{:ok, metadata} <- parse_metadata(metadata_lines, opts) do
{:ok,
%Open{
date: date,
account: account,
currencies: currencies,
booking: booking,
metadata: metadata
}, rest, next_line}
end
_ ->
{:error, error("invalid open directive", line_no, 1)}
end
end
defp parse_open_tail([], _opts), do: {:ok, [], nil}
defp parse_open_tail([token], opts) do
if quoted_token?(token) do
with {:ok, booking} <- parse_quoted_token(token, opts) do
{:ok, [], booking}
end
else
{:ok, String.split(token, ",", trim: true), nil}
end
end
defp parse_open_tail([currencies, booking], opts) do
with {:ok, booking} <- parse_quoted_token(booking, opts) do
{:ok, String.split(currencies, ",", trim: true), booking}
end
end
defp parse_open_tail(_tokens, opts),
do: {:error, error("invalid open directive", opts[:line], 1)}
defp parse_close(date, account, rest, line_no, opts) do
{metadata_lines, rest, next_line} = collect_metadata_lines(rest, line_no)
with {:ok, account} <- parse_account_token(account, opts),
{:ok, metadata} <- parse_metadata(metadata_lines, opts) do
{:ok, %Close{date: date, account: account, metadata: metadata}, rest, next_line}
end
end
defp parse_simple_metadata(date, token, rest, line_no, builder, opts) do
case Lexer.split_tokens(token) do
[value | _] ->
{metadata_lines, rest, next_line} = collect_metadata_lines(rest, line_no)
with {:ok, value} <- parse_commodity_token(value, opts),
{:ok, metadata} <- parse_metadata(metadata_lines, opts) do
{:ok, builder.(date, value, metadata), rest, next_line}
end
_ ->
{:error, error("invalid directive", line_no, 1)}
end
end
defp parse_balance(date, rest_line, rest, line_no, opts) do
case Lexer.split_tokens(rest_line) do
[account | amount_tokens] ->
{metadata_lines, rest, next_line} = collect_metadata_lines(rest, line_no)
with {:ok, account} <- parse_account_token(account, opts),
{:ok, amount, currency, tolerance} <- parse_balance_amount(amount_tokens, opts),
{:ok, metadata} <- parse_metadata(metadata_lines, opts) do
{:ok,
%Balance{
date: date,
account: account,
amount: amount,
currency: currency,
tolerance: tolerance,
metadata: metadata
}, rest, next_line}
end
_ ->
{:error, error("invalid balance directive", line_no, 1)}
end
end
defp parse_balance_amount(tokens, opts) do
case tokens do
[amount, "~", tolerance, currency] ->
with {:ok, amount} <- parse_number_token(amount, opts),
{:ok, tolerance} <- parse_number_token(tolerance, opts),
{:ok, currency} <- parse_commodity_token(currency, opts) do
{:ok, amount, currency, tolerance}
end
[amount, currency] ->
with {:ok, amount} <- parse_number_token(amount, opts),
{:ok, currency} <- parse_commodity_token(currency, opts) do
{:ok, amount, currency, nil}
end
_ ->
{:error, error("invalid balance amount", opts[:line], 1)}
end
end
defp parse_price(date, rest_line, rest, line_no, opts) do
tokens = Lexer.split_tokens(rest_line)
case tokens do
[currency, amount, quote_currency | _] ->
{metadata_lines, rest, next_line} = collect_metadata_lines(rest, line_no)
with {:ok, currency} <- parse_commodity_token(currency, opts),
{:ok, amount} <- parse_number_token(amount, opts),
{:ok, quote_currency} <- parse_commodity_token(quote_currency, opts),
{:ok, metadata} <- parse_metadata(metadata_lines, opts) do
{:ok,
%Price{
date: date,
commodity: currency,
amount: amount,
currency: quote_currency,
metadata: metadata
}, rest, next_line}
end
_ ->
{:error, error("invalid price directive", line_no, 1)}
end
end
defp parse_note(date, rest_line, rest, line_no, opts) do
case Lexer.split_tokens(rest_line) do
[account | comment_tokens] ->
{metadata_lines, rest, next_line} = collect_metadata_lines(rest, line_no)
with {:ok, account} <- parse_account_token(account, opts),
{:ok, comment} <- parse_quoted_token(Enum.join(comment_tokens, " "), opts),
{:ok, metadata} <- parse_metadata(metadata_lines, opts) do
{:ok, %Note{date: date, account: account, comment: comment, metadata: metadata}, rest,
next_line}
end
_ ->
{:error, error("invalid note directive", line_no, 1)}
end
end
defp parse_document(date, rest_line, rest, line_no, opts) do
case Lexer.split_tokens(rest_line) do
[account | path_tokens] ->
{metadata_lines, rest, next_line} = collect_metadata_lines(rest, line_no)
with {:ok, account} <- parse_account_token(account, opts),
{:ok, path} <- parse_quoted_token(Enum.join(path_tokens, " "), opts),
{:ok, metadata} <- parse_metadata(metadata_lines, opts) do
{:ok, %Document{date: date, account: account, path: path, metadata: metadata}, rest,
next_line}
end
_ ->
{:error, error("invalid document directive", line_no, 1)}
end
end
defp parse_event(date, rest_line, rest, line_no, opts) do
case Lexer.split_tokens(rest_line) do
[type | description_tokens] ->
{metadata_lines, rest, next_line} = collect_metadata_lines(rest, line_no)
with {:ok, type} <- parse_quoted_token(type, opts),
{:ok, description} <- parse_quoted_token(Enum.join(description_tokens, " "), opts),
{:ok, metadata} <- parse_metadata(metadata_lines, opts) do
{:ok, %Event{date: date, type: type, description: description, metadata: metadata},
rest, next_line}
end
_ ->
{:error, error("invalid event directive", line_no, 1)}
end
end
defp parse_custom(date, rest_line, rest, line_no, opts) do
case Lexer.split_tokens(rest_line) do
[type | value_tokens] ->
{metadata_lines, rest, next_line} = collect_metadata_lines(rest, line_no)
with {:ok, type} <- parse_quoted_token(type, opts),
{:ok, values} <- parse_custom_values(value_tokens, opts),
{:ok, metadata} <- parse_metadata(metadata_lines, opts) do
{:ok, %Custom{date: date, type: type, values: values, metadata: metadata}, rest,
next_line}
end
_ ->
{:error, error("invalid custom directive", line_no, 1)}
end
end
defp parse_pad(date, rest_line, rest, line_no, opts) do
case Lexer.split_tokens(rest_line) do
[account, source_account | _] ->
{metadata_lines, rest, next_line} = collect_metadata_lines(rest, line_no)
with {:ok, account} <- parse_account_token(account, opts),
{:ok, source_account} <- parse_account_token(source_account, opts),
{:ok, metadata} <- parse_metadata(metadata_lines, opts) do
{:ok,
%Pad{date: date, account: account, source_account: source_account, metadata: metadata},
rest, next_line}
end
_ ->
{:error, error("invalid pad directive", line_no, 1)}
end
end
defp parse_query(date, rest_line, rest, line_no, opts) do
case Lexer.split_tokens(rest_line) do
[name | bql_tokens] when bql_tokens != [] ->
with {:ok, name} <- parse_quoted_token(name, opts),
{:ok, bql, rest, next_line} <- parse_query_bql(bql_tokens, rest, line_no, opts) do
{:ok, %Query{date: date, name: name, bql: bql, metadata: %{}}, rest, next_line}
end
_ ->
{:error, error("invalid query directive", line_no, 1)}
end
end
defp parse_query_bql(["\""], rest, line_no, _opts) do
collect_multiline_quoted(rest, line_no, [])
end
defp parse_query_bql(tokens, rest, line_no, opts) do
with {:ok, bql} <- parse_quoted_token(Enum.join(tokens, " "), opts) do
{:ok, bql, rest, line_no}
end
end
defp collect_multiline_quoted([line | rest], line_no, acc) do
trimmed = String.trim_trailing(line)
cond do
trimmed == "\"" ->
{:ok, Enum.join(acc, "\n"), rest, line_no + 1}
String.ends_with?(trimmed, "\"") ->
content = String.slice(trimmed, 0, String.length(trimmed) - 1)
{:ok, Enum.join(acc ++ [content], "\n"), rest, line_no + 1}
true ->
collect_multiline_quoted(rest, line_no + 1, acc ++ [line])
end
end
defp collect_multiline_quoted([], line_no, _acc) do
{:error, error("unterminated query string", line_no, 1)}
end
defp parse_transaction(date, flag, rest_line, rest, line_no, opts) do
{tags, links, header} = extract_tags_links(rest_line)
{metadata_lines, posting_lines, rest, next_line} = collect_transaction_body(rest, line_no + 1)
with {:ok, payee, narration} <- parse_payee_narration(header, opts),
{:ok, txn_metadata} <- parse_metadata(metadata_lines, opts),
{:ok, postings} <- parse_postings(posting_lines, opts) do
{:ok,
%Transaction{
date: date,
flag: flag,
payee: payee,
narration: narration,
postings: postings,
tags: tags,
links: links,
metadata: txn_metadata
}, rest, next_line}
end
end
defp collect_transaction_body(rest, line_no, metadata \\ [], postings \\ []) do
case rest do
[line | tail] ->
collect_transaction_line(line, tail, line_no, metadata, postings)
[] ->
{Enum.reverse(metadata), Enum.reverse(postings), [], line_no}
end
end
defp collect_transaction_line(line, tail, line_no, metadata, postings) do
cond do
transaction_body_end?(line) ->
{Enum.reverse(metadata), Enum.reverse(postings), tail, line_no + 1}
posting_line?(line) ->
collect_transaction_body(tail, line_no + 1, metadata, [{line, line_no, []} | postings])
metadata_line?(line) ->
collect_transaction_metadata(line, tail, line_no, metadata, postings)
true ->
{Enum.reverse(metadata), Enum.reverse(postings), [line | tail], line_no}
end
end
defp collect_transaction_metadata(line, tail, line_no, metadata, postings) do
case postings do
[{posting_line, posting_line_no, posting_meta} | rest_postings] ->
collect_transaction_body(tail, line_no + 1, metadata, [
{posting_line, posting_line_no, [line | posting_meta]} | rest_postings
])
[] ->
collect_transaction_body(tail, line_no + 1, [line | metadata], postings)
end
end
defp transaction_body_end?(line), do: line == "" or String.starts_with?(line, ";")
defp collect_metadata_lines(rest, line_no, metadata \\ []) do
case rest do
[line | tail] ->
cond do
line == "" or String.starts_with?(line, ";") ->
{Enum.reverse(metadata), tail, line_no + 1}
metadata_line?(line) ->
collect_metadata_lines(tail, line_no + 1, [line | metadata])
true ->
{Enum.reverse(metadata), rest, line_no}
end
[] ->
{Enum.reverse(metadata), [], line_no}
end
end
defp parse_payee_narration("", _opts), do: {:ok, nil, ""}
defp parse_payee_narration(header, opts) do
tokens = Lexer.split_tokens(header)
case tokens do
[payee, narration] ->
with {:ok, payee} <- parse_quoted_token(payee, opts),
{:ok, narration} <- parse_quoted_token(narration, opts) do
{:ok, payee, narration}
end
[narration] ->
with {:ok, narration} <- parse_quoted_token(narration, opts) do
{:ok, nil, narration}
end
_ ->
{:error, error("invalid transaction header", opts[:line], 1)}
end
end
defp parse_postings(items, opts) do
Enum.reduce_while(items, {:ok, []}, fn {line, line_no, metadata_lines}, {:ok, acc} ->
line_opts = Keyword.put(opts, :line, line_no)
with {:ok, posting} <- Posting.parse_line(line, line_opts),
{:ok, metadata} <- parse_metadata(metadata_lines, line_opts) do
{:cont, {:ok, [%{posting | metadata: metadata} | acc]}}
else
{:error, _} = err -> {:halt, err}
end
end)
|> case do
{:ok, postings} -> {:ok, Enum.reverse(postings)}
other -> other
end
end
defp parse_metadata(lines, opts) do
lines
|> Enum.filter(&metadata_line?/1)
|> parse_metadata_map(opts)
end
defp parse_metadata_map(lines, opts) do
Enum.reduce_while(lines, {:ok, %{}}, fn line, {:ok, acc} ->
case Metadata.parse_line(line, opts) do
{:ok, {key, value}} -> {:cont, {:ok, Map.put(acc, key, value)}}
{:error, _} = err -> {:halt, err}
end
end)
end
defp metadata_line?(line) do
String.starts_with?(line, " ") and not posting_line?(line) and
Regex.match?(~r/^\s+\S+\s*:/, line)
end
defp posting_line?(line) do
String.starts_with?(line, " ") and Regex.match?(~r/^\s+[!?]?[A-Z]/, line) and
not Regex.match?(~r/^\s+\S+\s*:\s+/, line)
end
defp parse_option(text, opts) do
tokens = Lexer.split_tokens(String.trim(text))
case tokens do
[name | value_tokens] ->
with {:ok, name} <- parse_quoted_token(name, opts),
{:ok, value} <- Metadata.parse_value(Enum.join(value_tokens, " "), opts) do
{:ok, name, value}
end
_ ->
{:error, error("invalid option directive", opts[:line], 1)}
end
end
defp parse_plugin(text, opts) do
tokens = Lexer.split_tokens(String.trim(text))
case tokens do
[module] ->
with {:ok, module} <- parse_quoted_token(module, opts) do
{:ok, %Plugin{module: module, config: nil}}
end
[module, config | _] ->
with {:ok, module} <- parse_quoted_token(module, opts),
{:ok, config} <- parse_quoted_token(config, opts) do
{:ok, %Plugin{module: module, config: config}}
end
_ ->
{:error, error("invalid plugin directive", opts[:line], 1)}
end
end
defp parse_tag(text, opts) do
text = String.trim(text)
if String.starts_with?(text, "#") do
{:ok, String.slice(text, 1..-1//1)}
else
{:error, error("expected tag", opts[:line], 1)}
end
end
defp parse_custom_values(tokens, opts) do
values =
Enum.reduce_while(tokens, {:ok, []}, fn token, {:ok, acc} ->
case parse_custom_value(token, opts) do
{:ok, value} -> {:cont, {:ok, [value | acc]}}
{:error, _} = err -> {:halt, err}
end
end)
case values do
{:ok, values} -> {:ok, Enum.reverse(values)}
other -> other
end
end
defp parse_custom_value(token, opts) do
cond do
match?(~s(") <> _, token) ->
parse_quoted_token(token, opts)
String.starts_with?(token, "#") ->
{:ok, %Value.Tag{name: String.slice(token, 1..-1//1)}}
Regex.match?(~r/^\d{4}-\d{2}-\d{2}$/, token) ->
Date.from_iso8601(token)
token in ["TRUE", "FALSE"] ->
Metadata.parse_value(token, opts)
true ->
parse_custom_value_token(token, opts)
end
end
defp parse_custom_value_token(token, opts) do
case Lexer.split_tokens(token) do
[amount, currency | _] ->
with {:ok, amount} <- parse_number_token(amount, opts),
{:ok, currency} <- parse_commodity_token(currency, opts) do
{:ok, %Value.Amount{number: amount, currency: currency}}
end
_ ->
parse_custom_account_or_number(token, opts)
end
end
defp parse_custom_account_or_number(token, opts) do
case Lexer.parse_account(token) do
{:ok, _, ""} -> {:ok, token}
_ -> parse_number_token(token, opts)
end
end
defp extract_tags_links(line) do
tags =
Regex.scan(~r/#([A-Za-z0-9_\-]+)/, line)
|> Enum.map(fn [_, tag] -> tag end)
links =
Regex.scan(~r/\^([A-Za-z0-9_\-]+)/, line)
|> Enum.map(fn [_, link] -> link end)
cleaned =
line
|> String.replace(~r/#[A-Za-z0-9_\-]+/, "")
|> String.replace(~r/\^[A-Za-z0-9_\-]+/, "")
|> String.trim()
{tags, links, cleaned}
end
defp parse_quoted_rest(text, opts), do: parse_quoted_token(String.trim(text), opts)
defp parse_quoted_token(~s(") <> _rest = token, opts) do
if quoted_token?(token) do
case Lexer.parse_quoted_string(token) do
{:ok, value, ""} -> {:ok, value}
_ -> {:error, error("invalid quoted string", opts[:line], 1)}
end
else
{:error, error("expected quoted string", opts[:line], 1)}
end
end
defp parse_quoted_token(_token, opts) do
{:error, error("expected quoted string", opts[:line], 1)}
end
defp parse_account_token(token, opts) do
case Lexer.parse_account(token) do
{:ok, _, ""} -> {:ok, token}
_ -> {:error, error("invalid account", opts[:line], 1)}
end
end
defp parse_commodity_token(token, opts) do
case Lexer.parse_commodity(token) do
{:ok, _, ""} -> {:ok, token}
_ -> {:error, error("invalid commodity", opts[:line], 1)}
end
end
defp parse_number_token(token, opts) do
case Lexer.parse_number(token) do
{:ok, value, ""} -> {:ok, value}
_ -> {:error, error("invalid number", opts[:line], 1)}
end
end
defp quoted_token?(~s(") <> _rest = token), do: String.ends_with?(token, ~s("))
defp quoted_token?(_), do: false
defp error(message, line, column) do
%Error{message: message, line: line, column: column}
end
end