Current section
Files
Jump to
Current section
Files
lib/expug/expression_tokenizer.ex
defmodule Expug.ExpressionTokenizer do
@moduledoc ~S"""
Tokenizes an expression.
This is used by `Expug.Tokenizer` to match attribute values and support multiline.
`expression/2` is used to capture an expression token.
state
|> Expug.ExpressionTokenizer.expression(:attribute_value)
## Valid expressions
Expressions are combination of one or more of these:
- a word without spaces
- a balanced `(` ... `)` pair (or `[`, or `{`)
- a string with single quotes `'...'` or double quotes `"..."`
A balanced pair can have balanced pairs, words, and strings inside them.
Double-quote strings can have `#{...}` interpolation inside them.
## Examples
These are valid expressions:
hello
hello(1 + 2)
"Hello world" # strings
(hello world) # balanced (...) pair
These aren't:
hello world # spaces
hello(world[) # pairs not balanced
"hello #{foo(}" # not balanced inside an interpolation
"""
import Expug.TokenizerTools
def expression(state, token_name) do
state
|> start_empty(token_name)
|> many_of(&expression_fragment/1)
end
def expression_fragment(state) do
state
|> one_of([
&balanced_parentheses/1,
&balanced_braces/1,
&balanced_brackets/1,
&double_quote_string/1,
&single_quote_string/1,
&expression_term/1
])
end
@doc """
Matches simple expressions like `xyz` or even `a+b`.
"""
def expression_term(state) do
state
|> append(~r/^[^\(\)\[\]\{\}"', \n\t]+/)
end
@doc """
Matches simple expressions like `xyz`, but only for inside parentheses.
These can have spaces.
"""
def expression_term_inside(state) do
state
|> append(~r/^[^\(\)\[\]\{\}"']+/)
end
@doc """
Matches balanced `(...)` fragments
"""
def balanced_parentheses(state) do
state
|> balanced_pairs(~r/^\(/, ~r/^\)/)
end
@doc """
Matches balanced `{...}` fragments
"""
def balanced_braces(state) do
state
|> balanced_pairs(~r/^\{/, ~r/^\}/)
end
@doc """
Matches balanced `[...]` fragments
"""
def balanced_brackets(state) do
state
|> balanced_pairs(~r/^\[/, ~r/^\]/)
end
@doc """
Underlying implementation for `balanced_*` functions
"""
def balanced_pairs(state, left, right) do
state
|> append(left)
|> optional(fn s -> s
|> many_of(fn s -> s
|> one_of([
&expression_fragment/1,
&expression_term_inside/1
])
end)
end)
|> append(right)
end
@doc """
Matches an entire double-quoted string, taking care of interpolation and escaping
"""
def double_quote_string(state) do
state
|> append(~r/^"/)
|> optional_many_of(fn s -> s
|> one_of([
&(&1 |> append(~r/^#/) |> balanced_braces()),
&(&1 |> append(~r/^(?:(?:\\")|[^"])/))
])
end)
|> append(~r/^"/)
end
@doc """
Matches an entire double-quoted string, taking care of escaping
"""
def single_quote_string(state) do
state
|> append(~r/^'/)
|> optional_many_of(&(&1 |> append(~r/^(?:(?:\\')|[^'])/)))
|> append(~r/^'/)
end
end