Current section
Files
Jump to
Current section
Files
lib/jsonpath.ex
defmodule JsonPath do
@moduledoc """
RFC 9535 compliant JSONPath implementation for Elixir.
"""
@typedoc "JSONPath AST generated by the parser"
@type ast :: {:jsonpath, :root, list(any())} | map() | {:path, list(any())}
@typedoc "A node path in the JSON structure"
@type path :: list(String.t() | integer())
@typedoc "Result of evaluating a JSONPath AST: list of {path_string, value}"
@type eval_result :: list({String.t(), any()})
@typedoc "Result of tokenize/1"
@type tokenize_result :: {:ok, list(any()), integer()} | {:error, any()}
@typedoc "Result of parse/1"
@type parse_result :: {:ok, ast()} | {:error, any()}
@doc """
Tokenizes a JSONPath query string into a list of tokens using the LEEX lexer.
"""
@spec tokenize(String.t()) :: tokenize_result()
def tokenize(query) when is_binary(query) do
trimmed = String.trim(query)
if trimmed != query do
{:error, "JSONPath expressions cannot have leading or trailing whitespace"}
else
ensure_loaded(:jsonpath_lexer)
:jsonpath_lexer.string(String.to_charlist(query))
end
end
@doc """
Parses a list of tokens into an AST using the YECC parser.
"""
@spec parse(list(any())) :: parse_result()
def parse(tokens) when is_list(tokens) do
try do
ensure_loaded(:jsonpath_parser)
:jsonpath_parser.parse(tokens)
rescue
e in CaseClauseError -> {:error, e.term}
e -> {:error, e.term}
end
end
@doc """
Evaluate a JSONPath AST against data according to RFC 9535.
"""
@spec evaluate(ast(), map() | list(), keyword()) :: eval_result() | {:error, any()}
def evaluate(ast, data, opts \\ []) do
opts_with_root = Keyword.put(opts, :root_data, data)
case ast do
{:jsonpath, :root, segments} when is_list(segments) ->
traverse([{["$"], data}], segments, opts_with_root)
%{root: :root, segments: segs} when is_list(segs) ->
traverse([{["$"], data}], segs, opts_with_root)
%{segments: segs} when is_list(segs) ->
traverse([{["$"], data}], segs, opts_with_root)
{:path, segs} when is_list(segs) ->
traverse([{["$"], data}], segs, opts_with_root)
other ->
{:error, {:invalid_ast, other}}
end
end
@doc """
Convenience function: parse and evaluate a JSONPath query string in one call.
"""
@spec query(map() | list(), String.t(), keyword()) :: eval_result() | {:error, any()}
def query(data, path_string, opts \\ [])
when is_binary(path_string) and (is_map(data) or is_list(data)) do
with {:ok, tokens, _line} <- tokenize(path_string),
{:ok, ast} <- parse(tokens) do
# IO.inspect(ast, label: "DEBUG: AST")
evaluate(ast, data, opts)
else
{:error, reason} -> {:error, reason}
end
end
## PRIVATE HELPERS
@spec ensure_loaded(module()) :: :ok
defp ensure_loaded(module) do
case :code.is_loaded(module) do
{:file, _} -> :ok
_ -> :code.load_file(module)
end
end
@spec traverse(list({path(), any()}), list(any()), keyword()) :: eval_result()
defp traverse(nodes, [], _opts),
do: Enum.map(nodes, fn {p, v} -> {path_join(p), v} end)
defp traverse(nodes, [seg | rest], opts) do
# Pass root data through opts for absolute queries in filters
root_data = Keyword.get(opts, :root_data)
opts_with_root =
if root_data, do: opts, else: Keyword.put(opts, :root_data, get_root_data(nodes))
expanded =
Enum.flat_map(nodes, fn {path, node} ->
# IO.inspect({seg, path, node}, label: "traversal")
apply_segment(seg, {path, node}, opts_with_root)
end)
traverse(expanded, rest, opts_with_root)
end
# apply child or descendant segment
defp apply_segment({child_type, selectors}, {path, node}, opts) do
case child_type do
:child ->
Enum.flat_map(selectors, fn sel ->
apply_selector(sel, {path, node}, opts)
end)
:descendant ->
nodes = collect_descendants({path, node})
Enum.flat_map(nodes, fn n ->
Enum.flat_map(selectors, fn sel ->
apply_selector(sel, n, opts)
end)
end)
end
end
defp collect_descendants({path, value}) do
# include the node itself
base = [{path, value}]
children =
case value do
%{} = m ->
Enum.flat_map(Map.to_list(m), fn {k, v} ->
child_path = path ++ [to_string(k)]
collect_descendants({child_path, v})
end)
l when is_list(l) ->
Enum.with_index(l)
|> Enum.flat_map(fn {v, idx} ->
child_path = path ++ [idx]
collect_descendants({child_path, v})
end)
_ ->
[]
end
base ++ children
end
defp apply_selector({:name, name}, {path, node}, opts) do
case node do
%{} = m ->
case lookup(m, name, opts) do
{:ok, v, actual_key} -> [{path ++ [actual_key], v}]
:error -> []
end
_ ->
[]
end
end
defp apply_selector({:wildcard}, {path, node}, _opts) do
case node do
%{} = m ->
Enum.map(Map.to_list(m), fn {k, v} -> {path ++ [to_string(k)], v} end)
l when is_list(l) ->
Enum.with_index(l) |> Enum.map(fn {v, i} -> {path ++ [i], v} end)
_ ->
[]
end
end
defp apply_selector({:index, idx}, {path, node}, _opts) when is_integer(idx) do
case node do
l when is_list(l) ->
n = length(l)
real_i = if idx < 0, do: n + idx, else: idx
if real_i >= 0 and real_i < n, do: [{path ++ [real_i], Enum.at(l, real_i)}], else: []
_ ->
[]
end
end
defp apply_selector({:slice, slice}, {path, node}, _opts) do
case node do
l when is_list(l) ->
indices = normalize_slice(length(l), slice)
Enum.map(indices, fn i -> {path ++ [i], Enum.at(l, i)} end)
_ ->
[]
end
end
defp apply_selector({:union, items}, {path, node}, opts) do
Enum.flat_map(items, fn item ->
apply_selector(item, {path, node}, opts)
end)
end
defp apply_selector({:filter, expr}, {path, node}, opts) do
case node do
l when is_list(l) ->
Enum.with_index(l)
|> Enum.flat_map(fn {v, i} ->
if eval_filter(expr, v, opts), do: [{path ++ [i], v}], else: []
end)
%{} = m ->
# For objects, we filter the object itself
if eval_filter(expr, m, opts), do: [{path, m}], else: []
_ ->
[]
end
end
# slice normalization: accepts slice shape produced by parser
defp normalize_slice(len, {:start_end_step, s, e, step}) do
range_from_slice(len, s, e, step)
end
defp normalize_slice(len, {:start_end, s, e}), do: range_from_slice(len, s, e, 1)
defp normalize_slice(len, {:start_end_omitted_step, s, step}),
do: range_from_slice(len, s, nil, step)
defp normalize_slice(len, {:start_omitted_end, e}), do: range_from_slice(len, nil, e, 1)
defp normalize_slice(len, {:start_omitted_end_step, e, step}),
do: range_from_slice(len, nil, e, step)
defp normalize_slice(len, {:start_end_omitted, s}), do: range_from_slice(len, s, nil, 1)
defp normalize_slice(len, {:start_omitted_end_step, step}),
do: range_from_slice(len, nil, nil, step)
defp normalize_slice(len, {:omitted_all}), do: range_from_slice(len, nil, nil, 1)
defp range_from_slice(len, start, stop, step) do
step = step || 1
# Return empty for zero step
if step == 0 do
[]
else
# Handle negative indices and nil values
{st, sp} = normalize_start_stop(len, start, stop, step)
# Generate the range
generate_range(st, sp, step)
end
end
defp normalize_start_stop(len, start, stop, step) do
# Normalize start
st =
case start do
nil when step > 0 -> 0
nil when step < 0 -> len - 1
# Handle negative indices
n when n < 0 -> max(0, len + n)
# Out of bounds positive start
n when n >= len and step > 0 -> len
# Out of bounds positive start with negative step
n when n >= len and step < 0 -> len - 1
# Normal positive index, but ensure >= 0
n -> max(0, n)
end
# Normalize stop
sp =
case stop do
nil when step > 0 -> len
nil when step < 0 -> -1
# Handle negative indices, allow -1 for negative step
n when n < 0 -> max(-1, len + n)
# For positive step, cap at len
n when step > 0 -> min(n, len)
# For negative step, don't cap positive stop values
n -> n
end
# Additional bounds checking for start
st =
cond do
# Start beyond array for positive step
step > 0 and st >= len -> len
# Start before array for negative step
step < 0 and st < 0 -> -1
# Ensure non-negative for positive step
step > 0 -> max(0, st)
# Cap at last valid index for negative step
step < 0 -> min(st, len - 1)
true -> st
end
{st, sp}
end
defp generate_range(st, sp, step) when step > 0 do
if st >= sp do
[]
else
# For positive step: start..(stop-1)//step
Enum.to_list(st..(sp - 1)//step)
end
end
defp generate_range(st, sp, step) when step < 0 do
if st <= sp do
[]
else
# For negative step: start..(stop+1)//step
Enum.to_list(st..(sp + 1)//step)
end
end
## FILTER EVAL: RFC 9535 compliant filter evaluation
defp eval_filter({:or, a, b}, node, opts),
do: eval_filter(a, node, opts) or eval_filter(b, node, opts)
defp eval_filter({:and, a, b}, node, opts),
do: eval_filter(a, node, opts) and eval_filter(b, node, opts)
defp eval_filter({:not, a}, node, opts), do: not eval_filter(a, node, opts)
defp eval_filter({:cmp, op, left, right}, node, opts) do
l = eval_comparable(left, node, opts)
r = eval_comparable(right, node, opts)
compare_values(op, l, r)
end
# Test expression - checks existence or function result
defp eval_filter({:query, :relative, qsegs}, node, _opts) do
results = run_query(node, qsegs)
# According to RFC 9535: non-empty nodelist = true, empty = false
not Enum.empty?(results)
end
defp eval_filter({:query, :absolute, qsegs}, _node, opts) do
case Keyword.get(opts, :root_data) do
nil ->
false
root_data ->
results = run_query(root_data, qsegs)
not Enum.empty?(results)
end
end
defp eval_filter({:function, name, args}, node, opts) do
# Functions in test expressions must return LogicalType or NodesType
# IO.puts("DEBUG: eval_filter function call - #{name}")
# IO.inspect(args, label: "DEBUG: function args raw")
result = eval_function(name, args, node, opts)
# IO.inspect(result, label: "DEBUG: function result 1")
case result do
# LogicalType
:logical_true -> true
:logical_false -> false
# NodesType converted to LogicalType
nodelist when is_list(nodelist) -> not Enum.empty?(nodelist)
# Should not happen with well-typed expressions
_ -> false
end
end
# Comparable evaluation for comparison expressions
defp eval_comparable({:lit, :null}, _node, _opts), do: nil
defp eval_comparable({:lit, v}, _node, _opts), do: v
defp eval_comparable({:query, :relative, qsegs}, node, _opts) do
case run_query(node, qsegs) do
# Empty nodelist
[] -> :nothing
# Singular query - take first value
[{_path, val} | _] -> val
end
end
defp eval_comparable({:query, :absolute, qsegs}, _node, opts) do
case Keyword.get(opts, :root_data) do
nil ->
:nothing
root_data ->
case run_query(root_data, qsegs) do
[{_path, val} | _] -> val
[] -> :nothing
end
end
end
defp eval_comparable({:function, name, args}, node, opts) do
# Functions in comparables must return ValueType
eval_function(name, args, node, opts)
end
# Function evaluation according to RFC 9535 type system
defp eval_function(name, args, node, opts) do
# IO.puts("DEBUG: eval_function called - #{name}")
# IO.inspect(args, label: "DEBUG: raw args")
# IO.inspect(node, label: "DEBUG: current node")
evaluated_args =
Enum.map(args, fn arg ->
resp = eval_function_argument(arg, node, opts)
# IO.inspect({arg, node, opts, resp}, label: "DEBUG: eval_function_argument_arg - #{name}")
resp
end)
# IO.inspect({name, evaluated_args, node, opts}, label: "DEBUG: evaluated args - #{name}")
result = apply_function(name, evaluated_args, node, opts)
# IO.inspect(result, label: "DEBUG: function result 2")
result
end
# Function argument evaluation - preserves types
defp eval_function_argument({:lit, :null}, _node, _opts), do: nil
defp eval_function_argument({:lit, v}, _node, _opts), do: v
defp eval_function_argument({:query, :relative, qsegs}, node, _opts) do
# DEBUG: Let's see what queries return
result = run_query(node, qsegs)
# IO.inspect({:relative_query, qsegs, result}, label: "DEBUG: relative query")
result
end
defp eval_function_argument({:query, :absolute, qsegs}, _node, opts) do
result =
case Keyword.get(opts, :root_data) do
nil ->
[]
root_data ->
query_result = run_query(root_data, qsegs)
# IO.inspect({:absolute_query, qsegs, query_result}, label: "DEBUG: absolute query")
query_result
end
# IO.inspect(result, label: "DEBUG: absolute query result")
result
end
defp eval_function_argument({:function, name, args}, node, opts) do
# IO.inspect({name, node, opts}, label: "DEBUG: eval_function_argument: function")
# For logical expressions, return LogicalType
eval_function(name, args, node, opts)
end
defp eval_function_argument(logical_expr, node, opts) do
# IO.inspect({logical_expr, node, opts}, label: "DEBUG: eval_function_argument")
# For logical expressions, return LogicalType
if eval_filter(logical_expr, node, opts), do: :logical_true, else: :logical_false
end
# RFC 9535 compliant function implementations
defp apply_function("length", [value], _node, _opts) do
cond do
is_list(value) -> length(value)
is_map(value) -> map_size(value)
is_binary(value) -> String.length(value)
value == nil -> :nothing
true -> :nothing
end
end
defp apply_function("count", [nodelist], _node, _opts) when is_list(nodelist) do
# count() takes NodesType and returns ValueType (integer)
length(nodelist)
end
defp apply_function("value", [nodelist], _node, _opts) when is_list(nodelist) do
# IO.inspect({nodelist}, label: "DEBUG: value args")
# value() takes NodesType and returns ValueType
case nodelist do
[{_path, val} | _] -> val
[] -> :nothing
end
end
defp apply_function("match", [str_nodelist, pattern_nodelist], _node, _opts) do
# IO.inspect({str_nodelist, pattern_nodelist}, label: "DEBUG: match args")
# Extract values from nodelists
str =
case str_nodelist do
[{_path, val}] when is_binary(val) -> val
val when is_binary(val) -> val
_ -> nil
end
pattern =
case pattern_nodelist do
[{_path, val}] when is_binary(val) -> val
val when is_binary(val) -> val
_ -> nil
end
# IO.inspect({str, pattern}, label: "DEBUG: extracted match values")
if is_binary(str) and is_binary(pattern) do
case Regex.compile("^#{pattern}$") do
{:ok, regex} ->
result = if Regex.match?(regex, str), do: :logical_true, else: :logical_false
# IO.inspect(result, label: "DEBUG: match result")
result
{:error, _} ->
# IO.puts("DEBUG: regex compile failed")
:logical_false
end
else
# IO.puts("DEBUG: match - not both strings")
:logical_false
end
end
defp apply_function("search", [str_nodelist, pattern_nodelist], _node, _opts) do
# IO.inspect({str_nodelist, pattern_nodelist}, label: "DEBUG: search args")
# Extract values from nodelists
str =
case str_nodelist do
[{_path, val}] when is_binary(val) -> val
val when is_binary(val) -> val
_ -> nil
end
pattern =
case pattern_nodelist do
[{_path, val}] when is_binary(val) -> val
val when is_binary(val) -> val
_ -> nil
end
# IO.inspect({str, pattern}, label: "DEBUG: extracted search values")
if is_binary(str) and is_binary(pattern) do
case Regex.compile(pattern) do
{:ok, regex} ->
result = if Regex.run(regex, str), do: :logical_true, else: :logical_false
# IO.inspect(result, label: "DEBUG: search result")
result
{:error, _} ->
# IO.puts("DEBUG: regex compile failed")
:logical_false
end
else
# IO.puts("DEBUG: search - not both strings")
:logical_false
end
end
# Fallback for invalid arguments or unknown functions
defp apply_function(_name, _args, _node, _opts) do
# IO.inspect({name, args}, label: "DEBUG: unknown function")
:nothing
end
# RFC 9535 comparison semantics
defp compare_values(:eq, a, b), do: equal?(a, b)
defp compare_values(:lt, a, b), do: less?(a, b)
defp compare_values(:ne, a, b), do: not compare_values(:eq, a, b)
defp compare_values(:le, a, b), do: compare_values(:lt, a, b) or compare_values(:eq, a, b)
defp compare_values(:gt, a, b), do: compare_values(:lt, b, a)
defp compare_values(:ge, a, b), do: compare_values(:gt, a, b) or compare_values(:eq, a, b)
defp compare_values(_, _, _), do: false
# RFC 9535 equality semantics
defp equal?(a, b) do
cond do
a == :nothing and b == :nothing ->
true
a == :nothing or b == :nothing ->
false
a == nil and b == nil ->
true
is_number(a) and is_number(b) ->
a == b
is_binary(a) and is_binary(b) ->
a == b
is_boolean(a) and is_boolean(b) ->
a == b
is_list(a) and is_list(b) ->
length(a) == length(b) and Enum.zip(a, b) |> Enum.all?(fn {x, y} -> equal?(x, y) end)
is_map(a) and is_map(b) ->
Map.keys(a) |> Enum.sort() == Map.keys(b) |> Enum.sort() and
Enum.all?(Map.keys(a), fn k -> equal?(Map.fetch!(a, k), Map.fetch!(b, k)) end)
true ->
false
end
end
# RFC 9535 ordering semantics
defp less?(a, b) do
cond do
a == :nothing or b == :nothing ->
false
is_number(a) and is_number(b) ->
a < b
is_binary(a) and is_binary(b) ->
cond do
a == "" and b != "" -> true
a != "" and b == "" -> false
true -> a < b
end
true ->
false
end
end
defp run_query(node, qsegs) do
# RFC 9535 compliant query execution - returns nodelist
walk_query([{[], node}], qsegs)
end
defp walk_query(nodes, []), do: nodes
defp walk_query(nodes, [seg | rest]) do
# IO.inspect({nodes, seg, rest}, label: "DEBUG: walk_query")
next =
Enum.flat_map(nodes, fn {path, node} ->
case seg do
{:qname, name} ->
case node do
%{} = m ->
name_str = to_string(name)
case Map.fetch(m, name_str) do
{:ok, v} -> [{path ++ [name_str], v}]
:error -> []
end
_ ->
[]
end
{:qindex, idx} when is_integer(idx) ->
case node do
l when is_list(l) ->
n = length(l)
i = if idx < 0, do: n + idx, else: idx
if i >= 0 and i < n, do: [{path ++ [i], Enum.at(l, i)}], else: []
_ ->
[]
end
{:qwildcard} ->
case node do
%{} = m ->
Enum.map(Map.to_list(m), fn {k, v} -> {path ++ [to_string(k)], v} end)
l when is_list(l) ->
Enum.with_index(l) |> Enum.map(fn {v, i} -> {path ++ [i], v} end)
_ ->
[]
end
{:qslice, slice} ->
case node do
l when is_list(l) ->
indices = normalize_slice(length(l), slice)
Enum.map(indices, fn i -> {path ++ [i], Enum.at(l, i)} end)
_ ->
[]
end
# RFC 9535 descendant selectors in queries
{:qdescendant_name, name} ->
descendants = collect_descendants_for_query({path, node})
Enum.flat_map(descendants, fn {desc_path, desc_node} ->
case desc_node do
%{} = m ->
name_str = to_string(name)
case Map.fetch(m, name_str) do
{:ok, v} -> [{desc_path ++ [name_str], v}]
:error -> []
end
_ ->
[]
end
end)
{:qdescendant_wildcard} ->
descendants = collect_descendants_for_query({path, node})
Enum.flat_map(descendants, fn {desc_path, desc_node} ->
case desc_node do
%{} = m ->
Enum.map(Map.to_list(m), fn {k, v} -> {desc_path ++ [to_string(k)], v} end)
l when is_list(l) ->
Enum.with_index(l) |> Enum.map(fn {v, i} -> {desc_path ++ [i], v} end)
_ ->
[{desc_path, desc_node}]
end
end)
{:qdescendant_index, idx} when is_integer(idx) ->
descendants = collect_descendants_for_query({path, node})
Enum.flat_map(descendants, fn {desc_path, desc_node} ->
case desc_node do
l when is_list(l) ->
n = length(l)
i = if idx < 0, do: n + idx, else: idx
if i >= 0 and i < n, do: [{desc_path ++ [i], Enum.at(l, i)}], else: []
_ ->
[]
end
end)
{:qdescendant_slice, slice} ->
descendants = collect_descendants_for_query({path, node})
Enum.flat_map(descendants, fn {desc_path, desc_node} ->
case desc_node do
l when is_list(l) ->
indices = normalize_slice(length(l), slice)
Enum.map(indices, fn i -> {desc_path ++ [i], Enum.at(l, i)} end)
_ ->
[]
end
end)
end
end)
walk_query(next, rest)
end
defp collect_descendants_for_query({path, value}) do
case value do
%{} = m ->
base = [{path, value}]
children =
Enum.flat_map(Map.to_list(m), fn {k, v} ->
child_path = path ++ [to_string(k)]
collect_descendants_for_query({child_path, v})
end)
base ++ children
l when is_list(l) ->
base = [{path, value}]
children =
Enum.with_index(l)
|> Enum.flat_map(fn {v, idx} ->
child_path = path ++ [idx]
collect_descendants_for_query({child_path, v})
end)
base ++ children
_ ->
[{path, value}]
end
end
defp path_join(parts) do
"$" <>
Enum.map_join(tl(parts), "", fn
part when is_integer(part) ->
"[#{part}]"
part when is_binary(part) ->
"['#{escape_json_string(part)}']"
part when is_atom(part) ->
"['#{escape_json_string(Atom.to_string(part))}']"
end)
end
defp lookup(map, key, opts) when is_map(map) do
mode = Keyword.get(opts, :key_mode, :string)
case {mode, key} do
{:string, k} ->
case Map.fetch(map, to_string(k)) do
{:ok, v} -> {:ok, v, to_string(k)}
:error -> :error
end
{:atom, k} when is_binary(k) ->
atom_key = String.to_atom(k)
case Map.fetch(map, atom_key) do
{:ok, v} -> {:ok, v, atom_key}
:error -> :error
end
{:both, k} ->
str_k = to_string(k)
case Map.fetch(map, str_k) do
{:ok, v} ->
{:ok, v, str_k}
:error ->
atom_k = String.to_atom(str_k)
case Map.fetch(map, atom_k) do
{:ok, v} -> {:ok, v, atom_k}
:error -> :error
end
end
end
end
defp get_root_data([{["$"], root_data} | _]), do: root_data
defp get_root_data(_), do: nil
defp escape_json_string(s) do
s
|> String.replace("\\", "\\\\")
|> String.replace("\"", "\\\"")
|> String.replace("\'", "\\\'")
|> String.replace("\n", "\\n")
|> String.replace("\r", "\\r")
|> String.replace("\t", "\\t")
|> String.replace("\b", "\\b")
|> String.replace("\f", "\\f")
end
end