Packages
electric
1.4.5
1.7.7
1.7.6
1.7.5
1.7.4
1.7.3
1.7.2
1.7.1
1.7.0
1.6.10
1.6.9
1.6.8
1.6.7
1.6.6
1.6.5
1.6.4
1.6.3
1.6.2
1.6.1
1.6.0
1.5.1
1.5.0
1.4.16
1.4.16-beta-1
1.4.15
1.4.14
1.4.13
1.4.12
1.4.11
1.4.10
1.4.8
1.4.7
1.4.6
1.4.5
1.4.4
1.4.3
1.4.2
1.4.1
1.4.0
1.3.4
1.3.3
1.3.2
1.2.4
1.2.3
1.2.2
1.2.1
1.2.0
1.1.14
1.1.13
1.1.12
1.1.11
1.1.10
1.1.9
1.1.8
1.1.7
1.1.6
retired
1.1.5
retired
1.1.4
retired
1.1.3
retired
1.1.2
1.1.1
1.1.0
1.0.24
1.0.23
1.0.22
1.0.21
1.0.20
1.0.19
1.0.18
1.0.17
1.0.15
1.0.13
1.0.12
1.0.11
1.0.10
1.0.9
1.0.5
1.0.4
1.0.3
1.0.2
1.0.1
1.0.0
1.0.0-beta.23
1.0.0-beta.22
1.0.0-beta.20
1.0.0-beta.19
1.0.0-beta.18
1.0.0-beta.17
1.0.0-beta.16
1.0.0-beta.15
1.0.0-beta.14
1.0.0-beta.13
1.0.0-beta.12
1.0.0-beta.11
1.0.0-beta.10
1.0.0-beta.9
1.0.0-beta.8
1.0.0-beta.7
1.0.0-beta.6
1.0.0-beta.5
1.0.0-beta.4
1.0.0-beta.3
1.0.0-beta.2
1.0.0-beta.1
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.1
0.8.0
0.7.7
0.7.6
0.7.5
0.7.4
0.7.3
0.7.2
0.7.1
0.7.0
0.6.3
0.6.2
0.6.1
0.5.2
0.4.4
Postgres sync engine. Sync little subsets of your Postgres data into local apps and services.
Current section
Files
Jump to
Current section
Files
lib/pg_interop/array.ex
defmodule PgInterop.Array do
defguardp is_space(c) when c in [?\s, ?\t, ?\n, ?\r, ?\v, ?\f]
@doc ~S"""
Parse a Postgres string-serialized array into a list of strings, unwrapping the escapes. Parses nested arrays.
If a casting function is provided, it will be applied to each element.
Parsing follows SOME of the same rules as the postgres parser, in particular:
1. at most 6 nesting levels are allowed,
2. arrays must be of uniform dimension, i.e. all sub-arrays must have the same number of elements if at the same depth.
This implementation also breaks away from the postgres parser in that some bugs are NOT reimplemented:
- `select '{{1},{{2}}}'::text[];` yields `{{{1}},{{2}}}` in PG, we raise an error
- `select '{{{1}},{2}}'::text[];` yields `{}` in PG, we raise an error
- `select '{{{1}},{2},{{3}}}::text[];` yields `{{{1}},{{NULL}},{{3}}}` in PG, we raise an error
## Examples
iex> ~S|{"(\"2023-06-15 11:18:05.372698+00\",)"}| |> parse()
[~s|("2023-06-15 11:18:05.372698+00",)|]
iex> ~S|{"(\"2023-06-15 11:18:05.372698+00\",)","(\"2023-06-15 11:18:05.372698+00\",)"}| |> parse()
[~s|("2023-06-15 11:18:05.372698+00",)|, ~s|("2023-06-15 11:18:05.372698+00",)|]
iex> ~S|{hello, world, null, "null"}| |> parse()
["hello", "world", nil, "null"]
iex> ~S|{"2023-06-15 11:18:05.372698+00",2023-06-15 11:18:05.372698+00}| |> parse(fn x -> {:ok, n, _} = DateTime.from_iso8601(x); n end)
[~U[2023-06-15 11:18:05.372698Z], ~U[2023-06-15 11:18:05.372698Z]]
iex> ~s|{ "1" , 3 , "2" , 3 3 }| |> parse()
["1", "3", "2", "3 3"]
iex> ~s|{ {{1, 1}, { "2" , 2 }} ,{{"3", 3}, {4, 4} }, { {5, 5},{6, 6} }}| |> parse(&String.to_integer/1)
[[[1, 1], [2, 2]], [[3, 3], [4, 4]], [[5, 5], [6, 6]]]
iex> ~s|{ "1" , "2" , 3 3 , , 4}| |> parse()
** (RuntimeError) Unexpected ',' character
iex> ~s|{ "1" , 3, "2" , 3 3 , }| |> parse()
** (RuntimeError) Unexpected '}' character
iex> ~s|{ {1} ,{ 2 }, {3 }} }| |> parse()
** (RuntimeError) Invalid array syntax
iex> ~s|{{{1} ,{ 2 }, {3 }} | |> parse()
** (RuntimeError) Unexpected end of input
iex> ~s|{"}| |> parse()
** (RuntimeError) Unexpected end of input
iex> ~s|{{1},2,{3}}| |> parse(&String.to_integer/1)
** (RuntimeError) Unexpected array element
iex> ~s|{{{{{{{1}}}}}}}| |> parse()
** (RuntimeError) number of dimensions (7) exceeds maximum of 6
iex> ~s|{ {1} ,{ {2} }, {3 }}| |> parse()
** (RuntimeError) Inconsistent array dimensions
iex> ~s|{ {{1}} ,{2}, {3 }}| |> parse()
** (RuntimeError) Inconsistent array dimensions
"""
def parse(str, casting_fun \\ & &1)
def parse("{}", _), do: []
def parse(str, casting_fun) do
case parse_nested_arrays(str, casting_fun, %{cur_dim: 1}) do
{result, "", _} ->
result
{result, rest, _} ->
if String.match?(rest, ~r/^\s$/) do
result
else
raise "Invalid array syntax"
end
end
end
defp parse_nested_arrays(<<c>> <> rest, fun, dim_info) when is_space(c),
do: parse_nested_arrays(rest, fun, dim_info)
defp parse_nested_arrays(_, _, %{cur_dim: dim}) when dim > 6,
do: raise("number of dimensions (#{dim}) exceeds maximum of 6")
defp parse_nested_arrays(_, _, %{cur_dim: dim, max_dim: max_dim}) when dim > max_dim,
do: raise("Inconsistent array dimensions")
defp parse_nested_arrays("{" <> rest, fun, %{cur_dim: dim} = dim_info) do
# we're in an array, need to parse all the elements at this level
case String.trim_leading(rest) do
"" ->
raise "Unexpected end of input"
"{" <> _ = rest ->
parse_all_nested_arrays(rest, fun, [], 0, dim_info)
_ ->
# If we know max dimension but see a non-array element, before that, we know it's inconsistent
if is_map_key(dim_info, :max_dim) and dim_info.max_dim > dim do
raise "Inconsistent array dimensions"
end
{result, rest, dim_size} = parse_all_elements(rest, fun)
# If we've been at this depth, validate that new array is consistent with the previous ones,
# if not, save it
case Map.fetch(dim_info, dim) do
{:ok, ^dim_size} ->
{result, rest, dim_info}
:error ->
{result, rest, Map.put(dim_info, dim, dim_size)}
{:ok, _} ->
raise "Inconsistent array dimensions"
end
end
end
defp parse_nested_arrays(_, _, _), do: raise("Unexpected array element")
defp parse_all_nested_arrays(str, fun, acc, dim_size, %{cur_dim: dim} = dim_info) do
{result, rest, dim_info} = parse_nested_arrays(str, fun, %{dim_info | cur_dim: dim + 1})
dim_info = %{dim_info | cur_dim: dim}
# First time we reach this branch is when we followed all open braces at the start
# of the string, so we know the maximum dimension of the array
dim_info = Map.put_new(dim_info, :max_dim, dim + 1)
case scan_until_next_boundary(rest) do
# If next boundary is a comma, we're at the same depth, so keep parsing
{?,, rest} ->
parse_all_nested_arrays(rest, fun, [result | acc], dim_size + 1, dim_info)
# If next boundary is a closing brace, we're done with this array, so update what we can
{?}, rest} ->
dim_size = dim_size + 1
# If we've been at this depth, validate that new array is consistent with the previous ones,
# if not, save it
case Map.fetch(dim_info, dim) do
{:ok, ^dim_size} ->
{Enum.reverse([result | acc]), rest, dim_info}
:error ->
{Enum.reverse([result | acc]), rest, Map.put(dim_info, dim, dim_size)}
{:ok, _} ->
raise "Inconsistent array dimensions"
end
end
end
defp parse_all_elements(str, fun, acc \\ [], dim \\ 0)
defp parse_all_elements("", _, _, _), do: raise("Unexpected end of input")
defp parse_all_elements(<<c>> <> rest, fun, acc, dim) when is_space(c),
do: parse_all_elements(rest, fun, acc, dim)
defp parse_all_elements(str, fun, acc, dim) do
{type, {elem, rest}} = scan_next_element(str)
case scan_until_next_boundary(rest) do
{?,, rest} -> parse_all_elements(rest, fun, [apply_fun(type, elem, fun) | acc], dim + 1)
{?}, rest} -> {Enum.reverse([apply_fun(type, elem, fun) | acc]), rest, dim + 1}
end
end
defp scan_next_element(<<?{>> <> _), do: raise("Unexpected '{' character")
defp scan_next_element(<<?">> <> rest), do: {:quoted, scan_until_quote(rest, "")}
defp scan_next_element(rest), do: {:unquoted, scan_until_comma_or_end(rest, "", "")}
defp scan_until_quote("", _), do: raise("Unexpected end of input")
defp scan_until_quote(<<?">> <> rest, acc), do: {acc, rest}
defp scan_until_quote(~S'\"' <> str, acc), do: scan_until_quote(str, acc <> ~S'"')
defp scan_until_quote(~S'\\' <> str, acc), do: scan_until_quote(str, acc <> ~S'\\')
defp scan_until_quote(<<c>> <> str, acc), do: scan_until_quote(str, acc <> <<c>>)
defp scan_until_comma_or_end("", _, _), do: raise("Unexpected end of input")
defp scan_until_comma_or_end(<<c>> <> _, "", _) when c in [?,, ?}],
do: raise("Unexpected '#{[c]}' character")
defp scan_until_comma_or_end("}" <> _ = rest, acc, _acc_whitespace), do: {acc, rest}
defp scan_until_comma_or_end(<<?,>> <> _ = str, acc, _acc_whitespace), do: {acc, str}
defp scan_until_comma_or_end(<<c>> <> str, "", "") when is_space(c),
do: scan_until_comma_or_end(str, "", "")
defp scan_until_comma_or_end(<<c>> <> str, acc, acc_whitespace) when is_space(c),
do: scan_until_comma_or_end(str, acc, acc_whitespace <> <<c>>)
defp scan_until_comma_or_end(<<c>> <> str, acc, acc_whitespace),
do: scan_until_comma_or_end(str, acc <> acc_whitespace <> <<c>>, "")
defp scan_until_next_boundary(""), do: raise("Unexpected end of input")
defp scan_until_next_boundary(<<c>> <> rest) when c in [?,, ?}], do: {c, rest}
defp scan_until_next_boundary(<<c>> <> rest) when is_space(c),
do: scan_until_next_boundary(rest)
defp scan_until_next_boundary(<<c>> <> _), do: raise("Unexpected '#{[c]}' character")
defp apply_fun(:quoted, elem, fun), do: fun.(elem)
defp apply_fun(:unquoted, elem, fun) do
if String.downcase(elem) == "null", do: nil, else: fun.(elem)
end
@doc ~S"""
Serialize a list of strings into a postgres string-serialized array into a list of strings, wrapping the contents
## Examples
iex> [~s|("2023-06-15 11:18:05.372698+00",)|] |> serialize()
~S|{"(\"2023-06-15 11:18:05.372698+00\",)"}|
iex> [~s|("2023-06-15 11:18:05.372698+00",)|, ~s|("2023-06-15 11:18:05.372698+00",)|] |> serialize()
~S|{"(\"2023-06-15 11:18:05.372698+00\",)","(\"2023-06-15 11:18:05.372698+00\",)"}|
iex> str = ~S|{"(\"2023-06-15 11:18:05.372698+00\",)","(\"2023-06-15 11:18:05.372698+00\",)"}|
iex> str |> parse() |> serialize()
str
"""
def serialize(array, quote_char \\ ?") when is_list(array) do
array
|> Enum.map_join(",", fn
nil -> "null"
val when is_binary(val) -> val |> String.replace(~S|"|, ~S|\"|) |> enclose(<<quote_char>>)
end)
|> enclose("{", "}")
end
defp enclose(str, left, right \\ nil) do
left <> str <> (right || left)
end
@doc """
Access a slice or index of a postgres array.
## Examples
iex> ~S|{1,2,3,4,5}| |> parse(&String.to_integer/1) |> slice_access([{:slice, nil, 3}])
[1, 2, 3]
iex> ~S|{1,2,3,4,5}| |> parse(&String.to_integer/1) |> slice_access([{:slice, 3, nil}])
[3, 4, 5]
iex> ~S|{1,2,3,4,5}| |> parse(&String.to_integer/1) |> slice_access([{:slice, 3, 4}])
[3, 4]
iex> ~S|{{1,2},{3,4}}| |> parse(&String.to_integer/1) |> slice_access([{:slice, nil, nil}, {:index, 2}])
[[1, 2], [3, 4]]
iex> ~S|{{1,2},{3,4}}| |> parse(&String.to_integer/1) |> slice_access([{:slice, nil, nil}, {:slice, 2, 2}])
[[2], [4]]
iex> ~S|{{1,2},{3,4}}| |> parse(&String.to_integer/1) |> slice_access([{:slice, nil, nil}, {:slice, -1, 1}])
[[1], [3]]
iex> ~S|{{1,2},{3,4}}| |> parse(&String.to_integer/1) |> slice_access([{:slice, nil, nil}, {:slice, 1, -1}])
[]
"""
@spec slice_access(
list(),
list({:slice, nil | integer(), nil | integer()} | {:index, integer()})
) :: list()
def slice_access(array, instructions) do
do_slice_access(array, instructions)
catch
:out_of_bounds -> []
end
defp do_slice_access(elem, [_ | _]) when not is_list(elem), do: throw(:out_of_bounds)
defp do_slice_access(array, []), do: array
defp do_slice_access(array, [{:slice, nil, nil} | rest]),
do: Enum.map(array, &do_slice_access(&1, rest))
defp do_slice_access(_array, [{:slice, lower_idx, upper_idx} | _])
when is_integer(lower_idx) and is_integer(upper_idx) and
(lower_idx > upper_idx or upper_idx < 1),
do: throw(:out_of_bounds)
defp do_slice_access(array, [{:slice, lower_idx, upper_idx} | rest]),
do:
array
|> Enum.slice((normalize_idx(lower_idx) || 0)..(normalize_idx(upper_idx) || -1)//1)
|> Enum.map(&do_slice_access(&1, rest))
defp do_slice_access(array, [{:index, idx} | rest]),
do: do_slice_access(array, [{:slice, 1, idx} | rest])
@doc """
Access an index of a postgres array. If the index is out of bounds or array has more dimensions than the indices provided, returns `nil`.
## Examples
iex> ~S|{1,2,3,4,5}| |> parse(&String.to_integer/1) |> index_access([{:index, 3}])
3
iex> ~S|{{1,2},{3,4}}| |> parse(&String.to_integer/1) |> index_access([{:index, 2}, {:index, 1}])
3
iex> ~S|{{1,2},{3,4}}| |> parse(&String.to_integer/1) |> index_access([{:index, 3}])
nil
"""
@spec index_access(list(), list({:index, integer()})) :: list()
def index_access(array, list_of_indices) do
Enum.reduce_while(list_of_indices, array, fn
_, nil -> {:halt, nil}
{:index, idx}, _acc when idx < 1 -> {:halt, nil}
{:index, idx}, acc -> {:cont, Enum.at(acc, idx - 1)}
end)
|> case do
[] -> nil
result -> result
end
end
defp normalize_idx(nil), do: nil
defp normalize_idx(pg_index) when pg_index < 1, do: 0
defp normalize_idx(pg_index), do: pg_index - 1
def concat_arrays(arr1, []), do: arr1
def concat_arrays([], arr2), do: arr2
def concat_arrays(arr1, nil), do: arr1
def concat_arrays(nil, arr2), do: arr2
def concat_arrays(arr1, arr2) do
case {get_array_dim(arr1), get_array_dim(arr2)} do
{d1, d1} -> arr1 ++ arr2
{d1, d2} when d2 - d1 == 1 -> [arr1 | arr2]
{d1, d2} when d1 - d2 == 1 -> arr1 ++ [arr2]
{d1, d2} -> raise "Incompatible array dimensions: #{d1} and #{d2}"
end
end
@doc """
Get the dimension of a postgres array.
## Examples
iex> ~S|{}| |> parse() |> get_array_dim()
nil
iex> ~S|{1,2,3,4,5}| |> parse() |> get_array_dim()
1
iex> ~S|{{1,2},{3,4}}| |> parse() |> get_array_dim()
2
"""
@spec get_array_dim(list()) :: non_neg_integer()
def get_array_dim(arr, dim \\ 0)
def get_array_dim([], _), do: nil
def get_array_dim([hd | _], dim), do: get_array_dim(hd, dim + 1)
def get_array_dim(_, dim), do: dim
def array_prepend(elem, []), do: [elem]
def array_prepend(elem, [hd | tl]) when not is_list(hd), do: [elem, hd | tl]
def array_prepend_concat(elem, _) when is_binary(elem),
do: raise("Cannot prepend a binary to an array with ||")
def array_prepend_concat(nil, arr), do: arr
def array_prepend_concat(elem, arr) when not is_binary(elem), do: array_prepend(elem, arr)
def array_append([], elem), do: [elem]
def array_append([hd | _] = list, elem) when not is_list(hd), do: list ++ [elem]
def array_append_concat(_, elem) when is_binary(elem),
do: raise("Cannot append a binary to an array with ||")
def array_append_concat(arr, nil), do: arr
def array_append_concat(arr, elem), do: array_append(arr, elem)
end