Packages
ch
0.2.6
0.8.3
0.8.2
0.8.1
0.8.0
0.7.3
0.7.2
0.7.1
0.7.0
0.6.4
0.6.3
0.6.2
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
retired
0.5.3
retired
0.5.2
retired
0.5.1
retired
0.5.0
retired
0.4.3
0.4.2
0.4.1
0.4.0
0.3.6
0.3.5
0.3.4
0.3.3
0.3.2
0.3.1
0.3.0
0.2.10
0.2.9
0.2.8
0.2.7
0.2.6
0.2.5
0.2.5-rc.0
0.2.4
0.2.3
0.2.2
0.2.2-rc.3
0.2.2-rc.2
0.2.2-rc.1
0.2.2-rc.0
0.2.1
0.2.0
0.1.15
0.1.14
0.1.13
0.1.12
0.1.11
0.1.10
0.1.9
0.1.8
0.1.7
retired
0.1.6
retired
0.1.5
retired
0.1.4
retired
0.1.3
retired
0.1.2
retired
0.1.1
0.1.0
HTTP ClickHouse driver for Elixir
Current section
Files
Jump to
Current section
Files
lib/ch/types.ex
defmodule Ch.Types do
@moduledoc """
Helpers to turn ClickHouse types into Elixir terms for easier processing.
"""
types =
[
{_encoded = "String", _decoded = :string, _args = []},
{"Bool", :boolean, []},
for size <- [8, 16, 32, 64, 128, 256] do
[
{"UInt#{size}", :"u#{size}", []},
{"Int#{size}", :"i#{size}", []}
]
end,
for size <- [32, 64] do
{"Float#{size}", :"f#{size}", []}
end,
{"Array", :array, [:type]},
{"Tuple", :tuple, [:type]},
{"Map", :map, [:type]},
{"FixedString", :fixed_string, [:int]},
{"Nullable", :nullable, [:type]},
{"DateTime64", :datetime64, [:int, :string]},
{"DateTime", :datetime, [:string]},
# {"DateTime", :datetime, []},
{"Date32", :date32, []},
{"Date", :date, []},
{"LowCardinality", :low_cardinality, [:type]},
for size <- [32, 64, 128, 256] do
{"Decimal#{size}", :"decimal#{size}", [:int]}
end,
{"Decimal", :decimal, [:int, :int]},
{"SimpleAggregateFunction", :simple_aggregate_function, [:identifier, :type]},
{"Enum8", :enum8, [:string, :eq, :int]},
{"Enum16", :enum16, [:string, :eq, :int]},
{"UUID", :uuid, []},
{"IPv4", :ipv4, []},
{"IPv6", :ipv6, []},
{"Point", :point, []},
{"Ring", :ring, []},
{"Polygon", :polygon, []},
{"MultiPolygon", :multipolygon, []},
{"Nothing", :nothing, []}
]
|> List.flatten()
for {encoded, name, []} <- types do
@doc """
Helper for `#{encoded}` ClickHouse type:
iex> #{name}()
:#{name}
iex> encode(#{name}())
"#{encoded}"
iex> decode("#{encoded}")
#{name}()
"""
def unquote(name)(), do: unquote(name)
end
@doc """
Helper for `DateTime` ClickHouse type:
iex> datetime()
:datetime
iex> to_string(encode(datetime()))
"DateTime"
iex> decode("DateTime")
datetime()
"""
def datetime, do: :datetime
@doc """
Helper for `DateTime(timezone)` ClickHouse type:
iex> datetime("Europe/Vienna")
{:datetime, "Europe/Vienna"}
iex> to_string(encode(datetime("UTC")))
"DateTime('UTC')"
iex> decode("DateTime('UTC')")
datetime("UTC")
"""
def datetime(timezone) when is_binary(timezone), do: {:datetime, timezone}
@doc """
Helper for `DateTime64(precision)` ClickHouse type:
iex> datetime64(3)
{:datetime64, 3}
iex> to_string(encode(datetime64(3)))
"DateTime64(3)"
iex> decode("DateTime64(3)")
datetime64(3)
"""
def datetime64(precision) when is_integer(precision), do: {:datetime64, precision}
@doc """
Helper for `DateTime64(precision, timezone)` ClickHouse type:
iex> datetime64(3, "UTC")
{:datetime64, 3, "UTC"}
iex> to_string(encode(datetime64(3, "UTC")))
"DateTime64(3, 'UTC')"
iex> decode("DateTime64(3, 'UTC')")
datetime64(3, "UTC")
"""
def datetime64(precision, timezone) when is_integer(precision) and is_binary(timezone) do
{:datetime64, precision, timezone}
end
@doc """
Helper for `FixedString(n)` ClickHouse type:
iex> fixed_string(3)
{:fixed_string, 3}
iex> to_string(encode(fixed_string(16)))
"FixedString(16)"
iex> decode("FixedString(16)")
fixed_string(16)
"""
def fixed_string(n) when is_integer(n), do: {:fixed_string, n}
@doc """
Helper for `Decimal(P, S)` ClickHouse type:
iex> decimal(18, 4)
{:decimal, 18, 4}
iex> to_string(encode(decimal(18, 4)))
"Decimal(18, 4)"
iex> decode("Decimal(18, 4)")
decimal(18, 4)
"""
def decimal(precision, scale) when is_integer(precision) and is_integer(scale) do
{:decimal, precision, scale}
end
for size <- [32, 64, 128, 256] do
name = :"decimal#{size}"
# `select toTypeName(cast(1 as Decimal32(2)))` etc.
precision =
case size do
32 -> 9
64 -> 18
128 -> 38
256 -> 76
end
@doc """
Helper for `Decimal#{size}(S)` ClickHouse type:
iex> #{name}(4)
{:#{name}, 4}
iex> to_string(encode(#{name}(4)))
"Decimal(#{precision}, 4)"
iex> decode("Decimal#{size}(4)")
{:#{name}, 4}
"""
def unquote(name)(scale) when is_integer(scale), do: {unquote(name), scale}
end
defguardp is_type(type) when is_atom(type) or is_tuple(type)
@doc """
Helper for `Array(T)` ClickHouse type:
iex> array(u64())
{:array, :u64}
iex> to_string(encode(array(u64())))
"Array(UInt64)"
iex> decode("Array(UInt64)")
array(u64())
"""
def array(type) when is_type(type), do: {:array, type}
@doc """
Helper for `Tuple(T1, T2, ...)` ClickHouse type:
iex> tuple([u64(), array(string())])
{:tuple, [:u64, {:array, :string}]}
iex> to_string(encode(tuple([u64(), array(string())])))
"Tuple(UInt64, Array(String))"
iex> decode("Tuple(UInt64, Array(String))")
tuple([u64(), array(string())])
"""
def tuple(types) when is_list(types), do: {:tuple, types}
@doc """
Helper for `Map(K, V)` ClickHouse type:
iex> map(string(), array(string()))
{:map, :string, {:array, :string}}
iex> to_string(encode(map(string(), array(string()))))
"Map(String, Array(String))"
iex> decode("Map(String, Array(String))")
map(string(), array(string()))
"""
def map(key_type, value_type) when is_type(key_type) and is_type(value_type) do
{:map, key_type, value_type}
end
@doc """
Helper for `Nullable(T)` ClickHouse type:
iex> nullable(array(boolean()))
{:nullable, {:array, :boolean}}
iex> to_string(encode(nullable(array(boolean()))))
"Nullable(Array(Bool))"
iex> decode("Nullable(Array(Bool))")
nullable(array(boolean()))
"""
def nullable(type) when is_type(type), do: {:nullable, type}
@doc """
Helper for `LowCardinality(T)` ClickHouse type:
iex> low_cardinality(string())
{:low_cardinality, :string}
iex> to_string(encode(low_cardinality(string())))
"LowCardinality(String)"
iex> decode("LowCardinality(String)")
low_cardinality(string())
"""
def low_cardinality(type) when is_type(type), do: {:low_cardinality, type}
@doc """
Helper for `SimpleAggregateFunction(name, type)` ClickHouse type:
iex> simple_aggregate_function("any", u8())
{:simple_aggregate_function, "any", :u8}
iex> to_string(encode(simple_aggregate_function("any", u8())))
"SimpleAggregateFunction(any, UInt8)"
iex> decode("SimpleAggregateFunction(any, UInt8)")
simple_aggregate_function("any", u8())
"""
def simple_aggregate_function(name, type) when is_binary(name) and is_type(type) do
{:simple_aggregate_function, name, type}
end
for size <- [8, 16] do
name = :"enum#{size}"
@doc """
Helper for `Enum#{size}` ClickHouse type:
iex> #{name}([{"hello", 1}, {"world", 2}])
{:#{name}, [{"hello", 1}, {"world", 2}]}
iex> to_string(encode(#{name}([{"hello", 1}, {"world", 2}])))
"Enum#{size}('hello' = 1, 'world' = 2)"
iex> decode("Enum#{size}('hello' = 1, 'world' = 2)")
#{name}([{"hello", 1}, {"world", 2}])
"""
def unquote(name)(mapping) when is_list(mapping), do: {unquote(name), mapping}
end
@doc """
Decodes a ClickHouse type into an intermediary Elixir term.
iex> decode("String")
:string
iex> decode("Array(String)")
{:array, :string}
iex> decode("Enum8('hello' = 1, 'world' = 2)")
{:enum8, [{"hello", 1}, {"world", 2}]}
iex> decode("Nullable(Decimal(18, 4))")
{:nullable, {:decimal, 18, 4}}
"""
def decode(type)
for {encoded, decoded, []} <- types do
def decode(unquote(encoded)), do: unquote(decoded)
end
def decode("DateTime"), do: :datetime
def decode(type) do
try do
decode([:type], type, [])
rescue
e ->
message = "failed to decode #{inspect(type)} as ClickHouse type (#{Exception.message(e)})"
reraise(ArgumentError, message, __STACKTRACE__)
end
end
defguardp is_whitespace(char) when char == ?\s or char == ?\t
defp decode(stack, <<whitespace, rest::bytes>>, acc) when is_whitespace(whitespace) do
decode(stack, rest, acc)
end
for {encoded, decoded, [_ | _] = args} <- types do
defp decode([:type | stack], unquote(encoded) <> rest, acc) do
decode(
[:open | unquote(args)] ++ [:close, {unquote(decoded), unquote(args)}, acc | stack],
rest,
[]
)
end
end
for {encoded, decoded, []} <- types do
defp decode([:type | stack], unquote(encoded) <> rest, acc) do
decode(stack, rest, [unquote(decoded) | acc])
end
end
defp decode([:open | stack], <<rest::bytes>>, acc) do
case rest do
<<?(, rest::bytes>> ->
decode(stack, rest, acc)
_ ->
# handles DateTime and Type()
[{type, _args}, prev_acc | stack] = close(stack)
decode(stack, rest, [type | prev_acc])
end
end
defp decode(stack, <<?), rest::bytes>>, acc) do
[{type, _args}, prev_acc | stack] = close(stack)
decode(stack, rest, [build_type(type, acc) | prev_acc])
end
defp decode([:close, {_type, args} | _] = stack, <<?,, rest::bytes>>, acc) do
decode(args ++ stack, rest, acc)
end
defp decode(stack, <<?,, rest::bytes>>, acc) do
decode(stack, rest, acc)
end
defp decode([:string | stack], <<?', rest::bytes>>, acc) do
decode_string(rest, 0, rest, stack, acc)
end
defp decode([:int | stack], <<rest::bytes>>, acc) do
decode_int(rest, stack, acc)
end
defp decode([:identifier | stack], <<rest::bytes>>, acc) do
decode_identifier(rest, 0, rest, stack, acc)
end
defp decode([:eq | stack], <<?=, rest::bytes>>, acc) do
decode(stack, rest, acc)
end
defp decode([], <<>>, [type]), do: type
defp close([:close | stack]), do: stack
defp close([_ | stack]), do: close(stack)
defp build_type(:array = a, [t]), do: {a, t}
defp build_type(:tuple = t, ts), do: {t, :lists.reverse(ts)}
defp build_type(:fixed_string = fs, [n]), do: {fs, n}
defp build_type(:datetime = d, [tz]), do: {d, tz}
defp build_type(:datetime64 = d, [precision]), do: {d, precision}
defp build_type(:datetime64 = d, [tz, p]), do: {d, p, tz}
defp build_type(:map = m, [v, k]), do: {m, k, v}
defp build_type(:nullable = n, [t]), do: {n, t}
defp build_type(:low_cardinality = l, [t]), do: {l, t}
defp build_type(:enum8 = e, mapping), do: {e, build_enum_mapping(mapping)}
defp build_type(:enum16 = e, mapping), do: {e, build_enum_mapping(mapping)}
defp build_type(:simple_aggregate_function = saf, [t, f]), do: {saf, f, t}
defp build_type(:decimal32 = d, [s]), do: {d, s}
defp build_type(:decimal64 = d, [s]), do: {d, s}
defp build_type(:decimal128 = d, [s]), do: {d, s}
defp build_type(:decimal256 = d, [s]), do: {d, s}
defp build_type(:decimal = d, [s, p]), do: {d, p, s}
defp build_enum_mapping(mapping) do
mapping |> :lists.reverse() |> Enum.chunk_every(2) |> Enum.map(fn [k, v] -> {k, v} end)
end
# TODO '', \'
defp decode_string(<<?', rest::bytes>>, len, original, stack, acc) do
part = :binary.part(original, 0, len)
decode(stack, rest, [:binary.copy(part) | acc])
end
defp decode_string(<<u::utf8, rest::bytes>>, len, original, stack, acc) do
decode_string(rest, len + utf8_size(u), original, stack, acc)
end
@compile inline: [utf8_size: 1]
defp utf8_size(codepoint) when codepoint <= 0x7F, do: 1
defp utf8_size(codepoint) when codepoint <= 0x7FF, do: 2
defp utf8_size(codepoint) when codepoint <= 0xFFFF, do: 3
defp utf8_size(codepoint) when codepoint <= 0x10FFFF, do: 4
defguardp is_alpha(a) when (a >= ?a and a <= ?z) or (a >= ?A and a <= ?Z)
defp decode_identifier(<<a, rest::bytes>>, len, original, stack, acc) when is_alpha(a) do
decode_identifier(rest, len + 1, original, stack, acc)
end
defp decode_identifier(<<rest::bytes>>, len, original, stack, acc) do
part = :binary.part(original, 0, len)
decode(stack, rest, [:binary.copy(part) | acc])
end
defguardp is_numeric(char) when char >= ?0 and char <= ?9
defp decode_int(<<?-, i, rest::bytes>>, stack, outer_acc) when is_numeric(i) do
decode_int_cont(rest, -(i - ?0), stack, outer_acc)
end
defp decode_int(<<i, rest::bytes>>, stack, outer_acc) when is_numeric(i) do
decode_int_cont(rest, i - ?0, stack, outer_acc)
end
defp decode_int_cont(<<i, rest::bytes>>, acc, stack, outer_acc) when is_numeric(i) do
decode_int_cont(rest, acc * 10 + i - ?0, stack, outer_acc)
end
defp decode_int_cont(<<rest::bytes>>, int, stack, acc) do
decode(stack, rest, [int | acc])
end
@doc """
Encodes a type from Elixir atom / tuple to proper ClickHouse name.
iex> encode(:string)
"String"
iex> IO.iodata_to_binary(encode({:nullable, :i8}))
"Nullable(Int8)"
"""
def encode(type)
for {encoded, decoded, []} <- types do
def encode(unquote(decoded)), do: unquote(encoded)
end
def encode(:datetime), do: "DateTime"
def encode({:nullable, type}), do: ["Nullable(", encode(type), ?)]
def encode({:fixed_string, n}), do: ["FixedString(", String.Chars.Integer.to_string(n), ?)]
def encode({:array, type}), do: ["Array(", encode(type), ?)]
def encode({:tuple, types}), do: ["Tuple(", encode_intersperse(types, ", "), ?)]
def encode({:map, key_type, value_type}) do
["Map(", encode(key_type), ", ", encode(value_type), ?)]
end
def encode({:low_cardinality, type}), do: ["LowCardinality(", encode(type), ?)]
for size <- [32, 64, 128, 256] do
# `select toTypeName(cast(1 as Decimal32(2)))` etc.
precision =
case size do
32 -> 9
64 -> 18
128 -> 38
256 -> 76
end
def encode({unquote(:"decimal#{size}"), scale}) do
encode({:decimal, unquote(precision), scale})
end
end
def encode({:decimal, precision, scale}) do
[
"Decimal(",
String.Chars.Integer.to_string(precision),
", ",
String.Chars.Integer.to_string(scale),
?)
]
end
def encode({:datetime, timezone}) when is_binary(timezone) do
["DateTime('", timezone, "')"]
end
def encode({:datetime64, precision}) do
["DateTime64(", String.Chars.Integer.to_string(precision), ?)]
end
def encode({:datetime64, precision, timezone}) when is_binary(timezone) do
["DateTime64(", String.Chars.Integer.to_string(precision), ", '", timezone, "')"]
end
def encode({:enum8, mapping}) do
["Enum8('", encode_mapping(mapping), ?)]
end
def encode({:enum16, mapping}) do
["Enum16('", encode_mapping(mapping), ?)]
end
def encode({:simple_aggregate_function, name, type}) when is_binary(name) do
["SimpleAggregateFunction(", name, ", ", encode(type), ?)]
end
defp encode_intersperse([last_type], _separator) do
[encode(last_type)]
end
defp encode_intersperse([type | types], separator) do
[encode(type), separator | encode_intersperse(types, separator)]
end
defp encode_intersperse([] = empty, _separator), do: empty
defp encode_mapping([{k, v}]) when is_binary(k) do
[k, "' = ", String.Chars.Integer.to_string(v)]
end
defp encode_mapping([{k, v} | mapping]) when is_binary(k) do
[k, "' = ", String.Chars.Integer.to_string(v), ", '" | encode_mapping(mapping)]
end
defp encode_mapping([] = empty), do: empty
end