Current section
Files
Jump to
Current section
Files
lib/ccxt/response_transformer.ex
defmodule CCXT.ResponseTransformer do
@moduledoc """
Response transformers for converting raw exchange responses to unified format.
Some exchanges return responses in formats that don't match the unified API expectations.
For example:
- BitMEX returns `[%{ticker}]` instead of `%{ticker}` for fetch_ticker
- BitMEX returns flat order list instead of `%{bids: [], asks: []}` for fetch_order_book
This module provides transformers that can be configured per-endpoint in specs.
## Usage
Add a `response_transformer` field to an endpoint in the spec:
%{
name: :fetch_ticker,
path: "/instrument",
response_transformer: {:extract_path_unwrap, []}
}
Available transformers:
- `{:extract_path, path}` - Extracts nested data via key path
- `{:extract_path_unwrap, path}` - Extracts via path, then unwraps single-element list (returns `nil` for empty list)
- `{:extract_path_unwrap_map, path}` - Extracts via path, then unwraps single-key map to its value
- `:unwrap_single_element_list` - Unwraps `[item]` to `item` (returns `[]` for empty list)
- `:unwrap_single_element_map` - Unwraps `%{key => value}` to `value` (leaves 0 or 2+ key maps unchanged)
- `:extract_first_list_value` - Finds the first value that is a list in a map (ignores metadata keys like `"last"`)
- `:order_book_from_flat_list` - Converts flat order list to `%{bids: [], asks: []}`
- `{:positional_to_maps, field_names}` - Converts inner positional arrays to maps by zipping with field names
- `{:transpose_columns_to_rows, column_keys}` - Transposes column-oriented map to list of rows
- `{:extract_path_unwrap_merge, path, merge_keys}` - Like `extract_path_unwrap` but merges specified keys from the original envelope into the extracted result (inner fields win via `put_new`)
- `{:compose, [transformer]}` - Chains transformers left-to-right
"""
require Logger
@type transformer ::
:unwrap_single_element_list
| :unwrap_single_element_map
| :extract_first_list_value
| :order_book_from_flat_list
| {:extract_path, [String.t()]}
| {:extract_path_unwrap, [String.t()]}
| {:extract_path_unwrap_map, [String.t()]}
| {:extract_path_unwrap_merge, [String.t()], [String.t()]}
| {:positional_to_maps, [String.t()]}
| {:transpose_columns_to_rows, [String.t()]}
| {:compose, [transformer()]}
| nil
@doc """
Applies a transformer to the response body.
Returns the transformed body, or the original body if no transformer is specified.
"""
@spec transform(term(), transformer()) :: term()
def transform(body, nil), do: body
def transform(body, :unwrap_single_element_list), do: unwrap_single_element_list(body)
def transform(body, :order_book_from_flat_list), do: order_book_from_flat_list(body)
def transform(body, {:extract_path, path}) when is_list(path), do: extract_path(body, path)
# Note: unlike bare :unwrap_single_element_list (which returns [] for empty input),
# extract_path_unwrap returns nil for empty lists — signaling "no data" after envelope extraction.
def transform(body, {:extract_path_unwrap, path}) when is_list(path) do
case body |> extract_path(path) |> unwrap_single_element_list() do
[] -> nil
other -> other
end
end
def transform(body, {:extract_path_unwrap_merge, path, merge_keys}) when is_list(path) and is_list(merge_keys) do
case body |> extract_path(path) |> unwrap_single_element_list() do
[] -> nil
%{} = result -> merge_envelope_fields(body, result, merge_keys)
other -> other
end
end
def transform(body, :unwrap_single_element_map), do: unwrap_single_element_map(body)
def transform(body, :extract_first_list_value), do: extract_first_list_value(body)
def transform(body, {:extract_path_unwrap_map, path}) when is_list(path) do
body |> extract_path(path) |> unwrap_single_element_map()
end
def transform(body, {:positional_to_maps, field_names}) when is_list(field_names) do
positional_to_maps(body, field_names)
end
def transform(body, {:compose, transformers}) when is_list(transformers) do
Enum.reduce(transformers, body, fn t, acc -> transform(acc, t) end)
end
def transform(body, {:transpose_columns_to_rows, keys}) when is_list(keys) do
transpose_columns_to_rows(body, keys)
end
def transform(body, unknown) do
Logger.warning("[ResponseTransformer] Unknown transformer: #{inspect(unknown)}, returning body unchanged")
body
end
@doc """
Unwraps a single-element list to its element.
Used when an API returns `[item]` but the unified API expects `item`.
## Examples
iex> CCXT.ResponseTransformer.unwrap_single_element_list([%{"symbol" => "BTC"}])
%{"symbol" => "BTC"}
iex> CCXT.ResponseTransformer.unwrap_single_element_list([])
[]
iex> CCXT.ResponseTransformer.unwrap_single_element_list([%{a: 1}, %{b: 2}])
[%{a: 1}, %{b: 2}]
"""
@spec unwrap_single_element_list(term()) :: term()
def unwrap_single_element_list([single]) when is_map(single), do: single
def unwrap_single_element_list(other), do: other
@doc """
Unwraps a single-key map to its value.
Used when an API returns `%{"KEY" => value}` but the unified API expects `value` directly.
E.g., Kraken's /public/Ticker returns `{"result": {"XXBTUSD": {...}}}` where the key is
the symbol name (not a meaningful container).
Returns maps with 0 or 2+ keys unchanged.
## Examples
iex> CCXT.ResponseTransformer.unwrap_single_element_map(%{"XXBTUSD" => %{"a" => [1]}})
%{"a" => [1]}
iex> CCXT.ResponseTransformer.unwrap_single_element_map(%{})
%{}
iex> CCXT.ResponseTransformer.unwrap_single_element_map(%{"a" => 1, "b" => 2})
%{"a" => 1, "b" => 2}
iex> CCXT.ResponseTransformer.unwrap_single_element_map("not a map")
"not a map"
"""
@spec unwrap_single_element_map(term()) :: term()
def unwrap_single_element_map(%{} = map) when map_size(map) == 1 do
[{_key, value}] = :maps.to_list(map)
value
end
def unwrap_single_element_map(other), do: other
@doc """
Extracts the first list value from a map, ignoring non-list metadata keys.
Used when an API returns a map with a dynamic data key alongside metadata keys.
E.g., Kraken's fetch_trades returns `%{"XXBTZUSD" => [[...trades...]], "last" => "1541439421"}`
where `"last"` is metadata and the pair key holds the actual data.
Returns the map unchanged when no list values exist.
## Examples
iex> CCXT.ResponseTransformer.extract_first_list_value(%{"XXBTZUSD" => [[1], [2]], "last" => "123"})
[[1], [2]]
iex> CCXT.ResponseTransformer.extract_first_list_value(%{"XXBTZUSD" => [[1]]})
[[1]]
iex> CCXT.ResponseTransformer.extract_first_list_value(%{"a" => 1, "b" => 2})
%{"a" => 1, "b" => 2}
iex> CCXT.ResponseTransformer.extract_first_list_value("not a map")
"not a map"
"""
@spec extract_first_list_value(term()) :: term()
def extract_first_list_value(%{} = map) do
case Enum.find(map, fn {_key, value} -> is_list(value) end) do
{_key, list_value} -> list_value
nil -> map
end
end
def extract_first_list_value(other), do: other
@doc """
Converts a flat order list to structured order book format.
BitMEX returns orders as a flat list with "side" field:
`[%{"side" => "Sell", "price" => 100, "size" => 10}, %{"side" => "Buy", ...}]`
This transforms it to unified format:
`%{"bids" => [[price, size], ...], "asks" => [[price, size], ...]}`
## Examples
iex> orders = [
...> %{"side" => "Sell", "price" => 100.5, "size" => 10},
...> %{"side" => "Buy", "price" => 99.5, "size" => 20}
...> ]
iex> CCXT.ResponseTransformer.order_book_from_flat_list(orders)
%{"bids" => [[99.5, 20]], "asks" => [[100.5, 10]]}
"""
@spec order_book_from_flat_list(term()) :: term()
def order_book_from_flat_list(orders) when is_list(orders) do
{bids, asks} =
Enum.reduce(orders, {[], []}, fn order, {bids, asks} ->
price = order["price"]
size = order["size"]
case order["side"] do
"Buy" -> {[[price, size] | bids], asks}
"Sell" -> {bids, [[price, size] | asks]}
# Skip unknown sides
_ -> {bids, asks}
end
end)
# Bids sorted descending (highest first), asks sorted ascending (lowest first)
%{
"bids" => Enum.sort_by(bids, fn [price, _] -> price end, :desc),
"asks" => Enum.sort_by(asks, fn [price, _] -> price end, :asc)
}
end
def order_book_from_flat_list(other), do: other
@doc """
Transposes column-oriented data into a list of rows.
Some exchanges (e.g., Deribit OHLCV) return data as a map of columns:
`%{"ticks" => [1, 2], "open" => [10, 20]}` instead of rows `[[1, 10], [2, 20]]`.
Column keys determine the order of elements in each row. Missing keys or
non-list column values cause the data to be returned unchanged (defensive).
## Examples
iex> CCXT.ResponseTransformer.transpose_columns_to_rows(
...> %{"a" => [1, 2], "b" => [3, 4]},
...> ["a", "b"]
...> )
[[1, 3], [2, 4]]
iex> CCXT.ResponseTransformer.transpose_columns_to_rows("not a map", ["a"])
"not a map"
"""
@spec transpose_columns_to_rows(term(), [String.t()]) :: term()
def transpose_columns_to_rows(%{} = data, [_ | _] = column_keys) do
columns = Enum.map(column_keys, &Map.get(data, &1))
if Enum.all?(columns, &is_list/1) do
columns |> Enum.zip() |> Enum.map(&Tuple.to_list/1)
else
data
end
end
def transpose_columns_to_rows(data, _column_keys), do: data
@doc """
Converts positional arrays in a list to maps by zipping with field names.
Used when an API returns trades (or similar) as positional arrays instead of maps.
E.g., Kraken's fetchTrades returns `[["50000", "0.01", 1710327959, "b", "m", "", 123], ...]`
## Examples
iex> CCXT.ResponseTransformer.positional_to_maps(
...> [["50000", "0.01", "b"], ["51000", "0.02", "s"]],
...> ["price", "amount", "side"]
...> )
[%{"price" => "50000", "amount" => "0.01", "side" => "b"}, %{"price" => "51000", "amount" => "0.02", "side" => "s"}]
iex> CCXT.ResponseTransformer.positional_to_maps("not a list", ["a"])
"not a list"
"""
@spec positional_to_maps(term(), [String.t()]) :: term()
def positional_to_maps(rows, field_names) when is_list(rows) do
Enum.map(rows, fn
row when is_list(row) ->
field_names
|> Enum.zip(row)
|> Map.new()
other ->
other
end)
end
def positional_to_maps(other, _field_names), do: other
@doc """
Extracts nested data from a response envelope using a key path.
Walks into nested maps following the given keys. Returns the data at the
final key, or the current level's data if a key is not found (i.e., stops
walking and returns whatever map it reached).
## Examples
iex> body = %{"retCode" => 0, "result" => %{"list" => [[1, 2, 3]]}}
iex> CCXT.ResponseTransformer.extract_path(body, ["result", "list"])
[[1, 2, 3]]
iex> CCXT.ResponseTransformer.extract_path(%{"data" => "test"}, [])
%{"data" => "test"}
iex> CCXT.ResponseTransformer.extract_path(%{"a" => 1}, ["missing"])
%{"a" => 1}
iex> CCXT.ResponseTransformer.extract_path(%{"result" => %{"data" => "test"}}, ["result", "missing"])
%{"data" => "test"}
"""
@spec extract_path(term(), [String.t()]) :: term()
def extract_path(data, []), do: data
def extract_path(%{} = data, [key | rest]) do
case Map.get(data, key) do
nil -> data
value -> extract_path(value, rest)
end
end
def extract_path(data, _path), do: data
@doc false
# Merges specified keys from the envelope into the extracted result.
# Uses put_new so inner data fields always win over envelope fields.
@spec merge_envelope_fields(term(), map(), [String.t()]) :: map()
defp merge_envelope_fields(%{} = envelope, result, merge_keys) do
Enum.reduce(merge_keys, result, fn key, acc ->
case Map.get(envelope, key) do
nil -> acc
value -> Map.put_new(acc, key, value)
end
end)
end
defp merge_envelope_fields(_envelope, result, _merge_keys), do: result
end