Current section
Files
Jump to
Current section
Files
lib/ccxt/response_parser/mapping_compiler.ex
defmodule CCXT.ResponseParser.MappingCompiler do
@moduledoc """
Compiles P1 mapping analysis data into instruction lists for `CCXT.ResponseParser`.
Loads the mapping analysis JSON (produced by P1) at compile time and converts
exchange-specific field mappings into instruction tuples that the runtime parser
can execute efficiently.
## Instruction Format
{field_atom, coercion_atom, source_keys}
Where:
- `field_atom` - Unified field name (e.g., `:ask`, `:bid_volume`)
- `coercion_atom` - Type coercion to apply (e.g., `:number`, `:string`)
- `source_keys` - Exchange-specific key(s) to read from raw data
## Category Handling
| P1 Category | Action |
|-------------|--------|
| `safe_accessor` | Emit instruction with `fields` as source_keys |
| `resolved_safe_accessor` | Same as safe_accessor |
| `variable_ref` | Emit instruction with `raw` value as source key |
| `wrapped_safe_accessor` | Emit instruction from `inner.fields` (field-backed subset only) |
| `dynamic_field_accessor` | Emit instruction from `possible_fields` (when present) |
| `literal` | Emit `{:literal, value}` instruction — constant value |
| `composed_parse_call` | Emit instruction with inner accessor + parse function / enum table |
| `array_indexed_accessor` | Emit instruction indexing into array at specific position |
| `iso8601` | Skip — derived from timestamp by from_map |
| `computed` | Skip — calculated from other fields |
| `undefined` | Skip — CCXT doesn't map this for the exchange |
| `passthrough` | Emit `{:param, key}` instruction for known params; skip info-like params |
| `param_derived` | Emit `{:param, key}` instruction (param values already normalized) |
| `param_array_accessor` | Skip — indexes into array-format data (deferred, 229d) |
| `parse_call` | Skip — requires calling another parse method |
"""
# Maps parse method names to their schema modules and response type atoms
@method_schemas %{
"parseTicker" => {CCXT.Types.Schema.Ticker, :ticker},
"parseTrade" => {CCXT.Types.Schema.Trade, :trade},
"parseOrder" => {CCXT.Types.Schema.Order, :order},
"parseFundingRate" => {CCXT.Types.Schema.FundingRate, :funding_rate},
"parsePosition" => {CCXT.Types.Schema.Position, :position},
"parseTransaction" => {CCXT.Types.Schema.Transaction, :transaction},
"parseTransfer" => {CCXT.Types.Schema.TransferEntry, :transfer},
"parseDepositAddress" => {CCXT.Types.Schema.DepositAddress, :deposit_address},
"parseLedgerEntry" => {CCXT.Types.Schema.LedgerEntry, :ledger_entry},
"parseLeverage" => {CCXT.Types.Schema.Leverage, :leverage},
"parseTradingFee" => {CCXT.Types.Schema.TradingFeeInterface, :trading_fee},
"parseDepositWithdrawFee" => {CCXT.Types.Schema.DepositWithdrawFee, :deposit_withdraw_fee},
"parseMarginModification" => {CCXT.Types.Schema.MarginModification, :margin_modification},
"parseOpenInterest" => {CCXT.Types.Schema.OpenInterest, :open_interest},
"parseMarginMode" => {CCXT.Types.Schema.MarginMode, :margin_mode},
"parseLiquidation" => {CCXT.Types.Schema.Liquidation, :liquidation},
"parseFundingRateHistory" => {CCXT.Types.Schema.FundingRateHistory, :funding_rate_history},
"parseBorrowInterest" => {CCXT.Types.Schema.BorrowInterest, :borrow_interest},
"parseBorrowRate" => {CCXT.Types.Schema.CrossBorrowRate, :borrow_rate},
"parseConversion" => {CCXT.Types.Schema.Conversion, :conversion},
"parseGreeks" => {CCXT.Types.Schema.Greeks, :greeks},
"parseAccount" => {CCXT.Types.Schema.Account, :account},
"parseOption" => {CCXT.Types.Schema.Option, :option},
"parseFundingHistory" => {CCXT.Types.Schema.FundingHistory, :funding_history},
"parseIsolatedBorrowRate" => {CCXT.Types.Schema.IsolatedBorrowRate, :isolated_borrow_rate},
"parseLastPrice" => {CCXT.Types.Schema.LastPrice, :last_price},
"parseLongShortRatio" => {CCXT.Types.Schema.LongShortRatio, :long_short_ratio},
"parseLeverageTiers" => {CCXT.Types.Schema.LeverageTier, :leverage_tier},
"parseBalance" => {CCXT.Types.Schema.Balances, :balance},
"parseOrderBook" => {CCXT.Types.Schema.OrderBook, :order_book}
}
# Categories that produce instructions (the field maps to raw exchange data)
@generatable_categories MapSet.new([
"safe_accessor",
"resolved_safe_accessor",
"variable_ref",
"boolean_derivation",
"wrapped_safe_accessor",
"dynamic_field_accessor"
])
# Maps CCXT parse-function param names to endpoint param_map keys.
# Params not in this map (response, trade, ticker, order, entry) are skipped —
# they represent the raw input object, already handled by maybe_inject_info/1.
@parse_param_to_endpoint %{
"symbol" => :symbol,
"timestamp" => :since,
"code" => :code,
"currency" => :code
}
# Maps param_derived safe_fn to the param_map key containing the value.
# Our param values are already normalized (unified symbol, currency code),
# so the safe_fn distinction is informational only.
@safe_fn_to_param %{
"safeSymbol" => :symbol,
"safeCurrencyCode" => :code
}
@doc """
Compiles a mapping for a specific exchange and parse method.
Returns a list of instruction tuples, or nil if no mapping data exists.
## Parameters
- `exchange_id` - Exchange identifier (e.g., "binance")
- `parse_method` - CCXT parse method name (e.g., "parseTicker")
- `analysis` - Pre-loaded P1 analysis data
## Examples
iex> analysis = CCXT.ResponseParser.MappingCompiler.load_analysis()
iex> instructions = CCXT.ResponseParser.MappingCompiler.compile_mapping("binance", "parseTicker", analysis)
iex> Enum.find(instructions, fn {field, _, _} -> field == :ask end)
{:ask, :number, ["askPrice"]}
"""
@spec compile_mapping(String.t(), String.t(), map()) ::
[CCXT.ResponseParser.instruction()] | nil
def compile_mapping(exchange_id, parse_method, analysis) do
with {:ok, {schema_module, _type_atom}} <- Map.fetch(@method_schemas, parse_method),
exchange_fields when is_map(exchange_fields) <-
get_in(analysis, ["methods", parse_method, "exchange_mappings", exchange_id]) do
schema_fields = :attributes |> schema_module.__info__() |> Keyword.get(:fields, [])
field_type_map = build_field_type_map(schema_fields)
instructions =
exchange_fields
|> Enum.map(fn {unified_key, mapping} ->
compile_field(unified_key, mapping, field_type_map)
end)
|> Enum.reject(&is_nil/1)
if instructions == [], do: nil, else: instructions
else
_ -> nil
end
end
@doc """
Returns the supported parse methods and their schema/type mappings.
"""
@spec method_schemas() :: map()
def method_schemas, do: @method_schemas
@doc """
Loads the P1 mapping analysis JSON.
Intended to be called once at compile time and reused across multiple
`compile_mapping/3` calls.
"""
# sobelow_skip ["Traversal.FileModule"]
@spec load_analysis() :: map()
def load_analysis do
path = analysis_path()
case File.read(path) do
{:ok, content} ->
case JSON.decode(content) do
{:ok, data} ->
data
{:error, reason} ->
raise "MappingCompiler: failed to decode #{path}: #{inspect(reason)}"
end
{:error, :enoent} ->
# File doesn't exist yet — expected during initial setup
%{}
{:error, reason} ->
raise "MappingCompiler: failed to read #{path}: #{inspect(reason)}"
end
end
@doc """
Returns the path to the P1 mapping analysis JSON file.
"""
@spec analysis_path() :: String.t()
def analysis_path do
case :code.priv_dir(:ccxt_client) do
{:error, :bad_name} ->
[__DIR__, "..", "..", "priv", "extractor/ccxt_mapping_analysis.json"]
|> Path.join()
|> Path.expand()
priv when is_list(priv) ->
Path.join(List.to_string(priv), "extractor/ccxt_mapping_analysis.json")
end
end
# Compiles a single field mapping to an instruction tuple.
# Returns nil for categories we can't generate (computed, iso8601, etc.)
@doc false
@spec compile_field(String.t(), map(), %{String.t() => atom()}) ::
CCXT.ResponseParser.instruction() | nil
defp compile_field(unified_key, %{"category" => "boolean_derivation"} = mapping, field_type_map) do
compile_boolean_derivation(unified_key, mapping, field_type_map)
end
# Literal: hardcoded constant value (e.g., marginType: "isolated")
defp compile_field(unified_key, %{"category" => "literal", "value" => value}, field_type_map) do
if Map.has_key?(field_type_map, unified_key) do
field_atom = unified_key_to_field_atom(unified_key)
{field_atom, {:literal, value}, []}
end
end
# Wrapped safe accessor: unwrap inner.fields for source keys, inner.safe_fn for coercion
# Skips instances where inner has no "fields" key (computed inner values)
defp compile_field(
unified_key,
%{"category" => "wrapped_safe_accessor", "inner" => %{"fields" => inner_fields} = inner},
field_type_map
)
when is_list(inner_fields) and inner_fields != [] do
with coercion when not is_nil(coercion) <- Map.get(field_type_map, unified_key),
field_atom when is_atom(field_atom) <- unified_key_to_field_atom(unified_key) do
final_coercion = safe_fn_coercion_override(inner["safe_fn"], coercion)
{field_atom, final_coercion, inner_fields}
else
_ -> nil
end
end
# Composed parse call with enum_table: extract value, look up in table
defp compile_field(
unified_key,
%{"category" => "composed_parse_call", "enum_table" => enum_table, "argument" => argument},
field_type_map
) do
source_keys = extract_argument_source_keys(argument)
if source_keys != [] and Map.has_key?(field_type_map, unified_key) do
field_atom = unified_key_to_field_atom(unified_key)
extraction = safe_fn_coercion_override(argument["safe_fn"], :string)
{field_atom, {:enum_map, extraction, enum_table}, source_keys}
end
end
# Composed parse call without enum_table: whitelisted parse functions only
@whitelisted_parse_functions MapSet.new(~w(parse8601 parseDate parseNumber parseToInt))
defp compile_field(
unified_key,
%{"category" => "composed_parse_call", "function" => function, "argument" => argument},
field_type_map
) do
if MapSet.member?(@whitelisted_parse_functions, function) do
compile_whitelisted_parse_call(unified_key, function, argument, field_type_map)
end
end
# Array indexed accessor: index into array value at specific position
defp compile_field(
unified_key,
%{
"category" => "array_indexed_accessor",
"source_key" => source_key,
"array_index" => index,
"safe_fn" => safe_fn
},
field_type_map
) do
with coercion when not is_nil(coercion) <- Map.get(field_type_map, unified_key),
field_atom when is_atom(field_atom) <- unified_key_to_field_atom(unified_key) do
inner_coercion = safe_fn_coercion_override(safe_fn, coercion)
{field_atom, {:array_index, index, inner_coercion}, [source_key]}
else
_ -> nil
end
end
# Passthrough: value comes from endpoint function parameter, not response data.
# Only emits for known param names (symbol, timestamp, code, currency).
# Info-like params (response, trade, ticker, order, entry) are skipped —
# already handled by maybe_inject_info/1 in ResponseCoercer.
defp compile_field(unified_key, %{"category" => "passthrough", "param" => param}, field_type_map) do
with endpoint_key when not is_nil(endpoint_key) <- Map.get(@parse_param_to_endpoint, param),
true <- Map.has_key?(field_type_map, unified_key) do
field_atom = unified_key_to_field_atom(unified_key)
{field_atom, {:param, endpoint_key}, []}
else
_ -> nil
end
end
# Param derived: value derived from endpoint param via safe_fn (e.g., safeSymbol).
# Treated identically to passthrough since our param values are already normalized.
defp compile_field(unified_key, %{"category" => "param_derived", "safe_fn" => safe_fn}, field_type_map) do
with endpoint_key when not is_nil(endpoint_key) <- Map.get(@safe_fn_to_param, safe_fn),
true <- Map.has_key?(field_type_map, unified_key) do
field_atom = unified_key_to_field_atom(unified_key)
{field_atom, {:param, endpoint_key}, []}
else
_ -> nil
end
end
defp compile_field(unified_key, %{"category" => category} = mapping, field_type_map) do
if MapSet.member?(@generatable_categories, category) do
with coercion when not is_nil(coercion) <- Map.get(field_type_map, unified_key),
field_atom when is_atom(field_atom) <- unified_key_to_field_atom(unified_key),
source_keys when source_keys != [] <- extract_source_keys(category, mapping) do
final_coercion = safe_fn_coercion_override(mapping["safe_fn"], coercion)
{field_atom, final_coercion, source_keys}
else
_ -> nil
end
end
end
defp compile_field(_unified_key, _mapping, _field_type_map), do: nil
@doc false
# Compiles a boolean_derivation field: maps true/false boolean to enum values.
# Skips if source_keys is empty (unresolvable for some exchanges).
defp compile_boolean_derivation(unified_key, mapping, _field_type_map) do
source_keys = Map.get(mapping, "source_keys", [])
true_value = Map.get(mapping, "true_value")
false_value = Map.get(mapping, "false_value")
if source_keys != [] and not is_nil(true_value) and not is_nil(false_value) do
field_atom = unified_key_to_field_atom(unified_key)
{field_atom, {:bool_enum, true_value, false_value}, source_keys}
end
end
@doc false
# Overrides schema-derived coercion using the safe_fn from P1 analysis.
# E.g., safeStringLower → :string_lower, safeIntegerN → :integer
defp safe_fn_coercion_override(nil, schema_coercion), do: schema_coercion
defp safe_fn_coercion_override("safeStringLower" <> _, _schema), do: :string_lower
defp safe_fn_coercion_override("safeStringUpper" <> _, _schema), do: :string_upper
defp safe_fn_coercion_override("safeInteger" <> _, _schema), do: :integer
defp safe_fn_coercion_override("safeTimestamp" <> _, _schema), do: :timestamp
defp safe_fn_coercion_override(_safe_fn, schema_coercion), do: schema_coercion
# Extracts source keys from the mapping based on category
@doc false
@spec extract_source_keys(String.t(), map()) :: [String.t()]
defp extract_source_keys("variable_ref", %{"raw" => raw}) when is_binary(raw), do: [raw]
defp extract_source_keys("dynamic_field_accessor", %{"possible_fields" => fields}) when is_list(fields), do: fields
defp extract_source_keys(_category, %{"fields" => fields}) when is_list(fields), do: fields
defp extract_source_keys(_category, _mapping), do: []
# Extracts source keys from a composed_parse_call's inner argument
@doc false
@spec extract_argument_source_keys(map()) :: [String.t()]
defp extract_argument_source_keys(%{"category" => cat, "fields" => fields})
when cat in ~w(safe_accessor resolved_safe_accessor) and is_list(fields) and fields != [], do: fields
defp extract_argument_source_keys(%{"category" => "wrapped_safe_accessor", "inner" => %{"fields" => fields}})
when is_list(fields) and fields != [], do: fields
defp extract_argument_source_keys(_), do: []
@doc false
# Compiles a whitelisted composed_parse_call with extractable source keys
defp compile_whitelisted_parse_call(unified_key, function, argument, field_type_map) do
source_keys = extract_argument_source_keys(argument)
if source_keys != [] do
with coercion when not is_nil(coercion) <- Map.get(field_type_map, unified_key),
field_atom when is_atom(field_atom) <- unified_key_to_field_atom(unified_key) do
{field_atom, function_to_coercion(function), source_keys}
else
_ -> nil
end
end
end
# Maps whitelisted composed_parse_call function name to coercion type
@doc false
defp function_to_coercion("parse8601"), do: :iso8601_timestamp
defp function_to_coercion("parseDate"), do: :iso8601_timestamp
defp function_to_coercion("parseNumber"), do: :number
defp function_to_coercion("parseToInt"), do: :integer
# Converts a unified key string (camelCase from CCXT) to a struct field atom.
# E.g., "askVolume" -> :ask_volume, "bid" -> :bid
@doc false
# Keys come from trusted P1 analysis JSON, not user input
@spec unified_key_to_field_atom(String.t()) :: atom()
defp unified_key_to_field_atom(key) do
key |> Macro.underscore() |> String.to_atom()
end
# Builds a map from unified key strings to coercion atoms based on schema field types.
# E.g., %{"ask" => :number, "symbol" => :string, "timestamp" => :integer}
@doc false
@spec build_field_type_map([map()]) :: %{String.t() => atom()}
defp build_field_type_map(schema_fields) do
Map.new(schema_fields, fn %{source: source, type: type_str} ->
{source, type_to_coercion(type_str)}
end)
end
@doc """
Converts a schema type string to a coercion atom.
## Examples
iex> CCXT.ResponseParser.MappingCompiler.type_to_coercion("number() | nil")
:number
iex> CCXT.ResponseParser.MappingCompiler.type_to_coercion("String.t() | nil")
:string
iex> CCXT.ResponseParser.MappingCompiler.type_to_coercion("integer() | nil")
:integer
"""
@spec type_to_coercion(String.t()) :: atom()
def type_to_coercion(type_str) do
cond do
String.starts_with?(type_str, "integer") -> :integer
String.starts_with?(type_str, "number") -> :number
String.starts_with?(type_str, "String") -> :string
String.starts_with?(type_str, "boolean") -> :bool
true -> :value
end
end
end