cmdc_eval

defmodule CMDCEval.Reasoning do
  @moduledoc """
  Reasoning Eval adapter。

  接收 `CMDC.Reasoning.Runner` 返回值或 EventBus 中的 `reasoning_*` 事件，
  归一化为可做发布门禁的指标 map。该模块只依赖公开事件 shape，不启动 Agent，
  适合离线 fixture、CI 回放和 Hex 发布前的推理策略回归检查。
  """

  @type source :: map() | [term()]
  @type metrics :: map()
  @type failure :: %{metric: atom(), expected: term(), actual: term()}

  @reasoning_events [
    :reasoning_thought,
    :reasoning_branch,
    :reasoning_score,
    :reasoning_prune,
    :reasoning_done
  ]

  @doc "从 EventBus 事件列表构造 ReasoningEval context。"
  @spec from_events([term()], keyword()) :: map()
  def from_events(events, opts \\ []) when is_list(events) do
    normalized = Enum.flat_map(events, &normalize_event/1)

    context = %{
      events: normalized,
      result: last_payload(normalized, :reasoning_done),
      metadata: Keyword.get(opts, :metadata, %{})
    }

    Map.put(context, :metrics, metrics(context))
  end

  @doc "从 `CMDC.Reasoning.Runner.run/4` 返回 payload 构造 ReasoningEval context。"
  @spec from_result(map(), keyword()) :: map()
  def from_result(result, opts \\ []) when is_map(result) do
    context = %{
      events: [],
      result: result,
      metadata: Keyword.get(opts, :metadata, %{})
    }

    Map.put(context, :metrics, metrics(context))
  end

  @doc "计算 reasoning 指标。"
  @spec metrics(source()) :: metrics()
  def metrics(source) when is_list(source), do: source |> from_events() |> Map.fetch!(:metrics)

  def metrics(%{events: [], result: result}) do
    (result || %{})
    |> result_metrics()
    |> Map.put(:events_digest, nil)
  end

  def metrics(%{events: events, result: result}) when is_list(events) do
    event_metrics = event_metrics(events)
    result_metrics = result_metrics(result || %{})

    result_metrics
    |> Map.merge(event_metrics, fn
      _key, nil, value -> value
      _key, value, nil -> value
      _key, _left, right -> right
    end)
    |> Map.put(:events_digest, events_digest(events))
  end

  def metrics(%{} = result), do: result |> from_result() |> Map.fetch!(:metrics)

  @doc "按门禁策略返回 `:ok` 或结构化 failures。"
  @spec gate(source(), map() | keyword()) :: :ok | {:error, [failure()]}
  def gate(source, policy) do
    metrics = source |> ensure_context() |> Map.fetch!(:metrics)
    policy = Map.new(policy)

    []
    |> check_min(metrics, policy, :best_score, :best_score_min)
    |> check_min(metrics, policy, :thought_count, :thought_count_min)
    |> check_max(metrics, policy, :thought_count, :thought_count_lte)
    |> check_max(metrics, policy, :branch_count, :branch_count_lte)
    |> check_max(metrics, policy, :revision_count, :revision_count_lte)
    |> check_max(metrics, policy, :max_depth, :max_depth_lte)
    |> check_max(metrics, policy, :total_tokens, :total_tokens_lte)
    |> check_max(metrics, policy, :cost_usd, :cost_usd_lte)
    |> check_bool(metrics, policy, :done?, :require_done)
    |> check_bool(metrics, policy, :answer_present?, :require_answer)
    |> case do
      [] -> :ok
      failures -> {:error, Enum.reverse(failures)}
    end
  end

  defp ensure_context(%{metrics: %{}} = context), do: context
  defp ensure_context(events) when is_list(events), do: from_events(events)
  defp ensure_context(%{} = result), do: from_result(result)

  defp event_metrics(events) do
    thoughts = payloads(events, :reasoning_thought)
    branches = payloads(events, :reasoning_branch)
    scores = payloads(events, :reasoning_score)
    prunes = payloads(events, :reasoning_prune)
    done = last_payload(events, :reasoning_done) || %{}
    all_payloads = Enum.map(events, &elem(&1, 1))

    %{
      strategy: first_value(all_payloads, [:strategy]),
      thought_count: length(thoughts),
      branch_count: length(branches),
      score_count: length(scores),
      prune_count: length(prunes),
      revision_count: Enum.count(thoughts, &(field(&1, :stage) in [:revise, "revise"])),
      done?: done != %{},
      answer_present?: answer_present?(field(done, :answer)),
      max_depth: max_number(all_payloads, :depth, 0),
      best_score: best_score(scores ++ result_thoughts(done) ++ answer_candidates(done)),
      total_tokens: token_total(field(done, :token_usage)),
      cost_usd: token_cost(field(done, :token_usage))
    }
  end

  defp result_metrics(result) when is_map(result) do
    thoughts = result_thoughts(result)
    answer = field(result, :answer)
    usage = field(result, :token_usage)

    %{
      strategy: field(result, :strategy),
      thought_count: length(thoughts),
      branch_count: length(answer_candidates(result)),
      score_count: Enum.count(thoughts, &score?/1),
      prune_count: 0,
      revision_count: Enum.count(thoughts, &(field(&1, :stage) in [:revise, "revise"])),
      done?: field(result, :error) == nil and field(result, :reason) == nil,
      answer_present?: answer_present?(answer),
      max_depth: field(result, :depth) || 0,
      best_score: best_score(thoughts ++ answer_candidates(result)),
      total_tokens: token_total(usage),
      cost_usd: token_cost(usage)
    }
  end

  defp normalize_event({:cmdc_event, _sid, event}), do: normalize_event(event)

  defp normalize_event({type, payload}) when type in @reasoning_events and is_map(payload),
    do: [{type, payload}]

  defp normalize_event(%{type: type, payload: payload})
       when type in @reasoning_events and is_map(payload),
       do: [{type, payload}]

  defp normalize_event(%{"type" => type, "payload" => payload}) when is_map(payload) do
    case normalize_event_type(type) do
      nil -> []
      normalized -> normalize_event({normalized, payload})
    end
  end

  defp normalize_event(_), do: []

  defp normalize_event_type(type) when is_atom(type), do: type

  defp normalize_event_type(type) when is_binary(type) do
    Enum.find(@reasoning_events, &(Atom.to_string(&1) == type))
  end

  defp payloads(events, type) do
    events
    |> Enum.filter(&(elem(&1, 0) == type))
    |> Enum.map(&elem(&1, 1))
  end

  defp last_payload(events, type) do
    events
    |> payloads(type)
    |> List.last()
  end

  defp result_thoughts(result) when is_map(result) do
    case field(result, :thoughts) do
      thoughts when is_list(thoughts) -> thoughts
      _ -> []
    end
  end

  defp answer_candidates(result) when is_map(result) do
    result
    |> field(:answer)
    |> case do
      %{} = answer ->
        case field(answer, :candidates) do
          candidates when is_list(candidates) -> candidates
          _ -> []
        end

      _ ->
        []
    end
  end

  defp answer_present?(nil), do: false
  defp answer_present?(answer) when is_binary(answer), do: String.trim(answer) != ""

  defp answer_present?(%{} = answer) do
    answer
    |> first_value([:answer, :content, :text])
    |> answer_present?()
  end

  defp answer_present?(_), do: false

  defp score?(payload), do: is_number(field(payload, :score))

  defp best_score(payloads) do
    payloads
    |> Enum.map(&field(&1, :score))
    |> Enum.filter(&is_number/1)
    |> case do
      [] -> nil
      scores -> Enum.max(scores)
    end
  end

  defp max_number(payloads, key, default) do
    payloads
    |> Enum.map(&field(&1, key))
    |> Enum.filter(&is_number/1)
    |> case do
      [] -> default
      values -> Enum.max(values)
    end
  end

  defp token_total(nil), do: 0
  defp token_total(usage), do: field(usage, :total_tokens) || 0

  defp token_cost(nil), do: 0.0
  defp token_cost(usage), do: field(usage, :cost_usd) || 0.0

  defp check_min(failures, _metrics, policy, _metric, policy_key)
       when not is_map_key(policy, policy_key),
       do: failures

  defp check_min(failures, metrics, policy, metric, policy_key) do
    actual = Map.get(metrics, metric)
    expected = Map.fetch!(policy, policy_key)

    if is_number(actual) and actual >= expected do
      failures
    else
      [%{metric: metric, expected: {:>=, expected}, actual: actual} | failures]
    end
  end

  defp check_max(failures, _metrics, policy, _metric, policy_key)
       when not is_map_key(policy, policy_key),
       do: failures

  defp check_max(failures, metrics, policy, metric, policy_key) do
    actual = Map.get(metrics, metric)
    expected = Map.fetch!(policy, policy_key)

    if is_number(actual) and actual <= expected do
      failures
    else
      [%{metric: metric, expected: {:<=, expected}, actual: actual} | failures]
    end
  end

  defp check_bool(failures, _metrics, policy, _metric, policy_key)
       when not is_map_key(policy, policy_key),
       do: failures

  defp check_bool(failures, metrics, policy, metric, policy_key) do
    expected = Map.fetch!(policy, policy_key)
    actual = Map.get(metrics, metric)

    if actual == expected do
      failures
    else
      [%{metric: metric, expected: expected, actual: actual} | failures]
    end
  end

  defp events_digest([]), do: nil

  defp events_digest(events) do
    digest =
      events
      |> :erlang.term_to_binary()
      |> then(&:crypto.hash(:sha256, &1))
      |> Base.encode16(case: :lower)

    "sha256:" <> digest
  end

  defp first_value(values, keys) when is_list(values) do
    Enum.find_value(values, &first_value(&1, keys))
  end

  defp first_value(map, keys) when is_map(map) do
    Enum.find_value(keys, &field(map, &1))
  end

  defp first_value(_value, _keys), do: nil

  defp field(%_{} = struct, key), do: Map.get(struct, key)

  defp field(map, key) when is_map(map),
    do: Map.get(map, key) || Map.get(map, Atom.to_string(key))

  defp field(_value, _key), do: nil
end