tinkex

defmodule Tinkex.Examples.CheckpointMultiDeleteLive do
  @moduledoc """
  Live multi-delete demo: creates two checkpoints, caches their paths, then
  deletes both in a single CLI invocation. Requires only `TINKER_API_KEY`.
  """

  @base_model "meta-llama/Llama-3.1-8B"
  @checkpoint_cache Path.join(["tmp", "checkpoints", "default.path"])

  alias Tinkex.{CLI, Config, Error, ServiceClient, TrainingClient}
  alias Tinkex.Types.{LoraConfig, SaveWeightsResponse}

  def run do
    {:ok, _} = Application.ensure_all_started(:tinkex)
    File.mkdir_p!(Path.dirname(@checkpoint_cache))

    timestamp = System.system_time(:second)
    names = ["multi-delete-#{timestamp}-a", "multi-delete-#{timestamp}-b"]

    with {:ok, config} <- build_config(),
         await_timeout = compute_await_timeout(config),
         {:ok, service} <- ServiceClient.start_link(config: config),
         {:ok, training} <-
           ServiceClient.create_lora_training_client(service, @base_model,
             lora_config: %LoraConfig{rank: 8}
           ),
         {:ok, paths} <- save_checkpoints(training, names, await_timeout),
         :ok <- cache_default_path(paths),
         {:ok, delete_summary} <- run_multi_delete(paths),
         :ok <- shutdown([training, service]) do
      IO.puts("\nMulti-delete summary:")
      IO.inspect(delete_summary, label: "result")
    else
      {:error, %Error{} = error} ->
        IO.puts(:stderr, "Error: #{Error.format(error)}")
        if error.data, do: IO.inspect(error.data, label: "data")
        System.halt(1)

      {:error, reason} ->
        IO.puts(:stderr, "Error: #{inspect(reason)}")
        System.halt(1)
    end
  end

  defp build_config do
    {:ok, Config.new()}
  rescue
    e -> {:error, e}
  end

  defp compute_await_timeout(%Config{} = config) do
    # Saving checkpoints can exceed a single request timeout (server 408 + client retries).
    # Await long enough to cover worst-case (attempts + jitter), but keep bounded.
    worst_case = config.timeout * (config.max_retries + 1) + 30_000
    max(60_000, min(worst_case, 15 * 60_000))
  end

  defp save_checkpoints(training, names, await_timeout) do
    names
    |> Enum.map(&save_checkpoint(training, &1, await_timeout))
    |> Enum.reduce_while([], fn
      {:ok, path}, acc -> {:cont, [path | acc]}
      {:error, reason}, _acc -> {:halt, {:error, reason}}
    end)
    |> case do
      {:error, _} = error -> error
      paths -> {:ok, Enum.reverse(paths)}
    end
  end

  defp save_checkpoint(training, name, await_timeout) do
    with {:ok, task} <- TrainingClient.save_state(training, name),
         {:ok, %SaveWeightsResponse{path: path}} <-
           await(task, "save_state #{name}", await_timeout) do
      IO.puts("Saved checkpoint #{name}: #{path}")
      {:ok, path}
    else
      {:error, %Error{} = error} ->
        {:error, error}

      other ->
        {:error, other}
    end
  end

  defp run_multi_delete(paths) do
    IO.puts("\nDeleting #{length(paths)} checkpoints with one confirmation...")
    args = ["checkpoint", "delete"] ++ paths ++ ["--yes"]

    case CLI.run(args) do
      {:ok, summary} -> {:ok, summary}
      {:error, reason} -> {:error, reason}
    end
  end

  defp cache_default_path([first | _]) do
    File.write!(@checkpoint_cache, first)
    IO.puts("Cached default checkpoint path at #{@checkpoint_cache}: #{first}")
    :ok
  end

  defp cache_default_path([]), do: :ok

  defp await(task, label, timeout_ms) do
    try do
      case Task.await(task, timeout_ms) do
        {:ok, value} -> {:ok, value}
        {:error, %Error{} = error} -> {:error, error}
        other -> {:error, {:unexpected_reply, label, other}}
      end
    catch
      :exit, reason ->
        {:error, {:task_exit, label, reason}}
    end
  end

  defp shutdown(pids) do
    Enum.each(pids, fn pid ->
      if is_pid(pid), do: Process.exit(pid, :normal)
    end)

    :ok
  end
end

Tinkex.Examples.CheckpointMultiDeleteLive.run()