Packages

ex_llama

0.2.1
0.2.3 0.2.2 0.2.1 0.2.0 0.1.0 0.0.2 0.0.1

NIF Wrapper around llama.cpp allowing elixir code to load/infer against gguf format models.

HexDocs

Current section

Files

Jump to
Readme
7 Versions
5 Dependencies
Dependants
Files
Activity
ex_llama lib ex_llama session.ex
Raw

lib/ex_llama/session.ex 

defmodule ExLLama.Session do

  defstruct [

    seed: nil,

    model_name: nil,

    resource: nil

  ]



  def default_options(), do: ExLLama.Nif.__session_nif_default_session_options__()

  def advance_context_with_tokens(%__MODULE__{resource: _} = session, context), do: ExLLama.Nif.__session_nif_advance_context_with_tokens__(session.resource, context)

  def advance_context(%__MODULE__{resource: _} = session, context), do: ExLLama.Nif.__session_nif_advance_context__(session.resource, context)

  def start_completing_with(%__MODULE__{resource: _} = session, options) do

    max_tokens = options[:max_tokens] || 512

    pid = options[:pid] || self()

    # Capture context from process dict before spawning

    prompt = Process.get({:ex_llama_ctx, session.resource}, "")

    seed = Process.get({:ex_llama_seed, session.resource})

    opts = %{max_tokens: max_tokens}

    opts = if seed, do: Map.put(opts, :seed, seed), else: opts

    ExLLama.Nif.streaming_completion(session.resource, prompt, pid, opts)

    :ok

  end

  def completion(%__MODULE__{resource: _} = session, max_tokens, stop), do: ExLLama.Nif.__session_nif_completion__(session.resource, max_tokens, stop)

  def model(%__MODULE__{resource: _} = session), do: ExLLama.Nif.__session_nif_model__(session.resource)

  def params(%__MODULE__{resource: _} = session), do: ExLLama.Nif.__session_nif_params__(session.resource)

  def context_size(%__MODULE__{resource: _} = session), do: ExLLama.Nif.__session_nif_context_size__(session.resource)

  def context(%__MODULE__{resource: _} = session), do: ExLLama.Nif.__session_nif_context__(session.resource)

  def truncate_context(%__MODULE__{resource: _} = session, n_tokens), do: ExLLama.Nif.__session_nif_truncate_context__(session.resource, n_tokens)

  def set_context_to_tokens(%__MODULE__{resource: _} = session, tokens), do: ExLLama.Nif.__session_nif_set_context_to_tokens__(session.resource, tokens)

  def set_context(%__MODULE__{resource: _} = session, context), do: ExLLama.Nif.__session_nif_set_context__(session.resource, context)

  def deep_copy(%__MODULE__{resource: _} = session) do

    with {:ok, copy} <- ExLLama.Nif.__session_deep_copy__(session.resource) do

      {:ok, put_in(copy, [Access.key(:model_name)], session.model_name)}

    end

  end



end