Packages
llama_cpp_ex
0.8.31
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.28
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.10
0.8.9
0.8.8
0.8.7
0.8.6
0.8.5
0.8.4
0.8.3
0.8.2
0.8.1
0.8.0
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.4
0.7.3
0.7.2
0.7.0
0.6.14
0.6.13
0.6.12
0.6.11
0.6.10
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.0
0.4.4
0.4.3
0.4.2
0.4.1
0.3.0
0.2.0
Elixir bindings for llama.cpp — run LLMs locally with Metal, CUDA, Vulkan, or CPU acceleration.
Current section
Files
Jump to
Current section
Files
lib/llama_cpp_ex/model_manager/model_io.ex
defmodule LlamaCppEx.ModelManager.ModelIO do
@moduledoc """
Default `LlamaCppEx.ModelManager.Backend` implementation.
Delegates to `LlamaCppEx.Hub` for downloads, `LlamaCppEx.Model` for direct
loading, and `LlamaCppEx.Server` (under the manager's `DynamicSupervisor`,
named via the manager's `Registry`) for server-backed models.
"""
@behaviour LlamaCppEx.ModelManager.Backend
alias LlamaCppEx.{Hub, Model, Server}
@registry LlamaCppEx.ModelRegistry
@dynsup LlamaCppEx.ModelDynSup
# Hub-only options, stripped before handing opts to Model.load / Server.
@hub_keys [:cache_dir, :token, :revision, :force, :progress]
# Manager-only options, never forwarded to the native layer.
@manager_keys [:mode, :capabilities, :default, :memory_budget, :io]
@doc "The registry name backing servers are registered under."
def registry, do: @registry
@doc "The dynamic supervisor name backing servers run under."
def dynamic_supervisor, do: @dynsup
@impl true
def resolve_source({:path, path}, _opts) do
case File.stat(path) do
{:ok, %{size: size}} -> {:ok, path, size}
{:error, reason} -> {:error, {:stat_failed, path, reason}}
end
end
def resolve_source({:hub, repo_id, filename}, opts) do
hub_opts = Keyword.take(opts, @hub_keys)
with {:ok, path} <- Hub.download(repo_id, filename, hub_opts),
{:ok, %{size: size}} <- File.stat(path) do
{:ok, path, size}
end
end
@impl true
def load_model(path, opts) do
Model.load(path, native_opts(opts))
end
@impl true
def start_server(id, path, opts) do
server_opts = [model_path: path, name: via(id)] ++ native_opts(opts)
spec = %{
id: {:llama_server, id},
start: {Server, :start_link, [server_opts]},
restart: :temporary
}
DynamicSupervisor.start_child(@dynsup, spec)
end
@impl true
def stop_server(pid) when is_pid(pid) do
case DynamicSupervisor.terminate_child(@dynsup, pid) do
:ok -> :ok
{:error, :not_found} -> :ok
end
end
defp via(id), do: {:via, Registry, {@registry, id}}
defp native_opts(opts) do
Keyword.drop(opts, @hub_keys ++ @manager_keys)
end
end