Current section
Files
Jump to
Current section
Files
lib/snakebridge/generator.ex
defmodule SnakeBridge.Generator do
@moduledoc """
Generates Elixir source files from introspection data.
"""
alias SnakeBridge.Docs.{MarkdownConverter, RstParser}
alias SnakeBridge.Generator.{Class, Function, TypeMapper}
@reserved_words ~w(def defp defmodule class do end if unless case cond for while with fn when and or not true false nil in try catch rescue after else raise throw receive)
@dunder_mappings %{
"__init__" => "new",
"__str__" => "to_string",
"__repr__" => "inspect",
"__len__" => "length",
"__getitem__" => "get",
"__setitem__" => "put",
"__contains__" => "member?"
}
@spec render_library(SnakeBridge.Config.Library.t(), list(), list(), keyword()) :: String.t()
def render_library(library, functions, classes, opts \\ []) do
context = TypeMapper.build_context(classes)
TypeMapper.with_context(context, fn ->
do_render_library(library, functions, classes, opts)
end)
end
defp do_render_library(library, functions, classes, opts) do
version = Keyword.get(opts, :version, Application.spec(:snakebridge, :vsn) |> to_string())
module_docs = Keyword.get(opts, :module_docs) || %{}
lock_data = Keyword.get(opts, :lock_data)
module_name = module_to_string(library.module_name)
header = """
# Generated by SnakeBridge v#{version} - DO NOT EDIT MANUALLY
# Regenerate with: mix compile
# Library: #{library.python_name} #{library.version || "unknown"}
"""
moduledoc = build_moduledoc(library, module_name, library.python_name, module_docs, lock_data)
functions_by_module =
functions
|> Enum.map(&Map.put_new(&1, "python_module", library.python_name))
|> Enum.group_by(& &1["python_module"])
base_functions = Map.get(functions_by_module, library.python_name, [])
function_defs =
base_functions
|> Enum.sort_by(& &1["name"])
|> Enum.map_join("\n\n", &Function.render_function(&1, library))
submodule_modules =
module_docs
|> Map.keys()
|> Enum.filter(&String.starts_with?(&1, library.python_name <> "."))
submodule_modules =
submodule_modules ++ Map.keys(functions_by_module)
submodule_defs =
submodule_modules
|> Enum.reject(&(&1 == library.python_name))
|> Enum.uniq()
|> Enum.sort()
|> Enum.map_join("\n\n", fn python_module ->
funcs = Map.get(functions_by_module, python_module, [])
render_submodule(python_module, funcs, library, module_docs, lock_data)
end)
class_defs =
classes
|> Enum.sort_by(&class_sort_key/1)
|> Enum.map_join("\n\n", &Class.render_class(&1, library))
discovery = render_discovery(functions, classes)
"""
#{header}
defmodule #{module_name} do
#{moduledoc}
#{function_defs}
#{submodule_defs}
#{class_defs}
#{discovery}
end
"""
|> Code.format_string!()
|> IO.iodata_to_binary()
|> ensure_final_newline()
end
@spec generate_library(
SnakeBridge.Config.Library.t(),
list(),
list(),
SnakeBridge.Config.t(),
map(),
map() | nil
) :: :ok
def generate_library(library, functions, classes, config, module_docs \\ %{}, lock_data \\ nil) do
lock_data = lock_data || SnakeBridge.Lock.load()
case config.generated_layout do
:single -> generate_single_file(library, functions, classes, config, module_docs, lock_data)
:split -> generate_split_layout(library, functions, classes, config, module_docs, lock_data)
_ -> generate_split_layout(library, functions, classes, config, module_docs, lock_data)
end
end
# Legacy single-file generation
defp generate_single_file(library, functions, classes, config, module_docs, lock_data) do
start_time = System.monotonic_time()
File.mkdir_p!(config.generated_dir)
path = Path.join(config.generated_dir, "#{library.python_name}.ex")
source =
render_library(library, functions, classes,
version: Application.spec(:snakebridge, :vsn),
module_docs: module_docs,
lock_data: lock_data
)
result = write_if_changed(path, source)
cleanup_stale_generated_files(library, config, [path])
register_generated_library(library, functions, classes, config, [path])
bytes_written = if result == :written, do: byte_size(source), else: 0
SnakeBridge.Telemetry.generate_stop(
start_time,
library.name,
path,
bytes_written,
length(functions),
length(classes)
)
:ok
end
# Split layout: one file per Python module path
defp generate_split_layout(library, functions, classes, config, module_docs, lock_data) do
alias SnakeBridge.Generator.PathMapper
start_time = System.monotonic_time()
context = TypeMapper.build_context(classes)
TypeMapper.with_context(context, fn ->
do_generate_split_layout(
library,
functions,
classes,
config,
module_docs,
lock_data,
start_time
)
end)
end
defp do_generate_split_layout(
library,
functions,
classes,
config,
module_docs,
lock_data,
start_time
) do
alias SnakeBridge.Generator.PathMapper
# Group functions and classes by python_module
functions_by_module =
functions
|> Enum.map(&Map.put_new(&1, "python_module", library.python_name))
|> Enum.group_by(& &1["python_module"])
module_modules =
split_layout_module_modules(library, functions_by_module, classes, module_docs)
expected_paths = split_layout_expected_paths(library, module_modules, classes, config)
class_module_names =
classes
|> Enum.map(&class_module_name(&1, library))
|> MapSet.new()
doc_context = %{module_docs: module_docs, lock_data: lock_data}
module_files =
write_split_module_files(
library,
module_modules,
functions_by_module,
functions,
classes,
class_module_names,
config,
doc_context
)
class_files = write_split_class_files(library, classes, config)
all_paths = module_files ++ class_files
cleanup_stale_generated_files(library, config, expected_paths)
register_generated_library(library, functions, classes, config, all_paths)
total_bytes =
Enum.reduce(all_paths, 0, fn path, acc ->
case File.stat(path) do
{:ok, %{size: size}} -> acc + size
_ -> acc
end
end)
duration_ms =
System.convert_time_unit(System.monotonic_time() - start_time, :native, :millisecond)
duration_str = format_duration(duration_ms)
Mix.shell().info(
"[SnakeBridge] Generated #{length(all_paths)} files for #{library.python_name} in #{duration_str}"
)
SnakeBridge.Telemetry.generate_stop(
start_time,
library.name,
List.first(all_paths) || config.generated_dir,
total_bytes,
length(functions),
length(classes)
)
:ok
end
defp format_duration(ms) when ms < 1000, do: "#{ms}ms"
defp format_duration(ms), do: "#{Float.round(ms / 1000, 1)}s"
defp split_layout_module_modules(library, functions_by_module, classes, module_docs) do
function_modules = Map.keys(functions_by_module)
class_modules =
classes
|> Enum.map(&(&1["python_module"] || library.python_name))
|> Enum.reject(&is_nil/1)
module_doc_modules =
module_docs
|> Map.keys()
|> Enum.filter(&String.starts_with?(&1, library.python_name))
[library.python_name | function_modules ++ class_modules ++ module_doc_modules]
|> Enum.uniq()
|> Enum.sort()
end
defp split_layout_expected_paths(library, module_modules, classes, config) do
alias SnakeBridge.Generator.PathMapper
module_paths = Enum.map(module_modules, &PathMapper.module_to_path(&1, config.generated_dir))
class_paths =
classes
|> Enum.map(fn class ->
python_module = class["python_module"] || library.python_name
class_name = class["name"] || class["class"] || "Class"
PathMapper.class_file_path(python_module, class_name, config.generated_dir)
end)
module_paths ++ class_paths
end
defp write_split_module_files(
library,
module_modules,
functions_by_module,
functions,
classes,
class_module_names,
config,
doc_context
) do
alias SnakeBridge.Generator.PathMapper
%{module_docs: module_docs, lock_data: lock_data} = doc_context
Enum.map(module_modules, fn python_module ->
funcs = Map.get(functions_by_module, python_module, [])
path = PathMapper.module_to_path(python_module, config.generated_dir)
File.mkdir_p!(Path.dirname(path))
elixir_module =
python_module
|> PathMapper.python_module_to_elixir_module(library.module_name)
|> module_to_string()
|> resolve_module_collision(python_module, library, class_module_names)
discovery =
if python_module == library.python_name do
{functions, classes}
else
nil
end
source =
render_module_file(
library,
elixir_module,
python_module,
funcs,
discovery,
module_docs,
lock_data
)
write_if_changed(path, source)
path
end)
end
defp resolve_module_collision(elixir_module, python_module, library, class_module_names) do
if python_module == library.python_name do
elixir_module
else
if MapSet.member?(class_module_names, elixir_module) do
elixir_module <> ".Module"
else
elixir_module
end
end
end
defp write_split_class_files(library, classes, config) do
alias SnakeBridge.Generator.PathMapper
total = length(classes)
progress_interval = max(div(total, 10), 100)
if total > 0 do
Mix.shell().info(
"[SnakeBridge] Generating #{total} class files for #{library.python_name}..."
)
end
classes
|> Enum.with_index(1)
|> Enum.map(fn {class, index} ->
# Show progress at intervals
if total > 100 and rem(index, progress_interval) == 0 do
percent = div(index * 100, total)
Mix.shell().info("[SnakeBridge] Progress: #{index}/#{total} (#{percent}%)")
end
python_module = class["python_module"] || library.python_name
class_name = class["name"] || class["class"] || "Class"
path = PathMapper.class_file_path(python_module, class_name, config.generated_dir)
File.mkdir_p!(Path.dirname(path))
elixir_module = class_module_name(class, library)
source = render_class_file(library, elixir_module, python_module, class)
write_if_changed(path, source)
path
end)
|> Enum.uniq()
end
@doc false
def render_module_file(
library,
elixir_module,
python_module,
functions,
discovery \\ nil,
module_docs \\ %{},
lock_data \\ nil
) do
version = Application.spec(:snakebridge, :vsn) |> to_string()
module_name = module_to_string(elixir_module)
header = """
# Generated by SnakeBridge v#{version} - DO NOT EDIT MANUALLY
# Regenerate with: mix compile
# Library: #{library.python_name} #{library.version || "unknown"}
# Python module: #{python_module}
"""
moduledoc = build_moduledoc(library, module_name, python_module, module_docs, lock_data)
function_defs =
functions
|> Enum.sort_by(& &1["name"])
|> Enum.map_join("\n\n", &Function.render_function(&1, library))
discovery_source =
case discovery do
{discovery_functions, discovery_classes} ->
render_discovery(discovery_functions, discovery_classes)
_ ->
""
end
"""
#{header}
defmodule #{module_name} do
#{moduledoc}
#{function_defs}
#{discovery_source}
end
"""
|> Code.format_string!()
|> IO.iodata_to_binary()
|> ensure_final_newline()
end
defp build_moduledoc(library, module_name, python_module, module_docs, lock_data) do
module_docs = module_docs || %{}
module_info = Map.get(module_docs, python_module, %{})
raw_doc = module_info["docstring"] || ""
formatted_doc =
raw_doc
|> format_docstring([], nil)
|> String.trim()
fallback =
if python_module == library.python_name do
"SnakeBridge bindings for `#{library.python_name}`."
else
"Submodule bindings for `#{python_module}`."
end
doc_section = if formatted_doc == "", do: fallback, else: formatted_doc
error_note =
case module_info["doc_source"] do
"error" ->
reason = module_info["doc_missing_reason"] || "unknown error"
"## Notes\n\n- Module failed to import during generation: #{reason}"
_ ->
nil
end
sections =
[
doc_section,
error_note,
python_docs_section(library),
version_section(library, lock_data, module_info),
runtime_options_block(module_name)
]
|> Enum.filter(&present_section?/1)
body = Enum.join(sections, "\n\n")
moduledoc = """
@moduledoc \"\"\"
#{body}
\"\"\"
@doc false
def __snakebridge_python_name__, do: "#{python_module}"
@doc false
def __snakebridge_library__, do: "#{library.python_name}"
"""
indent(moduledoc, 2)
end
defp present_section?(section) when is_binary(section) do
String.trim(section) != ""
end
defp present_section?(_section), do: false
defp python_docs_section(library) do
case python_docs_url(library) do
nil -> nil
url -> "## Python Docs\n\n- [Python docs](#{url})"
end
end
defp python_docs_url(library) do
if library.version == :stdlib do
"https://docs.python.org/3/library/#{library.python_name}.html"
else
library.docs_url
end
end
defp version_section(library, lock_data, module_info) do
if library.version == :stdlib do
python_version = lock_python_version(lock_data)
if python_version do
"## Version\n\n- Python stdlib (Python #{python_version})"
else
"## Version\n\n- Python stdlib"
end
else
requested = requested_version(library)
observed = observed_version(library, lock_data, module_info)
lines =
[]
|> maybe_append_version("Requested", requested)
|> maybe_append_version("Observed at generation", observed)
lines =
if lines == [] do
["- Version unspecified (not pinned)"]
else
lines
end
"## Version\n\n" <> Enum.join(lines, "\n")
end
end
defp maybe_append_version(lines, _label, nil), do: lines
defp maybe_append_version(lines, label, value) do
lines ++ ["- #{label}: #{value}"]
end
defp requested_version(%{version: nil}), do: nil
defp requested_version(%{version: :stdlib}), do: nil
defp requested_version(%{version: version}) do
to_string(version)
end
defp observed_version(library, lock_data, module_info) do
lock_version = lock_package_version(lock_data, library)
module_version = module_info["module_version"]
cond do
is_binary(lock_version) and lock_version != "" -> lock_version
is_binary(module_version) and module_version != "" -> module_version
true -> nil
end
end
defp lock_package_version(nil, _library), do: nil
defp lock_package_version(lock_data, library) do
package_name = library.pypi_package || library.python_name
packages = Map.get(lock_data || %{}, "python_packages", %{})
normalized = normalize_package_name(package_name)
packages
|> Enum.find_value(fn {name, info} ->
if normalize_package_name(name) == normalized do
info["version"] || info[:version]
end
end)
end
defp normalize_package_name(name) when is_binary(name) do
name
|> String.downcase()
|> String.replace("_", "-")
end
defp normalize_package_name(_name), do: ""
defp lock_python_version(lock_data) do
get_in(lock_data || %{}, ["environment", "python_version"])
end
defp runtime_options_block(module_name) do
"""
## Runtime Options
All functions accept a `__runtime__` option for controlling execution behavior:
#{module_name}.some_function(args, __runtime__: [timeout: 120_000])
### Supported runtime options
- `:timeout` - Call timeout in milliseconds (default: 120,000ms / 2 minutes)
- `:timeout_profile` - Use a named profile (`:default`, `:ml_inference`, `:batch_job`, `:streaming`)
- `:stream_timeout` - Timeout for streaming operations (default: 1,800,000ms / 30 minutes)
- `:session_id` - Override the session ID for this call
- `:pool_name` - Target a specific Snakepit pool (multi-pool setups)
- `:affinity` - Override session affinity (`:hint`, `:strict_queue`, `:strict_fail_fast`)
### Timeout Profiles
- `:default` - 2 minute timeout for regular calls
- `:ml_inference` - 10 minute timeout for ML/LLM workloads
- `:batch_job` - Unlimited timeout for long-running jobs
- `:streaming` - 2 minute timeout, 30 minute stream_timeout
### Example with timeout override
# For a long-running ML inference call
#{module_name}.predict(data, __runtime__: [timeout_profile: :ml_inference])
# Or explicit timeout
#{module_name}.predict(data, __runtime__: [timeout: 600_000])
# Route to a pool and enforce strict affinity
#{module_name}.predict(data, __runtime__: [pool_name: :strict_pool, affinity: :strict_queue])
See `SnakeBridge.Defaults` for global timeout configuration.
"""
end
@doc false
def render_class_file(library, elixir_module, python_module, class_info) do
version = Application.spec(:snakebridge, :vsn) |> to_string()
class_name = class_info["name"] || class_info["class"] || "Class"
header = """
# Generated by SnakeBridge v#{version} - DO NOT EDIT MANUALLY
# Regenerate with: mix compile
# Library: #{library.python_name} #{library.version || "unknown"}
# Python module: #{python_module}
# Python class: #{class_name}
"""
# Render class as standalone module
class_source = Class.render_class_standalone(class_info, library, elixir_module)
"""
#{header}
#{class_source}
"""
|> Code.format_string!()
|> IO.iodata_to_binary()
|> ensure_final_newline()
end
@spec write_if_changed(String.t(), String.t()) :: :written | :unchanged
def write_if_changed(path, new_content) do
with_lock(path, fn ->
case File.read(path) do
{:ok, existing} when existing == new_content ->
:unchanged
_ ->
write_atomic(path, new_content)
end
end)
end
defp write_atomic(path, content) do
temp_path = "#{path}.tmp.#{System.unique_integer([:positive])}"
try do
File.mkdir_p!(Path.dirname(path))
File.write!(temp_path, content)
File.rename!(temp_path, path)
:written
rescue
exception ->
File.rm(temp_path)
reraise exception, __STACKTRACE__
end
end
@doc false
def ensure_final_newline(content) when is_binary(content) do
if String.ends_with?(content, "\n") do
content
else
content <> "\n"
end
end
defp with_lock(path, fun) when is_function(fun, 0) do
lock_path = Path.expand(path) <> ".lock"
File.mkdir_p!(Path.dirname(lock_path))
acquire_lock(lock_path, fun, 100)
end
defp acquire_lock(lock_path, fun, retries_left) do
case File.open(lock_path, [:write, :exclusive]) do
{:ok, device} ->
try do
fun.()
after
File.close(device)
File.rm(lock_path)
end
{:error, :eexist} when retries_left > 0 ->
Process.sleep(50)
acquire_lock(lock_path, fun, retries_left - 1)
{:error, reason} ->
raise "Failed to acquire lock #{lock_path}: #{inspect(reason)}"
end
end
defp cleanup_stale_generated_files(library, config, expected_paths) do
layout = config.generated_layout || :split
legacy_single = Path.expand(Path.join(config.generated_dir, "#{library.python_name}.ex"))
expected =
expected_paths
|> Enum.map(&Path.expand/1)
|> MapSet.new()
candidate_paths =
library_candidate_paths(library, config)
|> Enum.map(&Path.expand/1)
|> Enum.uniq()
|> Enum.reject(&MapSet.member?(expected, &1))
stale_paths =
candidate_paths
|> Enum.filter(fn path ->
if path == legacy_single and layout == :split do
true
else
generated_file_for_library?(path, library.python_name)
end
end)
base_dirs = cleanup_base_dirs(library, config)
Enum.each(stale_paths, fn path ->
_ = File.rm(path)
remove_empty_parent_dirs(path, base_dirs)
end)
end
defp library_candidate_paths(library, config) do
registry_paths = registry_paths_for_library(library, config)
library_dir_paths = library_dir_paths(library, config)
legacy_single = Path.join(config.generated_dir, "#{library.python_name}.ex")
registry_paths ++ library_dir_paths ++ [legacy_single]
end
defp registry_paths_for_library(library, config) do
case SnakeBridge.Registry.get(library.python_name) do
%{path: base, files: files} when is_list(files) ->
expand_registry_files(base, files, config.generated_dir)
_ ->
[]
end
end
defp expand_registry_files(base, files, generated_dir) do
base = Path.expand(base)
config_base = Path.expand(generated_dir)
if base == config_base do
Enum.map(files, fn file ->
Path.expand(Path.join(base, file))
end)
else
[]
end
end
defp library_dir_paths(library, config) do
alias SnakeBridge.Generator.PathMapper
dir = PathMapper.module_to_dir(library.python_name, config.generated_dir)
if File.dir?(dir) do
Path.wildcard(Path.join([dir, "**", "*.ex"]))
else
[]
end
end
defp cleanup_base_dirs(library, config) do
registry_base =
case SnakeBridge.Registry.get(library.python_name) do
%{path: base} when is_binary(base) ->
base = Path.expand(base)
config_base = Path.expand(config.generated_dir)
if base == config_base do
[base]
else
[]
end
_ ->
[]
end
[config.generated_dir | registry_base]
|> Enum.map(&Path.expand/1)
|> Enum.uniq()
end
defp generated_file_for_library?(path, library_python_name) do
case File.read(path) do
{:ok, content} ->
header =
content
|> String.split("\n")
|> Enum.take(6)
|> Enum.join("\n")
String.contains?(header, "# Generated by SnakeBridge") and
String.contains?(header, "# Library: #{library_python_name}")
_ ->
false
end
end
defp remove_empty_parent_dirs(path, base_dirs) do
base_dirs
|> Enum.filter(&path_within_base?(path, &1))
|> Enum.each(fn base -> remove_empty_parent_dir(Path.dirname(path), base) end)
end
defp remove_empty_parent_dir(dir, base) do
cond do
dir == base ->
:ok
not File.dir?(dir) ->
:ok
File.ls(dir) == {:ok, []} ->
_ = File.rmdir(dir)
remove_empty_parent_dir(Path.dirname(dir), base)
true ->
:ok
end
end
defp path_within_base?(path, base) do
expanded_path = Path.expand(path)
base_prefix = base |> Path.expand() |> Path.join("")
String.starts_with?(expanded_path, base_prefix)
end
defp register_generated_library(library, functions, classes, config, paths) do
entry = build_registry_entry(library, functions, classes, config, paths)
_ = SnakeBridge.Registry.register(library.python_name, entry)
end
defp build_registry_entry(library, functions, classes, config, paths) when is_list(paths) do
python_version =
case library.version do
nil -> "unknown"
:stdlib -> "stdlib"
value -> to_string(value)
end
# Get relative paths from base dir
relative_files =
paths
|> Enum.map(&Path.relative_to(&1, config.generated_dir))
%{
python_module: library.python_name,
python_version: python_version,
elixir_module: module_to_string(library.module_name),
generated_at: DateTime.utc_now(),
path: config.generated_dir,
files: relative_files,
stats: %{
functions: length(functions),
classes: length(classes),
submodules: count_submodules(library, functions, classes)
}
}
end
defp count_submodules(library, functions, classes) do
base = library.python_name
function_modules =
functions
|> Enum.map(&(&1["python_module"] || &1[:python_module] || base))
class_modules =
classes
|> Enum.map(&(&1["python_module"] || &1[:python_module] || base))
(function_modules ++ class_modules)
|> Enum.uniq()
|> Enum.reject(&(&1 == base))
|> length()
end
defp render_submodule(python_module, functions, library, module_docs, lock_data) do
module_name =
python_module
|> String.split(".")
|> Enum.drop(length(String.split(library.python_name, ".")))
|> Enum.map_join(".", &Macro.camelize/1)
full_module_name = module_to_string(library.module_name) <> "." <> module_name
function_defs =
functions
|> Enum.sort_by(& &1["name"])
|> Enum.map_join("\n\n", &Function.render_function(&1, library))
moduledoc = build_moduledoc(library, full_module_name, python_module, module_docs, lock_data)
"""
defmodule #{module_name} do
#{indent(moduledoc, 2)}
#{indent(function_defs, 4)}
end
"""
end
@doc false
defdelegate render_function(info, library), to: Function
@doc false
defdelegate render_class(class_info, library), to: Class
@doc false
def variadic_max_arity do
Application.get_env(:snakebridge, :variadic_max_arity, 8)
end
@doc false
def opts_guard do
"is_list(opts) and (opts == [] or (is_tuple(hd(opts)) and tuple_size(hd(opts)) == 2 and is_atom(elem(hd(opts), 0))))"
end
defp render_discovery(functions, classes) do
function_list =
functions
|> Enum.map_join(",\n ", fn info ->
name =
case info["elixir_name"] do
elixir when is_binary(elixir) ->
elixir
_ ->
info["name"] |> to_string() |> sanitize_function_name() |> elem(0)
end
arity = required_arity(info["parameters"] || [])
summary =
info["docstring"] |> normalize_docstring() |> String.split("\n") |> List.first() || ""
"{:#{name}, #{arity}, __MODULE__, #{inspect(summary, limit: :infinity, printable_limit: :infinity)}}"
end)
class_list =
classes
|> Enum.map_join(",\n ", fn info ->
module = class_module_name(info, nil)
doc = info["docstring"] |> normalize_docstring()
"{#{module}, #{inspect(doc, limit: :infinity, printable_limit: :infinity)}}"
end)
"""
@doc false
def __functions__ do
[
#{function_list}
]
end
@doc false
def __classes__ do
[
#{class_list}
]
end
@doc false
def __search__(query) do
SnakeBridge.Docs.search(__MODULE__, query)
end
@doc false
def doc(function) do
SnakeBridge.Docs.get(__MODULE__, function)
end
"""
end
@doc """
Normalize a docstring to a string, handling both raw strings and parsed docstring maps.
"""
@spec normalize_docstring(String.t() | map() | nil) :: String.t()
def normalize_docstring(nil), do: ""
def normalize_docstring(doc) when is_binary(doc), do: doc
def normalize_docstring(%{"raw" => raw}) when is_binary(raw), do: raw
def normalize_docstring(%{"summary" => summary, "description" => desc}) do
case {summary, desc} do
{nil, nil} -> ""
{s, nil} when is_binary(s) -> s
{nil, d} when is_binary(d) -> d
{s, d} when is_binary(s) and is_binary(d) -> "#{s}\n\n#{d}"
_ -> ""
end
end
def normalize_docstring(%{}), do: ""
@spec format_docstring(String.t() | map() | nil, list(), map() | nil) :: String.t()
def format_docstring(raw_doc, params \\ [], return_type \\ nil)
def format_docstring(nil, _params, _return_type), do: ""
def format_docstring("", _params, _return_type), do: ""
# Handle parsed docstring maps from full module introspection
def format_docstring(doc, params, return_type) when is_map(doc) do
format_docstring(normalize_docstring(doc), params, return_type)
end
def format_docstring(raw_doc, params, return_type) when is_binary(raw_doc) do
base =
raw_doc
|> RstParser.parse()
|> MarkdownConverter.convert()
extras = format_param_docs(params, return_type, base)
doc =
if extras == "" do
base
else
base <> "\n\n" <> extras
end
doc = rewrite_doc_links(doc)
escape_docstring(doc)
rescue
_ ->
extras = format_param_docs(params, return_type, raw_doc)
doc =
if extras == "" do
raw_doc
else
raw_doc <> "\n\n" <> extras
end
doc = rewrite_doc_links(doc)
escape_docstring(doc)
end
@doc false
@spec format_docstring_with_fallback(String.t() | map() | nil, list(), map() | nil, String.t()) ::
String.t()
def format_docstring_with_fallback(raw_doc, params, return_type, fallback) do
formatted =
raw_doc
|> format_docstring(params, return_type)
|> String.trim()
if formatted == "" do
fallback = fallback |> to_string() |> String.trim()
extras = format_param_docs(params, return_type, fallback)
doc =
cond do
fallback == "" -> ""
extras == "" -> fallback
true -> fallback <> "\n\n" <> extras
end
doc = rewrite_doc_links(doc)
escape_docstring(doc)
else
formatted
end
end
@pydantic_concepts %{
"../concepts/models.md" => "https://docs.pydantic.dev/latest/concepts/models/",
"../concepts/serialization.md" => "https://docs.pydantic.dev/latest/concepts/serialization/",
"../concepts/json.md" => "https://docs.pydantic.dev/latest/concepts/json/"
}
defp rewrite_doc_links(doc) when is_binary(doc) do
Enum.reduce(@pydantic_concepts, doc, fn {from, to}, acc ->
String.replace(acc, from, to)
end)
end
defp escape_docstring(doc) when is_binary(doc) do
doc
|> String.replace("\"\"\"", "\\\"\\\"\\\"")
|> String.replace("\#{", "\\\#{")
end
@spec build_params(list(), map()) :: %{
required: list(map()),
optional_positional: list(map()),
has_args: boolean(),
has_varargs: boolean(),
has_opts: boolean(),
is_variadic: boolean(),
required_keyword_only: list(map()),
optional_keyword_only: list(map()),
has_var_keyword: boolean()
}
def build_params(params, info \\ %{}) when is_list(params) do
signature_available = Map.get(info, "signature_available", true)
if params == [] and not signature_available do
%{
required: [],
optional_positional: [],
has_args: true,
has_varargs: true,
has_opts: true,
is_variadic: true,
required_keyword_only: [],
optional_keyword_only: [],
has_var_keyword: false
}
else
required =
params
|> Enum.filter(&required_positional?/1)
|> Enum.map(¶m_entry/1)
optional_pos =
params
|> Enum.filter(&optional_positional?/1)
|> Enum.map(¶m_entry/1)
required_kw_only = Enum.filter(params, &keyword_only_required?/1)
optional_kw_only = Enum.filter(params, &keyword_only_optional?/1)
has_varargs = Enum.any?(params, &varargs?/1)
has_args =
Enum.any?(params, fn param ->
optional_positional?(param) or varargs?(param)
end)
%{
required: required,
optional_positional: optional_pos,
has_args: has_args,
has_varargs: has_varargs,
has_opts: true,
is_variadic: false,
required_keyword_only: required_kw_only,
optional_keyword_only: optional_kw_only,
has_var_keyword: Enum.any?(params, &kwargs?/1)
}
end
end
defp required_arity(params) do
params
|> Enum.filter(fn param ->
param_kind(param) in ["POSITIONAL_ONLY", "POSITIONAL_OR_KEYWORD"]
end)
|> Enum.reject(¶m_default?/1)
|> length()
end
@doc false
def param_list(param_names, has_args, has_opts, args_name) do
param_names
|> maybe_add_args(has_args, args_name)
|> maybe_add_opts(has_opts)
|> Enum.join(", ")
end
@doc false
def runtime_call(name, python_name, args, opts_expr \\ "opts") do
"SnakeBridge.Runtime.call(__MODULE__, #{function_ref(name, python_name)}, #{args}, #{opts_expr})"
end
@doc false
def runtime_stream_call(name, python_name, args, opts_expr \\ "opts") do
"SnakeBridge.Runtime.stream(__MODULE__, #{function_ref(name, python_name)}, #{args}, #{opts_expr}, callback)"
end
@doc false
def function_ref(name, python_name) do
if python_name == name do
":#{name}"
else
inspect(python_name)
end
end
@doc false
def function_spec(name, param_entries, has_args, return_type) do
args =
param_entries
|> Enum.map(¶m_type_spec/1)
|> maybe_add_args_spec(has_args)
|> Kernel.++(["keyword()"])
return_spec = type_spec_string(return_type)
"@spec #{name}(#{Enum.join(args, ", ")}) :: {:ok, #{return_spec}} | {:error, Snakepit.Error.t()}"
end
@doc false
def method_spec(name, param_entries, has_args, return_type) do
args =
[ref_type_spec()]
|> Kernel.++(Enum.map(param_entries, ¶m_type_spec/1))
|> maybe_add_args_spec(has_args)
|> Kernel.++(["keyword()"])
return_spec = type_spec_string(return_type)
"@spec #{name}(#{Enum.join(args, ", ")}) :: {:ok, #{return_spec}} | {:error, Snakepit.Error.t()}"
end
@doc false
def runtime_method_call(name, python_name, param_names, has_args, args_name) do
args = args_expr(param_names, has_args, args_name)
"SnakeBridge.Runtime.call_method(ref, #{function_ref(name, python_name)}, #{args}, opts)"
end
@doc false
def method_param_suffix(param_names, has_args, has_opts, args_name) do
suffix =
param_names
|> maybe_add_args(has_args, args_name)
|> maybe_add_opts(has_opts)
", " <> Enum.join(suffix, ", ")
end
@doc false
def args_expr(param_names, true, args_name) do
base = "[" <> Enum.join(param_names, ", ") <> "]"
base <> " ++ List.wrap(" <> args_name <> ")"
end
def args_expr(param_names, false, _args_name) do
"[" <> Enum.join(param_names, ", ") <> "]"
end
@doc false
# When we have args, don't add a default because opts will also have a default
# This avoids the Elixir "defaults conflict" error
def maybe_add_args(items, true, args_name), do: items ++ [args_name]
def maybe_add_args(items, false, _args_name), do: items
defp maybe_add_opts(items, _has_opts), do: items ++ ["opts \\\\ []"]
@doc false
def maybe_add_args_spec(items, true), do: items ++ ["list(term())"]
def maybe_add_args_spec(items, false), do: items
@doc false
def normalize_args_line(true, args_name, indent) do
String.duplicate(" ", indent) <>
"{#{args_name}, opts} = SnakeBridge.Runtime.normalize_args_opts(#{args_name}, opts)\n"
end
def normalize_args_line(false, _args_name, _indent), do: ""
@doc false
def keyword_only_validation(required_keyword_only, indent) do
names =
required_keyword_only
|> Enum.map(¶m_name/1)
|> Enum.reject(&is_nil/1)
if names == [] do
""
else
padding = String.duplicate(" ", indent)
"""
#{padding}kw_keys = opts |> Keyword.keys() |> Enum.map(&to_string/1)
#{padding}missing_kw = #{inspect(names)} |> Enum.reject(&(&1 in kw_keys))
#{padding}if missing_kw != [] do
#{padding} raise ArgumentError,
#{padding} "Missing required keyword-only arguments: " <> Enum.join(missing_kw, ", ")
#{padding}end
"""
end
end
@doc false
def extra_args_name(param_names) do
if "args" in param_names do
"extra_args"
else
"args"
end
end
defp param_entry(param) do
%{
name: sanitize_name(param),
type: param_type(param)
}
end
defp param_type(%{"type" => type}) when is_map(type), do: type
defp param_type(%{type: type}) when is_map(type), do: type
defp param_type(_), do: %{"type" => "any"}
@doc false
def param_type_spec(%{type: type}), do: type_spec_string(type)
def param_type_spec(_), do: "term()"
@doc false
def type_spec_string(type) do
type
|> TypeMapper.to_spec()
|> Macro.to_string()
end
defp ref_type_spec do
"SnakeBridge.Ref.t()"
end
defp required_positional?(param) do
kind = param_kind(param)
kind in ["POSITIONAL_ONLY", "POSITIONAL_OR_KEYWORD"] and not param_default?(param)
end
defp optional_positional?(param) do
kind = param_kind(param)
kind in ["POSITIONAL_ONLY", "POSITIONAL_OR_KEYWORD"] and param_default?(param)
end
defp varargs?(param), do: param_kind(param) == "VAR_POSITIONAL"
defp kwargs?(param), do: param_kind(param) == "VAR_KEYWORD"
defp keyword_only_required?(param) do
param_kind(param) == "KEYWORD_ONLY" and not param_default?(param)
end
defp keyword_only_optional?(param) do
param_kind(param) == "KEYWORD_ONLY" and param_default?(param)
end
defp param_kind(%{"kind" => kind}) when is_binary(kind), do: String.upcase(kind)
defp param_kind(%{kind: kind}) when is_binary(kind), do: String.upcase(kind)
defp param_kind(%{kind: kind}), do: kind
defp param_kind(_), do: nil
defp param_default?(%{"default" => _}), do: true
defp param_default?(%{default: _}), do: true
defp param_default?(_), do: false
defp sanitize_name(%{"name" => name}), do: sanitize_name(name)
defp sanitize_name(name) when is_binary(name) do
sanitized =
name
|> Macro.underscore()
|> String.replace(~r/[^a-z0-9_]/, "_")
|> ensure_identifier()
# Escape reserved words for parameter names
if sanitized in @reserved_words do
"py_#{sanitized}"
else
sanitized
end
end
@doc false
def sanitize_function_name(python_name) when is_binary(python_name) do
elixir_name =
python_name
|> Macro.underscore()
|> String.replace(~r/[^a-z0-9_?!]/, "_")
|> ensure_valid_function_identifier()
elixir_name =
if elixir_name in @reserved_words do
"py_#{elixir_name}"
else
elixir_name
end
{elixir_name, python_name}
end
@doc false
@spec sanitize_method_name(String.t()) :: {String.t(), String.t()} | nil
def sanitize_method_name(python_name) when is_binary(python_name) do
cond do
Map.has_key?(@dunder_mappings, python_name) ->
{Map.get(@dunder_mappings, python_name), python_name}
String.starts_with?(python_name, "__") and String.ends_with?(python_name, "__") ->
nil
python_name in @reserved_words ->
{"py_#{python_name}", python_name}
true ->
elixir_name =
python_name
|> Macro.underscore()
|> String.replace(~r/[^a-z0-9_?!]/, "_")
|> ensure_valid_function_identifier()
{elixir_name, python_name}
end
end
defp ensure_valid_function_identifier(""), do: "_"
defp ensure_valid_function_identifier(name) do
if String.match?(name, ~r/^[a-z_][a-z0-9_?!]*$/) do
name
else
"_" <> name
end
end
defp ensure_identifier(name) when is_binary(name) do
trimmed = String.trim_leading(name, "_")
trimmed = if trimmed == "", do: "param", else: trimmed
case trimmed do
<<first::utf8, _rest::binary>> when first in ?a..?z ->
trimmed
<<first::utf8, _rest::binary>> when first in ?0..?9 ->
"param_" <> trimmed
_ ->
"param_" <> trimmed
end
end
defp module_to_string(module) when is_atom(module),
do: module |> Module.split() |> Enum.join(".")
defp module_to_string(module) when is_binary(module), do: module
defp class_sort_key(info) do
{class_module_name(info, nil), class_name(info)}
end
@doc false
def class_name(info) do
info["name"] || info["class"] || "Class"
end
@doc false
def class_python_module(info, library) do
info["python_module"] || library.python_name
end
@doc false
def class_module_name(info, nil) do
case info["module"] do
module when is_binary(module) -> camelize_module_name(module)
_ -> Macro.camelize(class_name(info))
end
end
def class_module_name(info, library) do
case info["module"] do
module when is_binary(module) ->
# Ensure the class name part (last segment) is properly camelized
camelize_module_name(module)
_ ->
python_module = class_python_module(info, library)
python_parts = String.split(python_module, ".")
library_parts = String.split(library.python_name, ".")
extra_parts = Enum.drop(python_parts, length(library_parts))
extra_parts = drop_class_suffix(extra_parts, class_name(info))
library.module_name
|> Module.split()
|> Kernel.++(Enum.map(extra_parts, &Macro.camelize/1))
|> Kernel.++([Macro.camelize(class_name(info))])
|> Module.concat()
|> module_to_string()
end
end
defp camelize_module_name(module) when is_binary(module) do
module
|> String.split(".")
|> Enum.map_join(".", &Macro.camelize/1)
end
@doc false
def relative_module_name(library, module_name) when is_binary(module_name) do
base = module_to_string(library.module_name) <> "."
if String.starts_with?(module_name, base) do
String.replace_prefix(module_name, base, "")
else
module_name
end
end
defp drop_class_suffix(parts, class_name) when is_list(parts) and is_binary(class_name) do
class_suffix = Macro.underscore(class_name)
case List.last(parts) do
^class_suffix -> Enum.drop(parts, -1)
_ -> parts
end
end
defp drop_class_suffix(parts, _class_name), do: parts
@doc false
def indent(text, spaces) do
prefix = String.duplicate(" ", spaces)
text
|> String.split("\n")
|> Enum.map_join("\n", fn line -> prefix <> line end)
end
defp format_param_docs(params, return_type, base) do
param_lines =
params
|> Enum.map(¶m_doc_line/1)
|> Enum.reject(&is_nil/1)
has_params_section? = String.contains?(base, "## Parameters")
has_returns_section? = String.contains?(base, "## Returns")
sections =
[]
|> maybe_add_params_section(param_lines, has_params_section?)
|> maybe_add_return_section(return_type, has_returns_section?)
Enum.join(sections, "\n\n")
end
defp maybe_add_params_section(sections, _lines, true), do: sections
defp maybe_add_params_section(sections, [], _has_section), do: sections
defp maybe_add_params_section(sections, lines, false) do
sections ++ ["## Parameters\n\n" <> Enum.join(lines, "\n")]
end
defp maybe_add_return_section(sections, _return_type, true), do: sections
defp maybe_add_return_section(sections, nil, _has_section), do: sections
defp maybe_add_return_section(sections, return_type, false) do
return_spec = type_spec_string(return_type)
sections ++ ["## Returns\n\n- `#{return_spec}`"]
end
defp param_doc_line(param) do
case sanitized_param_name(param) do
nil -> nil
name -> format_param_doc_line(name, param)
end
end
defp sanitized_param_name(param) do
case param_name(param) do
nil -> nil
name -> sanitize_name(name)
end
end
defp format_param_doc_line(name, param) do
type = param_type(param) |> type_spec_string()
kind_fragment = param_kind_fragment(param)
default_fragment = param_default_fragment(param)
"- `#{name}` (#{type}#{kind_fragment}#{default_fragment})"
end
defp param_kind_fragment(param) do
case param_kind(param) do
"KEYWORD_ONLY" -> keyword_only_fragment(param)
_ -> ""
end
end
defp keyword_only_fragment(param) do
if param_default?(param), do: " keyword-only", else: " keyword-only, required"
end
defp param_default_fragment(param) do
case param_default_value(param) do
nil -> ""
default -> " default: #{default}"
end
end
defp param_name(%{"name" => name}) when is_binary(name), do: name
defp param_name(%{name: name}) when is_binary(name), do: name
defp param_name(_), do: nil
defp param_default_value(%{"default" => default}), do: default
defp param_default_value(%{default: default}), do: default
defp param_default_value(_), do: nil
end