Current section

Files

Jump to
snakebridge lib mix tasks compile snakebridge.ex
Raw

lib/mix/tasks/compile/snakebridge.ex

defmodule Mix.Tasks.Compile.Snakebridge do
@moduledoc """
Mix compiler that runs the SnakeBridge pre-pass (scan → introspect → generate).
"""
use Mix.Task.Compiler
alias SnakeBridge.{
Config,
Generator,
HelperGenerator,
Helpers,
Introspector,
Lock,
Manifest,
PythonEnv,
Scanner,
Telemetry
}
@impl Mix.Task.Compiler
def run(_args) do
Mix.Task.run("loadconfig")
if skip_generation?() do
{:ok, []}
else
config = Config.load()
run_with_config(config)
end
end
defp run_with_config(%{libraries: []}), do: {:ok, []}
defp run_with_config(config) do
if strict_mode?(config) do
run_strict(config)
else
PythonEnv.ensure!(config)
run_normal(config)
end
end
@impl Mix.Task.Compiler
def manifests do
Mix.Task.run("loadconfig")
config = Config.load()
[
Path.join(config.metadata_dir, "manifest.json"),
"snakebridge.lock"
]
end
defp skip_generation? do
case System.get_env("SNAKEBRIDGE_SKIP") do
nil -> false
value -> value in ["1", "true", "TRUE", "yes", "YES"]
end
end
defp update_manifest(manifest, targets) do
targets
|> Introspector.introspect_batch()
|> Enum.reduce(manifest, fn {library, result, python_module}, acc ->
case result do
{:ok, infos} ->
{symbol_entries, class_entries} =
build_manifest_entries(library, python_module, infos)
acc
|> Manifest.put_symbols(symbol_entries)
|> Manifest.put_classes(class_entries)
{:error, _reason} ->
acc
end
end)
end
defp build_targets(missing, config, manifest) do
initial =
Enum.reduce(missing, %{}, fn entry, acc ->
accumulate_missing_target(entry, acc, config)
end)
with_includes =
Enum.reduce(config.libraries, initial, fn library, acc ->
includes =
library.include
|> Enum.reject(&function_or_class_present?(manifest, library, &1))
accumulate_includes(includes, library, acc)
end)
with_includes
|> Enum.map(fn {{library, python_module}, functions} ->
filtered = Enum.reject(functions, &(&1 in library.exclude))
{library, python_module, Enum.uniq(filtered)}
end)
|> Enum.reject(fn {_library, _python_module, functions} -> functions == [] end)
end
defp accumulate_missing_target({module, function, _arity}, acc, config) do
case library_for_module(config, module) do
nil ->
acc
library ->
python_module = python_module_for_elixir(library, module)
add_target(acc, library, python_module, function)
end
end
defp accumulate_includes([], _library, acc), do: acc
defp accumulate_includes(includes, library, acc) do
key = {library, library.python_name}
Map.update(acc, key, includes, fn funcs -> includes ++ funcs end)
end
defp add_target(acc, library, python_module, function) do
key = {library, python_module}
Map.update(acc, key, [function], fn funcs -> [function | funcs] end)
end
defp library_for_module(config, module) do
module_parts = Module.split(module)
Enum.find(config.libraries, fn library ->
library_parts = Module.split(library.module_name)
Enum.take(module_parts, length(library_parts)) == library_parts
end)
end
defp build_manifest_entries(library, python_module, infos) do
Enum.reduce(infos, {[], []}, fn info, {symbols, classes} ->
cond do
info["error"] ->
{symbols, classes}
info["type"] == "class" ->
class_entries = class_entries_for(library, python_module, info)
{symbols, class_entries ++ classes}
true ->
symbol_entry = symbol_entry_for(library, python_module, info)
{[symbol_entry | symbols], classes}
end
end)
end
defp symbol_entry_for(library, python_module, info) do
module = module_for_python(library, python_module)
function = info["name"]
arity = required_arity(info["parameters"] || [])
key = Manifest.symbol_key({module, String.to_atom(function), arity})
{
key,
%{
"module" => Module.split(module) |> Enum.join("."),
"function" => function,
"name" => function,
"python_module" => python_module,
"parameters" => info["parameters"] || [],
"docstring" => info["docstring"] || "",
"return_annotation" => info["return_annotation"],
"return_type" => info["return_type"]
}
}
end
defp class_entries_for(library, python_module, info) do
class_name = info["name"] || info["class"] || "Class"
class_python_module = info["python_module"] || python_module || library.python_name
class_module = class_module_for(library, class_python_module, class_name)
key = Manifest.class_key(class_module)
[
{
key,
%{
"module" => Module.split(class_module) |> Enum.join("."),
"class" => class_name,
"python_module" => class_python_module,
"docstring" => info["docstring"] || "",
"methods" => info["methods"] || [],
"attributes" => info["attributes"] || []
}
}
]
end
defp class_module_for(library, python_module, class_name) do
python_parts = String.split(python_module, ".")
library_parts = String.split(library.python_name, ".")
extra_parts = Enum.drop(python_parts, length(library_parts))
extra_parts = drop_class_suffix(extra_parts, class_name)
library.module_name
|> Module.split()
|> Kernel.++(Enum.map(extra_parts, &Macro.camelize/1))
|> Kernel.++([class_name])
|> Module.concat()
end
defp drop_class_suffix(parts, class_name) when is_list(parts) and is_binary(class_name) do
class_suffix = Macro.underscore(class_name)
case List.last(parts) do
^class_suffix -> Enum.drop(parts, -1)
_ -> parts
end
end
defp drop_class_suffix(parts, _class_name), do: parts
defp module_for_python(library, python_module) do
python_parts = String.split(python_module, ".")
library_parts = String.split(library.python_name, ".")
extra_parts = Enum.drop(python_parts, length(library_parts))
library.module_name
|> Module.split()
|> Kernel.++(Enum.map(extra_parts, &Macro.camelize/1))
|> Module.concat()
end
defp python_module_for_elixir(library, module) do
module_parts = Module.split(module)
library_parts = Module.split(library.module_name)
extra_parts = Enum.drop(module_parts, length(library_parts))
case Enum.map(extra_parts, &Macro.underscore/1) do
[] -> library.python_name
parts -> library.python_name <> "." <> Enum.join(parts, ".")
end
end
defp function_or_class_present?(manifest, library, name) do
module_prefix = Module.split(library.module_name) |> Enum.join(".")
function_prefix = "#{module_prefix}.#{name}/"
function_exists? =
manifest
|> Map.get("symbols", %{})
|> Map.keys()
|> Enum.any?(&String.starts_with?(&1, function_prefix))
class_exists? =
manifest
|> Map.get("classes", %{})
|> Map.values()
|> Enum.any?(fn info ->
info["class"] == name or String.ends_with?(info["module"] || "", ".#{name}")
end)
function_exists? or class_exists?
end
defp generate_from_manifest(config, manifest) do
Enum.each(config.libraries, fn library ->
functions = functions_for_library(manifest, library)
classes = classes_for_library(manifest, library)
Generator.generate_library(library, functions, classes, config)
end)
end
defp functions_for_library(manifest, library) do
Map.get(manifest, "symbols", %{})
|> Map.values()
|> Enum.filter(fn info ->
python_module = info["python_module"] || ""
String.starts_with?(python_module, library.python_name)
end)
end
defp classes_for_library(manifest, library) do
Map.get(manifest, "classes", %{})
|> Map.values()
|> Enum.filter(fn info ->
python_module = info["python_module"] || ""
String.starts_with?(python_module, library.python_name)
end)
end
defp strict_mode?(config) do
System.get_env("SNAKEBRIDGE_STRICT") == "1" || config.strict == true
end
defp run_strict(config) do
manifest = Manifest.load(config)
detected = scanner_module().scan_project(config)
missing = Manifest.missing(manifest, detected)
if missing != [] do
formatted = format_missing(missing)
raise SnakeBridge.CompileError, """
Strict mode: #{length(missing)} symbol(s) not in manifest.
Missing:
#{formatted}
To fix:
1. Run `mix snakebridge.setup` locally
2. Run `mix compile` to generate bindings
3. Commit the updated manifest and generated files
4. Re-run CI
Set SNAKEBRIDGE_STRICT=0 to disable strict mode.
"""
end
verify_generated_files_exist!(config)
verify_symbols_present!(config, manifest)
{:ok, []}
end
defp run_normal(config) do
start_time = System.monotonic_time()
libraries = Enum.map(config.libraries, & &1.name)
Telemetry.compile_start(libraries, false)
try do
detected = scanner_module().scan_project(config)
manifest = Manifest.load(config)
missing = Manifest.missing(manifest, detected)
targets = build_targets(missing, config, manifest)
updated_manifest =
if targets != [] do
update_manifest(manifest, targets)
else
manifest
end
Manifest.save(config, updated_manifest)
generate_from_manifest(config, updated_manifest)
generate_helper_wrappers(config)
Lock.update(config)
symbol_count = count_symbols(updated_manifest)
file_count = length(config.libraries)
Telemetry.compile_stop(start_time, symbol_count, file_count, libraries, :normal)
{:ok, []}
rescue
e ->
Telemetry.compile_exception(start_time, e, __STACKTRACE__)
reraise e, __STACKTRACE__
end
end
@spec verify_generated_files_exist!(Config.t()) :: :ok
def verify_generated_files_exist!(config) do
Enum.each(config.libraries, fn library ->
path = Path.join(config.generated_dir, "#{library.python_name}.ex")
unless File.exists?(path) do
raise SnakeBridge.CompileError, """
Strict mode: Generated file missing: #{path}
Run `mix compile` locally and commit the generated files.
"""
end
end)
:ok
end
@spec verify_symbols_present!(Config.t(), map()) :: :ok
def verify_symbols_present!(config, manifest) do
Enum.each(config.libraries, fn library ->
path = Path.join(config.generated_dir, "#{library.python_name}.ex")
content = read_generated_file!(path)
defs = parse_definitions!(content, path)
missing_functions = missing_functions_for_library(manifest, library, defs)
classes_for_library = classes_for_library(manifest, library)
missing_classes = missing_classes_for_library(classes_for_library, defs)
missing_class_members = missing_class_members_for_library(classes_for_library, defs)
maybe_raise_missing!(path, missing_functions, missing_classes, missing_class_members)
end)
:ok
end
defp missing_functions_for_library(manifest, library, defs) do
manifest
|> functions_for_library(library)
|> Enum.reject(fn info ->
function_defined?(defs, info["module"], info["name"])
end)
end
defp missing_classes_for_library(classes_for_library, defs) do
Enum.reject(classes_for_library, fn info ->
module = info["module"]
Map.has_key?(defs, module)
end)
end
defp missing_class_members_for_library(classes_for_library, defs) do
classes_for_library
|> Enum.filter(fn info -> Map.has_key?(defs, info["module"]) end)
|> Enum.flat_map(&missing_members_for_class(&1, defs))
end
defp missing_members_for_class(info, defs) do
module = info["module"]
method_names = missing_method_names(info["methods"] || [], defs, module)
attr_names = missing_attr_names(info["attributes"] || [], defs, module)
Enum.map(method_names ++ attr_names, fn name ->
{module, name}
end)
end
defp missing_method_names(methods, defs, module) do
methods
|> Enum.map(&method_expected_name/1)
|> Enum.reject(fn name ->
name == "" or function_defined?(defs, module, name)
end)
end
defp missing_attr_names(attrs, defs, module) do
attrs
|> Enum.map(&to_string/1)
|> Enum.reject(fn name ->
name == "" or function_defined?(defs, module, name)
end)
end
defp maybe_raise_missing!(path, missing_functions, missing_classes, missing_class_members) do
if missing_functions != [] or missing_classes != [] or missing_class_members != [] do
raise SnakeBridge.CompileError, """
Strict mode: Generated file #{path} is missing expected bindings.
#{missing_functions_message(missing_functions)}\
#{missing_classes_message(missing_classes)}\
#{missing_class_members_message(missing_class_members)}
Run `mix compile` locally to regenerate and commit the updated files.
"""
end
end
defp required_arity(params) do
params
|> Enum.filter(&(&1["kind"] in ["POSITIONAL_ONLY", "POSITIONAL_OR_KEYWORD"]))
|> Enum.reject(&Map.has_key?(&1, "default"))
|> length()
end
defp format_missing(missing) do
missing
|> Enum.sort()
|> Enum.map_join("\n", fn {mod, fun, arity} ->
module = Module.split(mod) |> Enum.join(".")
" - #{module}.#{fun}/#{arity}"
end)
end
defp scanner_module do
Application.get_env(:snakebridge, :scanner, Scanner)
end
defp read_generated_file!(path) do
case File.read(path) do
{:ok, content} ->
content
{:error, reason} ->
raise SnakeBridge.CompileError, """
Strict mode: Cannot read generated file #{path}: #{inspect(reason)}
"""
end
end
defp parse_definitions!(content, path) do
case Code.string_to_quoted(content, file: path) do
{:ok, ast} ->
collect_definitions(ast)
{:error, {line, error, token}} ->
raise SnakeBridge.CompileError, """
Strict mode: Cannot parse generated file #{path}: #{error} #{inspect(token)} on line #{line}
"""
end
end
defp collect_definitions(ast) do
initial = %{stack: [], defs: %{}}
{_, acc} =
Macro.traverse(ast, initial, &collect_pre/2, &collect_post/2)
acc.defs
end
defp collect_pre({:defmodule, _, [{:__aliases__, _, parts}, _]} = node, acc) do
segments = module_segments(parts, acc.stack)
module_name = Enum.join(segments, ".")
acc =
acc
|> Map.update!(:stack, &[segments | &1])
|> Map.update(:defs, %{}, &Map.put_new(&1, module_name, MapSet.new()))
{node, acc}
end
defp collect_pre({:def, _, [head | _]} = node, acc) do
{node, track_def(acc, head)}
end
defp collect_pre(node, acc), do: {node, acc}
defp track_def(acc, head) do
case {def_name(head), List.first(acc.stack)} do
{nil, _} ->
acc
{_, nil} ->
acc
{name, current_module} ->
module_name = Enum.join(current_module, ".")
Map.update(acc, :defs, %{}, fn defs ->
Map.update(defs, module_name, MapSet.new([name]), &MapSet.put(&1, name))
end)
end
end
defp collect_post({:defmodule, _, _} = node, acc) do
{_popped, rest} = List.pop_at(acc.stack, 0)
{node, %{acc | stack: rest}}
end
defp collect_post(node, acc), do: {node, acc}
defp module_segments(parts, []) do
Enum.map(parts, &Atom.to_string/1)
end
defp module_segments(parts, [parent | _]) do
parent ++ Enum.map(parts, &Atom.to_string/1)
end
defp def_name({:when, _, [inner | _]}), do: def_name(inner)
defp def_name({name, _, _}) when is_atom(name), do: Atom.to_string(name)
defp def_name(_), do: nil
defp function_defined?(defs, module, name) when is_binary(module) and is_binary(name) do
case Map.get(defs, module) do
nil -> false
set -> MapSet.member?(set, name)
end
end
defp method_expected_name(%{"name" => "__init__"}), do: "new"
defp method_expected_name(%{"name" => name}) when is_binary(name), do: name
defp method_expected_name(%{name: "__init__"}), do: "new"
defp method_expected_name(%{name: name}) when is_binary(name), do: name
defp method_expected_name(_), do: ""
defp missing_functions_message([]), do: ""
defp missing_functions_message(missing) do
formatted =
missing
|> Enum.map_join("\n", fn info ->
module = info["module"] || "Unknown"
name = info["name"] || "unknown"
" - #{module}.#{name}"
end)
"""
Missing functions:
#{formatted}
"""
end
defp missing_classes_message([]), do: ""
defp missing_classes_message(missing) do
formatted =
missing
|> Enum.map_join("\n", fn info ->
info["module"] || "Unknown"
end)
"""
Missing classes:
#{formatted}
"""
end
defp missing_class_members_message([]), do: ""
defp missing_class_members_message(missing) do
formatted =
missing
|> Enum.map_join("\n", fn {module, name} ->
" - #{module}.#{name}"
end)
"""
Missing class members:
#{formatted}
"""
end
defp count_symbols(manifest) do
symbols = Map.get(manifest, "symbols", %{}) |> map_size()
classes = Map.get(manifest, "classes", %{}) |> map_size()
symbols + classes
end
defp generate_helper_wrappers(config) do
if Helpers.enabled?(config) do
case Helpers.discover(config) do
{:ok, helpers} ->
HelperGenerator.generate_helpers(helpers, config)
{:error, %SnakeBridge.HelperRegistryError{} = error} ->
Mix.shell().error(Exception.message(error))
:ok
{:error, reason} ->
Mix.shell().error("Helper registry failed: #{inspect(reason)}")
:ok
end
else
:ok
end
end
end