Packages
snakepit
0.8.5
0.13.0
0.12.0
0.11.1
0.11.0
0.10.1
0.10.0
0.9.1
0.9.0
0.8.9
0.8.8
0.8.7
0.8.6
0.8.5
0.8.4
0.8.3
0.8.2
0.8.1
0.8.0
0.7.7
0.7.6
0.7.5
0.7.4
0.7.3
0.7.2
0.7.1
0.7.0
0.6.11
0.6.10
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.2
0.6.1
0.6.0
0.5.1
0.5.0
0.4.3
0.4.2
0.4.1
0.4.0
0.3.3
0.3.2
0.3.1
0.3.0
0.2.1
0.2.0
0.1.2
0.1.1
0.1.0
High-performance pooler and session manager for external language integrations. Supports Python, Node.js, Ruby, and more with gRPC streaming, session management, and production-ready process cleanup.
Current section
Files
Jump to
Current section
Files
lib/snakepit/compatibility.ex
defmodule Snakepit.Compatibility do
@moduledoc """
Thread-safety compatibility matrix for common Python libraries.
"""
@type thread_safety :: true | false | :conditional
@type library_info :: %{
thread_safe: thread_safety(),
notes: String.t()
}
@libraries %{
"numpy" => %{thread_safe: true, notes: "Releases GIL during computation"},
"scipy" => %{thread_safe: true, notes: "Releases GIL for numerical ops"},
"torch" => %{thread_safe: true, notes: "Configure with torch.set_num_threads/1"},
"tensorflow" => %{thread_safe: true, notes: "Use tf.config.threading APIs"},
"scikit-learn" => %{thread_safe: :conditional, notes: "Set n_jobs=1 per estimator"},
"polars" => %{thread_safe: true, notes: "Thread-safe DataFrame library"},
"requests" => %{thread_safe: true, notes: "Use separate Session per thread"},
"httpx" => %{thread_safe: true, notes: "Async-first; thread-safe clients"},
"aiohttp" => %{thread_safe: :conditional, notes: "One ClientSession per thread"},
"grpcio" => %{thread_safe: true, notes: "Thread-safe client with shared channels"},
"numexpr" => %{thread_safe: true, notes: "Releases GIL for expression evaluation"},
"onnxruntime" => %{thread_safe: true, notes: "Thread-safe inference sessions"},
"jax" => %{thread_safe: :conditional, notes: "Avoid shared mutable state"},
"opencv" => %{thread_safe: :conditional, notes: "Avoid shared mutable state"},
"pillow" => %{thread_safe: :conditional, notes: "Use separate Image objects"},
"pandas" => %{thread_safe: false, notes: "Not thread-safe; lock DataFrame ops"},
"matplotlib" => %{thread_safe: false, notes: "Global state; prefer process mode"},
"sqlite3" => %{thread_safe: false, notes: "Use separate connections per thread"},
"sqlalchemy" => %{thread_safe: :conditional, notes: "Use per-thread sessions"},
"spacy" => %{thread_safe: false, notes: "Models share global state"},
"fasttext" => %{thread_safe: false, notes: "Model objects are not thread-safe"},
"xgboost" => %{thread_safe: :conditional, notes: "Set num_threads and avoid shared state"},
"lightgbm" => %{thread_safe: :conditional, notes: "Set num_threads and avoid shared state"},
"cupy" => %{thread_safe: :conditional, notes: "Manage CUDA context per thread"},
"faiss" => %{thread_safe: :conditional, notes: "Avoid shared index mutation"},
"ray" => %{thread_safe: false, notes: "Prefer process-based workers"},
"celery" => %{thread_safe: false, notes: "Use process-based workers"}
}
@spec check(String.t() | atom(), :thread | :process) ::
{:ok, String.t()} | {:warning, String.t()} | {:error, String.t()}
def check(library, profile) when profile in [:thread, :process] do
name = normalize_name(library)
case Map.get(@libraries, name) do
nil ->
{:warning, "Unknown library: #{name}"}
_info when profile == :process ->
{:ok, "Process profile isolates workers"}
%{thread_safe: true} ->
{:ok, "Thread-safe"}
%{thread_safe: false, notes: notes} ->
{:error, "Not thread-safe: #{notes}"}
%{thread_safe: :conditional, notes: notes} ->
{:warning, "Conditionally thread-safe: #{notes}"}
end
end
def check(_library, _profile), do: {:error, "Unknown profile"}
@spec get_library_info(String.t() | atom()) :: library_info() | nil
def get_library_info(library) do
name = normalize_name(library)
Map.get(@libraries, name)
end
@spec list_all(:thread_safe | :thread_unsafe | :conditional | :all) :: [String.t()]
def list_all(:thread_safe) do
select_names(fn {_name, info} -> info.thread_safe == true end)
end
def list_all(:thread_unsafe) do
select_names(fn {_name, info} -> info.thread_safe == false end)
end
def list_all(:conditional) do
select_names(fn {_name, info} -> info.thread_safe == :conditional end)
end
def list_all(:all), do: Map.keys(@libraries)
def list_all(_), do: []
@spec generate_report([String.t() | atom()], :thread | :process) ::
{:ok, map()} | {:error, term()}
def generate_report(libraries, profile) when is_list(libraries) do
report =
Enum.reduce(libraries, %{safe: [], unsafe: [], conditional: [], unknown: []}, fn library,
acc ->
name = normalize_name(library)
case Map.get(@libraries, name) do
nil ->
Map.update!(acc, :unknown, &[name | &1])
%{thread_safe: true} ->
Map.update!(acc, :safe, &[name | &1])
%{thread_safe: false} ->
Map.update!(acc, :unsafe, &[name | &1])
%{thread_safe: :conditional} ->
Map.update!(acc, :conditional, &[name | &1])
end
end)
|> Map.put(:profile, profile)
|> normalize_report()
{:ok, report}
end
def generate_report(_libraries, _profile), do: {:error, :invalid_libraries}
defp normalize_name(library) when is_atom(library),
do: library |> Atom.to_string() |> normalize_name()
defp normalize_name(library) when is_binary(library) do
library
|> String.trim()
|> String.downcase()
|> String.replace("_", "-")
end
defp select_names(fun) do
@libraries
|> Enum.filter(fun)
|> Enum.map(fn {name, _info} -> name end)
end
defp normalize_report(report) do
report
|> Map.update!(:safe, &Enum.reverse/1)
|> Map.update!(:unsafe, &Enum.reverse/1)
|> Map.update!(:conditional, &Enum.reverse/1)
|> Map.update!(:unknown, &Enum.reverse/1)
end
end