Packages
kreuzberg
4.3.5
4.10.2
4.10.1
4.10.0
4.9.9
4.9.7
4.9.5
4.9.4
4.9.3
4.9.2
4.9.1
4.8.6
4.8.5
4.8.4
4.8.3
4.8.2
4.8.1
4.8.0
4.7.4
4.7.3
4.7.2
4.7.1
4.7.0
4.6.3
4.6.2
4.6.1
4.6.0
4.5.4
4.5.3
4.5.2
4.5.1
4.4.6
4.4.5
4.4.4
4.4.3
4.4.2
4.4.1
4.4.0
4.3.8
4.3.7
4.3.6
4.3.5
4.3.4
4.3.3
4.3.2
4.3.0
4.2.15
4.2.14
4.2.13
4.2.12
4.2.11
4.2.10
4.2.9
4.2.8
4.2.7
4.2.6
4.2.5
4.2.4
4.2.3
4.2.2
4.2.1
4.2.0
4.1.2
4.1.1
4.1.0
4.0.8
4.0.7
4.0.6
4.0.4
4.0.3
4.0.2
4.0.1
4.0.0
4.0.0-rc.27
4.0.0-rc.26
High-performance document intelligence library with OCR support
Current section
Files
Jump to
Current section
Files
lib/kreuzberg/plugin/registry.ex
defmodule Kreuzberg.Plugin.Registry do
@moduledoc """
GenServer for managing Kreuzberg plugins.
This module provides a centralized registry for managing different types of plugins:
- Post-processors: Transform extracted text content
- Validators: Validate extraction configuration parameters
- OCR backends: Provide OCR functionality
The registry maintains plugin metadata including module references, configuration,
priorities, stages, and language support. All operations are thread-safe through
the GenServer interface.
## State Structure
The internal state is a map with four top-level keys:
- `:post_processors`: Maps processor name to `%{module: ..., config: ..., stage: ...}`
- `:validators`: Maps validator name to `%{module: ..., priority: ...}`
- `:sorted_validators`: Pre-sorted list of validators by priority (descending) for performance
- `:ocr_backends`: Maps backend name to `%{module: ..., languages: ...}`
## Usage
Typically used during application startup to register available plugins:
{:ok, _pid} = Kreuzberg.Plugin.Registry.start_link([])
Kreuzberg.Plugin.Registry.register_post_processor(MyPostProcessor, %{enabled: true}, :pre)
Kreuzberg.Plugin.Registry.register_validator(MyValidator, priority: 10)
Kreuzberg.Plugin.Registry.register_ocr_backend(MyOCRBackend, languages: ["en", "de"])
"""
use GenServer
require Logger
# Client API
@doc """
Start the registry GenServer.
Options are passed directly to GenServer.start_link/3.
## Examples
{:ok, pid} = Kreuzberg.Plugin.Registry.start_link([])
{:ok, pid} = Kreuzberg.Plugin.Registry.start_link(name: :plugin_registry)
"""
@spec start_link(keyword()) :: GenServer.on_start()
def start_link(opts \\ []) do
GenServer.start_link(__MODULE__, [], opts)
end
# Post-Processor API
@doc """
Register a post-processor plugin.
## Parameters
* `module` - The module implementing the post-processor behavior
* `config` - Configuration map for the post-processor (optional, defaults to %{})
* `stage` - Processing stage (atom), e.g., `:pre`, `:post`, `:cleanup` (optional, defaults to `:post`)
* `server` - GenServer name/pid (optional, defaults to default registry)
## Returns
* `:ok` on success
* `{:error, reason}` on failure
## Examples
Kreuzberg.Plugin.Registry.register_post_processor(MyProcessor)
Kreuzberg.Plugin.Registry.register_post_processor(MyProcessor, %{enabled: true}, :pre)
"""
@spec register_post_processor(
atom() | module(),
map() | module() | nil,
atom() | nil,
GenServer.server() | nil
) ::
:ok | {:error, String.t()}
def register_post_processor(
name_or_module,
config_or_module \\ nil,
stage \\ nil,
server \\ nil
) do
server = server || __MODULE__
{:ok, name, module, config, stage} =
parse_post_processor_params(name_or_module, config_or_module, stage)
with :ok <- validate_module(module) do
# Extract metadata from module if available
module_name =
if function_exported?(module, :name, 0),
do: module.name(),
else: module_to_name(module)
module_version =
if function_exported?(module, :version, 0), do: module.version(), else: "1.0.0"
module_stage =
if function_exported?(module, :processing_stage, 0),
do: module.processing_stage(),
else: stage
GenServer.call(server, {
:register_post_processor,
name,
%{
module: module,
config: config,
stage: module_stage,
name: module_name,
version: module_version
}
})
end
end
@doc """
Unregister a post-processor plugin by name.
## Parameters
* `name` - The name of the post-processor (atom or string)
* `server` - GenServer name/pid (optional)
## Returns
* `:ok`
## Examples
Kreuzberg.Plugin.Registry.unregister_post_processor(:my_processor)
"""
@spec unregister_post_processor(atom() | String.t(), GenServer.server() | nil) :: :ok
def unregister_post_processor(name, server \\ nil) do
server = server || __MODULE__
GenServer.call(server, {:unregister_post_processor, normalize_name(name)})
end
@doc """
List all registered post-processors.
Returns a map of post-processor names to their metadata.
## Parameters
* `server` - GenServer name/pid (optional)
## Returns
A map where keys are processor names and values are metadata maps containing:
- `:module` - The processor module
- `:config` - The processor configuration
- `:stage` - The processing stage
## Examples
processors = Kreuzberg.Plugin.Registry.list_post_processors()
IO.inspect(processors)
"""
@spec list_post_processors(GenServer.server() | nil) :: map()
def list_post_processors(server \\ nil) do
server = server || __MODULE__
GenServer.call(server, :list_post_processors)
end
@doc """
Clear all registered post-processors.
## Parameters
* `server` - GenServer name/pid (optional)
## Returns
* `:ok`
## Examples
Kreuzberg.Plugin.Registry.clear_post_processors()
"""
@spec clear_post_processors(GenServer.server() | nil) :: :ok
def clear_post_processors(server \\ nil) do
server = server || __MODULE__
GenServer.call(server, :clear_post_processors)
end
@doc """
Get post-processors for a specific processing stage.
## Parameters
* `stage` - The processing stage to filter by (atom)
* `server` - GenServer name/pid (optional)
## Returns
A map of post-processor names to metadata for the specified stage
## Examples
pre_processors = Kreuzberg.Plugin.Registry.get_post_processors_by_stage(:pre)
"""
@spec get_post_processors_by_stage(atom(), GenServer.server() | nil) :: map()
def get_post_processors_by_stage(stage, server \\ nil) do
server = server || __MODULE__
GenServer.call(server, {:get_post_processors_by_stage, stage})
end
@doc """
Get a specific post-processor by name.
## Parameters
* `name` - The post-processor name
* `server` - GenServer name/pid (optional)
## Returns
* `{:ok, metadata}` - Post-processor metadata
* `{:error, "Not found"}` - Post-processor not found
## Examples
{:ok, metadata} = Kreuzberg.Plugin.Registry.get_post_processor(:my_processor)
"""
@spec get_post_processor(atom() | String.t(), GenServer.server() | nil) ::
{:ok, map()} | {:error, String.t()}
def get_post_processor(name, server \\ nil) do
server = server || __MODULE__
GenServer.call(server, {:get_post_processor, normalize_name(name)})
end
# Validator API
@doc """
Register a validator plugin.
## Parameters
* `module` - The module implementing the validator behavior
* `opts` - Options keyword list with:
- `:priority` - Validation priority (integer, higher runs first, optional, defaults to 0)
* `server` - GenServer name/pid (optional)
## Returns
* `:ok` on success
* `{:error, reason}` on failure
## Examples
Kreuzberg.Plugin.Registry.register_validator(MyValidator)
Kreuzberg.Plugin.Registry.register_validator(MyValidator, priority: 10)
"""
@spec register_validator(module(), keyword() | nil, GenServer.server() | nil) ::
:ok | {:error, String.t()}
def register_validator(module, opts \\ nil, server \\ nil) do
server = server || __MODULE__
opts = opts || []
with :ok <- validate_module(module) do
# Extract metadata from module if available
module_name =
if function_exported?(module, :name, 0), do: module.name(), else: module_to_name(module)
module_version =
if function_exported?(module, :version, 0), do: module.version(), else: "1.0.0"
module_priority =
if function_exported?(module, :priority, 0),
do: module.priority(),
else: Keyword.get(opts, :priority, 0)
GenServer.call(server, {
:register_validator,
module_name,
%{module: module, priority: module_priority, name: module_name, version: module_version}
})
end
end
@doc """
Unregister a validator plugin by name.
## Parameters
* `name` - The name of the validator
* `server` - GenServer name/pid (optional)
## Returns
* `:ok`
## Examples
Kreuzberg.Plugin.Registry.unregister_validator(:my_validator)
"""
@spec unregister_validator(atom() | String.t(), GenServer.server() | nil) :: :ok
def unregister_validator(name, server \\ nil) do
server = server || __MODULE__
GenServer.call(server, {:unregister_validator, normalize_name(name)})
end
@doc """
List all registered validators.
Returns a map of validator names to their metadata, not sorted.
For sorted validators, use `get_validators_by_priority/1`.
## Parameters
* `server` - GenServer name/pid (optional)
## Returns
A map where keys are validator names and values are metadata maps containing:
- `:module` - The validator module
- `:priority` - The validation priority
## Examples
validators = Kreuzberg.Plugin.Registry.list_validators()
"""
@spec list_validators(GenServer.server() | nil) :: map()
def list_validators(server \\ nil) do
server = server || __MODULE__
GenServer.call(server, :list_validators)
end
@doc """
Clear all registered validators.
## Parameters
* `server` - GenServer name/pid (optional)
## Returns
* `:ok`
## Examples
Kreuzberg.Plugin.Registry.clear_validators()
"""
@spec clear_validators(GenServer.server() | nil) :: :ok
def clear_validators(server \\ nil) do
server = server || __MODULE__
GenServer.call(server, :clear_validators)
end
@doc """
Get validators sorted by priority (highest first).
This is the primary way to retrieve validators for execution.
The list is pre-calculated and cached in the state for performance.
## Parameters
* `server` - GenServer name/pid (optional)
## Returns
A list of `{name, metadata}` tuples sorted by priority descending
## Examples
validators = Kreuzberg.Plugin.Registry.get_validators_by_priority()
Enum.each(validators, fn {name, metadata} ->
apply(metadata.module, :validate, [...])
end)
"""
@spec get_validators_by_priority(GenServer.server() | nil) :: list({atom(), map()})
def get_validators_by_priority(server \\ nil) do
server = server || __MODULE__
GenServer.call(server, :get_validators_by_priority)
end
@doc """
Get a specific validator by name.
## Parameters
* `name` - The validator name
* `server` - GenServer name/pid (optional)
## Returns
* `{:ok, metadata}` - Validator metadata
* `{:error, "Not found"}` - Validator not found
## Examples
{:ok, metadata} = Kreuzberg.Plugin.Registry.get_validator(:my_validator)
"""
@spec get_validator(atom() | String.t(), GenServer.server() | nil) ::
{:ok, map()} | {:error, String.t()}
def get_validator(name, server \\ nil) do
server = server || __MODULE__
GenServer.call(server, {:get_validator, normalize_name(name)})
end
# OCR Backend API
@doc """
Register an OCR backend plugin.
## Parameters
* `module` - The module implementing the OCR backend behavior
* `opts` - Options keyword list with:
- `:languages` - Supported language codes (list of strings, optional, defaults to [])
* `server` - GenServer name/pid (optional)
## Returns
* `:ok` on success
* `{:error, reason}` on failure
## Examples
Kreuzberg.Plugin.Registry.register_ocr_backend(MyOCR)
Kreuzberg.Plugin.Registry.register_ocr_backend(MyOCR, languages: ["en", "de", "fr"])
"""
@spec register_ocr_backend(module(), keyword() | nil, GenServer.server() | nil) ::
:ok | {:error, String.t()}
def register_ocr_backend(module, opts \\ nil, server \\ nil) do
server = server || __MODULE__
opts = opts || []
with :ok <- validate_module(module) do
# Extract metadata from module if available
module_name =
if function_exported?(module, :name, 0), do: module.name(), else: module_to_name(module)
module_version =
if function_exported?(module, :version, 0), do: module.version(), else: "1.0.0"
module_languages =
if function_exported?(module, :supported_languages, 0),
do: module.supported_languages(),
else: Keyword.get(opts, :languages, [])
GenServer.call(server, {
:register_ocr_backend,
module_name,
%{module: module, languages: module_languages, name: module_name, version: module_version}
})
end
end
@doc """
Unregister an OCR backend plugin by name.
## Parameters
* `name` - The name of the OCR backend
* `server` - GenServer name/pid (optional)
## Returns
* `:ok`
## Examples
Kreuzberg.Plugin.Registry.unregister_ocr_backend(:my_ocr)
"""
@spec unregister_ocr_backend(atom() | String.t(), GenServer.server() | nil) :: :ok
def unregister_ocr_backend(name, server \\ nil) do
server = server || __MODULE__
GenServer.call(server, {:unregister_ocr_backend, normalize_name(name)})
end
@doc """
List all registered OCR backends.
Returns a map of OCR backend names to their metadata.
## Parameters
* `server` - GenServer name/pid (optional)
## Returns
A map where keys are backend names and values are metadata maps containing:
- `:module` - The OCR backend module
- `:languages` - List of supported language codes
## Examples
backends = Kreuzberg.Plugin.Registry.list_ocr_backends()
"""
@spec list_ocr_backends(GenServer.server() | nil) :: map()
def list_ocr_backends(server \\ nil) do
server = server || __MODULE__
GenServer.call(server, :list_ocr_backends)
end
@doc """
Clear all registered OCR backends.
## Parameters
* `server` - GenServer name/pid (optional)
## Returns
* `:ok`
## Examples
Kreuzberg.Plugin.Registry.clear_ocr_backends()
"""
@spec clear_ocr_backends(GenServer.server() | nil) :: :ok
def clear_ocr_backends(server \\ nil) do
server = server || __MODULE__
GenServer.call(server, :clear_ocr_backends)
end
@doc """
Get OCR backends that support a specific language.
Uses a single-pass reduce for optimal performance (OPTIMIZATION 2).
## Parameters
* `language` - The language code (string)
* `server` - GenServer name/pid (optional)
## Returns
A map of backend names to metadata for backends supporting the language
## Examples
en_backends = Kreuzberg.Plugin.Registry.get_ocr_backends_by_language("en")
"""
@spec get_ocr_backends_by_language(String.t(), GenServer.server() | nil) :: map()
def get_ocr_backends_by_language(language, server \\ nil) do
server = server || __MODULE__
GenServer.call(server, {:get_ocr_backends_by_language, language})
end
@doc """
Get a specific OCR backend by name.
## Parameters
* `name` - The OCR backend name
* `server` - GenServer name/pid (optional)
## Returns
* `{:ok, metadata}` - OCR backend metadata
* `{:error, "Not found"}` - OCR backend not found
## Examples
{:ok, metadata} = Kreuzberg.Plugin.Registry.get_ocr_backend(:my_ocr)
"""
@spec get_ocr_backend(atom() | String.t(), GenServer.server() | nil) ::
{:ok, map()} | {:error, String.t()}
def get_ocr_backend(name, server \\ nil) do
server = server || __MODULE__
GenServer.call(server, {:get_ocr_backend, normalize_name(name)})
end
# GenServer Callbacks
@doc false
@impl GenServer
def init(_opts) do
state = %{
post_processors: %{},
validators: %{},
sorted_validators: [],
ocr_backends: %{}
}
Logger.debug("Initialized Kreuzberg.Plugin.Registry with empty state")
{:ok, state}
end
@doc false
@impl GenServer
def handle_call(
{:register_post_processor, name, metadata},
_from,
state
) do
if Map.has_key?(state.post_processors, name) do
{:reply, {:error, "Post-processor '#{inspect(name)}' is already registered"}, state}
else
new_state = put_in(state, [:post_processors, name], metadata)
Logger.debug("Registered post-processor: #{inspect(name)}")
{:reply, :ok, new_state}
end
end
def handle_call(
{:unregister_post_processor, name},
_from,
state
) do
new_state = update_in(state, [:post_processors], &Map.delete(&1, name))
Logger.debug("Unregistered post-processor: #{inspect(name)}")
{:reply, :ok, new_state}
end
def handle_call(:list_post_processors, _from, state) do
{:reply, state.post_processors, state}
end
def handle_call(:clear_post_processors, _from, state) do
new_state = %{state | post_processors: %{}}
Logger.debug("Cleared all post-processors")
{:reply, :ok, new_state}
end
def handle_call({:get_post_processors_by_stage, stage}, _from, state) do
# OPTIMIZATION 2: Single-pass reduce instead of filter+into (15-20% gain)
processors =
Enum.reduce(state.post_processors, %{}, fn
{name, metadata}, acc when metadata.stage == stage ->
Map.put(acc, name, metadata)
_, acc ->
acc
end)
{:reply, processors, state}
end
def handle_call({:get_post_processor, name}, _from, state) do
case Map.fetch(state.post_processors, name) do
{:ok, metadata} -> {:reply, {:ok, metadata}, state}
:error -> {:reply, {:error, "Post-processor not found: #{name}"}, state}
end
end
def handle_call(
{:register_validator, name, metadata},
_from,
state
) do
if Map.has_key?(state.validators, name) do
{:reply, {:error, "Validator '#{inspect(name)}' is already registered"}, state}
else
# OPTIMIZATION 1: Cache sorted validators on registration (30%+ gain if accessed frequently)
new_validators = Map.put(state.validators, name, metadata)
sorted = Enum.sort_by(new_validators, fn {_n, m} -> m.priority end, :desc)
new_state = %{state | validators: new_validators, sorted_validators: sorted}
Logger.debug("Registered validator: #{inspect(name)}")
{:reply, :ok, new_state}
end
end
def handle_call(
{:unregister_validator, name},
_from,
state
) do
# OPTIMIZATION 1: Update sorted cache on unregistration
new_validators = Map.delete(state.validators, name)
sorted = Enum.sort_by(new_validators, fn {_n, m} -> m.priority end, :desc)
new_state = %{state | validators: new_validators, sorted_validators: sorted}
Logger.debug("Unregistered validator: #{inspect(name)}")
{:reply, :ok, new_state}
end
def handle_call(:list_validators, _from, state) do
{:reply, state.validators, state}
end
def handle_call(:clear_validators, _from, state) do
# OPTIMIZATION 1: Clear sorted cache as well
new_state = %{state | validators: %{}, sorted_validators: []}
Logger.debug("Cleared all validators")
{:reply, :ok, new_state}
end
def handle_call(:get_validators_by_priority, _from, state) do
# OPTIMIZATION 1: Return pre-cached sorted list (30%+ gain)
{:reply, state.sorted_validators, state}
end
def handle_call({:get_validator, name}, _from, state) do
case Map.fetch(state.validators, name) do
{:ok, metadata} -> {:reply, {:ok, metadata}, state}
:error -> {:reply, {:error, "Validator not found: #{name}"}, state}
end
end
def handle_call(
{:register_ocr_backend, name, metadata},
_from,
state
) do
if Map.has_key?(state.ocr_backends, name) do
{:reply, {:error, "OCR backend '#{inspect(name)}' is already registered"}, state}
else
new_state = put_in(state, [:ocr_backends, name], metadata)
Logger.debug("Registered OCR backend: #{inspect(name)}")
{:reply, :ok, new_state}
end
end
def handle_call(
{:unregister_ocr_backend, name},
_from,
state
) do
new_state = update_in(state, [:ocr_backends], &Map.delete(&1, name))
Logger.debug("Unregistered OCR backend: #{inspect(name)}")
{:reply, :ok, new_state}
end
def handle_call(:list_ocr_backends, _from, state) do
{:reply, state.ocr_backends, state}
end
def handle_call(:clear_ocr_backends, _from, state) do
new_state = %{state | ocr_backends: %{}}
Logger.debug("Cleared all OCR backends")
{:reply, :ok, new_state}
end
def handle_call({:get_ocr_backends_by_language, language}, _from, state) do
# OPTIMIZATION 2: Single-pass reduce instead of filter+into (15-20% gain)
backends =
Enum.reduce(state.ocr_backends, %{}, fn {name, metadata}, acc ->
if Enum.member?(metadata.languages, language) do
Map.put(acc, name, metadata)
else
acc
end
end)
{:reply, backends, state}
end
def handle_call({:get_ocr_backend, name}, _from, state) do
case Map.fetch(state.ocr_backends, name) do
{:ok, metadata} -> {:reply, {:ok, metadata}, state}
:error -> {:reply, {:error, "OCR backend not found: #{name}"}, state}
end
end
# Private Helpers
@spec module_to_name(module()) :: String.t()
@doc false
def module_to_name(module) do
# Use the module's name() function if available, otherwise derive from module name
if function_exported?(module, :name, 0) do
module.name()
else
module
|> inspect()
|> String.split(".")
|> List.last()
|> String.downcase()
end
end
@spec normalize_name(atom() | String.t()) :: atom() | String.t()
defp normalize_name(name) when is_binary(name), do: name
defp normalize_name(name) when is_atom(name), do: name
defp normalize_name(name), do: to_string(name)
@spec parse_post_processor_params(
atom() | module(),
map() | module() | nil,
atom() | nil
) :: {:ok, atom() | String.t(), atom(), map(), atom()} | {:error, String.t()}
defp parse_post_processor_params(name_or_module, config_or_module, stage) do
if is_atom(name_or_module) and is_atom(config_or_module) and not is_nil(config_or_module) and
stage == nil do
# Called as: register_post_processor(name, module)
{:ok, name_or_module, config_or_module, %{}, :post}
else
# Called as: register_post_processor(module, config, stage, ...)
module = name_or_module
config = config_or_module || %{}
stage = stage || :post
name = module_to_name(module)
{:ok, name, module, config, stage}
end
end
@spec validate_module(atom()) :: :ok | {:error, String.t()}
defp validate_module(module) do
if is_atom(module) and Code.ensure_loaded?(module) do
:ok
else
{:error, "Invalid module: #{inspect(module)}"}
end
end
end