Packages
kreuzberg
4.10.2
4.10.2
4.10.1
4.10.0
4.9.9
4.9.7
4.9.5
4.9.4
4.9.3
4.9.2
4.9.1
4.8.6
4.8.5
4.8.4
4.8.3
4.8.2
4.8.1
4.8.0
4.7.4
4.7.3
4.7.2
4.7.1
4.7.0
4.6.3
4.6.2
4.6.1
4.6.0
4.5.4
4.5.3
4.5.2
4.5.1
4.4.6
4.4.5
4.4.4
4.4.3
4.4.2
4.4.1
4.4.0
4.3.8
4.3.7
4.3.6
4.3.5
4.3.4
4.3.3
4.3.2
4.3.0
4.2.15
4.2.14
4.2.13
4.2.12
4.2.11
4.2.10
4.2.9
4.2.8
4.2.7
4.2.6
4.2.5
4.2.4
4.2.3
4.2.2
4.2.1
4.2.0
4.1.2
4.1.1
4.1.0
4.0.8
4.0.7
4.0.6
4.0.4
4.0.3
4.0.2
4.0.1
4.0.0
4.0.0-rc.27
4.0.0-rc.26
High-performance document intelligence library with OCR support
Current section
Files
Jump to
Current section
Files
lib/kreuzberg/batch_api.ex
defmodule Kreuzberg.BatchAPI do
@moduledoc """
Batch extraction operations for processing multiple documents efficiently.
This module provides functions for extracting content from multiple files or
binary inputs in batch operations, which can be more efficient than processing
files individually when dealing with large numbers of documents.
"""
alias Kreuzberg.{Error, ExtractionConfig, ExtractionResult, Helpers, Native}
@doc """
Extract content from multiple files in a batch operation.
## Parameters
* `paths` - List of file paths (strings or Path.t())
* `mime_type` - MIME type for all files (optional, defaults to nil for auto-detection)
* `config` - ExtractionConfig struct or map with extraction options (optional)
## Returns
* `{:ok, results}` - List of ExtractionResult structs
* `{:error, reason}` - Error message if batch extraction fails
## Examples
# Extract multiple PDFs
paths = ["doc1.pdf", "doc2.pdf", "doc3.pdf"]
{:ok, results} = Kreuzberg.BatchAPI.batch_extract_files(paths, "application/pdf")
# Extract with config
config = %Kreuzberg.ExtractionConfig{images: %{"enabled" => true}}
{:ok, results} = Kreuzberg.BatchAPI.batch_extract_files(paths, "application/pdf", config)
# Auto-detect MIME types
{:ok, results} = Kreuzberg.BatchAPI.batch_extract_files(paths)
"""
@spec batch_extract_files(
[String.t() | Path.t()],
String.t() | nil,
ExtractionConfig.t() | map() | keyword() | nil
) :: {:ok, [ExtractionResult.t()]} | {:error, String.t()}
def batch_extract_files(paths, config_or_mime, third_arg \\ nil)
def batch_extract_files(paths, config, third_arg)
when is_list(paths) and
(is_map(config) or is_list(config)) and
not is_binary(config) and
is_nil(third_arg) do
batch_extract_files(paths, nil, config)
end
def batch_extract_files(paths, mime_type, config)
when is_list(paths) and (is_nil(mime_type) or is_binary(mime_type)) do
string_paths = Enum.map(paths, &to_string/1)
case call_native_batch_files(string_paths, mime_type, config) do
{:ok, results_list} when is_list(results_list) ->
process_batch_results(results_list, string_paths, "file")
{:error, _reason} = err ->
err
end
end
@doc """
Extract content from multiple files, raising on error.
Same as `batch_extract_files/3` but raises a `Kreuzberg.Error` exception if extraction fails.
## Examples
paths = ["doc1.pdf", "doc2.pdf", "doc3.pdf"]
results = Kreuzberg.BatchAPI.batch_extract_files!(paths, "application/pdf")
"""
@spec batch_extract_files!(
[String.t() | Path.t()],
String.t() | nil,
ExtractionConfig.t() | map() | keyword() | nil
) :: [ExtractionResult.t()]
def batch_extract_files!(paths, config_or_mime, third_arg \\ nil)
def batch_extract_files!(paths, config, third_arg)
when is_list(paths) and
(is_map(config) or is_list(config)) and
not is_binary(config) and
is_nil(third_arg) do
batch_extract_files!(paths, nil, config)
end
def batch_extract_files!(paths, mime_type, config) do
case batch_extract_files(paths, mime_type, config) do
{:ok, results} ->
results
{:error, reason} ->
raise Error, message: reason, reason: Kreuzberg.UtilityAPI.classify_error(reason)
end
end
@doc """
Extract content from multiple binary inputs in a batch operation.
## Parameters
* `data_list` - List of binary data inputs
* `mime_types` - List of MIME types (one per input) or single MIME type for all
* `config` - ExtractionConfig struct or map with extraction options (optional)
## Returns
* `{:ok, results}` - List of ExtractionResult structs
* `{:error, reason}` - Error message if batch extraction fails
## Examples
# Extract multiple PDFs from binary data
data_list = [pdf_binary1, pdf_binary2, pdf_binary3]
mime_types = ["application/pdf", "application/pdf", "application/pdf"]
{:ok, results} = Kreuzberg.BatchAPI.batch_extract_bytes(data_list, mime_types)
# Use single MIME type for all inputs
{:ok, results} = Kreuzberg.BatchAPI.batch_extract_bytes(data_list, "application/pdf")
# With config
config = %Kreuzberg.ExtractionConfig{ocr: %{"enabled" => true}}
{:ok, results} = Kreuzberg.BatchAPI.batch_extract_bytes(data_list, mime_types, config)
"""
@spec batch_extract_bytes(
[binary()],
String.t() | [String.t()],
ExtractionConfig.t() | map() | keyword() | nil
) :: {:ok, [ExtractionResult.t()]} | {:error, String.t()}
def batch_extract_bytes(data_list, mime_types, config \\ nil)
when is_list(data_list) and (is_binary(mime_types) or is_list(mime_types)) do
normalized_mime_types = normalize_mime_types(mime_types, data_list)
if length(data_list) != length(normalized_mime_types) do
mismatch_error(data_list, normalized_mime_types)
else
case call_native_batch_bytes(data_list, normalized_mime_types, config) do
{:ok, results_list} when is_list(results_list) ->
process_batch_results(results_list, normalized_mime_types, "mime_type")
{:error, _reason} = err ->
err
end
end
end
@doc """
Extract content from multiple binary inputs, raising on error.
Same as `batch_extract_bytes/3` but raises a `Kreuzberg.Error` exception if extraction fails.
## Examples
data_list = [pdf_binary1, pdf_binary2, pdf_binary3]
results = Kreuzberg.BatchAPI.batch_extract_bytes!(data_list, "application/pdf")
"""
@spec batch_extract_bytes!(
[binary()],
String.t() | [String.t()],
ExtractionConfig.t() | map() | keyword() | nil
) :: [ExtractionResult.t()]
def batch_extract_bytes!(data_list, mime_types, config \\ nil) do
case batch_extract_bytes(data_list, mime_types, config) do
{:ok, results} ->
results
{:error, reason} ->
raise Error, message: reason, reason: Kreuzberg.UtilityAPI.classify_error(reason)
end
end
defp normalize_mime_types(mime_types, data_list) do
if is_binary(mime_types) do
List.duplicate(mime_types, length(data_list))
else
mime_types
end
end
defp mismatch_error(data_list, mime_types) do
{:error,
"Mismatch between data_list length (#{length(data_list)}) and mime_types length (#{length(mime_types)})"}
end
defp process_batch_results(results_list, reference_list, reference_type) do
results =
results_list
|> Enum.with_index()
|> Enum.map(fn {result_map, index} ->
case Helpers.into_result(result_map) do
{:ok, result} -> {:ok, result}
{:error, reason} -> {:error, index, reason}
end
end)
case Enum.find(results, fn r -> match?({:error, _, _}, r) end) do
nil ->
{:ok, Enum.map(results, fn {:ok, result} -> result end)}
{:error, index, reason} ->
reference = Enum.at(reference_list, index, "unknown")
{:error, "Failed at index #{index} (#{reference_type}: '#{reference}'): #{reason}"}
end
end
defp call_native_batch_files(paths, mime_type, config) do
Helpers.call_native(
fn -> Native.batch_extract_files(paths, mime_type) end,
fn config_map ->
Native.batch_extract_files_with_options(paths, mime_type, config_map, nil)
end,
config
)
end
defp call_native_batch_bytes(data_list, mime_types, config) do
Helpers.call_native(
fn -> Native.batch_extract_bytes(data_list, mime_types) end,
fn config_map ->
Native.batch_extract_bytes_with_options(data_list, mime_types, config_map, nil)
end,
config
)
end
end