Packages

Automatically compare two implementations of the same problem with property-based testing and performance benchmarks. Perfect for refactoring, algorithm comparison, and validating that different implementations produce identical results.

Current section

Files

Jump to
ab lib ab robust_runner.ex
Raw

lib/ab/robust_runner.ex

defmodule AB.RobustRunner do
@moduledoc """
Executes robustness tests to verify functions properly handle invalid inputs.
This module generates invalid inputs that don't match the typespec and verifies
that functions either raise exceptions or properly validate inputs rather than
silently accepting incorrect data.
"""
import ExUnit.Assertions
alias AB.InvalidGenerators
@doc """
Runs robustness tests for a function with the given input types.
Generates invalid inputs and verifies proper error handling.
"""
@spec run_robust_test(module(), atom(), [any()], keyword()) :: :ok
def run_robust_test(module, function_name, input_types, opts) do
invalid_input_generator = InvalidGenerators.create_invalid_input_generator(input_types, module)
verbose = Keyword.get(opts, :verbose, false)
# Limit the generator to prevent infinite loops (StreamData default is 100)
limited_generator = Enum.take(invalid_input_generator, 100)
count =
Enum.reduce(limited_generator, 0, fn invalid_input, acc ->
test_invalid_input(module, function_name, invalid_input, verbose)
acc + 1
end)
IO.puts(" ✓ #{count} successful invalid input test runs")
end
@spec test_invalid_input(module(), atom(), any(), boolean()) :: :ok
defp test_invalid_input(module, function_name, invalid_input, verbose) do
try do
result = apply(module, function_name, invalid_input)
handle_unexpected_success(invalid_input, result, verbose)
rescue
e in [ExUnit.AssertionError] ->
reraise e, __STACKTRACE__
e ->
handle_expected_exception(invalid_input, e, verbose)
end
end
@spec handle_unexpected_success(any(), any(), boolean()) :: no_return()
defp handle_unexpected_success(invalid_input, result, verbose) do
if verbose do
IO.puts("Invalid input: #{inspect(invalid_input)} -> Output: #{inspect(result)}")
end
formatted_input = format_arguments(invalid_input)
flunk(
"Function accepted invalid input without validation and returned #{inspect(result)}. Expected function to either raise an exception or validate input types.\n\nGenerated arguments:\n#{formatted_input}"
)
end
@spec format_arguments(list()) :: String.t()
defp format_arguments(args) when is_list(args) do
args
|> Enum.with_index(1)
|> Enum.map(fn {arg, index} ->
" #{index}. #{inspect(arg, pretty: true, width: 80)}"
end)
|> Enum.join(",\n")
end
@spec handle_expected_exception(any(), Exception.t(), boolean()) :: :ok
defp handle_expected_exception(invalid_input, exception, true) do
IO.puts(
"Invalid input: #{inspect(invalid_input)} -> Exception: #{Exception.message(exception)} ✓"
)
end
defp handle_expected_exception(_invalid_input, _exception, false), do: :ok
end