Current section
Files
Jump to
Current section
Files
lib/ab.ex
defmodule AB do
@moduledoc """
A macro-based property testing generator that analyzes function typespecs
to automatically generate appropriate test data and output validators.
## Usage
defmodule MyModuleTest do
use ExUnit.Case
use ExUnitProperties
import AB
# Generate property test for a function with typespec
property_test MyModule, :my_function
end
"""
use ExUnitProperties
import ExUnit.Assertions
alias AB.{
TypeParser,
Generators,
Validators,
InvalidGenerators,
PropertyRunner,
CompareRunner,
BenchmarkRunner,
RobustRunner,
ModuleValidator,
StructValidator,
TypespecCorrector,
ConsistencyChecker,
TypeFormatter
}
@doc """
Macro that generates property testing utilities for a function based on its typespec.
Takes a module and function name, analyzes the typespec, and generates:
1. Input generators based on the function's parameter types
2. Output validators based on the function's return type
3. A complete property test
## Example
# For a function with spec: @spec sort_list([integer()]) :: [integer()]
property_test MyModule, :sort_list
# With verbose output
property_test MyModule, :sort_list, verbose: true
This will generate a property test that validates the function behavior.
"""
@spec property_test(module(), atom(), keyword()) :: Macro.t()
defmacro property_test(module, function_name, opts \\ []) do
quoted_test = property_test_quoted(module, function_name, opts)
quote do: unquote(quoted_test)
end
@spec property_test_quoted(module(), atom(), keyword()) :: Macro.t()
defp property_test_quoted(module, function_name, opts) do
quote do
property unquote("#{function_name} satisfies its typespec") do
AB.run_property_test(
unquote(module),
unquote(function_name),
unquote(opts)
)
end
property unquote("#{function_name} type consistency validation") do
AB.validate_type_consistency(unquote(module), unquote(function_name))
end
end
end
@doc false
@spec run_property_test(module(), atom(), keyword()) :: :ok
def run_property_test(module, function_name, opts) do
case get_function_spec(module, function_name) do
{:ok, {input_types, output_type}} ->
PropertyRunner.run_property_test(module, function_name, input_types, output_type, opts)
{:error, reason} ->
flunk("Could not generate property test: #{reason}")
end
end
@doc """
Runs a property test and returns detailed failure information including actual values.
This is used by the mix task to capture failure details for typespec suggestions.
Returns {:ok, count} on success or {:error, failure_details} on failure.
"""
@spec run_property_test_with_details(module(), atom(), keyword()) ::
{:ok, integer()} | {:error, map()}
def run_property_test_with_details(module, function_name, opts) do
case get_function_spec(module, function_name) do
{:ok, {input_types, output_type}} ->
PropertyRunner.run_property_test_with_details(
module,
function_name,
input_types,
output_type,
opts
)
{:error, reason} ->
{:error,
%{
module: module,
function_name: function_name,
reason: "Could not generate property test: #{reason}",
inputs: nil,
result: nil,
mismatch_type: nil
}}
end
end
@doc """
Macro that compares two functions with identical typespecs using the same generated test data.
Takes two module/function pairs, verifies their typespecs are identical, and if so,
runs both functions on the same generated inputs to ensure they produce identical outputs.
## Example
# Compare two sorting implementations
compare_test {AB, :a}, {AB, :b}
# With verbose output
compare_test {AB, :a}, {AB, :b}, verbose: true
This will generate a property test that validates both functions behave identically.
"""
@spec compare_test({module(), atom()}, {module(), atom()}, keyword()) :: Macro.t()
defmacro compare_test({module1, function1}, {module2, function2}, opts \\ []) do
quoted_test = compare_test_quoted(module1, function1, module2, function2, opts)
quote do: unquote(quoted_test)
end
@spec compare_test_quoted(module(), atom(), module(), atom(), keyword()) :: Macro.t()
defp compare_test_quoted(module1, function1, module2, function2, opts) do
quote do
property unquote("#{function1} and #{function2} produce identical results") do
AB.run_compare_test(
unquote(module1),
unquote(function1),
unquote(module2),
unquote(function2),
unquote(opts)
)
end
end
end
@doc false
@spec run_compare_test(module(), atom(), module(), atom(), keyword()) :: :ok
def run_compare_test(module1, function1, module2, function2, opts) do
with {:ok, spec1} <- get_function_spec(module1, function1),
{:ok, spec2} <- get_function_spec(module2, function2) do
CompareRunner.run_compare_test(module1, function1, module2, function2, spec1, spec2, opts)
else
{:error, reason} ->
flunk("Could not get typespec: #{reason}")
end
end
@doc """
Macro that benchmarks two functions with identical typespecs using Benchee.
Takes two module/function pairs, verifies their typespecs are identical, and if so,
runs a benchmark comparison using generated test data.
## Example
# Benchmark two sorting implementations
benchmark_test {AB, :a}, {AB, :b}
# With custom options
benchmark_test {AB, :a}, {AB, :b}, time: 5, memory_time: 2
This will generate a benchmark test that compares performance of both functions.
"""
@spec benchmark_test({module(), atom()}, {module(), atom()}, keyword()) :: Macro.t()
defmacro benchmark_test({module1, function1}, {module2, function2}, opts \\ []) do
quoted_test = benchmark_test_quoted(module1, function1, module2, function2, opts)
quote do: unquote(quoted_test)
end
@spec benchmark_test_quoted(module(), atom(), module(), atom(), keyword()) :: Macro.t()
defp benchmark_test_quoted(module1, function1, module2, function2, opts) do
quote do
test unquote("benchmark #{function1} vs #{function2}") do
AB.run_benchmark_test(
unquote(module1),
unquote(function1),
unquote(module2),
unquote(function2),
unquote(opts)
)
end
end
end
@doc false
@spec run_benchmark_test(module(), atom(), module(), atom(), keyword()) :: Benchee.Suite.t()
def run_benchmark_test(module1, function1, module2, function2, opts) do
with {:ok, spec1} <- get_function_spec(module1, function1),
{:ok, spec2} <- get_function_spec(module2, function2) do
BenchmarkRunner.run_benchmark_test(
module1,
function1,
module2,
function2,
spec1,
spec2,
opts
)
else
{:error, reason} ->
flunk("Could not get typespec: #{reason}")
end
end
@doc """
Macro that generates property tests for all public functions in a module.
Takes a module and automatically generates property tests for all exported functions
that have typespecs defined. This is useful for validating an entire module's API.
## Example
# Validate all public functions in a module
validate_module MyModule
# With verbose output
validate_module MyModule, verbose: true
This will generate property tests for each public function with a typespec.
"""
@spec validate_module(module(), keyword()) :: Macro.t()
defmacro validate_module(module, opts \\ []) do
quoted_test = validate_module_quoted(module, opts)
quote do: unquote(quoted_test)
end
@spec validate_module_quoted(module(), keyword()) :: Macro.t()
defp validate_module_quoted(module, opts) do
quote do
AB.run_validate_module(unquote(module), unquote(opts))
end
end
@doc false
@spec run_validate_module(module(), keyword()) :: :ok
def run_validate_module(module, opts) do
ModuleValidator.run_validate_module(module, opts)
end
@doc """
Macro that validates struct type definitions against function typespecs.
This catches inconsistencies where @type definitions don't match @spec definitions.
## Example
# This will fail if @type t :: %AB{a: atom()} but @spec expects integer()
validate_struct_consistency AB
"""
@spec validate_struct_consistency(module()) :: Macro.t()
defmacro validate_struct_consistency(module) do
module_name = extract_module_name(module)
quoted_test = validate_struct_consistency_quoted(module, module_name)
quote do: unquote(quoted_test)
end
@spec extract_module_name(module() | Macro.t()) :: atom()
defp extract_module_name(module) do
case module do
{:__aliases__, _, [name]} -> name
name when is_atom(name) -> name
_ -> module
end
end
@spec validate_struct_consistency_quoted(module(), atom()) :: Macro.t()
defp validate_struct_consistency_quoted(module, module_name) do
quote do
test unquote("#{module_name} struct type consistency") do
AB.run_struct_consistency_validation(unquote(module))
end
end
end
@doc false
@spec run_struct_consistency_validation(module()) :: :ok
def run_struct_consistency_validation(module) do
StructValidator.run_struct_consistency_validation(module)
end
@doc """
Macro that generates robustness tests for a function based on its typespec.
Takes a module and function name, analyzes the typespec, and generates:
1. Invalid input generators that create data NOT matching the function's parameter types
2. Tests that verify functions either raise errors or return invalid output when given invalid input
3. Ensures functions fail gracefully rather than silently accepting wrong input types
## Example
# For a function with spec: @spec process(integer()) :: string()
robust_test MyModule, :process
# With verbose output
robust_test MyModule, :process, verbose: true
This will generate tests that verify the function properly rejects invalid input.
"""
@spec robust_test(module(), atom(), keyword()) :: Macro.t()
defmacro robust_test(module, function_name, opts \\ []) do
quoted_test = robust_test_quoted(module, function_name, opts)
quote do: unquote(quoted_test)
end
@spec robust_test_quoted(module(), atom(), keyword()) :: Macro.t()
defp robust_test_quoted(module, function_name, opts) do
quote do
property unquote("#{function_name} properly rejects invalid input") do
AB.run_robust_test(
unquote(module),
unquote(function_name),
unquote(opts)
)
end
end
end
@doc false
@spec run_robust_test(module(), atom(), keyword()) :: :ok
def run_robust_test(module, function_name, opts) do
case get_function_spec(module, function_name) do
{:ok, {input_types, _output_type}} ->
RobustRunner.run_robust_test(module, function_name, input_types, opts)
{:error, reason} ->
flunk("Could not generate robust test: #{reason}")
end
end
# Public API functions delegating to submodules
@doc "Extracts the typespec for a given function."
defdelegate get_function_spec(module, function_name), to: TypeParser
@doc "Compares two type specifications for equivalence."
defdelegate types_equivalent?(type1, type2), to: TypeParser
@doc "Creates a struct generator from @type definition."
defdelegate create_struct_from_type_definition(module), to: TypeParser
@doc "Parses a spec into input and output types."
defdelegate parse_spec(spec), to: TypeParser, as: :parse_spec
@doc "Creates input generators from type specifications. Optionally accepts a module to resolve user-defined type aliases."
@spec create_input_generator_runtime([any()], module() | nil) :: Enumerable.t()
def create_input_generator_runtime(input_types, module \\ nil) do
Generators.create_input_generator(input_types, module)
end
@doc "Creates output validators from type specifications."
@spec create_output_validator_runtime(any()) :: (any() -> boolean())
def create_output_validator_runtime(output_type) do
Validators.create_output_validator(output_type)
end
@doc "Creates invalid input generators from type specifications."
@spec create_invalid_input_generator_runtime([any()], module() | nil) :: Enumerable.t()
def create_invalid_input_generator_runtime(input_types, module \\ nil) do
InvalidGenerators.create_invalid_input_generator(input_types, module)
end
@doc """
Validates type consistency between @type definitions and @spec definitions.
"""
@spec validate_type_consistency(module(), atom()) :: :ok
def validate_type_consistency(module, function_name) do
ConsistencyChecker.validate_type_consistency(module, function_name)
end
@doc """
Formats a typespec AST into a human-readable string using Elixir's standard functions where possible.
"""
@spec format_type_for_display(any()) :: String.t()
def format_type_for_display(type_ast) do
TypeFormatter.format_type_for_display(type_ast)
end
@doc """
Analyzes a failed test using runtime values and suggests a corrected typespec.
Takes actual runtime values (inputs and result) along with the current typespec,
infers the correct types from the values, and suggests a corrected typespec.
## Parameters
- `module`: The module containing the function
- `function_name`: The name of the function that failed
- `actual_inputs`: List of actual input values used in the test
- `actual_result`: The actual result value returned by the function
- `mismatch_type`: Either `:return` or `:argument` to indicate which type mismatched
## Returns
A map containing:
- `:old_spec` - The current typespec as a string
- `:new_spec` - The suggested corrected typespec as a string
- `:old_spec_ast` - The current typespec as AST {input_types, output_type}
- `:new_spec_ast` - The suggested typespec as AST {input_types, output_type}
- `:reason` - Why the typespec needs correction
## Example
iex> AB.suggest_typespec_correction(NumberFunctions, :double, [1.0], 2.0, :return)
%{
old_spec: "@spec double(number()) :: binary()",
new_spec: "@spec double(number()) :: float()",
old_spec_ast: {[{:type, 0, :number, []}], {:type, 0, :binary, []}},
new_spec_ast: {[{:type, 0, :number, []}], {:type, 0, :float, []}},
reason: "Return type mismatch: function returns float() but typespec declares binary()"
}
"""
@spec suggest_typespec_correction(module(), atom(), list(), any(), atom()) ::
%{
old_spec: String.t(),
new_spec: String.t(),
old_spec_ast: {[any()], any()},
new_spec_ast: {[any()], any()},
reason: String.t()
}
| {:error, String.t()}
def suggest_typespec_correction(
module,
function_name,
actual_inputs,
actual_result,
mismatch_type
) do
TypespecCorrector.suggest_typespec_correction(
module,
function_name,
actual_inputs,
actual_result,
mismatch_type
)
end
end