Current section
Files
Jump to
Current section
Files
lib/mix/tasks/ab.test.ex
defmodule Mix.Tasks.Ab.Test do
@moduledoc """
Mix task that compiles Elixir files, extracts modules, and runs AB.property_test on their exported functions.
## Usage
mix ab.test path/to/file.ex
mix ab.test path/to/directory
## Examples
# Test a single file
mix ab.test lib/my_module.ex
# Test all .ex files in a directory recursively
mix ab.test lib/
# Test with verbose output
mix ab.test lib/my_module.ex --verbose
# Show only failures in a structured format
mix ab.test lib/my_module.ex --failures-only
# Skip functions without typespecs (soft mode)
mix ab.test lib/my_module.ex --soft
# Test all files in a directory with soft mode
mix ab.test lib/ --soft
## Options
- `--verbose`, `-v`: Enable verbose output showing input/output for each test
- `--failures-only`, `-f`: Show only failed tests in a structured format with:
- Function name
- Why it failed
- Incorrect typespec
- Expected typespec
- File path and line number
- `--soft`, `-s`: Skip functions without typespecs instead of erroring out
This task will:
1. Compile the specified Elixir file(s) or all .ex files in a directory (recursively)
2. Extract the module from each compiled file
3. Find all exported functions with typespecs
4. Run AB.property_test on each function
5. Error out if functions without typespecs are found (unless --soft flag is used)
When processing directories, the task will:
- Recursively find all .ex files
- Skip hidden directories (starting with .)
- Skip common build directories (_build, deps, node_modules)
"""
use Mix.Task
@shortdoc "Compiles Elixir files and runs AB.property_test on their exported functions"
@impl Mix.Task
def run(args) do
# Start ExUnit
ExUnit.start(autorun: false)
{opts, paths} = parse_args(args)
if paths == [] do
Mix.shell().error(
"No file or directory specified. Usage: mix ab.test <file.ex> or <directory>"
)
System.halt(1)
end
# Expand paths to individual files
all_files = expand_paths_to_files(paths)
if all_files == [] do
Mix.shell().error("No .ex files found in the specified paths")
System.halt(1)
end
Mix.shell().info("Found #{length(all_files)} .ex files to process")
Enum.each(all_files, fn file ->
test_file(file, opts)
end)
end
defp parse_args(args) do
{opts, files, _} =
OptionParser.parse(args,
switches: [verbose: :boolean, failures_only: :boolean, soft: :boolean],
aliases: [v: :verbose, f: :failures_only, s: :soft]
)
{opts, files}
end
defp test_file(file_path, opts) do
Mix.shell().info("Testing file: #{file_path}")
case compile_and_extract_module(file_path) do
{:ok, module} ->
run_property_tests(module, opts, file_path)
{:error, reason} ->
Mix.shell().error("Failed to compile file: #{reason}")
System.halt(1)
end
end
defp compile_and_extract_module(file_path) do
# Extract module name from the file first
case extract_module_from_file(file_path) do
{:ok, module} ->
# Now compile the file using elixirc to preserve typespecs
compile_with_elixirc(file_path, module)
error ->
error
end
end
defp compile_with_elixirc(file_path, module) do
# Use Code.compile_file which has access to the current Mix environment
# This includes all dependencies that are already loaded
case Code.compile_file(file_path) do
[{^module, _}] ->
{:ok, module}
[{^module, _}, _] ->
{:ok, module}
[] ->
{:error, "No module compiled"}
_ ->
{:error, "Unexpected compilation result"}
end
end
defp extract_module_from_file(file_path) do
case File.read(file_path) do
{:ok, content} ->
# Parse the AST to extract the module name
case Code.string_to_quoted(content) do
{:ok, ast} ->
module_name = extract_module_name_from_ast(ast)
if module_name do
{:ok, module_name}
else
{:error, "Could not find module name in file"}
end
{:error, reason} ->
{:error, "Failed to parse file: #{inspect(reason)}"}
end
{:error, reason} ->
{:error, "Failed to read file: #{reason}"}
end
end
defp extract_module_name_from_ast({:defmodule, _, [module_alias | _]}) do
case module_alias do
{:__aliases__, _, parts} ->
Module.concat(parts)
name when is_atom(name) ->
name
_ ->
nil
end
end
defp extract_module_name_from_ast({:__block__, _, statements}) do
Enum.find_value(statements, fn statement ->
extract_module_name_from_ast(statement)
end)
end
defp extract_module_name_from_ast(_), do: nil
defp run_property_tests(module, opts, file_path) do
Mix.shell().info("Running property tests for module: #{module}")
# The module should already be loaded from compilation
run_tests_for_module(module, opts, file_path)
end
defp run_tests_for_module(module, opts, file_path) do
soft_mode = Keyword.get(opts, :soft, false)
# Get all exported functions with typespecs
functions_with_specs = get_functions_with_specs(module)
# Get all exported functions
all_exported_functions = module.__info__(:functions)
# Find functions without specs
functions_without_specs =
all_exported_functions
|> Enum.reject(fn {name, arity} ->
{name, arity} in functions_with_specs
end)
# Handle functions without specs
if functions_without_specs != [] do
if soft_mode do
Mix.shell().info(
"Warning: Found #{length(functions_without_specs)} functions without typespecs (skipping): #{format_function_list(functions_without_specs)}"
)
else
Mix.shell().error(
"Error: Found #{length(functions_without_specs)} functions without typespecs:"
)
Enum.each(functions_without_specs, fn {name, arity} ->
Mix.shell().error(
" - #{String.replace_leading(to_string(module), "Elixir.", "")}.#{name}/#{arity}"
)
end)
Mix.shell().error("Use --soft flag to skip functions without typespecs")
System.halt(1)
end
end
# Run tests for functions with specs
case functions_with_specs do
[] ->
Mix.shell().info("No functions with typespecs found in #{module}")
functions ->
Mix.shell().info("Found #{length(functions)} functions with typespecs")
run_tests_for_functions(module, functions, opts, file_path)
end
end
defp get_functions_with_specs(module) do
case Code.Typespec.fetch_specs(module) do
{:ok, specs} ->
exported_functions = module.__info__(:functions)
specs
|> Enum.map(fn {{function_name, arity}, _} -> {function_name, arity} end)
|> Enum.filter(fn {function_name, arity} ->
{function_name, arity} in exported_functions
end)
_ ->
[]
end
end
defp run_tests_for_functions(module, functions, opts, file_path) do
verbose = Keyword.get(opts, :verbose, false)
failures_only = Keyword.get(opts, :failures_only, false)
if not failures_only do
Mix.shell().info("\n=== Running property tests for #{length(functions)} functions ===")
end
results =
Enum.map(functions, fn {function_name, _arity} ->
if not failures_only do
Mix.shell().info("Testing #{module}.#{function_name}")
end
run_single_property_test(module, function_name, verbose, failures_only)
end)
# Summary
passed = Enum.count(results, &(&1 == :ok))
failed = length(results) - passed
if failures_only do
# Only show failures in the specified format
failures = Enum.filter(results, &(&1 != :ok))
if failures != [] do
Mix.shell().info("\n=== Failures ===")
Enum.with_index(failures, 1)
|> Enum.each(fn {failure, index} ->
format_failure_output(module, failure, index, file_path)
end)
else
Mix.shell().info("All tests passed!")
end
else
Mix.shell().info("\n=== Summary ===")
Mix.shell().info("✓ Passed: #{passed}")
Mix.shell().info("✗ Failed: #{failed}")
end
if failed > 0 do
System.halt(1)
end
end
defp run_single_property_test(module, function_name, verbose, failures_only) do
# Run the property test with details capture
case AB.run_property_test_with_details(module, function_name, verbose: verbose) do
{:ok, count} ->
if not failures_only do
Mix.shell().info(
" ✓ #{module}.#{function_name} passed all property tests (#{count} runs)"
)
end
:ok
{:error, details} ->
if not failures_only do
Mix.shell().error(" ✗ #{module}.#{function_name} failed property test")
Mix.shell().error(" #{details.reason}")
end
{:error, details}
end
end
defp format_failure_output(
_module,
{:error, details},
index,
file_path
) do
function_name = details.function_name
module = details.module
arity = get_arity_for_function(module, function_name)
# Get typespec correction suggestion if we have the actual values
suggestion =
if details.inputs && details.result && details.mismatch_type do
AB.suggest_typespec_correction(
module,
function_name,
details.inputs,
details.result,
details.mismatch_type
)
else
nil
end
# Get line number for the function - try both approaches
line_number =
case get_line_number_for_function(module, function_name) do
0 -> get_line_number_from_source(file_path, function_name)
line -> line
end
# Format the failure output - remove Elixir. prefix if present
module_name =
if to_string(module) |> String.starts_with?("Elixir.") do
to_string(module) |> String.replace_leading("Elixir.", "")
else
to_string(module)
end
Mix.shell().info("#{index}. #{module_name}.#{function_name}/#{arity}")
Mix.shell().info(" - Why it failed: #{details.reason}")
# Show the actual inputs and result if available
if details.inputs do
Mix.shell().info(" - Generated input: #{inspect(details.inputs)}")
end
if details.result do
Mix.shell().info(" - Actual result: #{inspect(details.result)}")
end
# Show typespec suggestion
case suggestion do
nil ->
# No suggestion available
:ok
{:error, error_msg} ->
Mix.shell().info(" - Could not analyze typespec: #{error_msg}")
%{old_spec: old_spec, new_spec: new_spec, reason: _suggestion_reason} ->
Mix.shell().info(" - Incorrect typespec: #{old_spec}")
Mix.shell().info(" - Suggested typespec: #{new_spec}")
end
Mix.shell().info(" - #{file_path}:#{line_number}")
Mix.shell().info("")
end
defp get_line_number_for_function(module, function_name) do
# Try to get line number from the function's metadata
case Code.Typespec.fetch_specs(module) do
{:ok, specs} ->
case Enum.find(specs, fn {{name, _arity}, _} -> name == function_name end) do
{{_name, _arity}, {_spec, context, metadata}} ->
# Try to get line number from context or metadata
cond do
# Check if context has line information
is_list(context) and Keyword.has_key?(context, :line) ->
Keyword.get(context, :line, 0)
# Check if metadata has line information
is_list(metadata) and Keyword.has_key?(metadata, :line) ->
Keyword.get(metadata, :line, 0)
# Look for line information in the metadata list
is_list(metadata) ->
case Enum.find(metadata, fn {key, _} -> key == :line end) do
{:line, line} -> line
_ -> 0
end
true ->
0
end
_ ->
0
end
_ ->
0
end
end
defp get_line_number_from_source(file_path, function_name) do
# Try to find the line number by parsing the source file
case File.read(file_path) do
{:ok, content} ->
lines = String.split(content, "\n")
# Look for the function definition
case Enum.with_index(lines, 1)
|> Enum.find({nil, 0}, fn {line, _index} ->
# Look for function definition patterns
String.contains?(line, "def #{function_name}") or
String.contains?(line, "def #{function_name}(") or
String.contains?(line, "def #{function_name} ")
end) do
{_line, line_number} -> line_number
_ -> 0
end
{:error, _} ->
0
end
end
defp get_arity_for_function(module, function_name) do
# Get arity from the module's exported functions
case module.__info__(:functions) do
functions when is_list(functions) ->
case Enum.find(functions, fn {name, _arity} -> name == function_name end) do
{_name, arity} -> arity
_ -> 0
end
_ ->
0
end
end
defp format_function_list(functions) do
functions
|> Enum.map(fn {name, arity} -> "#{name}/#{arity}" end)
|> Enum.join(", ")
end
defp expand_paths_to_files(paths) do
paths
|> Enum.flat_map(fn path ->
if File.dir?(path) do
find_ex_files_recursively(path)
else
if String.ends_with?(path, ".ex") do
[path]
else
[]
end
end
end)
|> Enum.uniq()
end
defp find_ex_files_recursively(directory) do
case File.ls(directory) do
{:ok, entries} ->
entries
|> Enum.flat_map(fn entry ->
full_path = Path.join(directory, entry)
cond do
File.dir?(full_path) ->
# Skip hidden directories and common build directories
if String.starts_with?(entry, ".") or entry in ["_build", "deps", "node_modules"] do
[]
else
find_ex_files_recursively(full_path)
end
String.ends_with?(entry, ".ex") ->
[full_path]
true ->
[]
end
end)
{:error, _reason} ->
Mix.shell().error("Could not read directory: #{directory}")
[]
end
end
end