Current section
Files
Jump to
Current section
Files
lib/repomix-output.xml
This file is a merged representation of the entire codebase, combined into a single document by Repomix.
<file_summary>
This section contains a summary of this file.
<purpose>
This file contains a packed representation of the entire repository's contents.
It is designed to be easily consumable by AI systems for analysis, code review,
or other automated processes.
</purpose>
<file_format>
The content is organized as follows:
1. This summary section
2. Repository information
3. Directory structure
4. Repository files (if enabled)
5. Multiple file entries, each consisting of:
- File path as an attribute
- Full contents of the file
</file_format>
<usage_guidelines>
- This file should be treated as read-only. Any changes should be made to the
original repository files, not this packed version.
- When processing this file, use the file path to distinguish
between different files in the repository.
- Be aware that this file may contain sensitive information. Handle it with
the same level of security as you would the original repository.
</usage_guidelines>
<notes>
- Some files may have been excluded based on .gitignore rules and Repomix's configuration
- Binary files are not included in this packed representation. Please refer to the Repository Structure section for a complete list of file paths, including binary files
- Files matching patterns in .gitignore are excluded
- Files matching default ignore patterns are excluded
- Files are sorted by Git change count (files with more changes are at the bottom)
</notes>
</file_summary>
<directory_structure>
crucible_bench/
stats/
anova.ex
confidence_interval.ex
distributions.ex
effect_size.ex
kruskal_wallis.ex
mann_whitney.ex
paired_t_test.ex
power.ex
t_test.ex
wilcoxon.ex
analysis.ex
experiment.ex
export.ex
result.ex
stats.ex
bench.ex
</directory_structure>
<files>
This section contains the contents of the repository's files.
<file path="crucible_bench/stats/anova.ex">
defmodule CrucibleBench.Stats.ANOVA do
@moduledoc """
One-way Analysis of Variance (ANOVA).
Compares means across 3+ independent groups to determine if at least
one group differs significantly from the others.
"""
alias CrucibleBench.{Stats, Result}
alias CrucibleBench.Stats.Distributions
@doc """
Perform one-way ANOVA.
## Options
- `:alpha` - Significance level (default: 0.05)
- `:labels` - Group labels for reporting
## Examples
iex> gpt4 = [0.89, 0.91, 0.88, 0.90, 0.92]
iex> claude = [0.87, 0.89, 0.86, 0.88, 0.90]
iex> gemini = [0.84, 0.86, 0.83, 0.85, 0.87]
iex> result = CrucibleBench.Stats.ANOVA.one_way([gpt4, claude, gemini])
iex> result.p_value < 0.05
true
"""
def one_way(groups, _opts \\ []) when is_list(groups) do
unless length(groups) >= 2 do
raise ArgumentError, "ANOVA requires at least 2 groups"
end
k = length(groups)
n_total = Enum.sum(Enum.map(groups, &length/1))
# Grand mean
all_values = List.flatten(groups)
grand_mean = Stats.mean(all_values)
# Between-group sum of squares
ss_between =
groups
|> Enum.map(fn group ->
n = length(group)
group_mean = Stats.mean(group)
n * :math.pow(group_mean - grand_mean, 2)
end)
|> Enum.sum()
# Within-group sum of squares
ss_within =
groups
|> Enum.map(fn group ->
group_mean = Stats.mean(group)
Enum.map(group, fn x -> :math.pow(x - group_mean, 2) end)
|> Enum.sum()
end)
|> Enum.sum()
# Total sum of squares
ss_total = Enum.map(all_values, fn x -> :math.pow(x - grand_mean, 2) end) |> Enum.sum()
# Degrees of freedom
df_between = k - 1
df_within = n_total - k
# Mean squares
ms_between = ss_between / df_between
ms_within = ss_within / df_within
# F-statistic
f_stat = ms_between / ms_within
# p-value from F-distribution
p_value = 1 - Distributions.f_cdf(f_stat, df_between, df_within)
# Effect sizes
eta_squared = ss_between / ss_total
omega_squared = (ss_between - df_between * ms_within) / (ss_total + ms_within)
effect_interpretation =
cond do
eta_squared >= 0.14 -> "large"
eta_squared >= 0.06 -> "medium"
eta_squared >= 0.01 -> "small"
true -> "negligible"
end
%Result{
test: :anova,
statistic: f_stat,
p_value: p_value,
effect_size: %{
eta_squared: eta_squared,
omega_squared: omega_squared,
interpretation: effect_interpretation
},
interpretation: interpret(p_value, eta_squared),
metadata: %{
df_between: df_between,
df_within: df_within,
ss_between: ss_between,
ss_within: ss_within,
ss_total: ss_total,
ms_between: ms_between,
ms_within: ms_within,
k: k,
n_total: n_total,
group_means: Enum.map(groups, &Stats.mean/1)
}
}
end
defp interpret(p_value, eta_squared) do
sig_text =
cond do
p_value < 0.001 -> "Highly significant"
p_value < 0.01 -> "Very significant"
p_value < 0.05 -> "Significant"
true -> "No significant"
end
effect_text =
cond do
eta_squared >= 0.14 -> "large"
eta_squared >= 0.06 -> "medium"
eta_squared >= 0.01 -> "small"
true -> "negligible"
end
"#{sig_text} difference between groups (#{effect_text} effect)"
end
end
</file>
<file path="crucible_bench/stats/confidence_interval.ex">
defmodule CrucibleBench.Stats.ConfidenceInterval do
@moduledoc """
Confidence interval calculations.
Supports both analytical and bootstrap methods for various statistics.
"""
alias CrucibleBench.Stats
alias CrucibleBench.Stats.Distributions
@doc """
Calculate confidence interval for a statistic.
## Options
- `:method` - :analytical (default) or :bootstrap
- `:confidence_level` - Confidence level (default: 0.95)
- `:iterations` - Bootstrap iterations (default: 10000)
- `:seed` - Random seed for reproducibility
## Examples
iex> data = [5.0, 5.2, 4.8, 5.1, 4.9, 5.3]
iex> ci = CrucibleBench.Stats.ConfidenceInterval.calculate(data, :mean)
iex> {lower, upper} = ci.interval
iex> lower < 5.05 and upper > 5.05
true
"""
def calculate(data, statistic, opts \\ []) do
method = Keyword.get(opts, :method, :analytical)
conf_level = Keyword.get(opts, :confidence_level, 0.95)
case method do
:analytical -> analytical_ci(data, statistic, conf_level)
:bootstrap -> bootstrap_ci(data, statistic, opts)
_ -> raise ArgumentError, "Unknown method: #{method}"
end
end
@doc """
Calculate analytical confidence interval for the mean.
Uses t-distribution for small samples.
"""
def analytical_ci(data, :mean, conf_level) do
n = length(data)
mean = Stats.mean(data)
sem = Stats.sem(data)
df = n - 1
alpha = 1 - conf_level
t_critical = Distributions.t_quantile(df, 1 - alpha / 2)
margin = t_critical * sem
interval = {mean - margin, mean + margin}
%{
statistic: :mean,
point_estimate: mean,
interval: interval,
confidence_level: conf_level,
method: :analytical,
margin_of_error: margin,
standard_error: sem
}
end
def analytical_ci(data, :median, conf_level) do
# For median, use bootstrap (no simple analytical formula)
bootstrap_ci(data, :median, confidence_level: conf_level)
end
def analytical_ci(data, :variance, conf_level) do
n = length(data)
var = Stats.variance(data)
df = n - 1
alpha = 1 - conf_level
# Chi-squared based CI for variance
chi2_lower = chi_squared_quantile(df, alpha / 2)
chi2_upper = chi_squared_quantile(df, 1 - alpha / 2)
lower = df * var / chi2_upper
upper = df * var / chi2_lower
%{
statistic: :variance,
point_estimate: var,
interval: {lower, upper},
confidence_level: conf_level,
method: :analytical
}
end
@doc """
Calculate bootstrap confidence interval.
Uses percentile method for bootstrap CI.
## Options
- `:confidence_level` - Confidence level (default: 0.95)
- `:iterations` - Number of bootstrap samples (default: 10000)
- `:seed` - Random seed for reproducibility
"""
def bootstrap_ci(data, statistic, opts \\ []) do
conf_level = Keyword.get(opts, :confidence_level, 0.95)
iterations = Keyword.get(opts, :iterations, 10_000)
seed = Keyword.get(opts, :seed, :os.system_time(:microsecond))
# Set random seed for reproducibility
:rand.seed(:exsplus, {seed, seed + 1, seed + 2})
# Calculate point estimate
point_estimate = calculate_statistic(data, statistic)
# Generate bootstrap samples
bootstrap_stats =
1..iterations
|> Enum.map(fn _ ->
bootstrap_sample = resample(data)
calculate_statistic(bootstrap_sample, statistic)
end)
|> Enum.sort()
# Percentile method
alpha = 1 - conf_level
lower_idx = round(alpha / 2 * iterations)
upper_idx = round((1 - alpha / 2) * iterations)
lower = Enum.at(bootstrap_stats, lower_idx)
upper = Enum.at(bootstrap_stats, upper_idx)
%{
statistic: statistic,
point_estimate: point_estimate,
interval: {lower, upper},
confidence_level: conf_level,
method: :bootstrap,
iterations: iterations,
bootstrap_distribution: %{
mean: Stats.mean(bootstrap_stats),
sd: Stats.stdev(bootstrap_stats)
}
}
end
defp calculate_statistic(data, :mean), do: Stats.mean(data)
defp calculate_statistic(data, :median), do: Stats.median(data)
defp calculate_statistic(data, :variance), do: Stats.variance(data)
defp calculate_statistic(data, :stdev), do: Stats.stdev(data)
defp calculate_statistic(data, fun) when is_function(fun, 1) do
fun.(data)
end
defp resample(data) do
n = length(data)
1..n
|> Enum.map(fn _ -> Enum.random(data) end)
end
# Chi-squared quantile approximation
defp chi_squared_quantile(df, p) do
# Wilson-Hilferty approximation
z = Distributions.normal_quantile(p)
df * :math.pow(1 - 2 / (9 * df) + z * :math.sqrt(2 / (9 * df)), 3)
end
end
</file>
<file path="crucible_bench/stats/distributions.ex">
defmodule CrucibleBench.Stats.Distributions do
@moduledoc """
Probability distributions and statistical functions.
Provides CDF and quantile functions for common distributions.
"""
@doc """
Standard normal cumulative distribution function (CDF).
Uses error function approximation for standard normal.
"""
def normal_cdf(z, mu \\ 0, sigma \\ 1) do
z_standard = (z - mu) / sigma
0.5 * (1 + erf(z_standard / :math.sqrt(2)))
end
@doc """
Standard normal quantile (inverse CDF).
Approximation of the inverse normal CDF.
"""
def normal_quantile(p) when p > 0 and p < 1 do
# Rational approximation for inverse normal CDF
# Beasley-Springer-Moro algorithm
a = [
2.50662823884,
-18.61500062529,
41.39119773534,
-25.44106049637
]
b = [
-8.47351093090,
23.08336743743,
-21.06224101826,
3.13082909833
]
c = [
0.3374754822726147,
0.9761690190917186,
0.1607979714918209,
0.0276438810333863,
0.0038405729373609,
0.0003951896511919,
0.0000321767881768,
0.0000002888167364,
0.0000003960315187
]
y = p - 0.5
if abs(y) < 0.42 do
r = y * y
numerator =
Enum.reduce(Enum.with_index(a), 0, fn {coef, i}, acc ->
acc + coef * :math.pow(r, i)
end)
denominator =
1 +
Enum.reduce(Enum.with_index(b), 0, fn {coef, i}, acc ->
acc + coef * :math.pow(r, i + 1)
end)
y * numerator / denominator
else
r = if y > 0, do: p, else: 1 - p
r = :math.log(-:math.log(r))
Enum.reduce(Enum.with_index(c), 0, fn {coef, i}, acc ->
acc + coef * :math.pow(r, i)
end)
|> then(fn result -> if y < 0, do: -result, else: result end)
end
end
@doc """
Student's t-distribution CDF.
Approximation using beta function relationship.
"""
def t_cdf(t, df) do
x = df / (df + t * t)
beta_cdf = incomplete_beta(df / 2, 0.5, x)
if t >= 0 do
1 - 0.5 * beta_cdf
else
0.5 * beta_cdf
end
end
@doc """
Student's t-distribution quantile.
Approximation for t-distribution inverse CDF.
"""
def t_quantile(df, p) when p > 0 and p < 1 do
# For large df, approximate with normal distribution
if df > 30 do
normal_quantile(p)
else
# Hill's algorithm for t-distribution quantile
a = 1 / (df - 0.5)
b = 48 / (a * a)
c = ((20700 * a / b - 98) * a - 16) * a + 96.36
d = ((94.5 / (b + c) - 3) / b + 1) * :math.sqrt(a * :math.pi() / 2) * df
y = :math.pow(d * p, 2 / df)
y =
if y > 0.05 + a do
x = normal_quantile(p)
y_temp = x * x
c =
if df < 5 do
c + 0.3 * (df - 4.5) * (x + 0.6)
else
c
end
c =
(((0.05 * d * x - 5) * x - 7) * x - 2) * x + b +
c
y_temp =
(((((0.4 * y_temp + 6.3) * y_temp + 36) * y_temp + 94.5) / c - y_temp - 3) / b + 1) *
x
y_temp = a * y_temp * y_temp
if y_temp > 0.002 do
:math.exp(y_temp) - 1
else
0.5 * y_temp * y_temp + y_temp
end
else
((1 / (((df + 6) / (df * y) - 0.089 * d - 0.822) * (df + 2) * 3) + 0.5 / (df + 4)) * y -
1) * (df + 1) / (df + 2) + 1 / y
end
:math.sqrt(df * y)
|> then(fn result -> if p < 0.5, do: -result, else: result end)
end
end
@doc """
F-distribution CDF.
Approximation using beta function relationship.
"""
def f_cdf(f, df1, df2) do
x = df1 * f / (df1 * f + df2)
incomplete_beta(df1 / 2, df2 / 2, x)
end
@doc """
Chi-squared distribution CDF.
Uses gamma function relationship.
"""
def chi_squared_cdf(x, df) do
lower_gamma(df / 2, x / 2) / gamma(df / 2)
end
# Error function approximation
defp erf(x) do
# Abramowitz and Stegun approximation
a1 = 0.254829592
a2 = -0.284496736
a3 = 1.421413741
a4 = -1.453152027
a5 = 1.061405429
p = 0.3275911
sign = if x < 0, do: -1, else: 1
x = abs(x)
t = 1.0 / (1.0 + p * x)
y = 1.0 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * :math.exp(-x * x)
sign * y
end
# Incomplete beta function (simplified approximation)
defp incomplete_beta(_a, _b, x) when x == 0, do: 0.0
defp incomplete_beta(_a, _b, x) when x == 1, do: 1.0
defp incomplete_beta(a, b, x) do
# Continued fraction approximation
beta_ab = beta_function(a, b)
beta_inc = incomplete_beta_continued_fraction(a, b, x)
beta_inc / beta_ab
end
defp beta_function(a, b) do
:math.exp(log_gamma(a) + log_gamma(b) - log_gamma(a + b))
end
defp incomplete_beta_continued_fraction(a, b, x, max_iter \\ 100) do
front = :math.pow(x, a) * :math.pow(1 - x, b) / a
c = 1.0
d = 1.0 - (a + b) * x / (a + 1)
d = if abs(d) < 1.0e-30, do: 1.0e-30, else: d
d = 1.0 / d
f = d
Enum.reduce_while(1..max_iter, {f, c, d}, fn m, {f, c, d} ->
m = m * 1.0
m2 = m * 2
aa = m * (b - m) * x / ((a + m2 - 1) * (a + m2))
d = 1.0 + aa * d
d = if abs(d) < 1.0e-30, do: 1.0e-30, else: d
c = 1.0 + aa / c
c = if abs(c) < 1.0e-30, do: 1.0e-30, else: c
d = 1.0 / d
f = f * d * c
aa = -(a + m) * (a + b + m) * x / ((a + m2) * (a + m2 + 1))
d = 1.0 + aa * d
d = if abs(d) < 1.0e-30, do: 1.0e-30, else: d
c = 1.0 + aa / c
c = if abs(c) < 1.0e-30, do: 1.0e-30, else: c
d = 1.0 / d
delta = d * c
f = f * delta
if abs(delta - 1.0) < 1.0e-10 do
{:halt, {f, c, d}}
else
{:cont, {f, c, d}}
end
end)
|> elem(0)
|> then(&(front * &1))
end
# Gamma function approximation
defp gamma(x) do
:math.exp(log_gamma(x))
end
# Log-gamma function (Lanczos approximation)
defp log_gamma(x) do
g = 7
coef = [
0.99999999999980993,
676.5203681218851,
-1259.1392167224028,
771.32342877765313,
-176.61502916214059,
12.507343278686905,
-0.13857109526572012,
9.9843695780195716e-6,
1.5056327351493116e-7
]
if x < 0.5 do
:math.log(:math.pi()) - :math.log(:math.sin(:math.pi() * x)) - log_gamma(1 - x)
else
x = x - 1
base = x + g + 0.5
series =
Enum.reduce(1..8, Enum.at(coef, 0), fn i, acc ->
acc + Enum.at(coef, i) / (x + i)
end)
:math.log(:math.sqrt(2 * :math.pi())) + :math.log(series) - base +
:math.log(base) * (x + 0.5)
end
end
# Lower incomplete gamma function
defp lower_gamma(a, x) do
gamma(a) * incomplete_gamma_p(a, x)
end
defp incomplete_gamma_p(a, x, max_iter \\ 100) do
if x < a + 1 do
# Use series representation
ap = a
del = 1.0 / a
sum = del
Enum.reduce_while(1..max_iter, {sum, del, ap}, fn _n, {sum, del, ap} ->
ap = ap + 1
del = del * x / ap
sum = sum + del
if abs(del) < abs(sum) * 1.0e-10 do
{:halt, sum * :math.exp(-x + a * :math.log(x) - log_gamma(a))}
else
{:cont, {sum, del, ap}}
end
end)
|> then(fn
{sum, _, _} -> sum * :math.exp(-x + a * :math.log(x) - log_gamma(a))
result -> result
end)
else
# Use continued fraction
1.0 - incomplete_gamma_q_cf(a, x)
end
end
defp incomplete_gamma_q_cf(a, x, max_iter \\ 100) do
b = x + 1 - a
c = 1.0 / 1.0e-30
d = 1.0 / b
h = d
Enum.reduce_while(1..max_iter, {h, d, c, b}, fn i, {h, d, c, b} ->
an = -i * (i - a)
b = b + 2
d = an * d + b
d = if abs(d) < 1.0e-30, do: 1.0e-30, else: d
c = b + an / c
c = if abs(c) < 1.0e-30, do: 1.0e-30, else: c
d = 1.0 / d
delta = d * c
h = h * delta
if abs(delta - 1.0) < 1.0e-10 do
{:halt, h * :math.exp(-x + a * :math.log(x) - log_gamma(a))}
else
{:cont, {h, d, c, b}}
end
end)
|> then(fn
{h, _, _, _} -> h * :math.exp(-x + a * :math.log(x) - log_gamma(a))
result -> result
end)
end
end
</file>
<file path="crucible_bench/stats/effect_size.ex">
defmodule CrucibleBench.Stats.EffectSize do
@moduledoc """
Effect size measures for statistical tests.
Effect sizes quantify the magnitude of differences, providing
practical significance beyond p-values.
"""
alias CrucibleBench.Stats
@doc """
Calculate Cohen's d for two independent groups.
Cohen's d is the standardized mean difference:
d = (mean1 - mean2) / pooled_sd
Interpretation (Cohen, 1988):
- Small: |d| = 0.2
- Medium: |d| = 0.5
- Large: |d| = 0.8
## Examples
iex> group1 = [5.0, 5.2, 4.8, 5.1, 4.9]
iex> group2 = [6.0, 6.2, 5.8, 6.1, 5.9]
iex> result = CrucibleBench.Stats.EffectSize.cohens_d(group1, group2)
iex> result.cohens_d > 0
true
"""
def cohens_d(group1, group2) do
mean1 = Stats.mean(group1)
mean2 = Stats.mean(group2)
var1 = Stats.variance(group1)
var2 = Stats.variance(group2)
# Pooled standard deviation
pooled_sd = :math.sqrt((var1 + var2) / 2)
d = (mean1 - mean2) / pooled_sd
%{
cohens_d: d,
interpretation: interpret_cohens_d(d),
mean1: mean1,
mean2: mean2,
pooled_sd: pooled_sd
}
end
@doc """
Calculate Hedges' g (bias-corrected Cohen's d).
Hedges' g applies a correction factor for small sample sizes,
making it less biased than Cohen's d.
## Examples
iex> group1 = [5.0, 5.2, 4.8]
iex> group2 = [6.0, 6.2, 5.8]
iex> result = CrucibleBench.Stats.EffectSize.hedges_g(group1, group2)
iex> result.hedges_g > 0
true
"""
def hedges_g(group1, group2) do
n1 = length(group1)
n2 = length(group2)
n = n1 + n2
# Calculate Cohen's d first
cohens = cohens_d(group1, group2)
d = cohens.cohens_d
# Correction factor
correction = 1 - 3 / (4 * n - 9)
g = d * correction
%{
hedges_g: g,
cohens_d: d,
correction_factor: correction,
interpretation: interpret_cohens_d(g)
}
end
@doc """
Calculate Glass's delta.
Uses only the control group's standard deviation,
useful when groups have different variances.
## Examples
iex> control = [5.0, 5.2, 4.8, 5.1, 4.9]
iex> treatment = [6.0, 6.5, 5.5, 6.2, 5.8]
iex> result = CrucibleBench.Stats.EffectSize.glass_delta(control, treatment)
iex> result.glass_delta > 0
true
"""
def glass_delta(control, treatment) do
mean_control = Stats.mean(control)
mean_treatment = Stats.mean(treatment)
sd_control = Stats.stdev(control)
delta = (mean_treatment - mean_control) / sd_control
%{
glass_delta: delta,
interpretation: interpret_cohens_d(delta),
mean_control: mean_control,
mean_treatment: mean_treatment,
sd_control: sd_control
}
end
@doc """
Calculate effect size for paired data.
Returns Cohen's d for paired samples, using the standard deviation
of the differences.
## Examples
iex> before = [0.72, 0.68, 0.75, 0.71, 0.69]
iex> after_values = [0.78, 0.73, 0.81, 0.76, 0.74]
iex> result = CrucibleBench.Stats.EffectSize.paired_cohens_d(before, after_values)
iex> result.cohens_d > 0
true
"""
def paired_cohens_d(group1, group2) do
unless length(group1) == length(group2) do
raise ArgumentError, "Paired effect size requires equal length groups"
end
differences = Enum.zip_with(group1, group2, fn x, y -> y - x end)
mean_diff = Stats.mean(differences)
sd_diff = Stats.stdev(differences)
d = if sd_diff == 0, do: 0.0, else: mean_diff / sd_diff
%{
cohens_d: d,
interpretation: interpret_cohens_d(d),
mean_diff: mean_diff,
sd_diff: sd_diff
}
end
@doc """
General effect size calculation with automatic method selection.
## Options
- `:type` - :cohens_d (default), :hedges_g, :glass_delta
- `:paired` - true for paired data (default: false)
## Examples
iex> group1 = [5.0, 5.2, 4.8, 5.1, 4.9]
iex> group2 = [6.0, 6.2, 5.8, 6.1, 5.9]
iex> result = CrucibleBench.Stats.EffectSize.calculate(group1, group2)
iex> Map.has_key?(result, :cohens_d)
true
"""
def calculate(group1, group2, opts \\ []) do
paired = Keyword.get(opts, :paired, false)
type = Keyword.get(opts, :type, :cohens_d)
if paired do
paired_cohens_d(group1, group2)
else
case type do
:cohens_d -> cohens_d(group1, group2)
:hedges_g -> hedges_g(group1, group2)
:glass_delta -> glass_delta(group1, group2)
_ -> raise ArgumentError, "Unknown effect size type: #{type}"
end
end
end
defp interpret_cohens_d(d) do
abs_d = abs(d)
cond do
abs_d < 0.2 -> "negligible"
abs_d < 0.5 -> "small"
abs_d < 0.8 -> "medium"
true -> "large"
end
end
end
</file>
<file path="crucible_bench/stats/kruskal_wallis.ex">
defmodule CrucibleBench.Stats.KruskalWallis do
@moduledoc """
Kruskal-Wallis H test.
Non-parametric alternative to one-way ANOVA.
Tests if multiple independent samples come from the same distribution.
"""
alias CrucibleBench.Result
alias CrucibleBench.Stats.Distributions
@doc """
Perform Kruskal-Wallis test.
## Options
- `:alpha` - Significance level (default: 0.05)
## Examples
iex> group1 = [5, 7, 8, 6, 9]
iex> group2 = [3, 4, 5, 4, 6]
iex> group3 = [1, 2, 3, 2, 4]
iex> result = CrucibleBench.Stats.KruskalWallis.test([group1, group2, group3])
iex> result.test == :kruskal_wallis
true
"""
def test(groups, _opts \\ []) when is_list(groups) do
unless length(groups) >= 2 do
raise ArgumentError, "Kruskal-Wallis test requires at least 2 groups"
end
# Combine all groups with labels
combined =
groups
|> Enum.with_index()
|> Enum.flat_map(fn {group, idx} ->
Enum.map(group, fn val -> {idx, val} end)
end)
# Sort and rank
sorted = Enum.sort_by(combined, fn {_, val} -> val end)
ranks = assign_ranks(sorted)
# Calculate rank sums for each group
k = length(groups)
n_total = length(sorted)
rank_sums =
ranks
|> Enum.group_by(fn {group_idx, _, _} -> group_idx end)
|> Enum.map(fn {idx, group_ranks} ->
{idx, Enum.sum(Enum.map(group_ranks, fn {_, _, rank} -> rank end))}
end)
|> Map.new()
# H statistic
h_stat =
12 / (n_total * (n_total + 1)) *
Enum.sum(
Enum.map(groups, fn group ->
idx = Enum.find_index(groups, &(&1 == group))
n = length(group)
r = Map.get(rank_sums, idx)
:math.pow(r, 2) / n
end)
) - 3 * (n_total + 1)
# p-value from chi-squared distribution
df = k - 1
p_value = 1 - Distributions.chi_squared_cdf(h_stat, df)
# Effect size (epsilon-squared)
epsilon_sq = h_stat / (n_total - 1)
%Result{
test: :kruskal_wallis,
statistic: h_stat,
p_value: p_value,
effect_size: %{
epsilon_squared: epsilon_sq,
interpretation: interpret_epsilon_squared(epsilon_sq)
},
interpretation: interpret(p_value, epsilon_sq),
metadata: %{
df: df,
k: k,
n_total: n_total,
rank_sums: rank_sums
}
}
end
defp assign_ranks(sorted_values) do
sorted_values
|> Enum.chunk_by(fn {_, val} -> val end)
|> Enum.reduce({[], 1}, fn chunk, {acc, start_rank} ->
chunk_size = length(chunk)
avg_rank = (start_rank + start_rank + chunk_size - 1) / 2
ranked_chunk = Enum.map(chunk, fn {group_idx, val} -> {group_idx, val, avg_rank} end)
{acc ++ ranked_chunk, start_rank + chunk_size}
end)
|> elem(0)
end
defp interpret_epsilon_squared(eps) do
cond do
eps >= 0.14 -> "large"
eps >= 0.06 -> "medium"
eps >= 0.01 -> "small"
true -> "negligible"
end
end
defp interpret(p_value, eps) do
sig_text =
cond do
p_value < 0.001 -> "Highly significant"
p_value < 0.01 -> "Very significant"
p_value < 0.05 -> "Significant"
true -> "No significant"
end
effect_text = interpret_epsilon_squared(eps)
"#{sig_text} difference between groups (#{effect_text} effect)"
end
end
</file>
<file path="crucible_bench/stats/mann_whitney.ex">
defmodule CrucibleBench.Stats.MannWhitney do
@moduledoc """
Mann-Whitney U test (Wilcoxon rank-sum test).
Non-parametric alternative to independent samples t-test.
Tests if two independent samples come from the same distribution.
"""
alias CrucibleBench.Result
alias CrucibleBench.Stats.Distributions
@doc """
Perform Mann-Whitney U test.
## Options
- `:alternative` - :two_sided (default), :less, :greater
## Examples
iex> control = [120, 135, 118, 142, 125, 890] # Has outlier
iex> treatment = [98, 105, 102, 110, 95, 108]
iex> result = CrucibleBench.Stats.MannWhitney.test(control, treatment)
iex> result.test == :mann_whitney
true
"""
def test(group1, group2, opts \\ []) do
n1 = length(group1)
n2 = length(group2)
# Combine and rank all values
combined = Enum.map(group1, &{:group1, &1}) ++ Enum.map(group2, &{:group2, &1})
sorted = Enum.sort_by(combined, fn {_, val} -> val end)
# Assign ranks (handle ties by averaging)
ranks = assign_ranks(sorted)
# Sum ranks for each group
r1 =
ranks
|> Enum.filter(fn {group, _, _} -> group == :group1 end)
|> Enum.map(fn {_, _, rank} -> rank end)
|> Enum.sum()
r2 =
ranks
|> Enum.filter(fn {group, _, _} -> group == :group2 end)
|> Enum.map(fn {_, _, rank} -> rank end)
|> Enum.sum()
# Calculate U statistics
u1 = n1 * n2 + n1 * (n1 + 1) / 2 - r1
u2 = n1 * n2 + n2 * (n2 + 1) / 2 - r2
# Use smaller U
u_stat = min(u1, u2)
# For large samples, use normal approximation
{p_value, z_stat} =
if n1 > 20 and n2 > 20 do
mu_u = n1 * n2 / 2
sigma_u = :math.sqrt(n1 * n2 * (n1 + n2 + 1) / 12)
# Continuity correction
z = (u_stat - mu_u + 0.5) / sigma_u
alternative = Keyword.get(opts, :alternative, :two_sided)
p =
case alternative do
:two_sided -> 2 * (1 - Distributions.normal_cdf(abs(z)))
:less -> Distributions.normal_cdf(z)
:greater -> 1 - Distributions.normal_cdf(z)
end
{p, z}
else
# For small samples, would use exact distribution
# For simplicity, using normal approximation with warning
mu_u = n1 * n2 / 2
sigma_u = :math.sqrt(n1 * n2 * (n1 + n2 + 1) / 12)
z = (u_stat - mu_u + 0.5) / sigma_u
p = 2 * (1 - Distributions.normal_cdf(abs(z)))
{p, z}
end
# Effect size (rank-biserial correlation)
r_rb = 1 - 2 * u_stat / (n1 * n2)
%Result{
test: :mann_whitney,
statistic: u_stat,
p_value: p_value,
effect_size: %{
rank_biserial: r_rb,
interpretation: interpret_rank_biserial(r_rb)
},
interpretation: interpret(p_value, r_rb),
metadata: %{
u1: u1,
u2: u2,
z_statistic: z_stat,
n1: n1,
n2: n2,
r1: r1,
r2: r2
}
}
end
defp assign_ranks(sorted_values) do
sorted_values
|> Enum.chunk_by(fn {_, val} -> val end)
|> Enum.reduce({[], 1}, fn chunk, {acc, start_rank} ->
chunk_size = length(chunk)
avg_rank = (start_rank + start_rank + chunk_size - 1) / 2
ranked_chunk = Enum.map(chunk, fn {group, val} -> {group, val, avg_rank} end)
{acc ++ ranked_chunk, start_rank + chunk_size}
end)
|> elem(0)
end
defp interpret_rank_biserial(r) do
cond do
abs(r) < 0.1 -> "negligible"
abs(r) < 0.3 -> "small"
abs(r) < 0.5 -> "medium"
true -> "large"
end
end
defp interpret(p_value, r_rb) do
sig_text =
cond do
p_value < 0.001 -> "Highly significant"
p_value < 0.01 -> "Very significant"
p_value < 0.05 -> "Significant"
true -> "No significant"
end
effect_text = interpret_rank_biserial(r_rb)
"#{sig_text} difference between groups (#{effect_text} effect)"
end
end
</file>
<file path="crucible_bench/stats/paired_t_test.ex">
defmodule CrucibleBench.Stats.PairedTTest do
@moduledoc """
Paired samples t-test.
Compares means of two related groups (e.g., before/after measurements).
"""
alias CrucibleBench.{Stats, Result}
alias CrucibleBench.Stats.Distributions
@doc """
Perform paired samples t-test.
## Options
- `:mu` - Hypothesized mean difference (default: 0.0)
- `:alternative` - :two_sided (default), :less, :greater
- `:confidence_level` - Confidence level for CI (default: 0.95)
## Examples
iex> before = [0.72, 0.68, 0.75, 0.71, 0.69]
iex> after = [0.78, 0.73, 0.81, 0.76, 0.74]
iex> result = CrucibleBench.Stats.PairedTTest.test(before, after)
iex> result.p_value < 0.05
true
"""
def test(group1, group2, opts \\ []) do
unless length(group1) == length(group2) do
raise ArgumentError, "Paired test requires equal length groups"
end
# Calculate differences
differences = Enum.zip_with(group1, group2, fn x, y -> y - x end)
n = length(differences)
mean_diff = Stats.mean(differences)
sd_diff = Stats.stdev(differences)
se_diff = sd_diff / :math.sqrt(n)
mu0 = Keyword.get(opts, :mu, 0.0)
t_stat = (mean_diff - mu0) / se_diff
df = n - 1
alternative = Keyword.get(opts, :alternative, :two_sided)
p_value = p_value_from_t(t_stat, df, alternative)
# Confidence interval
conf_level = Keyword.get(opts, :confidence_level, 0.95)
alpha = 1 - conf_level
t_critical = Distributions.t_quantile(df, 1 - alpha / 2)
margin = t_critical * se_diff
ci = {mean_diff - margin, mean_diff + margin}
%Result{
test: :paired_t_test,
statistic: t_stat,
p_value: p_value,
confidence_interval: ci,
interpretation: interpret(p_value, mean_diff),
metadata: %{
df: df,
mean_diff: mean_diff,
sd_diff: sd_diff,
se_diff: se_diff,
alternative: alternative,
n: n
}
}
end
defp p_value_from_t(t_stat, df, alternative) do
p_two_sided = 2 * (1 - Distributions.t_cdf(abs(t_stat), df))
case alternative do
:two_sided -> p_two_sided
:greater -> 1 - Distributions.t_cdf(t_stat, df)
:less -> Distributions.t_cdf(t_stat, df)
end
end
defp interpret(p_value, mean_diff) do
sig_text =
cond do
p_value < 0.001 -> "Highly significant"
p_value < 0.01 -> "Very significant"
p_value < 0.05 -> "Significant"
true -> "No significant"
end
direction = if mean_diff > 0, do: "increase", else: "decrease"
"#{sig_text} #{direction} from first to second group"
end
end
</file>
<file path="crucible_bench/stats/power.ex">
defmodule CrucibleBench.Stats.Power do
@moduledoc """
Power analysis for statistical tests.
Calculates statistical power and required sample sizes.
"""
alias CrucibleBench.Stats.Distributions
@doc """
Perform power analysis.
## Options
- `:analysis_type` - :a_priori (sample size) or :post_hoc (power)
- `:effect_size` - Expected or observed effect size (Cohen's d)
- `:alpha` - Significance level (default: 0.05)
- `:power` - Desired power for a priori (default: 0.80)
- `:n_per_group` - Sample size per group for post-hoc
- `:alternative` - :two_sided (default), :less, :greater
## Examples
# A priori: Calculate required sample size
iex> result = CrucibleBench.Stats.Power.analyze(:t_test,
...> analysis_type: :a_priori,
...> effect_size: 0.5,
...> alpha: 0.05,
...> power: 0.80)
iex> result.n_per_group > 0
true
# Post-hoc: Calculate achieved power
iex> result = CrucibleBench.Stats.Power.analyze(:t_test,
...> analysis_type: :post_hoc,
...> effect_size: 0.5,
...> n_per_group: 64,
...> alpha: 0.05)
iex> result.power > 0.7
true
"""
def analyze(test_type, opts \\ []) do
analysis_type = Keyword.get(opts, :analysis_type, :a_priori)
case {test_type, analysis_type} do
{:t_test, :a_priori} -> t_test_sample_size(opts)
{:t_test, :post_hoc} -> t_test_power(opts)
{:anova, :a_priori} -> anova_sample_size(opts)
{:anova, :post_hoc} -> anova_power(opts)
_ -> raise ArgumentError, "Unknown test type or analysis type"
end
end
@doc """
Calculate required sample size for t-test.
Based on Cohen (1988) power analysis formulas.
"""
def t_test_sample_size(opts) do
effect_size = Keyword.fetch!(opts, :effect_size)
alpha = Keyword.get(opts, :alpha, 0.05)
power = Keyword.get(opts, :power, 0.80)
alternative = Keyword.get(opts, :alternative, :two_sided)
# Critical values
z_alpha = critical_value_for_alpha(alpha, alternative)
z_beta = Distributions.normal_quantile(power)
# Sample size calculation
n_per_group = 2 * :math.pow((z_alpha + z_beta) / effect_size, 2)
n_per_group = ceil(n_per_group)
%{
analysis_type: :a_priori,
test: :t_test,
n_per_group: n_per_group,
total_n: n_per_group * 2,
effect_size: effect_size,
alpha: alpha,
power: power,
alternative: alternative,
recommendation:
"Collect at least #{n_per_group} samples per group (#{n_per_group * 2} total) " <>
"to detect an effect size of #{effect_size} with #{Float.round(power * 100, 1)}% power"
}
end
@doc """
Calculate achieved power for t-test.
"""
def t_test_power(opts) do
effect_size = Keyword.fetch!(opts, :effect_size)
n_per_group = Keyword.fetch!(opts, :n_per_group)
alpha = Keyword.get(opts, :alpha, 0.05)
alternative = Keyword.get(opts, :alternative, :two_sided)
# Non-centrality parameter
delta = effect_size * :math.sqrt(n_per_group / 2)
# Critical value
z_alpha = critical_value_for_alpha(alpha, alternative)
# Power calculation (using normal approximation)
power = 1 - Distributions.normal_cdf(z_alpha - delta)
recommendation =
cond do
power >= 0.8 ->
"Adequate power (#{Float.round(power * 100, 1)}%)"
power >= 0.6 ->
"Marginal power (#{Float.round(power * 100, 1)}%). Consider increasing sample size."
true ->
"Underpowered (#{Float.round(power * 100, 1)}%). Increase sample size significantly."
end
%{
analysis_type: :post_hoc,
test: :t_test,
power: power,
n_per_group: n_per_group,
total_n: n_per_group * 2,
effect_size: effect_size,
alpha: alpha,
alternative: alternative,
recommendation: recommendation
}
end
@doc """
Calculate required sample size for ANOVA.
"""
def anova_sample_size(opts) do
effect_size = Keyword.fetch!(opts, :effect_size)
# number of groups
k = Keyword.fetch!(opts, :k)
alpha = Keyword.get(opts, :alpha, 0.05)
power = Keyword.get(opts, :power, 0.80)
# Convert effect size to f (if given as eta-squared)
f = if effect_size < 1, do: :math.sqrt(effect_size / (1 - effect_size)), else: effect_size
# Critical values
z_alpha = Distributions.normal_quantile(1 - alpha)
z_beta = Distributions.normal_quantile(power)
# Sample size per group (simplified formula)
n_per_group = 1 + 2 * :math.pow((z_alpha + z_beta) / f, 2) / k
n_per_group = ceil(n_per_group)
%{
analysis_type: :a_priori,
test: :anova,
n_per_group: n_per_group,
total_n: n_per_group * k,
k: k,
effect_size: effect_size,
alpha: alpha,
power: power,
recommendation:
"Collect at least #{n_per_group} samples per group (#{n_per_group * k} total) " <>
"to detect an effect size of #{effect_size} with #{Float.round(power * 100, 1)}% power"
}
end
@doc """
Calculate achieved power for ANOVA.
"""
def anova_power(opts) do
effect_size = Keyword.fetch!(opts, :effect_size)
n_per_group = Keyword.fetch!(opts, :n_per_group)
k = Keyword.fetch!(opts, :k)
alpha = Keyword.get(opts, :alpha, 0.05)
# Convert effect size to f
f = if effect_size < 1, do: :math.sqrt(effect_size / (1 - effect_size)), else: effect_size
# Non-centrality parameter
lambda = n_per_group * k * f * f
# Critical F value (approximate)
df1 = k - 1
df2 = k * (n_per_group - 1)
# Power approximation (using normal approximation)
z_alpha = Distributions.normal_quantile(1 - alpha)
delta = :math.sqrt(lambda)
power = 1 - Distributions.normal_cdf(z_alpha - delta)
recommendation =
cond do
power >= 0.8 ->
"Adequate power (#{Float.round(power * 100, 1)}%)"
power >= 0.6 ->
"Marginal power (#{Float.round(power * 100, 1)}%). Consider increasing sample size."
true ->
"Underpowered (#{Float.round(power * 100, 1)}%). Increase sample size significantly."
end
%{
analysis_type: :post_hoc,
test: :anova,
power: power,
n_per_group: n_per_group,
total_n: n_per_group * k,
k: k,
effect_size: effect_size,
alpha: alpha,
df1: df1,
df2: df2,
recommendation: recommendation
}
end
defp critical_value_for_alpha(alpha, :two_sided) do
Distributions.normal_quantile(1 - alpha / 2)
end
defp critical_value_for_alpha(alpha, :less) do
Distributions.normal_quantile(alpha)
end
defp critical_value_for_alpha(alpha, :greater) do
Distributions.normal_quantile(1 - alpha)
end
end
</file>
<file path="crucible_bench/stats/t_test.ex">
defmodule CrucibleBench.Stats.TTest do
@moduledoc """
Independent samples t-test with Welch correction.
Compares means of two independent groups to determine if they
differ significantly.
"""
alias CrucibleBench.{Stats, Result}
alias CrucibleBench.Stats.Distributions
@doc """
Perform independent samples t-test.
Automatically uses Welch's t-test (does not assume equal variances)
unless explicitly specified otherwise.
## Options
- `:var_equal` - Assume equal variances (default: false)
- `:alternative` - :two_sided (default), :less, :greater
- `:confidence_level` - Confidence level for CI (default: 0.95)
## Examples
iex> group1 = [5.1, 4.9, 5.3, 5.0, 5.2]
iex> group2 = [6.2, 6.0, 6.4, 5.9, 6.1]
iex> result = CrucibleBench.Stats.TTest.test(group1, group2)
iex> result.p_value < 0.05
true
"""
def test(group1, group2, opts \\ []) do
n1 = length(group1)
n2 = length(group2)
mean1 = Stats.mean(group1)
mean2 = Stats.mean(group2)
var1 = Stats.variance(group1)
var2 = Stats.variance(group2)
var_equal = Keyword.get(opts, :var_equal, false)
{t_stat, df} =
if var_equal do
student_t(mean1, mean2, var1, var2, n1, n2)
else
welch_t(mean1, mean2, var1, var2, n1, n2)
end
alternative = Keyword.get(opts, :alternative, :two_sided)
p_value = p_value_from_t(t_stat, df, alternative)
# Confidence interval
conf_level = Keyword.get(opts, :confidence_level, 0.95)
ci = confidence_interval(mean1, mean2, var1, var2, n1, n2, df, conf_level, var_equal)
%Result{
test: if(var_equal, do: :student_t_test, else: :welch_t_test),
statistic: t_stat,
p_value: p_value,
confidence_interval: ci,
interpretation: interpret(p_value, t_stat),
metadata: %{
df: df,
mean1: mean1,
mean2: mean2,
mean_diff: mean1 - mean2,
alternative: alternative,
n1: n1,
n2: n2
}
}
end
defp student_t(mean1, mean2, var1, var2, n1, n2) do
# Pooled variance
pooled_var = ((n1 - 1) * var1 + (n2 - 1) * var2) / (n1 + n2 - 2)
# Standard error
se = :math.sqrt(pooled_var * (1 / n1 + 1 / n2))
# t-statistic
t = (mean1 - mean2) / se
df = n1 + n2 - 2
{t, df}
end
defp welch_t(mean1, mean2, var1, var2, n1, n2) do
# Standard error (no pooling)
se = :math.sqrt(var1 / n1 + var2 / n2)
# t-statistic
t = (mean1 - mean2) / se
# Welch-Satterthwaite degrees of freedom
numerator = :math.pow(var1 / n1 + var2 / n2, 2)
denominator = :math.pow(var1 / n1, 2) / (n1 - 1) + :math.pow(var2 / n2, 2) / (n2 - 1)
df = numerator / denominator
{t, df}
end
defp confidence_interval(mean1, mean2, var1, var2, n1, n2, df, conf_level, var_equal) do
mean_diff = mean1 - mean2
se =
if var_equal do
pooled_var = ((n1 - 1) * var1 + (n2 - 1) * var2) / (n1 + n2 - 2)
:math.sqrt(pooled_var * (1 / n1 + 1 / n2))
else
:math.sqrt(var1 / n1 + var2 / n2)
end
alpha = 1 - conf_level
t_critical = Distributions.t_quantile(df, 1 - alpha / 2)
margin = t_critical * se
{mean_diff - margin, mean_diff + margin}
end
defp p_value_from_t(t_stat, df, alternative) do
p_two_sided = 2 * (1 - Distributions.t_cdf(abs(t_stat), df))
case alternative do
:two_sided -> p_two_sided
:greater -> 1 - Distributions.t_cdf(t_stat, df)
:less -> Distributions.t_cdf(t_stat, df)
end
end
defp interpret(p_value, _t_stat) do
cond do
p_value < 0.001 -> "Highly significant difference between groups"
p_value < 0.01 -> "Very significant difference between groups"
p_value < 0.05 -> "Significant difference between groups"
true -> "No significant difference between groups"
end
end
end
</file>
<file path="crucible_bench/stats/wilcoxon.ex">
defmodule CrucibleBench.Stats.Wilcoxon do
@moduledoc """
Wilcoxon signed-rank test.
Non-parametric alternative to paired t-test.
Tests if the median of differences between paired samples is zero.
"""
alias CrucibleBench.Result
alias CrucibleBench.Stats.Distributions
@doc """
Perform Wilcoxon signed-rank test.
## Options
- `:alternative` - :two_sided (default), :less, :greater
## Examples
iex> before = [0.72, 0.68, 0.75, 0.71, 0.69]
iex> after = [0.78, 0.73, 0.81, 0.76, 0.74]
iex> result = CrucibleBench.Stats.Wilcoxon.test(before, after)
iex> result.p_value < 0.05
true
"""
def test(group1, group2, opts \\ []) do
unless length(group1) == length(group2) do
raise ArgumentError, "Wilcoxon test requires equal length groups"
end
# Calculate differences
differences = Enum.zip_with(group1, group2, fn x, y -> y - x end)
# Remove zeros
non_zero_diffs = Enum.filter(differences, fn d -> d != 0 end)
n = length(non_zero_diffs)
if n < 5 do
raise ArgumentError, "Wilcoxon test requires at least 5 non-zero differences"
end
# Rank absolute differences
abs_diffs = Enum.map(non_zero_diffs, &abs/1)
ranks = assign_ranks_with_ties(abs_diffs)
# Apply signs and sum
signed_ranks = Enum.zip(non_zero_diffs, ranks)
w_plus =
signed_ranks
|> Enum.filter(fn {diff, _} -> diff > 0 end)
|> Enum.map(fn {_, rank} -> rank end)
|> Enum.sum()
w_minus =
signed_ranks
|> Enum.filter(fn {diff, _} -> diff < 0 end)
|> Enum.map(fn {_, rank} -> rank end)
|> Enum.sum()
w_stat = min(w_plus, w_minus)
# For large samples (n > 25), use normal approximation
p_value =
if n > 25 do
mu = n * (n + 1) / 4
sigma = :math.sqrt(n * (n + 1) * (2 * n + 1) / 24)
z = (w_stat - mu) / sigma
alternative = Keyword.get(opts, :alternative, :two_sided)
case alternative do
:two_sided -> 2 * (1 - Distributions.normal_cdf(abs(z)))
:less -> Distributions.normal_cdf(z)
:greater -> 1 - Distributions.normal_cdf(z)
end
else
# For small samples, approximate p-value
# Would normally use exact distribution table
mu = n * (n + 1) / 4
sigma = :math.sqrt(n * (n + 1) * (2 * n + 1) / 24)
z = (w_stat - mu) / sigma
2 * (1 - Distributions.normal_cdf(abs(z)))
end
# Effect size (r = Z / sqrt(n))
effect_size =
if n > 25 do
mu = n * (n + 1) / 4
sigma = :math.sqrt(n * (n + 1) * (2 * n + 1) / 24)
z = (w_stat - mu) / sigma
abs(z) / :math.sqrt(n)
else
nil
end
%Result{
test: :wilcoxon_signed_rank,
statistic: w_stat,
p_value: p_value,
effect_size: if(effect_size, do: %{r: effect_size}, else: nil),
interpretation: interpret(p_value),
metadata: %{
w_plus: w_plus,
w_minus: w_minus,
n: n,
n_zero: length(differences) - n
}
}
end
defp assign_ranks_with_ties(values) do
indexed = Enum.with_index(values)
sorted = Enum.sort_by(indexed, fn {val, _idx} -> val end)
# Group by value to handle ties
grouped = Enum.chunk_by(sorted, fn {val, _idx} -> val end)
ranks =
Enum.reduce(grouped, {[], 1}, fn group, {acc, start_rank} ->
group_size = length(group)
avg_rank = (start_rank + start_rank + group_size - 1) / 2
ranks_for_group = Enum.map(group, fn {_val, idx} -> {idx, avg_rank} end)
{acc ++ ranks_for_group, start_rank + group_size}
end)
|> elem(0)
|> Enum.sort_by(fn {idx, _rank} -> idx end)
|> Enum.map(fn {_idx, rank} -> rank end)
ranks
end
defp interpret(p_value) do
cond do
p_value < 0.001 -> "Highly significant difference between paired groups"
p_value < 0.01 -> "Very significant difference between paired groups"
p_value < 0.05 -> "Significant difference between paired groups"
true -> "No significant difference between paired groups"
end
end
end
</file>
<file path="crucible_bench/analysis.ex">
defmodule CrucibleBench.Analysis do
@moduledoc """
High-level analysis functions with automatic test selection.
Provides smart defaults and automatic assumption checking.
"""
alias CrucibleBench.Stats
alias CrucibleBench.Stats.{
TTest,
PairedTTest,
ANOVA,
MannWhitney,
Wilcoxon,
KruskalWallis,
EffectSize
}
@doc """
Compare two independent groups with automatic test selection.
Automatically checks assumptions and selects appropriate test:
- Normal data + equal variance: Student's t-test
- Normal data + unequal variance: Welch's t-test (default)
- Non-normal data: Mann-Whitney U test
## Options
- `:test` - Force specific test (:t_test, :welch_t_test, :mann_whitney)
- `:confidence_level` - Confidence level for CI (default: 0.95)
- `:check_assumptions` - Test normality (default: true)
- `:alternative` - :two_sided (default), :less, :greater
"""
def compare_groups(group1, group2, opts \\ []) do
forced_test = Keyword.get(opts, :test)
check_assumptions = Keyword.get(opts, :check_assumptions, true)
test_to_use =
if forced_test do
forced_test
else
if check_assumptions and not normal_enough?(group1, group2) do
:mann_whitney
else
:welch_t_test
end
end
result =
case test_to_use do
:t_test -> TTest.test(group1, group2, Keyword.put(opts, :var_equal, true))
:welch_t_test -> TTest.test(group1, group2, Keyword.put(opts, :var_equal, false))
:mann_whitney -> MannWhitney.test(group1, group2, opts)
_ -> raise ArgumentError, "Unknown test: #{test_to_use}"
end
# Add effect size if not non-parametric
result =
if test_to_use in [:t_test, :welch_t_test] do
effect = EffectSize.cohens_d(group1, group2)
%{result | effect_size: effect}
else
result
end
result
end
@doc """
Compare paired groups with automatic test selection.
Selects paired t-test for normal differences, Wilcoxon for non-normal.
"""
def compare_paired(group1, group2, opts \\ []) do
forced_test = Keyword.get(opts, :test)
check_assumptions = Keyword.get(opts, :check_assumptions, true)
differences = Enum.zip_with(group1, group2, fn x, y -> y - x end)
test_to_use =
if forced_test do
forced_test
else
if check_assumptions and not normal_enough?(differences) do
:wilcoxon
else
:paired_t_test
end
end
result =
case test_to_use do
:paired_t_test -> PairedTTest.test(group1, group2, opts)
:wilcoxon -> Wilcoxon.test(group1, group2, opts)
_ -> raise ArgumentError, "Unknown test: #{test_to_use}"
end
# Add effect size for parametric test
result =
if test_to_use == :paired_t_test do
effect = EffectSize.paired_cohens_d(group1, group2)
%{result | effect_size: effect}
else
result
end
result
end
@doc """
Compare multiple groups with automatic test selection.
Selects ANOVA for normal data with equal variances,
Kruskal-Wallis for non-normal data.
"""
def compare_multiple(groups, opts \\ []) when is_list(groups) do
unless length(groups) >= 2 do
raise ArgumentError, "Need at least 2 groups"
end
forced_test = Keyword.get(opts, :test)
check_assumptions = Keyword.get(opts, :check_assumptions, true)
test_to_use =
if forced_test do
forced_test
else
if check_assumptions and not all_normal?(groups) do
:kruskal_wallis
else
:anova
end
end
case test_to_use do
:anova -> ANOVA.one_way(groups, opts)
:kruskal_wallis -> KruskalWallis.test(groups, opts)
_ -> raise ArgumentError, "Unknown test: #{test_to_use}"
end
end
# Simple normality check using skewness and kurtosis
defp normal_enough?(data) when is_list(data) do
n = length(data)
if n < 8 do
# Too small for reliable normality test, assume normal
true
else
skew = Stats.skewness(data)
kurt = Stats.kurtosis(data)
# Rough guidelines: |skew| < 2 and |kurt| < 7 suggests normality
skew_ok = skew == nil or abs(skew) < 2.0
kurt_ok = kurt == nil or abs(kurt) < 7.0
skew_ok and kurt_ok
end
end
defp normal_enough?(group1, group2) do
normal_enough?(group1) and normal_enough?(group2)
end
defp all_normal?(groups) do
Enum.all?(groups, &normal_enough?/1)
end
end
</file>
<file path="crucible_bench/experiment.ex">
defmodule CrucibleBench.Experiment do
@moduledoc """
High-level experiment DSL for common research patterns.
Provides convenient functions for A/B testing, ablation studies,
and hyperparameter sweeps.
"""
alias CrucibleBench.{Analysis, Stats}
alias CrucibleBench.Stats.{EffectSize, Power}
@doc """
Run an experiment with automatic analysis.
## Experiment Types
- `:ab_test` - Compare control vs treatment
- `:ablation` - Test impact of removing components
- `:hyperparameter_sweep` - Compare multiple configurations
## Examples
# A/B Test
CrucibleBench.Experiment.run(:ab_test,
control: [0.72, 0.68, 0.75],
treatment: [0.78, 0.73, 0.81],
name: "Prompt Engineering")
# Ablation Study
CrucibleBench.Experiment.run(:ablation,
baseline: [0.85, 0.87, 0.84],
without_component: [0.78, 0.76, 0.79],
component_name: "Ensemble Voting")
"""
def run(:ab_test, opts) do
control = Keyword.fetch!(opts, :control)
treatment = Keyword.fetch!(opts, :treatment)
name = Keyword.get(opts, :name, "A/B Test")
# Run test
result = Analysis.compare_groups(control, treatment)
# Calculate effect size
effect = EffectSize.cohens_d(control, treatment)
# Power analysis
power_result =
Power.analyze(:t_test,
analysis_type: :post_hoc,
effect_size: abs(effect.cohens_d),
n_per_group: length(control),
alpha: 0.05
)
# Compile comprehensive report
%{
experiment_type: :ab_test,
name: name,
significant?: result.p_value < 0.05,
p_value: result.p_value,
test_used: result.test,
effect_size: effect,
power: power_result.power,
sample_sizes: %{
control: length(control),
treatment: length(treatment)
},
means: %{
control: Stats.mean(control),
treatment: Stats.mean(treatment),
difference: Stats.mean(treatment) - Stats.mean(control)
},
confidence_interval: result.confidence_interval,
interpretation: interpret_ab_test(result, effect, power_result),
recommendation: generate_recommendation(result, effect, power_result),
raw_result: result
}
end
def run(:ablation, opts) do
baseline = Keyword.fetch!(opts, :baseline)
without_component = Keyword.fetch!(opts, :without_component)
component_name = Keyword.get(opts, :component_name, "Component")
# Run test (ablated version vs baseline)
result = Analysis.compare_groups(baseline, without_component)
# Calculate effect size
effect = EffectSize.cohens_d(baseline, without_component)
# Calculate performance drop
mean_baseline = Stats.mean(baseline)
mean_without = Stats.mean(without_component)
performance_drop = mean_baseline - mean_without
percent_drop = performance_drop / mean_baseline * 100
%{
experiment_type: :ablation,
component_name: component_name,
significant_impact?: result.p_value < 0.05,
p_value: result.p_value,
effect_size: effect,
performance_drop: %{
absolute: performance_drop,
percent: percent_drop
},
means: %{
baseline: mean_baseline,
without_component: mean_without
},
confidence_interval: result.confidence_interval,
interpretation: interpret_ablation(result, effect, component_name, percent_drop),
raw_result: result
}
end
def run(:hyperparameter_sweep, opts) do
configurations = Keyword.fetch!(opts, :configurations)
labels = Keyword.get(opts, :labels, Enum.map(1..length(configurations), &"Config #{&1}"))
unless length(configurations) >= 2 do
raise ArgumentError, "Need at least 2 configurations"
end
# Run omnibus test
omnibus_result = Analysis.compare_multiple(configurations)
# Calculate all pairwise comparisons
pairwise_results = calculate_pairwise_comparisons(configurations, labels)
# Find best configuration
means = Enum.map(configurations, &Stats.mean/1)
{best_mean, best_idx} = Enum.with_index(means) |> Enum.max_by(fn {m, _} -> m end)
best_config = Enum.at(labels, best_idx)
%{
experiment_type: :hyperparameter_sweep,
configurations_tested: length(configurations),
omnibus_test: %{
significant?: omnibus_result.p_value < 0.05,
p_value: omnibus_result.p_value,
test_used: omnibus_result.test,
effect_size: omnibus_result.effect_size
},
best_configuration: %{
name: best_config,
mean: best_mean,
rank: 1
},
configuration_means: Enum.zip(labels, means) |> Map.new(),
pairwise_comparisons: pairwise_results,
interpretation: interpret_sweep(omnibus_result, best_config, best_mean),
raw_result: omnibus_result
}
end
defp calculate_pairwise_comparisons(configurations, labels) do
for i <- 0..(length(configurations) - 2),
j <- (i + 1)..(length(configurations) - 1) do
config_i = Enum.at(configurations, i)
config_j = Enum.at(configurations, j)
label_i = Enum.at(labels, i)
label_j = Enum.at(labels, j)
result = Analysis.compare_groups(config_i, config_j)
effect = EffectSize.cohens_d(config_i, config_j)
%{
comparison: "#{label_i} vs #{label_j}",
significant?: result.p_value < 0.05,
p_value: result.p_value,
effect_size: effect.cohens_d,
mean_diff: Stats.mean(config_i) - Stats.mean(config_j)
}
end
end
defp interpret_ab_test(result, effect, power_result) do
sig_text = if result.p_value < 0.05, do: "significant", else: "not significant"
effect_text = effect.interpretation
power_text = if power_result.power >= 0.8, do: "adequate", else: "insufficient"
"""
The treatment group showed a #{sig_text} difference from control (p = #{Float.round(result.p_value, 4)}).
Effect size: #{effect_text} (Cohen's d = #{Float.round(effect.cohens_d, 3)}).
Statistical power: #{power_text} (#{Float.round(power_result.power * 100, 1)}%).
"""
|> String.trim()
end
defp interpret_ablation(result, effect, component_name, percent_drop) do
if result.p_value < 0.05 do
direction = if percent_drop > 0, do: "decreased", else: "increased"
"""
Removing #{component_name} significantly #{direction} performance by #{abs(Float.round(percent_drop, 2))}%.
Effect size: #{effect.interpretation} (Cohen's d = #{Float.round(effect.cohens_d, 3)}).
This component appears to be #{if percent_drop > 0, do: "important", else: "detrimental"} to the system.
"""
else
"""
Removing #{component_name} did not significantly affect performance (p = #{Float.round(result.p_value, 4)}).
This component may be redundant or its effect is too small to detect with current sample size.
"""
end
|> String.trim()
end
defp interpret_sweep(omnibus_result, best_config, best_mean) do
if omnibus_result.p_value < 0.05 do
"""
Significant differences were found between configurations (p = #{Float.round(omnibus_result.p_value, 4)}).
Best performing configuration: #{best_config} (mean = #{Float.round(best_mean, 4)}).
Effect size: #{omnibus_result.effect_size.interpretation}.
"""
else
"""
No significant differences were found between configurations (p = #{Float.round(omnibus_result.p_value, 4)}).
All configurations perform similarly. #{best_config} had the highest mean (#{Float.round(best_mean, 4)}),
but this difference is not statistically significant.
"""
end
|> String.trim()
end
defp generate_recommendation(result, effect, power_result) do
cond do
result.p_value >= 0.05 and power_result.power < 0.8 ->
"Increase sample size to achieve 80% power. Current power is only #{Float.round(power_result.power * 100, 1)}%."
result.p_value < 0.05 and abs(effect.cohens_d) < 0.2 ->
"While statistically significant, the effect size is negligible. Consider practical significance."
result.p_value < 0.05 and abs(effect.cohens_d) >= 0.5 ->
"Both statistically and practically significant. Treatment shows clear improvement."
true ->
"Results are inconclusive. Consider collecting more data or adjusting the intervention."
end
end
end
</file>
<file path="crucible_bench/export.ex">
defmodule CrucibleBench.Export do
@moduledoc """
Export statistical results to various formats.
Supports Markdown, LaTeX, and HTML output for publication.
"""
alias CrucibleBench.Result
@doc """
Export result to Markdown format.
## Examples
iex> result = %CrucibleBench.Result{
...> test: :welch_t_test,
...> statistic: 5.477,
...> p_value: 0.001307,
...> confidence_interval: {0.5, 1.5}
...> }
iex> markdown = CrucibleBench.Export.to_markdown(result)
iex> String.contains?(markdown, "Welch's t-test")
true
"""
def to_markdown(%Result{} = result) do
"""
## Statistical Test Results
**Test**: #{format_test_name(result.test)}
**Test Statistic**: #{format_number(result.statistic)}
**P-value**: #{format_p_value(result.p_value)}
**Significance**: #{if result.p_value < 0.05, do: "Yes (p < 0.05)", else: "No (p ≥ 0.05)"}
#{format_effect_size_md(result.effect_size)}
#{format_confidence_interval_md(result.confidence_interval)}
### Interpretation
#{result.interpretation || "No interpretation available"}
#{format_metadata_md(result.metadata)}
"""
|> String.trim()
end
@doc """
Export result to LaTeX format.
Generates LaTeX table suitable for academic papers.
"""
def to_latex(%Result{} = result) do
"""
\\begin{table}[h]
\\centering
\\begin{tabular}{ll}
\\hline
\\textbf{Test} & #{format_test_name(result.test)} \\\\
\\textbf{Test Statistic} & #{format_number(result.statistic)} \\\\
\\textbf{P-value} & #{format_p_value_latex(result.p_value)} \\\\
#{format_effect_size_latex(result.effect_size)}
#{format_confidence_interval_latex(result.confidence_interval)}
\\hline
\\end{tabular}
\\caption{Statistical test results}
\\label{tab:results}
\\end{table}
"""
|> String.trim()
end
@doc """
Export result to HTML format.
Generates styled HTML suitable for interactive reports.
"""
def to_html(%Result{} = result) do
"""
<div class="statistical-results">
<h2>Statistical Test Results</h2>
<table class="results-table">
<tr>
<th>Test</th>
<td>#{format_test_name(result.test)}</td>
</tr>
<tr>
<th>Test Statistic</th>
<td>#{format_number(result.statistic)}</td>
</tr>
<tr>
<th>P-value</th>
<td class="#{if result.p_value < 0.05, do: "significant", else: "not-significant"}">
#{format_p_value(result.p_value)}
</td>
</tr>
#{format_effect_size_html(result.effect_size)}
#{format_confidence_interval_html(result.confidence_interval)}
</table>
<div class="interpretation">
<h3>Interpretation</h3>
<p>#{result.interpretation || "No interpretation available"}</p>
</div>
#{format_metadata_html(result.metadata)}
</div>
<style>
.statistical-results { font-family: Arial, sans-serif; max-width: 800px; margin: 20px; }
.results-table { width: 100%; border-collapse: collapse; margin: 20px 0; }
.results-table th { text-align: left; padding: 8px; background-color: #f0f0f0; border: 1px solid #ddd; }
.results-table td { padding: 8px; border: 1px solid #ddd; }
.significant { color: #d9534f; font-weight: bold; }
.not-significant { color: #5bc0de; }
.interpretation { margin-top: 20px; padding: 15px; background-color: #f9f9f9; border-left: 4px solid #5bc0de; }
.metadata { margin-top: 20px; font-size: 0.9em; color: #666; }
</style>
"""
|> String.trim()
end
@doc """
Export experiment results to comprehensive report.
"""
def experiment_to_markdown(experiment_result) do
case experiment_result.experiment_type do
:ab_test -> ab_test_report(experiment_result)
:ablation -> ablation_report(experiment_result)
:hyperparameter_sweep -> sweep_report(experiment_result)
end
end
defp ab_test_report(exp) do
"""
# A/B Test Report: #{exp.name}
## Summary
#{if exp.significant?, do: "✓", else: "✗"} **Result**: #{if exp.significant?, do: "Significant difference detected", else: "No significant difference"}
## Statistics
- **P-value**: #{format_p_value(exp.p_value)}
- **Test Used**: #{format_test_name(exp.test_used)}
- **Effect Size**: #{exp.effect_size.interpretation} (Cohen's d = #{format_number(exp.effect_size.cohens_d)})
- **Statistical Power**: #{Float.round(exp.power * 100, 1)}%
## Sample Sizes
- Control: #{exp.sample_sizes.control}
- Treatment: #{exp.sample_sizes.treatment}
## Performance Metrics
| Group | Mean | Difference |
|-------|------|------------|
| Control | #{format_number(exp.means.control)} | - |
| Treatment | #{format_number(exp.means.treatment)} | #{format_diff(exp.means.difference)} |
## Confidence Interval (95%)
#{format_ci_range(exp.confidence_interval)}
## Interpretation
#{exp.interpretation}
## Recommendation
#{exp.recommendation}
"""
|> String.trim()
end
defp ablation_report(exp) do
"""
# Ablation Study: #{exp.component_name}
## Summary
#{if exp.significant_impact?, do: "✓ Significant impact", else: "✗ No significant impact"}
## Statistics
- **P-value**: #{format_p_value(exp.p_value)}
- **Effect Size**: #{exp.effect_size.interpretation} (Cohen's d = #{format_number(exp.effect_size.cohens_d)})
## Performance Impact
- **Absolute Drop**: #{format_number(exp.performance_drop.absolute)}
- **Percent Drop**: #{format_number(exp.performance_drop.percent)}%
## Performance Comparison
| Configuration | Mean Performance |
|--------------|------------------|
| Baseline (with component) | #{format_number(exp.means.baseline)} |
| Without component | #{format_number(exp.means.without_component)} |
## Interpretation
#{exp.interpretation}
"""
|> String.trim()
end
defp sweep_report(exp) do
"""
# Hyperparameter Sweep Results
## Summary
**Configurations Tested**: #{exp.configurations_tested}
**Best Configuration**: #{exp.best_configuration.name} (mean = #{format_number(exp.best_configuration.mean)})
## Omnibus Test
- **Significant Differences**: #{if exp.omnibus_test.significant?, do: "Yes", else: "No"}
- **P-value**: #{format_p_value(exp.omnibus_test.p_value)}
- **Test Used**: #{format_test_name(exp.omnibus_test.test_used)}
- **Effect Size**: #{exp.omnibus_test.effect_size.interpretation}
## Configuration Means
| Configuration | Mean Performance |
|--------------|------------------|
#{format_config_means(exp.configuration_means)}
## Pairwise Comparisons
#{format_pairwise_comparisons(exp.pairwise_comparisons)}
## Interpretation
#{exp.interpretation}
"""
|> String.trim()
end
# Formatting helpers
defp format_test_name(:welch_t_test), do: "Welch's t-test"
defp format_test_name(:student_t_test), do: "Student's t-test"
defp format_test_name(:paired_t_test), do: "Paired t-test"
defp format_test_name(:anova), do: "One-way ANOVA"
defp format_test_name(:mann_whitney), do: "Mann-Whitney U test"
defp format_test_name(:wilcoxon_signed_rank), do: "Wilcoxon signed-rank test"
defp format_test_name(:kruskal_wallis), do: "Kruskal-Wallis test"
defp format_test_name(other), do: to_string(other)
defp format_number(nil), do: "N/A"
defp format_number(n) when is_float(n), do: Float.round(n, 4) |> to_string()
defp format_number(n), do: to_string(n)
defp format_p_value(p) when p < 0.001, do: "< 0.001"
defp format_p_value(p), do: format_number(p)
defp format_p_value_latex(p) when p < 0.001, do: "$< 0.001$"
defp format_p_value_latex(p), do: "$#{format_number(p)}$"
defp format_diff(d) when d > 0, do: "+#{format_number(d)}"
defp format_diff(d), do: format_number(d)
defp format_ci_range({lower, upper}) do
"[#{format_number(lower)}, #{format_number(upper)}]"
end
defp format_ci_range(nil), do: "N/A"
defp format_effect_size_md(nil), do: ""
defp format_effect_size_md(effect) when is_map(effect) do
"**Effect Size**: #{format_effect_size_value(effect)}\n"
end
defp format_effect_size_value(%{cohens_d: d}), do: "Cohen's d = #{format_number(d)}"
defp format_effect_size_value(%{eta_squared: eta}), do: "η² = #{format_number(eta)}"
defp format_effect_size_value(%{rank_biserial: r}), do: "Rank-biserial r = #{format_number(r)}"
defp format_effect_size_value(_), do: "N/A"
defp format_confidence_interval_md(nil), do: ""
defp format_confidence_interval_md(ci) do
"**95% Confidence Interval**: #{format_ci_range(ci)}\n"
end
defp format_effect_size_latex(nil), do: ""
defp format_effect_size_latex(effect) when is_map(effect) do
"\\textbf{Effect Size} & #{format_effect_size_value(effect)} \\\\\n"
end
defp format_confidence_interval_latex(nil), do: ""
defp format_confidence_interval_latex(ci) do
"\\textbf{95\\% CI} & #{format_ci_range(ci)} \\\\\n"
end
defp format_effect_size_html(nil), do: ""
defp format_effect_size_html(effect) when is_map(effect) do
"""
<tr>
<th>Effect Size</th>
<td>#{format_effect_size_value(effect)}</td>
</tr>
"""
end
defp format_confidence_interval_html(nil), do: ""
defp format_confidence_interval_html(ci) do
"""
<tr>
<th>95% Confidence Interval</th>
<td>#{format_ci_range(ci)}</td>
</tr>
"""
end
defp format_metadata_md(metadata) when map_size(metadata) == 0, do: ""
defp format_metadata_md(metadata) do
"""
### Additional Details
#{Enum.map_join(metadata, "\n", fn {k, v} -> "- **#{k}**: #{format_number(v)}" end)}
"""
end
defp format_metadata_html(metadata) when map_size(metadata) == 0, do: ""
defp format_metadata_html(metadata) do
"""
<div class="metadata">
<h4>Additional Details</h4>
<ul>
#{Enum.map_join(metadata, "\n", fn {k, v} -> "<li><strong>#{k}</strong>: #{format_number(v)}</li>" end)}
</ul>
</div>
"""
end
defp format_config_means(means) do
means
|> Enum.map(fn {name, mean} -> "| #{name} | #{format_number(mean)} |" end)
|> Enum.join("\n")
end
defp format_pairwise_comparisons(comparisons) do
"""
| Comparison | P-value | Significant | Effect Size | Mean Diff |
|-----------|---------|-------------|-------------|-----------|
#{Enum.map_join(comparisons, "\n", &format_pairwise_row/1)}
"""
end
defp format_pairwise_row(comp) do
sig = if comp.significant?, do: "Yes", else: "No"
"| #{comp.comparison} | #{format_p_value(comp.p_value)} | #{sig} | #{format_number(comp.effect_size)} | #{format_diff(comp.mean_diff)} |"
end
end
</file>
<file path="crucible_bench/result.ex">
defmodule CrucibleBench.Result do
@moduledoc """
Standard result structure for statistical tests.
All statistical tests in Bench return a `CrucibleBench.Result` struct
containing test statistics, p-values, effect sizes, and interpretations.
"""
@type t :: %__MODULE__{
test: atom(),
statistic: float(),
p_value: float(),
effect_size: map() | nil,
confidence_interval: {float(), float()} | nil,
interpretation: String.t() | nil,
metadata: map()
}
defstruct [
:test,
:statistic,
:p_value,
:effect_size,
:confidence_interval,
:interpretation,
metadata: %{}
]
@doc """
Determine if result is statistically significant at given alpha level.
## Examples
iex> result = %CrucibleBench.Result{p_value: 0.03}
iex> CrucibleBench.Result.significant?(result, 0.05)
true
iex> result = %CrucibleBench.Result{p_value: 0.08}
iex> CrucibleBench.Result.significant?(result, 0.05)
false
"""
def significant?(%__MODULE__{p_value: p_value}, alpha \\ 0.05) do
p_value < alpha
end
@doc """
Generate human-readable summary of result.
"""
def summarize(%__MODULE__{} = result) do
sig_status = if significant?(result), do: "significant", else: "not significant"
effect_summary =
if result.effect_size do
"\nEffect size: #{format_effect_size(result.effect_size)}"
else
""
end
ci_summary =
if result.confidence_interval do
{lower, upper} = result.confidence_interval
"\n95% CI: [#{Float.round(lower, 4)}, #{Float.round(upper, 4)}]"
else
""
end
"""
Test: #{result.test}
Statistic: #{Float.round(result.statistic, 4)}
P-value: #{format_p_value(result.p_value)}
Result: #{sig_status}#{effect_summary}#{ci_summary}
"""
|> String.trim()
end
defp format_p_value(p) when p < 0.001, do: "< 0.001"
defp format_p_value(p), do: Float.round(p, 4) |> to_string()
defp format_effect_size(%{cohens_d: d}), do: "Cohen's d = #{Float.round(d, 3)}"
defp format_effect_size(%{eta_squared: eta}), do: "η² = #{Float.round(eta, 3)}"
defp format_effect_size(%{rank_biserial: r}), do: "r = #{Float.round(r, 3)}"
defp format_effect_size(_), do: "N/A"
end
</file>
<file path="crucible_bench/stats.ex">
defmodule CrucibleBench.Stats do
@moduledoc """
Core statistical functions and utilities.
Provides basic statistical calculations used throughout the framework.
"""
@doc """
Calculate mean (average) of a list of numbers.
## Examples
iex> CrucibleBench.Stats.mean([1, 2, 3, 4, 5])
3.0
"""
def mean([]), do: nil
def mean(values) when is_list(values) do
Enum.sum(values) / length(values)
end
@doc """
Calculate median of a list of numbers.
## Examples
iex> CrucibleBench.Stats.median([1, 2, 3, 4, 5])
3.0
iex> CrucibleBench.Stats.median([1, 2, 3, 4])
2.5
"""
def median([]), do: nil
def median(values) when is_list(values) do
sorted = Enum.sort(values)
n = length(sorted)
mid = div(n, 2)
if rem(n, 2) == 1 do
Enum.at(sorted, mid)
else
(Enum.at(sorted, mid - 1) + Enum.at(sorted, mid)) / 2
end
end
@doc """
Calculate variance of a list of numbers.
## Options
- `:sample` - If true (default), uses n-1 denominator (sample variance)
- `:population` - If true, uses n denominator (population variance)
## Examples
iex> CrucibleBench.Stats.variance([1, 2, 3, 4, 5])
2.5
"""
def variance(values, opts \\ [])
def variance([], _opts), do: nil
def variance([_single], _opts), do: 0.0
def variance(values, opts) when is_list(values) do
m = mean(values)
n = length(values)
sum_sq_diff =
Enum.reduce(values, 0, fn x, acc ->
acc + :math.pow(x - m, 2)
end)
denominator = if Keyword.get(opts, :population, false), do: n, else: n - 1
sum_sq_diff / denominator
end
@doc """
Calculate standard deviation of a list of numbers.
## Examples
iex> CrucibleBench.Stats.stdev([1, 2, 3, 4, 5])
1.5811388300841898
"""
def stdev(values, opts \\ []) do
case variance(values, opts) do
nil -> nil
var -> :math.sqrt(var)
end
end
@doc """
Calculate standard error of the mean.
## Examples
iex> CrucibleBench.Stats.sem([1, 2, 3, 4, 5])
0.7071067811865476
"""
def sem([]), do: nil
def sem(values) when is_list(values) do
sd = stdev(values)
n = length(values)
sd / :math.sqrt(n)
end
@doc """
Calculate quantile at given probability.
## Examples
iex> CrucibleBench.Stats.quantile([1, 2, 3, 4, 5], 0.5)
3.0
"""
def quantile([], _p), do: nil
def quantile(values, p) when is_list(values) and p >= 0 and p <= 1 do
sorted = Enum.sort(values)
n = length(sorted)
index = p * (n - 1)
lower_idx = floor(index)
upper_idx = ceil(index)
if lower_idx == upper_idx do
Enum.at(sorted, round(index))
else
lower_val = Enum.at(sorted, lower_idx)
upper_val = Enum.at(sorted, upper_idx)
fraction = index - lower_idx
lower_val + fraction * (upper_val - lower_val)
end
end
@doc """
Calculate z-score for each value in a list.
## Examples
iex> CrucibleBench.Stats.z_scores([1, 2, 3, 4, 5])
[-1.2649110640673518, -0.6324555320336759, 0.0, 0.6324555320336759, 1.2649110640673518]
"""
def z_scores([]), do: []
def z_scores(values) when is_list(values) do
m = mean(values)
sd = stdev(values)
Enum.map(values, fn x -> (x - m) / sd end)
end
@doc """
Calculate skewness of a distribution.
Positive skew means right tail is longer.
Negative skew means left tail is longer.
"""
def skewness([]), do: nil
def skewness(values) when is_list(values) and length(values) < 3, do: nil
def skewness(values) when is_list(values) do
n = length(values)
m = mean(values)
sd = stdev(values)
sum_cubed =
Enum.reduce(values, 0, fn x, acc ->
acc + :math.pow((x - m) / sd, 3)
end)
n / ((n - 1) * (n - 2)) * sum_cubed
end
@doc """
Calculate kurtosis of a distribution.
Excess kurtosis > 0 means heavy tails (leptokurtic).
Excess kurtosis < 0 means light tails (platykurtic).
"""
def kurtosis([]), do: nil
def kurtosis(values) when is_list(values) and length(values) < 4, do: nil
def kurtosis(values) when is_list(values) do
n = length(values)
m = mean(values)
sd = stdev(values)
sum_fourth =
Enum.reduce(values, 0, fn x, acc ->
acc + :math.pow((x - m) / sd, 4)
end)
numerator = n * (n + 1) * sum_fourth
denominator = (n - 1) * (n - 2) * (n - 3)
correction = 3 * :math.pow(n - 1, 2) / ((n - 2) * (n - 3))
numerator / denominator - correction
end
@doc """
Calculate correlation coefficient between two variables.
Returns Pearson correlation coefficient (-1 to 1).
## Examples
iex> x = [1, 2, 3, 4, 5]
iex> y = [2, 4, 6, 8, 10]
iex> CrucibleBench.Stats.correlation(x, y)
1.0
"""
def correlation(x, y) when length(x) != length(y) do
raise ArgumentError, "Lists must have equal length"
end
def correlation([], []), do: nil
def correlation(x, y) when is_list(x) and is_list(y) do
mean_x = mean(x)
mean_y = mean(y)
numerator =
Enum.zip(x, y)
|> Enum.reduce(0, fn {xi, yi}, acc ->
acc + (xi - mean_x) * (yi - mean_y)
end)
sum_sq_x =
Enum.reduce(x, 0, fn xi, acc ->
acc + :math.pow(xi - mean_x, 2)
end)
sum_sq_y =
Enum.reduce(y, 0, fn yi, acc ->
acc + :math.pow(yi - mean_y, 2)
end)
denominator = :math.sqrt(sum_sq_x * sum_sq_y)
if denominator == 0 do
0.0
else
numerator / denominator
end
end
@doc """
Rank values in ascending order, handling ties by averaging.
## Examples
iex> CrucibleBench.Stats.rank([5, 2, 8, 2, 9])
[3.0, 1.5, 4.0, 1.5, 5.0]
"""
def rank([]), do: []
def rank(values) when is_list(values) do
indexed = Enum.with_index(values)
sorted = Enum.sort_by(indexed, fn {val, _idx} -> val end)
# Group by value to handle ties
grouped = Enum.chunk_by(sorted, fn {val, _idx} -> val end)
ranks =
Enum.reduce(grouped, {[], 1}, fn group, {acc, start_rank} ->
group_size = length(group)
# Average rank for ties
avg_rank = (start_rank + start_rank + group_size - 1) / 2
ranks_for_group = Enum.map(group, fn {_val, idx} -> {idx, avg_rank} end)
{acc ++ ranks_for_group, start_rank + group_size}
end)
|> elem(0)
|> Enum.sort_by(fn {idx, _rank} -> idx end)
|> Enum.map(fn {_idx, rank} -> rank end)
ranks
end
end
</file>
<file path="bench.ex">
defmodule CrucibleBench do
@moduledoc """
Bench - Statistical Testing Framework for AI Research
A comprehensive statistical testing framework designed specifically for AI/ML research.
Provides rigorous statistical tests, effect size measures, power analysis, and
publication-ready reporting.
## Core Capabilities
- **Parametric Tests**: t-tests, ANOVA
- **Non-Parametric Tests**: Mann-Whitney U, Wilcoxon signed-rank, Kruskal-Wallis
- **Effect Sizes**: Cohen's d, Hedges' g, eta-squared, omega-squared
- **Power Analysis**: A priori and post-hoc power calculations
- **Confidence Intervals**: Bootstrap and analytical methods
- **Multiple Comparison Correction**: Bonferroni, Holm, Benjamini-Hochberg
## Quick Start
# Compare two groups
control = [0.72, 0.68, 0.75, 0.71, 0.69]
treatment = [0.78, 0.73, 0.81, 0.76, 0.74]
CrucibleBench.compare(control, treatment)
## Design Principles
1. **Statistical Rigor**: All implementations validated against R/SciPy
2. **Interpretability**: Every result includes effect sizes
3. **Reproducibility**: Complete audit trails
4. **Peer-Review Ready**: Publication-quality output
"""
alias CrucibleBench.Stats
alias CrucibleBench.Analysis
alias CrucibleBench.Experiment
@doc """
Compare two independent groups with automatic test selection.
Automatically selects appropriate test based on data characteristics and
assumption checking. Returns comprehensive results including p-value,
effect size, confidence interval, and interpretation.
## Options
- `:test` - Force specific test (:t_test, :welch_t_test, :mann_whitney)
- `:confidence_level` - Confidence level for CI (default: 0.95)
- `:check_assumptions` - Test normality and variance (default: true)
- `:alternative` - :two_sided (default), :less, :greater
## Examples
iex> control = [5.1, 4.9, 5.3, 5.0, 5.2]
iex> treatment = [6.2, 6.0, 6.4, 5.9, 6.1]
iex> result = CrucibleBench.compare(control, treatment)
iex> result.p_value < 0.05
true
## Returns
A `CrucibleBench.Result` struct containing:
- `test`: Test type used
- `statistic`: Test statistic value
- `p_value`: P-value
- `effect_size`: Effect size measure
- `confidence_interval`: CI for mean difference
- `interpretation`: Human-readable interpretation
"""
def compare(group1, group2, opts \\ []) do
Analysis.compare_groups(group1, group2, opts)
end
@doc """
Perform paired comparison between related groups.
Use when samples are paired (e.g., before/after measurements on same subjects).
## Examples
iex> before = [0.72, 0.68, 0.75, 0.71, 0.69]
iex> after_values = [0.78, 0.73, 0.81, 0.76, 0.74]
iex> result = CrucibleBench.compare_paired(before, after_values)
iex> result.effect_size.mean_diff > 0
true
"""
def compare_paired(group1, group2, opts \\ []) do
Analysis.compare_paired(group1, group2, opts)
end
@doc """
Compare multiple groups (3+) with ANOVA or Kruskal-Wallis.
Automatically selects parametric (ANOVA) or non-parametric (Kruskal-Wallis)
test based on assumption checking.
## Examples
iex> gpt4 = [0.89, 0.91, 0.88, 0.90, 0.92]
iex> claude = [0.87, 0.89, 0.86, 0.88, 0.90]
iex> gemini = [0.84, 0.86, 0.83, 0.85, 0.87]
iex> result = CrucibleBench.compare_multiple([gpt4, claude, gemini])
iex> result.test in [:anova, :kruskal_wallis]
true
"""
def compare_multiple(groups, opts \\ []) do
Analysis.compare_multiple(groups, opts)
end
@doc """
Calculate effect size between two groups.
Returns Cohen's d or appropriate effect size measure.
## Examples
iex> group1 = [5.0, 5.2, 4.8, 5.1, 4.9]
iex> group2 = [6.0, 6.2, 5.8, 6.1, 5.9]
iex> effect = CrucibleBench.effect_size(group1, group2)
iex> effect.cohens_d < 0
true
"""
def effect_size(group1, group2, opts \\ []) do
Stats.EffectSize.calculate(group1, group2, opts)
end
@doc """
Calculate confidence interval for a statistic.
Supports both analytical and bootstrap methods.
## Options
- `:method` - :analytical (default) or :bootstrap
- `:confidence_level` - Confidence level (default: 0.95)
- `:iterations` - Bootstrap iterations (default: 10000)
## Examples
iex> data = [5.0, 5.2, 4.8, 5.1, 4.9, 5.3]
iex> ci = CrucibleBench.confidence_interval(data, :mean)
iex> {lower, upper} = ci.interval
iex> lower < 5.05 and upper > 5.05
true
"""
def confidence_interval(data, statistic, opts \\ []) do
Stats.ConfidenceInterval.calculate(data, statistic, opts)
end
@doc """
Perform power analysis for a test.
Calculate required sample size or achieved power.
## Options
- `:analysis_type` - :a_priori (sample size) or :post_hoc (power)
- `:effect_size` - Expected or observed effect size
- `:alpha` - Significance level (default: 0.05)
- `:power` - Desired power (default: 0.80)
## Examples
iex> # Calculate required sample size
iex> result = CrucibleBench.power_analysis(:t_test,
...> effect_size: 0.5, alpha: 0.05, power: 0.80)
iex> result.n_per_group > 0
true
"""
def power_analysis(test_type, opts \\ []) do
Stats.Power.analyze(test_type, opts)
end
@doc """
Run an experiment with automatic analysis.
High-level DSL for common experiment patterns.
## Experiment Types
- `:ab_test` - A/B testing
- `:ablation` - Ablation study
- `:hyperparameter_sweep` - Hyperparameter optimization
## Examples
iex> control = [0.72, 0.68, 0.75, 0.71, 0.69]
iex> treatment = [0.78, 0.73, 0.81, 0.76, 0.74]
iex> result = CrucibleBench.experiment(:ab_test,
...> control: control, treatment: treatment,
...> name: "Prompt Engineering Test")
iex> result.significant?
true
"""
def experiment(type, opts \\ []) do
Experiment.run(type, opts)
end
end
</file>
</files>