Current section
Files
Jump to
Current section
Files
src/evaluation/adk_eval_report.erl
%% @doc Deterministic evaluation reports and baseline regression comparison.
-module(adk_eval_report).
-export([compare/3, render/2, validate_comparison/1]).
-define(REPORT_VERSION, 1).
-spec compare(map(), map(), map()) -> {ok, map()} | {error, term()}.
compare(Baseline0, Current0, Opts) when is_map(Opts) ->
case {adk_eval_set:encode_result(Baseline0),
adk_eval_set:encode_result(Current0),
validate_options(Opts)} of
{{ok, Baseline}, {ok, Current}, {ok, ValidOpts}} ->
compare_valid(Baseline, Current, ValidOpts);
{{error, Reason}, _, _} ->
{error, {invalid_baseline, reason_tag(Reason)}};
{_, {error, Reason}, _} ->
{error, {invalid_current_result, reason_tag(Reason)}};
{_, _, {error, _} = Error} -> Error
end;
compare(_, _, _) ->
{error, invalid_eval_comparison_arguments}.
-spec render(map(), json | markdown) -> {ok, binary()} | {error, term()}.
render(Value, json) when is_map(Value) ->
case checked_report(Value) of
{ok, Safe} ->
try jsx:encode(Safe) of
Encoded when is_binary(Encoded) -> {ok, Encoded}
catch
_:_ -> {error, invalid_eval_report}
end;
{error, _} = Error -> Error
end;
render(Value, markdown) when is_map(Value) ->
case checked_report(Value) of
{ok, Safe} -> {ok, markdown(Safe)};
{error, _} = Error -> Error
end;
render(_, _) ->
{error, invalid_eval_report_format}.
%% @doc Validate the canonical baseline-comparison schema without rendering.
%%
%% This is shared by the public report API and the developer UI boundary so a
%% malformed nested diff is rejected before any formatter can dereference it.
-spec validate_comparison(map()) -> ok | {error, term()}.
validate_comparison(Value) when is_map(Value) ->
case checked_comparison(Value) of
{ok, _Safe} -> ok;
{error, _} = Error -> Error
end;
validate_comparison(_) ->
{error, invalid_eval_comparison}.
compare_valid(Baseline, Current, Opts) ->
BaselineId = maps:get(<<"eval_set_id">>, Baseline),
CurrentId = maps:get(<<"eval_set_id">>, Current),
case BaselineId =:= CurrentId of
false -> {error, eval_set_mismatch};
true ->
BaselineRate = number(maps:get(<<"pass_rate">>, Baseline, 0.0)),
CurrentRate = number(maps:get(<<"pass_rate">>, Current, 0.0)),
RateDrop = erlang:max(0.0, BaselineRate - CurrentRate),
MetricDiffs = metric_diffs(
maps:get(<<"metrics">>, Baseline, []),
maps:get(<<"metrics">>, Current, []), Opts),
CaseDiffs = case_diffs(
maps:get(<<"cases">>, Baseline, []),
maps:get(<<"cases">>, Current, [])),
RateRegression =
RateDrop > maps:get(max_pass_rate_drop, Opts),
MetricRegression =
lists:any(fun(Diff) ->
maps:get(<<"regression">>, Diff)
end, MetricDiffs),
CaseRegression =
lists:any(fun(Diff) ->
maps:get(<<"regression">>, Diff)
end, CaseDiffs),
Passed = not (RateRegression orelse MetricRegression
orelse CaseRegression),
Report = #{
<<"report_schema_version">> => ?REPORT_VERSION,
<<"report_type">> => <<"baseline_comparison">>,
<<"eval_set_id">> => BaselineId,
<<"baseline_version">> =>
maps:get(<<"eval_set_version">>, Baseline),
<<"current_version">> =>
maps:get(<<"eval_set_version">>, Current),
<<"passed">> => Passed,
<<"baseline_pass_rate">> => BaselineRate,
<<"current_pass_rate">> => CurrentRate,
<<"pass_rate_drop">> => RateDrop,
<<"max_pass_rate_drop">> =>
maps:get(max_pass_rate_drop, Opts),
<<"metric_diffs">> => MetricDiffs,
<<"case_diffs">> => CaseDiffs
},
checked_comparison(Report)
end.
metric_diffs(Baseline, Current, Opts) ->
BaselineMap = maps:from_list(
[{maps:get(<<"metric_id">>, M), M} || M <- Baseline]),
CurrentMap = maps:from_list(
[{maps:get(<<"metric_id">>, M), M} || M <- Current]),
Ids = lists:sort(maps:keys(BaselineMap)),
[metric_diff(Id, maps:get(Id, BaselineMap),
maps:get(Id, CurrentMap, missing), Opts)
|| Id <- Ids].
metric_diff(Id, Baseline, missing, Opts) ->
#{<<"metric_id">> => Id,
<<"baseline_score">> =>
number(maps:get(<<"average_score">>, Baseline, 0.0)),
<<"current_score">> => 0.0,
<<"score_drop">> =>
number(maps:get(<<"average_score">>, Baseline, 0.0)),
<<"tolerance">> => metric_tolerance(Id, Opts),
<<"regression">> => true,
<<"reason">> => <<"missing_current_metric">>};
metric_diff(Id, Baseline, Current, Opts) ->
BaselineScore =
number(maps:get(<<"average_score">>, Baseline, 0.0)),
CurrentScore = number(maps:get(<<"average_score">>, Current, 0.0)),
Drop = erlang:max(0.0, BaselineScore - CurrentScore),
Tolerance = metric_tolerance(Id, Opts),
#{<<"metric_id">> => Id,
<<"baseline_score">> => BaselineScore,
<<"current_score">> => CurrentScore,
<<"score_drop">> => Drop,
<<"tolerance">> => Tolerance,
<<"regression">> => Drop > Tolerance,
<<"reason">> => case Drop > Tolerance of
true -> <<"score_drop">>;
false -> <<"none">>
end}.
case_diffs(Baseline, Current) ->
BaselineMap = maps:from_list(
[{maps:get(<<"case_id">>, C), C} || C <- Baseline]),
CurrentMap = maps:from_list(
[{maps:get(<<"case_id">>, C), C} || C <- Current]),
Ids = lists:sort(maps:keys(BaselineMap)),
[case_diff(Id, maps:get(Id, BaselineMap),
maps:get(Id, CurrentMap, missing))
|| Id <- Ids].
case_diff(Id, Baseline, missing) ->
#{<<"case_id">> => Id,
<<"baseline_status">> => maps:get(<<"status">>, Baseline),
<<"current_status">> => <<"missing">>,
<<"trajectory_changed">> => true,
<<"regression">> => true,
<<"reason">> => <<"missing_current_case">>};
case_diff(Id, Baseline, Current) ->
BaselinePassed = maps:get(<<"passed">>, Baseline, false),
CurrentPassed = maps:get(<<"passed">>, Current, false),
TrajectoryChanged =
maps:get(<<"trajectory">>, Baseline, []) =/=
maps:get(<<"trajectory">>, Current, []),
Regression = BaselinePassed andalso not CurrentPassed,
#{<<"case_id">> => Id,
<<"baseline_status">> => maps:get(<<"status">>, Baseline),
<<"current_status">> => maps:get(<<"status">>, Current),
<<"trajectory_changed">> => TrajectoryChanged,
<<"regression">> => Regression,
<<"reason">> => case Regression of
true -> <<"case_no_longer_passes">>;
false -> <<"none">>
end}.
validate_options(Opts) ->
MaxDrop = get(Opts, max_pass_rate_drop, 0.0),
Tolerances = get(Opts, metric_tolerances, #{}),
case {valid_fraction(MaxDrop), is_map(Tolerances),
valid_tolerances(Tolerances)} of
{true, true, true} ->
{ok, #{max_pass_rate_drop => MaxDrop,
metric_tolerances => Tolerances}};
_ -> {error, invalid_eval_comparison_options}
end.
valid_tolerances(Tolerances) ->
lists:all(
fun({Id, Value}) ->
is_binary(Id) andalso byte_size(Id) > 0 andalso
valid_fraction(Value)
end, maps:to_list(Tolerances)).
metric_tolerance(Id, Opts) ->
maps:get(Id, maps:get(metric_tolerances, Opts), 0.0).
checked_report(#{<<"report_type">> := <<"baseline_comparison">>} = Value) ->
checked_comparison(Value);
checked_report(Value) ->
adk_eval_set:encode_result(Value).
checked_comparison(Value) ->
case adk_eval_limits:check(Value) of
ok ->
case adk_context_guard:sanitize_value(Value) of
{ok, Safe} when is_map(Safe) ->
case valid_comparison(Safe) of
true -> {ok, Safe};
false -> {error, invalid_eval_comparison}
end;
_ -> {error, invalid_eval_comparison}
end;
{error, Reason} ->
{error, {invalid_eval_comparison, reason_tag(Reason)}}
end.
valid_comparison(Report) ->
Keys = [<<"report_schema_version">>, <<"report_type">>,
<<"eval_set_id">>, <<"baseline_version">>,
<<"current_version">>, <<"passed">>,
<<"baseline_pass_rate">>, <<"current_pass_rate">>,
<<"pass_rate_drop">>, <<"max_pass_rate_drop">>,
<<"metric_diffs">>, <<"case_diffs">>],
exact_keys(Report, Keys) andalso valid_comparison_fields(Report).
valid_comparison_fields(Report) ->
BaselineRate = maps:get(<<"baseline_pass_rate">>, Report),
CurrentRate = maps:get(<<"current_pass_rate">>, Report),
Drop = maps:get(<<"pass_rate_drop">>, Report),
MaxDrop = maps:get(<<"max_pass_rate_drop">>, Report),
MetricDiffs = maps:get(<<"metric_diffs">>, Report),
CaseDiffs = maps:get(<<"case_diffs">>, Report),
Basic = maps:get(<<"report_schema_version">>, Report) =:=
?REPORT_VERSION
andalso maps:get(<<"report_type">>, Report) =:=
<<"baseline_comparison">>
andalso valid_id(maps:get(<<"eval_set_id">>, Report))
andalso valid_id(maps:get(<<"baseline_version">>, Report))
andalso valid_id(maps:get(<<"current_version">>, Report))
andalso is_boolean(maps:get(<<"passed">>, Report))
andalso valid_fraction(BaselineRate)
andalso valid_fraction(CurrentRate)
andalso valid_fraction(Drop)
andalso valid_fraction(MaxDrop)
andalso is_list(MetricDiffs)
andalso is_list(CaseDiffs),
Basic andalso near(Drop, erlang:max(0.0,
BaselineRate - CurrentRate))
andalso valid_metric_diffs(MetricDiffs)
andalso valid_case_diffs(CaseDiffs)
andalso comparison_pass_is_consistent(Report, Drop, MaxDrop,
MetricDiffs, CaseDiffs).
comparison_pass_is_consistent(Report, Drop, MaxDrop,
MetricDiffs, CaseDiffs) ->
Regression = Drop > MaxDrop
orelse any_regression(MetricDiffs)
orelse any_regression(CaseDiffs),
maps:get(<<"passed">>, Report) =:= not Regression.
valid_metric_diffs(Diffs) ->
unique_ids(Diffs, <<"metric_id">>) andalso
lists:all(fun valid_metric_diff/1, Diffs).
valid_metric_diff(Diff) when is_map(Diff) ->
Keys = [<<"metric_id">>, <<"baseline_score">>,
<<"current_score">>, <<"score_drop">>,
<<"tolerance">>, <<"regression">>, <<"reason">>],
case exact_keys(Diff, Keys) of
false -> false;
true ->
Baseline = maps:get(<<"baseline_score">>, Diff),
Current = maps:get(<<"current_score">>, Diff),
Drop = maps:get(<<"score_drop">>, Diff),
Tolerance = maps:get(<<"tolerance">>, Diff),
Regression = maps:get(<<"regression">>, Diff),
Reason = maps:get(<<"reason">>, Diff),
valid_id(maps:get(<<"metric_id">>, Diff))
andalso valid_fraction(Baseline)
andalso valid_fraction(Current)
andalso valid_fraction(Drop)
andalso valid_fraction(Tolerance)
andalso is_boolean(Regression)
andalso near(Drop, erlang:max(0.0, Baseline - Current))
andalso valid_metric_reason(
Reason, Baseline, Current, Drop,
Tolerance, Regression)
end;
valid_metric_diff(_) -> false.
valid_metric_reason(<<"missing_current_metric">>, Baseline, Current,
Drop, _Tolerance, Regression) ->
Regression andalso near(Current, 0.0) andalso near(Drop, Baseline);
valid_metric_reason(<<"score_drop">>, _Baseline, _Current,
Drop, Tolerance, Regression) ->
Regression andalso Drop > Tolerance;
valid_metric_reason(<<"none">>, _Baseline, _Current,
Drop, Tolerance, Regression) ->
not Regression andalso Drop =< Tolerance;
valid_metric_reason(_, _, _, _, _, _) -> false.
valid_case_diffs(Diffs) ->
unique_ids(Diffs, <<"case_id">>) andalso
lists:all(fun valid_case_diff/1, Diffs).
valid_case_diff(Diff) when is_map(Diff) ->
Keys = [<<"case_id">>, <<"baseline_status">>,
<<"current_status">>, <<"trajectory_changed">>,
<<"regression">>, <<"reason">>],
case exact_keys(Diff, Keys) of
false -> false;
true ->
Baseline = maps:get(<<"baseline_status">>, Diff),
Current = maps:get(<<"current_status">>, Diff),
Changed = maps:get(<<"trajectory_changed">>, Diff),
Regression = maps:get(<<"regression">>, Diff),
Reason = maps:get(<<"reason">>, Diff),
valid_id(maps:get(<<"case_id">>, Diff))
andalso valid_case_status(Baseline)
andalso valid_current_status(Current)
andalso is_boolean(Changed)
andalso is_boolean(Regression)
andalso valid_case_reason(
Reason, Baseline, Current, Changed, Regression)
end;
valid_case_diff(_) -> false.
valid_case_reason(<<"missing_current_case">>, _Baseline, <<"missing">>,
true, true) -> true;
valid_case_reason(<<"case_no_longer_passes">>, <<"passed">>, Current,
_Changed, true) ->
Current =/= <<"passed">> andalso Current =/= <<"missing">>;
valid_case_reason(<<"none">>, _Baseline, _Current, _Changed, false) -> true;
valid_case_reason(_, _, _, _, _) -> false.
valid_case_status(<<"passed">>) -> true;
valid_case_status(<<"failed">>) -> true;
valid_case_status(<<"partial">>) -> true;
valid_case_status(<<"error">>) -> true;
valid_case_status(_) -> false.
valid_current_status(<<"missing">>) -> true;
valid_current_status(Status) -> valid_case_status(Status).
any_regression(Diffs) ->
lists:any(fun(Diff) -> maps:get(<<"regression">>, Diff) end, Diffs).
unique_ids(Values, Key) ->
Ids = [maps:get(Key, Value, undefined) || Value <- Values,
is_map(Value)],
length(Ids) =:= length(Values)
andalso length(Ids) =:= length(lists:usort(Ids)).
exact_keys(Map, Required) ->
lists:sort(maps:keys(Map)) =:= lists:sort(Required).
valid_id(Value) when is_binary(Value), byte_size(Value) > 0 ->
try unicode:characters_to_binary(Value) of
Value -> true;
_ -> false
catch
_:_ -> false
end;
valid_id(_) -> false.
near(A, B) when is_number(A), is_number(B) -> abs(A - B) =< 1.0e-12;
near(_, _) -> false.
markdown(#{<<"report_type">> := <<"baseline_comparison">>} = Report) ->
Status = pass_word(maps:get(<<"passed">>, Report)),
MetricLines =
[io_lib:format("| ~ts | ~.4f | ~.4f | ~.4f | ~ts |~n",
[maps:get(<<"metric_id">>, Diff),
maps:get(<<"baseline_score">>, Diff),
maps:get(<<"current_score">>, Diff),
maps:get(<<"score_drop">>, Diff),
pass_word(not maps:get(<<"regression">>, Diff))])
|| Diff <- maps:get(<<"metric_diffs">>, Report)],
CaseLines =
[io_lib:format("| ~ts | ~ts | ~ts | ~ts |~n",
[maps:get(<<"case_id">>, Diff),
maps:get(<<"baseline_status">>, Diff),
maps:get(<<"current_status">>, Diff),
pass_word(not maps:get(<<"regression">>, Diff))])
|| Diff <- maps:get(<<"case_diffs">>, Report)],
iolist_to_binary([
<<"# Evaluation baseline comparison\n\n">>,
<<"Status: **">>, Status, <<"**\n\n">>,
io_lib:format("Pass rate: ~.4f -> ~.4f (drop ~.4f)~n~n",
[maps:get(<<"baseline_pass_rate">>, Report),
maps:get(<<"current_pass_rate">>, Report),
maps:get(<<"pass_rate_drop">>, Report)]),
<<"## Metrics\n\n| Metric | Baseline | Current | Drop | Status |\n">>,
<<"|---|---:|---:|---:|---|\n">>, MetricLines,
<<"\n## Cases\n\n| Case | Baseline | Current | Status |\n">>,
<<"|---|---|---|---|\n">>, CaseLines
]);
markdown(#{<<"result_schema_version">> := 1} = Result) ->
%% Version 1 deliberately had a small saved-result contract. Do not
%% assume v2 aggregate or per-case fields while preserving its supported
%% read/render path.
Status = pass_word(maps:get(<<"passed">>, Result)),
iolist_to_binary([
<<"# Evaluation report (legacy schema v1)\n\n">>,
<<"Status: **">>, Status, <<"**\n\n">>,
<<"Evaluation set: `">>, maps:get(<<"eval_set_id">>, Result),
<<"` (version `">>, maps:get(<<"eval_set_version">>, Result),
<<"`)\n\n">>,
io_lib:format("Cases: ~B~n", [length(maps:get(<<"cases">>, Result))])
]);
markdown(Result) ->
Status = pass_word(maps:get(<<"passed">>, Result)),
MetricLines =
[io_lib:format("| ~ts | ~ts | ~.4f | ~ts |~n",
[maps:get(<<"metric_id">>, Metric),
maps:get(<<"scope">>, Metric, <<"turn">>),
maps:get(<<"average_score">>, Metric),
pass_word(maps:get(<<"passed">>, Metric))])
|| Metric <- maps:get(<<"metrics">>, Result, [])],
CaseLines =
[io_lib:format("| ~ts | ~ts | ~B | ~B |~n",
[maps:get(<<"case_id">>, Case),
maps:get(<<"status">>, Case),
maps:get(<<"passed_sample_count">>, Case, 0),
maps:get(<<"sample_count">>, Case, 0)])
|| Case <- maps:get(<<"cases">>, Result, [])],
iolist_to_binary([
<<"# Evaluation report\n\n">>,
<<"Status: **">>, Status, <<"**\n\n">>,
io_lib:format("Pass rate: ~.4f (~B/~B cases)~n~n",
[maps:get(<<"pass_rate">>, Result),
maps:get(<<"passed_case_count">>, Result),
maps:get(<<"case_count">>, Result)]),
<<"## Metrics\n\n| Metric | Scope | Average | Status |\n">>,
<<"|---|---|---:|---|\n">>, MetricLines,
<<"\n## Cases\n\n| Case | Status | Passed samples | Samples |\n">>,
<<"|---|---|---:|---:|\n">>, CaseLines
]).
pass_word(true) -> <<"PASS">>;
pass_word(false) -> <<"FAIL">>.
number(Value) when is_integer(Value) -> float(Value);
number(Value) when is_float(Value) -> Value;
number(_) -> 0.0.
valid_fraction(Value) when is_integer(Value) ->
Value >= 0 andalso Value =< 1;
valid_fraction(Value) when is_float(Value) ->
Value =:= Value andalso Value >= 0.0 andalso Value =< 1.0;
valid_fraction(_) -> false.
get(Map, Key, Default) ->
case maps:find(Key, Map) of
{ok, Value} -> Value;
error -> maps:get(atom_to_binary(Key, utf8), Map, Default)
end.
reason_tag({Tag, _}) when is_atom(Tag) -> Tag;
reason_tag({Tag, _, _}) when is_atom(Tag) -> Tag;
reason_tag(Tag) when is_atom(Tag) -> Tag;
reason_tag(_) -> evaluation_failed.