macula_neuroevolution

%% @doc Reward Signal Computation for Liquid Conglomerate Silos.
%%
%% Part of the Liquid Conglomerate v2 architecture. This module computes
%% reward signals for each silo's L0 TWEANN, including cooperative
%% penalties/bonuses to encourage alignment between silos.
%%
%% == Reward Structure ==
%%
%% Each silo has local objectives plus cooperation terms:
%%
%% Resource Silo:
%%   reward = 0.35*throughput + 0.25*stability + 0.15*efficiency
%%          - 0.15*task_blocked - 0.10*distribution_blocked
%%
%% Task Silo:
%%   reward = 0.40*improvement + 0.20*diversity - 0.15*complexity
%%          - 0.15*resource_pressure + 0.10*distribution_diversity
%%
%% Distribution Silo:
%%   reward = 0.30*load_balance + 0.25*migration + 0.20*network_eff
%%          - 0.15*resource_pressure + 0.10*task_diversity
%%
%% == Global Health Bonus ==
%%
%% All silos receive: +0.1 * global_health_improvement
%% where global_health = weighted sum of all silo performances
%%
%% @author Macula.io
%% @copyright 2025 Macula.io
-module(lc_reward).

%% API
-export([
    compute_resource_reward/1,
    compute_task_reward/1,
    compute_task_reward_breakdown/1,
    compute_trial_reward/1,
    compute_distribution_reward/1,
    compute_global_health/1,
    add_global_bonus/2
]).

%%% ============================================================================
%%% Resource Silo Reward
%%% ============================================================================

%% @doc Compute reward signal for Resource Silo L0.
%%
%% Input metrics map should contain:
%% - throughput: Evaluations per second (normalized 0-1)
%% - gc_triggers: Number of GC triggers in period (0 = best)
%% - pauses: Number of evolution pauses (0 = best)
%% - memory_utilization: Current memory usage (0-1)
%% - cpu_utilization: Current CPU usage (0-1)
%% - task_silo_blocked: Was Task Silo blocked? (0-1)
%% - distribution_blocked: Was Distribution blocked? (0-1)
-spec compute_resource_reward(map()) -> float().
compute_resource_reward(Metrics) ->
    %% Extract metrics with defaults
    Throughput = maps:get(throughput, Metrics, 0.0),
    GcTriggers = maps:get(gc_triggers, Metrics, 0),
    Pauses = maps:get(pauses, Metrics, 0),
    MemUtil = maps:get(memory_utilization, Metrics, 0.5),
    CpuUtil = maps:get(cpu_utilization, Metrics, 0.5),
    TaskBlocked = maps:get(task_silo_blocked, Metrics, 0.0),
    DistBlocked = maps:get(distribution_blocked, Metrics, 0.0),

    %% Compute component scores

    %% Throughput score: higher is better (already normalized)
    ThroughputScore = clamp(Throughput, 0.0, 1.0),

    %% Stability score: penalize GC triggers and pauses
    %% Each GC trigger costs 0.1, each pause costs 0.3
    StabilityScore = max(0.0, 1.0 - (GcTriggers * 0.1) - (Pauses * 0.3)),

    %% Efficiency score: balance utilization (not too high, not too low)
    %% Optimal around 0.6-0.7
    MemEfficiency = 1.0 - abs(MemUtil - 0.65) * 2.0,
    CpuEfficiency = 1.0 - abs(CpuUtil - 0.65) * 2.0,
    EfficiencyScore = clamp((MemEfficiency + CpuEfficiency) / 2.0, 0.0, 1.0),

    %% Cooperation penalties
    TaskBlockedPenalty = clamp(TaskBlocked, 0.0, 1.0),
    DistBlockedPenalty = clamp(DistBlocked, 0.0, 1.0),

    %% Weighted sum
    Reward = 0.35 * ThroughputScore
           + 0.25 * StabilityScore
           + 0.15 * EfficiencyScore
           - 0.15 * TaskBlockedPenalty
           - 0.10 * DistBlockedPenalty,

    clamp(Reward, -1.0, 1.0).

%%% ============================================================================
%%% Task Silo Reward (TWEANN Fitness Function)
%%% ============================================================================

%% @doc Compute reward signal for Task Silo L0 TWEANN.
%%
%% Designed to reward LC controllers that:
%% 1. MAXIMIZE fitness improvement velocity (faster learning)
%% 2. MINIMIZE training time to convergence (efficiency)
%% 3. PUNISH premature convergence (getting stuck at low fitness)
%% 4. MINIMIZE resource usage (memory/CPU efficiency)
%%
%% Input metrics map should contain:
%% - improvement_velocity: Fitness improvement rate per 1000 evals (-1 to 1)
%% - best_fitness: Current best fitness achieved (0-1)
%% - fitness_target: Target fitness to reach (0-1)
%% - evaluations_used: Total evaluations consumed
%% - evaluations_budget: Total evaluation budget
%% - stagnation_severity: How stuck evolution is (0=progressing, 1=stuck)
%% - diversity_index: Population diversity (0-1)
%% - memory_pressure: Current memory usage (0-1)
%% - cpu_pressure: Current CPU usage (0-1)
-spec compute_task_reward(map()) -> float().
compute_task_reward(Metrics) ->
    %% Extract metrics with defaults
    ImprovementVelocity = maps:get(improvement_velocity, Metrics, 0.0),
    BestFitness = maps:get(best_fitness, Metrics, 0.0),
    FitnessTarget = maps:get(fitness_target, Metrics, 1.0),
    EvalsUsed = maps:get(evaluations_used, Metrics, 0),
    EvalsBudget = maps:get(evaluations_budget, Metrics, 100000),
    StagnationSeverity = maps:get(stagnation_severity, Metrics, 0.0),
    DiversityIndex = maps:get(diversity_index, Metrics, 0.5),
    MemoryPressure = maps:get(memory_pressure, Metrics, 0.0),
    CpuPressure = maps:get(cpu_pressure, Metrics, 0.0),

    %% =========================================================================
    %% Component 1: Fitness Improvement Velocity (40% weight)
    %% Higher velocity = better. Strongly rewards rapid improvement.
    %% =========================================================================
    %% Map velocity from [-0.1, 0.1] to [0, 1] with scaling
    %% Positive velocity gets exponential boost
    VelocityScore = case ImprovementVelocity > 0 of
        true ->
            %% Positive velocity: exponential reward (accelerating improvement is great)
            min(1.0, math:pow(ImprovementVelocity * 10, 0.7));
        false ->
            %% Negative velocity: linear penalty (declining is bad)
            max(0.0, 0.5 + ImprovementVelocity * 5)
    end,

    %% =========================================================================
    %% Component 2: Training Efficiency (20% weight)
    %% Reward reaching target fitness with fewer evaluations.
    %% =========================================================================
    FitnessRatio = min(1.0, BestFitness / max(0.01, FitnessTarget)),
    EvalRatio = EvalsUsed / max(1, EvalsBudget),

    %% Efficiency = fitness achieved per evaluation used
    %% Early achievement of high fitness = high score
    EfficiencyScore = case FitnessRatio > 0.9 of
        true ->
            %% Near target: reward reaching it quickly
            1.0 - EvalRatio;
        false ->
            %% Not near target: score based on progress vs time
            FitnessRatio * (1.0 - EvalRatio * 0.5)
    end,

    %% =========================================================================
    %% Component 3: Stagnation Penalty (20% weight)
    %% PUNISH stagnation - LC TWEANN needs to know when to try different strategies.
    %% =========================================================================
    %% For LC control purposes, stagnation is ALWAYS bad because we want the
    %% controller to adapt hyperparameters when stuck, regardless of fitness level.
    %% The TWEANN should learn: high stagnation = low reward = need to change.
    %%
    %% Stagnation severity is already computed in velocity_state:
    %% - 0.0 = healthy (improving)
    %% - 1.0 = critical (completely stuck)
    StagnationPenalty = StagnationSeverity,

    %% Also penalize low diversity (premature collapse of exploration)
    DiversityCollapsePenalty = case DiversityIndex < 0.3 of
        true -> (0.3 - DiversityIndex) * 0.5;  %% Up to 0.15 penalty
        false -> 0.0
    end,

    ConvergenceScore = 1.0 - clamp(StagnationPenalty + DiversityCollapsePenalty, 0.0, 1.0),

    %% =========================================================================
    %% Component 4: Resource Efficiency (20% weight)
    %% Minimize memory and CPU usage while still making progress.
    %% =========================================================================
    %% Memory and CPU pressure are bad, but only if not improving
    %% If improving rapidly, some resource usage is acceptable
    VelocityBonus = max(0.0, ImprovementVelocity * 2.0),
    EffectiveMemPressure = max(0.0, MemoryPressure - VelocityBonus),
    EffectiveCpuPressure = max(0.0, CpuPressure - VelocityBonus),

    ResourceScore = 1.0 - (EffectiveMemPressure * 0.6 + EffectiveCpuPressure * 0.4),

    %% =========================================================================
    %% Composite Reward
    %% =========================================================================
    %% Weights: Velocity=40%, Efficiency=20%, Convergence=20%, Resources=20%
    Reward = 0.40 * VelocityScore
           + 0.20 * EfficiencyScore
           + 0.20 * ConvergenceScore
           + 0.20 * clamp(ResourceScore, 0.0, 1.0),

    clamp(Reward, -1.0, 1.0).

%% @doc Compute reward breakdown for Task Silo L0 - returns individual component scores.
%%
%% Returns a map containing:
%% - total_reward: The combined weighted reward (-1.0 to 1.0)
%% - velocity_score: Fitness improvement velocity score (40% weight)
%% - efficiency_score: Training time efficiency score (20% weight)
%% - convergence_score: Premature convergence penalty score (20% weight)
%% - resource_score: Memory/CPU efficiency score (20% weight)
%% - velocity_weight, efficiency_weight, etc.: The weights used
%%
%% This allows visualization of which factors are contributing to the reward.
-spec compute_task_reward_breakdown(map()) -> map().
compute_task_reward_breakdown(Metrics) ->
    %% Extract metrics with defaults
    ImprovementVelocity = maps:get(improvement_velocity, Metrics, 0.0),
    BestFitness = maps:get(best_fitness, Metrics, 0.0),
    FitnessTarget = maps:get(fitness_target, Metrics, 1.0),
    EvalsUsed = maps:get(evaluations_used, Metrics, 0),
    EvalsBudget = maps:get(evaluations_budget, Metrics, 100000),
    StagnationSeverity = maps:get(stagnation_severity, Metrics, 0.0),
    DiversityIndex = maps:get(diversity_index, Metrics, 0.5),
    MemoryPressure = maps:get(memory_pressure, Metrics, 0.0),
    CpuPressure = maps:get(cpu_pressure, Metrics, 0.0),

    %% Component 1: Velocity Score (40%)
    VelocityScore = case ImprovementVelocity > 0 of
        true ->
            min(1.0, math:pow(ImprovementVelocity * 10, 0.7));
        false ->
            max(0.0, 0.5 + ImprovementVelocity * 5)
    end,

    %% Component 2: Efficiency Score (20%)
    FitnessRatio = min(1.0, BestFitness / max(0.01, FitnessTarget)),
    EvalRatio = EvalsUsed / max(1, EvalsBudget),
    EfficiencyScore = case FitnessRatio > 0.9 of
        true ->
            1.0 - EvalRatio;
        false ->
            FitnessRatio * (1.0 - EvalRatio * 0.5)
    end,

    %% Component 3: Convergence Score (20%)
    LowFitnessMultiplier = max(0.0, 1.0 - FitnessRatio),
    PrematureConvergencePenalty = StagnationSeverity * LowFitnessMultiplier,
    DiversityCollapsePenalty = case DiversityIndex < 0.2 andalso FitnessRatio < 0.5 of
        true -> (0.2 - DiversityIndex) * (0.5 - FitnessRatio) * 2.0;
        false -> 0.0
    end,
    ConvergenceScore = 1.0 - clamp(PrematureConvergencePenalty + DiversityCollapsePenalty, 0.0, 1.0),

    %% Component 4: Resource Score (20%)
    VelocityBonus = max(0.0, ImprovementVelocity * 2.0),
    EffectiveMemPressure = max(0.0, MemoryPressure - VelocityBonus),
    EffectiveCpuPressure = max(0.0, CpuPressure - VelocityBonus),
    ResourceScore = 1.0 - (EffectiveMemPressure * 0.6 + EffectiveCpuPressure * 0.4),

    %% Composite Reward
    TotalReward = 0.40 * VelocityScore
               + 0.20 * EfficiencyScore
               + 0.20 * ConvergenceScore
               + 0.20 * clamp(ResourceScore, 0.0, 1.0),

    #{
        %% Total weighted reward
        total_reward => clamp(TotalReward, -1.0, 1.0),
        %% Individual component scores (0.0-1.0)
        velocity_score => VelocityScore,
        efficiency_score => EfficiencyScore,
        convergence_score => ConvergenceScore,
        resource_score => clamp(ResourceScore, 0.0, 1.0),
        %% Weights for visualization
        velocity_weight => 0.40,
        efficiency_weight => 0.20,
        convergence_weight => 0.20,
        resource_weight => 0.20,
        %% Input metrics (for debugging)
        improvement_velocity => ImprovementVelocity,
        stagnation_severity => StagnationSeverity,
        fitness_ratio => FitnessRatio,
        eval_ratio => EvalRatio
    }.

%% @doc Compute cumulative reward for an LC TWEANN trial.
%%
%% Called at the end of a trial period to evaluate the LC controller's performance.
%% Uses fitness improvement achieved during the trial as the primary metric.
%%
%% Input metrics map should contain:
%% - start_fitness: Fitness at trial start
%% - end_fitness: Fitness at trial end
%% - evaluations_used: Evaluations consumed during trial
%% - evaluation_budget: Evaluation budget for trial
%% - peak_memory: Peak memory usage during trial (0-1)
%% - avg_cpu: Average CPU usage during trial (0-1)
%% - stagnation_events: Number of stagnation events during trial
%% - convergence_reached: true if fitness target was reached
-spec compute_trial_reward(map()) -> float().
compute_trial_reward(Metrics) ->
    StartFitness = maps:get(start_fitness, Metrics, 0.0),
    EndFitness = maps:get(end_fitness, Metrics, 0.0),
    EvalsUsed = maps:get(evaluations_used, Metrics, 0),
    EvalsBudget = maps:get(evaluation_budget, Metrics, 10000),
    PeakMemory = maps:get(peak_memory, Metrics, 0.5),
    AvgCpu = maps:get(avg_cpu, Metrics, 0.5),
    StagnationEvents = maps:get(stagnation_events, Metrics, 0),
    ConvergenceReached = maps:get(convergence_reached, Metrics, false),

    %% Fitness improvement: main objective
    FitnessImprovement = EndFitness - StartFitness,

    %% Bonus for reaching convergence
    ConvergenceBonus = case ConvergenceReached of
        true -> 0.5 * (1.0 - EvalsUsed / max(1, EvalsBudget));  %% Faster = better
        false -> 0.0
    end,

    %% Penalty for stagnation events
    StagnationPenalty = min(0.3, StagnationEvents * 0.05),

    %% Resource efficiency
    ResourceEfficiency = 1.0 - (PeakMemory * 0.5 + AvgCpu * 0.5),

    %% Composite trial reward
    TrialReward = FitnessImprovement * 2.0      %% Main: fitness gained
               + ConvergenceBonus               %% Bonus: reached target quickly
               - StagnationPenalty              %% Penalty: got stuck
               + ResourceEfficiency * 0.2,      %% Bonus: used resources efficiently

    clamp(TrialReward, -1.0, 2.0).

%%% ============================================================================
%%% Distribution Silo Reward
%%% ============================================================================

%% @doc Compute reward signal for Distribution Silo L0.
%%
%% Input metrics map should contain:
%% - load_balance_score: How evenly distributed load is (0-1)
%% - migration_effectiveness: Do migrations improve diversity? (0-1)
%% - network_efficiency: Low overhead, successful transfers (0-1)
%% - resource_pressure_caused: Pressure on Resource Silo (0-1)
%% - task_diversity_contribution: Helping Task Silo diversity (0-1)
-spec compute_distribution_reward(map()) -> float().
compute_distribution_reward(Metrics) ->
    %% Extract metrics with defaults
    LoadBalanceScore = maps:get(load_balance_score, Metrics, 0.5),
    MigrationEffectiveness = maps:get(migration_effectiveness, Metrics, 0.5),
    NetworkEfficiency = maps:get(network_efficiency, Metrics, 0.8),
    ResourcePressure = maps:get(resource_pressure_caused, Metrics, 0.0),
    TaskDiversityContrib = maps:get(task_diversity_contribution, Metrics, 0.0),

    %% All scores are already in 0-1 range, just clamp
    LoadScore = clamp(LoadBalanceScore, 0.0, 1.0),
    MigrationScore = clamp(MigrationEffectiveness, 0.0, 1.0),
    NetworkScore = clamp(NetworkEfficiency, 0.0, 1.0),
    ResourcePenalty = clamp(ResourcePressure, 0.0, 1.0),
    TaskBonus = clamp(TaskDiversityContrib, 0.0, 1.0),

    %% Weighted sum
    Reward = 0.30 * LoadScore
           + 0.25 * MigrationScore
           + 0.20 * NetworkScore
           - 0.15 * ResourcePenalty
           + 0.10 * TaskBonus,

    clamp(Reward, -1.0, 1.0).

%%% ============================================================================
%%% Global Health and Bonus
%%% ============================================================================

%% @doc Compute global health metric from all silo rewards.
%%
%% Global health is a weighted average of silo rewards.
%% This provides a holistic view of system performance.
-spec compute_global_health(map()) -> float().
compute_global_health(SiloRewards) ->
    ResourceReward = maps:get(resource, SiloRewards, 0.0),
    TaskReward = maps:get(task, SiloRewards, 0.0),
    DistReward = maps:get(distribution, SiloRewards, 0.0),

    %% Equal weighting for global health
    GlobalHealth = (ResourceReward + TaskReward + DistReward) / 3.0,

    clamp(GlobalHealth, -1.0, 1.0).

%% @doc Add global bonus to a silo's reward based on global health improvement.
%%
%% Each silo gets +0.1 * (current_global - previous_global) as bonus.
%% This encourages silos to cooperate for overall system improvement.
-spec add_global_bonus(float(), map()) -> float().
add_global_bonus(SiloReward, GlobalState) ->
    CurrentGlobal = maps:get(current_global_health, GlobalState, 0.0),
    PreviousGlobal = maps:get(previous_global_health, GlobalState, 0.0),

    GlobalImprovement = CurrentGlobal - PreviousGlobal,

    %% Add 10% of global improvement as bonus
    Bonus = 0.1 * GlobalImprovement,

    clamp(SiloReward + Bonus, -1.0, 1.0).

%%% ============================================================================
%%% Internal Functions
%%% ============================================================================

%% @private Clamp value to range.
clamp(Value, Min, Max) ->
    max(Min, min(Max, Value)).