Packages

LiveView-style runtime for Gleam.

Current section

Files

Jump to
lightspeed src lightspeed ops slo_autopilot.gleam
Raw

src/lightspeed/ops/slo_autopilot.gleam

//// SLO burn-rate and mitigation-control contracts for M30.
import gleam/int
import gleam/list
import lightspeed/ops/telemetry
/// Burn-rate observation for one service/SLO target.
pub type BurnRateSample {
BurnRateSample(
service: String,
slo_name: String,
short_window_milli: Int,
long_window_milli: Int,
hotspot_tenant_id: String,
)
}
/// Threshold profile for burn-rate evaluation.
pub type Thresholds {
Thresholds(
warn_short_milli: Int,
warn_long_milli: Int,
critical_short_milli: Int,
critical_long_milli: Int,
)
}
/// Burn-rate status classification.
pub type Status {
Healthy
Warning
Critical
}
/// Mitigation controls managed by the autopilot runtime.
pub type Control {
ShedEvents(percent: Int)
PauseBackgroundJobs
IsolateHotspotTenant(tenant_id: String)
EnterReadOnlyMode
}
/// Auditable runtime decision.
pub type Decision {
Applied
RolledBack
Noop(reason: String)
}
/// One audit trail entry for mitigation actions.
pub type AuditEntry {
AuditEntry(
actor_id: String,
control: Control,
decision: Decision,
reason: String,
at_ms: Int,
status: Status,
)
}
/// SLO autopilot runtime.
pub opaque type Runtime {
Runtime(
thresholds: Thresholds,
active_controls_rev: List(Control),
audit_rev: List(AuditEntry),
)
}
/// Build one burn-rate sample.
pub fn sample(
service: String,
slo_name: String,
short_window_milli: Int,
long_window_milli: Int,
hotspot_tenant_id: String,
) -> BurnRateSample {
BurnRateSample(
service: service,
slo_name: slo_name,
short_window_milli: short_window_milli,
long_window_milli: long_window_milli,
hotspot_tenant_id: hotspot_tenant_id,
)
}
/// Build one threshold profile.
pub fn thresholds(
warn_short_milli: Int,
warn_long_milli: Int,
critical_short_milli: Int,
critical_long_milli: Int,
) -> Thresholds {
Thresholds(
warn_short_milli: warn_short_milli,
warn_long_milli: warn_long_milli,
critical_short_milli: critical_short_milli,
critical_long_milli: critical_long_milli,
)
}
/// Default M30 threshold profile.
pub fn default_thresholds() -> Thresholds {
Thresholds(
warn_short_milli: 1000,
warn_long_milli: 700,
critical_short_milli: 2000,
critical_long_milli: 1500,
)
}
/// Build one autopilot runtime.
pub fn new(thresholds: Thresholds) -> Runtime {
Runtime(thresholds: thresholds, active_controls_rev: [], audit_rev: [])
}
/// Evaluate one sample and reconcile active controls.
pub fn evaluate(
runtime: Runtime,
sample: BurnRateSample,
actor_id: String,
now_ms: Int,
) -> #(Runtime, Status) {
let status = classify(sample, runtime.thresholds)
let desired = recommended_controls(sample, status)
#(reconcile_controls(runtime, desired, actor_id, now_ms, status), status)
}
/// Classify one burn-rate sample against thresholds.
pub fn classify(sample: BurnRateSample, thresholds: Thresholds) -> Status {
case
sample.short_window_milli >= thresholds.critical_short_milli
|| sample.long_window_milli >= thresholds.critical_long_milli
{
True -> Critical
False ->
case
sample.short_window_milli >= thresholds.warn_short_milli
|| sample.long_window_milli >= thresholds.warn_long_milli
{
True -> Warning
False -> Healthy
}
}
}
/// Desired controls for one sample/status pair.
pub fn recommended_controls(
sample: BurnRateSample,
status: Status,
) -> List(Control) {
case status {
Healthy -> []
Warning ->
[ShedEvents(percent: 15), PauseBackgroundJobs]
|> append(hotspot_control(sample.hotspot_tenant_id))
Critical ->
[ShedEvents(percent: 40), PauseBackgroundJobs, EnterReadOnlyMode]
|> append(hotspot_control(sample.hotspot_tenant_id))
}
}
/// Apply one control with an audit trail entry.
pub fn apply(
runtime: Runtime,
control: Control,
actor_id: String,
reason: String,
at_ms: Int,
status: Status,
) -> Runtime {
case has_control(runtime.active_controls_rev, control) {
True ->
record_audit(
runtime,
AuditEntry(
actor_id: actor_id,
control: control,
decision: Noop(reason: "already_active"),
reason: reason,
at_ms: at_ms,
status: status,
),
)
False ->
Runtime(..runtime, active_controls_rev: [
control,
..runtime.active_controls_rev
])
|> record_audit(AuditEntry(
actor_id: actor_id,
control: control,
decision: Applied,
reason: reason,
at_ms: at_ms,
status: status,
))
}
}
/// Roll back one control with an audit trail entry.
pub fn rollback(
runtime: Runtime,
control: Control,
actor_id: String,
reason: String,
at_ms: Int,
status: Status,
) -> Runtime {
case has_control(runtime.active_controls_rev, control) {
True ->
Runtime(
..runtime,
active_controls_rev: remove_control(
runtime.active_controls_rev,
control,
),
)
|> record_audit(AuditEntry(
actor_id: actor_id,
control: control,
decision: RolledBack,
reason: reason,
at_ms: at_ms,
status: status,
))
False ->
record_audit(
runtime,
AuditEntry(
actor_id: actor_id,
control: control,
decision: Noop(reason: "not_active"),
reason: reason,
at_ms: at_ms,
status: status,
),
)
}
}
/// Threshold accessor.
pub fn runtime_thresholds(runtime: Runtime) -> Thresholds {
runtime.thresholds
}
/// Active controls in stable order.
pub fn active_controls(runtime: Runtime) -> List(Control) {
list.reverse(runtime.active_controls_rev)
}
/// Audit entries in stable order.
pub fn audit_entries(runtime: Runtime) -> List(AuditEntry) {
list.reverse(runtime.audit_rev)
}
/// Return `True` when one control can be rolled back.
pub fn reversible(control: Control) -> Bool {
case control {
ShedEvents(_) -> True
PauseBackgroundJobs -> True
IsolateHotspotTenant(_) -> True
EnterReadOnlyMode -> True
}
}
/// Stable status label.
pub fn status_label(status: Status) -> String {
case status {
Healthy -> "healthy"
Warning -> "warning"
Critical -> "critical"
}
}
/// Stable control label.
pub fn control_label(control: Control) -> String {
case control {
ShedEvents(percent) -> "shed_events:" <> int.to_string(percent)
PauseBackgroundJobs -> "pause_background_jobs"
IsolateHotspotTenant(tenant_id) -> "isolate_tenant:" <> tenant_id
EnterReadOnlyMode -> "enter_read_only_mode"
}
}
/// Stable decision label.
pub fn decision_label(decision: Decision) -> String {
case decision {
Applied -> "applied"
RolledBack -> "rolled_back"
Noop(reason) -> "noop:" <> reason
}
}
/// Stable sample label.
pub fn sample_label(sample: BurnRateSample) -> String {
"service="
<> sample.service
<> "|slo="
<> sample.slo_name
<> "|short="
<> int.to_string(sample.short_window_milli)
<> "|long="
<> int.to_string(sample.long_window_milli)
<> "|hotspot_tenant="
<> hotspot_label(sample.hotspot_tenant_id)
}
/// Stable threshold label.
pub fn thresholds_label(thresholds: Thresholds) -> String {
"warn_short="
<> int.to_string(thresholds.warn_short_milli)
<> ",warn_long="
<> int.to_string(thresholds.warn_long_milli)
<> ",critical_short="
<> int.to_string(thresholds.critical_short_milli)
<> ",critical_long="
<> int.to_string(thresholds.critical_long_milli)
}
/// Stable audit-entry label.
pub fn audit_label(entry: AuditEntry) -> String {
"actor="
<> entry.actor_id
<> "|control="
<> control_label(entry.control)
<> "|decision="
<> decision_label(entry.decision)
<> "|reason="
<> entry.reason
<> "|status="
<> status_label(entry.status)
<> "|at_ms="
<> int.to_string(entry.at_ms)
}
/// Convert one audit entry into a metric point.
pub fn audit_metric(entry: AuditEntry) -> telemetry.Metric {
telemetry.Counter(
name: "lightspeed.slo.autopilot_control_total",
value: 1,
tags: [
telemetry.Tag(key: "actor_id", value: entry.actor_id),
telemetry.Tag(key: "control", value: control_label(entry.control)),
telemetry.Tag(key: "decision", value: decision_label(entry.decision)),
telemetry.Tag(key: "status", value: status_label(entry.status)),
],
)
}
/// Runtime invariants.
pub fn valid(runtime: Runtime) -> Bool {
let thresholds = runtime.thresholds
thresholds.warn_short_milli > 0
&& thresholds.warn_long_milli > 0
&& thresholds.critical_short_milli >= thresholds.warn_short_milli
&& thresholds.critical_long_milli >= thresholds.warn_long_milli
&& unique_controls(active_controls(runtime), [])
&& audits_reversible(audit_entries(runtime))
}
/// Stable runtime signature for deterministic fixtures.
pub fn signature(runtime: Runtime) -> String {
"thresholds="
<> thresholds_label(runtime.thresholds)
<> "|active="
<> join_with(",", list.map(active_controls(runtime), control_label))
<> "|audit="
<> int.to_string(list.length(runtime.audit_rev))
}
fn reconcile_controls(
runtime: Runtime,
desired: List(Control),
actor_id: String,
now_ms: Int,
status: Status,
) -> Runtime {
let applied =
apply_missing_controls(runtime, desired, actor_id, now_ms, status)
rollback_extra_controls(applied, desired, actor_id, now_ms, status)
}
fn apply_missing_controls(
runtime: Runtime,
desired: List(Control),
actor_id: String,
now_ms: Int,
status: Status,
) -> Runtime {
case desired {
[] -> runtime
[control, ..rest] ->
apply_missing_controls(
apply(runtime, control, actor_id, "auto_reconcile", now_ms, status),
rest,
actor_id,
now_ms,
status,
)
}
}
fn rollback_extra_controls(
runtime: Runtime,
desired: List(Control),
actor_id: String,
now_ms: Int,
status: Status,
) -> Runtime {
rollback_loop(
active_controls(runtime),
runtime,
desired,
actor_id,
now_ms,
status,
)
}
fn rollback_loop(
current: List(Control),
runtime: Runtime,
desired: List(Control),
actor_id: String,
now_ms: Int,
status: Status,
) -> Runtime {
case current {
[] -> runtime
[control, ..rest] ->
case has_control(desired, control) {
True -> rollback_loop(rest, runtime, desired, actor_id, now_ms, status)
False ->
rollback_loop(
rest,
rollback(
runtime,
control,
actor_id,
"auto_reconcile",
now_ms,
status,
),
desired,
actor_id,
now_ms,
status,
)
}
}
}
fn record_audit(runtime: Runtime, entry: AuditEntry) -> Runtime {
Runtime(..runtime, audit_rev: [entry, ..runtime.audit_rev])
}
fn hotspot_control(tenant_id: String) -> List(Control) {
case tenant_id {
"" -> []
_ -> [IsolateHotspotTenant(tenant_id: tenant_id)]
}
}
fn hotspot_label(tenant_id: String) -> String {
case tenant_id {
"" -> "none"
_ -> tenant_id
}
}
fn has_control(controls: List(Control), target: Control) -> Bool {
case controls {
[] -> False
[control, ..rest] ->
case control == target {
True -> True
False -> has_control(rest, target)
}
}
}
fn remove_control(controls: List(Control), target: Control) -> List(Control) {
case controls {
[] -> []
[control, ..rest] ->
case control == target {
True -> rest
False -> [control, ..remove_control(rest, target)]
}
}
}
fn unique_controls(controls: List(Control), seen: List(String)) -> Bool {
case controls {
[] -> True
[control, ..rest] -> {
let label = control_label(control)
case contains(seen, label) {
True -> False
False -> unique_controls(rest, [label, ..seen])
}
}
}
}
fn audits_reversible(entries: List(AuditEntry)) -> Bool {
case entries {
[] -> True
[entry, ..rest] -> reversible(entry.control) && audits_reversible(rest)
}
}
fn contains(values: List(String), target: String) -> Bool {
case values {
[] -> False
[value, ..rest] ->
case value == target {
True -> True
False -> contains(rest, target)
}
}
}
fn append(left: List(Control), right: List(Control)) -> List(Control) {
case left {
[] -> right
[entry, ..rest] -> [entry, ..append(rest, right)]
}
}
fn join_with(separator: String, values: List(String)) -> String {
case values {
[] -> ""
[value] -> value
[value, ..rest] -> value <> separator <> join_with(separator, rest)
}
}