Packages

LiveView-style runtime for Gleam.

Current section

Files

Jump to
lightspeed src lightspeed ops operations_kit.gleam
Raw

src/lightspeed/ops/operations_kit.gleam

//// Production app operations-kit contracts for M40.
import gleam/list
/// Operations domain.
pub type Domain {
Runtime
Transport
Data
Etl
}
/// Operator-safe workflow action.
pub type WorkflowAction {
Rollout
Rollback
Drain
Replay
Recover
}
/// One runbook step.
pub type Step {
Step(id: String, instruction: String, evidence_tag: String, reversible: Bool)
}
/// Incident-ready runbook.
pub type Runbook {
Runbook(domain: Domain, name: String, trigger: String, steps: List(Step))
}
/// One operator workflow contract.
pub type WorkflowPlan {
WorkflowPlan(
action: WorkflowAction,
requires_dual_control: Bool,
reversible: Bool,
preflight_checks: List(String),
audit_markers: List(String),
post_checks: List(String),
)
}
/// Alert contract attached to one dashboard.
pub type AlertContract {
AlertContract(
metric: String,
threshold: String,
window: String,
severity: String,
runbook: String,
)
}
/// Dashboard contract with required metrics and alerts.
pub type Dashboard {
Dashboard(name: String, metrics: List(String), alerts: List(AlertContract))
}
/// Deterministic operations drill scenario.
pub type DrillScenario {
DrillScenario(
name: String,
domain: Domain,
workflow: WorkflowAction,
runbook: String,
evidence_artifacts: List(String),
seed: String,
)
}
/// Canonical M40 runbooks.
pub fn canonical_runbooks() -> List(Runbook) {
[
runtime_incident_runbook(),
transport_degradation_runbook(),
data_integrity_runbook(),
etl_reliability_runbook(),
]
}
/// Required operator-safe workflows for M40.
pub fn required_workflows() -> List(WorkflowAction) {
[Rollout, Rollback, Drain, Replay, Recover]
}
/// M40 workflow plans.
pub fn workflow_plans() -> List(WorkflowPlan) {
[
workflow_contract(Rollout),
workflow_contract(Rollback),
workflow_contract(Drain),
workflow_contract(Replay),
workflow_contract(Recover),
]
}
/// Contract for one operator workflow.
pub fn workflow_contract(action: WorkflowAction) -> WorkflowPlan {
case action {
Rollout ->
WorkflowPlan(
action: Rollout,
requires_dual_control: True,
reversible: True,
preflight_checks: [
"change_ticket_linked",
"canary_budget_green",
"rollback_artifact_ready",
],
audit_markers: [
"operator_identity",
"approver_identity",
"artifact_digest",
],
post_checks: [
"synthetic_smoke_green",
"error_budget_guard_green",
"customer_path_sample_green",
],
)
Rollback ->
WorkflowPlan(
action: Rollback,
requires_dual_control: True,
reversible: True,
preflight_checks: [
"incident_ticket_linked",
"last_known_good_digest_available",
"data_migration_guard_confirmed",
],
audit_markers: [
"operator_identity",
"rollback_reason",
"rollback_target_digest",
],
post_checks: [
"crash_rate_recovered",
"latency_budget_recovered",
"drift_gate_revalidated",
],
)
Drain ->
WorkflowPlan(
action: Drain,
requires_dual_control: True,
reversible: True,
preflight_checks: [
"backlog_depth_measured",
"downstream_sinks_healthy",
"replay_window_checkpointed",
],
audit_markers: [
"operator_identity",
"drain_scope",
"checkpoint_revision",
],
post_checks: [
"in_flight_zero",
"queue_depth_flat",
"replay_window_consistent",
],
)
Replay ->
WorkflowPlan(
action: Replay,
requires_dual_control: True,
reversible: True,
preflight_checks: [
"checkpoint_revision_selected",
"payload_quality_gate_green",
"idempotency_keys_confirmed",
],
audit_markers: [
"operator_identity",
"replay_start_checkpoint",
"replay_target_scope",
],
post_checks: [
"replay_completion_green",
"duplicate_sink_guard_green",
"etl_budget_recovered",
],
)
Recover ->
WorkflowPlan(
action: Recover,
requires_dual_control: True,
reversible: True,
preflight_checks: [
"incident_classified",
"safe_mode_controls_applied",
"communications_window_open",
],
audit_markers: [
"operator_identity",
"recovery_strategy",
"timeline_reference",
],
post_checks: [
"all_alerts_acknowledged",
"runbook_followups_created",
"handoff_note_published",
],
)
}
}
/// Standardized M40 dashboards and alert contracts.
pub fn dashboards() -> List(Dashboard) {
[
Dashboard(
name: "runtime_transport_baseline",
metrics: [
"lightspeed.session.crashed_total",
"lightspeed.session.heartbeat_timeout_total",
"lightspeed.transport.receive.error_total",
"lightspeed.transport.ack_latency_p95_ms",
],
alerts: [
AlertContract(
metric: "lightspeed.session.crashed_total",
threshold: ">=3",
window: "5m",
severity: "page",
runbook: "runtime_incident",
),
AlertContract(
metric: "lightspeed.transport.ack_latency_p95_ms",
threshold: ">=250",
window: "10m",
severity: "ticket",
runbook: "transport_degradation",
),
],
),
Dashboard(
name: "data_pipeline_baseline",
metrics: [
"lightspeed.pipeline.lag_ms",
"lightspeed.pipeline.retry_total",
"lightspeed.pipeline.dead_letter_total",
"lightspeed.pipeline.checkpoint_drift_total",
],
alerts: [
AlertContract(
metric: "lightspeed.pipeline.checkpoint_drift_total",
threshold: ">=1",
window: "5m",
severity: "page",
runbook: "data_integrity",
),
AlertContract(
metric: "lightspeed.pipeline.lag_ms",
threshold: ">=120000",
window: "15m",
severity: "ticket",
runbook: "etl_reliability",
),
],
),
Dashboard(
name: "etl_reliability_baseline",
metrics: [
"lightspeed.etl.slo_budget_failures_total",
"lightspeed.etl.replay_recovery_ms",
"lightspeed.etl.poison_message_total",
"lightspeed.etl.schema_quality_failures_total",
],
alerts: [
AlertContract(
metric: "lightspeed.etl.slo_budget_failures_total",
threshold: ">=1",
window: "5m",
severity: "page",
runbook: "etl_reliability",
),
AlertContract(
metric: "lightspeed.etl.schema_quality_failures_total",
threshold: ">=1",
window: "10m",
severity: "ticket",
runbook: "data_integrity",
),
],
),
]
}
/// Deterministic M40 operations drills.
pub fn drill_scenarios() -> List(DrillScenario) {
[
DrillScenario(
name: "runtime_rollback_drill",
domain: Runtime,
workflow: Rollback,
runbook: "runtime_incident",
evidence_artifacts: [
"audit.runtime.rollback.timeline",
"trace.runtime.recovery.span",
"dashboard.runtime_transport_baseline.snapshot",
],
seed: "m40-runtime-rollback-001",
),
DrillScenario(
name: "transport_drain_recover_drill",
domain: Transport,
workflow: Drain,
runbook: "transport_degradation",
evidence_artifacts: [
"audit.transport.drain.timeline",
"queue.transport.backlog.snapshot",
"dashboard.runtime_transport_baseline.snapshot",
],
seed: "m40-transport-drain-001",
),
DrillScenario(
name: "data_replay_integrity_drill",
domain: Data,
workflow: Replay,
runbook: "data_integrity",
evidence_artifacts: [
"audit.data.replay.timeline",
"checkpoint.data.replay.selection",
"dashboard.data_pipeline_baseline.snapshot",
],
seed: "m40-data-replay-001",
),
DrillScenario(
name: "etl_recovery_drill",
domain: Etl,
workflow: Recover,
runbook: "etl_reliability",
evidence_artifacts: [
"audit.etl.recovery.timeline",
"slo.etl.budget.after_mitigation",
"dashboard.etl_reliability_baseline.snapshot",
],
seed: "m40-etl-recovery-001",
),
]
}
/// Validate one runbook contract.
pub fn runbook_valid(runbook: Runbook) -> Bool {
runbook.name != ""
&& runbook.trigger != ""
&& list.length(runbook.steps) >= 4
&& steps_valid(runbook.steps)
&& has_reversible_step(runbook.steps)
}
/// Validate M40 runbook-domain coverage.
pub fn runbooks_cover_domains() -> Bool {
let runbooks = canonical_runbooks()
has_domain(runbooks, Runtime)
&& has_domain(runbooks, Transport)
&& has_domain(runbooks, Data)
&& has_domain(runbooks, Etl)
&& all_runbooks_valid(runbooks)
}
/// Validate operator-safe workflow contracts.
pub fn workflow_contracts_safe() -> Bool {
workflow_plans()
|> all_workflow_plans_valid
}
/// Validate dashboard + alert contracts.
pub fn dashboards_cover_alert_contracts() -> Bool {
let contracts = dashboards()
list.length(contracts) >= 3 && all_dashboards_valid(contracts)
}
/// Validate deterministic and reproducible drill contracts.
pub fn drills_reproducible() -> Bool {
let drills = drill_scenarios()
list.length(drills) >= 4
&& all_drills_valid(drills)
&& unique_drill_names(drills)
&& unique_drill_seeds(drills)
}
/// Full M40 readiness guard.
pub fn operations_kit_ready() -> Bool {
runbooks_cover_domains()
&& workflow_contracts_safe()
&& dashboards_cover_alert_contracts()
&& drills_reproducible()
}
/// Domain label.
pub fn domain_label(domain: Domain) -> String {
case domain {
Runtime -> "runtime"
Transport -> "transport"
Data -> "data"
Etl -> "etl"
}
}
/// Workflow label.
pub fn workflow_label(action: WorkflowAction) -> String {
case action {
Rollout -> "rollout"
Rollback -> "rollback"
Drain -> "drain"
Replay -> "replay"
Recover -> "recover"
}
}
/// Stable runbook signature.
pub fn runbook_signature(runbook: Runbook) -> String {
"runbook:"
<> domain_label(runbook.domain)
<> ":"
<> runbook.name
<> "|trigger="
<> runbook.trigger
<> "|steps="
<> join_with(";", list.map(runbook.steps, step_signature))
}
/// Stable workflow-plan signature.
pub fn workflow_plan_signature(plan: WorkflowPlan) -> String {
"workflow:"
<> workflow_label(plan.action)
<> "|dual_control="
<> bool_label(plan.requires_dual_control)
<> "|reversible="
<> bool_label(plan.reversible)
<> "|preflight="
<> join_with(",", plan.preflight_checks)
<> "|audit="
<> join_with(",", plan.audit_markers)
<> "|post="
<> join_with(",", plan.post_checks)
}
/// Stable dashboard signature.
pub fn dashboard_signature(dashboard: Dashboard) -> String {
"dashboard:"
<> dashboard.name
<> "|metrics="
<> join_with(",", dashboard.metrics)
<> "|alerts="
<> join_with(";", list.map(dashboard.alerts, alert_signature))
}
/// Stable drill signature.
pub fn drill_signature(drill: DrillScenario) -> String {
"drill:"
<> drill.name
<> "|domain="
<> domain_label(drill.domain)
<> "|workflow="
<> workflow_label(drill.workflow)
<> "|runbook="
<> drill.runbook
<> "|seed="
<> drill.seed
<> "|evidence="
<> join_with(",", drill.evidence_artifacts)
}
/// Stable full-kit signature for deterministic M40 fixtures.
pub fn operations_kit_signature() -> String {
"runbooks="
<> join_with(";", list.map(canonical_runbooks(), runbook_signature))
<> "|workflows="
<> join_with(";", list.map(workflow_plans(), workflow_plan_signature))
<> "|dashboards="
<> join_with(";", list.map(dashboards(), dashboard_signature))
<> "|drills="
<> join_with(";", list.map(drill_scenarios(), drill_signature))
}
fn runtime_incident_runbook() -> Runbook {
Runbook(
domain: Runtime,
name: "runtime_incident",
trigger: "crash_rate_or_heartbeat_timeout_spike",
steps: [
Step(
id: "capture_release_fingerprint",
instruction: "Capture deploy digest and scope impacted sessions.",
evidence_tag: "audit.release.digest",
reversible: False,
),
Step(
id: "enable_guardrails",
instruction: "Enable mitigation controls for high-risk routes.",
evidence_tag: "audit.runtime.guardrail_change",
reversible: True,
),
Step(
id: "execute_safe_rollback",
instruction: "Rollback to last known good artifact with dual control.",
evidence_tag: "audit.runtime.rollback",
reversible: True,
),
Step(
id: "verify_session_recovery",
instruction: "Verify reconnect and patch-ack recovery across canary set.",
evidence_tag: "trace.runtime.recovery",
reversible: False,
),
Step(
id: "publish_handoff_summary",
instruction: "Publish incident timeline and next actions.",
evidence_tag: "audit.incident.handoff",
reversible: False,
),
],
)
}
fn transport_degradation_runbook() -> Runbook {
Runbook(
domain: Transport,
name: "transport_degradation",
trigger: "ack_latency_or_receive_errors_over_budget",
steps: [
Step(
id: "confirm_transport_profile",
instruction: "Confirm active transport profile and fallback state.",
evidence_tag: "audit.transport.profile",
reversible: False,
),
Step(
id: "drain_high_risk_topics",
instruction: "Drain high-risk channels and freeze high-churn pushes.",
evidence_tag: "audit.transport.drain",
reversible: True,
),
Step(
id: "activate_fallback_mode",
instruction: "Activate progressive fallback and rate controls.",
evidence_tag: "audit.transport.fallback",
reversible: True,
),
Step(
id: "verify_ack_recovery",
instruction: "Verify p95 ack-latency and receive-error recovery.",
evidence_tag: "dashboard.transport.recovery",
reversible: False,
),
Step(
id: "resume_normal_flow",
instruction: "Resume normal transport profile after guard checks.",
evidence_tag: "audit.transport.resume",
reversible: True,
),
],
)
}
fn data_integrity_runbook() -> Runbook {
Runbook(
domain: Data,
name: "data_integrity",
trigger: "checkpoint_drift_or_schema_quality_failure",
steps: [
Step(
id: "freeze_mutating_writes",
instruction: "Freeze mutating writes for affected streams.",
evidence_tag: "audit.data.freeze",
reversible: True,
),
Step(
id: "select_replay_checkpoint",
instruction: "Select replay checkpoint and validate idempotency scope.",
evidence_tag: "audit.data.checkpoint",
reversible: True,
),
Step(
id: "run_quality_guard",
instruction: "Run payload-quality and schema compatibility guards.",
evidence_tag: "audit.data.quality_guard",
reversible: False,
),
Step(
id: "execute_replay",
instruction: "Replay affected windows with duplicate-sink suppression.",
evidence_tag: "audit.data.replay",
reversible: True,
),
Step(
id: "thaw_writes",
instruction: "Thaw writes after deterministic reconciliation checks.",
evidence_tag: "audit.data.thaw",
reversible: True,
),
],
)
}
fn etl_reliability_runbook() -> Runbook {
Runbook(
domain: Etl,
name: "etl_reliability",
trigger: "slo_budget_failure_or_poison_message_escalation",
steps: [
Step(
id: "pause_affected_pipelines",
instruction: "Pause affected pipelines and capture backlog profile.",
evidence_tag: "audit.etl.pause",
reversible: True,
),
Step(
id: "classify_failure_path",
instruction: "Classify outage, poison, replay, or schema fault class.",
evidence_tag: "audit.etl.classification",
reversible: False,
),
Step(
id: "apply_recovery_strategy",
instruction: "Apply drain or replay strategy with checkpoint evidence.",
evidence_tag: "audit.etl.recovery_strategy",
reversible: True,
),
Step(
id: "verify_slo_recovery",
instruction: "Verify lag, throughput, retry-rate, and replay SLO recovery.",
evidence_tag: "dashboard.etl.slo_recovery",
reversible: False,
),
Step(
id: "resume_and_handoff",
instruction: "Resume flows and publish on-call handoff summary.",
evidence_tag: "audit.etl.handoff",
reversible: True,
),
],
)
}
fn steps_valid(steps: List(Step)) -> Bool {
case steps {
[] -> True
[step, ..rest] ->
step.id != ""
&& step.instruction != ""
&& step.evidence_tag != ""
&& steps_valid(rest)
}
}
fn has_reversible_step(steps: List(Step)) -> Bool {
case steps {
[] -> False
[step, ..rest] -> step.reversible || has_reversible_step(rest)
}
}
fn has_domain(runbooks: List(Runbook), domain: Domain) -> Bool {
case runbooks {
[] -> False
[runbook, ..rest] -> runbook.domain == domain || has_domain(rest, domain)
}
}
fn all_runbooks_valid(runbooks: List(Runbook)) -> Bool {
case runbooks {
[] -> True
[runbook, ..rest] -> runbook_valid(runbook) && all_runbooks_valid(rest)
}
}
fn all_workflow_plans_valid(plans: List(WorkflowPlan)) -> Bool {
case plans {
[] -> True
[plan, ..rest] ->
workflow_plan_valid(plan) && all_workflow_plans_valid(rest)
}
}
fn workflow_plan_valid(plan: WorkflowPlan) -> Bool {
plan.requires_dual_control
&& plan.reversible
&& list.length(plan.preflight_checks) >= 3
&& list.length(plan.audit_markers) >= 3
&& list.length(plan.post_checks) >= 3
&& string_entries_non_empty(plan.preflight_checks)
&& string_entries_non_empty(plan.audit_markers)
&& string_entries_non_empty(plan.post_checks)
}
fn all_dashboards_valid(contracts: List(Dashboard)) -> Bool {
case contracts {
[] -> True
[contract, ..rest] ->
dashboard_valid(contract) && all_dashboards_valid(rest)
}
}
fn dashboard_valid(contract: Dashboard) -> Bool {
contract.name != ""
&& list.length(contract.metrics) >= 4
&& list.length(contract.alerts) >= 2
&& string_entries_non_empty(contract.metrics)
&& alerts_valid(contract.alerts, contract.metrics)
}
fn alerts_valid(alerts: List(AlertContract), metrics: List(String)) -> Bool {
case alerts {
[] -> True
[alert, ..rest] ->
alert_valid(alert, metrics) && alerts_valid(rest, metrics)
}
}
fn alert_valid(alert: AlertContract, metrics: List(String)) -> Bool {
alert.metric != ""
&& alert.threshold != ""
&& alert.window != ""
&& alert.severity != ""
&& alert.runbook != ""
&& has_string(metrics, alert.metric)
&& { alert.severity == "page" || alert.severity == "ticket" }
&& standard_window(alert.window)
&& known_runbook(alert.runbook)
}
fn standard_window(window: String) -> Bool {
window == "5m" || window == "10m" || window == "15m"
}
fn known_runbook(name: String) -> Bool {
canonical_runbooks()
|> has_runbook(name)
}
fn has_runbook(runbooks: List(Runbook), name: String) -> Bool {
case runbooks {
[] -> False
[runbook, ..rest] -> runbook.name == name || has_runbook(rest, name)
}
}
fn all_drills_valid(drills: List(DrillScenario)) -> Bool {
case drills {
[] -> True
[drill, ..rest] -> drill_valid(drill) && all_drills_valid(rest)
}
}
fn drill_valid(drill: DrillScenario) -> Bool {
drill.name != ""
&& drill.runbook != ""
&& drill.seed != ""
&& list.length(drill.evidence_artifacts) >= 3
&& string_entries_non_empty(drill.evidence_artifacts)
&& known_runbook(drill.runbook)
&& required_workflow(drill.workflow)
}
fn required_workflow(action: WorkflowAction) -> Bool {
required_workflows()
|> has_workflow(action)
}
fn has_workflow(
workflows: List(WorkflowAction),
action: WorkflowAction,
) -> Bool {
case workflows {
[] -> False
[workflow, ..rest] -> workflow == action || has_workflow(rest, action)
}
}
fn unique_drill_names(drills: List(DrillScenario)) -> Bool {
drills
|> collect_drill_names([])
|> unique_strings
}
fn unique_drill_seeds(drills: List(DrillScenario)) -> Bool {
drills
|> collect_drill_seeds([])
|> unique_strings
}
fn collect_drill_names(
drills: List(DrillScenario),
acc: List(String),
) -> List(String) {
case drills {
[] -> list.reverse(acc)
[drill, ..rest] -> collect_drill_names(rest, [drill.name, ..acc])
}
}
fn collect_drill_seeds(
drills: List(DrillScenario),
acc: List(String),
) -> List(String) {
case drills {
[] -> list.reverse(acc)
[drill, ..rest] -> collect_drill_seeds(rest, [drill.seed, ..acc])
}
}
fn unique_strings(values: List(String)) -> Bool {
unique_strings_loop(values, [])
}
fn unique_strings_loop(values: List(String), seen: List(String)) -> Bool {
case values {
[] -> True
[value, ..rest] ->
case has_string(seen, value) {
True -> False
False -> unique_strings_loop(rest, [value, ..seen])
}
}
}
fn has_string(values: List(String), target: String) -> Bool {
case values {
[] -> False
[value, ..rest] -> value == target || has_string(rest, target)
}
}
fn string_entries_non_empty(values: List(String)) -> Bool {
case values {
[] -> True
[value, ..rest] -> value != "" && string_entries_non_empty(rest)
}
}
fn step_signature(step: Step) -> String {
step.id
<> ":reversible="
<> bool_label(step.reversible)
<> ":evidence="
<> step.evidence_tag
}
fn alert_signature(alert: AlertContract) -> String {
alert.metric
<> ":threshold="
<> alert.threshold
<> ":window="
<> alert.window
<> ":severity="
<> alert.severity
<> ":runbook="
<> alert.runbook
}
fn join_with(separator: String, values: List(String)) -> String {
case values {
[] -> ""
[value] -> value
[value, ..rest] -> value <> separator <> join_with(separator, rest)
}
}
fn bool_label(value: Bool) -> String {
case value {
True -> "true"
False -> "false"
}
}