Current section
Files
Jump to
Current section
Files
priv/toolchain/verdicts.py
#!/usr/bin/env python3
"""verdicts.py — the ONE verdict-verification implementation (protocol 2).
Both consumers run the SAME checks over a verdict record, so the gate
and the round-time validator can never drift (the W25 disposition):
* check_join.py (the gate) — rejects rule-quote / evidence claims the
tree does not reproduce; a rejected verdict returns to the missing
list and the run is INCOMPLETE until redone;
* batch_judge.py (round time) — the same checks run early so the
model can correct before the gate refuses.
A verdict record:
{"adr", "verdict", "rule_quote", "basis",
"evidence": [{"query" | "query_ref", "in",
"expect": "some" | "none",
"matches": [{"at": "file:line", "text"}], "note"}],
"dormant_mechanism"?}
The checks:
* rule_quote is required and must be a whitespace-normalized
substring of the ADR's ## Rule section (protocol 2 — the rule is
the yardstick, not the whole file; W11);
* every evidence record resolves to a replayable query — `call
<name>` or a profile query_ref — and is RE-EXECUTED against the
tree; expect:"some" must reproduce every listed match (at + text);
expect:"none" must have zero structural matches AND zero textual
hits;
* on replayable platforms a `partial` needs >=1 evidence record and
a `not_verifiable` must name the searches it ran.
Stdlib + tsq (tree-sitter) when an engine is available; without one,
evidence degrades to the quote-level fallback and says so.
"""
import os
import re
VERDICTS = ["conformant", "partial", "unwitnessed", "not_verifiable"]
CALL_RE = re.compile(r"^call\s+(\S+)$")
RULE_SECTION = re.compile(r"^#+\s*Rule\s*$(.*?)(?=^#+\s|\Z)", re.M | re.S)
def norm_ws(s):
return re.sub(r"\s+", " ", s).strip()
def rule_scope(adr_text):
"""The ## Rule section's bytes, or None when the ADR has none."""
m = RULE_SECTION.search(adr_text or "")
return m.group(1) if m else None
class Verifier:
"""repo + profile queries + a tsq module (None = no engine: evidence
falls back to quote-level verification, reported in notes)."""
def __init__(self, repo, profile_queries=None, tsq_mod=None):
self.repo = repo
self.profile_queries = profile_queries or {}
self.tsq = tsq_mod
# ── query resolution ─────────────────────────────────────────────
def resolve_query(self, ev):
"""An evidence record's replayable query, or None. Citable
forms: `call <name>` (the vetted built-in), a profile-ratified
`query_ref` (kind call or scm), or a CONSTRUCTED tree-sitter
query (`query_scm`) — the retrieval net the judge cast:
presence citations replay as-is; absence citations additionally
require positive controls (run_controls)."""
if ev.get("query_ref"):
named = self.profile_queries.get(ev["query_ref"])
if isinstance(named, dict) and named.get("kind") == "call":
return {"name": named["name"], "in": named.get("in") or ev.get("in")}
if isinstance(named, dict) and named.get("kind") == "scm":
return {"scm": named["scm"], "in": named.get("in") or ev.get("in")}
return None
if ev.get("query_scm"):
return {"scm": ev["query_scm"], "in": ev.get("in")}
m = CALL_RE.match((ev.get("query") or "").strip())
return {"name": m.group(1), "in": ev.get("in")} if m else None
# ── replay ───────────────────────────────────────────────────────
def replay(self, q):
"""Re-execute a query (call or scm). [{'at','text'}] — or None
(no engine). A malformed constructed query raises ValueError."""
if self.tsq is None:
return None
eng = self.tsq
lang = eng.LANGS["elixir"]
prefixes = [q["in"]] if q.get("in") else []
out = []
parser = eng.tree_sitter.Parser(lang)
if q.get("scm"):
try:
eng.tree_sitter.Query(lang, q["scm"])
except Exception as e:
raise ValueError(f"constructed query does not compile: {e}")
for rel in eng.code_files(self.repo, prefixes):
try:
src = open(os.path.join(self.repo, rel), "rb").read()
except OSError:
continue
root = parser.parse(src).root_node
nodes = (
eng.run_scm(lang, src, root, q["scm"])
if q.get("scm")
else eng.find_calls(src, root, q["name"])
)
for node in nodes:
out.append(
{
"at": f"{rel}:{node.start_point[0] + 1}",
"text": src[node.start_byte : node.end_byte].decode(errors="replace"),
}
)
return out
def run_controls(self, ev):
"""Positive controls for a constructed-query ABSENCE citation:
each control is a code fragment plus the match count the query
must find in it. 'The net caught nothing' is citable only when
the net has proven teeth. Returns problems."""
problems = []
controls = ev.get("controls") or []
if not controls:
return [
"an absence citation from a constructed query requires >=1 "
"positive control ({source, matches}) proving the query "
"matches what it claims to search for"
]
eng = self.tsq
lang = eng.LANGS["elixir"]
parser = eng.tree_sitter.Parser(lang)
any_teeth = False
for j, c in enumerate(controls):
src = (c.get("source") or "").encode()
root = parser.parse(src).root_node
try:
nodes = eng.run_scm(lang, src, root, ev["query_scm"])
except Exception as e:
problems.append(f"controls[{j}]: query failed on control: {e}")
continue
want = c.get("matches")
if want is not None and len(nodes) != want:
problems.append(
f"controls[{j}]: declared {want} match(es), query found {len(nodes)}"
)
if len(nodes) >= 1:
any_teeth = True
if not any_teeth:
problems.append(
"no control demonstrates the query matching anything — "
"the net has no proven teeth; the absence is not citable"
)
return problems
def textual_hits(self, name, prefix):
if self.tsq is None:
return 0
return self.tsq.textual_hits(self.repo, [prefix] if prefix else [], name)
# ── the checks ───────────────────────────────────────────────────
def verify_evidence(self, v):
"""Replay a verdict's evidence records.
Returns (problems, notes): problems reject the verdict; notes
are degradations to report (engine unavailable)."""
problems, notes = [], []
for i, ev in enumerate(v.get("evidence") or []):
if not isinstance(ev, dict):
problems.append(f"evidence[{i}]: not a record")
continue
q = self.resolve_query(ev)
if q is None:
problems.append(
f"evidence[{i}] is not a replayable query "
"(use `call <name>`, a profile query_ref, or a constructed `query_scm`)"
)
continue
try:
found = self.replay(q)
except ValueError as e:
problems.append(f"evidence[{i}]: {e}")
continue
if found is None:
# no engine here. Presence claims degrade to the quote-level
# fallback; a constructed ABSENCE cannot be verified at all
# without replay — reject rather than trust.
if ev.get("expect") == "none" and q.get("scm"):
problems.append(
f"evidence[{i}]: constructed absence citation cannot be "
"verified without the replay engine (tree-sitter)"
)
continue
notes.append(
"evidence replay unavailable (tree-sitter not installed) "
"— quote-level fallback"
)
for m in ev.get("matches") or []:
rel = (m.get("at") or "").split(":")[0]
try:
body = open(
os.path.join(self.repo, rel), encoding="utf-8", errors="replace"
).read()
except OSError:
body = ""
if not m.get("text") or norm_ws(m["text"]) not in norm_ws(body):
problems.append(f"evidence[{i}]: fallback quote failed at {m.get('at')}")
continue
if ev.get("expect") == "none":
if found:
problems.append(
f"evidence[{i}]: expect none but {len(found)} structural "
f"match(es) — first {found[0]['at']}"
)
elif q.get("scm"):
problems += [f"evidence[{i}]: {p}" for p in self.run_controls(ev)]
else:
hits = self.textual_hits(q["name"], q.get("in"))
if hits:
problems.append(
f"evidence[{i}]: 0 structural matches but {hits} textual "
f"hit(s) for '{q['name']}' — inspect before citing absence"
)
else:
have = {m["at"]: m["text"] for m in found}
if not (ev.get("matches") or []):
problems.append(f"evidence[{i}]: expect some but no matches recorded")
for m in ev.get("matches") or []:
at = (m or {}).get("at")
if at not in have:
problems.append(f"evidence[{i}]: claimed match {at} not reproduced")
elif m.get("text") and norm_ws(m["text"]) not in norm_ws(have[at]):
problems.append(f"evidence[{i}]: text at {at} does not match the tree")
return problems, notes
def requirements(self, v, replayable):
"""The hard requirements (replayable platforms only): a partial
must point at code; a not_verifiable must name its searches."""
if not replayable:
return []
ev = v.get("evidence") or []
if v.get("verdict") == "partial" and not ev:
return ["a partial verdict requires >=1 replayable evidence record"]
if v.get("verdict") == "not_verifiable" and not ev:
return ["a not_verifiable verdict must name the queries it ran"]
return []
def validate(self, scope_text, rec, replayable=True):
"""All problems for one verdict record against the ADR's Rule
scope. Returns (problems, notes)."""
problems, notes = [], []
quote = rec.get("rule_quote") or ""
if not quote.strip():
problems.append("missing rule_quote — quote the rule's load-bearing sentence VERBATIM")
elif norm_ws(quote) not in norm_ws(scope_text or ""):
problems.append(
"rule_quote does not verify against the ADR's Rule section — quote VERBATIM"
)
ev_problems, ev_notes = self.verify_evidence(rec)
problems += ev_problems
notes += ev_notes
problems += self.requirements(rec, replayable)
return problems, notes