Current section
Files
Jump to
Current section
Files
priv/toolchain/local_judge.py
#!/usr/bin/env python3
"""local_judge.py — the judged leg on Claude Code PLAN tokens.
Drives one headless `claude -p` session to judge the nominees, instead
of the Message Batches API: no ANTHROPIC_API_KEY, no API billing —
the user's Claude Code subscription does the judging. The trust
boundary is unchanged and indifferent to the runtime: the session
writes .dds-verdicts.json and the gate quote-verifies and REPLAYS
every record exactly as it does for batch or interactive verdicts.
The session is told to self-verify: run the instrument with
--verdicts itself and repair rejections before finishing — the
interactive agent leg's re-run-until-clean loop, in one headless run.
Stability (issue #45): any verdict landing non-conformant is
borderline — extra INDEPENDENT sessions re-judge just those ADRs and
the modal verdict stands (a tie resolves to the most severe). The
chosen record carries a `samples` list; the instrument surfaces
splits as stability notes. Bounded: at most 2 extra sessions.
Refuses unprofiled platforms (extraction is not a judging problem),
mirroring batch_judge.
"""
import json
import os
import shutil
import subprocess
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
from batch_judge import modal_verdict # noqa: E402 — one modal law, both runtimes
PROMPT = """Judge the nominated ADRs of this repository's dds check and write the
verdicts file. You are the judged leg of /dds-check: narrow per-ADR
conformance verdicts, produced by investigation, enforced externally —
your record is verified by an instrument you do not control.
## Procedure
1. Run the instrument for the nominee list and witnessed facts:
{py} {instrument} --repo . --base {base} --json /tmp/dds-pre.json
Read the `nominees` from /tmp/dds-pre.json (paths of ADRs to judge)
and use the walk for coverage/context. Read each nominated ADR.
{scope}
2. Investigate with structural retrieval nets:
{py} {tsq} call <name> --repo . [--in lib/]
A query is a NET defining the evaluated population; YOU evaluate the
retrieved source (read files freely for context). Prefer broad nets
and filter visibly in your basis. Constructed tree-sitter queries:
write the query to a file and run `{py} {tsq} --scm <file> --repo .`.
3. Write the verdicts to {out} — a BARE JSON list, one record per
nominee:
{{"adr": "docs/adr/....md",
"verdict": "conformant|partial|unwitnessed|not_verifiable",
"rule_quote": "<the load-bearing sentence, VERBATIM from the ADR's ## Rule section>",
"basis": "your evaluation, incl. visible filtering of matches",
"evidence": [{{"query": "call <name>", // or "query_scm": "<raw query>"
"in": "lib/", "expect": "some"|"none",
"matches": [{{"at": "file:line", "text": "<span EXACTLY as tsq printed>"}}],
"controls": [{{"source": "<code the query provably matches>", "matches": 1}}], // absence-via-scm only
"note": "what this operationalizes"}}],
"dormant_mechanism": "<optional>"}}
4. SELF-VERIFY, then finish: re-run
{py} {instrument} --repo . --base {base} --verdicts {out}
If it reports rejected verdicts (DEGRADED lines naming them) or
missing nominees, fix the records and re-run — at most 3 repair
rounds; then stop and leave the honest state.
## The law (violations are rejected by the instrument)
- rule_quote: verbatim substring of the ADR's ## Rule section — never
the preamble, never paraphrase.
- matches: copied EXACTLY from tsq output, never from memory or a file
read. Presence evidence must reproduce on replay.
- absence via `call`: citable only when tsq reports 0 structural AND
0 textual hits (tsq warns otherwise). Absence via a constructed
query: REQUIRES positive controls (code the query provably matches).
- conformant needs POSITIVELY shown state; an interim rule accurately
declaring the witnessed state is conformant AS an interim. A rule
that resists operationalization is honestly not_verifiable, naming
the searches run. Judge witnessed state, never your recollection of
how such apps usually work.
- The verdicts file is the deliverable — no report, no commentary,
never commit anything.
"""
def _session(repo, base, out, say, only=None):
claude = shutil.which("claude")
if not claude:
return False
instrument = os.path.join(HERE, "check_join.py")
tsq = os.path.join(HERE, "tsq.py")
scope = ""
if only:
scope = (
" JUDGE ONLY THESE NOMINEES (an independent stability "
"re-sample — investigate FRESH from the tree; cite only what "
"the tools show you THIS session):\n"
+ "".join(f" {a}\n" for a in only)
+ " Write records for only these ADRs; when self-verifying, "
"ignore missing-verdict reports for nominees outside this list.\n"
)
prompt = PROMPT.format(
py=sys.executable, instrument=instrument, tsq=tsq, base=base, out=out,
scope=scope,
)
say(f"judging via local claude (plan tokens) — headless session, self-verifying…")
r = subprocess.run(
[
claude, "-p", prompt,
"--output-format", "json",
"--max-turns", "60",
"--allowedTools", "Bash,Read,Write,Edit,Glob,Grep",
],
cwd=repo, capture_output=True, text=True, timeout=1800,
)
if r.returncode != 0:
say(f"local claude exited {r.returncode}: {(r.stderr or r.stdout or '').strip()[:200]}")
return os.path.exists(out)
try:
doc = json.loads(r.stdout)
cost = doc.get("total_cost_usd")
# claude -p reports computed usage regardless of billing mode;
# on plan/subscription auth this is covered usage, not a bill
say(
f"local judge done — {doc.get('num_turns', '?')} turns, "
f"{round((doc.get('duration_ms') or 0) / 1000)}s"
+ (f", usage ≈ ${cost:.2f} (plan-covered)" if cost else "")
)
except (json.JSONDecodeError, TypeError):
say("local judge done (unparseable session summary)")
return os.path.exists(out)
def _load(path):
try:
recs = json.load(open(path, encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return []
if not isinstance(recs, list):
return []
return [r for r in recs if isinstance(r, dict) and r.get("adr") and r.get("verdict")]
def run(repo, base, out, say, samples=3):
if not _session(repo, base, out, say):
return False
if samples <= 1:
return True
recs = _load(out)
samples_by = {r["adr"]: [r] for r in recs if r["verdict"] != "conformant"}
if not samples_by:
return True
say(f"{len(samples_by)} borderline verdict(s) — drawing independent re-sample(s)…")
tmp = out + ".sample"
for _round in range(2, samples + 1):
# another sample only where no verdict has 2 votes yet
todo = sorted(
a for a, rs in samples_by.items()
if max(sum(1 for r in rs if r["verdict"] == v)
for v in {r["verdict"] for r in rs}) < 2
)
if not todo:
break
if os.path.exists(tmp):
os.unlink(tmp)
if not _session(repo, base, tmp, say, only=todo):
break
for r in _load(tmp):
if r["adr"] in samples_by:
samples_by[r["adr"]].append(r)
if os.path.exists(tmp):
os.unlink(tmp)
final = []
for r in recs:
rs = samples_by.get(r["adr"])
final.append(modal_verdict(rs) if rs and len(rs) > 1 else r)
json.dump(final, open(out, "w", encoding="utf-8"), indent=1)
for r in final:
s = r.get("samples")
if s and len(set(s)) > 1:
say(f" {os.path.basename(r['adr'])}: samples {s} → {r['verdict']} (modal)")
return True