Add src/baseline_runner.py

src/baseline_runner.py

@@ -0,0 +1,114 @@
+"""Baseline runners — normal LLM answer and prompt-only honesty baseline."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import time
+from pathlib import Path
+
+from . import config
+from .llm_client import llm_call
+from .pipeline_runner import load_gold_cases
+from .schemas import BaselineResult, GoldCase
+
+# ── Prompts ────────────────────────────────────────────────────────────
+
+BASELINE_NORMAL_SYSTEM = "Answer the user's question."
+
+BASELINE_HONESTY_SYSTEM = (
+    "Answer only if supported by the provided evidence. "
+    "If not supported, say you do not know."
+)
+
+BASELINE_USER_TEMPLATE = """\
+QUESTION:
+{question}
+
+EVIDENCE:
+{evidence_text}
+"""
+
+
+# ── Runner ─────────────────────────────────────────────────────────────
+
+def run_baseline(
+    cases: list[GoldCase],
+    mode: str,
+) -> list[BaselineResult]:
+    system = BASELINE_NORMAL_SYSTEM if mode == "normal" else BASELINE_HONESTY_SYSTEM
+    results: list[BaselineResult] = []
+
+    for i, case in enumerate(cases, 1):
+        print(f"[baseline:{mode}] {i}/{len(cases)}  case={case.id}")
+        t0 = time.perf_counter()
+        try:
+            user_msg = BASELINE_USER_TEMPLATE.format(
+                question=case.question,
+                evidence_text=case.evidence_text,
+            )
+            answer = llm_call(system, user_msg)
+            elapsed = (time.perf_counter() - t0) * 1000
+            results.append(
+                BaselineResult(
+                    case_id=case.id,
+                    category=case.category,
+                    question=case.question,
+                    answer=answer,
+                    latency_ms=round(elapsed, 2),
+                )
+            )
+        except Exception as exc:
+            elapsed = (time.perf_counter() - t0) * 1000
+            results.append(
+                BaselineResult(
+                    case_id=case.id,
+                    category=case.category,
+                    question=case.question,
+                    answer="",
+                    error=str(exc),
+                    latency_ms=round(elapsed, 2),
+                )
+            )
+    return results
+
+
+def save_baseline(results: list[BaselineResult], path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w") as f:
+        for r in results:
+            f.write(r.model_dump_json() + "\n")
+    print(f"[baseline] saved {len(results)} results → {path}")
+
+
+# ── CLI ────────────────────────────────────────────────────────────────
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Run baselines")
+    parser.add_argument(
+        "--mode",
+        choices=["normal", "honesty"],
+        required=True,
+        help="Baseline mode: 'normal' or 'honesty'",
+    )
+    parser.add_argument("--cases", type=str, default=None)
+    parser.add_argument("--output", type=str, default=None)
+    args = parser.parse_args()
+
+    cases = load_gold_cases(Path(args.cases) if args.cases else None)
+    print(f"[baseline:{args.mode}] loaded {len(cases)} cases")
+
+    results = run_baseline(cases, args.mode)
+
+    if args.output:
+        out = Path(args.output)
+    elif args.mode == "normal":
+        out = config.BASELINE_NORMAL_PATH
+    else:
+        out = config.BASELINE_HONESTY_PATH
+
+    save_baseline(results, out)
+
+
+if __name__ == "__main__":
+    main()