"""Interpretation eval harness.

A small fixed eval set scored against the live interpret service. Run
nightly (or manually after any prompt change) to catch regressions
before they ship to learners.

Usage:

    cd apps/ekg-tutor/interpret
    .venv/bin/python scripts/eval.py \
        --eval-set scripts/eval-set.json \
        --interpret-url http://127.0.0.1:3112 \
        --images-dir /var/lib/ekg-tutor/seed-images \
        --report /var/log/ekg-tutor-eval-$(date +%Y%m%d).json

Per-case scoring:
  - schema_valid     — did the model output validate against the schema?
  - primary_correct  — does final_synthesis.primary_diagnosis match
                       the canonical (loose keyword match)?
  - rate_within_10   — is rate within ±10 bpm of canonical?
  - axis_category    — does axis.category match exactly?
  - bbb_match        — does qrs_morphology.bundle_branch_block match?
  - av_block_match   — does conduction.av_block match?
  - st_elevation     — does st_t_changes.st_elevation.present match?

Per-run aggregate: counts + percentages for each metric. The job exits
non-zero if any metric drops below its configured floor — the prompt
needs work.

The eval set is intentionally small (5-10 cases) and hand-validated.
Growing it requires the same clinician review as the case library —
the eval set is the calibration anchor for everything else.
"""

from __future__ import annotations

import argparse
import json
import sys
import time
from dataclasses import dataclass, asdict
from pathlib import Path
from typing import Any

import httpx


@dataclass
class CaseEval:
    case_id: str
    image_filename: str
    canonical_summary: dict          # the hand-validated subset to score against
    notes: str | None = None


@dataclass
class CaseResult:
    case_id: str
    schema_valid: bool
    primary_correct: bool | None
    rate_within_10: bool | None
    axis_category_match: bool | None
    bbb_match: bool | None
    av_block_match: bool | None
    st_elevation_match: bool | None
    latency_ms: int
    error: str | None


# Floors: a metric below its floor fails the eval and exits non-zero.
FLOORS = {
    "schema_valid": 0.95,
    "primary_correct": 0.70,
    "rate_within_10": 0.80,
    "axis_category_match": 0.65,
    "bbb_match": 0.80,
    "av_block_match": 0.75,
    "st_elevation_match": 0.85,
}


def load_eval_set(path: Path) -> list[CaseEval]:
    raw = json.loads(path.read_text(encoding="utf-8"))
    return [CaseEval(**c) for c in raw["cases"]]


def loose_match(actual: str, expected: str) -> bool:
    """Substring match on the most distinctive keyword of expected. Handles
    minor word-order differences."""
    a = (actual or "").lower()
    e = (expected or "").lower()
    if not e:
        return False
    if a == e or e in a or a in e:
        return True
    # Match the longest 4+ character token from expected.
    tokens = [t for t in e.replace(",", " ").split() if len(t) > 3]
    if not tokens:
        return False
    tokens.sort(key=len, reverse=True)
    return tokens[0] in a


def score_one(canonical_summary: dict, output: dict) -> CaseResult:
    case_id = canonical_summary.get("case_id", "?")
    try:
        primary_expected = canonical_summary.get("primary_diagnosis")
        primary_actual = output["final_synthesis"]["primary_diagnosis"]
        primary_correct = (
            loose_match(primary_actual, primary_expected) if primary_expected else None
        )

        rate_expected = canonical_summary.get("rate_bpm")
        rate_actual = output["rate"]["measurement"]["value"]
        rate_within_10 = (
            abs(rate_actual - rate_expected) <= 10 if rate_expected is not None else None
        )

        axis_expected = canonical_summary.get("axis_category")
        axis_actual = output["axis"]["category"]
        axis_category_match = (
            axis_actual == axis_expected if axis_expected is not None else None
        )

        bbb_expected = canonical_summary.get("bundle_branch_block")
        bbb_actual = output["qrs_morphology"]["bundle_branch_block"]
        bbb_match = bbb_actual == bbb_expected if bbb_expected is not None else None

        av_expected = canonical_summary.get("av_block")
        av_actual = output["conduction_abnormalities"]["av_block"]
        av_block_match = av_actual == av_expected if av_expected is not None else None

        ste_expected = canonical_summary.get("st_elevation_present")
        ste_actual = output["st_t_changes"]["st_elevation"]["present"]
        st_elevation_match = (
            ste_actual == ste_expected if ste_expected is not None else None
        )

        return CaseResult(
            case_id=case_id,
            schema_valid=True,
            primary_correct=primary_correct,
            rate_within_10=rate_within_10,
            axis_category_match=axis_category_match,
            bbb_match=bbb_match,
            av_block_match=av_block_match,
            st_elevation_match=st_elevation_match,
            latency_ms=0,
            error=None,
        )
    except Exception as e:
        return CaseResult(
            case_id=case_id,
            schema_valid=False,
            primary_correct=None,
            rate_within_10=None,
            axis_category_match=None,
            bbb_match=None,
            av_block_match=None,
            st_elevation_match=None,
            latency_ms=0,
            error=f"score_error: {e}",
        )


def call_interpret(client: httpx.Client, url: str, image_path: Path) -> tuple[dict | None, int, str | None]:
    start = time.monotonic()
    try:
        resp = client.post(
            f"{url}/v1/interpret",
            json={"image_url": f"file://{image_path.resolve()}"},
            timeout=120.0,
        )
    except Exception as e:
        return None, int((time.monotonic() - start) * 1000), f"http_error: {e}"
    latency_ms = int((time.monotonic() - start) * 1000)
    if resp.status_code != 200:
        return None, latency_ms, f"status_{resp.status_code}: {resp.text[:200]}"
    body = resp.json()
    return body.get("interpretation"), latency_ms, None


def aggregate(results: list[CaseResult]) -> dict[str, Any]:
    """Compute per-metric pass rates, ignoring None entries."""
    metrics = [
        "schema_valid",
        "primary_correct",
        "rate_within_10",
        "axis_category_match",
        "bbb_match",
        "av_block_match",
        "st_elevation_match",
    ]
    out: dict[str, Any] = {"n": len(results), "metrics": {}}
    for m in metrics:
        applicable = [getattr(r, m) for r in results if getattr(r, m) is not None]
        if not applicable:
            out["metrics"][m] = {"n": 0, "pass_rate": None}
            continue
        passed = sum(1 for v in applicable if v)
        out["metrics"][m] = {
            "n": len(applicable),
            "passed": passed,
            "pass_rate": passed / len(applicable),
        }
    out["mean_latency_ms"] = (
        sum(r.latency_ms for r in results) / len(results) if results else 0
    )
    return out


def check_floors(agg: dict[str, Any]) -> list[str]:
    failures: list[str] = []
    for metric, floor in FLOORS.items():
        m = agg["metrics"].get(metric)
        if not m or m["pass_rate"] is None:
            continue
        if m["pass_rate"] < floor:
            failures.append(f"{metric}: {m['pass_rate']:.2%} below floor {floor:.0%}")
    return failures


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--eval-set", required=True)
    parser.add_argument("--interpret-url", default="http://127.0.0.1:3112")
    parser.add_argument("--images-dir", required=True)
    parser.add_argument("--report", help="optional JSON report output path")
    args = parser.parse_args()

    eval_path = Path(args.eval_set)
    cases = load_eval_set(eval_path)
    images_dir = Path(args.images_dir)

    results: list[CaseResult] = []
    with httpx.Client() as client:
        for ec in cases:
            image_path = images_dir / ec.image_filename
            if not image_path.exists():
                print(f"fail {ec.case_id} — image missing: {image_path}", file=sys.stderr)
                results.append(
                    CaseResult(
                        case_id=ec.case_id,
                        schema_valid=False,
                        primary_correct=None,
                        rate_within_10=None,
                        axis_category_match=None,
                        bbb_match=None,
                        av_block_match=None,
                        st_elevation_match=None,
                        latency_ms=0,
                        error="image_missing",
                    )
                )
                continue

            print(f"eval  {ec.case_id} — calling interpret")
            output, latency_ms, err = call_interpret(client, args.interpret_url, image_path)
            if err is not None:
                print(f"fail {ec.case_id} — {err}", file=sys.stderr)
                results.append(
                    CaseResult(
                        case_id=ec.case_id,
                        schema_valid=False,
                        primary_correct=None,
                        rate_within_10=None,
                        axis_category_match=None,
                        bbb_match=None,
                        av_block_match=None,
                        st_elevation_match=None,
                        latency_ms=latency_ms,
                        error=err,
                    )
                )
                continue

            ec.canonical_summary["case_id"] = ec.case_id  # for scoring helper
            r = score_one(ec.canonical_summary, output or {})
            r.latency_ms = latency_ms
            results.append(r)
            print(
                f"  primary={r.primary_correct} rate={r.rate_within_10} axis={r.axis_category_match} "
                f"bbb={r.bbb_match} ste={r.st_elevation_match}  ({latency_ms}ms)"
            )

    agg = aggregate(results)
    failures = check_floors(agg)
    report = {"aggregate": agg, "results": [asdict(r) for r in results], "floor_failures": failures}

    print()
    print("==== aggregate ====")
    for m, v in agg["metrics"].items():
        if v["pass_rate"] is None:
            print(f"  {m:>22s}: (no applicable cases)")
        else:
            print(f"  {m:>22s}: {v['pass_rate']:.0%} ({v['passed']}/{v['n']})")
    print(f"  {'mean_latency_ms':>22s}: {agg['mean_latency_ms']:.0f}")

    if args.report:
        Path(args.report).write_text(json.dumps(report, indent=2), encoding="utf-8")
        print(f"\nreport written to {args.report}")

    if failures:
        print("\n!!! floor failures:")
        for f in failures:
            print(f"  - {f}")
        return 1
    return 0


if __name__ == "__main__":
    raise SystemExit(main())