#!/usr/bin/env python3
"""Dependency-free JSONL evaluation harness for deterministic LLM checks."""

from __future__ import annotations

import argparse
import json
import re
import statistics
import sys
import time
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any, Callable, Iterable


@dataclass(frozen=True)
class Result:
    case_id: str
    passed: bool
    score: float
    checks: dict[str, bool]
    latency_ms: float
    error: str | None = None


def load_jsonl(path: Path) -> list[dict[str, Any]]:
    cases: list[dict[str, Any]] = []
    with path.open(encoding="utf-8") as source:
        for line_number, line in enumerate(source, 1):
            if not line.strip():
                continue
            value = json.loads(line)
            if not isinstance(value, dict) or not value.get("id"):
                raise ValueError(f"line {line_number}: object with non-empty id required")
            cases.append(value)
    if not cases:
        raise ValueError("dataset contains no cases")
    return cases


def exact_match(output: str, expected: str) -> bool:
    return output.strip() == expected.strip()


def contains_all(output: str, required: Iterable[str]) -> bool:
    lowered = output.casefold()
    return all(value.casefold() in lowered for value in required)


def excludes_all(output: str, forbidden: Iterable[str]) -> bool:
    lowered = output.casefold()
    return all(value.casefold() not in lowered for value in forbidden)


def matches_pattern(output: str, pattern: str | None) -> bool:
    return pattern is None or re.search(pattern, output, re.MULTILINE) is not None


def grade(case: dict[str, Any], output: str) -> tuple[bool, float, dict[str, bool]]:
    expected = case.get("expected", {})
    checks: dict[str, bool] = {}
    if "exact" in expected:
        checks["exact"] = exact_match(output, str(expected["exact"]))
    checks["contains_all"] = contains_all(output, expected.get("contains", []))
    checks["excludes_all"] = excludes_all(output, expected.get("forbidden", []))
    checks["pattern"] = matches_pattern(output, expected.get("pattern"))
    score = sum(checks.values()) / len(checks)
    return all(checks.values()), score, checks


def run_cases(
    cases: list[dict[str, Any]],
    candidate: Callable[[dict[str, Any]], str],
) -> list[Result]:
    results: list[Result] = []
    for case in cases:
        started = time.perf_counter()
        try:
            output = candidate(case)
            if not isinstance(output, str):
                raise TypeError("candidate must return str")
            passed, score, checks = grade(case, output)
            error = None
        except Exception as exc:  # candidate failures are evaluation results
            passed, score, checks, error = False, 0.0, {}, f"{type(exc).__name__}: {exc}"
        results.append(
            Result(
                case_id=str(case["id"]),
                passed=passed,
                score=score,
                checks=checks,
                latency_ms=(time.perf_counter() - started) * 1000,
                error=error,
            )
        )
    return results


def summarize(results: list[Result]) -> dict[str, Any]:
    return {
        "cases": len(results),
        "passed": sum(result.passed for result in results),
        "pass_rate": sum(result.passed for result in results) / len(results),
        "mean_score": statistics.fmean(result.score for result in results),
        "median_latency_ms": statistics.median(result.latency_ms for result in results),
    }


def replay_candidate(case: dict[str, Any]) -> str:
    """Replace this adapter with a call to the system under evaluation."""
    return str(case.get("candidate_output", ""))


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("dataset", type=Path)
    parser.add_argument("--report", type=Path)
    parser.add_argument("--min-pass-rate", type=float, default=1.0)
    args = parser.parse_args()
    if not 0 <= args.min_pass_rate <= 1:
        parser.error("--min-pass-rate must be between 0 and 1")

    results = run_cases(load_jsonl(args.dataset), replay_candidate)
    report = {"summary": summarize(results), "results": [asdict(result) for result in results]}
    rendered = json.dumps(report, indent=2, sort_keys=True)
    if args.report:
        args.report.write_text(rendered + "\n", encoding="utf-8")
    print(rendered)
    return 0 if report["summary"]["pass_rate"] >= args.min_pass_rate else 1


if __name__ == "__main__":
    sys.exit(main())