"""
audit_programs.py — the test that was missing: run EVERY approved question with a program set,
and flag the ones that wrongly escalate (a real user on that program would get no answer).

For each answerable KB entry, ask its own question with program=P3 and program=P9. If the entry
applies to that program (tag is 'all' or includes it) but the bot escalates, that is a gap to fix.

Output: eval/audit_programs_report.md (the gap list, grouped) + eval/audit_programs.json
Run:  python3 eval/audit_programs.py
"""
import json
import os
import sys
from concurrent.futures import ThreadPoolExecutor

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT)

import app  # noqa: E402
from app import ChatRequest  # noqa: E402

KB = [e for e in json.load(open(os.path.join(ROOT, "data", "kb.json")))
      if not e.get("coach_only") and not e.get("has_placeholder")]


def applies(entry, prog):
    return "all" in entry["programs"] or prog in entry["programs"]


def check(args):
    entry, prog = args
    try:
        o = app.chat(ChatRequest(message=entry["question"], program=prog))
        answered = not o["escalate"] and o.get("escalate_reason") != "out_of_scope"
        return {"q": entry["question"], "prog": prog, "tag": "/".join(entry["programs"]),
                "source": entry["source"], "answered": answered, "reason": o.get("escalate_reason")}
    except Exception as e:
        return {"q": entry["question"], "prog": prog, "tag": "/".join(entry["programs"]),
                "source": entry["source"], "answered": False, "reason": "ERR:" + str(e)[:40]}


def main():
    tasks = []
    for e in KB:
        for prog in ("P3", "P9"):
            # only test programs the entry is supposed to apply to (that's where escalating = a gap)
            if applies(e, prog):
                tasks.append((e, prog))
    print(f"Auditing {len(tasks)} (question x applicable-program) pairs...")

    results = []
    with ThreadPoolExecutor(max_workers=8) as ex:
        for i, r in enumerate(ex.map(check, tasks)):
            results.append(r)
            if (i + 1) % 150 == 0:
                print(f"  {i+1}/{len(tasks)}")

    gaps = [r for r in results if not r["answered"]]
    json.dump(results, open(os.path.join(ROOT, "eval", "audit_programs.json"), "w"), indent=2, ensure_ascii=False)

    L = ["# Program audit — questions that wrongly escalate for a program they apply to\n"]
    L.append(f"Checked {len(tasks)} pairs. **{len(gaps)} gaps** ({100*len(gaps)//max(len(tasks),1)}%).\n")
    # group gaps by source sheet
    from collections import defaultdict
    by_src = defaultdict(list)
    for g in gaps:
        by_src[g["source"]].append(g)
    for src, items in sorted(by_src.items(), key=lambda x: -len(x[1])):
        L.append(f"## {src} ({len(items)})")
        for g in items:
            L.append(f"- [P{g['prog'][-1]}, tag {g['tag']}] {g['q'][:70]}  ({g['reason']})")
        L.append("")

    open(os.path.join(ROOT, "eval", "audit_programs_report.md"), "w").write("\n".join(L))
    print(f"\n{len(gaps)} gaps found. Report -> eval/audit_programs_report.md")
    print("\n".join(L[:3]))


if __name__ == "__main__":
    main()