instant-bom/bom_assistant/suppliers/lcsc/eval_ground_truth.py

#!/usr/bin/env python3
"""
One-off: measure current LCSC category-resolver accuracy against
ground_truth.ndjson (built by scout_ground_truth.py).

For each ground-truth row, re-runs the normal ingestion pipeline on its
source BOM file to get the same BomRow the app would produce, calls the
real resolver, and compares its top-3 guess against the real LCSC category.

Also flags cases where the real category isn't even in the candidate list
`categories.get_subcategories_for()` offers for that row's ComponentCategory —
those are candidate-list gaps, not resolver mistakes.

Usage:
    python -m bom_assistant.suppliers.lcsc.eval_ground_truth
"""
from __future__ import annotations

import json
from pathlib import Path

from bom_assistant.ingestion.column_mapper import map_columns
from bom_assistant.ingestion.normalizer import normalize
from bom_assistant.ingestion.parser import parse_bom
from bom_assistant.session.models import BomRow
from bom_assistant.suppliers.lcsc.categories import get_subcategories_for
from bom_assistant.suppliers.lcsc.category_resolver import resolve

_EXAMPLES_DIR = Path(__file__).resolve().parents[3] / "examples" / "full"
_GT_PATH = Path(__file__).parent / "ground_truth.ndjson"


def _rows_for(source: str) -> list[BomRow]:
    path = _EXAMPLES_DIR / source
    raw = parse_bom(path.read_bytes(), path.name)
    mapped = map_columns(raw)
    return normalize(mapped)


def main() -> None:
    if not _GT_PATH.exists():
        print(f"no ground truth found at {_GT_PATH} — run scout_ground_truth.py first")
        return

    gt_entries = [json.loads(line) for line in _GT_PATH.read_text(encoding="utf-8").splitlines() if line.strip()]

    rows_by_source: dict[str, list[BomRow]] = {}
    for entry in gt_entries:
        src = entry["source"]
        if src not in rows_by_source:
            rows_by_source[src] = _rows_for(src)

    top1 = top3 = candidate_gap = total = 0
    mismatches: list[str] = []

    for entry in gt_entries:
        rows = rows_by_source[entry["source"]]
        row = next((r for r in rows if entry["designator"] in r.designators), None)
        if row is None:
            print(f"warn: no matching row for designator {entry['designator']!r} in {entry['source']}")
            continue

        total += 1
        real_id = entry["category_id"]
        real_name = entry["category_name"]

        candidates = get_subcategories_for(row.category)
        candidate_ids = {c["id"] for c in candidates}
        in_candidates = real_id in candidate_ids
        if not in_candidates:
            candidate_gap += 1

        guesses = resolve(row)
        guess_ids = [g.id for g in guesses]

        hit1 = bool(guess_ids) and guess_ids[0] == real_id
        hit3 = real_id in guess_ids
        top1 += hit1
        top3 += hit3

        if not hit3:
            guess_str = " | ".join(f"{g.name}({g.id})" for g in guesses) if guesses else "???"
            gap_note = " [NOT IN CANDIDATE LIST]" if not in_candidates else ""
            mismatches.append(
                f"  {entry['designator']:6} [{row.category.value:10}] val={entry['value']!r:20} "
                f"real={real_name}({real_id}){gap_note} guessed={guess_str}"
            )

    print(f"\n=== Eval results ({total} ground-truth rows) ===")
    print(f"top-1 accuracy: {top1}/{total} ({top1 / total:.0%})" if total else "no rows")
    print(f"top-3 accuracy: {top3}/{total} ({top3 / total:.0%})" if total else "")
    print(f"real category missing from candidate list: {candidate_gap}/{total}")

    if mismatches:
        print(f"\n=== Mismatches ({len(mismatches)}) ===")
        for m in mismatches:
            print(m)


if __name__ == "__main__":
    main()