instant-bom/bom_assistant/suppliers/lcsc/product_lookup.py

#!/usr/bin/env python3
"""
Ground-truth helper: fetch a real LCSC product's actual category by part number.

Scrapes https://www.lcsc.com/product-detail/{part}.html, which embeds a plain
JSON blob in <script id="__NEXT_DATA__">. The relevant fields live at
props.pageProps.webData: `catalogId`/`catalogName` (leaf category — matches
the `id`/`name` used in categories.json) and `parentCatalogId`/`parentCatalogName`.

Usage (library):
    from bom_assistant.suppliers.lcsc.product_lookup import lookup_product
    info = lookup_product("C437643")
    # ProductInfo(part="C437643", category_id=1140, category_name="Aluminum Electrolytic Capacitors",
    #             parent_category_name="Capacitors", product_name="CAP ALUM 330uF ±20% 63V TH")
"""
from __future__ import annotations

import json
import random
import re
import time
import urllib.request
from dataclasses import dataclass

_DETAIL_URL = "https://www.lcsc.com/product-detail/{part}.html"
_UA = "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0"
_GRACE = 1.5
_JITTER = 0.5
_RETRY_WAIT = 5.0

_NEXT_DATA_RE = re.compile(
    r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', re.DOTALL
)


@dataclass
class ProductInfo:
    part: str
    category_id: int
    category_name: str
    parent_category_name: str
    product_name: str


def _fetch(url: str) -> str | None:
    req = urllib.request.Request(
        url, headers={"User-Agent": _UA, "Accept-Language": "en-US,en;q=0.9"}
    )
    for attempt in range(2):
        try:
            with urllib.request.urlopen(req, timeout=15) as resp:
                return resp.read().decode("utf-8", errors="replace")
        except Exception as exc:
            if attempt == 0:
                print(f"  warn: {exc} — retrying in {_RETRY_WAIT}s")
                time.sleep(_RETRY_WAIT)
            else:
                return None
    return None


def sleep_between_requests() -> None:
    time.sleep(max(0.5, _GRACE + random.uniform(-_JITTER, _JITTER)))


def lookup_product(part: str) -> ProductInfo | None:
    """Fetch the real LCSC category for a part number. Returns None on 404/parse failure."""
    html = _fetch(_DETAIL_URL.format(part=part))
    if html is None:
        return None

    m = _NEXT_DATA_RE.search(html)
    if not m:
        return None

    try:
        data = json.loads(m.group(1))
        web_data = data["props"]["pageProps"].get("webData")
    except (json.JSONDecodeError, KeyError):
        return None

    if not web_data or not web_data.get("catalogId"):
        return None

    return ProductInfo(
        part=part,
        category_id=web_data["catalogId"],
        category_name=web_data.get("catalogName", ""),
        parent_category_name=web_data.get("parentCatalogName", ""),
        product_name=web_data.get("productNameEn", ""),
    )


if __name__ == "__main__":
    import sys

    for p in sys.argv[1:]:
        info = lookup_product(p)
        print(f"{p}: {info}")
        sleep_between_requests()