#!/usr/bin/env python3
"""Build per-language search indexes for the offline site assistant.
Parses every EN/MK/SI page, chunks the main content by heading, and writes
assistant/index-<lang>.json  (loaded lazily in the browser, no API needed)."""
import os, re, json, glob
from bs4 import BeautifulSoup

# Repo root = parent of this script's tools/ folder. Works locally and in CI
# (override with SITE_ROOT env var if ever needed).
ROOT = os.environ.get("SITE_ROOT") or os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
os.chdir(ROOT)
OUT = os.path.join(ROOT, "assistant")
os.makedirs(OUT, exist_ok=True)

DROP_SUFFIX = re.compile(r"\s*[-|]\s*(MSOS|Macedonian Student Organisation.*|Македонска.*|Makedonska.*)\s*$", re.I)
WS = re.compile(r"\s+")

def clean(t):
    return WS.sub(" ", (t or "")).strip()

def page_url(path):
    # en/foo/index.html -> /en/foo/
    rel = path.replace("\\", "/")
    rel = rel[:-len("index.html")] if rel.endswith("index.html") else rel
    return "/" + rel.strip("/") + "/"

def build(lang):
    chunks = []
    pages = sorted(glob.glob(f"{lang}/**/index.html", recursive=True)) + \
            ([f"{lang}/index.html"] if os.path.exists(f"{lang}/index.html") else [])
    seen = set()
    for path in pages:
        if path in seen: continue
        seen.add(path)
        html = open(path, encoding="utf-8").read()
        soup = BeautifulSoup(html, "html.parser")
        title = clean(soup.title.get_text()) if soup.title else ""
        title_short = DROP_SUFFIX.sub("", title).strip() or title
        desc_el = soup.find("meta", attrs={"name": "description"})
        desc = clean(desc_el["content"]) if desc_el and desc_el.get("content") else ""
        main = soup.find("main") or soup.body
        if not main: continue
        # strip non-content
        for sel in ["nav", "header", "footer", "script", "style", "form",
                    ".guide-help", ".ms-section", ".mm-sticky-cta", "button"]:
            for el in main.select(sel): el.decompose()
        url = page_url(path)

        # Landing chunk: title + description + first paragraph
        first_p = main.find("p")
        intro = desc or (clean(first_p.get_text()) if first_p else "")
        chunks.append({"u": url, "t": title_short, "h": title_short,
                       "x": clean((title_short + ". " + intro))[:600], "lang": lang})

        # Section chunks: each h2/h3 + following text until the next heading
        headings = main.find_all(["h2", "h3"])
        for hd in headings:
            htext = clean(hd.get_text())
            if not htext: continue
            parts = []
            for sib in hd.next_siblings:
                name = getattr(sib, "name", None)
                if name in ("h2", "h3"): break
                if name in ("p", "ul", "ol", "li", "table", "figure", "div"):
                    parts.append(clean(sib.get_text(" ")))
                if sum(len(p) for p in parts) > 700: break
            body = clean(" ".join([p for p in parts if p]))
            if len(htext) + len(body) < 25:  # skip near-empty sections
                continue
            chunks.append({"u": url, "t": title_short, "h": htext,
                           "x": clean(htext + ". " + body)[:600], "lang": lang})
    return chunks

summary = {}
for lang in ("en", "mk", "si"):
    ch = build(lang)
    out = os.path.join(OUT, f"index-{lang}.json")
    json.dump(ch, open(out, "w", encoding="utf-8"), ensure_ascii=False, separators=(",", ":"))
    summary[lang] = {"chunks": len(ch), "kb": round(os.path.getsize(out)/1024, 1)}

print(json.dumps(summary, indent=2))