#!/usr/bin/env python3
"""
build_botlinks.py  --  AI / crawler discovery files for msosorg.com

Generates two things from the pages that already exist on disk, so they stay
current automatically whenever a page is added or removed:

  1. /llms.txt                 an llmstxt.org-style Markdown index (English)
  2. /links-for-bots/index.html a single, lightweight, fully-linked crawl hub
                                covering EN / MK / SI

Run after adding or removing pages:

    python tools/build_botlinks.py

It only reads each page's <title> and <meta name="description">, so it never
depends on the visual markup and is safe to re-run (idempotent).
"""
import io, os, re

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
BASE = "https://msosorg.com"
LANGS = ("en", "mk", "si")
LANG_LABEL = {"en": "English", "mk": "Македонски", "si": "Slovenščina"}

# (top-level directory under a language, human label). Order = display order.
SECTIONS = [
    ("student-welcome-guide", "Student guide"),
    ("blog", "Blog"),
    ("news", "News"),
    ("projects", "Events & projects"),
    ("get-to-know-slovenia", "Get to know Slovenia"),
    ("get-to-know-macedonia", "Get to know Macedonia"),
]
# Single top-level pages (their own index.html directly under the language dir).
MAIN_PAGES = [
    "about-us", "msos-hub", "become-a-member", "support", "gallery",
    "faq", "timeline-and-milestones", "legal", "privacy-policy", "cookie-policy",
]

TITLE_RE = re.compile(r"<title>(.*?)</title>", re.DOTALL | re.I)
DESC_RE = re.compile(r'<meta name="description" content="(.*?)"', re.I)


def read_meta(path):
    txt = io.open(path, encoding="utf-8").read()
    tm = TITLE_RE.search(txt)
    dm = DESC_RE.search(txt)
    title = tm.group(1).strip() if tm else ""
    # drop the " - MSOS" / " | MSOS" style suffix for a cleaner label
    title = re.sub(r"\s*[-|]\s*(MSOS|Macedonian Student Organisation).*$", "", title).strip() or title
    desc = (dm.group(1).strip() if dm else "")
    return title, desc


def page_url(path):
    rel = os.path.relpath(path, ROOT).replace("\\", "/")
    if rel.endswith("index.html"):
        rel = rel[:-len("index.html")]
    return BASE + "/" + rel


def collect(lang):
    """Return {section_label: [(title, url, desc), ...]} for one language."""
    out = {}
    base = os.path.join(ROOT, lang)

    # sectioned content (one folder per article/subpage)
    for folder, label in SECTIONS:
        rows = []
        sec_dir = os.path.join(base, folder)
        if not os.path.isdir(sec_dir):
            continue
        # section landing page first
        idx = os.path.join(sec_dir, "index.html")
        if os.path.exists(idx):
            t, d = read_meta(idx)
            rows.append((t, page_url(idx), d))
        # child pages
        for name in sorted(os.listdir(sec_dir)):
            child = os.path.join(sec_dir, name, "index.html")
            if os.path.exists(child):
                t, d = read_meta(child)
                rows.append((t, page_url(child), d))
        if rows:
            out[label] = rows

    # single main pages
    rows = []
    for name in MAIN_PAGES:
        p = os.path.join(base, name, "index.html")
        if os.path.exists(p):
            t, d = read_meta(p)
            rows.append((t, page_url(p), d))
    # language home
    home = os.path.join(base, "index.html")
    if os.path.exists(home):
        t, d = read_meta(home)
        rows.insert(0, (t or "Home", page_url(home), d))
    if rows:
        out["Main pages"] = rows
    return out


def esc(s):
    return (s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
             .replace('"', "&quot;"))


def build_llms_txt():
    """English llmstxt.org-style index."""
    data = collect("en")
    lines = []
    lines.append("# MSOS — Macedonian Student Organisation in Slovenia")
    lines.append("")
    lines.append("> MSOS is the first official Macedonian student organisation in Slovenia. "
                 "This site helps Macedonian students move to and study in Slovenia — "
                 "enrolment guides, events, community news and support. Content is "
                 "student-written and informative; official sources should always be verified.")
    lines.append("")
    lines.append(f"Full multilingual crawl index (EN/MK/SI): {BASE}/links-for-bots/")
    lines.append("")
    # preferred section order for llms.txt
    order = ["Main pages", "Student guide", "Blog", "News", "Events & projects",
             "Get to know Slovenia", "Get to know Macedonia"]
    for label in order:
        rows = data.get(label)
        if not rows:
            continue
        lines.append(f"## {label}")
        for title, url, desc in rows:
            if desc:
                lines.append(f"- [{title}]({url}): {desc}")
            else:
                lines.append(f"- [{title}]({url})")
        lines.append("")
    txt = "\n".join(lines).rstrip() + "\n"
    io.open(os.path.join(ROOT, "llms.txt"), "w", encoding="utf-8", newline="\n").write(txt)
    return sum(len(v) for v in data.values())


def build_links_page():
    """A single lightweight HTML hub linking every page, grouped by language + section."""
    blocks = []
    total = 0
    for lang in LANGS:
        data = collect(lang)
        order = ["Main pages", "Student guide", "Blog", "News", "Events & projects",
                 "Get to know Slovenia", "Get to know Macedonia"]
        secs = []
        for label in order:
            rows = data.get(label)
            if not rows:
                continue
            items = "\n".join(
                f'      <li><a href="{esc(u)}">{esc(t)}</a>{(" — " + esc(d)) if d else ""}</li>'
                for t, u, d in rows
            )
            total += len(rows)
            secs.append(f'    <h3>{esc(label)}</h3>\n    <ul>\n{items}\n    </ul>')
        blocks.append(
            f'  <section>\n    <h2 id="{lang}">{LANG_LABEL[lang]} ({lang})</h2>\n'
            + "\n".join(secs) + "\n  </section>"
        )

    body = "\n".join(blocks)
    html = f"""<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Links for bots — MSOS</title>
<meta name="description" content="A plain, fully-linked index of every page on msosorg.com in English, Macedonian and Slovenian, for search engines and AI crawlers.">
<link rel="canonical" href="{BASE}/links-for-bots/">
<meta name="robots" content="index, follow">
<style>
  body {{ font-family: system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif;
          max-width: 860px; margin: 0 auto; padding: 32px 20px 64px; color: #1d2939; line-height: 1.5; }}
  h1 {{ font-size: 26px; margin: 0 0 8px; }}
  p.intro {{ color: #475467; }}
  nav.langs a {{ margin-right: 14px; font-weight: 600; }}
  h2 {{ margin-top: 40px; border-bottom: 1px solid #e4e7ec; padding-bottom: 6px; }}
  h3 {{ margin: 22px 0 6px; font-size: 15px; text-transform: uppercase; letter-spacing: .04em; color: #667085; }}
  ul {{ margin: 0 0 8px; padding-left: 20px; }}
  li {{ margin: 3px 0; }}
  a {{ color: #2d738c; }}
</style>
</head>
<body>
  <h1>Links for bots</h1>
  <p class="intro">A plain, fully-linked index of every page on msosorg.com, for search engines and AI crawlers.
  See also <a href="{BASE}/llms.txt">/llms.txt</a> and <a href="{BASE}/sitemap.xml">/sitemap.xml</a>.</p>
  <nav class="langs">{" ".join(f'<a href="#{l}">{LANG_LABEL[l]}</a>' for l in LANGS)}</nav>
{body}
</body>
</html>
"""
    out_dir = os.path.join(ROOT, "links-for-bots")
    os.makedirs(out_dir, exist_ok=True)
    io.open(os.path.join(out_dir, "index.html"), "w", encoding="utf-8", newline="\n").write(html)
    return total


if __name__ == "__main__":
    n1 = build_llms_txt()
    n2 = build_links_page()
    print(f"llms.txt written           ({n1} English pages listed)")
    print(f"links-for-bots/index.html  ({n2} links across EN/MK/SI)")