80 lines
3.3 KiB
Python
80 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Build per-language search indexes for the offline site assistant.
|
|
Parses every EN/MK/SI page, chunks the main content by heading, and writes
|
|
assistant/index-<lang>.json (loaded lazily in the browser, no API needed)."""
|
|
import os, re, json, glob
|
|
from bs4 import BeautifulSoup
|
|
|
|
ROOT = r"C:\Coding\msosorg.com"
|
|
os.chdir(ROOT)
|
|
OUT = os.path.join(ROOT, "assistant")
|
|
os.makedirs(OUT, exist_ok=True)
|
|
|
|
DROP_SUFFIX = re.compile(r"\s*[-|]\s*(MSOS|Macedonian Student Organisation.*|Македонска.*|Makedonska.*)\s*$", re.I)
|
|
WS = re.compile(r"\s+")
|
|
|
|
def clean(t):
|
|
return WS.sub(" ", (t or "")).strip()
|
|
|
|
def page_url(path):
|
|
# en/foo/index.html -> /en/foo/
|
|
rel = path.replace("\\", "/")
|
|
rel = rel[:-len("index.html")] if rel.endswith("index.html") else rel
|
|
return "/" + rel.strip("/") + "/"
|
|
|
|
def build(lang):
|
|
chunks = []
|
|
pages = sorted(glob.glob(f"{lang}/**/index.html", recursive=True)) + \
|
|
([f"{lang}/index.html"] if os.path.exists(f"{lang}/index.html") else [])
|
|
seen = set()
|
|
for path in pages:
|
|
if path in seen: continue
|
|
seen.add(path)
|
|
html = open(path, encoding="utf-8").read()
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
title = clean(soup.title.get_text()) if soup.title else ""
|
|
title_short = DROP_SUFFIX.sub("", title).strip() or title
|
|
desc_el = soup.find("meta", attrs={"name": "description"})
|
|
desc = clean(desc_el["content"]) if desc_el and desc_el.get("content") else ""
|
|
main = soup.find("main") or soup.body
|
|
if not main: continue
|
|
# strip non-content
|
|
for sel in ["nav", "header", "footer", "script", "style", "form",
|
|
".guide-help", ".ms-section", ".mm-sticky-cta", "button"]:
|
|
for el in main.select(sel): el.decompose()
|
|
url = page_url(path)
|
|
|
|
# Landing chunk: title + description + first paragraph
|
|
first_p = main.find("p")
|
|
intro = desc or (clean(first_p.get_text()) if first_p else "")
|
|
chunks.append({"u": url, "t": title_short, "h": title_short,
|
|
"x": clean((title_short + ". " + intro))[:600], "lang": lang})
|
|
|
|
# Section chunks: each h2/h3 + following text until the next heading
|
|
headings = main.find_all(["h2", "h3"])
|
|
for hd in headings:
|
|
htext = clean(hd.get_text())
|
|
if not htext: continue
|
|
parts = []
|
|
for sib in hd.next_siblings:
|
|
name = getattr(sib, "name", None)
|
|
if name in ("h2", "h3"): break
|
|
if name in ("p", "ul", "ol", "li", "table", "figure", "div"):
|
|
parts.append(clean(sib.get_text(" ")))
|
|
if sum(len(p) for p in parts) > 700: break
|
|
body = clean(" ".join([p for p in parts if p]))
|
|
if len(htext) + len(body) < 25: # skip near-empty sections
|
|
continue
|
|
chunks.append({"u": url, "t": title_short, "h": htext,
|
|
"x": clean(htext + ". " + body)[:600], "lang": lang})
|
|
return chunks
|
|
|
|
summary = {}
|
|
for lang in ("en", "mk", "si"):
|
|
ch = build(lang)
|
|
out = os.path.join(OUT, f"index-{lang}.json")
|
|
json.dump(ch, open(out, "w", encoding="utf-8"), ensure_ascii=False, separators=(",", ":"))
|
|
summary[lang] = {"chunks": len(ch), "kb": round(os.path.getsize(out)/1024, 1)}
|
|
|
|
print(json.dumps(summary, indent=2))
|