209 lines
7.8 KiB
Python
209 lines
7.8 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
build_botlinks.py -- AI / crawler discovery files for msosorg.com
|
|
|
|
Generates two things from the pages that already exist on disk, so they stay
|
|
current automatically whenever a page is added or removed:
|
|
|
|
1. /llms.txt an llmstxt.org-style Markdown index (English)
|
|
2. /links-for-bots/index.html a single, lightweight, fully-linked crawl hub
|
|
covering EN / MK / SI
|
|
|
|
Run after adding or removing pages:
|
|
|
|
python tools/build_botlinks.py
|
|
|
|
It only reads each page's <title> and <meta name="description">, so it never
|
|
depends on the visual markup and is safe to re-run (idempotent).
|
|
"""
|
|
import io, os, re
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
BASE = "https://msosorg.com"
|
|
LANGS = ("en", "mk", "si")
|
|
LANG_LABEL = {"en": "English", "mk": "Македонски", "si": "Slovenščina"}
|
|
|
|
# (top-level directory under a language, human label). Order = display order.
|
|
SECTIONS = [
|
|
("student-welcome-guide", "Student guide"),
|
|
("blog", "Blog"),
|
|
("news", "News"),
|
|
("projects", "Events & projects"),
|
|
("get-to-know-slovenia", "Get to know Slovenia"),
|
|
("get-to-know-macedonia", "Get to know Macedonia"),
|
|
]
|
|
# Single top-level pages (their own index.html directly under the language dir).
|
|
MAIN_PAGES = [
|
|
"about-us", "msos-hub", "become-a-member", "support", "gallery",
|
|
"faq", "timeline-and-milestones", "legal", "privacy-policy", "cookie-policy",
|
|
]
|
|
|
|
TITLE_RE = re.compile(r"<title>(.*?)</title>", re.DOTALL | re.I)
|
|
DESC_RE = re.compile(r'<meta name="description" content="(.*?)"', re.I)
|
|
|
|
|
|
def read_meta(path):
|
|
txt = io.open(path, encoding="utf-8").read()
|
|
tm = TITLE_RE.search(txt)
|
|
dm = DESC_RE.search(txt)
|
|
title = tm.group(1).strip() if tm else ""
|
|
# drop the " - MSOS" / " | MSOS" style suffix for a cleaner label
|
|
title = re.sub(r"\s*[-|]\s*(MSOS|Macedonian Student Organisation).*$", "", title).strip() or title
|
|
desc = (dm.group(1).strip() if dm else "")
|
|
return title, desc
|
|
|
|
|
|
def page_url(path):
|
|
rel = os.path.relpath(path, ROOT).replace("\\", "/")
|
|
if rel.endswith("index.html"):
|
|
rel = rel[:-len("index.html")]
|
|
return BASE + "/" + rel
|
|
|
|
|
|
def collect(lang):
|
|
"""Return {section_label: [(title, url, desc), ...]} for one language."""
|
|
out = {}
|
|
base = os.path.join(ROOT, lang)
|
|
|
|
# sectioned content (one folder per article/subpage)
|
|
for folder, label in SECTIONS:
|
|
rows = []
|
|
sec_dir = os.path.join(base, folder)
|
|
if not os.path.isdir(sec_dir):
|
|
continue
|
|
# section landing page first
|
|
idx = os.path.join(sec_dir, "index.html")
|
|
if os.path.exists(idx):
|
|
t, d = read_meta(idx)
|
|
rows.append((t, page_url(idx), d))
|
|
# child pages
|
|
for name in sorted(os.listdir(sec_dir)):
|
|
child = os.path.join(sec_dir, name, "index.html")
|
|
if os.path.exists(child):
|
|
t, d = read_meta(child)
|
|
rows.append((t, page_url(child), d))
|
|
if rows:
|
|
out[label] = rows
|
|
|
|
# single main pages
|
|
rows = []
|
|
for name in MAIN_PAGES:
|
|
p = os.path.join(base, name, "index.html")
|
|
if os.path.exists(p):
|
|
t, d = read_meta(p)
|
|
rows.append((t, page_url(p), d))
|
|
# language home
|
|
home = os.path.join(base, "index.html")
|
|
if os.path.exists(home):
|
|
t, d = read_meta(home)
|
|
rows.insert(0, (t or "Home", page_url(home), d))
|
|
if rows:
|
|
out["Main pages"] = rows
|
|
return out
|
|
|
|
|
|
def esc(s):
|
|
return (s.replace("&", "&").replace("<", "<").replace(">", ">")
|
|
.replace('"', """))
|
|
|
|
|
|
def build_llms_txt():
|
|
"""English llmstxt.org-style index."""
|
|
data = collect("en")
|
|
lines = []
|
|
lines.append("# MSOS — Macedonian Student Organisation in Slovenia")
|
|
lines.append("")
|
|
lines.append("> MSOS is the first official Macedonian student organisation in Slovenia. "
|
|
"This site helps Macedonian students move to and study in Slovenia — "
|
|
"enrolment guides, events, community news and support. Content is "
|
|
"student-written and informative; official sources should always be verified.")
|
|
lines.append("")
|
|
lines.append(f"Full multilingual crawl index (EN/MK/SI): {BASE}/links-for-bots/")
|
|
lines.append("")
|
|
# preferred section order for llms.txt
|
|
order = ["Main pages", "Student guide", "Blog", "News", "Events & projects",
|
|
"Get to know Slovenia", "Get to know Macedonia"]
|
|
for label in order:
|
|
rows = data.get(label)
|
|
if not rows:
|
|
continue
|
|
lines.append(f"## {label}")
|
|
for title, url, desc in rows:
|
|
if desc:
|
|
lines.append(f"- [{title}]({url}): {desc}")
|
|
else:
|
|
lines.append(f"- [{title}]({url})")
|
|
lines.append("")
|
|
txt = "\n".join(lines).rstrip() + "\n"
|
|
io.open(os.path.join(ROOT, "llms.txt"), "w", encoding="utf-8", newline="\n").write(txt)
|
|
return sum(len(v) for v in data.values())
|
|
|
|
|
|
def build_links_page():
|
|
"""A single lightweight HTML hub linking every page, grouped by language + section."""
|
|
blocks = []
|
|
total = 0
|
|
for lang in LANGS:
|
|
data = collect(lang)
|
|
order = ["Main pages", "Student guide", "Blog", "News", "Events & projects",
|
|
"Get to know Slovenia", "Get to know Macedonia"]
|
|
secs = []
|
|
for label in order:
|
|
rows = data.get(label)
|
|
if not rows:
|
|
continue
|
|
items = "\n".join(
|
|
f' <li><a href="{esc(u)}">{esc(t)}</a>{(" — " + esc(d)) if d else ""}</li>'
|
|
for t, u, d in rows
|
|
)
|
|
total += len(rows)
|
|
secs.append(f' <h3>{esc(label)}</h3>\n <ul>\n{items}\n </ul>')
|
|
blocks.append(
|
|
f' <section>\n <h2 id="{lang}">{LANG_LABEL[lang]} ({lang})</h2>\n'
|
|
+ "\n".join(secs) + "\n </section>"
|
|
)
|
|
|
|
body = "\n".join(blocks)
|
|
html = f"""<!doctype html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<title>Links for bots — MSOS</title>
|
|
<meta name="description" content="A plain, fully-linked index of every page on msosorg.com in English, Macedonian and Slovenian, for search engines and AI crawlers.">
|
|
<link rel="canonical" href="{BASE}/links-for-bots/">
|
|
<meta name="robots" content="index, follow">
|
|
<style>
|
|
body {{ font-family: system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif;
|
|
max-width: 860px; margin: 0 auto; padding: 32px 20px 64px; color: #1d2939; line-height: 1.5; }}
|
|
h1 {{ font-size: 26px; margin: 0 0 8px; }}
|
|
p.intro {{ color: #475467; }}
|
|
nav.langs a {{ margin-right: 14px; font-weight: 600; }}
|
|
h2 {{ margin-top: 40px; border-bottom: 1px solid #e4e7ec; padding-bottom: 6px; }}
|
|
h3 {{ margin: 22px 0 6px; font-size: 15px; text-transform: uppercase; letter-spacing: .04em; color: #667085; }}
|
|
ul {{ margin: 0 0 8px; padding-left: 20px; }}
|
|
li {{ margin: 3px 0; }}
|
|
a {{ color: #2d738c; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>Links for bots</h1>
|
|
<p class="intro">A plain, fully-linked index of every page on msosorg.com, for search engines and AI crawlers.
|
|
See also <a href="{BASE}/llms.txt">/llms.txt</a> and <a href="{BASE}/sitemap.xml">/sitemap.xml</a>.</p>
|
|
<nav class="langs">{" ".join(f'<a href="#{l}">{LANG_LABEL[l]}</a>' for l in LANGS)}</nav>
|
|
{body}
|
|
</body>
|
|
</html>
|
|
"""
|
|
out_dir = os.path.join(ROOT, "links-for-bots")
|
|
os.makedirs(out_dir, exist_ok=True)
|
|
io.open(os.path.join(out_dir, "index.html"), "w", encoding="utf-8", newline="\n").write(html)
|
|
return total
|
|
|
|
|
|
if __name__ == "__main__":
|
|
n1 = build_llms_txt()
|
|
n2 = build_links_page()
|
|
print(f"llms.txt written ({n1} English pages listed)")
|
|
print(f"links-for-bots/index.html ({n2} links across EN/MK/SI)")
|