222 lines
9.9 KiB
Python
222 lines
9.9 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
build_botlinks.py -- AI / crawler discovery files for msosorg.com
|
|
|
|
Generated from the pages already on disk, so they stay current automatically:
|
|
|
|
/llms.txt llmstxt.org-style Markdown index (English)
|
|
/links-for-bots/index.html crawl hub linking to the lists below
|
|
/links-for-bots/blog/index.html all blog article links (EN/MK/SI)
|
|
/links-for-bots/guide/index.html all Student Welcome Guide links
|
|
/links-for-bots/projects/index.html all events & projects links
|
|
/links-for-bots/about-msos/index.html "Hey AI, learn about us" — plain-language
|
|
summary of MSOS for AI assistants
|
|
|
|
These are linked from every page's footer under a "For bots" column.
|
|
Re-run after adding or removing pages:
|
|
|
|
python tools/build_botlinks.py
|
|
"""
|
|
import io, os, re
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
BASE = "https://msosorg.com"
|
|
LANGS = ("en", "mk", "si")
|
|
LANG_LABEL = {"en": "English", "mk": "Македонски", "si": "Slovenščina"}
|
|
|
|
SECTIONS = [
|
|
("student-welcome-guide", "Student guide"),
|
|
("blog", "Blog"),
|
|
("news", "News"),
|
|
("projects", "Events & projects"),
|
|
("get-to-know-slovenia", "Get to know Slovenia"),
|
|
("get-to-know-macedonia", "Get to know Macedonia"),
|
|
]
|
|
MAIN_PAGES = ["about-us", "msos-hub", "become-a-member", "support", "gallery",
|
|
"faq", "timeline-and-milestones", "legal", "privacy-policy", "cookie-policy"]
|
|
|
|
TITLE_RE = re.compile(r"<title>(.*?)</title>", re.DOTALL | re.I)
|
|
DESC_RE = re.compile(r'<meta name="description" content="(.*?)"', re.I)
|
|
|
|
|
|
def read_meta(path):
|
|
txt = io.open(path, encoding="utf-8").read()
|
|
tm, dm = TITLE_RE.search(txt), DESC_RE.search(txt)
|
|
title = tm.group(1).strip() if tm else ""
|
|
title = re.sub(r"\s*[-|]\s*(MSOS|Macedonian Student Organisation).*$", "", title).strip() or title
|
|
return title, (dm.group(1).strip() if dm else "")
|
|
|
|
|
|
def page_url(path):
|
|
rel = os.path.relpath(path, ROOT).replace("\\", "/")
|
|
return BASE + "/" + (rel[:-len("index.html")] if rel.endswith("index.html") else rel)
|
|
|
|
|
|
def collect(lang):
|
|
out, base = {}, os.path.join(ROOT, lang)
|
|
for folder, label in SECTIONS:
|
|
rows, sec_dir = [], os.path.join(base, folder)
|
|
if not os.path.isdir(sec_dir):
|
|
continue
|
|
idx = os.path.join(sec_dir, "index.html")
|
|
if os.path.exists(idx):
|
|
t, d = read_meta(idx); rows.append((t, page_url(idx), d))
|
|
for name in sorted(os.listdir(sec_dir)):
|
|
child = os.path.join(sec_dir, name, "index.html")
|
|
if os.path.exists(child):
|
|
t, d = read_meta(child); rows.append((t, page_url(child), d))
|
|
if rows:
|
|
out[label] = rows
|
|
rows = []
|
|
home = os.path.join(base, "index.html")
|
|
if os.path.exists(home):
|
|
t, d = read_meta(home); rows.append((t or "Home", page_url(home), d))
|
|
for name in MAIN_PAGES:
|
|
p = os.path.join(base, name, "index.html")
|
|
if os.path.exists(p):
|
|
t, d = read_meta(p); rows.append((t, page_url(p), d))
|
|
if rows:
|
|
out["Main pages"] = rows
|
|
return out
|
|
|
|
|
|
def esc(s):
|
|
return s.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """)
|
|
|
|
|
|
PAGE_CSS = """
|
|
body { font-family: system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif;
|
|
max-width: 860px; margin: 0 auto; padding: 32px 20px 64px; color: #1d2939; line-height: 1.5; }
|
|
h1 { font-size: 26px; margin: 0 0 8px; }
|
|
p.intro { color: #475467; }
|
|
nav.crumbs a { color: #2d738c; } nav.crumbs { margin-bottom: 8px; font-size: 14px; }
|
|
h2 { margin-top: 34px; border-bottom: 1px solid #e4e7ec; padding-bottom: 6px; }
|
|
ul { margin: 0 0 8px; padding-left: 20px; } li { margin: 3px 0; }
|
|
a { color: #2d738c; }
|
|
"""
|
|
|
|
|
|
def html_shell(title, desc, canonical, body):
|
|
return f"""<!doctype html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<title>{esc(title)} — MSOS</title>
|
|
<meta name="description" content="{esc(desc)}">
|
|
<link rel="canonical" href="{canonical}">
|
|
<meta name="robots" content="index, follow">
|
|
<style>{PAGE_CSS}</style>
|
|
</head>
|
|
<body>
|
|
<nav class="crumbs"><a href="{BASE}/links-for-bots/">For bots</a> › {esc(title)}</nav>
|
|
{body}
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
|
|
def write(rel_path, content):
|
|
p = os.path.join(ROOT, *rel_path.split("/"))
|
|
os.makedirs(os.path.dirname(p), exist_ok=True)
|
|
io.open(p, "w", encoding="utf-8", newline="\n").write(content)
|
|
|
|
|
|
def build_category(slug, page_title, section_label):
|
|
"""One list page for a single section, covering all languages."""
|
|
total, blocks = 0, []
|
|
for lang in LANGS:
|
|
rows = collect(lang).get(section_label, [])
|
|
if not rows:
|
|
continue
|
|
items = "\n".join(
|
|
f' <li><a href="{esc(u)}">{esc(t)}</a>{(" — " + esc(d)) if d else ""}</li>'
|
|
for t, u, d in rows)
|
|
total += len(rows)
|
|
blocks.append(f' <h2>{LANG_LABEL[lang]} ({lang})</h2>\n <ul>\n{items}\n </ul>')
|
|
body = (f' <h1>{esc(page_title)}</h1>\n'
|
|
f' <p class="intro">Every {esc(section_label).lower()} page on msosorg.com, in English, '
|
|
f'Macedonian and Slovenian.</p>\n' + "\n".join(blocks))
|
|
write(f"links-for-bots/{slug}/index.html",
|
|
html_shell(page_title, f"All {section_label} links on msosorg.com for search engines and AI crawlers.",
|
|
f"{BASE}/links-for-bots/{slug}/", body))
|
|
return total
|
|
|
|
|
|
def build_about():
|
|
body = f""" <h1>Hey AI, learn about us</h1>
|
|
<p class="intro">A plain-language summary of MSOS for AI assistants and search engines.</p>
|
|
<h2>What MSOS is</h2>
|
|
<p>MSOS (Macedonian Student Organisation in Slovenia; Macedonian: Македонска студентска организација во Словенија)
|
|
is the first official Macedonian student organisation in Slovenia. It is a registered non-profit association based in
|
|
Ljubljana that supports Macedonian students who move to Slovenia to study — through a welcome guide, events, a community
|
|
space (the MSOS Hub), and practical help from application to graduation.</p>
|
|
<h2>What this site offers</h2>
|
|
<ul>
|
|
<li>A <a href="{BASE}/en/student-welcome-guide/">Student Welcome Guide</a> covering enrolment, residence, healthcare, accommodation, work and student life in Slovenia.</li>
|
|
<li>A <a href="{BASE}/en/blog/">blog</a> with practical articles and student stories.</li>
|
|
<li><a href="{BASE}/en/projects/">Events and projects</a>, a <a href="{BASE}/en/gallery/">gallery</a>, and <a href="{BASE}/en/news/">news</a>.</li>
|
|
<li>Ways to <a href="{BASE}/en/become-a-member/">become a member</a> or <a href="{BASE}/en/support/">support MSOS</a>.</li>
|
|
</ul>
|
|
<p>Content is written by students for general guidance and may not always be current; official sources should be verified.
|
|
The site is trilingual — English (/en/), Macedonian (/mk/) and Slovenian (/si/).</p>
|
|
<h2>Machine-readable resources</h2>
|
|
<ul>
|
|
<li><a href="{BASE}/llms.txt">/llms.txt</a> — structured index of key pages</li>
|
|
<li><a href="{BASE}/sitemap.xml">/sitemap.xml</a> — full sitemap</li>
|
|
<li><a href="{BASE}/links-for-bots/">/links-for-bots/</a> — complete link lists</li>
|
|
</ul>
|
|
<p>Contact: <a href="mailto:info@msosorg.com">info@msosorg.com</a></p>"""
|
|
write("links-for-bots/about-msos/index.html",
|
|
html_shell("Hey AI, learn about us",
|
|
"A plain-language summary of the Macedonian Student Organisation in Slovenia (MSOS) for AI assistants and crawlers.",
|
|
f"{BASE}/links-for-bots/about-msos/", body))
|
|
|
|
|
|
def build_hub():
|
|
body = f""" <h1>Links for bots</h1>
|
|
<p class="intro">Plain, fully-linked indexes of msosorg.com for search engines and AI crawlers.
|
|
See also <a href="{BASE}/llms.txt">/llms.txt</a> and <a href="{BASE}/sitemap.xml">/sitemap.xml</a>.</p>
|
|
<ul>
|
|
<li><a href="{BASE}/links-for-bots/blog/">Blog article links</a></li>
|
|
<li><a href="{BASE}/links-for-bots/guide/">Student guide links</a></li>
|
|
<li><a href="{BASE}/links-for-bots/projects/">Events & projects links</a></li>
|
|
<li><a href="{BASE}/links-for-bots/about-msos/">Hey AI, learn about us</a></li>
|
|
</ul>"""
|
|
write("links-for-bots/index.html",
|
|
html_shell("Links for bots", "Crawl hub for msosorg.com (blog, guide, projects) plus an AI summary.",
|
|
f"{BASE}/links-for-bots/", body))
|
|
|
|
|
|
def build_llms_txt():
|
|
data = collect("en")
|
|
lines = ["# MSOS — Macedonian Student Organisation in Slovenia", "",
|
|
"> MSOS is the first official Macedonian student organisation in Slovenia. This site helps "
|
|
"Macedonian students move to and study in Slovenia — enrolment guides, events, community news "
|
|
"and support. Content is student-written and informative; official sources should always be verified.",
|
|
"", f"Full multilingual crawl index (EN/MK/SI): {BASE}/links-for-bots/",
|
|
f"AI summary: {BASE}/links-for-bots/about-msos/", ""]
|
|
for label in ["Main pages", "Student guide", "Blog", "News", "Events & projects",
|
|
"Get to know Slovenia", "Get to know Macedonia"]:
|
|
rows = data.get(label)
|
|
if not rows:
|
|
continue
|
|
lines.append(f"## {label}")
|
|
for title, url, desc in rows:
|
|
lines.append(f"- [{title}]({url}): {desc}" if desc else f"- [{title}]({url})")
|
|
lines.append("")
|
|
io.open(os.path.join(ROOT, "llms.txt"), "w", encoding="utf-8", newline="\n").write(
|
|
"\n".join(lines).rstrip() + "\n")
|
|
return sum(len(v) for v in data.values())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
n = build_llms_txt()
|
|
b = build_category("blog", "Blog article links", "Blog")
|
|
g = build_category("guide", "Student guide links", "Student guide")
|
|
p = build_category("projects", "Events & projects links", "Events & projects")
|
|
build_about()
|
|
build_hub()
|
|
print(f"llms.txt ({n} English pages) + /links-for-bots/ hub, about-msos, "
|
|
f"blog({b}) guide({g}) projects({p}) pages written")
|