Bot pages: expose them in the footer under a "For bots" column
Previously /llms.txt and /links-for-bots/ existed but were not linked anywhere, so users/bots couldn't find them. Following Proxify's pattern, add a "For bots" footer column (all 225 pages, EN/MK/SI) linking to category link lists and an AI summary page. build_botlinks.py now also generates: /links-for-bots/blog|guide|projects/ per-section link lists (EN/MK/SI) /links-for-bots/about-msos/ "Hey AI, learn about us" summary /links-for-bots/ hub + /llms.txt Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
popovskikandClaude Opus 4.8 committed
1 parent
b79ad2d0de
commit
ac052c2b09
238 files changed
+2086
-444
No files matched your search
+147
-134
@@ -2,19 +2,20 @@
|
||||
"""
|
||||
build_botlinks.py -- AI / crawler discovery files for msosorg.com
|
||||
|
||||
Generates two things from the pages that already exist on disk, so they stay
|
||||
current automatically whenever a page is added or removed:
|
||||
Generated from the pages already on disk, so they stay current automatically:
|
||||
|
||||
1. /llms.txt an llmstxt.org-style Markdown index (English)
|
||||
2. /links-for-bots/index.html a single, lightweight, fully-linked crawl hub
|
||||
covering EN / MK / SI
|
||||
/llms.txt llmstxt.org-style Markdown index (English)
|
||||
/links-for-bots/index.html crawl hub linking to the lists below
|
||||
/links-for-bots/blog/index.html all blog article links (EN/MK/SI)
|
||||
/links-for-bots/guide/index.html all Student Welcome Guide links
|
||||
/links-for-bots/projects/index.html all events & projects links
|
||||
/links-for-bots/about-msos/index.html "Hey AI, learn about us" — plain-language
|
||||
summary of MSOS for AI assistants
|
||||
|
||||
Run after adding or removing pages:
|
||||
These are linked from every page's footer under a "For bots" column.
|
||||
Re-run after adding or removing pages:
|
||||
|
||||
python tools/build_botlinks.py
|
||||
|
||||
It only reads each page's <title> and <meta name="description">, so it never
|
||||
depends on the visual markup and is safe to re-run (idempotent).
|
||||
"""
|
||||
import io, os, re
|
||||
|
||||
@@ -23,7 +24,6 @@ BASE = "https://msosorg.com"
|
||||
LANGS = ("en", "mk", "si")
|
||||
LANG_LABEL = {"en": "English", "mk": "Македонски", "si": "Slovenščina"}
|
||||
|
||||
# (top-level directory under a language, human label). Order = display order.
|
||||
SECTIONS = [
|
||||
("student-welcome-guide", "Student guide"),
|
||||
("blog", "Blog"),
|
||||
@@ -32,11 +32,8 @@ SECTIONS = [
|
||||
("get-to-know-slovenia", "Get to know Slovenia"),
|
||||
("get-to-know-macedonia", "Get to know Macedonia"),
|
||||
]
|
||||
# Single top-level pages (their own index.html directly under the language dir).
|
||||
MAIN_PAGES = [
|
||||
"about-us", "msos-hub", "become-a-member", "support", "gallery",
|
||||
"faq", "timeline-and-milestones", "legal", "privacy-policy", "cookie-policy",
|
||||
]
|
||||
MAIN_PAGES = ["about-us", "msos-hub", "become-a-member", "support", "gallery",
|
||||
"faq", "timeline-and-milestones", "legal", "privacy-policy", "cookie-policy"]
|
||||
|
||||
TITLE_RE = re.compile(r"<title>(.*?)</title>", re.DOTALL | re.I)
|
||||
DESC_RE = re.compile(r'<meta name="description" content="(.*?)"', re.I)
|
||||
@@ -44,165 +41,181 @@ DESC_RE = re.compile(r'<meta name="description" content="(.*?)"', re.I)
|
||||
|
||||
def read_meta(path):
|
||||
txt = io.open(path, encoding="utf-8").read()
|
||||
tm = TITLE_RE.search(txt)
|
||||
dm = DESC_RE.search(txt)
|
||||
tm, dm = TITLE_RE.search(txt), DESC_RE.search(txt)
|
||||
title = tm.group(1).strip() if tm else ""
|
||||
# drop the " - MSOS" / " | MSOS" style suffix for a cleaner label
|
||||
title = re.sub(r"\s*[-|]\s*(MSOS|Macedonian Student Organisation).*$", "", title).strip() or title
|
||||
desc = (dm.group(1).strip() if dm else "")
|
||||
return title, desc
|
||||
return title, (dm.group(1).strip() if dm else "")
|
||||
|
||||
|
||||
def page_url(path):
|
||||
rel = os.path.relpath(path, ROOT).replace("\\", "/")
|
||||
if rel.endswith("index.html"):
|
||||
rel = rel[:-len("index.html")]
|
||||
return BASE + "/" + rel
|
||||
return BASE + "/" + (rel[:-len("index.html")] if rel.endswith("index.html") else rel)
|
||||
|
||||
|
||||
def collect(lang):
|
||||
"""Return {section_label: [(title, url, desc), ...]} for one language."""
|
||||
out = {}
|
||||
base = os.path.join(ROOT, lang)
|
||||
|
||||
# sectioned content (one folder per article/subpage)
|
||||
out, base = {}, os.path.join(ROOT, lang)
|
||||
for folder, label in SECTIONS:
|
||||
rows = []
|
||||
sec_dir = os.path.join(base, folder)
|
||||
rows, sec_dir = [], os.path.join(base, folder)
|
||||
if not os.path.isdir(sec_dir):
|
||||
continue
|
||||
# section landing page first
|
||||
idx = os.path.join(sec_dir, "index.html")
|
||||
if os.path.exists(idx):
|
||||
t, d = read_meta(idx)
|
||||
rows.append((t, page_url(idx), d))
|
||||
# child pages
|
||||
t, d = read_meta(idx); rows.append((t, page_url(idx), d))
|
||||
for name in sorted(os.listdir(sec_dir)):
|
||||
child = os.path.join(sec_dir, name, "index.html")
|
||||
if os.path.exists(child):
|
||||
t, d = read_meta(child)
|
||||
rows.append((t, page_url(child), d))
|
||||
t, d = read_meta(child); rows.append((t, page_url(child), d))
|
||||
if rows:
|
||||
out[label] = rows
|
||||
|
||||
# single main pages
|
||||
rows = []
|
||||
home = os.path.join(base, "index.html")
|
||||
if os.path.exists(home):
|
||||
t, d = read_meta(home); rows.append((t or "Home", page_url(home), d))
|
||||
for name in MAIN_PAGES:
|
||||
p = os.path.join(base, name, "index.html")
|
||||
if os.path.exists(p):
|
||||
t, d = read_meta(p)
|
||||
rows.append((t, page_url(p), d))
|
||||
# language home
|
||||
home = os.path.join(base, "index.html")
|
||||
if os.path.exists(home):
|
||||
t, d = read_meta(home)
|
||||
rows.insert(0, (t or "Home", page_url(home), d))
|
||||
t, d = read_meta(p); rows.append((t, page_url(p), d))
|
||||
if rows:
|
||||
out["Main pages"] = rows
|
||||
return out
|
||||
|
||||
|
||||
def esc(s):
|
||||
return (s.replace("&", "&").replace("<", "<").replace(">", ">")
|
||||
.replace('"', """))
|
||||
return s.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """)
|
||||
|
||||
|
||||
PAGE_CSS = """
|
||||
body { font-family: system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif;
|
||||
max-width: 860px; margin: 0 auto; padding: 32px 20px 64px; color: #1d2939; line-height: 1.5; }
|
||||
h1 { font-size: 26px; margin: 0 0 8px; }
|
||||
p.intro { color: #475467; }
|
||||
nav.crumbs a { color: #2d738c; } nav.crumbs { margin-bottom: 8px; font-size: 14px; }
|
||||
h2 { margin-top: 34px; border-bottom: 1px solid #e4e7ec; padding-bottom: 6px; }
|
||||
ul { margin: 0 0 8px; padding-left: 20px; } li { margin: 3px 0; }
|
||||
a { color: #2d738c; }
|
||||
"""
|
||||
|
||||
|
||||
def html_shell(title, desc, canonical, body):
|
||||
return f"""<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{esc(title)} — MSOS</title>
|
||||
<meta name="description" content="{esc(desc)}">
|
||||
<link rel="canonical" href="{canonical}">
|
||||
<meta name="robots" content="index, follow">
|
||||
<style>{PAGE_CSS}</style>
|
||||
</head>
|
||||
<body>
|
||||
<nav class="crumbs"><a href="{BASE}/links-for-bots/">For bots</a> › {esc(title)}</nav>
|
||||
{body}
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
def write(rel_path, content):
|
||||
p = os.path.join(ROOT, *rel_path.split("/"))
|
||||
os.makedirs(os.path.dirname(p), exist_ok=True)
|
||||
io.open(p, "w", encoding="utf-8", newline="\n").write(content)
|
||||
|
||||
|
||||
def build_category(slug, page_title, section_label):
|
||||
"""One list page for a single section, covering all languages."""
|
||||
total, blocks = 0, []
|
||||
for lang in LANGS:
|
||||
rows = collect(lang).get(section_label, [])
|
||||
if not rows:
|
||||
continue
|
||||
items = "\n".join(
|
||||
f' <li><a href="{esc(u)}">{esc(t)}</a>{(" — " + esc(d)) if d else ""}</li>'
|
||||
for t, u, d in rows)
|
||||
total += len(rows)
|
||||
blocks.append(f' <h2>{LANG_LABEL[lang]} ({lang})</h2>\n <ul>\n{items}\n </ul>')
|
||||
body = (f' <h1>{esc(page_title)}</h1>\n'
|
||||
f' <p class="intro">Every {esc(section_label).lower()} page on msosorg.com, in English, '
|
||||
f'Macedonian and Slovenian.</p>\n' + "\n".join(blocks))
|
||||
write(f"links-for-bots/{slug}/index.html",
|
||||
html_shell(page_title, f"All {section_label} links on msosorg.com for search engines and AI crawlers.",
|
||||
f"{BASE}/links-for-bots/{slug}/", body))
|
||||
return total
|
||||
|
||||
|
||||
def build_about():
|
||||
body = f""" <h1>Hey AI, learn about us</h1>
|
||||
<p class="intro">A plain-language summary of MSOS for AI assistants and search engines.</p>
|
||||
<h2>What MSOS is</h2>
|
||||
<p>MSOS (Macedonian Student Organisation in Slovenia; Macedonian: Македонска студентска организација во Словенија)
|
||||
is the first official Macedonian student organisation in Slovenia. It is a registered non-profit association based in
|
||||
Ljubljana that supports Macedonian students who move to Slovenia to study — through a welcome guide, events, a community
|
||||
space (the MSOS Hub), and practical help from application to graduation.</p>
|
||||
<h2>What this site offers</h2>
|
||||
<ul>
|
||||
<li>A <a href="{BASE}/en/student-welcome-guide/">Student Welcome Guide</a> covering enrolment, residence, healthcare, accommodation, work and student life in Slovenia.</li>
|
||||
<li>A <a href="{BASE}/en/blog/">blog</a> with practical articles and student stories.</li>
|
||||
<li><a href="{BASE}/en/projects/">Events and projects</a>, a <a href="{BASE}/en/gallery/">gallery</a>, and <a href="{BASE}/en/news/">news</a>.</li>
|
||||
<li>Ways to <a href="{BASE}/en/become-a-member/">become a member</a> or <a href="{BASE}/en/support/">support MSOS</a>.</li>
|
||||
</ul>
|
||||
<p>Content is written by students for general guidance and may not always be current; official sources should be verified.
|
||||
The site is trilingual — English (/en/), Macedonian (/mk/) and Slovenian (/si/).</p>
|
||||
<h2>Machine-readable resources</h2>
|
||||
<ul>
|
||||
<li><a href="{BASE}/llms.txt">/llms.txt</a> — structured index of key pages</li>
|
||||
<li><a href="{BASE}/sitemap.xml">/sitemap.xml</a> — full sitemap</li>
|
||||
<li><a href="{BASE}/links-for-bots/">/links-for-bots/</a> — complete link lists</li>
|
||||
</ul>
|
||||
<p>Contact: <a href="mailto:info@msosorg.com">info@msosorg.com</a></p>"""
|
||||
write("links-for-bots/about-msos/index.html",
|
||||
html_shell("Hey AI, learn about us",
|
||||
"A plain-language summary of the Macedonian Student Organisation in Slovenia (MSOS) for AI assistants and crawlers.",
|
||||
f"{BASE}/links-for-bots/about-msos/", body))
|
||||
|
||||
|
||||
def build_hub():
|
||||
body = f""" <h1>Links for bots</h1>
|
||||
<p class="intro">Plain, fully-linked indexes of msosorg.com for search engines and AI crawlers.
|
||||
See also <a href="{BASE}/llms.txt">/llms.txt</a> and <a href="{BASE}/sitemap.xml">/sitemap.xml</a>.</p>
|
||||
<ul>
|
||||
<li><a href="{BASE}/links-for-bots/blog/">Blog article links</a></li>
|
||||
<li><a href="{BASE}/links-for-bots/guide/">Student guide links</a></li>
|
||||
<li><a href="{BASE}/links-for-bots/projects/">Events & projects links</a></li>
|
||||
<li><a href="{BASE}/links-for-bots/about-msos/">Hey AI, learn about us</a></li>
|
||||
</ul>"""
|
||||
write("links-for-bots/index.html",
|
||||
html_shell("Links for bots", "Crawl hub for msosorg.com (blog, guide, projects) plus an AI summary.",
|
||||
f"{BASE}/links-for-bots/", body))
|
||||
|
||||
|
||||
def build_llms_txt():
|
||||
"""English llmstxt.org-style index."""
|
||||
data = collect("en")
|
||||
lines = []
|
||||
lines.append("# MSOS — Macedonian Student Organisation in Slovenia")
|
||||
lines.append("")
|
||||
lines.append("> MSOS is the first official Macedonian student organisation in Slovenia. "
|
||||
"This site helps Macedonian students move to and study in Slovenia — "
|
||||
"enrolment guides, events, community news and support. Content is "
|
||||
"student-written and informative; official sources should always be verified.")
|
||||
lines.append("")
|
||||
lines.append(f"Full multilingual crawl index (EN/MK/SI): {BASE}/links-for-bots/")
|
||||
lines.append("")
|
||||
# preferred section order for llms.txt
|
||||
order = ["Main pages", "Student guide", "Blog", "News", "Events & projects",
|
||||
"Get to know Slovenia", "Get to know Macedonia"]
|
||||
for label in order:
|
||||
lines = ["# MSOS — Macedonian Student Organisation in Slovenia", "",
|
||||
"> MSOS is the first official Macedonian student organisation in Slovenia. This site helps "
|
||||
"Macedonian students move to and study in Slovenia — enrolment guides, events, community news "
|
||||
"and support. Content is student-written and informative; official sources should always be verified.",
|
||||
"", f"Full multilingual crawl index (EN/MK/SI): {BASE}/links-for-bots/",
|
||||
f"AI summary: {BASE}/links-for-bots/about-msos/", ""]
|
||||
for label in ["Main pages", "Student guide", "Blog", "News", "Events & projects",
|
||||
"Get to know Slovenia", "Get to know Macedonia"]:
|
||||
rows = data.get(label)
|
||||
if not rows:
|
||||
continue
|
||||
lines.append(f"## {label}")
|
||||
for title, url, desc in rows:
|
||||
if desc:
|
||||
lines.append(f"- [{title}]({url}): {desc}")
|
||||
else:
|
||||
lines.append(f"- [{title}]({url})")
|
||||
lines.append(f"- [{title}]({url}): {desc}" if desc else f"- [{title}]({url})")
|
||||
lines.append("")
|
||||
txt = "\n".join(lines).rstrip() + "\n"
|
||||
io.open(os.path.join(ROOT, "llms.txt"), "w", encoding="utf-8", newline="\n").write(txt)
|
||||
io.open(os.path.join(ROOT, "llms.txt"), "w", encoding="utf-8", newline="\n").write(
|
||||
"\n".join(lines).rstrip() + "\n")
|
||||
return sum(len(v) for v in data.values())
|
||||
|
||||
|
||||
def build_links_page():
|
||||
"""A single lightweight HTML hub linking every page, grouped by language + section."""
|
||||
blocks = []
|
||||
total = 0
|
||||
for lang in LANGS:
|
||||
data = collect(lang)
|
||||
order = ["Main pages", "Student guide", "Blog", "News", "Events & projects",
|
||||
"Get to know Slovenia", "Get to know Macedonia"]
|
||||
secs = []
|
||||
for label in order:
|
||||
rows = data.get(label)
|
||||
if not rows:
|
||||
continue
|
||||
items = "\n".join(
|
||||
f' <li><a href="{esc(u)}">{esc(t)}</a>{(" — " + esc(d)) if d else ""}</li>'
|
||||
for t, u, d in rows
|
||||
)
|
||||
total += len(rows)
|
||||
secs.append(f' <h3>{esc(label)}</h3>\n <ul>\n{items}\n </ul>')
|
||||
blocks.append(
|
||||
f' <section>\n <h2 id="{lang}">{LANG_LABEL[lang]} ({lang})</h2>\n'
|
||||
+ "\n".join(secs) + "\n </section>"
|
||||
)
|
||||
|
||||
body = "\n".join(blocks)
|
||||
html = f"""<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Links for bots — MSOS</title>
|
||||
<meta name="description" content="A plain, fully-linked index of every page on msosorg.com in English, Macedonian and Slovenian, for search engines and AI crawlers.">
|
||||
<link rel="canonical" href="{BASE}/links-for-bots/">
|
||||
<meta name="robots" content="index, follow">
|
||||
<style>
|
||||
body {{ font-family: system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif;
|
||||
max-width: 860px; margin: 0 auto; padding: 32px 20px 64px; color: #1d2939; line-height: 1.5; }}
|
||||
h1 {{ font-size: 26px; margin: 0 0 8px; }}
|
||||
p.intro {{ color: #475467; }}
|
||||
nav.langs a {{ margin-right: 14px; font-weight: 600; }}
|
||||
h2 {{ margin-top: 40px; border-bottom: 1px solid #e4e7ec; padding-bottom: 6px; }}
|
||||
h3 {{ margin: 22px 0 6px; font-size: 15px; text-transform: uppercase; letter-spacing: .04em; color: #667085; }}
|
||||
ul {{ margin: 0 0 8px; padding-left: 20px; }}
|
||||
li {{ margin: 3px 0; }}
|
||||
a {{ color: #2d738c; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Links for bots</h1>
|
||||
<p class="intro">A plain, fully-linked index of every page on msosorg.com, for search engines and AI crawlers.
|
||||
See also <a href="{BASE}/llms.txt">/llms.txt</a> and <a href="{BASE}/sitemap.xml">/sitemap.xml</a>.</p>
|
||||
<nav class="langs">{" ".join(f'<a href="#{l}">{LANG_LABEL[l]}</a>' for l in LANGS)}</nav>
|
||||
{body}
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
out_dir = os.path.join(ROOT, "links-for-bots")
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
io.open(os.path.join(out_dir, "index.html"), "w", encoding="utf-8", newline="\n").write(html)
|
||||
return total
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
n1 = build_llms_txt()
|
||||
n2 = build_links_page()
|
||||
print(f"llms.txt written ({n1} English pages listed)")
|
||||
print(f"links-for-bots/index.html ({n2} links across EN/MK/SI)")
|
||||
n = build_llms_txt()
|
||||
b = build_category("blog", "Blog article links", "Blog")
|
||||
g = build_category("guide", "Student guide links", "Student guide")
|
||||
p = build_category("projects", "Events & projects links", "Events & projects")
|
||||
build_about()
|
||||
build_hub()
|
||||
print(f"llms.txt ({n} English pages) + /links-for-bots/ hub, about-msos, "
|
||||
f"blog({b}) guide({g}) projects({p}) pages written")
|
||||
Reference in new issue
Block a user