changes.txt #2: AI/crawler discovery files (llms.txt + links-for-bots)
Add tools/build_botlinks.py which regenerates, from the pages already on disk: - /llms.txt llmstxt.org-style Markdown index (English, with descriptions) - /links-for-bots/ a lightweight fully-linked crawl hub covering EN/MK/SI Re-run `python tools/build_botlinks.py` after adding or removing pages; it only reads each page's <title> and meta description, so it stays current automatically. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
popovskikandClaude Opus 4.8 committed
1 parent
5abda945ed
commit
c14a227ac7
3 files changed
+628
No files matched your search
@@ -0,0 +1,208 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
build_botlinks.py -- AI / crawler discovery files for msosorg.com
|
||||
|
||||
Generates two things from the pages that already exist on disk, so they stay
|
||||
current automatically whenever a page is added or removed:
|
||||
|
||||
1. /llms.txt an llmstxt.org-style Markdown index (English)
|
||||
2. /links-for-bots/index.html a single, lightweight, fully-linked crawl hub
|
||||
covering EN / MK / SI
|
||||
|
||||
Run after adding or removing pages:
|
||||
|
||||
python tools/build_botlinks.py
|
||||
|
||||
It only reads each page's <title> and <meta name="description">, so it never
|
||||
depends on the visual markup and is safe to re-run (idempotent).
|
||||
"""
|
||||
import io, os, re
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
BASE = "https://msosorg.com"
|
||||
LANGS = ("en", "mk", "si")
|
||||
LANG_LABEL = {"en": "English", "mk": "Македонски", "si": "Slovenščina"}
|
||||
|
||||
# (top-level directory under a language, human label). Order = display order.
|
||||
SECTIONS = [
|
||||
("student-welcome-guide", "Student guide"),
|
||||
("blog", "Blog"),
|
||||
("news", "News"),
|
||||
("projects", "Events & projects"),
|
||||
("get-to-know-slovenia", "Get to know Slovenia"),
|
||||
("get-to-know-macedonia", "Get to know Macedonia"),
|
||||
]
|
||||
# Single top-level pages (their own index.html directly under the language dir).
|
||||
MAIN_PAGES = [
|
||||
"about-us", "msos-hub", "become-a-member", "support", "gallery",
|
||||
"faq", "timeline-and-milestones", "legal", "privacy-policy", "cookie-policy",
|
||||
]
|
||||
|
||||
TITLE_RE = re.compile(r"<title>(.*?)</title>", re.DOTALL | re.I)
|
||||
DESC_RE = re.compile(r'<meta name="description" content="(.*?)"', re.I)
|
||||
|
||||
|
||||
def read_meta(path):
|
||||
txt = io.open(path, encoding="utf-8").read()
|
||||
tm = TITLE_RE.search(txt)
|
||||
dm = DESC_RE.search(txt)
|
||||
title = tm.group(1).strip() if tm else ""
|
||||
# drop the " - MSOS" / " | MSOS" style suffix for a cleaner label
|
||||
title = re.sub(r"\s*[-|]\s*(MSOS|Macedonian Student Organisation).*$", "", title).strip() or title
|
||||
desc = (dm.group(1).strip() if dm else "")
|
||||
return title, desc
|
||||
|
||||
|
||||
def page_url(path):
|
||||
rel = os.path.relpath(path, ROOT).replace("\\", "/")
|
||||
if rel.endswith("index.html"):
|
||||
rel = rel[:-len("index.html")]
|
||||
return BASE + "/" + rel
|
||||
|
||||
|
||||
def collect(lang):
|
||||
"""Return {section_label: [(title, url, desc), ...]} for one language."""
|
||||
out = {}
|
||||
base = os.path.join(ROOT, lang)
|
||||
|
||||
# sectioned content (one folder per article/subpage)
|
||||
for folder, label in SECTIONS:
|
||||
rows = []
|
||||
sec_dir = os.path.join(base, folder)
|
||||
if not os.path.isdir(sec_dir):
|
||||
continue
|
||||
# section landing page first
|
||||
idx = os.path.join(sec_dir, "index.html")
|
||||
if os.path.exists(idx):
|
||||
t, d = read_meta(idx)
|
||||
rows.append((t, page_url(idx), d))
|
||||
# child pages
|
||||
for name in sorted(os.listdir(sec_dir)):
|
||||
child = os.path.join(sec_dir, name, "index.html")
|
||||
if os.path.exists(child):
|
||||
t, d = read_meta(child)
|
||||
rows.append((t, page_url(child), d))
|
||||
if rows:
|
||||
out[label] = rows
|
||||
|
||||
# single main pages
|
||||
rows = []
|
||||
for name in MAIN_PAGES:
|
||||
p = os.path.join(base, name, "index.html")
|
||||
if os.path.exists(p):
|
||||
t, d = read_meta(p)
|
||||
rows.append((t, page_url(p), d))
|
||||
# language home
|
||||
home = os.path.join(base, "index.html")
|
||||
if os.path.exists(home):
|
||||
t, d = read_meta(home)
|
||||
rows.insert(0, (t or "Home", page_url(home), d))
|
||||
if rows:
|
||||
out["Main pages"] = rows
|
||||
return out
|
||||
|
||||
|
||||
def esc(s):
|
||||
return (s.replace("&", "&").replace("<", "<").replace(">", ">")
|
||||
.replace('"', """))
|
||||
|
||||
|
||||
def build_llms_txt():
|
||||
"""English llmstxt.org-style index."""
|
||||
data = collect("en")
|
||||
lines = []
|
||||
lines.append("# MSOS — Macedonian Student Organisation in Slovenia")
|
||||
lines.append("")
|
||||
lines.append("> MSOS is the first official Macedonian student organisation in Slovenia. "
|
||||
"This site helps Macedonian students move to and study in Slovenia — "
|
||||
"enrolment guides, events, community news and support. Content is "
|
||||
"student-written and informative; official sources should always be verified.")
|
||||
lines.append("")
|
||||
lines.append(f"Full multilingual crawl index (EN/MK/SI): {BASE}/links-for-bots/")
|
||||
lines.append("")
|
||||
# preferred section order for llms.txt
|
||||
order = ["Main pages", "Student guide", "Blog", "News", "Events & projects",
|
||||
"Get to know Slovenia", "Get to know Macedonia"]
|
||||
for label in order:
|
||||
rows = data.get(label)
|
||||
if not rows:
|
||||
continue
|
||||
lines.append(f"## {label}")
|
||||
for title, url, desc in rows:
|
||||
if desc:
|
||||
lines.append(f"- [{title}]({url}): {desc}")
|
||||
else:
|
||||
lines.append(f"- [{title}]({url})")
|
||||
lines.append("")
|
||||
txt = "\n".join(lines).rstrip() + "\n"
|
||||
io.open(os.path.join(ROOT, "llms.txt"), "w", encoding="utf-8", newline="\n").write(txt)
|
||||
return sum(len(v) for v in data.values())
|
||||
|
||||
|
||||
def build_links_page():
|
||||
"""A single lightweight HTML hub linking every page, grouped by language + section."""
|
||||
blocks = []
|
||||
total = 0
|
||||
for lang in LANGS:
|
||||
data = collect(lang)
|
||||
order = ["Main pages", "Student guide", "Blog", "News", "Events & projects",
|
||||
"Get to know Slovenia", "Get to know Macedonia"]
|
||||
secs = []
|
||||
for label in order:
|
||||
rows = data.get(label)
|
||||
if not rows:
|
||||
continue
|
||||
items = "\n".join(
|
||||
f' <li><a href="{esc(u)}">{esc(t)}</a>{(" — " + esc(d)) if d else ""}</li>'
|
||||
for t, u, d in rows
|
||||
)
|
||||
total += len(rows)
|
||||
secs.append(f' <h3>{esc(label)}</h3>\n <ul>\n{items}\n </ul>')
|
||||
blocks.append(
|
||||
f' <section>\n <h2 id="{lang}">{LANG_LABEL[lang]} ({lang})</h2>\n'
|
||||
+ "\n".join(secs) + "\n </section>"
|
||||
)
|
||||
|
||||
body = "\n".join(blocks)
|
||||
html = f"""<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Links for bots — MSOS</title>
|
||||
<meta name="description" content="A plain, fully-linked index of every page on msosorg.com in English, Macedonian and Slovenian, for search engines and AI crawlers.">
|
||||
<link rel="canonical" href="{BASE}/links-for-bots/">
|
||||
<meta name="robots" content="index, follow">
|
||||
<style>
|
||||
body {{ font-family: system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif;
|
||||
max-width: 860px; margin: 0 auto; padding: 32px 20px 64px; color: #1d2939; line-height: 1.5; }}
|
||||
h1 {{ font-size: 26px; margin: 0 0 8px; }}
|
||||
p.intro {{ color: #475467; }}
|
||||
nav.langs a {{ margin-right: 14px; font-weight: 600; }}
|
||||
h2 {{ margin-top: 40px; border-bottom: 1px solid #e4e7ec; padding-bottom: 6px; }}
|
||||
h3 {{ margin: 22px 0 6px; font-size: 15px; text-transform: uppercase; letter-spacing: .04em; color: #667085; }}
|
||||
ul {{ margin: 0 0 8px; padding-left: 20px; }}
|
||||
li {{ margin: 3px 0; }}
|
||||
a {{ color: #2d738c; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Links for bots</h1>
|
||||
<p class="intro">A plain, fully-linked index of every page on msosorg.com, for search engines and AI crawlers.
|
||||
See also <a href="{BASE}/llms.txt">/llms.txt</a> and <a href="{BASE}/sitemap.xml">/sitemap.xml</a>.</p>
|
||||
<nav class="langs">{" ".join(f'<a href="#{l}">{LANG_LABEL[l]}</a>' for l in LANGS)}</nav>
|
||||
{body}
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
out_dir = os.path.join(ROOT, "links-for-bots")
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
io.open(os.path.join(out_dir, "index.html"), "w", encoding="utf-8", newline="\n").write(html)
|
||||
return total
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
n1 = build_llms_txt()
|
||||
n2 = build_links_page()
|
||||
print(f"llms.txt written ({n1} English pages listed)")
|
||||
print(f"links-for-bots/index.html ({n2} links across EN/MK/SI)")
|
||||
Reference in new issue
Block a user