252 lines
8.4 KiB
Python
252 lines
8.4 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
migrate_pages_to_content.py -- one-shot back-fill of the content/ source tree
|
|
from the existing hand-authored detail pages.
|
|
|
|
Implements U1 of docs/plans/2026-08-08-001-feat-no-code-cms-plan.md. See
|
|
docs/content-model.md for the front-matter schema this produces.
|
|
|
|
It walks en|mk|si/{blog,news,projects}/<slug>/index.html, extracts the
|
|
structured header fields (title, subtitle, category, date, hero image + caption)
|
|
and preserves the article body verbatim as an HTML fragment (body_format: html),
|
|
then writes content/<type>/<slug>/<lang>.md.
|
|
|
|
This is a SCAFFOLD a human proofs, not a guaranteed lossless transform. Pages it
|
|
cannot parse cleanly are reported and skipped, never emitted half-formed.
|
|
|
|
Usage:
|
|
python tools/migrate_pages_to_content.py # dry run: report only
|
|
python tools/migrate_pages_to_content.py --write # write content/ files
|
|
"""
|
|
import argparse
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
from bs4 import BeautifulSoup
|
|
import yaml
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import seo_inject # noqa: E402 (central ARTICLE_DATES + date helpers)
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
LANGS = ("en", "mk", "si")
|
|
|
|
# source folder -> content type
|
|
FOLDER_TO_TYPE = {"blog": "blog", "news": "news", "projects": "event"}
|
|
# content type -> content/ subfolder
|
|
TYPE_TO_DIR = {"blog": "blog", "news": "news", "event": "events"}
|
|
|
|
|
|
def _rootrel_img(src):
|
|
"""Normalise an image src to a root-relative /images/... path."""
|
|
if not src:
|
|
return src
|
|
m = re.search(r"(images/.*)$", src)
|
|
return "/" + m.group(1) if m else src
|
|
|
|
|
|
def _inner_html(node):
|
|
"""Verbatim inner HTML of a bs4 node (children concatenated)."""
|
|
return "".join(str(c) for c in node.contents).strip()
|
|
|
|
|
|
def _body_fragment(body):
|
|
"""Verbatim markup of the body element PLUS every following sibling within
|
|
its container, with wrappers intact.
|
|
|
|
Event pages keep the photo gallery (`section.event-gallery`) as a sibling
|
|
that follows `.article-body`, so inner-HTML alone would drop it. Walking
|
|
from the body element to the end of the container preserves the body wrapper
|
|
and any trailing sections losslessly.
|
|
"""
|
|
parts, node = [], body
|
|
while node is not None:
|
|
parts.append(str(node))
|
|
node = node.next_sibling
|
|
return "".join(parts).strip()
|
|
|
|
|
|
def _text(node):
|
|
return node.get_text(" ", strip=True) if node else ""
|
|
|
|
|
|
def _iso_date(source_folder, slug, en_html=None):
|
|
"""ISO date: prefer the central ARTICLE_DATES registry, then fall back to
|
|
reading it off the English page (mirrors build_listings' behaviour)."""
|
|
key = f"{source_folder}/{slug}"
|
|
rec = seo_inject.ARTICLE_DATES.get(key, {})
|
|
iso = rec.get("published") or rec.get("modified")
|
|
if iso:
|
|
return iso
|
|
if en_html is not None:
|
|
try:
|
|
return seo_inject.extract_publish_date(en_html)
|
|
except Exception:
|
|
return None
|
|
return None
|
|
|
|
|
|
def _parse_hero(figure):
|
|
"""Extract hero image + caption from a <figure> node."""
|
|
if not figure:
|
|
return None
|
|
img = figure.find("img")
|
|
hero = {}
|
|
if img:
|
|
hero["src"] = _rootrel_img(img.get("src"))
|
|
if img.get("alt"):
|
|
hero["alt"] = img.get("alt")
|
|
if img.get("width"):
|
|
hero["width"] = str(img.get("width"))
|
|
if img.get("height"):
|
|
hero["height"] = str(img.get("height"))
|
|
cap = figure.find("figcaption")
|
|
if cap:
|
|
caption = _inner_html(cap)
|
|
if caption:
|
|
hero["caption_html"] = caption
|
|
return hero or None
|
|
|
|
|
|
def parse_page(html, source_folder, slug, lang, en_html=None):
|
|
"""Parse one detail page into a content record dict, or return None with a
|
|
reason if the page shape is not recognised.
|
|
|
|
Returns (record, None) on success or (None, reason) on failure.
|
|
"""
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
ctype = FOLDER_TO_TYPE[source_folder]
|
|
|
|
h1 = soup.find("h1")
|
|
if not h1:
|
|
return None, "no <h1> found"
|
|
title = _text(h1)
|
|
|
|
record = {
|
|
"type": ctype,
|
|
"slug": slug,
|
|
"lang": lang,
|
|
"source_folder": source_folder,
|
|
"title": title,
|
|
}
|
|
|
|
if source_folder == "news":
|
|
body = soup.select_one(".news-article-body")
|
|
hero = _parse_hero(soup.select_one("figure.news-article-hero"))
|
|
date_line = _text(soup.select_one(".news-article-date"))
|
|
else: # blog + projects share the article-* structure
|
|
body = soup.select_one(".article-body")
|
|
hero = _parse_hero(soup.select_one("figure.article-main-image"))
|
|
date_line = _text(soup.select_one(".article-publish-date"))
|
|
subtitle = _text(soup.select_one(".article-subtitle"))
|
|
if subtitle:
|
|
record["subtitle"] = subtitle
|
|
|
|
if body is None:
|
|
return None, "no article body element found"
|
|
|
|
# Best-effort category: blog date line is "Category · Last checked …".
|
|
if date_line and "·" in date_line:
|
|
cat, _sep, _rest = date_line.partition("·")
|
|
cat = cat.strip()
|
|
# Only treat as a category if it doesn't itself look like a date phrase.
|
|
if cat and not re.search(r"\bpublished\b|\d{4}", cat, re.I):
|
|
record["category"] = cat
|
|
|
|
iso = _iso_date(source_folder, slug, en_html=en_html)
|
|
if not iso:
|
|
return None, "no date (missing from ARTICLE_DATES and unparseable)"
|
|
record["date"] = iso
|
|
if date_line:
|
|
record["date_line"] = date_line
|
|
|
|
if hero:
|
|
record["hero"] = hero
|
|
|
|
record["body_format"] = "html"
|
|
record["_body"] = _body_fragment(body)
|
|
if not record["_body"]:
|
|
return None, "article body is empty"
|
|
|
|
return record, None
|
|
|
|
|
|
# Deterministic front-matter key order for readable, stable output.
|
|
_KEY_ORDER = [
|
|
"type", "slug", "lang", "source_folder",
|
|
"title", "subtitle", "category", "date", "date_line", "hero", "body_format",
|
|
]
|
|
|
|
|
|
def to_file_text(record):
|
|
"""Serialise a content record to front-matter + body text."""
|
|
body = record.get("_body", "")
|
|
meta = {k: record[k] for k in _KEY_ORDER if k in record}
|
|
front = yaml.safe_dump(
|
|
meta, allow_unicode=True, sort_keys=False, default_flow_style=False, width=1000
|
|
)
|
|
return f"---\n{front}---\n{body}\n"
|
|
|
|
|
|
def iter_pages():
|
|
"""Yield (lang, source_folder, slug, page_path) for every detail page."""
|
|
for lang in LANGS:
|
|
for folder in ("blog", "news", "projects"):
|
|
base = os.path.join(ROOT, lang, folder)
|
|
if not os.path.isdir(base):
|
|
continue
|
|
for slug in sorted(os.listdir(base)):
|
|
page = os.path.join(base, slug, "index.html")
|
|
if os.path.exists(page):
|
|
yield lang, folder, slug, page
|
|
|
|
|
|
def _read(path):
|
|
with open(path, encoding="utf-8") as f:
|
|
return f.read()
|
|
|
|
|
|
def run(write=False):
|
|
written, skipped = [], []
|
|
# Cache English pages so non-en pages can fall back to the en date.
|
|
en_cache = {}
|
|
for lang, folder, slug, page in iter_pages():
|
|
if lang == "en":
|
|
en_cache[(folder, slug)] = _read(page)
|
|
|
|
for lang, folder, slug, page in iter_pages():
|
|
html = _read(page)
|
|
en_html = en_cache.get((folder, slug))
|
|
record, reason = parse_page(html, folder, slug, lang, en_html=en_html)
|
|
if record is None:
|
|
skipped.append((f"{lang}/{folder}/{slug}", reason))
|
|
continue
|
|
out_dir = os.path.join(ROOT, "content", TYPE_TO_DIR[record["type"]], slug)
|
|
out_path = os.path.join(out_dir, f"{lang}.md")
|
|
rel = os.path.relpath(out_path, ROOT).replace(os.sep, "/")
|
|
if write:
|
|
os.makedirs(out_dir, exist_ok=True)
|
|
with open(out_path, "w", encoding="utf-8", newline="\n") as f:
|
|
f.write(to_file_text(record))
|
|
written.append(rel)
|
|
|
|
verb = "Wrote" if write else "Would write"
|
|
print(f"{verb} {len(written)} content files:")
|
|
for rel in written:
|
|
print(f" + {rel}")
|
|
if skipped:
|
|
print(f"\nSkipped {len(skipped)} page(s) that need manual attention:")
|
|
for name, reason in skipped:
|
|
print(f" ! {name}: {reason}")
|
|
if not write:
|
|
print("\n(dry run — re-run with --write to create the files)")
|
|
return written, skipped
|
|
|
|
|
|
if __name__ == "__main__":
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument("--write", action="store_true", help="write content/ files")
|
|
args = ap.parse_args()
|
|
run(write=args.write)
|