msos/tools/migrate_pages_to_content.py

#!/usr/bin/env python3
"""
migrate_pages_to_content.py  --  one-shot back-fill of the content/ source tree
from the existing hand-authored detail pages.

Implements U1 of docs/plans/2026-08-08-001-feat-no-code-cms-plan.md. See
docs/content-model.md for the front-matter schema this produces.

It walks en|mk|si/{blog,news,projects}/<slug>/index.html, extracts the
structured header fields (title, subtitle, category, date, hero image + caption)
and preserves the article body verbatim as an HTML fragment (body_format: html),
then writes content/<type>/<slug>/<lang>.md.

This is a SCAFFOLD a human proofs, not a guaranteed lossless transform. Pages it
cannot parse cleanly are reported and skipped, never emitted half-formed.

Usage:
    python tools/migrate_pages_to_content.py            # dry run: report only
    python tools/migrate_pages_to_content.py --write    # write content/ files
"""
import argparse
import os
import re
import sys

from bs4 import BeautifulSoup
import yaml

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import seo_inject  # noqa: E402  (central ARTICLE_DATES + date helpers)

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
LANGS = ("en", "mk", "si")

# source folder -> content type
FOLDER_TO_TYPE = {"blog": "blog", "news": "news", "projects": "event"}
# content type -> content/ subfolder
TYPE_TO_DIR = {"blog": "blog", "news": "news", "event": "events"}


def _rootrel_img(src):
    """Normalise an image src to a root-relative /images/... path."""
    if not src:
        return src
    m = re.search(r"(images/.*)$", src)
    return "/" + m.group(1) if m else src


def _inner_html(node):
    """Verbatim inner HTML of a bs4 node (children concatenated)."""
    return "".join(str(c) for c in node.contents).strip()


def _body_fragment(body):
    """Verbatim markup of the body element PLUS every following sibling within
    its container, with wrappers intact.

    Event pages keep the photo gallery (`section.event-gallery`) as a sibling
    that follows `.article-body`, so inner-HTML alone would drop it. Walking
    from the body element to the end of the container preserves the body wrapper
    and any trailing sections losslessly.
    """
    parts, node = [], body
    while node is not None:
        parts.append(str(node))
        node = node.next_sibling
    return "".join(parts).strip()


def _text(node):
    return node.get_text(" ", strip=True) if node else ""


def _iso_date(source_folder, slug, en_html=None):
    """(iso, kind) for an entry: prefer the central ARTICLE_DATES registry
    (preserving whether the date is a 'published' or 'modified' one), then fall
    back to reading it off the English page (mirrors build_listings)."""
    key = f"{source_folder}/{slug}"
    rec = seo_inject.ARTICLE_DATES.get(key, {})
    if rec.get("published"):
        return rec["published"], "published"
    if rec.get("modified"):
        return rec["modified"], "modified"
    if en_html is not None:
        try:
            iso = seo_inject.extract_publish_date(en_html)
            if iso:
                return iso, "published"
        except Exception:
            pass
    return None, None


def _parse_hero(figure):
    """Extract hero image + caption from a <figure> node."""
    if not figure:
        return None
    img = figure.find("img")
    hero = {}
    if img:
        hero["src"] = _rootrel_img(img.get("src"))
        if img.get("alt"):
            hero["alt"] = img.get("alt")
        if img.get("width"):
            hero["width"] = str(img.get("width"))
        if img.get("height"):
            hero["height"] = str(img.get("height"))
    cap = figure.find("figcaption")
    if cap:
        caption = _inner_html(cap)
        if caption:
            hero["caption_html"] = caption
    return hero or None


def parse_page(html, source_folder, slug, lang, en_html=None):
    """Parse one detail page into a content record dict, or return None with a
    reason if the page shape is not recognised.

    Returns (record, None) on success or (None, reason) on failure.
    """
    soup = BeautifulSoup(html, "html.parser")
    ctype = FOLDER_TO_TYPE[source_folder]

    h1 = soup.find("h1")
    if not h1:
        return None, "no <h1> found"
    title = _text(h1)

    record = {
        "type": ctype,
        "slug": slug,
        "lang": lang,
        "source_folder": source_folder,
        "title": title,
    }

    if source_folder == "news":
        body = soup.select_one(".news-article-body")
        hero = _parse_hero(soup.select_one("figure.news-article-hero"))
        date_line = _text(soup.select_one(".news-article-date"))
    else:  # blog + projects share the article-* structure
        body = soup.select_one(".article-body")
        hero = _parse_hero(soup.select_one("figure.article-main-image"))
        date_line = _text(soup.select_one(".article-publish-date"))
        subtitle = _text(soup.select_one(".article-subtitle"))
        if subtitle:
            record["subtitle"] = subtitle

    if body is None:
        return None, "no article body element found"

    # Best-effort category: blog date line is "Category · Last checked …".
    if date_line and "·" in date_line:
        cat, _sep, _rest = date_line.partition("·")
        cat = cat.strip()
        # Only treat as a category if it doesn't itself look like a date phrase.
        if cat and not re.search(r"\bpublished\b|\d{4}", cat, re.I):
            record["category"] = cat

    iso, kind = _iso_date(source_folder, slug, en_html=en_html)
    if not iso:
        return None, "no date (missing from ARTICLE_DATES and unparseable)"
    record["date"] = iso
    if kind == "modified":
        # 'published' is the implicit default; only record the exception.
        record["date_type"] = "modified"
    if date_line:
        record["date_line"] = date_line

    if hero:
        record["hero"] = hero

    record["body_format"] = "html"
    record["_body"] = _body_fragment(body)
    if not record["_body"]:
        return None, "article body is empty"

    return record, None


# Deterministic front-matter key order for readable, stable output.
_KEY_ORDER = [
    "type", "slug", "lang", "source_folder",
    "title", "subtitle", "category", "date", "date_type", "date_line", "hero", "body_format",
]


def to_file_text(record):
    """Serialise a content record to front-matter + body text."""
    body = record.get("_body", "")
    meta = {k: record[k] for k in _KEY_ORDER if k in record}
    front = yaml.safe_dump(
        meta, allow_unicode=True, sort_keys=False, default_flow_style=False, width=1000
    )
    return f"---\n{front}---\n{body}\n"


def iter_pages():
    """Yield (lang, source_folder, slug, page_path) for every detail page."""
    for lang in LANGS:
        for folder in ("blog", "news", "projects"):
            base = os.path.join(ROOT, lang, folder)
            if not os.path.isdir(base):
                continue
            for slug in sorted(os.listdir(base)):
                page = os.path.join(base, slug, "index.html")
                if os.path.exists(page):
                    yield lang, folder, slug, page


def _read(path):
    with open(path, encoding="utf-8") as f:
        return f.read()


def run(write=False):
    written, skipped = [], []
    # Cache English pages so non-en pages can fall back to the en date.
    en_cache = {}
    for lang, folder, slug, page in iter_pages():
        if lang == "en":
            en_cache[(folder, slug)] = _read(page)

    for lang, folder, slug, page in iter_pages():
        html = _read(page)
        en_html = en_cache.get((folder, slug))
        record, reason = parse_page(html, folder, slug, lang, en_html=en_html)
        if record is None:
            skipped.append((f"{lang}/{folder}/{slug}", reason))
            continue
        out_dir = os.path.join(ROOT, "content", TYPE_TO_DIR[record["type"]], slug)
        out_path = os.path.join(out_dir, f"{lang}.md")
        rel = os.path.relpath(out_path, ROOT).replace(os.sep, "/")
        if write:
            os.makedirs(out_dir, exist_ok=True)
            with open(out_path, "w", encoding="utf-8", newline="\n") as f:
                f.write(to_file_text(record))
        written.append(rel)

    verb = "Wrote" if write else "Would write"
    print(f"{verb} {len(written)} content files:")
    for rel in written:
        print(f"  + {rel}")
    if skipped:
        print(f"\nSkipped {len(skipped)} page(s) that need manual attention:")
        for name, reason in skipped:
            print(f"  ! {name}: {reason}")
    if not write:
        print("\n(dry run — re-run with --write to create the files)")
    return written, skipped


if __name__ == "__main__":
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--write", action="store_true", help="write content/ files")
    args = ap.parse_args()
    run(write=args.write)