ThothII/scripts/verify-public-docs.py

#!/usr/bin/env python3
"""Verify the public manual boundary, including generated pages and search."""

import argparse
import json
import re
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import unquote, urljoin, urlsplit

import yaml


PUBLIC_ASSETS = {
    "stylesheets/extra.css",
    "javascripts/layout-init.js",
    "install/examples/thothii-installation.local.yaml",
    "install/examples/thothii-installation.server.yaml",
    "install/examples/workspace-bindings.env.example",
}
# Exact static assets shipped by the locked Windmill theme. Sources in docs/ must
# not shadow these paths, or an internal file could bypass the public allowlist.
THEME_ASSETS = {
    "css/base.css",
    "css/bootstrap-3.3.7.css",
    "css/bootstrap-3.3.7.min.css",
    "css/font-awesome-4.7.0.css",
    "css/font-awesome-4.7.0.min.css",
    "css/highlight.css",
    "fonts/fontawesome-webfont.eot",
    "fonts/fontawesome-webfont.svg",
    "fonts/fontawesome-webfont.ttf",
    "fonts/fontawesome-webfont.woff",
    "fonts/fontawesome-webfont.woff2",
    "fonts/glyphicons-halflings-regular.eot",
    "fonts/glyphicons-halflings-regular.svg",
    "fonts/glyphicons-halflings-regular.ttf",
    "fonts/glyphicons-halflings-regular.woff",
    "fonts/glyphicons-halflings-regular.woff2",
    "img/favicon.ico",
    "js/base.js",
    "js/bootstrap-3.3.7.js",
    "js/bootstrap-3.3.7.min.js",
    "js/elasticlunr.js",
    "js/elasticlunr.min.js",
    "js/highlight.pack.js",
    "js/jquery-3.2.1.js",
    "js/jquery-3.2.1.min.js",
}
INTERNAL_DIRS = {
    "adr", "agents", "architecture", "contracts", "maintenance", "plans",
    "reports", "research", "testing",
}
INTERNAL_PAGES = {
    "disambiguazione-iniziale.md", "gestione-memory.md", "installazione-docker-4-contesti.md",
    "install/authentication-upstream.md", "operations/docker-refresh.md",
    "operations/shell-and-localization.md",
    "operations/compose-reference.md", "operations/public-docs-publication.md",
}


def nav_pages(value):
    if isinstance(value, str):
        yield value
    elif isinstance(value, list):
        for item in value:
            yield from nav_pages(item)
    elif isinstance(value, dict):
        for item in value.values():
            yield from nav_pages(item)


def output_path(source):
    path = Path(source)
    if path.suffix != ".md":
        return path
    return path.with_suffix("") / "index.html" if path.name != "index.md" else path.with_suffix(".html")


class AssetLinks(HTMLParser):
    def __init__(self):
        super().__init__()
        self.urls = []

    def handle_starttag(self, tag, attrs):
        attrs = dict(attrs)
        if tag == "link" and "stylesheet" in attrs.get("rel", "").split():
            self.urls.append(attrs.get("href", ""))
        elif tag in ("script", "img") and attrs.get("src"):
            self.urls.append(attrs["src"])


def check_assets(site, pages, site_url):
    """Follow page assets and CSS font/image references using browser URL rules."""
    base = site_url.rstrip("/") + "/"
    origin = urlsplit(base)
    visited = set()

    def visit(reference, parent):
        if not reference or reference.startswith("#"):
            return
        url = urlsplit(urljoin(parent, reference))
        if url.scheme not in ("http", "https") or url.netloc != origin.netloc:
            return
        path = unquote(url.path)
        if not path.startswith(origin.path):
            raise ValueError(f"asset escapes public site prefix: {reference}")
        relative = path[len(origin.path):]
        asset = (site / relative).resolve()
        if site.resolve() not in asset.parents or not asset.is_file():
            raise ValueError(f"missing generated page asset: {relative}")
        if relative in visited:
            return
        visited.add(relative)
        if asset.suffix == ".css":
            css = asset.read_text()
            refs = re.findall(r"url\(\s*['\"]?([^)'\"\s]+)", css)
            refs += re.findall(r"@import\s+['\"]([^'\"]+)", css)
            for child in refs:
                visit(child, url.geturl())

    for page in pages:
        output = output_path(page)
        parser = AssetLinks()
        parser.feed((site / output).read_text())
        page_url = urljoin(base, output.as_posix())
        for reference in parser.urls:
            visit(reference, page_url)


def check(root, site):
    # BaseLoader reads configuration without executing custom Python YAML tags.
    config = yaml.load((root / "mkdocs.yml").read_text(), Loader=yaml.BaseLoader)
    pages = list(nav_pages(config["nav"]))
    if len(pages) != len(set(pages)):
        raise ValueError("duplicate public navigation page")
    for page in pages:
        path = Path(page)
        if path.is_absolute() or ".." in path.parts or path.suffix != ".md":
            raise ValueError(f"invalid public page: {page}")
        if (path.parts[0] in INTERNAL_DIRS or page in INTERNAL_PAGES
                or page.startswith("operations/server-")):
            raise ValueError(f"internal page in public navigation: {page}")
        if not (root / "docs" / page).is_file():
            raise ValueError(f"missing public source: {page}")

    rules = config.get("exclude_docs", "").splitlines()
    if not rules or rules[0] != "*" or any(not r.startswith("!/") for r in rules[1:]):
        raise ValueError("public docs must use deny-by-default exclusions and explicit exceptions")
    published = [r[2:] for r in rules[1:]]
    if len(published) != len(set(published)) or set(published) != set(pages) | PUBLIC_ASSETS | THEME_ASSETS:
        raise ValueError("publication exceptions must match nav pages and approved assets exactly")

    for source in published:
        if source in THEME_ASSETS:
            if (root / "docs" / source).exists():
                raise ValueError(f"docs source shadows theme asset: {source}")
            continue
        if not (root / "docs" / source).is_file():
            raise ValueError(f"missing public source: {source}")
        if not (site / output_path(source)).is_file():
            raise ValueError(f"missing generated public file: {source}")
    for source in (root / "docs").rglob("*"):
        if not source.is_file():
            continue
        relative = source.relative_to(root / "docs").as_posix()
        if relative not in published and (site / output_path(relative)).exists():
            raise ValueError(f"internal file leaked into site: {relative}")

    expected_urls = {
        "" if page == "index.md" else str(output_path(page).parent).replace("\\", "/") + "/"
        for page in pages
    }
    search = json.loads((site / "search/search_index.json").read_text())
    indexed = set()
    for entry in search["docs"]:
        location = unquote(urlsplit(entry["location"]).path)
        if location not in expected_urls:
            raise ValueError(f"non-public page in search index: {location}")
        indexed.add(location)
    if indexed != expected_urls:
        raise ValueError("search index does not cover exactly the public pages")
    check_assets(site, pages, config.get("site_url", "https://docs.example.invalid/"))
    for asset in sorted(THEME_ASSETS):
        if not (site / asset).is_file():
            raise ValueError(f"missing generated theme asset: {asset}")
    return len(pages)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1])
    parser.add_argument("--site-dir", type=Path)
    args = parser.parse_args()
    root = args.root.resolve()
    try:
        count = check(root, args.site_dir or root / "site")
    except (ValueError, KeyError, OSError, yaml.YAMLError) as error:
        raise SystemExit(f"public docs verification failed: {error}") from error
    print(f"Public documentation boundary passed: {count} pages; internal files and search excluded")