#!/usr/bin/env python3 """Verify the public manual boundary, including generated pages and search.""" import argparse import json import re from html.parser import HTMLParser from pathlib import Path from urllib.parse import unquote, urljoin, urlsplit import yaml PUBLIC_ASSETS = { "stylesheets/extra.css", "javascripts/layout-init.js", "install/examples/thothii-installation.local.yaml", "install/examples/thothii-installation.server.yaml", "install/examples/workspace-bindings.env.example", } # Exact static assets shipped by the locked Windmill theme. Sources in docs/ must # not shadow these paths, or an internal file could bypass the public allowlist. THEME_ASSETS = { "css/base.css", "css/bootstrap-3.3.7.css", "css/bootstrap-3.3.7.min.css", "css/font-awesome-4.7.0.css", "css/font-awesome-4.7.0.min.css", "css/highlight.css", "fonts/fontawesome-webfont.eot", "fonts/fontawesome-webfont.svg", "fonts/fontawesome-webfont.ttf", "fonts/fontawesome-webfont.woff", "fonts/fontawesome-webfont.woff2", "fonts/glyphicons-halflings-regular.eot", "fonts/glyphicons-halflings-regular.svg", "fonts/glyphicons-halflings-regular.ttf", "fonts/glyphicons-halflings-regular.woff", "fonts/glyphicons-halflings-regular.woff2", "img/favicon.ico", "js/base.js", "js/bootstrap-3.3.7.js", "js/bootstrap-3.3.7.min.js", "js/elasticlunr.js", "js/elasticlunr.min.js", "js/highlight.pack.js", "js/jquery-3.2.1.js", "js/jquery-3.2.1.min.js", } INTERNAL_DIRS = { "adr", "agents", "architecture", "contracts", "maintenance", "plans", "reports", "research", "testing", } INTERNAL_PAGES = { "disambiguazione-iniziale.md", "gestione-memory.md", "installazione-docker-4-contesti.md", "install/authentication-upstream.md", "operations/docker-refresh.md", "operations/shell-and-localization.md", "operations/compose-reference.md", "operations/public-docs-publication.md", } def nav_pages(value): if isinstance(value, str): yield value elif isinstance(value, list): for item in value: yield from nav_pages(item) elif isinstance(value, dict): for item in value.values(): yield from nav_pages(item) def output_path(source): path = Path(source) if path.suffix != ".md": return path return path.with_suffix("") / "index.html" if path.name != "index.md" else path.with_suffix(".html") class AssetLinks(HTMLParser): def __init__(self): super().__init__() self.urls = [] def handle_starttag(self, tag, attrs): attrs = dict(attrs) if tag == "link" and "stylesheet" in attrs.get("rel", "").split(): self.urls.append(attrs.get("href", "")) elif tag in ("script", "img") and attrs.get("src"): self.urls.append(attrs["src"]) def check_assets(site, pages, site_url): """Follow page assets and CSS font/image references using browser URL rules.""" base = site_url.rstrip("/") + "/" origin = urlsplit(base) visited = set() def visit(reference, parent): if not reference or reference.startswith("#"): return url = urlsplit(urljoin(parent, reference)) if url.scheme not in ("http", "https") or url.netloc != origin.netloc: return path = unquote(url.path) if not path.startswith(origin.path): raise ValueError(f"asset escapes public site prefix: {reference}") relative = path[len(origin.path):] asset = (site / relative).resolve() if site.resolve() not in asset.parents or not asset.is_file(): raise ValueError(f"missing generated page asset: {relative}") if relative in visited: return visited.add(relative) if asset.suffix == ".css": css = asset.read_text() refs = re.findall(r"url\(\s*['\"]?([^)'\"\s]+)", css) refs += re.findall(r"@import\s+['\"]([^'\"]+)", css) for child in refs: visit(child, url.geturl()) for page in pages: output = output_path(page) parser = AssetLinks() parser.feed((site / output).read_text()) page_url = urljoin(base, output.as_posix()) for reference in parser.urls: visit(reference, page_url) def check(root, site): # BaseLoader reads configuration without executing custom Python YAML tags. config = yaml.load((root / "mkdocs.yml").read_text(), Loader=yaml.BaseLoader) pages = list(nav_pages(config["nav"])) if len(pages) != len(set(pages)): raise ValueError("duplicate public navigation page") for page in pages: path = Path(page) if path.is_absolute() or ".." in path.parts or path.suffix != ".md": raise ValueError(f"invalid public page: {page}") if (path.parts[0] in INTERNAL_DIRS or page in INTERNAL_PAGES or page.startswith("operations/server-")): raise ValueError(f"internal page in public navigation: {page}") if not (root / "docs" / page).is_file(): raise ValueError(f"missing public source: {page}") rules = config.get("exclude_docs", "").splitlines() if not rules or rules[0] != "*" or any(not r.startswith("!/") for r in rules[1:]): raise ValueError("public docs must use deny-by-default exclusions and explicit exceptions") published = [r[2:] for r in rules[1:]] if len(published) != len(set(published)) or set(published) != set(pages) | PUBLIC_ASSETS | THEME_ASSETS: raise ValueError("publication exceptions must match nav pages and approved assets exactly") for source in published: if source in THEME_ASSETS: if (root / "docs" / source).exists(): raise ValueError(f"docs source shadows theme asset: {source}") continue if not (root / "docs" / source).is_file(): raise ValueError(f"missing public source: {source}") if not (site / output_path(source)).is_file(): raise ValueError(f"missing generated public file: {source}") for source in (root / "docs").rglob("*"): if not source.is_file(): continue relative = source.relative_to(root / "docs").as_posix() if relative not in published and (site / output_path(relative)).exists(): raise ValueError(f"internal file leaked into site: {relative}") expected_urls = { "" if page == "index.md" else str(output_path(page).parent).replace("\\", "/") + "/" for page in pages } search = json.loads((site / "search/search_index.json").read_text()) indexed = set() for entry in search["docs"]: location = unquote(urlsplit(entry["location"]).path) if location not in expected_urls: raise ValueError(f"non-public page in search index: {location}") indexed.add(location) if indexed != expected_urls: raise ValueError("search index does not cover exactly the public pages") check_assets(site, pages, config.get("site_url", "https://docs.example.invalid/")) for asset in sorted(THEME_ASSETS): if not (site / asset).is_file(): raise ValueError(f"missing generated theme asset: {asset}") return len(pages) if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) parser.add_argument("--site-dir", type=Path) args = parser.parse_args() root = args.root.resolve() try: count = check(root, args.site_dir or root / "site") except (ValueError, KeyError, OSError, yaml.YAMLError) as error: raise SystemExit(f"public docs verification failed: {error}") from error print(f"Public documentation boundary passed: {count} pages; internal files and search excluded")