docs: define workspace evidence registry contract
This commit is contained in:
@@ -235,6 +235,276 @@ if embedding["dimensions"] != 1024:
|
||||
PY
|
||||
}
|
||||
|
||||
|
||||
verify_workspace_evidence_contract() {
|
||||
local base_root="${1:-$root}"
|
||||
python3 - "$base_root" <<'PY'
|
||||
import pathlib, re, sys, yaml
|
||||
from pathlib import PurePosixPath
|
||||
|
||||
base = pathlib.Path(sys.argv[1])
|
||||
contract_path = base / "docs/contracts/workspace-evidence-v3.md"
|
||||
local_path = base / "docs/install/local-workspace-registry.md"
|
||||
server_path = base / "docs/install/server-workspace-registry.md"
|
||||
bindings_path = base / "docs/install/examples/workspace-bindings.env.example"
|
||||
paths = [contract_path, local_path, server_path, bindings_path]
|
||||
for path in paths:
|
||||
if not path.is_file():
|
||||
raise SystemExit(f"missing workspace Evidence contract input: {path.relative_to(base)}")
|
||||
|
||||
# Generic descriptors must publish an explicit, ID-derived canonical filesystem contract.
|
||||
for relative in ("deploy/workspaces/example.yaml", "deploy/workspaces/psd.yaml.example"):
|
||||
path = base / relative
|
||||
document = yaml.safe_load(path.read_text())
|
||||
workspace_id = document["workspace"]["id"]
|
||||
evidence = document.get("evidence")
|
||||
if not isinstance(evidence, dict) or not isinstance(evidence.get("source"), dict):
|
||||
raise SystemExit(f"{relative}: missing explicit filesystem Evidence contract")
|
||||
uri = evidence["source"].get("uri")
|
||||
expected_uri = f"workspace-content/{workspace_id}/evidence"
|
||||
if not isinstance(uri, str) or uri.startswith("/") or "\\" in uri or ".." in uri.split("/"):
|
||||
raise SystemExit(f"{relative}: noncanonical filesystem Evidence URI")
|
||||
if uri != expected_uri:
|
||||
raise SystemExit(f"{relative}: Evidence namespace mismatch")
|
||||
expected = {
|
||||
"source": {
|
||||
"type": "filesystem",
|
||||
"uri": expected_uri,
|
||||
"patterns": ["**/*.md"],
|
||||
"max_bytes": 10485760,
|
||||
},
|
||||
"policy": {"max_chunk_chars": 4000, "retain_published_generations": 3},
|
||||
}
|
||||
if evidence != expected:
|
||||
raise SystemExit(f"{relative}: explicit filesystem Evidence object mismatch")
|
||||
|
||||
contract = contract_path.read_text()
|
||||
|
||||
def named_example(name):
|
||||
match = re.search(
|
||||
rf"^### Example: {re.escape(name)}\s*$\n\s*```yaml\n(.*?)^```\s*$",
|
||||
contract,
|
||||
re.MULTILINE | re.DOTALL,
|
||||
)
|
||||
if not match:
|
||||
raise SystemExit(f"missing named {name} Evidence YAML example")
|
||||
return yaml.safe_load(match.group(1))
|
||||
|
||||
examples = {
|
||||
"filesystem": {
|
||||
"evidence": {
|
||||
"source": {
|
||||
"type": "filesystem", "uri": "workspace-content/example/evidence",
|
||||
"patterns": ["**/*.md"], "max_bytes": 10485760,
|
||||
},
|
||||
"policy": {"max_chunk_chars": 4000, "retain_published_generations": 3},
|
||||
},
|
||||
},
|
||||
"http": {
|
||||
"evidence": {
|
||||
"source": {
|
||||
"type": "http", "uris": ["https://evidence.example.invalid/report.md"],
|
||||
"authentication": "signed_urls_file", "connect_timeout_ms": 5000,
|
||||
"read_timeout_ms": 30000, "max_bytes": 10485760, "max_redirects": 5,
|
||||
"allow_private_hosts": False, "max_cache_bytes": 67108864,
|
||||
},
|
||||
"policy": {"max_chunk_chars": 4000, "retain_published_generations": 3},
|
||||
},
|
||||
},
|
||||
"s3": {
|
||||
"evidence": {
|
||||
"source": {
|
||||
"type": "s3", "uri": "s3://example-evidence/curated/",
|
||||
"credentials": "static_files", "trusted_endpoint": False,
|
||||
"allow_private_endpoint": False, "allow_insecure_endpoint": False,
|
||||
"max_bytes": 10485760, "max_objects": 10000, "max_pages": 100,
|
||||
"page_size": 1000,
|
||||
},
|
||||
"policy": {"max_chunk_chars": 4000, "retain_published_generations": 3},
|
||||
},
|
||||
},
|
||||
}
|
||||
for name, expected in examples.items():
|
||||
if named_example(name) != expected:
|
||||
raise SystemExit(f"{name} Evidence YAML example shape/default mismatch")
|
||||
|
||||
required_contract_phrases = [
|
||||
"Evidence is optional: a valid v3 descriptor without it remains operational.",
|
||||
"reject unknown keys",
|
||||
"nonempty list of unique, normalized relative POSIX globs",
|
||||
"no whitespace, control character, backslash, userinfo, query, or fragment",
|
||||
"A custom endpoint requires",
|
||||
"HTTP endpoint additionally requires",
|
||||
"page size cannot exceed 1000",
|
||||
"Public docs, exports, and rendered YAML never expose file contents.",
|
||||
"THT_WORKSPACE_SECRET_ROOTS",
|
||||
"readable regular file",
|
||||
"strictly below",
|
||||
"Content-only revision",
|
||||
"read-only Evidence summary",
|
||||
"excludes Evidence bytes",
|
||||
]
|
||||
for phrase in required_contract_phrases:
|
||||
if phrase not in contract:
|
||||
raise SystemExit(f"workspace Evidence contract lacks required rule: {phrase}")
|
||||
|
||||
# The canonical one-repository tree is exact, including generated docs outside workspaces/.
|
||||
legacy_docs = re.compile(r"workspaces/(?:<[^>]+>|[^\s`/]+)\.(?:env\.example|md)")
|
||||
all_public = "\n".join(path.read_text() for path in paths)
|
||||
if legacy_docs.search(all_public) or "workspaces/<id>.env.example" in all_public:
|
||||
raise SystemExit("generated docs path invalid")
|
||||
required_tree_lines = [
|
||||
"registry.git/", "├── workspaces/", "│ ├── example.yaml", "│ └── another.yaml",
|
||||
"├── workspace-content/", "│ ├── example/evidence/...",
|
||||
"│ └── another/evidence/...", "└── workspace-docs/",
|
||||
" ├── example/{contract.env.example,README.md}",
|
||||
" └── another/{contract.env.example,README.md}",
|
||||
]
|
||||
if any(line not in contract for line in required_tree_lines):
|
||||
raise SystemExit("missing canonical Evidence layout")
|
||||
|
||||
def table_for(heading):
|
||||
match = re.search(rf"^## {re.escape(heading)}\s*$", contract, re.MULTILINE)
|
||||
if not match:
|
||||
raise SystemExit(f"missing structured Evidence section: {heading}")
|
||||
rows = []
|
||||
for line in contract[match.end():].splitlines():
|
||||
if line.startswith("## "):
|
||||
break
|
||||
if line.startswith("|"):
|
||||
cells = [cell.strip() for cell in line.strip().strip("|").split("|")]
|
||||
if len(cells) >= 2 and not all(set(cell) <= {"-", ":"} for cell in cells):
|
||||
rows.append(cells)
|
||||
return rows[1:] if rows else []
|
||||
|
||||
relationships = {row[0]: row[1] for row in table_for("Registry revision and phase ownership")}
|
||||
revision_text = relationships.get("Revision identity", "")
|
||||
if "same 40-hex Git commit" not in revision_text or "descriptor blob" not in revision_text or "root tree" not in revision_text:
|
||||
raise SystemExit("missing same-revision ownership")
|
||||
if "Evidence-only commit" not in relationships.get("Content-only revision", "") or "revision.commit" not in relationships.get("Content-only revision", ""):
|
||||
raise SystemExit("missing content-only revision identity")
|
||||
p1 = relationships.get("P1", "")
|
||||
p6 = relationships.get("P6", "")
|
||||
if not all(token in p1 for token in ("lexical URI", "Git tree", "same commit", "does not recursively inspect nested symlinks")):
|
||||
raise SystemExit("missing P1 lexical/tree ownership")
|
||||
if not all(token in p6 for token in ("commit-addressed materialization", "realpath", "recursive containment", "nested-symlink", "race")):
|
||||
raise SystemExit("missing P6 materialization ownership")
|
||||
no_scope = "P1 performs no acquisition, extraction, preprocessing/indexing, embeddings, Qdrant writes, `ACTIVE` publication, retention, or GC."
|
||||
if no_scope not in contract or re.search(r"P1\s+(?:materializes|extracts|indexes)", contract, re.IGNORECASE):
|
||||
raise SystemExit("P1 scope violation")
|
||||
|
||||
installation_rows = {row[0]: row[1:] for row in table_for("Installation files")}
|
||||
http_row = " ".join(installation_rows.get("Signed HTTP", []))
|
||||
if "THT_WS_<NAMESPACE>_EVIDENCE_SIGNED_URLS_FILE" not in http_row or not all(
|
||||
token in http_row for token in ("nonempty JSON string array", "declared-URI order", "query-stripped identities")
|
||||
):
|
||||
raise SystemExit("missing signed HTTP file boundary")
|
||||
s3_pair = " ".join(installation_rows.get("Static S3 pair", []))
|
||||
if not all(token in s3_pair for token in (
|
||||
"THT_WS_<NAMESPACE>_EVIDENCE_ACCESS_KEY_FILE",
|
||||
"THT_WS_<NAMESPACE>_EVIDENCE_SECRET_KEY_FILE", "Required together",
|
||||
)):
|
||||
raise SystemExit("missing static S3 file boundary")
|
||||
s3_token = " ".join(installation_rows.get("Static S3 session", []))
|
||||
if "THT_WS_<NAMESPACE>_EVIDENCE_SESSION_TOKEN_FILE" not in s3_token or "Optional" not in s3_token:
|
||||
raise SystemExit("missing static S3 session-token boundary")
|
||||
|
||||
if "tht config check -c <path>" not in contract:
|
||||
raise SystemExit("exact config-check ordering missing")
|
||||
automated = re.findall(r"^automated integration: (?:PENDING|PASS|FAIL)$", contract, re.MULTILINE)
|
||||
manual = re.findall(r"^manual acceptance: (?:PENDING|PASS|FAIL)$", contract, re.MULTILINE)
|
||||
if len(automated) != 1 or len(manual) != 1:
|
||||
raise SystemExit("separate automated/manual states missing")
|
||||
|
||||
flow_tokens = [
|
||||
"Clone the one shared registry", "workspace-content/<id>/evidence", "commit and push",
|
||||
"Validate and publish the descriptor against that base commit",
|
||||
"workspace-docs/<id>/contract.env.example", "workspace-docs/<id>/README.md",
|
||||
"Evidence `*_FILE` files outside Git", "THT_WORKSPACE_SECRET_ROOTS", "`*_SOURCE` paths",
|
||||
"tht config check -c <path>", "P2/P6 later performs preprocessing and materialization",
|
||||
]
|
||||
for guide in (local_path, server_path):
|
||||
text = guide.read_text()
|
||||
match = re.search(
|
||||
r"^## Curator flow for shared-registry Evidence\s*$\n(.*?)(?=^## |\Z)",
|
||||
text,
|
||||
re.MULTILINE | re.DOTALL,
|
||||
)
|
||||
if not match:
|
||||
raise SystemExit(f"{guide.name}: missing curator flow")
|
||||
section = match.group(1)
|
||||
positions = [section.find(token) for token in flow_tokens]
|
||||
if any(position < 0 for position in positions) or positions != sorted(positions):
|
||||
raise SystemExit(f"{guide.name}: curator flow out of order")
|
||||
|
||||
# Parse dotenv assignments in the core example and fenced public guide blocks. Public examples may
|
||||
# contain paths and query-free identities, but never credential values or unsafe placeholders.
|
||||
def dotenv_lines(path):
|
||||
text = path.read_text()
|
||||
if path == bindings_path:
|
||||
sources = [text]
|
||||
else:
|
||||
sources = re.findall(r"```(?:dotenv|sh)\n(.*?)```", text, re.DOTALL)
|
||||
assignments = []
|
||||
for source in sources:
|
||||
for line in source.splitlines():
|
||||
match = re.match(r"\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$", line)
|
||||
if match:
|
||||
assignments.append((match.group(1), match.group(2).strip().strip("\"'")))
|
||||
return assignments
|
||||
|
||||
def safe_absolute(value):
|
||||
if not value.startswith("/") or "//" in value:
|
||||
return False
|
||||
return all(part not in (".", "..") for part in PurePosixPath(value).parts)
|
||||
|
||||
assignments = []
|
||||
for path in (bindings_path, local_path, server_path):
|
||||
assignments.extend(dotenv_lines(path))
|
||||
unsafe_placeholders = ("changeme", "replace-me", "your_secret", "<secret>")
|
||||
for name, value in assignments:
|
||||
lowered = value.lower()
|
||||
if any(token in lowered for token in unsafe_placeholders):
|
||||
raise SystemExit("unsafe file placeholder/path")
|
||||
if name.endswith(("_FILE", "_SOURCE")) and value and not safe_absolute(value):
|
||||
raise SystemExit("unsafe file placeholder/path")
|
||||
if "_EVIDENCE_" in name and name.endswith("_FILE") and not value.startswith("/run/secrets/"):
|
||||
raise SystemExit("unsafe file placeholder/path")
|
||||
if "_EVIDENCE_" in name and name.endswith("_SOURCE") and not (
|
||||
value.startswith("/srv/thothii/secrets/")
|
||||
or value.startswith("/absolute/path/installation-secrets/")
|
||||
):
|
||||
raise SystemExit("unsafe file placeholder/path")
|
||||
credential_name = re.search(r"(?:SECRET_KEY|ACCESS_KEY|PASSWORD|SESSION_TOKEN|SIGNED_URLS|CREDENTIAL)$", name)
|
||||
if credential_name and value:
|
||||
raise SystemExit("credential literal forbidden")
|
||||
if re.match(r"(?i)(?:AKIA|ASIA)[A-Z0-9]{12,}", value):
|
||||
raise SystemExit("credential literal forbidden")
|
||||
if re.match(r"https?://", value) and "?" in value:
|
||||
raise SystemExit("query-bearing public URI forbidden")
|
||||
|
||||
for uri in re.findall(r"https?://[^\s`\"'<>]+", all_public):
|
||||
if "?" in uri:
|
||||
raise SystemExit("query-bearing public URI forbidden")
|
||||
authority = uri.split("//", 1)[1].split("/", 1)[0]
|
||||
if "@" in authority:
|
||||
raise SystemExit("credential literal forbidden")
|
||||
|
||||
expected_evidence_bindings = {
|
||||
"THT_WS_SIGNED_HTTP_EVIDENCE_SIGNED_URLS_FILE": "/run/secrets/signed-http-evidence-urls.json",
|
||||
"THT_WS_STATIC_S3_EVIDENCE_ACCESS_KEY_FILE": "/run/secrets/static-s3-evidence-access-key",
|
||||
"THT_WS_STATIC_S3_EVIDENCE_SECRET_KEY_FILE": "/run/secrets/static-s3-evidence-secret-key",
|
||||
"THT_WS_STATIC_S3_EVIDENCE_SESSION_TOKEN_FILE": "/run/secrets/static-s3-evidence-session-token",
|
||||
}
|
||||
binding_values = dict(dotenv_lines(bindings_path))
|
||||
for name, value in expected_evidence_bindings.items():
|
||||
if binding_values.get(name) != value:
|
||||
raise SystemExit(f"workspace bindings example mismatch: {name}")
|
||||
|
||||
print("workspace Evidence documentation contract passed")
|
||||
PY
|
||||
}
|
||||
|
||||
verify_vector_helper_interfaces() {
|
||||
local output status
|
||||
output="$(mktemp "${TMPDIR:-/tmp}/thoth-vector-backup-help.XXXXXX")"
|
||||
@@ -1756,6 +2026,7 @@ case "$mode" in
|
||||
[[ $# -eq 1 ]] || { echo "usage: $0 --fixtures-only" >&2; exit 2; }
|
||||
verify_internal_semantic_infrastructure_docs
|
||||
echo "internal semantic infrastructure documentation contract passed"
|
||||
verify_workspace_evidence_contract
|
||||
verify_local_guide
|
||||
verify_windows_line_endings_guide
|
||||
verify_pi_management_guide
|
||||
@@ -1774,12 +2045,14 @@ case "$mode" in
|
||||
|| { echo "usage: $0 --profile {local|server}" >&2; exit 2; }
|
||||
if [[ "$profile" == local ]]; then
|
||||
verify_internal_semantic_infrastructure_docs
|
||||
verify_workspace_evidence_contract
|
||||
verify_local_guide
|
||||
verify_windows_line_endings_guide
|
||||
verify_pi_management_guide
|
||||
verify_local_installation_example
|
||||
else
|
||||
verify_internal_semantic_infrastructure_docs
|
||||
verify_workspace_evidence_contract
|
||||
verify_server_guide
|
||||
verify_reverse_proxy_nginx_guide
|
||||
verify_reverse_proxy_caddy_guide
|
||||
|
||||
Reference in New Issue
Block a user