diff --git a/deploy/workspaces/example.yaml b/deploy/workspaces/example.yaml index d85f5e2d..9c6e1978 100644 --- a/deploy/workspaces/example.yaml +++ b/deploy/workspaces/example.yaml @@ -24,6 +24,17 @@ semantic_index: model: qwen3-embedding:0.6b dimensions: 1024 +evidence: + source: + type: filesystem + uri: workspace-content/example/evidence + patterns: + - "**/*.md" + max_bytes: 10485760 + policy: + max_chunk_chars: 4000 + retain_published_generations: 3 + llm_policy: default: zai/glm-5.2 allowed: diff --git a/deploy/workspaces/psd.yaml.example b/deploy/workspaces/psd.yaml.example index 3e59dbdd..d80862ae 100644 --- a/deploy/workspaces/psd.yaml.example +++ b/deploy/workspaces/psd.yaml.example @@ -24,6 +24,17 @@ semantic_index: model: qwen3-embedding:0.6b dimensions: 1024 +evidence: + source: + type: filesystem + uri: workspace-content/example-workspace/evidence + patterns: + - "**/*.md" + max_bytes: 10485760 + policy: + max_chunk_chars: 4000 + retain_published_generations: 3 + llm_policy: default: zai/glm-5.2 allowed: diff --git a/docs/contracts/workspace-evidence-v3.md b/docs/contracts/workspace-evidence-v3.md new file mode 100644 index 00000000..03b35aea --- /dev/null +++ b/docs/contracts/workspace-evidence-v3.md @@ -0,0 +1,172 @@ +# Workspace Evidence v3 contract + +This is the canonical public contract for the optional `evidence` object in a schema-v3 +workspace descriptor. Evidence is optional: a valid v3 descriptor without it remains operational. +When present, `evidence` is strict: it contains `source` and a defaulted strict `policy`; every +source variant and the policy reject unknown keys. + +## Filesystem source + +A filesystem source uses the exact URI `workspace-content//evidence`. `patterns` is +a nonempty list of unique, normalized relative POSIX globs. Its defaults are +`patterns: ["**/*.md"]` and `max_bytes: 10485760`. + +### Example: filesystem + +```yaml +evidence: + source: + type: filesystem + uri: workspace-content/example/evidence + patterns: + - "**/*.md" + max_bytes: 10485760 + policy: + max_chunk_chars: 4000 + retain_published_generations: 3 +``` + +Safe: `workspace-content/example/evidence`. Unsafe filesystem identities include `/srv/evidence`, +`workspace-content/another/evidence`, and `workspace-content/example/../another/evidence` because +absolute, cross-namespace, and traversal paths are not canonical. + +## HTTP source + +`uris` is a nonempty, unique list of canonical HTTP or HTTPS provenance identities. Each URI must +have no whitespace, control character, backslash, userinfo, query, or fragment. Public mode uses +`authentication: none`; signed mode uses `authentication: signed_urls_file`. Defaults are +`authentication: none`, `connect_timeout_ms: 5000`, `read_timeout_ms: 30000`, +`max_bytes: 10485760`, `max_redirects: 5`, `allow_private_hosts: false`, and +`max_cache_bytes: 67108864`. + +### Example: http + +```yaml +evidence: + source: + type: http + uris: + - https://evidence.example.invalid/report.md + authentication: signed_urls_file + connect_timeout_ms: 5000 + read_timeout_ms: 30000 + max_bytes: 10485760 + max_redirects: 5 + allow_private_hosts: false + max_cache_bytes: 67108864 + policy: + max_chunk_chars: 4000 + retain_published_generations: 3 +``` + +The shown provenance URI is safe and query-free. An HTTP fragment identity such as +`https://evidence.example.invalid/report.md#section` is unsafe. Query-bearing and userinfo-bearing +HTTP identities are rejected; public documentation must not spell or publish a signed transport +URL. + +## S3 source + +`uri` is a canonical `s3://` identity with a valid bucket and no port, userinfo, query, or fragment. +`endpoint_url`, when present, is an origin-only HTTP(S) URL; `region`, when present, is nonblank. +Defaults are `credentials: ambient`, `trusted_endpoint: false`, `allow_private_endpoint: false`, +`allow_insecure_endpoint: false`, `max_bytes: 10485760`, `max_objects: 10000`, `max_pages: 100`, +and `page_size: 1000` (and page size cannot exceed 1000). A custom endpoint requires +`trusted_endpoint: true`; an HTTP endpoint additionally requires `allow_insecure_endpoint: true`. +Static mode uses `credentials: static_files`, requires access-key and secret-key files together, +and permits an optional session-token file. + +### Example: s3 + +```yaml +evidence: + source: + type: s3 + uri: s3://example-evidence/curated/ + credentials: static_files + trusted_endpoint: false + allow_private_endpoint: false + allow_insecure_endpoint: false + max_bytes: 10485760 + max_objects: 10000 + max_pages: 100 + page_size: 1000 + policy: + max_chunk_chars: 4000 + retain_published_generations: 3 +``` + +Safe: `s3://example-evidence/curated/`. Unsafe identities include +`s3://Invalid_Bucket/evidence` and `s3://example-evidence/evidence#section`. S3 userinfo and query +identities are rejected in prose and implementation; no credential-bearing example is published. + +## Policy + +The strict policy defaults to `max_chunk_chars: 4000` and +`retain_published_generations: 3`. + +## Installation files + +The namespace is the workspace ID uppercased with every `-` changed to `_`. All Evidence variables +hold file paths, never credential or signed-URL values. + +| Mode | Variable | File contract | +| --- | --- | --- | +| Signed HTTP | `THT_WS__EVIDENCE_SIGNED_URLS_FILE` | Required for `signed_urls_file`; nonempty JSON string array in declared-URI order; query-stripped identities must match `uris`. | +| Static S3 pair | `THT_WS__EVIDENCE_ACCESS_KEY_FILE` and `THT_WS__EVIDENCE_SECRET_KEY_FILE` | Required together for `static_files`. | +| Static S3 session | `THT_WS__EVIDENCE_SESSION_TOKEN_FILE` | Optional, and valid only with the required access/secret pair. | + +Every variable is an absolute path to a readable regular file whose resolved target is strictly below one of the roots configured by `THT_WORKSPACE_SECRET_ROOTS`. Scalar S3 files are nonempty +UTF-8 tokens without whitespace. Public docs, exports, and rendered YAML never expose file contents. `changeme`, `replace-me`, `YOUR_SECRET`, ``, access-key-looking strings, and any +credential-bearing or query-bearing URI are forbidden as public placeholder values. + +## One shared registry repository + +All workspace namespaces live in one Git repository: + +```text +registry.git/ +├── workspaces/ +│ ├── example.yaml +│ └── another.yaml +├── workspace-content/ +│ ├── example/evidence/... +│ └── another/evidence/... +└── workspace-docs/ + ├── example/{contract.env.example,README.md} + └── another/{contract.env.example,README.md} +``` + +Curators change only `workspace-content//evidence/**` through a normal clone. The API publishes +only `workspaces/.yaml` and +`workspace-docs//{contract.env.example,README.md}`. It never writes Evidence source bytes. + +## Registry revision and phase ownership + +| Relationship | Contract | +| --- | --- | +| Revision identity | The descriptor blob and filesystem Evidence root tree are checked at the same 40-hex Git commit. | +| Content-only revision | An Evidence-only commit changes authoritative `revision.commit` even when the descriptor blob is unchanged. | +| Browser | Create and edit flows preserve and show a read-only Evidence summary. | +| Export | Export remains exactly manifest, descriptor, contract, and README; it excludes Evidence bytes. | +| P1 | Validates the lexical URI and proves the declared filesystem root object is a Git tree at that same commit; it does not recursively inspect nested symlinks. | +| P6 | Owns commit-addressed materialization, realpath and recursive containment, nested-symlink checks, and race checks. | + +P1 performs no acquisition, extraction, preprocessing/indexing, embeddings, Qdrant writes, `ACTIVE` publication, retention, or GC. + +## Operator validation + +After the runtime configuration is rendered or acquired, validate it with the exact per-command +option ordering: + +```sh +tht config check -c +``` + +Stop after validation. P2/P6 later owns preprocessing and materialization. + +## Acceptance states + +These gates are independent and are not implied by this documentation contract. + +automated integration: PENDING +manual acceptance: PENDING diff --git a/docs/install/examples/workspace-bindings.env.example b/docs/install/examples/workspace-bindings.env.example index 770a76cf..09b0bfbd 100644 --- a/docs/install/examples/workspace-bindings.env.example +++ b/docs/install/examples/workspace-bindings.env.example @@ -5,3 +5,10 @@ THT_WS_NORTH_STAR_RESEARCH_DWH_HOST=dwh.internal.example THT_WS_NORTH_STAR_RESEARCH_DWH_PORT=5432 THT_WS_NORTH_STAR_RESEARCH_DWH_USER=thoth_reader THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password + +# Evidence examples use separate illustrative namespaces because one descriptor selects one mode. +# Values are container file paths only; signed URLs and credential contents stay in those files. +THT_WS_SIGNED_HTTP_EVIDENCE_SIGNED_URLS_FILE=/run/secrets/signed-http-evidence-urls.json +THT_WS_STATIC_S3_EVIDENCE_ACCESS_KEY_FILE=/run/secrets/static-s3-evidence-access-key +THT_WS_STATIC_S3_EVIDENCE_SECRET_KEY_FILE=/run/secrets/static-s3-evidence-secret-key +THT_WS_STATIC_S3_EVIDENCE_SESSION_TOKEN_FILE=/run/secrets/static-s3-evidence-session-token diff --git a/docs/install/local-workspace-registry.md b/docs/install/local-workspace-registry.md index 11d217e3..5bc7f6cb 100644 --- a/docs/install/local-workspace-registry.md +++ b/docs/install/local-workspace-registry.md @@ -39,10 +39,13 @@ Create one private repository such as `thoth-workspaces.git`. It contains canoni definitions and generated artifacts only: ```text -thoth-workspaces.yaml -workspaces/.yaml -workspaces/.env.example -workspaces/.md +registry.git/ +├── workspaces/ +│ └── .yaml +├── workspace-content/ +│ └── /evidence/... +└── workspace-docs/ + └── /{contract.env.example,README.md} ``` For SSH, use a scoped deploy key, a verified `known_hosts` file, and strict host-key checking. For @@ -66,6 +69,32 @@ THT_WORKSPACE_GIT_CA_FILE=/absolute/path/installation-secrets/git-ca.pem For HTTPS set `THT_WORKSPACE_GIT_CREDENTIALS_FILE` instead of the SSH key/known-hosts pair. Remote and branch are non-secret; every `*_FILE` is a local path whose content never enters Git or logs. +## Curator flow for shared-registry Evidence + +Follow this order; the [canonical Evidence contract](../contracts/workspace-evidence-v3.md) defines +the source shapes and safety boundary. + +1. Clone the one shared registry, or update the review clone with `git pull --ff-only`. +2. Add source bytes below `workspace-content//evidence`, then commit and push. +3. Validate and publish the descriptor against that base commit. +4. Inspect `workspace-docs//contract.env.example` and `workspace-docs//README.md`. +5. Provision only the selected Evidence `*_FILE` files outside Git and strictly below a root in `THT_WORKSPACE_SECRET_ROOTS`; add matching host-only `*_SOURCE` paths for the generated connector override. +6. Render or acquire the runtime config, then run `tht config check -c `. +7. Stop: P2/P6 later performs preprocessing and materialization. + +For example, a signed-HTTP workspace and a different static-S3 workspace can use these host-only +connector sources; the values are paths, not file contents: + +```dotenv +THT_WS_SIGNED_HTTP_EVIDENCE_SIGNED_URLS_SOURCE=/absolute/path/installation-secrets/signed-http-evidence-urls.json +THT_WS_STATIC_S3_EVIDENCE_ACCESS_KEY_SOURCE=/absolute/path/installation-secrets/static-s3-evidence-access-key +THT_WS_STATIC_S3_EVIDENCE_SECRET_KEY_SOURCE=/absolute/path/installation-secrets/static-s3-evidence-secret-key +THT_WS_STATIC_S3_EVIDENCE_SESSION_TOKEN_SOURCE=/absolute/path/installation-secrets/static-s3-evidence-session-token +``` + +The descriptor and declared filesystem root are validated at the same registry commit. The +browser shows a read-only Evidence summary, while exports omit Evidence bytes. + ## Shared Git values, local bindings, and secret files | Location | Contains | Never contains | diff --git a/docs/install/server-workspace-registry.md b/docs/install/server-workspace-registry.md index 841bb9a7..6b3bc043 100644 --- a/docs/install/server-workspace-registry.md +++ b/docs/install/server-workspace-registry.md @@ -50,8 +50,10 @@ account Gitea administration, database-superuser rights, or a shell in the Git h Create a private Gitea (or compatible Git) repository such as `platform/thoth-workspaces`. Protect `main` according to the release policy and grant the ThothII publisher only the intended repository -scope. Commit canonical schema-v3 descriptors and generated `.md`/`.env.example` artifacts only; -do not commit installation bindings or secret material. +scope. Commit canonical schema-v3 descriptors under `workspaces/.yaml`, curated Evidence +under `workspace-content//evidence/**`, and generated public artifacts only at +`workspace-docs//README.md` and `workspace-docs//contract.env.example`; do not commit +installation bindings or secret material. For SSH, create a least-privilege deploy key, record Gitea's host key in managed known-hosts, and use `ssh://git@git.example.invalid/platform/thoth-workspaces.git`. For HTTPS, create a scoped @@ -62,6 +64,32 @@ Bootstrap an empty remote from a temporary review clone: migrate legacy descript schema-v3 identity and generated artifacts, commit, and push `main`. The running server is not an authoring environment for migration. +## Curator flow for shared-registry Evidence + +Follow this order; the [canonical Evidence contract](../contracts/workspace-evidence-v3.md) defines +the source shapes and safety boundary. + +1. Clone the one shared registry, or update the review clone with `git pull --ff-only`. +2. Add source bytes below `workspace-content//evidence`, then commit and push. +3. Validate and publish the descriptor against that base commit. +4. Inspect `workspace-docs//contract.env.example` and `workspace-docs//README.md`. +5. Provision only the selected Evidence `*_FILE` files outside Git and strictly below a root in `THT_WORKSPACE_SECRET_ROOTS`; add matching host-only `*_SOURCE` paths for the generated connector override. +6. Render or acquire the runtime config, then run `tht config check -c `. +7. Stop: P2/P6 later performs preprocessing and materialization. + +For example, separate signed-HTTP and static-S3 workspaces can use these host-only connector source +paths: + +```dotenv +THT_WS_SIGNED_HTTP_EVIDENCE_SIGNED_URLS_SOURCE=/srv/thothii/secrets/signed-http-evidence-urls.json +THT_WS_STATIC_S3_EVIDENCE_ACCESS_KEY_SOURCE=/srv/thothii/secrets/static-s3-evidence-access-key +THT_WS_STATIC_S3_EVIDENCE_SECRET_KEY_SOURCE=/srv/thothii/secrets/static-s3-evidence-secret-key +THT_WS_STATIC_S3_EVIDENCE_SESSION_TOKEN_SOURCE=/srv/thothii/secrets/static-s3-evidence-session-token +``` + +The descriptor and declared filesystem root are validated at the same registry commit. The +browser shows a read-only Evidence summary, while exports omit Evidence bytes. + ## Git credentials, CA, SSH key, and known-hosts mounts Use the secret manager or a protected host-only procedure to create independent regular files under diff --git a/scripts/test-verify-workspace-install-docs.sh b/scripts/test-verify-workspace-install-docs.sh index d307dbe9..6c586bf0 100755 --- a/scripts/test-verify-workspace-install-docs.sh +++ b/scripts/test-verify-workspace-install-docs.sh @@ -12,6 +12,7 @@ trap 'rm -f "$output" "$verifier_functions"; rm -rf "$negative_root"' EXIT HUP I for fixture in \ "internal semantic infrastructure documentation contract" \ + "workspace Evidence documentation contract" \ "local installation guide contract" \ "source update fail-closed semantics" \ "Windows line-ending recovery guide contract" \ @@ -430,6 +431,126 @@ if [[ $adapted_status -eq 0 ]] || ! grep -Fq "Caddy adapted frontend path bypass fi negative_failures=0 +expect_evidence_fixture_rejected() { + local label="$1" target="$2" mutation="$3" expected_error="$4" + local fixture_root="$negative_root/evidence-${label// /-}" + local fixture_output="$fixture_root/output" + + # Before Task 7's dedicated verifier exists, every mutation is deliberately accepted. This + # makes the complete Evidence test matrix RED without allowing command-not-found to abort it. + if ! declare -F verify_workspace_evidence_contract >/dev/null; then + echo "negative fixture accepted: $label (Evidence verifier missing)" >&2 + negative_failures=$((negative_failures + 1)) + return + fi + + mkdir -p \ + "$fixture_root/deploy/workspaces" \ + "$fixture_root/docs/contracts" \ + "$fixture_root/docs/install/examples" + cp "$root/deploy/workspaces/example.yaml" "$fixture_root/deploy/workspaces/example.yaml" + cp "$root/deploy/workspaces/psd.yaml.example" "$fixture_root/deploy/workspaces/psd.yaml.example" + cp "$root/docs/contracts/workspace-evidence-v3.md" \ + "$fixture_root/docs/contracts/workspace-evidence-v3.md" + cp "$root/docs/install/local-workspace-registry.md" \ + "$fixture_root/docs/install/local-workspace-registry.md" + cp "$root/docs/install/server-workspace-registry.md" \ + "$fixture_root/docs/install/server-workspace-registry.md" + cp "$root/docs/install/examples/workspace-bindings.env.example" \ + "$fixture_root/docs/install/examples/workspace-bindings.env.example" + + python3 - "$fixture_root/$target" "$mutation" <<'PY' +import pathlib, sys, yaml +path = pathlib.Path(sys.argv[1]) +mutation = sys.argv[2] +original = path.read_text() +changed = original +if mutation == "layout-omitted": + changed = original.replace("│ ├── example/evidence/...\n", "", 1) +elif mutation == "same-commit-omitted": + changed = original.replace( + "| Revision identity | The descriptor blob and filesystem Evidence root tree are checked at the same 40-hex Git commit. |\n", + "", + 1, + ) +elif mutation in {"absolute-filesystem", "cross-workspace"}: + document = yaml.safe_load(original) + document["evidence"]["source"]["uri"] = ( + "/srv/evidence" if mutation == "absolute-filesystem" + else "workspace-content/example/evidence" + ) + changed = yaml.safe_dump(document, sort_keys=False) +elif mutation == "wrong-docs-directory": + changed = original.replace( + "workspace-docs/\n ├── example/{contract.env.example,README.md}\n └── another/{contract.env.example,README.md}", + "workspaces/.env.example\nworkspaces/.md", + 1, + ) +elif mutation == "http-file-boundary-omitted": + changed = original.replace( + "| Signed HTTP | `THT_WS__EVIDENCE_SIGNED_URLS_FILE` | Required for `signed_urls_file`; nonempty JSON string array in declared-URI order; query-stripped identities must match `uris`. |\n", + "", + 1, + ) +elif mutation == "s3-pair-boundary-omitted": + changed = original.replace( + "| Static S3 pair | `THT_WS__EVIDENCE_ACCESS_KEY_FILE` and `THT_WS__EVIDENCE_SECRET_KEY_FILE` | Required together for `static_files`. |\n", + "", + 1, + ) +elif mutation == "s3-token-boundary-omitted": + changed = original.replace( + "| Static S3 session | `THT_WS__EVIDENCE_SESSION_TOKEN_FILE` | Optional, and valid only with the required access/secret pair. |\n", + "", + 1, + ) +elif mutation == "credential-literal": + changed = original + "\nTHT_WS_STATIC_S3_EVIDENCE_SECRET_KEY=AKIAEXAMPLECREDENTIAL\n" +elif mutation == "signed-query-example": + signed_query = "https://evidence.example.invalid/report" + "?X-Amz-Signature=unsafe" + changed = original + f"\nTHT_EVIDENCE_URI={signed_query}\n" +elif mutation == "unsafe-placeholder": + changed = original.replace( + "/run/secrets/signed-http-evidence-urls.json", "changeme", 1 + ) +elif mutation == "p1-scope-inversion": + changed = original.replace( + "P1 performs no acquisition, extraction, preprocessing/indexing, embeddings, Qdrant writes, `ACTIVE` publication, retention, or GC.", + "P1 materializes, extracts, and indexes Evidence before publication.", + 1, + ) +elif mutation == "config-ordering": + changed = original.replace( + "tht config check -c ", "tht -c config check", 1 + ) +elif mutation == "acceptance-conflation": + changed = original.replace("manual acceptance: PENDING\n", "", 1) +elif mutation == "curator-order": + second = "2. Add source bytes below `workspace-content//evidence`, then commit and push." + third = "3. Validate and publish the descriptor against that base commit." + changed = original.replace(second + "\n" + third, third + "\n" + second, 1) +else: + raise SystemExit(f"unknown Evidence mutation: {mutation}") +if changed == original: + raise SystemExit(f"Evidence mutation made no change: {mutation}") +path.write_text(changed) +PY + + set +e + verify_workspace_evidence_contract "$fixture_root" >"$fixture_output" 2>&1 + local status=$? + set -e + if [[ $status -eq 0 ]]; then + echo "negative fixture accepted: $label" >&2 + cat "$fixture_output" >&2 + negative_failures=$((negative_failures + 1)) + elif ! grep -Fq -- "$expected_error" "$fixture_output"; then + echo "negative fixture failed for the wrong reason: $label" >&2 + cat "$fixture_output" >&2 + negative_failures=$((negative_failures + 1)) + fi +} + expect_guide_rejected() { local label="$1" validator="$2" source_guide="$3" relative_path="$4" local mutation="$5" expected_error="$6" @@ -784,6 +905,52 @@ expect_guide_rejected \ "$root/docs/install/windows-line-endings.md" docs/install/windows-line-endings.md powershell-crlf-failure \ "PowerShell CRLF repair lacks failure propagation: Assert-NativeSuccess 'index export'" +expect_evidence_fixture_rejected \ + "canonical Evidence layout omitted" docs/contracts/workspace-evidence-v3.md layout-omitted \ + "missing canonical Evidence layout" +expect_evidence_fixture_rejected \ + "same revision ownership omitted" docs/contracts/workspace-evidence-v3.md same-commit-omitted \ + "missing same-revision ownership" +expect_evidence_fixture_rejected \ + "absolute filesystem Evidence path" deploy/workspaces/example.yaml absolute-filesystem \ + "noncanonical filesystem Evidence URI" +expect_evidence_fixture_rejected \ + "cross-workspace Evidence path" deploy/workspaces/psd.yaml.example cross-workspace \ + "Evidence namespace mismatch" +expect_evidence_fixture_rejected \ + "generated docs in wrong directory" docs/contracts/workspace-evidence-v3.md wrong-docs-directory \ + "generated docs path invalid" +expect_evidence_fixture_rejected \ + "signed HTTP file boundary omitted" docs/contracts/workspace-evidence-v3.md http-file-boundary-omitted \ + "missing signed HTTP file boundary" +expect_evidence_fixture_rejected \ + "static S3 pair boundary omitted" docs/contracts/workspace-evidence-v3.md s3-pair-boundary-omitted \ + "missing static S3 file boundary" +expect_evidence_fixture_rejected \ + "static S3 optional token boundary omitted" docs/contracts/workspace-evidence-v3.md s3-token-boundary-omitted \ + "missing static S3 session-token boundary" +expect_evidence_fixture_rejected \ + "credential literal in public bindings" docs/install/examples/workspace-bindings.env.example credential-literal \ + "credential literal forbidden" +expect_evidence_fixture_rejected \ + "signed query in public bindings" docs/install/examples/workspace-bindings.env.example signed-query-example \ + "query-bearing public URI forbidden" +expect_evidence_fixture_rejected \ + "unsafe Evidence file placeholder" docs/install/examples/workspace-bindings.env.example unsafe-placeholder \ + "unsafe file placeholder/path" +expect_evidence_fixture_rejected \ + "P1 Evidence scope inversion" docs/contracts/workspace-evidence-v3.md p1-scope-inversion \ + "P1 scope violation" +expect_evidence_fixture_rejected \ + "config check option reordered" docs/contracts/workspace-evidence-v3.md config-ordering \ + "exact config-check ordering missing" +expect_evidence_fixture_rejected \ + "acceptance states conflated" docs/contracts/workspace-evidence-v3.md acceptance-conflation \ + "separate automated/manual states missing" +expect_evidence_fixture_rejected \ + "local curator flow reordered" docs/install/local-workspace-registry.md curator-order \ + "curator flow out of order" + if (( negative_failures != 0 )); then echo "$negative_failures unsafe installation-document fixtures were accepted" >&2 exit 1 diff --git a/scripts/verify-workspace-install-docs.sh b/scripts/verify-workspace-install-docs.sh index 0abd9c1c..5a13022d 100755 --- a/scripts/verify-workspace-install-docs.sh +++ b/scripts/verify-workspace-install-docs.sh @@ -235,6 +235,276 @@ if embedding["dimensions"] != 1024: PY } + +verify_workspace_evidence_contract() { + local base_root="${1:-$root}" + python3 - "$base_root" <<'PY' +import pathlib, re, sys, yaml +from pathlib import PurePosixPath + +base = pathlib.Path(sys.argv[1]) +contract_path = base / "docs/contracts/workspace-evidence-v3.md" +local_path = base / "docs/install/local-workspace-registry.md" +server_path = base / "docs/install/server-workspace-registry.md" +bindings_path = base / "docs/install/examples/workspace-bindings.env.example" +paths = [contract_path, local_path, server_path, bindings_path] +for path in paths: + if not path.is_file(): + raise SystemExit(f"missing workspace Evidence contract input: {path.relative_to(base)}") + +# Generic descriptors must publish an explicit, ID-derived canonical filesystem contract. +for relative in ("deploy/workspaces/example.yaml", "deploy/workspaces/psd.yaml.example"): + path = base / relative + document = yaml.safe_load(path.read_text()) + workspace_id = document["workspace"]["id"] + evidence = document.get("evidence") + if not isinstance(evidence, dict) or not isinstance(evidence.get("source"), dict): + raise SystemExit(f"{relative}: missing explicit filesystem Evidence contract") + uri = evidence["source"].get("uri") + expected_uri = f"workspace-content/{workspace_id}/evidence" + if not isinstance(uri, str) or uri.startswith("/") or "\\" in uri or ".." in uri.split("/"): + raise SystemExit(f"{relative}: noncanonical filesystem Evidence URI") + if uri != expected_uri: + raise SystemExit(f"{relative}: Evidence namespace mismatch") + expected = { + "source": { + "type": "filesystem", + "uri": expected_uri, + "patterns": ["**/*.md"], + "max_bytes": 10485760, + }, + "policy": {"max_chunk_chars": 4000, "retain_published_generations": 3}, + } + if evidence != expected: + raise SystemExit(f"{relative}: explicit filesystem Evidence object mismatch") + +contract = contract_path.read_text() + +def named_example(name): + match = re.search( + rf"^### Example: {re.escape(name)}\s*$\n\s*```yaml\n(.*?)^```\s*$", + contract, + re.MULTILINE | re.DOTALL, + ) + if not match: + raise SystemExit(f"missing named {name} Evidence YAML example") + return yaml.safe_load(match.group(1)) + +examples = { + "filesystem": { + "evidence": { + "source": { + "type": "filesystem", "uri": "workspace-content/example/evidence", + "patterns": ["**/*.md"], "max_bytes": 10485760, + }, + "policy": {"max_chunk_chars": 4000, "retain_published_generations": 3}, + }, + }, + "http": { + "evidence": { + "source": { + "type": "http", "uris": ["https://evidence.example.invalid/report.md"], + "authentication": "signed_urls_file", "connect_timeout_ms": 5000, + "read_timeout_ms": 30000, "max_bytes": 10485760, "max_redirects": 5, + "allow_private_hosts": False, "max_cache_bytes": 67108864, + }, + "policy": {"max_chunk_chars": 4000, "retain_published_generations": 3}, + }, + }, + "s3": { + "evidence": { + "source": { + "type": "s3", "uri": "s3://example-evidence/curated/", + "credentials": "static_files", "trusted_endpoint": False, + "allow_private_endpoint": False, "allow_insecure_endpoint": False, + "max_bytes": 10485760, "max_objects": 10000, "max_pages": 100, + "page_size": 1000, + }, + "policy": {"max_chunk_chars": 4000, "retain_published_generations": 3}, + }, + }, +} +for name, expected in examples.items(): + if named_example(name) != expected: + raise SystemExit(f"{name} Evidence YAML example shape/default mismatch") + +required_contract_phrases = [ + "Evidence is optional: a valid v3 descriptor without it remains operational.", + "reject unknown keys", + "nonempty list of unique, normalized relative POSIX globs", + "no whitespace, control character, backslash, userinfo, query, or fragment", + "A custom endpoint requires", + "HTTP endpoint additionally requires", + "page size cannot exceed 1000", + "Public docs, exports, and rendered YAML never expose file contents.", + "THT_WORKSPACE_SECRET_ROOTS", + "readable regular file", + "strictly below", + "Content-only revision", + "read-only Evidence summary", + "excludes Evidence bytes", +] +for phrase in required_contract_phrases: + if phrase not in contract: + raise SystemExit(f"workspace Evidence contract lacks required rule: {phrase}") + +# The canonical one-repository tree is exact, including generated docs outside workspaces/. +legacy_docs = re.compile(r"workspaces/(?:<[^>]+>|[^\s`/]+)\.(?:env\.example|md)") +all_public = "\n".join(path.read_text() for path in paths) +if legacy_docs.search(all_public) or "workspaces/.env.example" in all_public: + raise SystemExit("generated docs path invalid") +required_tree_lines = [ + "registry.git/", "├── workspaces/", "│ ├── example.yaml", "│ └── another.yaml", + "├── workspace-content/", "│ ├── example/evidence/...", + "│ └── another/evidence/...", "└── workspace-docs/", + " ├── example/{contract.env.example,README.md}", + " └── another/{contract.env.example,README.md}", +] +if any(line not in contract for line in required_tree_lines): + raise SystemExit("missing canonical Evidence layout") + +def table_for(heading): + match = re.search(rf"^## {re.escape(heading)}\s*$", contract, re.MULTILINE) + if not match: + raise SystemExit(f"missing structured Evidence section: {heading}") + rows = [] + for line in contract[match.end():].splitlines(): + if line.startswith("## "): + break + if line.startswith("|"): + cells = [cell.strip() for cell in line.strip().strip("|").split("|")] + if len(cells) >= 2 and not all(set(cell) <= {"-", ":"} for cell in cells): + rows.append(cells) + return rows[1:] if rows else [] + +relationships = {row[0]: row[1] for row in table_for("Registry revision and phase ownership")} +revision_text = relationships.get("Revision identity", "") +if "same 40-hex Git commit" not in revision_text or "descriptor blob" not in revision_text or "root tree" not in revision_text: + raise SystemExit("missing same-revision ownership") +if "Evidence-only commit" not in relationships.get("Content-only revision", "") or "revision.commit" not in relationships.get("Content-only revision", ""): + raise SystemExit("missing content-only revision identity") +p1 = relationships.get("P1", "") +p6 = relationships.get("P6", "") +if not all(token in p1 for token in ("lexical URI", "Git tree", "same commit", "does not recursively inspect nested symlinks")): + raise SystemExit("missing P1 lexical/tree ownership") +if not all(token in p6 for token in ("commit-addressed materialization", "realpath", "recursive containment", "nested-symlink", "race")): + raise SystemExit("missing P6 materialization ownership") +no_scope = "P1 performs no acquisition, extraction, preprocessing/indexing, embeddings, Qdrant writes, `ACTIVE` publication, retention, or GC." +if no_scope not in contract or re.search(r"P1\s+(?:materializes|extracts|indexes)", contract, re.IGNORECASE): + raise SystemExit("P1 scope violation") + +installation_rows = {row[0]: row[1:] for row in table_for("Installation files")} +http_row = " ".join(installation_rows.get("Signed HTTP", [])) +if "THT_WS__EVIDENCE_SIGNED_URLS_FILE" not in http_row or not all( + token in http_row for token in ("nonempty JSON string array", "declared-URI order", "query-stripped identities") +): + raise SystemExit("missing signed HTTP file boundary") +s3_pair = " ".join(installation_rows.get("Static S3 pair", [])) +if not all(token in s3_pair for token in ( + "THT_WS__EVIDENCE_ACCESS_KEY_FILE", + "THT_WS__EVIDENCE_SECRET_KEY_FILE", "Required together", +)): + raise SystemExit("missing static S3 file boundary") +s3_token = " ".join(installation_rows.get("Static S3 session", [])) +if "THT_WS__EVIDENCE_SESSION_TOKEN_FILE" not in s3_token or "Optional" not in s3_token: + raise SystemExit("missing static S3 session-token boundary") + +if "tht config check -c " not in contract: + raise SystemExit("exact config-check ordering missing") +automated = re.findall(r"^automated integration: (?:PENDING|PASS|FAIL)$", contract, re.MULTILINE) +manual = re.findall(r"^manual acceptance: (?:PENDING|PASS|FAIL)$", contract, re.MULTILINE) +if len(automated) != 1 or len(manual) != 1: + raise SystemExit("separate automated/manual states missing") + +flow_tokens = [ + "Clone the one shared registry", "workspace-content//evidence", "commit and push", + "Validate and publish the descriptor against that base commit", + "workspace-docs//contract.env.example", "workspace-docs//README.md", + "Evidence `*_FILE` files outside Git", "THT_WORKSPACE_SECRET_ROOTS", "`*_SOURCE` paths", + "tht config check -c ", "P2/P6 later performs preprocessing and materialization", +] +for guide in (local_path, server_path): + text = guide.read_text() + match = re.search( + r"^## Curator flow for shared-registry Evidence\s*$\n(.*?)(?=^## |\Z)", + text, + re.MULTILINE | re.DOTALL, + ) + if not match: + raise SystemExit(f"{guide.name}: missing curator flow") + section = match.group(1) + positions = [section.find(token) for token in flow_tokens] + if any(position < 0 for position in positions) or positions != sorted(positions): + raise SystemExit(f"{guide.name}: curator flow out of order") + +# Parse dotenv assignments in the core example and fenced public guide blocks. Public examples may +# contain paths and query-free identities, but never credential values or unsafe placeholders. +def dotenv_lines(path): + text = path.read_text() + if path == bindings_path: + sources = [text] + else: + sources = re.findall(r"```(?:dotenv|sh)\n(.*?)```", text, re.DOTALL) + assignments = [] + for source in sources: + for line in source.splitlines(): + match = re.match(r"\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$", line) + if match: + assignments.append((match.group(1), match.group(2).strip().strip("\"'"))) + return assignments + +def safe_absolute(value): + if not value.startswith("/") or "//" in value: + return False + return all(part not in (".", "..") for part in PurePosixPath(value).parts) + +assignments = [] +for path in (bindings_path, local_path, server_path): + assignments.extend(dotenv_lines(path)) +unsafe_placeholders = ("changeme", "replace-me", "your_secret", "") +for name, value in assignments: + lowered = value.lower() + if any(token in lowered for token in unsafe_placeholders): + raise SystemExit("unsafe file placeholder/path") + if name.endswith(("_FILE", "_SOURCE")) and value and not safe_absolute(value): + raise SystemExit("unsafe file placeholder/path") + if "_EVIDENCE_" in name and name.endswith("_FILE") and not value.startswith("/run/secrets/"): + raise SystemExit("unsafe file placeholder/path") + if "_EVIDENCE_" in name and name.endswith("_SOURCE") and not ( + value.startswith("/srv/thothii/secrets/") + or value.startswith("/absolute/path/installation-secrets/") + ): + raise SystemExit("unsafe file placeholder/path") + credential_name = re.search(r"(?:SECRET_KEY|ACCESS_KEY|PASSWORD|SESSION_TOKEN|SIGNED_URLS|CREDENTIAL)$", name) + if credential_name and value: + raise SystemExit("credential literal forbidden") + if re.match(r"(?i)(?:AKIA|ASIA)[A-Z0-9]{12,}", value): + raise SystemExit("credential literal forbidden") + if re.match(r"https?://", value) and "?" in value: + raise SystemExit("query-bearing public URI forbidden") + +for uri in re.findall(r"https?://[^\s`\"'<>]+", all_public): + if "?" in uri: + raise SystemExit("query-bearing public URI forbidden") + authority = uri.split("//", 1)[1].split("/", 1)[0] + if "@" in authority: + raise SystemExit("credential literal forbidden") + +expected_evidence_bindings = { + "THT_WS_SIGNED_HTTP_EVIDENCE_SIGNED_URLS_FILE": "/run/secrets/signed-http-evidence-urls.json", + "THT_WS_STATIC_S3_EVIDENCE_ACCESS_KEY_FILE": "/run/secrets/static-s3-evidence-access-key", + "THT_WS_STATIC_S3_EVIDENCE_SECRET_KEY_FILE": "/run/secrets/static-s3-evidence-secret-key", + "THT_WS_STATIC_S3_EVIDENCE_SESSION_TOKEN_FILE": "/run/secrets/static-s3-evidence-session-token", +} +binding_values = dict(dotenv_lines(bindings_path)) +for name, value in expected_evidence_bindings.items(): + if binding_values.get(name) != value: + raise SystemExit(f"workspace bindings example mismatch: {name}") + +print("workspace Evidence documentation contract passed") +PY +} + verify_vector_helper_interfaces() { local output status output="$(mktemp "${TMPDIR:-/tmp}/thoth-vector-backup-help.XXXXXX")" @@ -1756,6 +2026,7 @@ case "$mode" in [[ $# -eq 1 ]] || { echo "usage: $0 --fixtures-only" >&2; exit 2; } verify_internal_semantic_infrastructure_docs echo "internal semantic infrastructure documentation contract passed" + verify_workspace_evidence_contract verify_local_guide verify_windows_line_endings_guide verify_pi_management_guide @@ -1774,12 +2045,14 @@ case "$mode" in || { echo "usage: $0 --profile {local|server}" >&2; exit 2; } if [[ "$profile" == local ]]; then verify_internal_semantic_infrastructure_docs + verify_workspace_evidence_contract verify_local_guide verify_windows_line_endings_guide verify_pi_management_guide verify_local_installation_example else verify_internal_semantic_infrastructure_docs + verify_workspace_evidence_contract verify_server_guide verify_reverse_proxy_nginx_guide verify_reverse_proxy_caddy_guide