docs: harden internal semantic doc state
This commit is contained in:
@@ -55,6 +55,18 @@ require_absent() {
|
||||
done
|
||||
}
|
||||
|
||||
require_pattern() {
|
||||
local source="$1" label="$2" pattern="$3"
|
||||
python3 - "$source" "$label" "$pattern" <<'PY'
|
||||
import pathlib, re, sys
|
||||
source = pathlib.Path(sys.argv[1]).read_text()
|
||||
label = sys.argv[2]
|
||||
pattern = sys.argv[3]
|
||||
if not re.search(pattern, source, re.MULTILINE | re.DOTALL):
|
||||
raise SystemExit(f"{label} lacks required pattern: {pattern}")
|
||||
PY
|
||||
}
|
||||
|
||||
require_headings() {
|
||||
local source="$1" label="$2"
|
||||
shift 2
|
||||
@@ -79,10 +91,136 @@ require_text() {
|
||||
done
|
||||
}
|
||||
|
||||
verify_compose_internal_semantic_contract() {
|
||||
python3 - "$root/compose.yaml" <<'PY'
|
||||
import sys, yaml, pathlib
|
||||
doc = yaml.safe_load(pathlib.Path(sys.argv[1]).read_text())
|
||||
services = doc["services"]
|
||||
expected = {"core", "frontend", "qdrant", "embedding", "embedding-model-init"}
|
||||
if set(services) != expected:
|
||||
raise SystemExit(f"compose.yaml services mismatch: {sorted(services)}")
|
||||
core = services["core"]
|
||||
env = core["environment"]
|
||||
for key, value in {
|
||||
"THT_INTERNAL_QDRANT_URL": "http://qdrant:6333",
|
||||
"THT_INTERNAL_EMBEDDING_URL": "http://embedding:11434",
|
||||
"THT_INTERNAL_EMBEDDING_MODEL": "qwen3-embedding:0.6b",
|
||||
"THT_INTERNAL_EMBEDDING_DIMENSIONS": "1024",
|
||||
}.items():
|
||||
if env.get(key) != value:
|
||||
raise SystemExit(f"core missing semantic env {key}={value}")
|
||||
for forbidden in ("THT_VEC_REST_URL", "THT_VEC_WRITE_REST_URL", "THT_OLLAMA_URL"):
|
||||
if forbidden in env:
|
||||
raise SystemExit(f"core still exposes deprecated env {forbidden}")
|
||||
if core["depends_on"]["qdrant"]["condition"] != "service_healthy":
|
||||
raise SystemExit("core must wait for qdrant health")
|
||||
if core["depends_on"]["embedding-model-init"]["condition"] != "service_completed_successfully":
|
||||
raise SystemExit("core must wait for model init success")
|
||||
for name, port in (("qdrant", "6333"), ("embedding", "11434")):
|
||||
service = services[name]
|
||||
if "ports" in service:
|
||||
raise SystemExit(f"{name} must stay private")
|
||||
if service.get("expose") != [port]:
|
||||
raise SystemExit(f"{name} expose mismatch")
|
||||
if "devices" in str(services["embedding"]):
|
||||
raise SystemExit("base embedding service must stay CPU-first")
|
||||
volumes = set(doc["volumes"])
|
||||
for required in ("qdrant-data", "embedding-models"):
|
||||
if required not in volumes:
|
||||
raise SystemExit(f"missing volume {required}")
|
||||
model_init = services["embedding-model-init"]
|
||||
if model_init["environment"].get("OLLAMA_MODEL") != "qwen3-embedding:0.6b":
|
||||
raise SystemExit("model init must pin qwen3-embedding:0.6b")
|
||||
PY
|
||||
}
|
||||
|
||||
verify_workspace_descriptor_semantic_contract() {
|
||||
local source="${1:?source required}"
|
||||
local label="${2:-$source}"
|
||||
python3 - "$source" "$label" <<'PY'
|
||||
import pathlib, sys, yaml
|
||||
path = pathlib.Path(sys.argv[1])
|
||||
label = sys.argv[2]
|
||||
doc = yaml.safe_load(path.read_text())
|
||||
ws = doc["workspace"]
|
||||
semantic = doc["semantic_index"]
|
||||
vector = semantic["vector_store"]
|
||||
embedding = semantic["embedding"]
|
||||
if ws["schema_version"] != 3:
|
||||
raise SystemExit(f"{label}: schema_version must be 3")
|
||||
if vector["engine"] != "qdrant":
|
||||
raise SystemExit(f"{label}: vector store must be qdrant")
|
||||
if vector["collection"] != ws["id"]:
|
||||
raise SystemExit(f"{label}: collection must equal workspace id")
|
||||
if vector["dimensions"] != 1024 or vector["distance"] != "cosine":
|
||||
raise SystemExit(f"{label}: vector contract must be 1024/cosine")
|
||||
if embedding["provider"] != "ollama_internal":
|
||||
raise SystemExit(f"{label}: embedding provider must be ollama_internal")
|
||||
if embedding["model"] != "qwen3-embedding:0.6b":
|
||||
raise SystemExit(f"{label}: embedding model must be qwen3-embedding:0.6b")
|
||||
if embedding["dimensions"] != 1024:
|
||||
raise SystemExit(f"{label}: embedding dimensions must be 1024")
|
||||
PY
|
||||
}
|
||||
|
||||
verify_vector_helper_interfaces() {
|
||||
local output status
|
||||
output="$(mktemp "${TMPDIR:-/tmp}/thoth-vector-backup-help.XXXXXX")"
|
||||
set +e
|
||||
"$root/scripts/vector-backup.sh" >"$output" 2>&1
|
||||
status=$?
|
||||
set -e
|
||||
[[ $status -eq 2 ]] || { cat "$output" >&2; rm -f "$output"; echo "vector-backup usage exit mismatch" >&2; return 1; }
|
||||
grep -Eq 'usage: .*--project-name NAME --output FILE' "$output" || { cat "$output" >&2; rm -f "$output"; echo "vector-backup usage contract changed" >&2; return 1; }
|
||||
set +e
|
||||
"$root/scripts/vector-restore.sh" >"$output" 2>&1
|
||||
status=$?
|
||||
set -e
|
||||
[[ $status -eq 2 ]] || { cat "$output" >&2; rm -f "$output"; echo "vector-restore usage exit mismatch" >&2; return 1; }
|
||||
grep -Eq 'usage: .*--project-name NAME --input FILE --confirm-project NAME' "$output" || { cat "$output" >&2; rm -f "$output"; echo "vector-restore usage contract changed" >&2; return 1; }
|
||||
rm -f "$output"
|
||||
}
|
||||
|
||||
verify_project_state_current_contract() {
|
||||
local source="${1:-$root/PROJECT_STATE.md}"
|
||||
local label="${2:-PROJECT_STATE.md}"
|
||||
python3 - "$source" "$label" <<'PY'
|
||||
import pathlib, re, sys
|
||||
text = pathlib.Path(sys.argv[1]).read_text()
|
||||
label = sys.argv[2]
|
||||
marker = re.search(r"^## Historical snapshots\b", text, re.MULTILINE)
|
||||
if not marker:
|
||||
raise SystemExit(f"{label}: missing Historical snapshots boundary")
|
||||
current = text[:marker.start()]
|
||||
historical = text[marker.start():]
|
||||
required = [
|
||||
r"Internal Qdrant \+ Ollama semantic infrastructure",
|
||||
r"Schema-v3 descriptors are operational",
|
||||
r"migration_required",
|
||||
r"One workspace owns one Qdrant collection",
|
||||
r"Only DWH and LLM remain external",
|
||||
r"embedding-model-init",
|
||||
]
|
||||
for pattern in required:
|
||||
if not re.search(pattern, current, re.MULTILINE):
|
||||
raise SystemExit(f"{label}: current section missing {pattern}")
|
||||
forbidden = [
|
||||
r"Schema-v2 descriptors are operational",
|
||||
r"supported Compose stack is exactly `frontend` plus `core`",
|
||||
r"DWH, vector DB, embedding, LLM",
|
||||
r"vector DB, embedding, and LLM remain external",
|
||||
]
|
||||
for pattern in forbidden:
|
||||
if re.search(pattern, current, re.MULTILINE):
|
||||
raise SystemExit(f"{label}: current section still contains contradictory active text: {pattern}")
|
||||
if re.search(r"^## .*— LIVE", historical, re.MULTILINE):
|
||||
raise SystemExit(f"{label}: historical section still contains LIVE headings")
|
||||
PY
|
||||
}
|
||||
|
||||
verify_internal_semantic_infrastructure_docs() {
|
||||
local readme="$root/README.md"
|
||||
local agents="$root/AGENTS.md"
|
||||
local project_state="$root/PROJECT_STATE.md"
|
||||
local local_manual="$root/docs/install/local-workspace-registry.md"
|
||||
local server_manual="$root/docs/install/server-workspace-registry.md"
|
||||
local compact_manual="$root/docs/installazione-docker-4-contesti.md"
|
||||
@@ -90,67 +228,45 @@ verify_internal_semantic_infrastructure_docs() {
|
||||
local memory="$root/docs/gestione-memory.md"
|
||||
local secrets="$root/deploy/secrets/README.md"
|
||||
|
||||
require_text "$readme" "README" \
|
||||
'mandatory stack is `frontend`, `core`, `qdrant`, `embedding`, and the one-shot `embedding-model-init`' \
|
||||
'`qwen3-embedding:0.6b`' \
|
||||
'`1024`' \
|
||||
'`qdrant-data`' \
|
||||
'`embedding-models`' \
|
||||
'`--confirm-project`' || return 1
|
||||
require_text "$agents" "AGENTS.md" \
|
||||
'Run the local Docker stack with `./scripts/run-stack.sh` after creating `deploy/env/local.env`; it starts the base+local Compose' \
|
||||
'Qdrant and Ollama are internal Compose services' \
|
||||
'DWH and LLM remain external configuration endpoints.' || return 1
|
||||
require_text "$project_state" "PROJECT_STATE.md" \
|
||||
'## Internal Qdrant + Ollama semantic infrastructure' \
|
||||
'Schema-v3 descriptors are operational; schema-v1/v2 descriptors remain `migration_required` until an explicit reviewed migration writes schema version 3.' \
|
||||
'One workspace owns one Qdrant collection' || return 1
|
||||
verify_compose_internal_semantic_contract || return 1
|
||||
verify_workspace_descriptor_semantic_contract "$root/deploy/workspaces/example.yaml" "example workspace" || return 1
|
||||
verify_workspace_descriptor_semantic_contract "$root/deploy/workspaces/psd.yaml.example" "psd workspace example" || return 1
|
||||
verify_vector_helper_interfaces || return 1
|
||||
verify_project_state_current_contract "$root/PROJECT_STATE.md" "PROJECT_STATE.md" || return 1
|
||||
|
||||
require_pattern "$readme" "README" 'mandatory stack.+qdrant.+embedding.+embedding-model-init' || return 1
|
||||
require_pattern "$readme" "README" 'qwen3-embedding:0\.6b' || return 1
|
||||
require_pattern "$readme" "README" 'qdrant-data.+embedding-models' || return 1
|
||||
require_pattern "$readme" "README" 'confirm-project' || return 1
|
||||
require_pattern "$agents" "AGENTS.md" 'Qdrant and Ollama are internal Compose services' || return 1
|
||||
require_pattern "$agents" "AGENTS.md" 'DWH and LLM remain external configuration endpoints' || return 1
|
||||
for manual in "$local_manual" "$server_manual"; do
|
||||
require_text "$manual" "$(basename "$manual")" \
|
||||
'`frontend`, `core`, `qdrant`, `embedding`, and the one-shot `embedding-model-init`' \
|
||||
'`qwen3-embedding:0.6b`' \
|
||||
'`1024`' \
|
||||
'`migration_required`' \
|
||||
'One workspace owns one Qdrant collection' || return 1
|
||||
require_pattern "$manual" "$(basename "$manual")" 'qwen3-embedding:0\.6b' || return 1
|
||||
require_pattern "$manual" "$(basename "$manual")" 'migration_required' || return 1
|
||||
require_pattern "$manual" "$(basename "$manual")" 'One workspace owns one Qdrant collection' || return 1
|
||||
done
|
||||
require_text "$local_manual" "local workspace manual" \
|
||||
'CPU-first' \
|
||||
'THOTH_ENABLE_EMBEDDING_GPU=1' \
|
||||
'Qdrant is a derived but persistent index' || return 1
|
||||
require_text "$server_manual" "server workspace manual" \
|
||||
'Only the Git remote, DWH, LLM, and optional bastion endpoints stay external.' \
|
||||
'Ollama model cache' \
|
||||
'Qdrant backup/restore' || return 1
|
||||
|
||||
require_text "$compact_manual" "four-context install note" \
|
||||
'Qdrant e Ollama embedding sono servizi interni obbligatori del progetto Compose. Restano esterni solo DWH e LLM.' \
|
||||
'qwen3-embedding:0.6b' \
|
||||
'1024 dimensioni' || return 1
|
||||
|
||||
require_text "$diagnostics" "workspace diagnostic protocol" \
|
||||
'schema version 3' \
|
||||
'One workspace owns one Qdrant collection.' \
|
||||
'`semantic_index_incompatible`' || return 1
|
||||
require_pattern "$local_manual" "local workspace manual" 'CPU-first' || return 1
|
||||
require_pattern "$local_manual" "local workspace manual" 'THOTH_ENABLE_EMBEDDING_GPU=1' || return 1
|
||||
require_pattern "$server_manual" "server workspace manual" 'Qdrant backup/restore' || return 1
|
||||
require_pattern "$server_manual" "server workspace manual" 'Only the Git remote, DWH, LLM, and optional bastion endpoints stay external' || return 1
|
||||
require_pattern "$compact_manual" "four-context install note" 'Qdrant e Ollama embedding.+Restano esterni solo DWH e LLM' || return 1
|
||||
require_pattern "$compact_manual" "four-context install note" '1024 dimensioni' || return 1
|
||||
require_pattern "$diagnostics" "workspace diagnostic protocol" 'schema version 3' || return 1
|
||||
require_pattern "$diagnostics" "workspace diagnostic protocol" 'semantic_index_incompatible' || return 1
|
||||
require_absent "$diagnostics" "workspace diagnostic protocol" \
|
||||
'engine: pgvector' \
|
||||
'provider: ollama_compatible' \
|
||||
'THT_WS_<NAMESPACE>_VECTOR_TRANSPORT' \
|
||||
'THT_WS_<NAMESPACE>_EMBEDDING_BASE_URL' || return 1
|
||||
|
||||
require_text "$memory" "memory guide" \
|
||||
'Indice Qdrant' \
|
||||
'Qdrant resta un indice derivato ma persistente' \
|
||||
'`kind`' || return 1
|
||||
require_pattern "$memory" "memory guide" 'Indice Qdrant' || return 1
|
||||
require_pattern "$memory" "memory guide" 'indice derivato ma persistente' || return 1
|
||||
require_pattern "$memory" "memory guide" '`kind`' || return 1
|
||||
require_absent "$memory" "memory guide" \
|
||||
'Indice pgvector' \
|
||||
'save-one costruisce un solo `VectorRecord` e lo invia all''indice pgvector.' || return 1
|
||||
|
||||
require_text "$secrets" "deploy secrets guide" \
|
||||
'`THT_MODEL_API_KEY`, `THT_DWH_API_KEY`, `THT_CA`, and `THT_SSL_CA`' \
|
||||
'Do not add vector or embedding endpoint credentials to the bundle.' || return 1
|
||||
require_absent "$secrets" "deploy secrets guide" \
|
||||
'PI_PROVIDER_API_KEY' || return 1
|
||||
"all'indice pgvector" || return 1
|
||||
require_pattern "$secrets" "deploy secrets guide" 'THT_MODEL_API_KEY.+THT_DWH_API_KEY.+THT_CA.+THT_SSL_CA' || return 1
|
||||
require_pattern "$secrets" "deploy secrets guide" 'Do not add vector or embedding endpoint credentials to the bundle' || return 1
|
||||
require_absent "$secrets" "deploy secrets guide" 'PI_PROVIDER_API_KEY' || return 1
|
||||
}
|
||||
|
||||
verify_local_guide() {
|
||||
|
||||
Reference in New Issue
Block a user