From 3588a7749bcd229307c63202812da414c250fc2a Mon Sep 17 00:00:00 2001 From: mptyl Date: Sun, 12 Jul 2026 01:32:33 +0200 Subject: [PATCH] fix(vector): harden packaged migrations --- .superpowers/sdd/pgvector-task-2-report.md | 30 +++++ docker/core.Dockerfile | 3 +- harness/migrations/vector/001_extensions.sql | 1 - harness/pyproject.toml | 3 + harness/tests/l0/test_pgvector_store.py | 4 +- harness/tests/l0/test_vector_migrations.py | 124 ++++++++++++++++-- .../tests/test_vector_migration_packaging.py | 59 +++++++++ harness/tht/adapters/vector/pgvector.py | 31 +++-- harness/tht/cli/vector_migrate_cmd.py | 70 +++++++--- .../tht/migrations/vector/001_extensions.sql | 3 + .../migrations/vector/002_schema_tables.sql | 16 +-- .../{ => tht}/migrations/vector/003_roles.sql | 4 +- scripts/test-vector-migration-image.sh | 42 ++++++ scripts/verify-container-images.sh | 1 + 14 files changed, 337 insertions(+), 54 deletions(-) delete mode 100644 harness/migrations/vector/001_extensions.sql create mode 100644 harness/tests/test_vector_migration_packaging.py create mode 100644 harness/tht/migrations/vector/001_extensions.sql rename harness/{ => tht}/migrations/vector/002_schema_tables.sql (68%) rename harness/{ => tht}/migrations/vector/003_roles.sql (81%) create mode 100755 scripts/test-vector-migration-image.sh diff --git a/.superpowers/sdd/pgvector-task-2-report.md b/.superpowers/sdd/pgvector-task-2-report.md index 7fdb5ea2..6fcac409 100644 --- a/.superpowers/sdd/pgvector-task-2-report.md +++ b/.superpowers/sdd/pgvector-task-2-report.md @@ -50,3 +50,33 @@ No unresolved Task 2 correctness concern found. One deliberate contract choice i writer `INSERT` and `UPDATE` are table-level because the approved direct adapter health probe uses `has_table_privilege` for those authorities. Least privilege is retained by withholding broad `SELECT`, `DELETE`, DDL, ownership, and credentials. + +## Review fix wave + +The post-implementation review found four production-boundary gaps. They are fixed as follows: + +- Migration SQL now ships inside the `tht` wheel (`tht/migrations/vector`) via explicit + setuptools package-data and is discovered through `importlib.resources`, rather than relying on + a source-checkout-relative directory. +- Both status and apply reject ledger versions absent from the installed manifest, including + nonnumeric future version labels. This treats a binary/database downgrade as drift instead of + silently reporting a healthy state. +- Migration files are ordered by parsed integer version; spellings such as `2` and `02` are + rejected as duplicate versions. +- Every migration transaction pins `search_path` locally to `pg_catalog, pg_temp`; catalog calls + and the ledger are schema-qualified. pgvector is installed into the locked `vectors` schema, + tables use `vectors.vector`, and `PgVectorStore` qualifies vector casts and the cosine operator. + A hostile admin default path with a writable shadow schema cannot redirect migration objects. +- The core image build asserts CLI discovery. Image verification now starts an ephemeral pgvector + database, runs the installed image's migration command, and compares pristine apply/status JSON. + +Additional verification after the fix wave: + +- Focused migration, adapter, hostile-path, and wheel suite: `27 passed`. +- Full harness: `477 passed, 5 deselected`. +- Production core image build: passed, including build-time CLI discovery. +- Core-image apply/status smoke against `pgvector/pgvector:pg16`: passed. +- Changed production and test files: Ruff clean; `git diff --check` clean. +- Full Ruff remains at the same 34 pre-existing unrelated test-file findings documented above. + +No dependency changed, so the committed Python requirements lock did not require regeneration. diff --git a/docker/core.Dockerfile b/docker/core.Dockerfile index ef1b5942..7347f2da 100644 --- a/docker/core.Dockerfile +++ b/docker/core.Dockerfile @@ -40,7 +40,8 @@ COPY docker/python-runtime/requirements.lock /app/docker/python-runtime/requirem RUN python -m venv /opt/venv \ && /opt/venv/bin/pip install --no-cache-dir --require-hashes \ --requirement /app/docker/python-runtime/requirements.lock \ - && /opt/venv/bin/pip install --no-cache-dir --no-deps --no-build-isolation /app/harness + && /opt/venv/bin/pip install --no-cache-dir --no-deps --no-build-isolation /app/harness \ + && /opt/venv/bin/tht vector migrate --help >/dev/null COPY --from=backend-build /src/backend/dist /app/backend/dist COPY --from=backend-build /src/backend/node_modules /app/backend/node_modules diff --git a/harness/migrations/vector/001_extensions.sql b/harness/migrations/vector/001_extensions.sql deleted file mode 100644 index 0aa0fc22..00000000 --- a/harness/migrations/vector/001_extensions.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS vector; diff --git a/harness/pyproject.toml b/harness/pyproject.toml index 12b952be..28f7d23d 100644 --- a/harness/pyproject.toml +++ b/harness/pyproject.toml @@ -31,6 +31,9 @@ dev = [ [tool.setuptools.packages.find] include = ["tht*"] +[tool.setuptools.package-data] +tht = ["migrations/vector/*.sql"] + [tool.ruff] line-length = 100 diff --git a/harness/tests/l0/test_pgvector_store.py b/harness/tests/l0/test_pgvector_store.py index 4dea38ea..69414491 100644 --- a/harness/tests/l0/test_pgvector_store.py +++ b/harness/tests/l0/test_pgvector_store.py @@ -45,8 +45,8 @@ def vector_configs(): ) engine = create_engine(pg.get_connection_url()) with engine.begin() as connection: - connection.exec_driver_sql("CREATE EXTENSION vector") connection.exec_driver_sql("CREATE SCHEMA vectors") + connection.exec_driver_sql("CREATE EXTENSION vector WITH SCHEMA vectors") for table in ("schema_records", "evidence", "memory"): connection.exec_driver_sql(f""" CREATE TABLE vectors.{table} ( @@ -55,7 +55,7 @@ def vector_configs(): kind text NOT NULL, content_hash text NOT NULL, metadata jsonb NOT NULL, - embedding vector(2) NOT NULL, + embedding vectors.vector(2) NOT NULL, indexed_at timestamptz NOT NULL DEFAULT now() ) """) diff --git a/harness/tests/l0/test_vector_migrations.py b/harness/tests/l0/test_vector_migrations.py index 15a208bf..b5f660ea 100644 --- a/harness/tests/l0/test_vector_migrations.py +++ b/harness/tests/l0/test_vector_migrations.py @@ -59,9 +59,9 @@ def test_schema_matches_direct_adapter_contract(database_url): for table in tables: assert {row.column_name for row in rows if row.table_name == table} == required assert vector_types == [ - ("evidence", "vector(768)"), - ("memory", "vector(768)"), - ("schema_records", "vector(768)"), + ("evidence", "vectors.vector(768)"), + ("memory", "vectors.vector(768)"), + ("schema_records", "vectors.vector(768)"), ] @@ -119,7 +119,7 @@ def test_roles_have_runtime_privileges_only(database_url): "INSERT INTO vectors.memory " "(record_key, kind, content_hash, metadata, embedding) " "VALUES ('reader-write', 'memory', 'x', '{}', " - "array_fill(0, ARRAY[768])::vector)" + "array_fill(0, ARRAY[768])::vectors.vector)" ) ) @@ -128,7 +128,8 @@ def test_roles_have_runtime_privileges_only(database_url): text( "INSERT INTO vectors.memory " "(record_key, kind, content_hash, metadata, embedding) " - "VALUES ('writer-ok', 'memory', 'x', '{}', array_fill(0, ARRAY[768])::vector)" + "VALUES ('writer-ok', 'memory', 'x', '{}', " + "array_fill(0, ARRAY[768])::vectors.vector)" ) ) assert connection.execute( @@ -176,15 +177,112 @@ def test_checksum_drift_is_reported_and_refused(database_url, tmp_path): migrate(database_url, migrations) +def test_unknown_applied_version_is_downgrade_drift(database_url): + from tht.cli.vector_migrate_cmd import MigrationError, migrate, migration_status + + migrate(database_url) + engine = create_engine(database_url) + with engine.begin() as connection: + connection.execute( + text( + "INSERT INTO public.tht_vector_migrations (version, name, checksum) " + "VALUES ('999', 'future', 'future-checksum'), " + "('future_x', 'future_named', 'future-checksum')" + ) + ) + try: + with pytest.raises( + MigrationError, match="absent from local manifest: 999, future_x" + ): + migration_status(database_url) + with pytest.raises( + MigrationError, match="absent from local manifest: 999, future_x" + ): + migrate(database_url) + finally: + with engine.begin() as connection: + connection.execute( + text( + "DELETE FROM public.tht_vector_migrations " + "WHERE version IN ('999', 'future_x')" + ) + ) + engine.dispose() + + +def test_migration_versions_sort_numerically_and_reject_numeric_duplicates(tmp_path): + from tht.cli.vector_migrate_cmd import MigrationError, _discover + + migrations = tmp_path / "ordered" + migrations.mkdir() + (migrations / "10_tenth.sql").write_text("SELECT 10;\n") + (migrations / "2_second.sql").write_text("SELECT 2;\n") + assert [item.version for item in _discover(migrations)] == ["2", "10"] + + (migrations / "02_duplicate.sql").write_text("SELECT 2;\n") + with pytest.raises(MigrationError, match="Duplicate migration version: 2"): + _discover(migrations) + + +def test_hostile_admin_search_path_cannot_shadow_migration_objects(database_url): + from tht.cli.vector_migrate_cmd import migrate + + admin = create_engine(database_url, isolation_level="AUTOCOMMIT") + with admin.connect() as connection: + connection.exec_driver_sql("DROP DATABASE IF EXISTS vector_hostile") + connection.exec_driver_sql("CREATE DATABASE vector_hostile") + hostile_url = admin.url.set(database="vector_hostile") + hostile = create_engine(hostile_url) + try: + with hostile.begin() as connection: + connection.exec_driver_sql("CREATE SCHEMA shadow") + connection.exec_driver_sql( + "CREATE TABLE shadow.tht_vector_migrations " + "(version text, checksum text, poisoned boolean DEFAULT true)" + ) + connection.exec_driver_sql("ALTER ROLE test SET search_path = shadow, public") + hostile.dispose() + + migrate(hostile_url.render_as_string(hide_password=False)) + + verification = create_engine(hostile_url) + with verification.connect() as connection: + assert connection.execute( + text("SELECT count(*) FROM public.tht_vector_migrations") + ).scalar_one() == 3 + assert connection.execute( + text("SELECT count(*) FROM shadow.tht_vector_migrations") + ).scalar_one() == 0 + assert connection.execute( + text( + "SELECT format_type(a.atttypid, a.atttypmod) " + "FROM pg_catalog.pg_attribute a " + "WHERE a.attrelid = 'vectors.memory'::pg_catalog.regclass " + "AND a.attname = 'embedding'" + ) + ).scalar_one() == "vectors.vector(768)" + verification.dispose() + finally: + cleanup = create_engine(database_url, isolation_level="AUTOCOMMIT") + with cleanup.connect() as connection: + connection.exec_driver_sql("ALTER ROLE test RESET search_path") + connection.exec_driver_sql( + "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity " + "WHERE datname = 'vector_hostile' AND pid <> pg_catalog.pg_backend_pid()" + ) + connection.exec_driver_sql("DROP DATABASE IF EXISTS vector_hostile") + cleanup.dispose() + admin.dispose() + + def test_failed_batch_rolls_back_schema_and_ledger(database_url, tmp_path): from tht.cli.vector_migrate_cmd import MigrationError, migrate, migration_status - migrations = tmp_path / "failed" - migrations.mkdir() - (migrations / "101_first.sql").write_text("CREATE TABLE public.must_rollback (id int);\n") - (migrations / "102_broken.sql").write_text("THIS IS NOT SQL;\n") + migrations = _copy_migrations(tmp_path) + (migrations / "004_first.sql").write_text("CREATE TABLE public.must_rollback (id int);\n") + (migrations / "005_broken.sql").write_text("THIS IS NOT SQL;\n") - with pytest.raises(MigrationError, match="102_broken.sql"): + with pytest.raises(MigrationError, match="005_broken.sql"): migrate(database_url, migrations) engine = create_engine(database_url) @@ -192,12 +290,12 @@ def test_failed_batch_rolls_back_schema_and_ledger(database_url, tmp_path): assert connection.execute(text("SELECT to_regclass('public.must_rollback')")).scalar() is None engine.dispose() status = migration_status(database_url, migrations) - assert status.applied == () - assert [item.version for item in status.pending] == ["101", "102"] + assert [item.version for item in status.applied] == ["001", "002", "003"] + assert [item.version for item in status.pending] == ["004", "005"] def _copy_migrations(tmp_path: Path) -> Path: - source = Path(__file__).parents[2] / "migrations" / "vector" + source = Path(__file__).parents[2] / "tht" / "migrations" / "vector" target = tmp_path / "migrations" target.mkdir() for migration in source.glob("*.sql"): diff --git a/harness/tests/test_vector_migration_packaging.py b/harness/tests/test_vector_migration_packaging.py new file mode 100644 index 00000000..24544c97 --- /dev/null +++ b/harness/tests/test_vector_migration_packaging.py @@ -0,0 +1,59 @@ +import os +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path + + +def test_built_wheel_installs_vector_migrations_and_discovers_cli(tmp_path): + harness = Path(__file__).parents[1] + wheelhouse = tmp_path / "wheelhouse" + target = tmp_path / "site" + wheelhouse.mkdir() + uv = shutil.which("uv") + assert uv is not None, "uv is required to verify the production wheel" + build_env = {**os.environ, "UV_CACHE_DIR": str(tmp_path / "uv-cache")} + subprocess.run( + [ + uv, + "build", + "--wheel", + "--out-dir", + str(wheelhouse), + str(harness), + ], + check=True, + capture_output=True, + text=True, + env=build_env, + ) + wheel = next(wheelhouse.glob("tht-*.whl")) + with zipfile.ZipFile(wheel) as archive: + names = set(archive.namelist()) + assert "tht/migrations/vector/001_extensions.sql" in names + assert "tht/migrations/vector/003_roles.sql" in names + + subprocess.run( + [sys.executable, "-m", "pip", "install", "--no-deps", "--target", str(target), wheel], + check=True, + capture_output=True, + text=True, + ) + env = {**os.environ, "PYTHONPATH": str(target)} + probe = subprocess.run( + [ + sys.executable, + "-c", + "from typer.testing import CliRunner; from tht.cli import app; " + "r=CliRunner().invoke(app, ['vector','migrate','--help']); " + "print(r.output); raise SystemExit(r.exit_code)", + ], + env=env, + check=False, + capture_output=True, + text=True, + cwd=tmp_path, + ) + assert probe.returncode == 0, probe.stderr + probe.stdout + assert "--status" in probe.stdout diff --git a/harness/tht/adapters/vector/pgvector.py b/harness/tht/adapters/vector/pgvector.py index fd975da3..09797b86 100644 --- a/harness/tht/adapters/vector/pgvector.py +++ b/harness/tht/adapters/vector/pgvector.py @@ -27,7 +27,7 @@ COLLECTION_KINDS = { } ALLOWED_COLLECTIONS = frozenset(COLLECTION_KINDS) ALLOWED_KINDS = frozenset().union(*COLLECTION_KINDS.values()) -_VECTOR_DIMENSION = re.compile(r"^vector\((\d+)\)$") +_VECTOR_DIMENSION = re.compile(r"^(?:[a-z_][a-z0-9_]*\.)?vector\((\d+)\)$") def _collection(schema: str, name: str) -> sql.Identifier: @@ -40,6 +40,14 @@ def _vector_literal(values: list[float]) -> str: return "[" + ",".join(str(float(value)) for value in values) + "]" +def _vector_type(schema: str) -> sql.Identifier: + return sql.Identifier(schema, "vector") + + +def _cosine_operator(schema: str) -> sql.Composed: + return sql.SQL("OPERATOR({}.<=>)").format(sql.Identifier(schema)) + + def _validate_collection_kinds(collection: str, kinds: list[str]) -> None: invalid = set(kinds) - COLLECTION_KINDS[collection] if invalid: @@ -248,9 +256,16 @@ class PgVectorStore: continue where = sql.SQL(" WHERE kind = ANY(%s)") if collection_kinds else sql.SQL("") query = sql.SQL( - "SELECT metadata, 1 - (embedding <=> %s::vector) AS similarity " - "FROM {}{} ORDER BY embedding <=> %s::vector LIMIT %s" - ).format(table, where) + "SELECT metadata, 1 - (embedding {} %s::{}) AS similarity " + "FROM {}{} ORDER BY embedding {} %s::{} LIMIT %s" + ).format( + _cosine_operator(self._schema), + _vector_type(self._schema), + table, + where, + _cosine_operator(self._schema), + _vector_type(self._schema), + ) params = [_vector_literal(embedding)] if collection_kinds: params.append(collection_kinds) @@ -295,13 +310,13 @@ class PgVectorStore: raise VectorStoreError("Embedding dimension does not match configured dimension") insert = sql.SQL( "INSERT INTO {} (record_key, kind, content_hash, metadata, embedding) " - "VALUES (%s, %s, %s, %s::jsonb, %s::vector) " + "VALUES (%s, %s, %s, %s::jsonb, %s::{}) " "ON CONFLICT (record_key) DO NOTHING" - ).format(table) + ).format(table, _vector_type(self._schema)) update = sql.SQL( "UPDATE {} SET kind = %s, content_hash = %s, metadata = %s::jsonb, " - "embedding = %s::vector, indexed_at = now() WHERE record_key = %s" - ).format(table) + "embedding = %s::{}, indexed_at = pg_catalog.now() WHERE record_key = %s" + ).format(table, _vector_type(self._schema)) raw = engine.raw_connection() try: with raw.cursor() as cursor: diff --git a/harness/tht/cli/vector_migrate_cmd.py b/harness/tht/cli/vector_migrate_cmd.py index 51e9c1f8..3ed4243b 100644 --- a/harness/tht/cli/vector_migrate_cmd.py +++ b/harness/tht/cli/vector_migrate_cmd.py @@ -6,6 +6,8 @@ import hashlib import json import re from dataclasses import dataclass +from importlib.resources import files +from importlib.resources.abc import Traversable from pathlib import Path import typer @@ -14,7 +16,7 @@ from sqlalchemy.exc import SQLAlchemyError from tht.cli.vector_cmd import vector_app -MIGRATIONS_DIR = Path(__file__).parents[2] / "migrations" / "vector" +MIGRATIONS_DIR = files("tht").joinpath("migrations", "vector") _MIGRATION_NAME = re.compile(r"^(?P\d+)_(?P[a-z0-9_]+)\.sql$") _LOCK_KEY = 7_304_708_654_221_909_028 @@ -27,7 +29,7 @@ class MigrationError(RuntimeError): class Migration: version: str name: str - path: Path + path: Traversable checksum: str @@ -38,32 +40,44 @@ class MigrationStatus: drifted: tuple[Migration, ...] -def _discover(directory: Path) -> tuple[Migration, ...]: +def _migration_source(directory: Traversable | Path | str) -> Traversable: + return Path(directory) if isinstance(directory, (str, Path)) else directory + + +def _discover(directory: Traversable | Path | str) -> tuple[Migration, ...]: + source = _migration_source(directory) migrations = [] - seen_versions: set[str] = set() - for path in sorted(directory.glob("*.sql")): + seen_versions: set[int] = set() + paths = [path for path in source.iterdir() if path.name.endswith(".sql")] + parsed = [] + for path in paths: match = _MIGRATION_NAME.fullmatch(path.name) if match is None: raise MigrationError(f"Invalid migration filename: {path.name}") version = match.group("version") - if version in seen_versions: - raise MigrationError(f"Duplicate migration version: {version}") - seen_versions.add(version) + numeric_version = int(version) + if numeric_version in seen_versions: + raise MigrationError(f"Duplicate migration version: {numeric_version}") + seen_versions.add(numeric_version) + parsed.append((numeric_version, version, match.group("name"), path)) + for _, version, name, path in sorted(parsed, key=lambda item: item[0]): migrations.append( Migration( version=version, - name=match.group("name"), + name=name, path=path, checksum=hashlib.sha256(path.read_bytes()).hexdigest(), ) ) if not migrations: - raise MigrationError(f"No migrations found in {directory}") + raise MigrationError(f"No migrations found in {source}") return tuple(migrations) def _applied(connection) -> dict[str, str]: - exists = connection.execute(text("SELECT to_regclass('public.tht_vector_migrations')")).scalar() + exists = connection.execute( + text("SELECT pg_catalog.to_regclass('public.tht_vector_migrations')") + ).scalar() if exists is None: return {} return dict( @@ -73,16 +87,32 @@ def _applied(connection) -> dict[str, str]: ) +def _reject_unknown_versions( + migrations: tuple[Migration, ...], applied_checksums: dict[str, str] +) -> None: + local_versions = {migration.version for migration in migrations} + unknown = sorted( + set(applied_checksums) - local_versions, + key=lambda version: (0, int(version)) if version.isdigit() else (1, version), + ) + if unknown: + raise MigrationError( + "Database migration versions absent from local manifest: " + ", ".join(unknown) + ) + + def migration_status( - database_url: str, migrations_dir: Path | str = MIGRATIONS_DIR + database_url: str, migrations_dir: Traversable | Path | str = MIGRATIONS_DIR ) -> MigrationStatus: - migrations = _discover(Path(migrations_dir)) + migrations = _discover(migrations_dir) engine = create_engine(database_url) try: with engine.connect() as connection: + connection.exec_driver_sql("SET LOCAL search_path = pg_catalog, pg_temp") applied_checksums = _applied(connection) finally: engine.dispose() + _reject_unknown_versions(migrations, applied_checksums) applied = tuple( migration for migration in migrations @@ -100,25 +130,31 @@ def migration_status( return MigrationStatus(applied=applied, pending=pending, drifted=drifted) -def migrate(database_url: str, migrations_dir: Path | str = MIGRATIONS_DIR) -> MigrationStatus: - migrations = _discover(Path(migrations_dir)) +def migrate( + database_url: str, migrations_dir: Traversable | Path | str = MIGRATIONS_DIR +) -> MigrationStatus: + migrations = _discover(migrations_dir) engine = create_engine(database_url) current: Migration | None = None try: with engine.begin() as connection: - connection.execute(text("SELECT pg_advisory_xact_lock(:key)"), {"key": _LOCK_KEY}) + connection.exec_driver_sql("SET LOCAL search_path = pg_catalog, pg_temp") + connection.execute( + text("SELECT pg_catalog.pg_advisory_xact_lock(:key)"), {"key": _LOCK_KEY} + ) connection.exec_driver_sql( """CREATE TABLE IF NOT EXISTS public.tht_vector_migrations ( version text PRIMARY KEY, name text NOT NULL, checksum text NOT NULL, - applied_at timestamptz NOT NULL DEFAULT now() + applied_at timestamptz NOT NULL DEFAULT pg_catalog.now() )""" ) connection.exec_driver_sql( "REVOKE ALL ON public.tht_vector_migrations FROM PUBLIC" ) applied_checksums = _applied(connection) + _reject_unknown_versions(migrations, applied_checksums) drifted = [ item for item in migrations diff --git a/harness/tht/migrations/vector/001_extensions.sql b/harness/tht/migrations/vector/001_extensions.sql new file mode 100644 index 00000000..f64f725a --- /dev/null +++ b/harness/tht/migrations/vector/001_extensions.sql @@ -0,0 +1,3 @@ +CREATE SCHEMA IF NOT EXISTS vectors; +REVOKE ALL ON SCHEMA vectors FROM PUBLIC; +CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA vectors; diff --git a/harness/migrations/vector/002_schema_tables.sql b/harness/tht/migrations/vector/002_schema_tables.sql similarity index 68% rename from harness/migrations/vector/002_schema_tables.sql rename to harness/tht/migrations/vector/002_schema_tables.sql index 640d593c..2cffaea8 100644 --- a/harness/migrations/vector/002_schema_tables.sql +++ b/harness/tht/migrations/vector/002_schema_tables.sql @@ -1,15 +1,11 @@ -CREATE SCHEMA IF NOT EXISTS vectors; - -REVOKE ALL ON SCHEMA vectors FROM PUBLIC; - CREATE TABLE IF NOT EXISTS vectors.schema_records ( id bigserial PRIMARY KEY, record_key text UNIQUE NOT NULL, kind text NOT NULL, content_hash text NOT NULL, metadata jsonb NOT NULL, - embedding vector(768) NOT NULL, - indexed_at timestamptz NOT NULL DEFAULT now() + embedding vectors.vector(768) NOT NULL, + indexed_at timestamptz NOT NULL DEFAULT pg_catalog.now() ); CREATE TABLE IF NOT EXISTS vectors.evidence ( @@ -18,8 +14,8 @@ CREATE TABLE IF NOT EXISTS vectors.evidence ( kind text NOT NULL, content_hash text NOT NULL, metadata jsonb NOT NULL, - embedding vector(768) NOT NULL, - indexed_at timestamptz NOT NULL DEFAULT now() + embedding vectors.vector(768) NOT NULL, + indexed_at timestamptz NOT NULL DEFAULT pg_catalog.now() ); CREATE TABLE IF NOT EXISTS vectors.memory ( @@ -28,8 +24,8 @@ CREATE TABLE IF NOT EXISTS vectors.memory ( kind text NOT NULL, content_hash text NOT NULL, metadata jsonb NOT NULL, - embedding vector(768) NOT NULL, - indexed_at timestamptz NOT NULL DEFAULT now() + embedding vectors.vector(768) NOT NULL, + indexed_at timestamptz NOT NULL DEFAULT pg_catalog.now() ); REVOKE ALL ON ALL TABLES IN SCHEMA vectors FROM PUBLIC; diff --git a/harness/migrations/vector/003_roles.sql b/harness/tht/migrations/vector/003_roles.sql similarity index 81% rename from harness/migrations/vector/003_roles.sql rename to harness/tht/migrations/vector/003_roles.sql index 34de7cf5..ba88bd2b 100644 --- a/harness/migrations/vector/003_roles.sql +++ b/harness/tht/migrations/vector/003_roles.sql @@ -1,9 +1,9 @@ DO $roles$ BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'vector_reader') THEN + IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'vector_reader') THEN CREATE ROLE vector_reader NOLOGIN; END IF; - IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'vector_writer') THEN + IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'vector_writer') THEN CREATE ROLE vector_writer NOLOGIN; END IF; END diff --git a/scripts/test-vector-migration-image.sh b/scripts/test-vector-migration-image.sh new file mode 100755 index 00000000..cfeffb97 --- /dev/null +++ b/scripts/test-vector-migration-image.sh @@ -0,0 +1,42 @@ +#!/bin/sh +set -eu + +image=${1:?usage: test-vector-migration-image.sh IMAGE [PLATFORM]} +platform=${2:-${PLATFORM:-linux/arm64}} +slug=$$ +network="thoth-vector-migration-$slug" +database="thoth-vector-db-$slug" + +cleanup() { + docker rm --force "$database" >/dev/null 2>&1 || true + docker network rm "$network" >/dev/null 2>&1 || true +} +trap cleanup EXIT INT TERM + +docker network create "$network" >/dev/null +docker run --detach --rm --platform "$platform" --name "$database" --network "$network" \ + -e POSTGRES_DB=thoth -e POSTGRES_USER=thoth_admin -e POSTGRES_PASSWORD=test-only \ + pgvector/pgvector:pg16 >/dev/null + +attempt=0 +until docker exec "$database" pg_isready -U thoth_admin -d thoth >/dev/null 2>&1; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge 30 ]; then + echo "pgvector test database did not become ready" >&2 + exit 1 + fi + sleep 1 +done + +database_url="postgresql+psycopg2://thoth_admin:test-only@$database:5432/thoth" +applied=$(docker run --rm --platform "$platform" --network "$network" \ + --entrypoint /opt/venv/bin/tht -e THT_VECTOR_ADMIN_URL="$database_url" \ + "$image" vector migrate --json) +status=$(docker run --rm --platform "$platform" --network "$network" \ + --entrypoint /opt/venv/bin/tht -e THT_VECTOR_ADMIN_URL="$database_url" \ + "$image" vector migrate --status --json) + +expected='{"applied": ["001", "002", "003"], "drifted": [], "pending": []}' +test "$applied" = "$expected" +test "$status" = "$expected" +echo "core image vector migration discovery/status smoke passed" diff --git a/scripts/verify-container-images.sh b/scripts/verify-container-images.sh index 15513859..eee3da93 100755 --- a/scripts/verify-container-images.sh +++ b/scripts/verify-container-images.sh @@ -18,6 +18,7 @@ docker buildx build --platform "$platform" --load \ docker run --rm --platform "$platform" --entrypoint /app/docker/smoke/core-smoke.sh \ "$core_image" +./scripts/test-vector-migration-image.sh "$core_image" "$platform" docker run --rm --platform "$platform" -e BACKEND_BASE_URL=/api \ "$frontend_image" frontend-config-smoke docker run --rm --platform "$platform" -e BACKEND_BASE_URL= \