mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(vector): harden packaged migrations

Browse Source
This commit is contained in:
marcopan
2026-07-12 01:32:33 +02:00
parent c0d50e9b08
commit 3588a7749b
14 changed files with 337 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tests/l0/test_pgvector_store.py
+2 -2
View File
@@ -45,8 +45,8 @@ def vector_configs():
)
engine = create_engine(pg.get_connection_url())
with engine.begin() as connection:
connection.exec_driver_sql("CREATE EXTENSION vector")
connection.exec_driver_sql("CREATE SCHEMA vectors")
connection.exec_driver_sql("CREATE EXTENSION vector WITH SCHEMA vectors")
for table in ("schema_records", "evidence", "memory"):
connection.exec_driver_sql(f"""
CREATE TABLE vectors.{table} (
@@ -55,7 +55,7 @@ def vector_configs():
kind text NOT NULL,
content_hash text NOT NULL,
metadata jsonb NOT NULL,
embedding vector(2) NOT NULL,
embedding vectors.vector(2) NOT NULL,
indexed_at timestamptz NOT NULL DEFAULT now()
)
""")
harness/tests/l0/test_vector_migrations.py
+111 -13
View File
@@ -59,9 +59,9 @@ def test_schema_matches_direct_adapter_contract(database_url):
for table in tables:
assert {row.column_name for row in rows if row.table_name == table} == required
assert vector_types == [
("evidence", "vector(768)"),
("memory", "vector(768)"),
("schema_records", "vector(768)"),
("evidence", "vectors.vector(768)"),
("memory", "vectors.vector(768)"),
("schema_records", "vectors.vector(768)"),
]
@@ -119,7 +119,7 @@ def test_roles_have_runtime_privileges_only(database_url):
"INSERT INTO vectors.memory "
"(record_key, kind, content_hash, metadata, embedding) "
"VALUES ('reader-write', 'memory', 'x', '{}', "
"array_fill(0, ARRAY[768])::vector)"
"array_fill(0, ARRAY[768])::vectors.vector)"
)
)
@@ -128,7 +128,8 @@ def test_roles_have_runtime_privileges_only(database_url):
text(
"INSERT INTO vectors.memory "
"(record_key, kind, content_hash, metadata, embedding) "
"VALUES ('writer-ok', 'memory', 'x', '{}', array_fill(0, ARRAY[768])::vector)"
"VALUES ('writer-ok', 'memory', 'x', '{}', "
"array_fill(0, ARRAY[768])::vectors.vector)"
)
)
assert connection.execute(
@@ -176,15 +177,112 @@ def test_checksum_drift_is_reported_and_refused(database_url, tmp_path):
migrate(database_url, migrations)
def test_unknown_applied_version_is_downgrade_drift(database_url):
from tht.cli.vector_migrate_cmd import MigrationError, migrate, migration_status
migrate(database_url)
engine = create_engine(database_url)
with engine.begin() as connection:
connection.execute(
text(
"INSERT INTO public.tht_vector_migrations (version, name, checksum) "
"VALUES ('999', 'future', 'future-checksum'), "
"('future_x', 'future_named', 'future-checksum')"
)
)
try:
with pytest.raises(
MigrationError, match="absent from local manifest: 999, future_x"
):
migration_status(database_url)
with pytest.raises(
MigrationError, match="absent from local manifest: 999, future_x"
):
migrate(database_url)
finally:
with engine.begin() as connection:
connection.execute(
text(
"DELETE FROM public.tht_vector_migrations "
"WHERE version IN ('999', 'future_x')"
)
)
engine.dispose()
def test_migration_versions_sort_numerically_and_reject_numeric_duplicates(tmp_path):
from tht.cli.vector_migrate_cmd import MigrationError, _discover
migrations = tmp_path / "ordered"
migrations.mkdir()
(migrations / "10_tenth.sql").write_text("SELECT 10;\n")
(migrations / "2_second.sql").write_text("SELECT 2;\n")
assert [item.version for item in _discover(migrations)] == ["2", "10"]
(migrations / "02_duplicate.sql").write_text("SELECT 2;\n")
with pytest.raises(MigrationError, match="Duplicate migration version: 2"):
_discover(migrations)
def test_hostile_admin_search_path_cannot_shadow_migration_objects(database_url):
from tht.cli.vector_migrate_cmd import migrate
admin = create_engine(database_url, isolation_level="AUTOCOMMIT")
with admin.connect() as connection:
connection.exec_driver_sql("DROP DATABASE IF EXISTS vector_hostile")
connection.exec_driver_sql("CREATE DATABASE vector_hostile")
hostile_url = admin.url.set(database="vector_hostile")
hostile = create_engine(hostile_url)
try:
with hostile.begin() as connection:
connection.exec_driver_sql("CREATE SCHEMA shadow")
connection.exec_driver_sql(
"CREATE TABLE shadow.tht_vector_migrations "
"(version text, checksum text, poisoned boolean DEFAULT true)"
)
connection.exec_driver_sql("ALTER ROLE test SET search_path = shadow, public")
hostile.dispose()
migrate(hostile_url.render_as_string(hide_password=False))
verification = create_engine(hostile_url)
with verification.connect() as connection:
assert connection.execute(
text("SELECT count(*) FROM public.tht_vector_migrations")
).scalar_one() == 3
assert connection.execute(
text("SELECT count(*) FROM shadow.tht_vector_migrations")
).scalar_one() == 0
assert connection.execute(
text(
"SELECT format_type(a.atttypid, a.atttypmod) "
"FROM pg_catalog.pg_attribute a "
"WHERE a.attrelid = 'vectors.memory'::pg_catalog.regclass "
"AND a.attname = 'embedding'"
)
).scalar_one() == "vectors.vector(768)"
verification.dispose()
finally:
cleanup = create_engine(database_url, isolation_level="AUTOCOMMIT")
with cleanup.connect() as connection:
connection.exec_driver_sql("ALTER ROLE test RESET search_path")
connection.exec_driver_sql(
"SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity "
"WHERE datname = 'vector_hostile' AND pid <> pg_catalog.pg_backend_pid()"
)
connection.exec_driver_sql("DROP DATABASE IF EXISTS vector_hostile")
cleanup.dispose()
admin.dispose()
def test_failed_batch_rolls_back_schema_and_ledger(database_url, tmp_path):
from tht.cli.vector_migrate_cmd import MigrationError, migrate, migration_status
migrations = tmp_path / "failed"
migrations.mkdir()
(migrations / "101_first.sql").write_text("CREATE TABLE public.must_rollback (id int);\n")
(migrations / "102_broken.sql").write_text("THIS IS NOT SQL;\n")
migrations = _copy_migrations(tmp_path)
(migrations / "004_first.sql").write_text("CREATE TABLE public.must_rollback (id int);\n")
(migrations / "005_broken.sql").write_text("THIS IS NOT SQL;\n")
with pytest.raises(MigrationError, match="102_broken.sql"):
with pytest.raises(MigrationError, match="005_broken.sql"):
migrate(database_url, migrations)
engine = create_engine(database_url)
@@ -192,12 +290,12 @@ def test_failed_batch_rolls_back_schema_and_ledger(database_url, tmp_path):
assert connection.execute(text("SELECT to_regclass('public.must_rollback')")).scalar() is None
engine.dispose()
status = migration_status(database_url, migrations)
assert status.applied == ()
assert [item.version for item in status.pending] == ["101", "102"]
assert [item.version for item in status.applied] == ["001", "002", "003"]
assert [item.version for item in status.pending] == ["004", "005"]
def _copy_migrations(tmp_path: Path) -> Path:
source = Path(__file__).parents[2] / "migrations" / "vector"
source = Path(__file__).parents[2] / "tht" / "migrations" / "vector"
target = tmp_path / "migrations"
target.mkdir()
for migration in source.glob("*.sql"):
harness/tests/test_vector_migration_packaging.py
+59
View File
@@ -0,0 +1,59 @@
import os
import shutil
import subprocess
import sys
import zipfile
from pathlib import Path
def test_built_wheel_installs_vector_migrations_and_discovers_cli(tmp_path):
harness = Path(__file__).parents[1]
wheelhouse = tmp_path / "wheelhouse"
target = tmp_path / "site"
wheelhouse.mkdir()
uv = shutil.which("uv")
assert uv is not None, "uv is required to verify the production wheel"
build_env = {**os.environ, "UV_CACHE_DIR": str(tmp_path / "uv-cache")}
subprocess.run(
[
uv,
"build",
"--wheel",
"--out-dir",
str(wheelhouse),
str(harness),
],
check=True,
capture_output=True,
text=True,
env=build_env,
)
wheel = next(wheelhouse.glob("tht-*.whl"))
with zipfile.ZipFile(wheel) as archive:
names = set(archive.namelist())
assert "tht/migrations/vector/001_extensions.sql" in names
assert "tht/migrations/vector/003_roles.sql" in names
subprocess.run(
[sys.executable, "-m", "pip", "install", "--no-deps", "--target", str(target), wheel],
check=True,
capture_output=True,
text=True,
)
env = {**os.environ, "PYTHONPATH": str(target)}
probe = subprocess.run(
[
sys.executable,
"-c",
"from typer.testing import CliRunner; from tht.cli import app; "
"r=CliRunner().invoke(app, ['vector','migrate','--help']); "
"print(r.output); raise SystemExit(r.exit_code)",
],
env=env,
check=False,
capture_output=True,
text=True,
cwd=tmp_path,
)
assert probe.returncode == 0, probe.stderr + probe.stdout
assert "--status" in probe.stdout