fix(vector): harden packaged migrations
This commit is contained in:
@@ -50,3 +50,33 @@ No unresolved Task 2 correctness concern found. One deliberate contract choice i
|
||||
writer `INSERT` and `UPDATE` are table-level because the approved direct adapter health probe uses
|
||||
`has_table_privilege` for those authorities. Least privilege is retained by withholding broad
|
||||
`SELECT`, `DELETE`, DDL, ownership, and credentials.
|
||||
|
||||
## Review fix wave
|
||||
|
||||
The post-implementation review found four production-boundary gaps. They are fixed as follows:
|
||||
|
||||
- Migration SQL now ships inside the `tht` wheel (`tht/migrations/vector`) via explicit
|
||||
setuptools package-data and is discovered through `importlib.resources`, rather than relying on
|
||||
a source-checkout-relative directory.
|
||||
- Both status and apply reject ledger versions absent from the installed manifest, including
|
||||
nonnumeric future version labels. This treats a binary/database downgrade as drift instead of
|
||||
silently reporting a healthy state.
|
||||
- Migration files are ordered by parsed integer version; spellings such as `2` and `02` are
|
||||
rejected as duplicate versions.
|
||||
- Every migration transaction pins `search_path` locally to `pg_catalog, pg_temp`; catalog calls
|
||||
and the ledger are schema-qualified. pgvector is installed into the locked `vectors` schema,
|
||||
tables use `vectors.vector`, and `PgVectorStore` qualifies vector casts and the cosine operator.
|
||||
A hostile admin default path with a writable shadow schema cannot redirect migration objects.
|
||||
- The core image build asserts CLI discovery. Image verification now starts an ephemeral pgvector
|
||||
database, runs the installed image's migration command, and compares pristine apply/status JSON.
|
||||
|
||||
Additional verification after the fix wave:
|
||||
|
||||
- Focused migration, adapter, hostile-path, and wheel suite: `27 passed`.
|
||||
- Full harness: `477 passed, 5 deselected`.
|
||||
- Production core image build: passed, including build-time CLI discovery.
|
||||
- Core-image apply/status smoke against `pgvector/pgvector:pg16`: passed.
|
||||
- Changed production and test files: Ruff clean; `git diff --check` clean.
|
||||
- Full Ruff remains at the same 34 pre-existing unrelated test-file findings documented above.
|
||||
|
||||
No dependency changed, so the committed Python requirements lock did not require regeneration.
|
||||
|
||||
@@ -40,7 +40,8 @@ COPY docker/python-runtime/requirements.lock /app/docker/python-runtime/requirem
|
||||
RUN python -m venv /opt/venv \
|
||||
&& /opt/venv/bin/pip install --no-cache-dir --require-hashes \
|
||||
--requirement /app/docker/python-runtime/requirements.lock \
|
||||
&& /opt/venv/bin/pip install --no-cache-dir --no-deps --no-build-isolation /app/harness
|
||||
&& /opt/venv/bin/pip install --no-cache-dir --no-deps --no-build-isolation /app/harness \
|
||||
&& /opt/venv/bin/tht vector migrate --help >/dev/null
|
||||
|
||||
COPY --from=backend-build /src/backend/dist /app/backend/dist
|
||||
COPY --from=backend-build /src/backend/node_modules /app/backend/node_modules
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
CREATE EXTENSION IF NOT EXISTS vector;
|
||||
@@ -31,6 +31,9 @@ dev = [
|
||||
[tool.setuptools.packages.find]
|
||||
include = ["tht*"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
tht = ["migrations/vector/*.sql"]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
|
||||
|
||||
@@ -45,8 +45,8 @@ def vector_configs():
|
||||
)
|
||||
engine = create_engine(pg.get_connection_url())
|
||||
with engine.begin() as connection:
|
||||
connection.exec_driver_sql("CREATE EXTENSION vector")
|
||||
connection.exec_driver_sql("CREATE SCHEMA vectors")
|
||||
connection.exec_driver_sql("CREATE EXTENSION vector WITH SCHEMA vectors")
|
||||
for table in ("schema_records", "evidence", "memory"):
|
||||
connection.exec_driver_sql(f"""
|
||||
CREATE TABLE vectors.{table} (
|
||||
@@ -55,7 +55,7 @@ def vector_configs():
|
||||
kind text NOT NULL,
|
||||
content_hash text NOT NULL,
|
||||
metadata jsonb NOT NULL,
|
||||
embedding vector(2) NOT NULL,
|
||||
embedding vectors.vector(2) NOT NULL,
|
||||
indexed_at timestamptz NOT NULL DEFAULT now()
|
||||
)
|
||||
""")
|
||||
|
||||
@@ -59,9 +59,9 @@ def test_schema_matches_direct_adapter_contract(database_url):
|
||||
for table in tables:
|
||||
assert {row.column_name for row in rows if row.table_name == table} == required
|
||||
assert vector_types == [
|
||||
("evidence", "vector(768)"),
|
||||
("memory", "vector(768)"),
|
||||
("schema_records", "vector(768)"),
|
||||
("evidence", "vectors.vector(768)"),
|
||||
("memory", "vectors.vector(768)"),
|
||||
("schema_records", "vectors.vector(768)"),
|
||||
]
|
||||
|
||||
|
||||
@@ -119,7 +119,7 @@ def test_roles_have_runtime_privileges_only(database_url):
|
||||
"INSERT INTO vectors.memory "
|
||||
"(record_key, kind, content_hash, metadata, embedding) "
|
||||
"VALUES ('reader-write', 'memory', 'x', '{}', "
|
||||
"array_fill(0, ARRAY[768])::vector)"
|
||||
"array_fill(0, ARRAY[768])::vectors.vector)"
|
||||
)
|
||||
)
|
||||
|
||||
@@ -128,7 +128,8 @@ def test_roles_have_runtime_privileges_only(database_url):
|
||||
text(
|
||||
"INSERT INTO vectors.memory "
|
||||
"(record_key, kind, content_hash, metadata, embedding) "
|
||||
"VALUES ('writer-ok', 'memory', 'x', '{}', array_fill(0, ARRAY[768])::vector)"
|
||||
"VALUES ('writer-ok', 'memory', 'x', '{}', "
|
||||
"array_fill(0, ARRAY[768])::vectors.vector)"
|
||||
)
|
||||
)
|
||||
assert connection.execute(
|
||||
@@ -176,15 +177,112 @@ def test_checksum_drift_is_reported_and_refused(database_url, tmp_path):
|
||||
migrate(database_url, migrations)
|
||||
|
||||
|
||||
def test_unknown_applied_version_is_downgrade_drift(database_url):
|
||||
from tht.cli.vector_migrate_cmd import MigrationError, migrate, migration_status
|
||||
|
||||
migrate(database_url)
|
||||
engine = create_engine(database_url)
|
||||
with engine.begin() as connection:
|
||||
connection.execute(
|
||||
text(
|
||||
"INSERT INTO public.tht_vector_migrations (version, name, checksum) "
|
||||
"VALUES ('999', 'future', 'future-checksum'), "
|
||||
"('future_x', 'future_named', 'future-checksum')"
|
||||
)
|
||||
)
|
||||
try:
|
||||
with pytest.raises(
|
||||
MigrationError, match="absent from local manifest: 999, future_x"
|
||||
):
|
||||
migration_status(database_url)
|
||||
with pytest.raises(
|
||||
MigrationError, match="absent from local manifest: 999, future_x"
|
||||
):
|
||||
migrate(database_url)
|
||||
finally:
|
||||
with engine.begin() as connection:
|
||||
connection.execute(
|
||||
text(
|
||||
"DELETE FROM public.tht_vector_migrations "
|
||||
"WHERE version IN ('999', 'future_x')"
|
||||
)
|
||||
)
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def test_migration_versions_sort_numerically_and_reject_numeric_duplicates(tmp_path):
|
||||
from tht.cli.vector_migrate_cmd import MigrationError, _discover
|
||||
|
||||
migrations = tmp_path / "ordered"
|
||||
migrations.mkdir()
|
||||
(migrations / "10_tenth.sql").write_text("SELECT 10;\n")
|
||||
(migrations / "2_second.sql").write_text("SELECT 2;\n")
|
||||
assert [item.version for item in _discover(migrations)] == ["2", "10"]
|
||||
|
||||
(migrations / "02_duplicate.sql").write_text("SELECT 2;\n")
|
||||
with pytest.raises(MigrationError, match="Duplicate migration version: 2"):
|
||||
_discover(migrations)
|
||||
|
||||
|
||||
def test_hostile_admin_search_path_cannot_shadow_migration_objects(database_url):
|
||||
from tht.cli.vector_migrate_cmd import migrate
|
||||
|
||||
admin = create_engine(database_url, isolation_level="AUTOCOMMIT")
|
||||
with admin.connect() as connection:
|
||||
connection.exec_driver_sql("DROP DATABASE IF EXISTS vector_hostile")
|
||||
connection.exec_driver_sql("CREATE DATABASE vector_hostile")
|
||||
hostile_url = admin.url.set(database="vector_hostile")
|
||||
hostile = create_engine(hostile_url)
|
||||
try:
|
||||
with hostile.begin() as connection:
|
||||
connection.exec_driver_sql("CREATE SCHEMA shadow")
|
||||
connection.exec_driver_sql(
|
||||
"CREATE TABLE shadow.tht_vector_migrations "
|
||||
"(version text, checksum text, poisoned boolean DEFAULT true)"
|
||||
)
|
||||
connection.exec_driver_sql("ALTER ROLE test SET search_path = shadow, public")
|
||||
hostile.dispose()
|
||||
|
||||
migrate(hostile_url.render_as_string(hide_password=False))
|
||||
|
||||
verification = create_engine(hostile_url)
|
||||
with verification.connect() as connection:
|
||||
assert connection.execute(
|
||||
text("SELECT count(*) FROM public.tht_vector_migrations")
|
||||
).scalar_one() == 3
|
||||
assert connection.execute(
|
||||
text("SELECT count(*) FROM shadow.tht_vector_migrations")
|
||||
).scalar_one() == 0
|
||||
assert connection.execute(
|
||||
text(
|
||||
"SELECT format_type(a.atttypid, a.atttypmod) "
|
||||
"FROM pg_catalog.pg_attribute a "
|
||||
"WHERE a.attrelid = 'vectors.memory'::pg_catalog.regclass "
|
||||
"AND a.attname = 'embedding'"
|
||||
)
|
||||
).scalar_one() == "vectors.vector(768)"
|
||||
verification.dispose()
|
||||
finally:
|
||||
cleanup = create_engine(database_url, isolation_level="AUTOCOMMIT")
|
||||
with cleanup.connect() as connection:
|
||||
connection.exec_driver_sql("ALTER ROLE test RESET search_path")
|
||||
connection.exec_driver_sql(
|
||||
"SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity "
|
||||
"WHERE datname = 'vector_hostile' AND pid <> pg_catalog.pg_backend_pid()"
|
||||
)
|
||||
connection.exec_driver_sql("DROP DATABASE IF EXISTS vector_hostile")
|
||||
cleanup.dispose()
|
||||
admin.dispose()
|
||||
|
||||
|
||||
def test_failed_batch_rolls_back_schema_and_ledger(database_url, tmp_path):
|
||||
from tht.cli.vector_migrate_cmd import MigrationError, migrate, migration_status
|
||||
|
||||
migrations = tmp_path / "failed"
|
||||
migrations.mkdir()
|
||||
(migrations / "101_first.sql").write_text("CREATE TABLE public.must_rollback (id int);\n")
|
||||
(migrations / "102_broken.sql").write_text("THIS IS NOT SQL;\n")
|
||||
migrations = _copy_migrations(tmp_path)
|
||||
(migrations / "004_first.sql").write_text("CREATE TABLE public.must_rollback (id int);\n")
|
||||
(migrations / "005_broken.sql").write_text("THIS IS NOT SQL;\n")
|
||||
|
||||
with pytest.raises(MigrationError, match="102_broken.sql"):
|
||||
with pytest.raises(MigrationError, match="005_broken.sql"):
|
||||
migrate(database_url, migrations)
|
||||
|
||||
engine = create_engine(database_url)
|
||||
@@ -192,12 +290,12 @@ def test_failed_batch_rolls_back_schema_and_ledger(database_url, tmp_path):
|
||||
assert connection.execute(text("SELECT to_regclass('public.must_rollback')")).scalar() is None
|
||||
engine.dispose()
|
||||
status = migration_status(database_url, migrations)
|
||||
assert status.applied == ()
|
||||
assert [item.version for item in status.pending] == ["101", "102"]
|
||||
assert [item.version for item in status.applied] == ["001", "002", "003"]
|
||||
assert [item.version for item in status.pending] == ["004", "005"]
|
||||
|
||||
|
||||
def _copy_migrations(tmp_path: Path) -> Path:
|
||||
source = Path(__file__).parents[2] / "migrations" / "vector"
|
||||
source = Path(__file__).parents[2] / "tht" / "migrations" / "vector"
|
||||
target = tmp_path / "migrations"
|
||||
target.mkdir()
|
||||
for migration in source.glob("*.sql"):
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_built_wheel_installs_vector_migrations_and_discovers_cli(tmp_path):
|
||||
harness = Path(__file__).parents[1]
|
||||
wheelhouse = tmp_path / "wheelhouse"
|
||||
target = tmp_path / "site"
|
||||
wheelhouse.mkdir()
|
||||
uv = shutil.which("uv")
|
||||
assert uv is not None, "uv is required to verify the production wheel"
|
||||
build_env = {**os.environ, "UV_CACHE_DIR": str(tmp_path / "uv-cache")}
|
||||
subprocess.run(
|
||||
[
|
||||
uv,
|
||||
"build",
|
||||
"--wheel",
|
||||
"--out-dir",
|
||||
str(wheelhouse),
|
||||
str(harness),
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=build_env,
|
||||
)
|
||||
wheel = next(wheelhouse.glob("tht-*.whl"))
|
||||
with zipfile.ZipFile(wheel) as archive:
|
||||
names = set(archive.namelist())
|
||||
assert "tht/migrations/vector/001_extensions.sql" in names
|
||||
assert "tht/migrations/vector/003_roles.sql" in names
|
||||
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "pip", "install", "--no-deps", "--target", str(target), wheel],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
env = {**os.environ, "PYTHONPATH": str(target)}
|
||||
probe = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"from typer.testing import CliRunner; from tht.cli import app; "
|
||||
"r=CliRunner().invoke(app, ['vector','migrate','--help']); "
|
||||
"print(r.output); raise SystemExit(r.exit_code)",
|
||||
],
|
||||
env=env,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd=tmp_path,
|
||||
)
|
||||
assert probe.returncode == 0, probe.stderr + probe.stdout
|
||||
assert "--status" in probe.stdout
|
||||
@@ -27,7 +27,7 @@ COLLECTION_KINDS = {
|
||||
}
|
||||
ALLOWED_COLLECTIONS = frozenset(COLLECTION_KINDS)
|
||||
ALLOWED_KINDS = frozenset().union(*COLLECTION_KINDS.values())
|
||||
_VECTOR_DIMENSION = re.compile(r"^vector\((\d+)\)$")
|
||||
_VECTOR_DIMENSION = re.compile(r"^(?:[a-z_][a-z0-9_]*\.)?vector\((\d+)\)$")
|
||||
|
||||
|
||||
def _collection(schema: str, name: str) -> sql.Identifier:
|
||||
@@ -40,6 +40,14 @@ def _vector_literal(values: list[float]) -> str:
|
||||
return "[" + ",".join(str(float(value)) for value in values) + "]"
|
||||
|
||||
|
||||
def _vector_type(schema: str) -> sql.Identifier:
|
||||
return sql.Identifier(schema, "vector")
|
||||
|
||||
|
||||
def _cosine_operator(schema: str) -> sql.Composed:
|
||||
return sql.SQL("OPERATOR({}.<=>)").format(sql.Identifier(schema))
|
||||
|
||||
|
||||
def _validate_collection_kinds(collection: str, kinds: list[str]) -> None:
|
||||
invalid = set(kinds) - COLLECTION_KINDS[collection]
|
||||
if invalid:
|
||||
@@ -248,9 +256,16 @@ class PgVectorStore:
|
||||
continue
|
||||
where = sql.SQL(" WHERE kind = ANY(%s)") if collection_kinds else sql.SQL("")
|
||||
query = sql.SQL(
|
||||
"SELECT metadata, 1 - (embedding <=> %s::vector) AS similarity "
|
||||
"FROM {}{} ORDER BY embedding <=> %s::vector LIMIT %s"
|
||||
).format(table, where)
|
||||
"SELECT metadata, 1 - (embedding {} %s::{}) AS similarity "
|
||||
"FROM {}{} ORDER BY embedding {} %s::{} LIMIT %s"
|
||||
).format(
|
||||
_cosine_operator(self._schema),
|
||||
_vector_type(self._schema),
|
||||
table,
|
||||
where,
|
||||
_cosine_operator(self._schema),
|
||||
_vector_type(self._schema),
|
||||
)
|
||||
params = [_vector_literal(embedding)]
|
||||
if collection_kinds:
|
||||
params.append(collection_kinds)
|
||||
@@ -295,13 +310,13 @@ class PgVectorStore:
|
||||
raise VectorStoreError("Embedding dimension does not match configured dimension")
|
||||
insert = sql.SQL(
|
||||
"INSERT INTO {} (record_key, kind, content_hash, metadata, embedding) "
|
||||
"VALUES (%s, %s, %s, %s::jsonb, %s::vector) "
|
||||
"VALUES (%s, %s, %s, %s::jsonb, %s::{}) "
|
||||
"ON CONFLICT (record_key) DO NOTHING"
|
||||
).format(table)
|
||||
).format(table, _vector_type(self._schema))
|
||||
update = sql.SQL(
|
||||
"UPDATE {} SET kind = %s, content_hash = %s, metadata = %s::jsonb, "
|
||||
"embedding = %s::vector, indexed_at = now() WHERE record_key = %s"
|
||||
).format(table)
|
||||
"embedding = %s::{}, indexed_at = pg_catalog.now() WHERE record_key = %s"
|
||||
).format(table, _vector_type(self._schema))
|
||||
raw = engine.raw_connection()
|
||||
try:
|
||||
with raw.cursor() as cursor:
|
||||
|
||||
@@ -6,6 +6,8 @@ import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from importlib.resources import files
|
||||
from importlib.resources.abc import Traversable
|
||||
from pathlib import Path
|
||||
|
||||
import typer
|
||||
@@ -14,7 +16,7 @@ from sqlalchemy.exc import SQLAlchemyError
|
||||
|
||||
from tht.cli.vector_cmd import vector_app
|
||||
|
||||
MIGRATIONS_DIR = Path(__file__).parents[2] / "migrations" / "vector"
|
||||
MIGRATIONS_DIR = files("tht").joinpath("migrations", "vector")
|
||||
_MIGRATION_NAME = re.compile(r"^(?P<version>\d+)_(?P<name>[a-z0-9_]+)\.sql$")
|
||||
_LOCK_KEY = 7_304_708_654_221_909_028
|
||||
|
||||
@@ -27,7 +29,7 @@ class MigrationError(RuntimeError):
|
||||
class Migration:
|
||||
version: str
|
||||
name: str
|
||||
path: Path
|
||||
path: Traversable
|
||||
checksum: str
|
||||
|
||||
|
||||
@@ -38,32 +40,44 @@ class MigrationStatus:
|
||||
drifted: tuple[Migration, ...]
|
||||
|
||||
|
||||
def _discover(directory: Path) -> tuple[Migration, ...]:
|
||||
def _migration_source(directory: Traversable | Path | str) -> Traversable:
|
||||
return Path(directory) if isinstance(directory, (str, Path)) else directory
|
||||
|
||||
|
||||
def _discover(directory: Traversable | Path | str) -> tuple[Migration, ...]:
|
||||
source = _migration_source(directory)
|
||||
migrations = []
|
||||
seen_versions: set[str] = set()
|
||||
for path in sorted(directory.glob("*.sql")):
|
||||
seen_versions: set[int] = set()
|
||||
paths = [path for path in source.iterdir() if path.name.endswith(".sql")]
|
||||
parsed = []
|
||||
for path in paths:
|
||||
match = _MIGRATION_NAME.fullmatch(path.name)
|
||||
if match is None:
|
||||
raise MigrationError(f"Invalid migration filename: {path.name}")
|
||||
version = match.group("version")
|
||||
if version in seen_versions:
|
||||
raise MigrationError(f"Duplicate migration version: {version}")
|
||||
seen_versions.add(version)
|
||||
numeric_version = int(version)
|
||||
if numeric_version in seen_versions:
|
||||
raise MigrationError(f"Duplicate migration version: {numeric_version}")
|
||||
seen_versions.add(numeric_version)
|
||||
parsed.append((numeric_version, version, match.group("name"), path))
|
||||
for _, version, name, path in sorted(parsed, key=lambda item: item[0]):
|
||||
migrations.append(
|
||||
Migration(
|
||||
version=version,
|
||||
name=match.group("name"),
|
||||
name=name,
|
||||
path=path,
|
||||
checksum=hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
)
|
||||
)
|
||||
if not migrations:
|
||||
raise MigrationError(f"No migrations found in {directory}")
|
||||
raise MigrationError(f"No migrations found in {source}")
|
||||
return tuple(migrations)
|
||||
|
||||
|
||||
def _applied(connection) -> dict[str, str]:
|
||||
exists = connection.execute(text("SELECT to_regclass('public.tht_vector_migrations')")).scalar()
|
||||
exists = connection.execute(
|
||||
text("SELECT pg_catalog.to_regclass('public.tht_vector_migrations')")
|
||||
).scalar()
|
||||
if exists is None:
|
||||
return {}
|
||||
return dict(
|
||||
@@ -73,16 +87,32 @@ def _applied(connection) -> dict[str, str]:
|
||||
)
|
||||
|
||||
|
||||
def _reject_unknown_versions(
|
||||
migrations: tuple[Migration, ...], applied_checksums: dict[str, str]
|
||||
) -> None:
|
||||
local_versions = {migration.version for migration in migrations}
|
||||
unknown = sorted(
|
||||
set(applied_checksums) - local_versions,
|
||||
key=lambda version: (0, int(version)) if version.isdigit() else (1, version),
|
||||
)
|
||||
if unknown:
|
||||
raise MigrationError(
|
||||
"Database migration versions absent from local manifest: " + ", ".join(unknown)
|
||||
)
|
||||
|
||||
|
||||
def migration_status(
|
||||
database_url: str, migrations_dir: Path | str = MIGRATIONS_DIR
|
||||
database_url: str, migrations_dir: Traversable | Path | str = MIGRATIONS_DIR
|
||||
) -> MigrationStatus:
|
||||
migrations = _discover(Path(migrations_dir))
|
||||
migrations = _discover(migrations_dir)
|
||||
engine = create_engine(database_url)
|
||||
try:
|
||||
with engine.connect() as connection:
|
||||
connection.exec_driver_sql("SET LOCAL search_path = pg_catalog, pg_temp")
|
||||
applied_checksums = _applied(connection)
|
||||
finally:
|
||||
engine.dispose()
|
||||
_reject_unknown_versions(migrations, applied_checksums)
|
||||
applied = tuple(
|
||||
migration
|
||||
for migration in migrations
|
||||
@@ -100,25 +130,31 @@ def migration_status(
|
||||
return MigrationStatus(applied=applied, pending=pending, drifted=drifted)
|
||||
|
||||
|
||||
def migrate(database_url: str, migrations_dir: Path | str = MIGRATIONS_DIR) -> MigrationStatus:
|
||||
migrations = _discover(Path(migrations_dir))
|
||||
def migrate(
|
||||
database_url: str, migrations_dir: Traversable | Path | str = MIGRATIONS_DIR
|
||||
) -> MigrationStatus:
|
||||
migrations = _discover(migrations_dir)
|
||||
engine = create_engine(database_url)
|
||||
current: Migration | None = None
|
||||
try:
|
||||
with engine.begin() as connection:
|
||||
connection.execute(text("SELECT pg_advisory_xact_lock(:key)"), {"key": _LOCK_KEY})
|
||||
connection.exec_driver_sql("SET LOCAL search_path = pg_catalog, pg_temp")
|
||||
connection.execute(
|
||||
text("SELECT pg_catalog.pg_advisory_xact_lock(:key)"), {"key": _LOCK_KEY}
|
||||
)
|
||||
connection.exec_driver_sql(
|
||||
"""CREATE TABLE IF NOT EXISTS public.tht_vector_migrations (
|
||||
version text PRIMARY KEY,
|
||||
name text NOT NULL,
|
||||
checksum text NOT NULL,
|
||||
applied_at timestamptz NOT NULL DEFAULT now()
|
||||
applied_at timestamptz NOT NULL DEFAULT pg_catalog.now()
|
||||
)"""
|
||||
)
|
||||
connection.exec_driver_sql(
|
||||
"REVOKE ALL ON public.tht_vector_migrations FROM PUBLIC"
|
||||
)
|
||||
applied_checksums = _applied(connection)
|
||||
_reject_unknown_versions(migrations, applied_checksums)
|
||||
drifted = [
|
||||
item
|
||||
for item in migrations
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
CREATE SCHEMA IF NOT EXISTS vectors;
|
||||
REVOKE ALL ON SCHEMA vectors FROM PUBLIC;
|
||||
CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA vectors;
|
||||
+6
-10
@@ -1,15 +1,11 @@
|
||||
CREATE SCHEMA IF NOT EXISTS vectors;
|
||||
|
||||
REVOKE ALL ON SCHEMA vectors FROM PUBLIC;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS vectors.schema_records (
|
||||
id bigserial PRIMARY KEY,
|
||||
record_key text UNIQUE NOT NULL,
|
||||
kind text NOT NULL,
|
||||
content_hash text NOT NULL,
|
||||
metadata jsonb NOT NULL,
|
||||
embedding vector(768) NOT NULL,
|
||||
indexed_at timestamptz NOT NULL DEFAULT now()
|
||||
embedding vectors.vector(768) NOT NULL,
|
||||
indexed_at timestamptz NOT NULL DEFAULT pg_catalog.now()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS vectors.evidence (
|
||||
@@ -18,8 +14,8 @@ CREATE TABLE IF NOT EXISTS vectors.evidence (
|
||||
kind text NOT NULL,
|
||||
content_hash text NOT NULL,
|
||||
metadata jsonb NOT NULL,
|
||||
embedding vector(768) NOT NULL,
|
||||
indexed_at timestamptz NOT NULL DEFAULT now()
|
||||
embedding vectors.vector(768) NOT NULL,
|
||||
indexed_at timestamptz NOT NULL DEFAULT pg_catalog.now()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS vectors.memory (
|
||||
@@ -28,8 +24,8 @@ CREATE TABLE IF NOT EXISTS vectors.memory (
|
||||
kind text NOT NULL,
|
||||
content_hash text NOT NULL,
|
||||
metadata jsonb NOT NULL,
|
||||
embedding vector(768) NOT NULL,
|
||||
indexed_at timestamptz NOT NULL DEFAULT now()
|
||||
embedding vectors.vector(768) NOT NULL,
|
||||
indexed_at timestamptz NOT NULL DEFAULT pg_catalog.now()
|
||||
);
|
||||
|
||||
REVOKE ALL ON ALL TABLES IN SCHEMA vectors FROM PUBLIC;
|
||||
+2
-2
@@ -1,9 +1,9 @@
|
||||
DO $roles$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'vector_reader') THEN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'vector_reader') THEN
|
||||
CREATE ROLE vector_reader NOLOGIN;
|
||||
END IF;
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'vector_writer') THEN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'vector_writer') THEN
|
||||
CREATE ROLE vector_writer NOLOGIN;
|
||||
END IF;
|
||||
END
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
image=${1:?usage: test-vector-migration-image.sh IMAGE [PLATFORM]}
|
||||
platform=${2:-${PLATFORM:-linux/arm64}}
|
||||
slug=$$
|
||||
network="thoth-vector-migration-$slug"
|
||||
database="thoth-vector-db-$slug"
|
||||
|
||||
cleanup() {
|
||||
docker rm --force "$database" >/dev/null 2>&1 || true
|
||||
docker network rm "$network" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
docker network create "$network" >/dev/null
|
||||
docker run --detach --rm --platform "$platform" --name "$database" --network "$network" \
|
||||
-e POSTGRES_DB=thoth -e POSTGRES_USER=thoth_admin -e POSTGRES_PASSWORD=test-only \
|
||||
pgvector/pgvector:pg16 >/dev/null
|
||||
|
||||
attempt=0
|
||||
until docker exec "$database" pg_isready -U thoth_admin -d thoth >/dev/null 2>&1; do
|
||||
attempt=$((attempt + 1))
|
||||
if [ "$attempt" -ge 30 ]; then
|
||||
echo "pgvector test database did not become ready" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
database_url="postgresql+psycopg2://thoth_admin:test-only@$database:5432/thoth"
|
||||
applied=$(docker run --rm --platform "$platform" --network "$network" \
|
||||
--entrypoint /opt/venv/bin/tht -e THT_VECTOR_ADMIN_URL="$database_url" \
|
||||
"$image" vector migrate --json)
|
||||
status=$(docker run --rm --platform "$platform" --network "$network" \
|
||||
--entrypoint /opt/venv/bin/tht -e THT_VECTOR_ADMIN_URL="$database_url" \
|
||||
"$image" vector migrate --status --json)
|
||||
|
||||
expected='{"applied": ["001", "002", "003"], "drifted": [], "pending": []}'
|
||||
test "$applied" = "$expected"
|
||||
test "$status" = "$expected"
|
||||
echo "core image vector migration discovery/status smoke passed"
|
||||
@@ -18,6 +18,7 @@ docker buildx build --platform "$platform" --load \
|
||||
|
||||
docker run --rm --platform "$platform" --entrypoint /app/docker/smoke/core-smoke.sh \
|
||||
"$core_image"
|
||||
./scripts/test-vector-migration-image.sh "$core_image" "$platform"
|
||||
docker run --rm --platform "$platform" -e BACKEND_BASE_URL=/api \
|
||||
"$frontend_image" frontend-config-smoke
|
||||
docker run --rm --platform "$platform" -e BACKEND_BASE_URL= \
|
||||
|
||||
Reference in New Issue
Block a user