mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

test(workflow): complete observable baseline (#21)

Browse Source
This commit is contained in:
marcopan
2026-08-24 00:40:11 +02:00
parent dc9726cb35
commit d15bb59c3d
4 changed files with 626 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/.pi/extensions/gate/__tests__/gate_workflow_observable_contract.test.js
+137 -159
View File
@@ -42,23 +42,45 @@ const PHASE_META = JSON.stringify({
function useShell({ phase, preview = [], fail = () => null }) {
const calls = [];
shell.current = (_file, args, options = {}) => {
const command = args.join(" ");
calls.push({ command, input: options.input });
const failure = fail(command);
calls.push({ args: [...args], input: options.input });
const failure = fail(args);
if (failure) {
const error = new Error(failure.stderr);
error.status = failure.status;
error.stderr = failure.stderr;
throw error;
}
if (command === "phase meta --json") return PHASE_META;
if (command.startsWith("phase show --session ")) return `Fase corrente: ${phase}\n`;
if (command.startsWith("memory promote --session ")) return JSON.stringify(preview);
if (sameArgs(args, ["phase", "meta", "--json"])) return PHASE_META;
if (startsWithArgs(args, ["phase", "show", "--session"])) return `Fase corrente: ${phase}\n`;
if (startsWithArgs(args, ["memory", "promote", "--session"])) return JSON.stringify(preview);
return "";
};
return calls;
}
function sameArgs(actual, expected) {
return actual.length === expected.length && actual.every((value, index) => value === expected[index]);
}
function startsWithArgs(actual, prefix) {
return prefix.every((value, index) => actual[index] === value);
}
function mutationArgs(calls, prefixes) {
return calls
.filter(({ args }) => prefixes.some((prefix) => startsWithArgs(args, prefix)))
.map(({ args }) => args);
}
function memoryPromotionMutationArgs(calls) {
return mutationArgs(calls, [
["memory", "save-one"],
["decision", "add"],
["phase", "advance"],
["session", "finalize"],
]);
}
async function setupGate(shellOptions) {
const calls = useShell(shellOptions);
const runtime = createFakePi();
@@ -83,65 +105,27 @@ test("the Pi gate exposes the approved tool names and parameter boundaries", ()
const publicContract = [...tools].map(([name, { def }]) => ({
name,
required: def.parameters.required ?? [],
properties: Object.keys(def.parameters.properties),
parameters: stripDescriptions(def.parameters),
}));
assert.deepEqual(publicContract, [
{
name: "reviewer_select",
required: ["session", "title", "options"],
properties: ["session", "title", "options", "intro", "advance"],
},
{
name: "reviewer_datamart",
required: ["session"],
properties: ["session"],
},
{
name: "reviewer_decide",
required: ["session", "title", "options"],
properties: ["session", "title", "options", "allow_empty", "advance"],
},
{
name: "reviewer_schema_linking",
required: ["session", "title", "tables"],
properties: ["session", "title", "tables", "advance"],
},
{
name: "reviewer_confirm",
required: ["session", "kind", "title", "artifact"],
properties: ["session", "kind", "title", "artifact", "names"],
},
{
name: "reviewer_memory_promote",
required: ["session"],
properties: ["session"],
},
{
name: "rewrite_question",
required: ["session", "question"],
properties: ["session", "question", "assumptions"],
},
{
name: "write_schema_linking",
required: ["session", "schema_linking"],
properties: ["session", "schema_linking"],
},
{
name: "write_cte_sql",
required: ["session", "name", "sql"],
properties: ["session", "name", "sql"],
},
{
name: "write_final_sql",
required: ["session", "sql"],
properties: ["session", "sql"],
},
]);
const expected = JSON.parse(fs.readFileSync(
path.join(__dirname, "golden", "pi_tool_schemas.json"),
"utf8",
));
assert.deepEqual(publicContract, expected);
});
test("the injected skill and workflow definition stay byte-identical during extraction", () => {
function stripDescriptions(value, insideProperties = false) {
if (Array.isArray(value)) return value.map((item) => stripDescriptions(item));
if (value && typeof value === "object") {
return Object.fromEntries(Object.entries(value)
.filter(([key]) => insideProperties || key !== "description")
.map(([key, item]) => [key, stripDescriptions(item, key === "properties")]));
}
return value;
}
test("the injected skill stays byte-identical during extraction", () => {
const harnessRoot = path.resolve(__dirname, "..", "..", "..", "..");
const digest = (relativePath) => crypto
.createHash("sha256")
@@ -152,10 +136,6 @@ test("the injected skill and workflow definition stay byte-identical during extr
digest(path.join(".pi", "skills", "tht-sessione", "SKILL.md")),
"626a794071c095a4f20fffabb3bab901f05c101590adbdc58e45adfae56f3219",
);
assert.equal(
digest("workflow.yaml"),
"a0604c3dfac7960dcfbf4a584383a602ee953a9fffac97a469281ceef1284d38",
);
});
test("F1 persists a concrete clarification directly from the select widget", async () => {
@@ -214,13 +194,13 @@ test("F1 persists a concrete clarification directly from the select widget", asy
reserved: ["back", "exit", "other"],
},
);
assert.ok(calls.some(({ command }) => command === [
"decision add --session s1",
"--type concept_clarified",
"--subject ablazione",
"--detail procedura clinica",
"--rationale scelta dal reviewer",
].join(" ")));
assert.ok(calls.some(({ args }) => sameArgs(args, [
"decision", "add", "--session", "s1",
"--type", "concept_clarified",
"--subject", "ablazione",
"--detail", "procedura clinica",
"--rationale", "scelta dal reviewer",
])));
assert.match(result.content[0].text, /Decisione registrata \(concept_clarified\)/);
});
@@ -240,7 +220,7 @@ test("F1 ask-only and control choices do not mutate the ledger", async () => {
ctx,
);
assert.equal(calls.some(({ command }) => command.startsWith("decision add ")), false);
assert.equal(calls.some(({ args }) => startsWithArgs(args, ["decision", "add"])), false);
assert.match(result.content[0].text, /Scelta del reviewer: Chiedi un dettaglio/);
});
@@ -269,13 +249,13 @@ test("F1 persists an explicitly accepted open ambiguity", async () => {
ctx,
);
assert.equal(calls.some(({ command }) => command === [
"decision add --session s1",
"--type ambiguity_open",
"--subject termine clinico",
"--detail nessuna definizione conclusiva",
"--rationale rischio accettato dal reviewer",
].join(" ")), true);
assert.equal(calls.some(({ args }) => sameArgs(args, [
"decision", "add", "--session", "s1",
"--type", "ambiguity_open",
"--subject", "termine clinico",
"--detail", "nessuna definizione conclusiva",
"--rationale", "rischio accettato dal reviewer",
])), true);
assert.match(result.content[0].text, /Decisione registrata \(ambiguity_open\)/);
});
@@ -297,7 +277,7 @@ const MEMORY_OPTIONS = [
test("F2 applies selected Memory content and keeps the substantive phase open", async () => {
const { ctx, tools, calls } = await setupGate({
phase: 2,
fail: (command) => command.startsWith("phase advance --auto")
fail: (args) => startsWithArgs(args, ["phase", "advance", "--auto"])
? { status: 6, stderr: "phase not auto-eligible" }
: null,
});
@@ -322,21 +302,15 @@ test("F2 applies selected Memory content and keeps the substantive phase open",
{
phase: descriptor.phase,
widget: descriptor.widget,
title: descriptor.title,
allowEmpty: descriptor.allow_empty,
selected: descriptor.selected,
selectionLabel: descriptor.selection_label,
confirmLabel: descriptor.confirm_label,
options: descriptor.options,
},
{
phase: "F2",
widget: "multiselect",
title: "Seleziona le memory da applicare alla domanda",
allowEmpty: true,
selected: ["mem-0042"],
selectionLabel: "memory da applicare",
confirmLabel: "Applica le memory selezionate",
options: [
{
id: "mem-0042",
@@ -349,13 +323,17 @@ test("F2 applies selected Memory content and keeps the substantive phase open",
],
},
);
const mutations = calls
.map(({ command }) => command)
.filter((command) => command.startsWith("decision ") || command.startsWith("phase advance"));
assert.equal(typeof descriptor.title, "string");
assert.equal(typeof descriptor.selection_label, "string");
assert.equal(typeof descriptor.confirm_label, "string");
const mutations = mutationArgs(calls, [["decision"], ["phase", "advance"]]);
assert.deepEqual(mutations, [
"decision add --session s1 --type concept_clarified --subject paziente attivo " +
"--detail flag_attivo = TRUE --rationale Riusa mem-0042 per la stessa definizione",
"phase advance --auto --session s1",
[
"decision", "add", "--session", "s1", "--type", "concept_clarified",
"--subject", "paziente attivo", "--detail", "flag_attivo = TRUE",
"--rationale", "Riusa mem-0042 per la stessa definizione",
],
["phase", "advance", "--auto", "--session", "s1"],
]);
assert.match(result.content[0].text, /La fase resta aperta/);
});
@@ -378,8 +356,10 @@ test("F2 accepts an empty selection without recording a rejection", async () =>
ctx,
);
assert.equal(calls.some(({ command }) => command.startsWith("decision add ")), false);
assert.equal(calls.some(({ command }) => command === "phase advance --auto --session s1"), true);
assert.equal(calls.some(({ args }) => startsWithArgs(args, ["decision", "add"])), false);
assert.equal(calls.some(({ args }) => sameArgs(
args, ["phase", "advance", "--auto", "--session", "s1"],
)), true);
assert.match(result.content[0].text, /Nessuna decisione registrata.*Fase avanzata/s);
});
@@ -401,19 +381,20 @@ test("F2 with no Memory candidates notifies once and advances without an empty w
ctx,
);
assert.deepEqual(ctx.notifications, [{
message: "Nessuna memory riutilizzabile per questa domanda — passo alla fase successiva.",
level: "info",
}]);
assert.equal(calls.some(({ command }) => command.startsWith("decision add ")), false);
assert.equal(calls.some(({ command }) => command === "phase advance --auto --session s1"), true);
assert.equal(ctx.notifications.length, 1);
assert.equal(ctx.notifications[0].level, "info");
assert.equal(typeof ctx.notifications[0].message, "string");
assert.equal(calls.some(({ args }) => startsWithArgs(args, ["decision", "add"])), false);
assert.equal(calls.some(({ args }) => sameArgs(
args, ["phase", "advance", "--auto", "--session", "s1"],
)), true);
assert.match(result.content[0].text, /Fase memoria vuota/);
});
test("F3 stops before ledger and advance when writing the rewritten question fails", async () => {
const { ctx, tools, calls } = await setupGate({
phase: 3,
fail: (command) => command.startsWith("session set-question ")
fail: (args) => startsWithArgs(args, ["session", "set-question"])
? { status: 1, stderr: "question write failed" }
: null,
});
@@ -426,12 +407,14 @@ test("F3 stops before ledger and advance when writing the rewritten question fai
ctx,
);
const mutations = calls.map(({ command }) => command).filter((command) =>
command.startsWith("session set-question ") ||
command.startsWith("decision add ") ||
command.startsWith("phase advance"));
const mutations = mutationArgs(calls, [
["session", "set-question"], ["decision", "add"], ["phase", "advance"],
]);
assert.deepEqual(mutations, [
"session set-question s1 --question Domanda riscritta --assumption Assunzione A",
[
"session", "set-question", "s1", "--question", "Domanda riscritta",
"--assumption", "Assunzione A",
],
]);
assert.match(result.content[0].text, /question write failed/);
});
@@ -439,7 +422,7 @@ test("F3 stops before ledger and advance when writing the rewritten question fai
test("F3 keeps the rewritten question but does not advance when the ledger write fails", async () => {
const { ctx, tools, calls } = await setupGate({
phase: 3,
fail: (command) => command.startsWith("decision add ")
fail: (args) => startsWithArgs(args, ["decision", "add"])
? { status: 1, stderr: "ledger write failed" }
: null,
});
@@ -452,14 +435,18 @@ test("F3 keeps the rewritten question but does not advance when the ledger write
ctx,
);
const mutations = calls.map(({ command }) => command).filter((command) =>
command.startsWith("session set-question ") ||
command.startsWith("decision add ") ||
command.startsWith("phase advance"));
const mutations = mutationArgs(calls, [
["session", "set-question"], ["decision", "add"], ["phase", "advance"],
]);
assert.deepEqual(mutations, [
"session set-question s1 --question Domanda riscritta --assumption Assunzione A",
"decision add --session s1 --type question_rewritten --subject domanda " +
"--detail Domanda riscritta",
[
"session", "set-question", "s1", "--question", "Domanda riscritta",
"--assumption", "Assunzione A",
],
[
"decision", "add", "--session", "s1", "--type", "question_rewritten",
"--subject", "domanda", "--detail", "Domanda riscritta",
],
]);
assert.match(result.content[0].text, /ledger write failed/);
});
@@ -507,12 +494,10 @@ test("F4 persists exactly the reviewer-selected Evidence disposition", async ()
{ id: "accept", label: "Usa la fonte", detail: undefined },
{ id: "reject", label: "Scarta la fonte", detail: undefined },
]);
const decisions = calls
.map(({ command }) => command)
.filter((command) => command.startsWith("decision add "));
const decisions = mutationArgs(calls, [["decision", "add"]]);
const expectedType = selected === "accept" ? "evidence_accepted" : "evidence_rejected";
assert.equal(decisions.length, 1);
assert.match(decisions[0], new RegExp(`--type ${expectedType} `));
assert.equal(decisions[0][decisions[0].indexOf("--type") + 1], expectedType);
assert.match(result.content[0].text, new RegExp(expectedType));
}
});
@@ -545,17 +530,16 @@ test("F8 saves an accepted Memory before its ledger marker and finalizes", async
assert.equal(descriptor.widget, "multiselect");
assert.deepEqual(descriptor.selected, ["seq-5"]);
assert.match(descriptor.content, /flag_attivo = TRUE/);
const mutations = calls.map(({ command }) => command).filter((command) =>
command.startsWith("memory save-one ") ||
command.startsWith("decision add ") ||
command.startsWith("phase advance") ||
command.startsWith("session finalize"));
const mutations = memoryPromotionMutationArgs(calls);
assert.deepEqual(mutations, [
"memory save-one --session s1 --decision 5 --json",
"decision add --session s1 --type memory_promoted --subject paziente attivo " +
"--detail seq:5 --rationale scelta dal reviewer",
"phase advance --session s1",
"session finalize s1",
["memory", "save-one", "--session", "s1", "--decision", "5", "--json"],
[
"decision", "add", "--session", "s1", "--type", "memory_promoted",
"--subject", "paziente attivo", "--detail", "seq:5",
"--rationale", "scelta dal reviewer",
],
["phase", "advance", "--session", "s1"],
["session", "finalize", "s1"],
]);
assert.match(result.content[0].text, /1 memorie salvate.*sessione finalizzata/s);
});
@@ -572,16 +556,14 @@ test("F8 records a declined candidate without saving it and finalizes", async ()
ctx,
);
const mutations = calls.map(({ command }) => command).filter((command) =>
command.startsWith("memory save-one ") ||
command.startsWith("decision add ") ||
command.startsWith("phase advance") ||
command.startsWith("session finalize"));
const mutations = memoryPromotionMutationArgs(calls);
assert.deepEqual(mutations, [
"decision add --session s1 --type memory_promotion_declined " +
"--subject paziente attivo --detail seq:5",
"phase advance --session s1",
"session finalize s1",
[
"decision", "add", "--session", "s1", "--type", "memory_promotion_declined",
"--subject", "paziente attivo", "--detail", "seq:5",
],
["phase", "advance", "--session", "s1"],
["session", "finalize", "s1"],
]);
assert.match(result.content[0].text, /1 candidati scartati.*sessione finalizzata/s);
});
@@ -598,12 +580,11 @@ test("F8 with no promotion candidates finalizes without showing a widget", async
ctx,
);
assert.equal(calls.some(({ command }) => command.startsWith("memory save-one ")), false);
assert.equal(calls.some(({ command }) => command.startsWith("decision add ")), false);
assert.equal(calls.some(({ args }) => startsWithArgs(args, ["memory", "save-one"])), false);
assert.equal(calls.some(({ args }) => startsWithArgs(args, ["decision", "add"])), false);
assert.deepEqual(
calls.map(({ command }) => command).filter((command) =>
command.startsWith("phase advance") || command.startsWith("session finalize")),
["phase advance --session s1", "session finalize s1"],
mutationArgs(calls, [["phase", "advance"], ["session", "finalize"]]),
[["phase", "advance", "--session", "s1"], ["session", "finalize", "s1"]],
);
assert.equal(ctx.notifications.length, 1);
assert.match(result.content[0].text, /Nessun candidato.*sessione finalizzata/s);
@@ -613,7 +594,7 @@ test("F8 does not write the ledger or finalize when the vector save fails", asyn
const { ctx, tools, calls } = await setupGate({
phase: 8,
preview: [PROMOTION_CANDIDATE],
fail: (command) => command.startsWith("memory save-one ")
fail: (args) => startsWithArgs(args, ["memory", "save-one"])
? { status: 1, stderr: "vector save failed" }
: null,
});
@@ -627,12 +608,10 @@ test("F8 does not write the ledger or finalize when the vector save fails", asyn
ctx,
);
const mutations = calls.map(({ command }) => command).filter((command) =>
command.startsWith("memory save-one ") ||
command.startsWith("decision add ") ||
command.startsWith("phase advance") ||
command.startsWith("session finalize"));
assert.deepEqual(mutations, ["memory save-one --session s1 --decision 5 --json"]);
const mutations = memoryPromotionMutationArgs(calls);
assert.deepEqual(mutations, [
["memory", "save-one", "--session", "s1", "--decision", "5", "--json"],
]);
assert.match(result.content[0].text, /vector save failed.*Recupero manuale/s);
});
@@ -640,7 +619,7 @@ test("F8 reports manual recovery and does not finalize after save succeeds but l
const { ctx, tools, calls } = await setupGate({
phase: 8,
preview: [PROMOTION_CANDIDATE],
fail: (command) => command.includes("--type memory_promoted")
fail: (args) => args.includes("--type") && args.includes("memory_promoted")
? { status: 1, stderr: "promotion ledger failed" }
: null,
});
@@ -654,15 +633,14 @@ test("F8 reports manual recovery and does not finalize after save succeeds but l
ctx,
);
const mutations = calls.map(({ command }) => command).filter((command) =>
command.startsWith("memory save-one ") ||
command.startsWith("decision add ") ||
command.startsWith("phase advance") ||
command.startsWith("session finalize"));
const mutations = memoryPromotionMutationArgs(calls);
assert.deepEqual(mutations, [
"memory save-one --session s1 --decision 5 --json",
"decision add --session s1 --type memory_promoted --subject paziente attivo " +
"--detail seq:5 --rationale scelta dal reviewer",
["memory", "save-one", "--session", "s1", "--decision", "5", "--json"],
[
"decision", "add", "--session", "s1", "--type", "memory_promoted",
"--subject", "paziente attivo", "--detail", "seq:5",
"--rationale", "scelta dal reviewer",
],
]);
assert.match(result.content[0].text, /vectordb.*NON registrata.*recupero manuale/is);
});
harness/.pi/extensions/gate/__tests__/golden/pi_tool_schemas.json
+199
View File
@@ -0,0 +1,199 @@
[
{
"name": "reviewer_select",
"parameters": {
"type": "object",
"required": ["session", "title", "options"],
"properties": {
"session": { "type": "string" },
"title": { "type": "string" },
"options": {
"type": "array",
"items": {
"type": "object",
"required": ["id", "label"],
"properties": {
"id": { "type": "string" },
"label": { "type": "string" },
"decision": {
"type": "object",
"required": ["type", "subject"],
"properties": {
"type": { "type": "string" },
"subject": { "type": "string" },
"detail": { "type": "string" },
"rationale": { "type": "string" }
}
},
"recommended": { "type": "boolean" }
}
}
},
"intro": { "type": "string" },
"advance": { "type": "boolean" }
}
}
},
{
"name": "reviewer_datamart",
"parameters": {
"type": "object",
"required": ["session"],
"properties": { "session": { "type": "string" } }
}
},
{
"name": "reviewer_decide",
"parameters": {
"type": "object",
"required": ["session", "title", "options"],
"properties": {
"session": { "type": "string" },
"title": { "type": "string" },
"options": {
"type": "array",
"items": {
"type": "object",
"required": ["id", "label", "decision"],
"properties": {
"id": { "type": "string" },
"label": { "type": "string" },
"description": { "type": "string" },
"decision": {
"type": "object",
"required": ["type", "subject"],
"properties": {
"type": { "type": "string" },
"subject": { "type": "string" },
"detail": { "type": "string" },
"rationale": { "type": "string" }
}
},
"recommended": { "type": "boolean" }
}
}
},
"allow_empty": { "type": "boolean" },
"advance": { "type": "boolean" }
}
}
},
{
"name": "reviewer_schema_linking",
"parameters": {
"type": "object",
"required": ["session", "title", "tables"],
"properties": {
"session": { "type": "string" },
"title": { "type": "string" },
"tables": {
"type": "array",
"items": {
"type": "object",
"required": ["id", "name", "kind"],
"properties": {
"id": { "type": "string" },
"name": { "type": "string" },
"kind": {
"anyOf": [
{ "type": "string", "const": "promote" },
{ "type": "string", "const": "exclude" }
]
},
"rationale": { "type": "string" },
"suggested_columns": {
"type": "array",
"items": { "type": "string" }
},
"recommended": { "type": "boolean" }
}
}
},
"advance": { "type": "boolean" }
}
}
},
{
"name": "reviewer_confirm",
"parameters": {
"type": "object",
"required": ["session", "kind", "title", "artifact"],
"properties": {
"session": { "type": "string" },
"kind": {
"anyOf": [
{ "type": "string", "const": "phase" },
{ "type": "string", "const": "cte_plan" },
{ "type": "string", "const": "cte_result" },
{ "type": "string", "const": "sql" }
]
},
"title": { "type": "string" },
"artifact": {
"type": "object",
"required": ["kind", "data"],
"properties": {
"kind": { "type": "string" },
"data": {},
"version": { "type": "number" }
}
},
"names": { "type": "array", "items": { "type": "string" } }
}
}
},
{
"name": "reviewer_memory_promote",
"parameters": {
"type": "object",
"required": ["session"],
"properties": { "session": { "type": "string" } }
}
},
{
"name": "rewrite_question",
"parameters": {
"type": "object",
"required": ["session", "question"],
"properties": {
"session": { "type": "string" },
"question": { "type": "string" },
"assumptions": {}
}
}
},
{
"name": "write_schema_linking",
"parameters": {
"type": "object",
"required": ["session", "schema_linking"],
"properties": {
"session": { "type": "string" },
"schema_linking": {}
}
}
},
{
"name": "write_cte_sql",
"parameters": {
"type": "object",
"required": ["session", "name", "sql"],
"properties": {
"session": { "type": "string" },
"name": { "type": "string" },
"sql": { "type": "string" }
}
}
},
{
"name": "write_final_sql",
"parameters": {
"type": "object",
"required": ["session", "sql"],
"properties": {
"session": { "type": "string" },
"sql": { "type": "string" }
}
}
}
]