feat(harness): cross-model behavior matrix (G) — harness + results

Tier 1 (clean-room first-turn harness, harness/scripts/model-matrix.mjs): kickoff + resume chain in-turn on ALL available models — zai/glm-5.2, deepseek/deepseek-v4-{pro,flash}, aritmolab/qwen3.6-35b-a3b, zai/glm-4.5-air. The resume cold-start stall recurs on none (closes A's cross-model robustness). aritmolab/gemma4-26b-a4b is a 404 at the endpoint (listed but not served) — an availability gap classified as MODEL_ERROR, not a workflow issue. Tier 2 (live, baseline zai/glm-5.2): F single-select auto-confirm verified end-to-end — answering the first reviewer_select persisted a concept_clarified decision (review_decisions.jsonl 0->1) with no follow-up confirmation gate. Closes F's deferred live check. No prompt hardening needed. Results in the G plan doc + memory. Throwaway psd sessions used and deleted; real sessions untouched. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-30 18:46:55 +02:00
parent e8cdd0037c
commit cbb8e184f1
3 changed files with 201 additions and 5 deletions
@@ -0,0 +1,160 @@
+// model-matrix.mjs — workstream G Tier 1: clean-room cross-model behavior matrix.
+//
+// Drives `pi --mode rpc` exactly as the backend does (set_model -> set_thinking_level ->
+// prompt) for each (model x scenario) and classifies the FIRST turn. Decisive signal: a
+// `toolcall_start`/`tool_execution_start` means the model chained into work in-turn; its
+// absence + quiescence = the narrate-and-stop stall.
+//
+// Scenarios (first-turn only, cheap ~15-30s each):
+//   new    -> prompt `/nuova-domanda "kickoff"` (THT_SESSION set => PROVIDED kickoff; first
+//             tool should be `read` of SKILL.md)
+//   resume -> prompt `/riprendi-sessione <id>` (first tools should be `tht session show` +
+//             `read SKILL.md` in-turn) — workstream A robustness across models.
+//
+// Usage:
+//   node harness/scripts/model-matrix.mjs <sessionId> [provider/model,...] [thinking] [outDir]
+// Defaults: the named primaries + one weak/local model; thinking=medium.
+//
+// NOTE: spawns real models against the configured workspace. Use a THROWAWAY session and
+// delete it afterwards (psd is real client data). Kills each child after the first decisive
+// signal, so mutation is minimal (reads only; no gate is answered).
+
+import { spawn } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const harnessDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+
+const sessionId = process.argv[2];
+if (!sessionId) {
+  console.error("usage: node model-matrix.mjs <sessionId> [provider/model,...] [thinking] [outDir]");
+  process.exit(2);
+}
+const models = (process.argv[3] ??
+  "zai/glm-5.2,deepseek/deepseek-v4-pro,aritmolab/qwen3.6-35b-a3b,aritmolab/gemma4-26b-a4b")
+  .split(",").map((s) => s.trim()).filter(Boolean);
+const thinking = process.argv[4] ?? "medium";
+const outDir = process.argv[5] ?? "/tmp/model-matrix";
+fs.mkdirSync(outDir, { recursive: true });
+
+const scenarios = ["new", "resume"];
+const DEADLINE = 90000;   // hard cap per cell
+const QUIESCENCE = 14000; // turn ended idle if this long with no new events and no toolcall
+const sleep = (n) => new Promise((r) => setTimeout(r, n));
+
+function runCell(provider, modelId, scenario, rawOut) {
+  return new Promise((resolve) => {
+    const env = {
+      ...process.env,
+      THT_SESSION: sessionId,
+      THT_AUTHOR: "matrix@local",
+      PATH: `${harnessDir}/.venv/bin:${process.env.PATH ?? ""}`,
+    };
+    const child = spawn("pi", ["--mode", "rpc"], { cwd: harnessDir, env });
+    const raw = fs.createWriteStream(rawOut);
+    child.stderr.resume();
+
+    const t0 = Date.now();
+    const ms = () => Date.now() - t0;
+    const counts = {};
+    let firstTool = null;
+    let assistantText = "";
+    let modelError = null; // a 404/endpoint error is "model unavailable", not a behavioral stall
+    let lastEventAtMs = 0;
+    let promptAtMs = Infinity;
+    let buf = "";
+
+    child.stdout.on("data", (d) => {
+      raw.write(d);
+      buf += d.toString();
+      let i;
+      while ((i = buf.indexOf("\n")) >= 0) {
+        const line = buf.slice(0, i);
+        buf = buf.slice(i + 1);
+        if (!line.trim()) continue;
+        lastEventAtMs = ms();
+        let e;
+        try { e = JSON.parse(line); } catch { continue; }
+        const type = e.type ?? "?";
+        counts[type] = (counts[type] || 0) + 1;
+        if ((type === "toolcall_start" || type === "tool_execution_start") && !firstTool) {
+          firstTool = {
+            atMs: ms(),
+            name: e.toolName ?? e.name ?? e.tool?.name ?? e.input?.command ?? "?",
+          };
+        }
+        if (type === "message_update") {
+          const delta = e.assistantMessageEvent?.delta;
+          if (typeof delta === "string") assistantText += delta;
+        }
+        if (!modelError) {
+          const m = e.message;
+          if (m && (m.errorMessage || m.stopReason === "error")) {
+            modelError = m.errorMessage || "stopReason=error";
+          }
+        }
+      }
+    });
+
+    function send(obj) { try { child.stdin.write(JSON.stringify(obj) + "\n"); } catch {} }
+
+    function finish(verdict) {
+      clearInterval(timer);
+      try { child.kill("SIGKILL"); } catch {}
+      resolve({
+        provider, modelId, scenario,
+        verdict: modelError ? "MODEL_ERROR" : verdict,
+        firstToolAtMs: firstTool?.atMs ?? null,
+        firstToolName: firstTool?.name ?? null,
+        error: modelError,
+        events: counts,
+        textTail: assistantText.slice(-240).replace(/\s+/g, " ").trim(),
+      });
+    }
+
+    (async () => {
+      await sleep(700);
+      send({ type: "set_model", provider, modelId });
+      await sleep(1800);
+      send({ type: "set_thinking_level", level: thinking });
+      await sleep(1000);
+      const message = scenario === "resume"
+        ? `/riprendi-sessione ${sessionId}`
+        : `/nuova-domanda "kickoff"`;
+      send({ type: "prompt", message });
+      promptAtMs = ms();
+    })();
+
+    const timer = setInterval(() => {
+      if (modelError) return finish("MODEL_ERROR");
+      if (firstTool) return finish("CHAINED");
+      const sawTurn = lastEventAtMs > promptAtMs && (counts.agent_start || counts.turn_start || counts.message_start);
+      const quiet = ms() - lastEventAtMs > QUIESCENCE;
+      if (sawTurn && quiet) return finish("STALLED");
+      if (ms() > DEADLINE) return finish(sawTurn ? "STALLED" : "NO_TURN");
+    }, 1000);
+
+    child.on("exit", () => { if (!firstTool) finish("CHILD_EXIT"); });
+  });
+}
+
+const results = [];
+for (const m of models) {
+  const [provider, modelId] = m.split("/");
+  for (const scenario of scenarios) {
+    const tag = `${provider}_${modelId}_${scenario}`.replace(/[^a-z0-9_.-]/gi, "-");
+    const r = await runCell(provider, modelId, scenario, path.join(outDir, `${tag}.jsonl`));
+    results.push(r);
+    console.error(`done ${m} ${scenario}: ${r.verdict} (${r.firstToolName ?? "-"} @ ${r.firstToolAtMs ?? "-"}ms)`);
+  }
+}
+
+// Markdown summary table + raw JSON for the record.
+const rows = results.map((r) =>
+  `| ${r.provider}/${r.modelId} | ${r.scenario} | ${r.verdict} | ${r.firstToolName ?? "-"} | ${r.firstToolAtMs ?? "-"} | ${r.error ?? ""} |`,
+);
+console.log("\n| model | scenario | verdict | first tool | t(ms) | error |");
+console.log("|---|---|---|---|---|---|");
+console.log(rows.join("\n"));
+console.log("\nJSON " + JSON.stringify(results));