test: repair MCP validation assertions and reuse model sessions

This commit is contained in:
gensokyo-dev committed 2026-10-29 08:57:42 +00:00
1 parent c88e1b0b43
commit f97ce32a55
5 files changed
+106 -69

No files matched your search

+27 -27
View File
@@ -9,31 +9,31 @@ import type { ToolContract } from "../toolContractTypes.js";
import { toolContractMetadata } from "../toolEffects.js";
import { evidenceResultOf } from "../toolOutputSchemaPrimitives.js";
export const MODEL_TOOL_CONTRACTS = [
{
name: "inspect_model_artifact",
...toolContractMetadata("inspect_model_artifact"),
kind: "artifact-provider",
requiresActiveTarget: true,
description:
"Inspect local GGUF v2/v3 or safetensors declarations by content, without loading models or executing code. Default hash header reads only the header: full-file SHA-256 and Evidence subject are unknown; artifact.header_sha256 is a scoped prefix commitment, not weight identity. hash full additionally streams all file bytes for SHA-256. Optional view returns a selected metadata/tensor/type page plus model context in this call. Summary retains complete Evidence for inspect_analysis_view kind model-artifact; its pages support selection, stable header-scoped indices and paging. Metadata arrays retain eight-element typed previews; scalar metadata is complete. Tensor counts, shapes, numeric type IDs and offsets are observed; textual types and parameter/storage totals use established format facts. Unknown or historically ambiguous GGUF types retain IDs with unknown nonempty byte lengths; ambiguous identities use AMBIGUOUS_<id> labels. Unresolved Q8_1 and K-block layouts also keep nonempty byte lengths unknown. Tensor payload, model behavior and inference quality are not inspected. One active inspection, 100 MB header ceiling and 32 MiB retained-facts budget; resource refusals are explicit. No network, external files, Python, model runtime, extraction or deserialization of pickle.",
inputSchema: inspectModelArtifactInputSchema,
outputSchema: evidenceResultOf(
z.union([
modelArtifactSchema,
modelArtifactSummaryViewResultSchema,
modelArtifactPageViewResultSchema,
]),
),
examples: [
{
title: "Read model declarations without reading weights",
input: {
path: "/artifacts/model.gguf",
hash: "header",
detail: "summary",
},
export const INSPECT_MODEL_ARTIFACT_CONTRACT = {
name: "inspect_model_artifact",
...toolContractMetadata("inspect_model_artifact"),
kind: "artifact-provider",
requiresActiveTarget: true,
description:
"Inspect local GGUF v2/v3 or safetensors declarations by content, without loading models or executing code. Default hash header reads only the header: full-file SHA-256 and Evidence subject are unknown; artifact.header_sha256 is a scoped prefix commitment, not weight identity. hash full additionally streams all file bytes for SHA-256. Optional view returns a selected metadata/tensor/type page plus model context in this call. Summary retains complete Evidence for inspect_analysis_view kind model-artifact; its pages support selection, stable header-scoped indices and paging. Metadata arrays retain eight-element typed previews; scalar metadata is complete. Tensor counts, shapes, numeric type IDs and offsets are observed; textual types and parameter/storage totals use established format facts. Unknown or historically ambiguous GGUF types retain IDs with unknown nonempty byte lengths; ambiguous identities use AMBIGUOUS_<id> labels. Unresolved Q8_1 and K-block layouts also keep nonempty byte lengths unknown. Tensor payload, model behavior and inference quality are not inspected. One active inspection, 100 MB header ceiling and 32 MiB retained-facts budget; resource refusals are explicit. No network, external files, Python, model runtime, extraction or deserialization of pickle.",
inputSchema: inspectModelArtifactInputSchema,
outputSchema: evidenceResultOf(
z.union([
modelArtifactSchema,
modelArtifactSummaryViewResultSchema,
modelArtifactPageViewResultSchema,
]),
),
examples: [
{
title: "Read model declarations without reading weights",
input: {
path: "/artifacts/model.gguf",
hash: "header",
detail: "summary",
},
],
},
] as const satisfies readonly ToolContract[];
},
],
} as const satisfies ToolContract;
export const MODEL_TOOL_CONTRACTS = [INSPECT_MODEL_ARTIFACT_CONTRACT] as const;
+30 -23
View File
@@ -1,8 +1,8 @@
import { createHash } from "node:crypto";
import { readFile, writeFile, unlink } from "node:fs/promises";
import { join } from "node:path";
import { expect, it } from "vitest";
import { toolContract } from "../../src/contracts/toolContracts.js";
import { afterAll, expect, it } from "vitest";
import { INSPECT_MODEL_ARTIFACT_CONTRACT } from "../../src/contracts/model/modelToolContracts.js";
import {
modelArtifactSchema,
type ModelArtifact,
@@ -22,7 +22,15 @@ import {
safetensorsFixture,
} from "../fixtures/model/model.js";
import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
import { cliTest } from "../support/cli/cliFixture.js";
import { cliTest, createSharedMcpSession } from "../support/cli/cliFixture.js";
// Model queries are target-free; share immutable registration and schema setup.
// Portable export keeps its separate post-CLI connection below.
const modelMcp = createSharedMcpSession(async (registerCleanup) => {
const { call } = await connectLocalToolsMcp({}, registerCleanup);
return call;
});
afterAll(() => modelMcp.close());
cliTest(
"preserves lowercase hyphenated GGUF architecture metadata from the published writer",
@@ -60,7 +68,7 @@ cliTest(
});
expect(response.exitCode, response.stderr).toBe(0);
const report = modelArtifactSchema.parse(
toolContract("inspect_model_artifact").outputSchema.parse(response.json)
INSPECT_MODEL_ARTIFACT_CONTRACT.outputSchema.parse(response.json)
.normalized_result,
);
expect(report.metadata.map(({ key, value }) => ({ key, value }))).toEqual(
@@ -78,7 +86,7 @@ cliTest(
"preserves unresolved Q8_1 layout through CLI, MCP summaries and retained tensor/type pages",
async ({ cli }) => {
const root = await createTestTempDirectory("rea-model-q8-layout-");
const { call } = await connectLocalToolsMcp();
const call = await modelMcp.session.call();
for (const blockBytes of [36, 40]) {
const path = join(root, `blocks-${String(blockBytes)}.gguf`);
const base = ggufFixture();
@@ -305,7 +313,7 @@ cliTest(
"preserves released K and shifted integer/IQ uncertainty through CLI and retained MCP views",
async ({ cli }) => {
const root = await createTestTempDirectory("rea-model-historical-layouts-");
const { call } = await connectLocalToolsMcp();
const call = await modelMcp.session.call();
for (const layout of ["legacy", "modern"] as const) {
const path = join(root, `${layout}.gguf`);
const base = ggufFixture();
@@ -454,7 +462,7 @@ cliTest(
});
expect(result.exitCode, result.stderr).toBe(0);
const report = modelArtifactSchema.parse(
toolContract("inspect_model_artifact").outputSchema.parse(result.json)
INSPECT_MODEL_ARTIFACT_CONTRACT.outputSchema.parse(result.json)
.normalized_result,
);
expect(
@@ -487,7 +495,7 @@ cliTest(
expect(report.artifact.sha256).toBe(
createHash("sha256").update(fixture.bytes).digest("hex"),
);
const { call } = await connectLocalToolsMcp();
const call = await modelMcp.session.call();
const response = await call("inspect_model_artifact", {
path,
detail: "complete",
@@ -545,7 +553,7 @@ it.each(["named", "default", "inventory-only", "absent"] as const)(
? [names]
: [];
await writeFile(path, ggufFixture({ metadata }).bytes);
const { call } = await connectLocalToolsMcp();
const call = await modelMcp.session.call();
const response = await call("inspect_model_artifact", { path });
expect(response.isError, JSON.stringify(response)).not.toBe(true);
const summary = modelArtifactSummaryViewResultSchema.parse(
@@ -591,7 +599,7 @@ it("keeps generated headline keys and imported compound metadata out of compact
],
}).bytes,
);
const { call } = await connectLocalToolsMcp();
const call = await modelMcp.session.call();
const compact = await call("inspect_model_artifact", { path });
expect(compact.isError, JSON.stringify(compact)).not.toBe(true);
const summary = modelArtifactSummaryViewResultSchema.parse(
@@ -628,7 +636,7 @@ it("rejects noncanonical imported model indexes and duplicate identities through
const root = await createTestTempDirectory("rea-model-import-");
const path = join(root, "model");
await writeFile(path, ggufFixture().bytes);
const { call } = await connectLocalToolsMcp();
const call = await modelMcp.session.call();
const response = await call("inspect_model_artifact", {
path,
detail: "complete",
@@ -691,7 +699,7 @@ it("bounds compact model warnings and omits unselected long labels while retaini
Buffer.alloc(0),
).bytes,
);
const { call } = await connectLocalToolsMcp();
const call = await modelMcp.session.call();
const response = await call("inspect_model_artifact", { path });
expect(response.isError, JSON.stringify(response)).not.toBe(true);
expect(JSON.stringify(response).length).toBeLessThan(16_000);
@@ -868,7 +876,7 @@ cliTest(
"shares scoped model facts through real CLI/MCP and pages retained metadata after source removal",
async ({ cli }) => {
const root = await createTestTempDirectory("rea-model-cli-");
const { call } = await connectLocalToolsMcp();
const call = await modelMcp.session.call();
for (const [format, fixture] of [
["gguf", ggufFixture()],
["safetensors", safetensorsFixture()],
@@ -892,9 +900,9 @@ cliTest(
],
});
expect(result.exitCode, result.stderr).toBe(0);
const evidence = toolContract(
"inspect_model_artifact",
).outputSchema.parse(result.json);
const evidence = INSPECT_MODEL_ARTIFACT_CONTRACT.outputSchema.parse(
result.json,
);
const report = modelArtifactSchema.parse(evidence.normalized_result);
expect(report.format).toBe(format);
expect(report.artifact.sha256).toBeNull();
@@ -965,7 +973,7 @@ cliTest(
"preserves optional safetensors fields and aligns empty GGUF data in CLI and MCP",
async ({ cli }) => {
const root = await createTestTempDirectory("rea-model-boundary-compat-");
const { call } = await connectLocalToolsMcp();
const call = await modelMcp.session.call();
const safetensors = safetensorsFixture(
'{"__metadata__":null,"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":{"source":"local","flags":[null,false,-1,1.25]}}}',
Buffer.alloc(4),
@@ -993,7 +1001,7 @@ cliTest(
});
expect(result.exitCode, result.stderr).toBe(0);
const report = modelArtifactSchema.parse(
toolContract("inspect_model_artifact").outputSchema.parse(result.json)
INSPECT_MODEL_ARTIFACT_CONTRACT.outputSchema.parse(result.json)
.normalized_result,
);
if (name.endsWith("safetensors")) {
@@ -1041,7 +1049,7 @@ cliTest(
});
expect(mapCli.exitCode, mapCli.stderr).toBe(0);
const mapReport = modelArtifactSchema.parse(
toolContract("inspect_model_artifact").outputSchema.parse(mapCli.json)
INSPECT_MODEL_ARTIFACT_CONTRACT.outputSchema.parse(mapCli.json)
.normalized_result,
);
expect(
@@ -1092,7 +1100,7 @@ cliTest(
"inspects unpadded zero-tensor vocab GGUF through CLI and MCP",
async ({ cli }) => {
const root = await createTestTempDirectory("rea-model-vocab-only-");
const { call } = await connectLocalToolsMcp();
const call = await modelMcp.session.call();
const base = ggufFixture();
const vocab = ggufFixture({
tensors: [],
@@ -1132,9 +1140,8 @@ cliTest(
});
expect(cliResult.exitCode, cliResult.stderr).toBe(0);
const report = modelArtifactSchema.parse(
toolContract("inspect_model_artifact").outputSchema.parse(
cliResult.json,
).normalized_result,
INSPECT_MODEL_ARTIFACT_CONTRACT.outputSchema.parse(cliResult.json)
.normalized_result,
);
expect(report.metadata.map(({ key, value }) => ({ key, value }))).toEqual(
[
+14 -5
View File
@@ -13,6 +13,7 @@ import {
serializeEvidenceBundle,
} from "../../../src/domain/evidenceBundle.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
import { cliTest } from "../../support/cli/cliFixture.js";
cliTest("rejects the removed compare-bundles command", async ({ cli }) => {
@@ -219,11 +220,19 @@ cliTest(
name: "compare_bundles",
arguments: { left_bundle_path: left, right_bundle_path: right },
});
expect(result.isError).toBe(true);
expect(result.content[0]).toMatchObject({
text: expect.stringContaining(
`${rejectedField}: must be an absolute local filesystem path`,
),
expect(parseMcpToolError(result).error).toMatchObject({
code: "invalid_request",
category: "invalid_input",
details: {
operation: "compare_bundles",
issues: expect.arrayContaining([
expect.objectContaining({
path: [rejectedField],
reason: "invalid_format",
message: "must be an absolute local filesystem path",
}),
]),
},
});
}
+20 -8
View File
@@ -164,11 +164,16 @@ it("requires absolute bundle paths without changing server-cwd semantics", async
expect(parseMcpToolError(result).error).toMatchObject({
code: "invalid_request",
category: "invalid_input",
});
expect(result.content[0]).toMatchObject({
text: expect.stringContaining(
`${rejectedField}: must be an absolute local filesystem path`,
),
details: {
operation: "compare_bundles",
issues: expect.arrayContaining([
expect.objectContaining({
path: [rejectedField],
reason: "invalid_format",
message: "must be an absolute local filesystem path",
}),
]),
},
});
}
expect(
@@ -195,9 +200,16 @@ it("requires absolute bundle paths without changing server-cwd semantics", async
expect(parseMcpToolError(relativeImport).error).toMatchObject({
code: "invalid_request",
category: "invalid_input",
});
expect(relativeImport.content[0]).toMatchObject({
text: expect.stringContaining("absolute local filesystem path"),
details: {
operation: "import_evidence_bundle",
issues: expect.arrayContaining([
expect.objectContaining({
path: ["path"],
reason: "invalid_format",
message: "must be an absolute local filesystem path",
}),
]),
},
});
const absolute = await compare(
join(root, "left.json"),
+15 -6
View File
@@ -14,6 +14,7 @@ import { flutterResultSchemas } from "../../../src/domain/flutter/flutterBuildAn
import { aotFixtureImage } from "../../../src/flutter/FlutterBuild.fixture.js";
import { createFlutterApk } from "../../fixtures/flutterApk.js";
import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
it.each([
{ label: "ARM64-V8A", paired: false },
@@ -255,12 +256,20 @@ it("advertises and enforces the same Unicode selector grammar through MCP", asyn
name: "inspect_dart_aot",
arguments: arguments_,
});
expect(refused.isError).toBe(true);
expect(refused.content).toContainEqual({
type: "text",
text: expect.stringContaining(
"abi: ABI must be the exact lib/ directory label",
),
expect(parseMcpToolError(refused).error).toMatchObject({
code: "invalid_request",
category: "invalid_input",
details: {
operation: "inspect_dart_aot",
issues: [
expect.objectContaining({
path: ["abi"],
message: expect.stringContaining(
"ABI must be the exact lib/ directory label",
),
}),
],
},
});
}
});