test: repair MCP validation assertions and reuse model sessions
This commit is contained in:
1 parent
c88e1b0b43
commit
f97ce32a55
5 files changed
+106
-69
No files matched your search
@@ -9,31 +9,31 @@ import type { ToolContract } from "../toolContractTypes.js";
|
||||
import { toolContractMetadata } from "../toolEffects.js";
|
||||
import { evidenceResultOf } from "../toolOutputSchemaPrimitives.js";
|
||||
|
||||
export const MODEL_TOOL_CONTRACTS = [
|
||||
{
|
||||
name: "inspect_model_artifact",
|
||||
...toolContractMetadata("inspect_model_artifact"),
|
||||
kind: "artifact-provider",
|
||||
requiresActiveTarget: true,
|
||||
description:
|
||||
"Inspect local GGUF v2/v3 or safetensors declarations by content, without loading models or executing code. Default hash header reads only the header: full-file SHA-256 and Evidence subject are unknown; artifact.header_sha256 is a scoped prefix commitment, not weight identity. hash full additionally streams all file bytes for SHA-256. Optional view returns a selected metadata/tensor/type page plus model context in this call. Summary retains complete Evidence for inspect_analysis_view kind model-artifact; its pages support selection, stable header-scoped indices and paging. Metadata arrays retain eight-element typed previews; scalar metadata is complete. Tensor counts, shapes, numeric type IDs and offsets are observed; textual types and parameter/storage totals use established format facts. Unknown or historically ambiguous GGUF types retain IDs with unknown nonempty byte lengths; ambiguous identities use AMBIGUOUS_<id> labels. Unresolved Q8_1 and K-block layouts also keep nonempty byte lengths unknown. Tensor payload, model behavior and inference quality are not inspected. One active inspection, 100 MB header ceiling and 32 MiB retained-facts budget; resource refusals are explicit. No network, external files, Python, model runtime, extraction or deserialization of pickle.",
|
||||
inputSchema: inspectModelArtifactInputSchema,
|
||||
outputSchema: evidenceResultOf(
|
||||
z.union([
|
||||
modelArtifactSchema,
|
||||
modelArtifactSummaryViewResultSchema,
|
||||
modelArtifactPageViewResultSchema,
|
||||
]),
|
||||
),
|
||||
examples: [
|
||||
{
|
||||
title: "Read model declarations without reading weights",
|
||||
input: {
|
||||
path: "/artifacts/model.gguf",
|
||||
hash: "header",
|
||||
detail: "summary",
|
||||
},
|
||||
export const INSPECT_MODEL_ARTIFACT_CONTRACT = {
|
||||
name: "inspect_model_artifact",
|
||||
...toolContractMetadata("inspect_model_artifact"),
|
||||
kind: "artifact-provider",
|
||||
requiresActiveTarget: true,
|
||||
description:
|
||||
"Inspect local GGUF v2/v3 or safetensors declarations by content, without loading models or executing code. Default hash header reads only the header: full-file SHA-256 and Evidence subject are unknown; artifact.header_sha256 is a scoped prefix commitment, not weight identity. hash full additionally streams all file bytes for SHA-256. Optional view returns a selected metadata/tensor/type page plus model context in this call. Summary retains complete Evidence for inspect_analysis_view kind model-artifact; its pages support selection, stable header-scoped indices and paging. Metadata arrays retain eight-element typed previews; scalar metadata is complete. Tensor counts, shapes, numeric type IDs and offsets are observed; textual types and parameter/storage totals use established format facts. Unknown or historically ambiguous GGUF types retain IDs with unknown nonempty byte lengths; ambiguous identities use AMBIGUOUS_<id> labels. Unresolved Q8_1 and K-block layouts also keep nonempty byte lengths unknown. Tensor payload, model behavior and inference quality are not inspected. One active inspection, 100 MB header ceiling and 32 MiB retained-facts budget; resource refusals are explicit. No network, external files, Python, model runtime, extraction or deserialization of pickle.",
|
||||
inputSchema: inspectModelArtifactInputSchema,
|
||||
outputSchema: evidenceResultOf(
|
||||
z.union([
|
||||
modelArtifactSchema,
|
||||
modelArtifactSummaryViewResultSchema,
|
||||
modelArtifactPageViewResultSchema,
|
||||
]),
|
||||
),
|
||||
examples: [
|
||||
{
|
||||
title: "Read model declarations without reading weights",
|
||||
input: {
|
||||
path: "/artifacts/model.gguf",
|
||||
hash: "header",
|
||||
detail: "summary",
|
||||
},
|
||||
],
|
||||
},
|
||||
] as const satisfies readonly ToolContract[];
|
||||
},
|
||||
],
|
||||
} as const satisfies ToolContract;
|
||||
|
||||
export const MODEL_TOOL_CONTRACTS = [INSPECT_MODEL_ARTIFACT_CONTRACT] as const;
|
||||
@@ -1,8 +1,8 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile, unlink } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { expect, it } from "vitest";
|
||||
import { toolContract } from "../../src/contracts/toolContracts.js";
|
||||
import { afterAll, expect, it } from "vitest";
|
||||
import { INSPECT_MODEL_ARTIFACT_CONTRACT } from "../../src/contracts/model/modelToolContracts.js";
|
||||
import {
|
||||
modelArtifactSchema,
|
||||
type ModelArtifact,
|
||||
@@ -22,7 +22,15 @@ import {
|
||||
safetensorsFixture,
|
||||
} from "../fixtures/model/model.js";
|
||||
import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
|
||||
import { cliTest } from "../support/cli/cliFixture.js";
|
||||
import { cliTest, createSharedMcpSession } from "../support/cli/cliFixture.js";
|
||||
|
||||
// Model queries are target-free; share immutable registration and schema setup.
|
||||
// Portable export keeps its separate post-CLI connection below.
|
||||
const modelMcp = createSharedMcpSession(async (registerCleanup) => {
|
||||
const { call } = await connectLocalToolsMcp({}, registerCleanup);
|
||||
return call;
|
||||
});
|
||||
afterAll(() => modelMcp.close());
|
||||
|
||||
cliTest(
|
||||
"preserves lowercase hyphenated GGUF architecture metadata from the published writer",
|
||||
@@ -60,7 +68,7 @@ cliTest(
|
||||
});
|
||||
expect(response.exitCode, response.stderr).toBe(0);
|
||||
const report = modelArtifactSchema.parse(
|
||||
toolContract("inspect_model_artifact").outputSchema.parse(response.json)
|
||||
INSPECT_MODEL_ARTIFACT_CONTRACT.outputSchema.parse(response.json)
|
||||
.normalized_result,
|
||||
);
|
||||
expect(report.metadata.map(({ key, value }) => ({ key, value }))).toEqual(
|
||||
@@ -78,7 +86,7 @@ cliTest(
|
||||
"preserves unresolved Q8_1 layout through CLI, MCP summaries and retained tensor/type pages",
|
||||
async ({ cli }) => {
|
||||
const root = await createTestTempDirectory("rea-model-q8-layout-");
|
||||
const { call } = await connectLocalToolsMcp();
|
||||
const call = await modelMcp.session.call();
|
||||
for (const blockBytes of [36, 40]) {
|
||||
const path = join(root, `blocks-${String(blockBytes)}.gguf`);
|
||||
const base = ggufFixture();
|
||||
@@ -305,7 +313,7 @@ cliTest(
|
||||
"preserves released K and shifted integer/IQ uncertainty through CLI and retained MCP views",
|
||||
async ({ cli }) => {
|
||||
const root = await createTestTempDirectory("rea-model-historical-layouts-");
|
||||
const { call } = await connectLocalToolsMcp();
|
||||
const call = await modelMcp.session.call();
|
||||
for (const layout of ["legacy", "modern"] as const) {
|
||||
const path = join(root, `${layout}.gguf`);
|
||||
const base = ggufFixture();
|
||||
@@ -454,7 +462,7 @@ cliTest(
|
||||
});
|
||||
expect(result.exitCode, result.stderr).toBe(0);
|
||||
const report = modelArtifactSchema.parse(
|
||||
toolContract("inspect_model_artifact").outputSchema.parse(result.json)
|
||||
INSPECT_MODEL_ARTIFACT_CONTRACT.outputSchema.parse(result.json)
|
||||
.normalized_result,
|
||||
);
|
||||
expect(
|
||||
@@ -487,7 +495,7 @@ cliTest(
|
||||
expect(report.artifact.sha256).toBe(
|
||||
createHash("sha256").update(fixture.bytes).digest("hex"),
|
||||
);
|
||||
const { call } = await connectLocalToolsMcp();
|
||||
const call = await modelMcp.session.call();
|
||||
const response = await call("inspect_model_artifact", {
|
||||
path,
|
||||
detail: "complete",
|
||||
@@ -545,7 +553,7 @@ it.each(["named", "default", "inventory-only", "absent"] as const)(
|
||||
? [names]
|
||||
: [];
|
||||
await writeFile(path, ggufFixture({ metadata }).bytes);
|
||||
const { call } = await connectLocalToolsMcp();
|
||||
const call = await modelMcp.session.call();
|
||||
const response = await call("inspect_model_artifact", { path });
|
||||
expect(response.isError, JSON.stringify(response)).not.toBe(true);
|
||||
const summary = modelArtifactSummaryViewResultSchema.parse(
|
||||
@@ -591,7 +599,7 @@ it("keeps generated headline keys and imported compound metadata out of compact
|
||||
],
|
||||
}).bytes,
|
||||
);
|
||||
const { call } = await connectLocalToolsMcp();
|
||||
const call = await modelMcp.session.call();
|
||||
const compact = await call("inspect_model_artifact", { path });
|
||||
expect(compact.isError, JSON.stringify(compact)).not.toBe(true);
|
||||
const summary = modelArtifactSummaryViewResultSchema.parse(
|
||||
@@ -628,7 +636,7 @@ it("rejects noncanonical imported model indexes and duplicate identities through
|
||||
const root = await createTestTempDirectory("rea-model-import-");
|
||||
const path = join(root, "model");
|
||||
await writeFile(path, ggufFixture().bytes);
|
||||
const { call } = await connectLocalToolsMcp();
|
||||
const call = await modelMcp.session.call();
|
||||
const response = await call("inspect_model_artifact", {
|
||||
path,
|
||||
detail: "complete",
|
||||
@@ -691,7 +699,7 @@ it("bounds compact model warnings and omits unselected long labels while retaini
|
||||
Buffer.alloc(0),
|
||||
).bytes,
|
||||
);
|
||||
const { call } = await connectLocalToolsMcp();
|
||||
const call = await modelMcp.session.call();
|
||||
const response = await call("inspect_model_artifact", { path });
|
||||
expect(response.isError, JSON.stringify(response)).not.toBe(true);
|
||||
expect(JSON.stringify(response).length).toBeLessThan(16_000);
|
||||
@@ -868,7 +876,7 @@ cliTest(
|
||||
"shares scoped model facts through real CLI/MCP and pages retained metadata after source removal",
|
||||
async ({ cli }) => {
|
||||
const root = await createTestTempDirectory("rea-model-cli-");
|
||||
const { call } = await connectLocalToolsMcp();
|
||||
const call = await modelMcp.session.call();
|
||||
for (const [format, fixture] of [
|
||||
["gguf", ggufFixture()],
|
||||
["safetensors", safetensorsFixture()],
|
||||
@@ -892,9 +900,9 @@ cliTest(
|
||||
],
|
||||
});
|
||||
expect(result.exitCode, result.stderr).toBe(0);
|
||||
const evidence = toolContract(
|
||||
"inspect_model_artifact",
|
||||
).outputSchema.parse(result.json);
|
||||
const evidence = INSPECT_MODEL_ARTIFACT_CONTRACT.outputSchema.parse(
|
||||
result.json,
|
||||
);
|
||||
const report = modelArtifactSchema.parse(evidence.normalized_result);
|
||||
expect(report.format).toBe(format);
|
||||
expect(report.artifact.sha256).toBeNull();
|
||||
@@ -965,7 +973,7 @@ cliTest(
|
||||
"preserves optional safetensors fields and aligns empty GGUF data in CLI and MCP",
|
||||
async ({ cli }) => {
|
||||
const root = await createTestTempDirectory("rea-model-boundary-compat-");
|
||||
const { call } = await connectLocalToolsMcp();
|
||||
const call = await modelMcp.session.call();
|
||||
const safetensors = safetensorsFixture(
|
||||
'{"__metadata__":null,"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":{"source":"local","flags":[null,false,-1,1.25]}}}',
|
||||
Buffer.alloc(4),
|
||||
@@ -993,7 +1001,7 @@ cliTest(
|
||||
});
|
||||
expect(result.exitCode, result.stderr).toBe(0);
|
||||
const report = modelArtifactSchema.parse(
|
||||
toolContract("inspect_model_artifact").outputSchema.parse(result.json)
|
||||
INSPECT_MODEL_ARTIFACT_CONTRACT.outputSchema.parse(result.json)
|
||||
.normalized_result,
|
||||
);
|
||||
if (name.endsWith("safetensors")) {
|
||||
@@ -1041,7 +1049,7 @@ cliTest(
|
||||
});
|
||||
expect(mapCli.exitCode, mapCli.stderr).toBe(0);
|
||||
const mapReport = modelArtifactSchema.parse(
|
||||
toolContract("inspect_model_artifact").outputSchema.parse(mapCli.json)
|
||||
INSPECT_MODEL_ARTIFACT_CONTRACT.outputSchema.parse(mapCli.json)
|
||||
.normalized_result,
|
||||
);
|
||||
expect(
|
||||
@@ -1092,7 +1100,7 @@ cliTest(
|
||||
"inspects unpadded zero-tensor vocab GGUF through CLI and MCP",
|
||||
async ({ cli }) => {
|
||||
const root = await createTestTempDirectory("rea-model-vocab-only-");
|
||||
const { call } = await connectLocalToolsMcp();
|
||||
const call = await modelMcp.session.call();
|
||||
const base = ggufFixture();
|
||||
const vocab = ggufFixture({
|
||||
tensors: [],
|
||||
@@ -1132,9 +1140,8 @@ cliTest(
|
||||
});
|
||||
expect(cliResult.exitCode, cliResult.stderr).toBe(0);
|
||||
const report = modelArtifactSchema.parse(
|
||||
toolContract("inspect_model_artifact").outputSchema.parse(
|
||||
cliResult.json,
|
||||
).normalized_result,
|
||||
INSPECT_MODEL_ARTIFACT_CONTRACT.outputSchema.parse(cliResult.json)
|
||||
.normalized_result,
|
||||
);
|
||||
expect(report.metadata.map(({ key, value }) => ({ key, value }))).toEqual(
|
||||
[
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
serializeEvidenceBundle,
|
||||
} from "../../../src/domain/evidenceBundle.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
|
||||
import { cliTest } from "../../support/cli/cliFixture.js";
|
||||
|
||||
cliTest("rejects the removed compare-bundles command", async ({ cli }) => {
|
||||
@@ -219,11 +220,19 @@ cliTest(
|
||||
name: "compare_bundles",
|
||||
arguments: { left_bundle_path: left, right_bundle_path: right },
|
||||
});
|
||||
expect(result.isError).toBe(true);
|
||||
expect(result.content[0]).toMatchObject({
|
||||
text: expect.stringContaining(
|
||||
`${rejectedField}: must be an absolute local filesystem path`,
|
||||
),
|
||||
expect(parseMcpToolError(result).error).toMatchObject({
|
||||
code: "invalid_request",
|
||||
category: "invalid_input",
|
||||
details: {
|
||||
operation: "compare_bundles",
|
||||
issues: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
path: [rejectedField],
|
||||
reason: "invalid_format",
|
||||
message: "must be an absolute local filesystem path",
|
||||
}),
|
||||
]),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -164,11 +164,16 @@ it("requires absolute bundle paths without changing server-cwd semantics", async
|
||||
expect(parseMcpToolError(result).error).toMatchObject({
|
||||
code: "invalid_request",
|
||||
category: "invalid_input",
|
||||
});
|
||||
expect(result.content[0]).toMatchObject({
|
||||
text: expect.stringContaining(
|
||||
`${rejectedField}: must be an absolute local filesystem path`,
|
||||
),
|
||||
details: {
|
||||
operation: "compare_bundles",
|
||||
issues: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
path: [rejectedField],
|
||||
reason: "invalid_format",
|
||||
message: "must be an absolute local filesystem path",
|
||||
}),
|
||||
]),
|
||||
},
|
||||
});
|
||||
}
|
||||
expect(
|
||||
@@ -195,9 +200,16 @@ it("requires absolute bundle paths without changing server-cwd semantics", async
|
||||
expect(parseMcpToolError(relativeImport).error).toMatchObject({
|
||||
code: "invalid_request",
|
||||
category: "invalid_input",
|
||||
});
|
||||
expect(relativeImport.content[0]).toMatchObject({
|
||||
text: expect.stringContaining("absolute local filesystem path"),
|
||||
details: {
|
||||
operation: "import_evidence_bundle",
|
||||
issues: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
path: ["path"],
|
||||
reason: "invalid_format",
|
||||
message: "must be an absolute local filesystem path",
|
||||
}),
|
||||
]),
|
||||
},
|
||||
});
|
||||
const absolute = await compare(
|
||||
join(root, "left.json"),
|
||||
|
||||
@@ -14,6 +14,7 @@ import { flutterResultSchemas } from "../../../src/domain/flutter/flutterBuildAn
|
||||
import { aotFixtureImage } from "../../../src/flutter/FlutterBuild.fixture.js";
|
||||
import { createFlutterApk } from "../../fixtures/flutterApk.js";
|
||||
import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
|
||||
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
|
||||
|
||||
it.each([
|
||||
{ label: "ARM64-V8A", paired: false },
|
||||
@@ -255,12 +256,20 @@ it("advertises and enforces the same Unicode selector grammar through MCP", asyn
|
||||
name: "inspect_dart_aot",
|
||||
arguments: arguments_,
|
||||
});
|
||||
expect(refused.isError).toBe(true);
|
||||
expect(refused.content).toContainEqual({
|
||||
type: "text",
|
||||
text: expect.stringContaining(
|
||||
"abi: ABI must be the exact lib/ directory label",
|
||||
),
|
||||
expect(parseMcpToolError(refused).error).toMatchObject({
|
||||
code: "invalid_request",
|
||||
category: "invalid_input",
|
||||
details: {
|
||||
operation: "inspect_dart_aot",
|
||||
issues: [
|
||||
expect.objectContaining({
|
||||
path: ["abi"],
|
||||
message: expect.stringContaining(
|
||||
"ABI must be the exact lib/ directory label",
|
||||
),
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user