diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index 478656e1c5..6c9d99475c 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -119,6 +119,8 @@ export const SCOPES = { "scripts/ci/npm-release-state.*", "scripts/verify-release-artifact.mjs", "scripts/lib/release-package-canary.mjs", + "scripts/lib/verify-package-core.*", + "tests/boundary/mcp/packageCatalogVerification.test.ts", "scripts/verify-published-package.mjs", "tests/boundary/filesystem/releaseRecovery.test.ts", "tests/boundary/filesystem/mcpRegistryPublication.test.ts", diff --git a/scripts/lib/release-package-canary.mjs b/scripts/lib/release-package-canary.mjs index b00bb9788c..9af44c06dc 100644 --- a/scripts/lib/release-package-canary.mjs +++ b/scripts/lib/release-package-canary.mjs @@ -6,11 +6,7 @@ import { join } from "node:path"; import { promisify } from "node:util"; import { FULL_CATALOG_IDENTITY } from "../../dist/catalogIdentity.js"; -import { TOOL_CONTRACTS } from "../../dist/contracts/toolContracts.js"; -import { - requireMcpLifecycleResult, - sortVerifierNames, -} from "./mcp-verifier-results.mjs"; +import { verifyCompleteToolCatalog } from "./verify-package-core.mjs"; const execute = promisify(execFile); @@ -71,37 +67,7 @@ export async function verifyInstalledCanaryPackage( }); try { await client.connect(transport); - const tools = await client.listTools(); - const toolNames = tools.tools.map(({ name }) => name); - const canonicalToolNames = new Set(TOOL_CONTRACTS.map(({ name }) => name)); - const publishedToolNames = new Set(toolNames); - const unknown = toolNames.filter((name) => !canonicalToolNames.has(name)); - const missing = [...canonicalToolNames].filter( - (name) => !publishedToolNames.has(name), - ); - if ( - toolNames.length === 0 || - publishedToolNames.size !== toolNames.length || - unknown.length || - missing.length - ) - throw new Error( - `Installed MCP exposed an invalid canonical tool catalog (${toolNames.length} tools, unknown: ${unknown.join(", ") || "none"}, missing: ${missing.join(", ") || "none"})`, - ); - const status = requireMcpLifecycleResult( - await client.callTool({ name: "binary_session", arguments: {} }), - "binary_session", - ); - const availability = sortVerifierNames( - status.tool_availability.map(({ name }) => name), - ); - if ( - JSON.stringify(availability) !== - JSON.stringify(sortVerifierNames(canonicalToolNames)) - ) - throw new Error( - "Installed binary_session omitted canonical tool availability", - ); + const toolNames = await verifyCompleteToolCatalog(client); return { mcpTools: toolNames.length, canonicalMcpTools: FULL_CATALOG_IDENTITY.counts.mcp_tools, diff --git a/scripts/lib/verify-package-core.d.mts b/scripts/lib/verify-package-core.d.mts new file mode 100644 index 0000000000..88d04b9142 --- /dev/null +++ b/scripts/lib/verify-package-core.d.mts @@ -0,0 +1,6 @@ +import type { Client } from "@modelcontextprotocol/client"; +export function verifyCompleteToolCatalog( + client: Pick, + options?: Parameters[1], +): Promise; +export function pathExists(path: string): Promise; diff --git a/scripts/lib/verify-package-core.mjs b/scripts/lib/verify-package-core.mjs index 2c5a8977e8..297a18e046 100644 --- a/scripts/lib/verify-package-core.mjs +++ b/scripts/lib/verify-package-core.mjs @@ -1,15 +1,31 @@ import { lstat } from "node:fs/promises"; import { Ajv2020 } from "ajv/dist/2020.js"; -// Only catalog verification needs built MCP contracts; path checks work -// from source without a runtime build. -const loadMcpVerifierResults = () => import("./mcp-verifier-results.mjs"); - -/** Verify schema validity and that the MCP catalog matches session metadata. */ +/** + * Verify canonical names, converted JSON Schemas, and session catalog metadata. + * Only this operation needs built contracts; path checks remain source-only. + */ export const verifyCompleteToolCatalog = async (client, options) => { - const { requireMcpLifecycleResult, sortVerifierNames } = - await loadMcpVerifierResults(); + const [{ requireMcpLifecycleResult, sortVerifierNames }, { TOOL_CONTRACTS }] = + await Promise.all([ + import("./mcp-verifier-results.mjs"), + import("../../dist/contracts/toolContracts.js"), + ]); const listed = await client.listTools(undefined, options); + const canonical = sortVerifierNames(TOOL_CONTRACTS.map(({ name }) => name)); + const observed = sortVerifierNames(listed.tools.map(({ name }) => name)); + if (JSON.stringify(observed) !== JSON.stringify(canonical)) { + const canonicalNames = new Set(canonical); + const observedNames = new Set(observed); + const unknown = observed.filter((name) => !canonicalNames.has(name)); + const missing = canonical.filter((name) => !observedNames.has(name)); + const duplicates = observed.filter( + (name, index) => index > 0 && name === observed[index - 1], + ); + throw new Error( + `packaged MCP exposed an invalid canonical tool catalog (${observed.length} tools, unknown: ${unknown.join(", ") || "none"}, missing: ${missing.join(", ") || "none"}, duplicates: ${duplicates.join(", ") || "none"})`, + ); + } const ajv = new Ajv2020({ strict: false, validateFormats: false }); for (const tool of listed.tools) { for (const kind of ["inputSchema", "outputSchema"]) { @@ -27,7 +43,6 @@ export const verifyCompleteToolCatalog = async (client, options) => { const expanded = requireMcpLifecycleResult(status, "binary_session"); const availability = expanded.tool_availability; const expected = sortVerifierNames(availability.map(({ name }) => name)); - const observed = sortVerifierNames(listed.tools.map(({ name }) => name)); if (JSON.stringify(observed) !== JSON.stringify(expected)) throw new Error( "packaged MCP tool inventory diverged from session metadata", diff --git a/tests/boundary/mcp/packageCatalogVerification.test.ts b/tests/boundary/mcp/packageCatalogVerification.test.ts new file mode 100644 index 0000000000..b1fa0066b2 --- /dev/null +++ b/tests/boundary/mcp/packageCatalogVerification.test.ts @@ -0,0 +1,132 @@ +import { expect, it } from "vitest"; +import { z } from "zod"; +import { verifyCompleteToolCatalog } from "../../../scripts/lib/verify-package-core.mjs"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; + +it("verifies canonical names, converted schemas and session metadata together", async () => { + const harness = await createApplicationMcpHarness(); + try { + const tools = await harness.client.listTools(); + const status = await harness.client.callTool({ + name: "binary_session", + arguments: {}, + }); + const forwarded: unknown[][] = []; + const client = { + listTools: async (...args: unknown[]) => { + forwarded.push(args); + return tools; + }, + callTool: async (...args: unknown[]) => { + forwarded.push(args); + return status; + }, + }; + const options = { timeout: 12_345 }; + const canonical = await verifyCompleteToolCatalog(client, options); + expect(forwarded).toEqual([ + [undefined, options], + [{ name: "binary_session", arguments: {} }, options], + ]); + expect(canonical).toEqual(tools.tools.map(({ name }) => name).sort()); + + const omitted = tools.tools[0]; + if (omitted === undefined) throw new Error("Expected canonical tools"); + const reduced = { ...tools, tools: tools.tools.slice(1) }; + const body = z + .object({ + result: z + .object({ + tool_availability: z + .object({ + groups: z.array( + z.object({ names: z.array(z.string()) }).passthrough(), + ), + }) + .passthrough(), + }) + .passthrough(), + }) + .passthrough() + .parse( + JSON.parse( + status.content.find((item) => item.type === "text")?.text ?? "null", + ), + ); + body.result.tool_availability.groups = body.result.tool_availability.groups + .map((group) => ({ + ...group, + names: group.names.filter((name) => name !== omitted.name), + })) + .filter((group) => group.names.length > 0); + toolContract("binary_session").outputSchema.parse(body); + const matchingOmission = { + ...status, + content: [{ type: "text" as const, text: JSON.stringify(body) }], + }; + // Equal inventories are insufficient when both omit the same canonical tool. + await expect( + verifyCompleteToolCatalog({ + listTools: async () => reduced, + callTool: async () => matchingOmission, + }), + ).rejects.toThrow("invalid canonical tool catalog"); + await expect( + verifyCompleteToolCatalog({ + ...client, + listTools: async () => ({ ...tools, tools: [...tools.tools, omitted] }), + }), + ).rejects.toThrow("invalid canonical tool catalog"); + await expect( + verifyCompleteToolCatalog({ + ...client, + listTools: async () => ({ + ...tools, + tools: [ + { ...omitted, name: "unknown_tool" }, + ...tools.tools.slice(1), + ], + }), + }), + ).rejects.toThrow("invalid canonical tool catalog"); + + for (const kind of ["inputSchema", "outputSchema"] as const) { + const malformed = { + ...omitted, + [kind]: { + ...omitted[kind], + type: "object" as const, + properties: { broken: { type: "invalid-json-schema-type" } }, + }, + }; + await expect( + verifyCompleteToolCatalog({ + ...client, + listTools: async () => ({ + ...tools, + tools: [malformed, ...tools.tools.slice(1)], + }), + }), + ).rejects.toThrow(`invalid ${omitted.name}.${kind}`); + } + await expect( + verifyCompleteToolCatalog({ + ...client, + callTool: async () => matchingOmission, + }), + ).rejects.toThrow("tool inventory diverged from session metadata"); + // Optional output schemas remain valid for clients using that output profile. + await expect( + verifyCompleteToolCatalog({ + ...client, + listTools: async () => ({ + ...tools, + tools: tools.tools.map(({ outputSchema: _output, ...tool }) => tool), + }), + }), + ).resolves.toEqual(canonical); + } finally { + await harness.close(); + } +}); diff --git a/tests/boundary/process/releaseArtifactCanary.test.ts b/tests/boundary/process/releaseArtifactCanary.test.ts index 690e58a120..5ee9c82140 100644 --- a/tests/boundary/process/releaseArtifactCanary.test.ts +++ b/tests/boundary/process/releaseArtifactCanary.test.ts @@ -84,6 +84,7 @@ if (process.argv[2] === "mcp") { for (const file of [ "scripts/verify-release-artifact.mjs", "scripts/lib/release-package-canary.mjs", + "scripts/lib/verify-package-core.mjs", "scripts/lib/mcp-verifier-results.mjs", "scripts/lib/verifier-run.mjs", ]) {