fix(artifacts): compare leaf inventories using explicit coverage
Classify coverage once at inventory production and require it in current outputs. Normalize older retained Evidence at the inventory parser, reject inconsistent page statuses, and remove comparison heuristics over limitation prose. Validation: public CLI-to-MCP plist comparison, changed-behavior and unknowns workflow; boundary checks for legacy Evidence, partial coverage, page disagreement, content and executable changes; generated docs checks. Fixes #2173.
This commit is contained in:
1 parent
8d4d631606
commit
ed485a3dc8
11 files changed
+379
-59
No files matched your search
@@ -0,0 +1,84 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js";
|
||||
import { parseEvidence } from "../../src/domain/evidence.js";
|
||||
import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js";
|
||||
import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const cli = fileURLToPath(new URL("../../scripts/rea.mjs", import.meta.url));
|
||||
const plist =
|
||||
'<?xml version="1.0"?><plist version="1.0"><dict><key>label</key><string>hello</string></dict></plist>';
|
||||
|
||||
it("compares identical complete plist inventories across CLI and MCP and creates no unknowns", async () => {
|
||||
const root = await createTestTempDirectory("rea-plist-compare-acceptance-");
|
||||
const leftPath = join(root, "left.plist");
|
||||
const rightPath = join(root, "right.plist");
|
||||
await Promise.all([writeFile(leftPath, plist), writeFile(rightPath, plist)]);
|
||||
|
||||
const cliResult = await execute(process.execPath, [
|
||||
cli,
|
||||
"inspect-artifact",
|
||||
leftPath,
|
||||
"--json",
|
||||
]);
|
||||
const cliInspection = artifactInspectionResultSchema.parse(
|
||||
parseEvidence(JSON.parse(cliResult.stdout)).normalized_result,
|
||||
);
|
||||
const cliInventory = cliInspection.substeps[0]?.evidence;
|
||||
if (cliInventory === undefined)
|
||||
throw new Error("CLI inspection did not retain inventory Evidence");
|
||||
expect(cliInventory.normalized_result).toMatchObject({
|
||||
coverage_status: "complete",
|
||||
limitations: ["Artifact has no child container entries."],
|
||||
});
|
||||
|
||||
const { call } = await connectLocalToolsMcp();
|
||||
const opened = await call("open_binary", { path: rightPath });
|
||||
expect(opened.isError, JSON.stringify(opened)).not.toBe(true);
|
||||
const inspected = await call("inspect_artifact", {});
|
||||
expect(inspected.isError, JSON.stringify(inspected)).not.toBe(true);
|
||||
const mcpInspection = artifactInspectionResultSchema.parse(
|
||||
parseEvidence(inspected.structuredContent).normalized_result,
|
||||
);
|
||||
const mcpInventory = mcpInspection.substeps[0]?.evidence;
|
||||
if (mcpInventory === undefined)
|
||||
throw new Error("MCP inspection did not retain inventory Evidence");
|
||||
expect(mcpInventory.normalized_result).toMatchObject({
|
||||
coverage_status: "complete",
|
||||
limitations: ["Artifact has no child container entries."],
|
||||
});
|
||||
|
||||
const compared = await call("compare_artifacts", {
|
||||
left: cliInventory,
|
||||
right: mcpInventory,
|
||||
});
|
||||
expect(compared.isError, JSON.stringify(compared)).not.toBe(true);
|
||||
const comparison = parseEvidence(compared.structuredContent);
|
||||
expect(comparison.normalized_result).toMatchObject({
|
||||
status: "unchanged",
|
||||
summary: { unchanged: 1, unknown: 0 },
|
||||
changes: [],
|
||||
});
|
||||
|
||||
const behavior = await call("find_changed_behavior", {
|
||||
comparisons: [comparison],
|
||||
});
|
||||
expect(behavior.isError, JSON.stringify(behavior)).not.toBe(true);
|
||||
expect(behavior.structuredContent).toMatchObject({
|
||||
normalized_result: {
|
||||
behavior_status: "unknown",
|
||||
findings: { items: [] },
|
||||
},
|
||||
});
|
||||
const unknowns = await call("list_unknowns", {
|
||||
domain: "artifact-comparison",
|
||||
});
|
||||
expect(unknowns.isError, JSON.stringify(unknowns)).not.toBe(true);
|
||||
expect(unknowns.structuredContent).toMatchObject({ result: { items: [] } });
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { chmod, mkdir, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
artifactComparisonResultSchema,
|
||||
compareArtifacts,
|
||||
} from "../../../src/domain/artifactComparison.js";
|
||||
import { parseArtifactInventoryEvidence } from "../../../src/domain/artifactInventoryEvidence.js";
|
||||
import { createEvidence } from "../../../src/domain/evidence.js";
|
||||
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
|
||||
|
||||
@@ -61,6 +62,184 @@ const changedArtifactComparison = async () => {
|
||||
};
|
||||
};
|
||||
|
||||
describe("leaf artifact comparison", () => {
|
||||
it("accepts persisted complete leaf inventory Evidence without coverage_status", async () => {
|
||||
const root = await createTestTempDirectory("rea-artifact-plist-compare-");
|
||||
const leftPath = join(root, "left.plist");
|
||||
const rightPath = join(root, "right.plist");
|
||||
const xml =
|
||||
'<?xml version="1.0"?><plist version="1.0"><dict><key>label</key><string>hello</string></dict></plist>';
|
||||
await Promise.all([writeFile(leftPath, xml), writeFile(rightPath, xml)]);
|
||||
const [left, right] = await Promise.all([
|
||||
observe(leftPath),
|
||||
observe(rightPath),
|
||||
]);
|
||||
// Persisted inventory Evidence predates coverage_status. The known leaf
|
||||
// notice remains complete when normalized at the inventory boundary.
|
||||
const parsed = artifactInventoryResultSchema.parse(left.normalized_result);
|
||||
const { coverage_status: _coverageStatus, ...legacyInventory } = parsed;
|
||||
const legacy = createEvidence(
|
||||
{
|
||||
path: left.subject!.local_path,
|
||||
sha256: left.subject!.digest.sha256,
|
||||
format: left.subject!.format,
|
||||
},
|
||||
PROVIDER,
|
||||
{
|
||||
operation: "inventory_artifact",
|
||||
parameters: {},
|
||||
result: jsonValueSchema.parse(legacyInventory),
|
||||
confidence: "observed",
|
||||
authority: "shipped-artifact",
|
||||
},
|
||||
);
|
||||
expect(compareArtifacts(legacy, right)).toMatchObject({
|
||||
status: "unchanged",
|
||||
summary: { unchanged: 1, unknown: 0 },
|
||||
changes: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps content and executable changes visible for complete plist roots", async () => {
|
||||
const root = await createTestTempDirectory("rea-artifact-plist-deltas-");
|
||||
const leftPath = join(root, "left.plist");
|
||||
const rightPath = join(root, "right.plist");
|
||||
const plist = (label: string) =>
|
||||
`<?xml version="1.0"?><plist version="1.0"><dict><key>label</key><string>${label}</string></dict></plist>`;
|
||||
await Promise.all([
|
||||
writeFile(leftPath, plist("hello"), { mode: 0o644 }),
|
||||
writeFile(rightPath, plist("world"), { mode: 0o644 }),
|
||||
]);
|
||||
const [left, changed] = await Promise.all([
|
||||
observe(leftPath),
|
||||
observe(rightPath),
|
||||
]);
|
||||
expect(compareArtifacts(left, changed)).toMatchObject({
|
||||
status: "changed",
|
||||
summary: { changed: 1, unknown: 0 },
|
||||
changes: [
|
||||
expect.objectContaining({
|
||||
logical_path: ".",
|
||||
classification: "changed",
|
||||
dimensions: expect.arrayContaining(["content"]),
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
if (process.platform !== "win32") {
|
||||
await writeFile(rightPath, plist("hello"), { mode: 0o644 });
|
||||
await chmod(rightPath, 0o755);
|
||||
const executable = await observe(rightPath);
|
||||
expect(compareArtifacts(left, executable)).toMatchObject({
|
||||
status: "changed",
|
||||
summary: { changed: 1, unknown: 0 },
|
||||
changes: [
|
||||
expect.objectContaining({
|
||||
logical_path: ".",
|
||||
classification: "changed",
|
||||
dimensions: expect.arrayContaining(["executable"]),
|
||||
}),
|
||||
],
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("artifact inventory coverage", () => {
|
||||
it("keeps explicit and legacy unknown coverage fail-closed", async () => {
|
||||
const root = await createTestTempDirectory("rea-artifact-coverage-");
|
||||
const path = join(root, "complete.plist");
|
||||
await writeFile(
|
||||
path,
|
||||
'<?xml version="1.0"?><plist version="1.0"><dict/></plist>',
|
||||
);
|
||||
const complete = await observe(path);
|
||||
const graph = artifactInventoryResultSchema.parse(
|
||||
complete.normalized_result,
|
||||
);
|
||||
const partial = createEvidence(
|
||||
{
|
||||
path: complete.subject!.local_path,
|
||||
sha256: complete.subject!.digest.sha256,
|
||||
format: complete.subject!.format,
|
||||
},
|
||||
PROVIDER,
|
||||
{
|
||||
operation: "inventory_artifact",
|
||||
parameters: {},
|
||||
result: jsonValueSchema.parse({ ...graph, coverage_status: "partial" }),
|
||||
confidence: "observed",
|
||||
authority: "shipped-artifact",
|
||||
},
|
||||
);
|
||||
expect(compareArtifacts(partial, partial)).toMatchObject({
|
||||
status: "unknown",
|
||||
summary: { unchanged: 0, unknown: 1 },
|
||||
});
|
||||
|
||||
const { coverage_status: _coverageStatus, ...legacyGraph } = graph;
|
||||
const unknownLegacy = createEvidence(
|
||||
{
|
||||
path: complete.subject!.local_path,
|
||||
sha256: complete.subject!.digest.sha256,
|
||||
format: complete.subject!.format,
|
||||
},
|
||||
PROVIDER,
|
||||
{
|
||||
operation: "inventory_artifact",
|
||||
parameters: {},
|
||||
result: jsonValueSchema.parse({
|
||||
...legacyGraph,
|
||||
limitations: ["Producer omitted unexamined child entries."],
|
||||
}),
|
||||
confidence: "observed",
|
||||
authority: "shipped-artifact",
|
||||
},
|
||||
);
|
||||
expect(compareArtifacts(unknownLegacy, unknownLegacy)).toMatchObject({
|
||||
status: "unknown",
|
||||
summary: { unchanged: 0, unknown: 1 },
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects inventory fragments that disagree on producer coverage", async () => {
|
||||
const parent = await createTestTempDirectory("rea-artifact-page-coverage-");
|
||||
const path = join(parent, "fixture.app");
|
||||
await mkdir(path);
|
||||
await writeFile(join(path, "main.js"), "main();");
|
||||
const source = await observe(path);
|
||||
const fragments = fragmentInventoryEvidence(source, 2);
|
||||
const firstFragment = fragments[0];
|
||||
const fragment = fragments[1];
|
||||
if (firstFragment === undefined || fragment === undefined)
|
||||
throw new Error("Missing test fragment");
|
||||
const fragmentGraph = artifactInventoryResultSchema.parse(
|
||||
fragment.normalized_result,
|
||||
);
|
||||
const mixed = createEvidence(
|
||||
{
|
||||
path: fragment.subject!.local_path,
|
||||
sha256: fragment.subject!.digest.sha256,
|
||||
format: fragment.subject!.format,
|
||||
},
|
||||
PROVIDER,
|
||||
{
|
||||
operation: "inventory_artifact",
|
||||
parameters: fragment.parameters,
|
||||
result: jsonValueSchema.parse({
|
||||
...fragmentGraph,
|
||||
coverage_status: "partial",
|
||||
}),
|
||||
confidence: "derived",
|
||||
authority: "shipped-artifact",
|
||||
},
|
||||
);
|
||||
expect(() =>
|
||||
parseArtifactInventoryEvidence([firstFragment, mixed]),
|
||||
).toThrow("Artifact inventory pages disagree on coverage status");
|
||||
});
|
||||
});
|
||||
|
||||
describe("artifact comparison", () => {
|
||||
it("classifies deterministic path changes and cites both inventories", async () => {
|
||||
const firstObservation = await changedArtifactComparison();
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
artifactInventoryResultSchema,
|
||||
type ArtifactInventoryResult,
|
||||
} from "../../../src/domain/artifactGraph.js";
|
||||
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
|
||||
import {
|
||||
analysisViewResultSchema,
|
||||
type AnalysisViewResult,
|
||||
@@ -266,7 +267,7 @@ const assertHistoricalInventory = async (
|
||||
{
|
||||
operation: "inventory_artifact",
|
||||
parameters: {},
|
||||
result: legacyInventory,
|
||||
result: jsonValueSchema.parse(legacyInventory),
|
||||
},
|
||||
);
|
||||
const legacyInspection = createArtifactInspection(legacySource);
|
||||
@@ -490,7 +491,10 @@ it("keeps complete ZIP output and reads retained inventory views after the sourc
|
||||
{
|
||||
operation: "inventory_artifact",
|
||||
parameters: {},
|
||||
result: { ...graph, occurrences: graph.occurrences.slice(0, 1) },
|
||||
result: jsonValueSchema.parse({
|
||||
...graph,
|
||||
occurrences: graph.occurrences.slice(0, 1),
|
||||
}),
|
||||
},
|
||||
);
|
||||
const partialReply = await client.callTool({
|
||||
|
||||
Reference in new issue
Block a user