From f483eab5435b0cbbc41b49959651cd719535bc70 Mon Sep 17 00:00:00 2001 From: ShuaixinHuang <1836656514@qq.com> Date: Wed, 28 Oct 2026 18:46:30 +0800 Subject: [PATCH] fix(captures): exclude normalized native numeric values (#7764) --- .../fixtures/generate-mitmproxy-capture.py | 1 + scripts/verify-web-network-captures.mjs | 39 ++++++++ .../history/HistoricalCaptureDecoder.ts | 1 + .../history/HistoricalCaptureReply.test.ts | 97 +++++++++++++++++++ src/browser/history/HistoricalCaptureReply.ts | 49 +++++++++- .../browser/historicalCaptureDecoder.test.ts | 67 +++++++++++++ 6 files changed, 253 insertions(+), 1 deletion(-) diff --git a/scripts/fixtures/generate-mitmproxy-capture.py b/scripts/fixtures/generate-mitmproxy-capture.py index 880c5e690b..cf539a65f5 100644 --- a/scripts/fixtures/generate-mitmproxy-capture.py +++ b/scripts/fixtures/generate-mitmproxy-capture.py @@ -66,6 +66,7 @@ class Generator: first.request.path = "http://user:password@example.test/a?token=ordinary#fragment" first.response = http.Response.make(200, b"\x00\xfeanswer", [(b"Set-Cookie", b"session=cookie-secret"), (b"Location", b"https://redirect-user:redirect-password@example.test/b?token=ordinary")]) first.metadata = {"big_integer": 9007199254740993, "nonfinite": float("inf"), "isLosslessNumber": True, "value": "ordinary-extension", "response": {"headers": [["Authorization", "ordinary-extension-credential-name"]]}} + first.metadata["numeric_spellings"] = {"small": 1e-7, "large": 1e20, "ordinary": 42} first.websocket = WebSocketData(messages=[WebSocketMessage(2, True, b"\x00\xfdmessage", 123.5)]) first.backup() first.comment = "caller-selected-fixture" diff --git a/scripts/verify-web-network-captures.mjs b/scripts/verify-web-network-captures.mjs index 3c8d813f30..5ae44e81c3 100644 --- a/scripts/verify-web-network-captures.mjs +++ b/scripts/verify-web-network-captures.mjs @@ -206,6 +206,45 @@ try { ); cases++; } + // Verify Python float spellings through the actual native producer and both public adapters. + for (const mode of ["cli", "mcp"]) { + for (const marked of [false, true]) { + const value = await inspect(mode, { + capture_path: join(runtime.path, "flows.mitm"), + format: "mitmproxy", + sensitive_values: marked ? ["1e-7", "000000"] : [], + }); + for (const record of value.records) { + assert.deepEqual(record.reported.metadata.numeric_spellings, { + small: marked ? null : 1e-7, + large: marked ? null : 1e20, + ordinary: 42, + }); + for (const [key, literal] of [ + ["small", "1e-07"], + ["large", "1e+20"], + ]) { + const pointer = "/metadata/numeric_spellings/" + key; + assert.equal( + record.numeric_literals.some( + (number) => + number.pointer === pointer && number.literal === literal, + ), + !marked, + ); + assert.equal( + record.redactions.some( + (redaction) => + redaction.pointer === pointer && + redaction.reason === "explicit-sensitive-value", + ), + marked, + ); + } + } + cases++; + } + } for (const sensitive_values of [ ["REDACTED", "["], ["secret", "REDACTED"], diff --git a/src/browser/history/HistoricalCaptureDecoder.ts b/src/browser/history/HistoricalCaptureDecoder.ts index 89057e7c80..421ab8b21f 100644 --- a/src/browser/history/HistoricalCaptureDecoder.ts +++ b/src/browser/history/HistoricalCaptureDecoder.ts @@ -219,6 +219,7 @@ export class HistoricalCaptureDecoder { input.capture_path, runtimeResources, options?.signal, + input.sensitive_values, ); const reply = decodedReply.value; encodedReplyBytes = decodedReply.bytes; diff --git a/src/browser/history/HistoricalCaptureReply.test.ts b/src/browser/history/HistoricalCaptureReply.test.ts index 684c50a693..1fd6453246 100644 --- a/src/browser/history/HistoricalCaptureReply.test.ts +++ b/src/browser/history/HistoricalCaptureReply.test.ts @@ -84,3 +84,100 @@ it.each(["{broken", "\uFEFF{}"])( }); }, ); + +// Python's wire spelling differs from the JavaScript number retained in Evidence. +const nativeNumericReply = (literal: string) => { + const payload = decodeHarCapture(JSON.stringify(historicalHar()), []); + payload.records[0]!.reported = { + metadata: { + ["__proto__"]: { ["ratio/~"]: [Number(literal)] }, + ordinary: 42, + }, + }; + payload.records[0]!.numeric_literals = [ + { + pointer: "/metadata/__proto__/ratio~1~0/0", + producer_type: "float", + literal, + }, + ]; + return JSON.stringify({ ok: true, value: payload }); +}; + +it.each([ + ["1e-07", "1e-7"], + ["1e+20", "000000"], +] as const)( + "excludes the public spelling of native number %s and its numeric sidecar", + async (literal, sensitive) => { + const bytes = Buffer.from(nativeNumericReply(literal)); + const file = await replyFile(bytes); + const result = await readHistoricalCaptureReply( + file.path, + "/capture.mitm", + file.resources, + undefined, + [sensitive], + ); + if (!result.value.ok) throw new Error("Expected a capture reply"); + const record = result.value.value.records[0]!; + expect(record.reported).toEqual({ + metadata: { ["__proto__"]: { ["ratio/~"]: [null] }, ordinary: 42 }, + }); + expect(record.numeric_literals).toEqual([]); + expect(record.redactions).toContainEqual({ + pointer: "/metadata/__proto__/ratio~1~0/0", + reason: "explicit-sensitive-value", + }); + expect(JSON.stringify(record)).not.toContain(sensitive); + expect(result.bytes).toBe(bytes.length); + expect(Reflect.get(Object.prototype, "ratio/~")).toBeUndefined(); + }, +); + +it("preserves unmarked native numbers and exact producer literals", async () => { + const text = nativeNumericReply("1e-07"); + const file = await replyFile(Buffer.from(text)); + const result = await readHistoricalCaptureReply( + file.path, + "/capture.mitm", + file.resources, + undefined, + ["unrelated"], + ); + expect(result.value).toEqual(JSON.parse(text)); +}); + +it.each(["container", "record"] as const)( + "excludes a normalized number at the %s root", + async (kind) => { + const payload = decodeHarCapture(JSON.stringify(historicalHar()), []); + const projection = + kind === "container" ? payload.container : payload.records[0]!; + projection.reported = 1e-7; + projection.numeric_literals = [ + { pointer: "", producer_type: "float", literal: "1e-07" }, + ]; + projection.redactions = []; + const file = await replyFile( + Buffer.from(JSON.stringify({ ok: true, value: payload })), + ); + const result = await readHistoricalCaptureReply( + file.path, + "/capture.mitm", + file.resources, + undefined, + ["1e-7"], + ); + if (!result.value.ok) throw new Error("Expected a capture reply"); + const retained = + kind === "container" + ? result.value.value.container + : result.value.value.records[0]!; + expect(retained.reported).toBeNull(); + expect(retained.numeric_literals).toEqual([]); + expect(retained.redactions).toEqual([ + { pointer: "", reason: "explicit-sensitive-value" }, + ]); + }, +); diff --git a/src/browser/history/HistoricalCaptureReply.ts b/src/browser/history/HistoricalCaptureReply.ts index a4b7eadd27..9f9f035862 100644 --- a/src/browser/history/HistoricalCaptureReply.ts +++ b/src/browser/history/HistoricalCaptureReply.ts @@ -1,4 +1,7 @@ import { z } from "zod"; +import type { JsonValue } from "../../domain/jsonValue.js"; +import { CaptureRedaction, capturePointer } from "./CaptureRedaction.js"; +import type { WebNetworkCaptureRecord } from "../../domain/webNetworkCapture.js"; import { safeParseUtf8Json } from "../../domain/safeJson.js"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; @@ -45,6 +48,7 @@ export const readHistoricalCaptureReply = async ( capturePath: string, resources: ArtifactResourceScope, signal?: AbortSignal, + sensitiveValues: readonly string[] = [], ) => { let finalization: ArtifactReaderFailure | undefined; try { @@ -66,8 +70,14 @@ export const readHistoricalCaptureReply = async ( } const decoded = safeParseUtf8Json(file.bytes); if (!decoded.ok) throw decoded.cause; + const value = replySchema.parse(decoded.value); + if (value.ok && sensitiveValues.length > 0) { + excludeNormalizedNumbers(value.value.container, sensitiveValues); + for (const record of value.value.records) + excludeNormalizedNumbers(record, sensitiveValues); + } return { - value: replySchema.parse(decoded.value), + value, bytes: file.bytes.length, finalization, }; @@ -115,3 +125,40 @@ const replyFailureReason = (cause: unknown): string => { return `Reply file could not be read (${String(cause.code)}).`; return "Reply could not be validated."; }; + +/** The bridge excludes producer literals; this boundary owns their JavaScript spelling. */ +const excludeNormalizedNumbers = ( + projection: Pick< + WebNetworkCaptureRecord, + "reported" | "numeric_literals" | "redactions" + >, + sensitiveValues: readonly string[], +): void => { + const redaction = new CaptureRedaction(sensitiveValues); + const excluded = new Set(); + const visit = (value: JsonValue, pointer: string): JsonValue => { + if (typeof value === "number") { + if (redaction.text(String(value), pointer) === null) { + excluded.add(pointer); + return null; + } + } else if (Array.isArray(value)) { + for (let index = 0; index < value.length; index++) + value[index] = visit( + value[index]!, + capturePointer(pointer, String(index)), + ); + } else if (value !== null && typeof value === "object") { + for (const key of Object.keys(value)) + value[key] = visit(value[key]!, capturePointer(pointer, key)); + } + return value; + }; + // The schema-validated reply is exclusively owned here, before Evidence retention. + projection.reported = visit(projection.reported, ""); + if (excluded.size === 0) return; + projection.numeric_literals = projection.numeric_literals.filter( + (number) => !excluded.has(number.pointer), + ); + projection.redactions.push(...redaction.redactions); +}; diff --git a/tests/boundary/browser/historicalCaptureDecoder.test.ts b/tests/boundary/browser/historicalCaptureDecoder.test.ts index 5f0a7e8452..bdb757c4c5 100644 --- a/tests/boundary/browser/historicalCaptureDecoder.test.ts +++ b/tests/boundary/browser/historicalCaptureDecoder.test.ts @@ -1,3 +1,4 @@ +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import type { AnalysisError } from "../../../src/domain/analysisErrorBase.js"; import { ProviderProcessSupervisor } from "../../../src/process/ProviderProcess.js"; import { createDeferred } from "../../fixtures/binarySession.js"; @@ -1331,3 +1332,69 @@ it.each([ } }, ); + +it("excludes normalized native numbers before retaining a completed decoder reply", async () => { + const root = await createTestTempDirectory("rea-native-number-redaction-"); + const capturePath = join(root, "capture.mitm"); + const original = "controlled native decoder input"; + await writeFile(capturePath, original); + const payload = decodeHarCapture(JSON.stringify(historicalHar()), []); + payload.decoder = MITMPROXY_CAPTURE_PROVIDER_IDENTITY; + const record = payload.records[0]!; + record.reported = { metadata: { small: 1e-7, large: 1e20, ordinary: 42 } }; + record.binary_fields = []; + record.redactions = []; + record.numeric_literals = [ + { pointer: "/metadata/small", producer_type: "float", literal: "1e-07" }, + { pointer: "/metadata/large", producer_type: "float", literal: "1e+20" }, + ]; + const decoder = new HistoricalCaptureDecoder( + [ + { + format: "mitmproxy", + identity: MITMPROXY_CAPTURE_PROVIDER_IDENTITY, + command: async (_, runtime) => ({ + command: process.execPath, + arguments: [ + "-e", + "require('node:fs').writeFileSync(process.argv[1], process.argv[2])", + join(runtime, "reply.json"), + JSON.stringify({ ok: true, value: payload }), + ], + }), + }, + ], + process.env, + ); + onTestFinished(() => decoder.close()); + for (const marked of [false, true]) { + const result = await decoder.inspect( + inspectWebNetworkCaptureInputSchema.parse({ + capture_path: capturePath, + format: "mitmproxy", + sensitive_values: marked ? ["1e-7", "000000"] : [], + }), + ); + if (!result.ok) throw result.error; + const retained = result.value.records[0]!; + expect(retained.reported).toEqual({ + metadata: { + small: marked ? null : 1e-7, + large: marked ? null : 1e20, + ordinary: 42, + }, + }); + expect(retained.numeric_literals).toEqual( + marked ? [] : record.numeric_literals, + ); + expect(retained.redactions).toEqual( + marked + ? [ + { pointer: "/metadata/small", reason: "explicit-sensitive-value" }, + { pointer: "/metadata/large", reason: "explicit-sensitive-value" }, + ] + : [], + ); + expect(await readFile(capturePath, "utf8")).toBe(original); + } +});