fix(captures): exclude normalized native numeric values (#7764)

scripts/fixtures/generate-mitmproxy-capture.py

@@ -66,6 +66,7 @@ class Generator:
         first.request.path = "http://user:[email protected]/a?token=ordinary#fragment"
         first.response = http.Response.make(200, b"\x00\xfeanswer", [(b"Set-Cookie", b"session=cookie-secret"), (b"Location", b"https://redirect-user:[email protected]/b?token=ordinary")])
         first.metadata = {"big_integer": 9007199254740993, "nonfinite": float("inf"), "isLosslessNumber": True, "value": "ordinary-extension", "response": {"headers": [["Authorization", "ordinary-extension-credential-name"]]}}
+        first.metadata["numeric_spellings"] = {"small": 1e-7, "large": 1e20, "ordinary": 42}
         first.websocket = WebSocketData(messages=[WebSocketMessage(2, True, b"\x00\xfdmessage", 123.5)])
         first.backup()
         first.comment = "caller-selected-fixture"

scripts/verify-web-network-captures.mjs

@@ -206,6 +206,45 @@ try {
     );
     cases++;
   }
+  // Verify Python float spellings through the actual native producer and both public adapters.
+  for (const mode of ["cli", "mcp"]) {
+    for (const marked of [false, true]) {
+      const value = await inspect(mode, {
+        capture_path: join(runtime.path, "flows.mitm"),
+        format: "mitmproxy",
+        sensitive_values: marked ? ["1e-7", "000000"] : [],
+      });
+      for (const record of value.records) {
+        assert.deepEqual(record.reported.metadata.numeric_spellings, {
+          small: marked ? null : 1e-7,
+          large: marked ? null : 1e20,
+          ordinary: 42,
+        });
+        for (const [key, literal] of [
+          ["small", "1e-07"],
+          ["large", "1e+20"],
+        ]) {
+          const pointer = "/metadata/numeric_spellings/" + key;
+          assert.equal(
+            record.numeric_literals.some(
+              (number) =>
+                number.pointer === pointer && number.literal === literal,
+            ),
+            !marked,
+          );
+          assert.equal(
+            record.redactions.some(
+              (redaction) =>
+                redaction.pointer === pointer &&
+                redaction.reason === "explicit-sensitive-value",
+            ),
+            marked,
+          );
+        }
+      }
+      cases++;
+    }
+  }
   for (const sensitive_values of [
     ["REDACTED", "["],
     ["secret", "REDACTED"],

src/browser/history/HistoricalCaptureDecoder.ts

@@ -219,6 +219,7 @@ export class HistoricalCaptureDecoder {
         input.capture_path,
         runtimeResources,
         options?.signal,
+        input.sensitive_values,
       );
       const reply = decodedReply.value;
       encodedReplyBytes = decodedReply.bytes;

src/browser/history/HistoricalCaptureReply.test.ts

@@ -84,3 +84,100 @@ it.each(["{broken", "\uFEFF{}"])(
     });
   },
 );
+
+// Python's wire spelling differs from the JavaScript number retained in Evidence.
+const nativeNumericReply = (literal: string) => {
+  const payload = decodeHarCapture(JSON.stringify(historicalHar()), []);
+  payload.records[0]!.reported = {
+    metadata: {
+      ["__proto__"]: { ["ratio/~"]: [Number(literal)] },
+      ordinary: 42,
+    },
+  };
+  payload.records[0]!.numeric_literals = [
+    {
+      pointer: "/metadata/__proto__/ratio~1~0/0",
+      producer_type: "float",
+      literal,
+    },
+  ];
+  return JSON.stringify({ ok: true, value: payload });
+};
+
+it.each([
+  ["1e-07", "1e-7"],
+  ["1e+20", "000000"],
+] as const)(
+  "excludes the public spelling of native number %s and its numeric sidecar",
+  async (literal, sensitive) => {
+    const bytes = Buffer.from(nativeNumericReply(literal));
+    const file = await replyFile(bytes);
+    const result = await readHistoricalCaptureReply(
+      file.path,
+      "/capture.mitm",
+      file.resources,
+      undefined,
+      [sensitive],
+    );
+    if (!result.value.ok) throw new Error("Expected a capture reply");
+    const record = result.value.value.records[0]!;
+    expect(record.reported).toEqual({
+      metadata: { ["__proto__"]: { ["ratio/~"]: [null] }, ordinary: 42 },
+    });
+    expect(record.numeric_literals).toEqual([]);
+    expect(record.redactions).toContainEqual({
+      pointer: "/metadata/__proto__/ratio~1~0/0",
+      reason: "explicit-sensitive-value",
+    });
+    expect(JSON.stringify(record)).not.toContain(sensitive);
+    expect(result.bytes).toBe(bytes.length);
+    expect(Reflect.get(Object.prototype, "ratio/~")).toBeUndefined();
+  },
+);
+
+it("preserves unmarked native numbers and exact producer literals", async () => {
+  const text = nativeNumericReply("1e-07");
+  const file = await replyFile(Buffer.from(text));
+  const result = await readHistoricalCaptureReply(
+    file.path,
+    "/capture.mitm",
+    file.resources,
+    undefined,
+    ["unrelated"],
+  );
+  expect(result.value).toEqual(JSON.parse(text));
+});
+
+it.each(["container", "record"] as const)(
+  "excludes a normalized number at the %s root",
+  async (kind) => {
+    const payload = decodeHarCapture(JSON.stringify(historicalHar()), []);
+    const projection =
+      kind === "container" ? payload.container : payload.records[0]!;
+    projection.reported = 1e-7;
+    projection.numeric_literals = [
+      { pointer: "", producer_type: "float", literal: "1e-07" },
+    ];
+    projection.redactions = [];
+    const file = await replyFile(
+      Buffer.from(JSON.stringify({ ok: true, value: payload })),
+    );
+    const result = await readHistoricalCaptureReply(
+      file.path,
+      "/capture.mitm",
+      file.resources,
+      undefined,
+      ["1e-7"],
+    );
+    if (!result.value.ok) throw new Error("Expected a capture reply");
+    const retained =
+      kind === "container"
+        ? result.value.value.container
+        : result.value.value.records[0]!;
+    expect(retained.reported).toBeNull();
+    expect(retained.numeric_literals).toEqual([]);
+    expect(retained.redactions).toEqual([
+      { pointer: "", reason: "explicit-sensitive-value" },
+    ]);
+  },
+);

src/browser/history/HistoricalCaptureReply.ts

@@ -1,4 +1,7 @@
 import { z } from "zod";
+import type { JsonValue } from "../../domain/jsonValue.js";
+import { CaptureRedaction, capturePointer } from "./CaptureRedaction.js";
+import type { WebNetworkCaptureRecord } from "../../domain/webNetworkCapture.js";
 import { safeParseUtf8Json } from "../../domain/safeJson.js";
 import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js";
 import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
@@ -45,6 +48,7 @@ export const readHistoricalCaptureReply = async (
   capturePath: string,
   resources: ArtifactResourceScope,
   signal?: AbortSignal,
+  sensitiveValues: readonly string[] = [],
 ) => {
   let finalization: ArtifactReaderFailure | undefined;
   try {
@@ -66,8 +70,14 @@ export const readHistoricalCaptureReply = async (
     }
     const decoded = safeParseUtf8Json(file.bytes);
     if (!decoded.ok) throw decoded.cause;
+    const value = replySchema.parse(decoded.value);
+    if (value.ok && sensitiveValues.length > 0) {
+      excludeNormalizedNumbers(value.value.container, sensitiveValues);
+      for (const record of value.value.records)
+        excludeNormalizedNumbers(record, sensitiveValues);
+    }
     return {
-      value: replySchema.parse(decoded.value),
+      value,
       bytes: file.bytes.length,
       finalization,
     };
@@ -115,3 +125,40 @@ const replyFailureReason = (cause: unknown): string => {
     return `Reply file could not be read (${String(cause.code)}).`;
   return "Reply could not be validated.";
 };
+
+/** The bridge excludes producer literals; this boundary owns their JavaScript spelling. */
+const excludeNormalizedNumbers = (
+  projection: Pick<
+    WebNetworkCaptureRecord,
+    "reported" | "numeric_literals" | "redactions"
+  >,
+  sensitiveValues: readonly string[],
+): void => {
+  const redaction = new CaptureRedaction(sensitiveValues);
+  const excluded = new Set<string>();
+  const visit = (value: JsonValue, pointer: string): JsonValue => {
+    if (typeof value === "number") {
+      if (redaction.text(String(value), pointer) === null) {
+        excluded.add(pointer);
+        return null;
+      }
+    } else if (Array.isArray(value)) {
+      for (let index = 0; index < value.length; index++)
+        value[index] = visit(
+          value[index]!,
+          capturePointer(pointer, String(index)),
+        );
+    } else if (value !== null && typeof value === "object") {
+      for (const key of Object.keys(value))
+        value[key] = visit(value[key]!, capturePointer(pointer, key));
+    }
+    return value;
+  };
+  // The schema-validated reply is exclusively owned here, before Evidence retention.
+  projection.reported = visit(projection.reported, "");
+  if (excluded.size === 0) return;
+  projection.numeric_literals = projection.numeric_literals.filter(
+    (number) => !excluded.has(number.pointer),
+  );
+  projection.redactions.push(...redaction.redactions);
+};

tests/boundary/browser/historicalCaptureDecoder.test.ts

@@ -1,3 +1,4 @@
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
 import type { AnalysisError } from "../../../src/domain/analysisErrorBase.js";
 import { ProviderProcessSupervisor } from "../../../src/process/ProviderProcess.js";
 import { createDeferred } from "../../fixtures/binarySession.js";
@@ -1331,3 +1332,69 @@ it.each([
     }
   },
 );
+
+it("excludes normalized native numbers before retaining a completed decoder reply", async () => {
+  const root = await createTestTempDirectory("rea-native-number-redaction-");
+  const capturePath = join(root, "capture.mitm");
+  const original = "controlled native decoder input";
+  await writeFile(capturePath, original);
+  const payload = decodeHarCapture(JSON.stringify(historicalHar()), []);
+  payload.decoder = MITMPROXY_CAPTURE_PROVIDER_IDENTITY;
+  const record = payload.records[0]!;
+  record.reported = { metadata: { small: 1e-7, large: 1e20, ordinary: 42 } };
+  record.binary_fields = [];
+  record.redactions = [];
+  record.numeric_literals = [
+    { pointer: "/metadata/small", producer_type: "float", literal: "1e-07" },
+    { pointer: "/metadata/large", producer_type: "float", literal: "1e+20" },
+  ];
+  const decoder = new HistoricalCaptureDecoder(
+    [
+      {
+        format: "mitmproxy",
+        identity: MITMPROXY_CAPTURE_PROVIDER_IDENTITY,
+        command: async (_, runtime) => ({
+          command: process.execPath,
+          arguments: [
+            "-e",
+            "require('node:fs').writeFileSync(process.argv[1], process.argv[2])",
+            join(runtime, "reply.json"),
+            JSON.stringify({ ok: true, value: payload }),
+          ],
+        }),
+      },
+    ],
+    process.env,
+  );
+  onTestFinished(() => decoder.close());
+  for (const marked of [false, true]) {
+    const result = await decoder.inspect(
+      inspectWebNetworkCaptureInputSchema.parse({
+        capture_path: capturePath,
+        format: "mitmproxy",
+        sensitive_values: marked ? ["1e-7", "000000"] : [],
+      }),
+    );
+    if (!result.ok) throw result.error;
+    const retained = result.value.records[0]!;
+    expect(retained.reported).toEqual({
+      metadata: {
+        small: marked ? null : 1e-7,
+        large: marked ? null : 1e20,
+        ordinary: 42,
+      },
+    });
+    expect(retained.numeric_literals).toEqual(
+      marked ? [] : record.numeric_literals,
+    );
+    expect(retained.redactions).toEqual(
+      marked
+        ? [
+            { pointer: "/metadata/small", reason: "explicit-sensitive-value" },
+            { pointer: "/metadata/large", reason: "explicit-sensitive-value" },
+          ]
+        : [],
+    );
+    expect(await readFile(capturePath, "utf8")).toBe(original);
+  }
+});