rea/src/python/PythonPycParser.ts

import { createHash } from "node:crypto";
import {
  PYTHON_PYC_CHECK_SOURCE,
  PYTHON_PYC_CONST_PREVIEW_LIMIT,
  PYTHON_PYC_HASH_BASED,
  PYTHON_PYC_HEADER_BYTES,
  PYTHON_PYC_INT_PREVIEW_BITS,
  pythonPycMagicEntry,
  type InspectPythonPycInput,
  type PythonPyc,
  type PythonPycCodeLayout,
  type PythonPycCodeObject,
  type PythonPycConst,
} from "../domain/python/pythonPyc.js";
import {
  PythonPycFormatFailure,
  PythonPycResourceFailure,
} from "./PythonPyc.errors.js";

const FLAG_REF = 0x80;
const TYPE_NULL = 0x30;
const TYPE_NONE = 0x4e;
const TYPE_FALSE = 0x46;
const TYPE_TRUE = 0x54;
const TYPE_STOPITER = 0x53;
const TYPE_ELLIPSIS = 0x2e;
const TYPE_INT = 0x69;
const TYPE_FLOAT_TEXT = 0x66;
const TYPE_FLOAT = 0x67;
const TYPE_COMPLEX_TEXT = 0x78;
const TYPE_COMPLEX = 0x79;
const TYPE_LONG = 0x6c;
const TYPE_BYTES = 0x73;
const TYPE_INTERNED = 0x74;
const TYPE_REF = 0x72;
const TYPE_TUPLE = 0x28;
const TYPE_LIST = 0x5b;
const TYPE_DICT = 0x7b;
const TYPE_CODE = 0x63;
const TYPE_UNICODE = 0x75;
const TYPE_SET = 0x3c;
const TYPE_FROZENSET = 0x3e;
const TYPE_ASCII = 0x61;
const TYPE_ASCII_INTERNED = 0x41;
const TYPE_SMALL_TUPLE = 0x29;
const TYPE_SHORT_ASCII = 0x7a;
const TYPE_SHORT_ASCII_INTERNED = 0x5a;
const TYPES_WITHOUT_REFERENCE_SLOTS = new Set([
  TYPE_NULL,
  TYPE_NONE,
  TYPE_FALSE,
  TYPE_TRUE,
  TYPE_STOPITER,
  TYPE_ELLIPSIS,
]);
/** CPython `CO_VARARGS` / `CO_VARKEYWORDS`. Each consumes one local name. */
const CO_VARARGS = 0x04;
const CO_VARKEYWORDS = 0x08;
/** CPython `_PyLocals_Kind` bit counted by `_PyCode_Validate` as a local. */
const CO_FAST_LOCAL = 0x20;

type MarshalValue =
  | { readonly kind: "null" }
  | { readonly kind: "none" }
  | { readonly kind: "bool"; readonly value: boolean }
  | { readonly kind: "ellipsis" }
  | { readonly kind: "stopiter" }
  | { readonly kind: "int"; readonly value: bigint; readonly bits: number }
  | { readonly kind: "float"; readonly value: number }
  | { readonly kind: "complex"; readonly real: number; readonly imag: number }
  | { readonly kind: "bytes"; readonly value: Buffer }
  | {
      readonly kind: "str";
      readonly value: string;
      readonly bytes: number;
      /** Marshal bytes. JavaScript strings merge lone surrogates with pairs. */
      readonly raw: Buffer;
    }
  | { readonly kind: "tuple"; readonly items: MarshalValue[] }
  | { readonly kind: "list"; readonly items: MarshalValue[] }
  | { readonly kind: "set"; readonly items: MarshalValue[] }
  | { readonly kind: "frozenset"; readonly items: MarshalValue[] }
  | {
      readonly kind: "dict";
      readonly items: [MarshalValue, MarshalValue][];
    }
  | { readonly kind: "code"; readonly value: CodeNode };

/**
 * A decoded code object. `consts` and `nested` stay empty until the marshal
 * graph is complete: a container can refer to itself before its items exist.
 * TYPE_REF children are the same node, not copies.
 */
type CodeNode = Omit<PythonPycCodeObject, "nested_code" | "consts"> & {
  consts: PythonPycConst[];
  nested: CodeNode[];
};

const EMPTY_CODES: readonly CodeNode[] = [];

type KeyMode = "hash" | "const";

/**
 * Equality token plus the size of the expanded CPython-style key.
 * Container tokens are interned so a reference chain does not copy every
 * ancestor into the next string. `expanded` is still the size that string
 * would have, so a repeated child cannot double without bound.
 */
type ResolvedKey = {
  readonly token: string;
  readonly expanded: number;
};

type KeyFrame = {
  value: MarshalValue;
  index: number;
  parts: string[];
  expanded: number[];
  failed: boolean;
};

const isContainer = (value: MarshalValue): boolean =>
  value.kind === "tuple" ||
  value.kind === "list" ||
  value.kind === "set" ||
  value.kind === "frozenset" ||
  value.kind === "dict";

const containerChildren = (value: MarshalValue): readonly MarshalValue[] => {
  if (value.kind === "dict") {
    const children: MarshalValue[] = [];
    for (const [key, mapped] of value.items) children.push(key, mapped);
    return children;
  }
  if (
    value.kind === "tuple" ||
    value.kind === "list" ||
    value.kind === "set" ||
    value.kind === "frozenset"
  )
    return value.items;
  return [];
};

/**
 * Fields CPython uses for code equality that the public report does not keep.
 * `consts` stays the decoded graph so a code key can follow Python's rules
 * without rebuilding marshal text.
 */
type CodeEquality = {
  readonly consts: readonly MarshalValue[];
  readonly linetable: string;
  readonly exceptiontable: string;
};

const malformed = (message: string): never => {
  throw new PythonPycFormatFailure("malformed", message);
};

/** Locals CPython's 3.11+ validator counts toward `co_varnames`. */
const fastLocalCount = (kinds: Buffer): number => {
  let count = 0;
  for (let index = 0; index < kinds.length; index += 1) {
    const kind = kinds[index] ?? 0;
    if ((kind & CO_FAST_LOCAL) !== 0) count += 1;
  }
  return count;
};

const unsupported = (message: string): never => {
  throw new PythonPycFormatFailure("unsupported", message);
};

const digests = new WeakMap<Buffer, string>();
const sha256 = (bytes: Buffer): string => {
  const cached = digests.get(bytes);
  if (cached !== undefined) return cached;
  const digest = createHash("sha256").update(bytes).digest("hex");
  digests.set(bytes, digest);
  return digest;
};

/** Decode one bounded CPython 3.7–3.13 pyc without executing bytecode. */
export const parsePythonPyc = (
  bytes: Buffer,
  artifact: { path: string; sha256: string; bytes: number },
  input: InspectPythonPycInput,
): PythonPyc => {
  if (bytes.length !== artifact.bytes)
    malformed("Selected artifact length disagrees with the retained bytes.");
  if (bytes.length < 4) malformed("Selected file is shorter than a pyc magic.");
  if (bytes[2] !== 0x0d || bytes[3] !== 0x0a)
    malformed("Pyc magic is not a little-endian integer followed by CR LF.");
  const magic = bytes.readUInt16LE(0);
  const mapped = pythonPycMagicEntry(magic);
  if (mapped !== null && mapped.header_bytes !== PYTHON_PYC_HEADER_BYTES)
    return unsupported(
      `Pyc magic ${magic} uses an unsupported ${mapped.header_bytes}-byte CPython header.`,
    );
  if (bytes.length < PYTHON_PYC_HEADER_BYTES)
    malformed(
      "Selected file is shorter than a 16-byte CPython 3.7+ pyc header.",
    );
  if (mapped === null)
    return unsupported(
      `Pyc magic ${magic} is outside the pinned CPython 3.7–3.13 table.`,
    );
  const flags = bytes.readUInt32LE(4);
  const hashBased = (flags & PYTHON_PYC_HASH_BASED) !== 0;
  const header = {
    magic,
    python_version: mapped.python_version,
    code_layout: mapped.code_layout,
    flags,
    hash_based: hashBased,
    check_source: (flags & PYTHON_PYC_CHECK_SOURCE) !== 0,
    source_hash_hex: hashBased ? bytes.subarray(8, 16).toString("hex") : null,
    source_mtime: hashBased ? null : bytes.readUInt32LE(8),
    source_size: hashBased ? null : bytes.readUInt32LE(12),
  };
  const marshal = bytes.subarray(PYTHON_PYC_HEADER_BYTES);
  const reader = new MarshalReader(
    marshal,
    mapped.code_layout,
    input,
    mapped.python_version !== "3.7",
  );
  const root = reader.readObject(0);
  if (reader.offset !== marshal.length)
    malformed("Marshal stream has trailing bytes after the top-level object.");
  if (root.kind !== "code")
    return malformed("Top-level marshal object is not a code object.");
  reader.finishCodes();
  const codeObjects = flattenCode(root.value);
  const unknownFlags =
    flags & ~(PYTHON_PYC_HASH_BASED | PYTHON_PYC_CHECK_SOURCE);
  return {
    artifact: {
      path: artifact.path,
      sha256: artifact.sha256,
      bytes: artifact.bytes,
    },
    header,
    code_objects: codeObjects,
    coverage: {
      status: "complete",
      objects: reader.objects,
      code_objects: codeObjects.length,
      marshal_bytes: marshal.length,
      marshal_depth: reader.depthReached,
    },
    limitations: [
      "Bytecode is retained as length and SHA-256 only; this workflow does not decompile, disassemble, or recover source.",
      "The selected file is not executed and CPython is not invoked.",
      hashBased
        ? "source_hash_hex is the PEP 552 header field; the original .py is not read."
        : "source_mtime and source_size are header fields; they do not prove a matching .py is present.",
      "python_version is the pinned CPython magic table for this header, not a runtime interpreter.",
      ...(unknownFlags === 0
        ? []
        : [
            `Header flags ${flags} include unrecognized bits; only hash-based and check-source bits are decoded.`,
          ]),
    ],
  };
};

class MarshalReader {
  offset = 0;
  objects = 0;
  depthReached = 0;
  readonly #intern: (MarshalValue | undefined)[] = [];
  /** Name and identifier bytes the report carries across all code objects. */
  #reportedStringBytes = 0;
  /** Const summaries the report copies, including tables shared by TYPE_REF. */
  #reportedConsts = 0;
  /** Bytes of hash-key descriptors materialized for set and dict rules. */
  #keyBytes = 0;
  /** Identity keys for NaN values, which are not equal to themselves. */
  readonly #identities = new WeakMap<MarshalValue, number>();
  #identityNext = 0;
  /** One Python equality key per object. Null means the value is unhashable. */
  readonly #keyCache = new WeakMap<MarshalValue, ResolvedKey | null>();
  /** Constant keys keep signed zeroes distinct and give cycles an identity. */
  readonly #constCache = new WeakMap<MarshalValue, ResolvedKey>();
  /** Interned descriptors so equal values share one token within this walk. */
  readonly #keyTokens = new Map<string, string>();
  #keyTokenNext = 0;
  /** Explicit key-walk steps. Stream `max_depth` does not bound reference chains. */
  #keySteps = 0;
  /** One code-equality key per code object. Empty while that key is in progress. */
  readonly #codeKeys = new WeakMap<CodeNode, string>();
  readonly #codeIdentities = new WeakMap<CodeNode, number>();
  readonly #codeEquality = new WeakMap<CodeNode, CodeEquality>();
  /** Code objects whose const summaries wait until the graph is fully decoded. */
  readonly #codes: CodeNode[] = [];
  /**
   * Nested code objects reached from one container. Pure wrappers store the
   * child's array; fresh arrays are counted so copies cannot grow without a
   * bound. Absent while a container is still on the explicit walk stack.
   */
  readonly #nestedCache = new WeakMap<MarshalValue, readonly CodeNode[]>();
  #nestedSteps = 0;
  #nestedRetained = 0;

  constructor(
    readonly bytes: Buffer,
    readonly layout: PythonPycCodeLayout,
    readonly input: InspectPythonPycInput,
    readonly rejectNegativeCodeFlags: boolean,
  ) {}

  readObject(depth: number): MarshalValue {
    this.objects += 1;
    if (this.objects > this.input.max_objects)
      throw new PythonPycResourceFailure(
        "objects",
        { max_objects: this.input.max_objects },
        `Marshal object count exceeds max_objects ${this.input.max_objects}.`,
      );
    if (depth > this.input.max_depth)
      throw new PythonPycResourceFailure(
        "depth",
        { max_depth: this.input.max_depth },
        `Marshal nesting exceeds max_depth ${this.input.max_depth}.`,
      );
    this.depthReached = Math.max(this.depthReached, depth);
    const raw = this.u8();
    const flagged = (raw & FLAG_REF) !== 0;
    const type = raw & ~FLAG_REF;
    // CPython masks FLAG_REF before dispatch; TYPE_REF returns its existing
    // slot directly, and singleton tags bypass reference registration.
    if (type === TYPE_REF) return this.readRef();
    const slot =
      flagged && !TYPES_WITHOUT_REFERENCE_SLOTS.has(type)
        ? this.#intern.length
        : undefined;
    if (slot !== undefined) this.#intern.push(undefined);
    const value = this.readTyped(type, depth, slot);
    if (slot !== undefined && this.#intern[slot] === undefined)
      this.#intern[slot] = value;
    return value;
  }

  readTyped(
    type: number,
    depth: number,
    slot: number | undefined,
  ): MarshalValue {
    switch (type) {
      case TYPE_NULL:
        return { kind: "null" };
      case TYPE_NONE:
        return { kind: "none" };
      case TYPE_FALSE:
        return { kind: "bool", value: false };
      case TYPE_TRUE:
        return { kind: "bool", value: true };
      case TYPE_STOPITER:
        return { kind: "stopiter" };
      case TYPE_ELLIPSIS:
        return { kind: "ellipsis" };
      case TYPE_INT: {
        const value = BigInt(this.i32());
        return { kind: "int", value, bits: smallBitLength(value) };
      }
      case TYPE_FLOAT_TEXT:
        return { kind: "float", value: this.textFloat() };
      case TYPE_FLOAT:
        return { kind: "float", value: this.f64() };
      case TYPE_COMPLEX_TEXT:
        return {
          kind: "complex",
          real: this.textFloat(),
          imag: this.textFloat(),
        };
      case TYPE_COMPLEX:
        return { kind: "complex", real: this.f64(), imag: this.f64() };
      case TYPE_LONG: {
        const decoded = this.long();
        return { kind: "int", value: decoded.value, bits: decoded.bits };
      }
      case TYPE_BYTES:
        return { kind: "bytes", value: this.blob(this.u32()) };
      case TYPE_INTERNED:
      case TYPE_UNICODE:
        return this.utf8(this.u32(), false);
      case TYPE_ASCII:
      case TYPE_ASCII_INTERNED:
        return this.utf8(this.u32(), true);
      case TYPE_SHORT_ASCII:
      case TYPE_SHORT_ASCII_INTERNED:
        return this.utf8(this.u8(), true);
      case TYPE_TUPLE:
        return this.sequenceValue("tuple", this.u32(), depth, slot);
      case TYPE_SMALL_TUPLE:
        return this.sequenceValue("tuple", this.u8(), depth, slot);
      case TYPE_LIST:
        return this.sequenceValue("list", this.u32(), depth, slot);
      case TYPE_SET:
        return this.setValue("set", this.u32(), depth, slot);
      case TYPE_FROZENSET:
        return this.setValue("frozenset", this.u32(), depth, slot);
      case TYPE_DICT:
        return this.dict(depth, slot);
      case TYPE_CODE:
        return { kind: "code", value: this.code(depth) };
      default:
        return malformed(`Unsupported marshal type 0x${type.toString(16)}.`);
    }
  }

  code(depth: number): CodeNode {
    const argcount = this.nonNegativeInt32("co_argcount");
    const posonlyargcount =
      this.layout === "py37"
        ? null
        : this.nonNegativeInt32("co_posonlyargcount");
    if (posonlyargcount !== null && argcount < posonlyargcount)
      malformed("co_argcount is smaller than co_posonlyargcount.");
    const kwonlyargcount = this.nonNegativeInt32("co_kwonlyargcount");
    const nlocals =
      this.layout === "py311" ? null : this.nonNegativeInt32("co_nlocals");
    const stacksize = this.nonNegativeInt32("co_stacksize");
    const flags = this.i32();
    if (this.rejectNegativeCodeFlags && flags < 0)
      malformed("co_flags is negative.");
    const bytecode = asBytes(this.readObject(depth + 1), "co_code");
    // 3.11+ `_PyCode_Validate` rejects a length that is not a whole code unit.
    // 3.7–3.10 constructors do not, so an odd `co_code` stays readable there.
    if (this.layout === "py311" && bytecode.length % 2 !== 0)
      malformed("co_code is malformed.");
    const consts = asTuple(this.readObject(depth + 1), "co_consts");
    this.#chargeConsts(consts.length);
    const names = this.#nameTable(
      asTuple(this.readObject(depth + 1), "co_names"),
    );
    let varnames: string[] | null = null;
    let freevars: string[] | null = null;
    let cellvars: string[] | null = null;
    let localsplusnames: string[] | null = null;
    let localspluskinds_hex: string | null = null;
    if (this.layout === "py311") {
      localsplusnames = this.#nameTable(
        asTuple(this.readObject(depth + 1), "co_localsplusnames"),
      );
      const localspluskinds = asBytes(
        this.readObject(depth + 1),
        "co_localspluskinds",
      );
      if (localspluskinds.length !== localsplusnames.length)
        malformed(
          "co_localspluskinds length does not match co_localsplusnames.",
        );
      localspluskinds_hex = localspluskinds.toString("hex");
      this.#rejectUndersizedLocals(
        argcount,
        kwonlyargcount,
        flags,
        fastLocalCount(localspluskinds),
      );
    } else {
      varnames = this.#nameTable(
        asTuple(this.readObject(depth + 1), "co_varnames"),
      );
      freevars = this.#nameTable(
        asTuple(this.readObject(depth + 1), "co_freevars"),
      );
      cellvars = this.#nameTable(
        asTuple(this.readObject(depth + 1), "co_cellvars"),
      );
      this.#rejectUndersizedLocals(
        argcount,
        kwonlyargcount,
        flags,
        varnames.length,
      );
    }
    const filename = asString(this.readObject(depth + 1), "co_filename");
    const name = asString(this.readObject(depth + 1), "co_name");
    const qualname =
      this.layout === "py311"
        ? asString(this.readObject(depth + 1), "co_qualname")
        : null;
    const firstlineno = this.i32();
    const linetable = asBytes(this.readObject(depth + 1), "co_linetable");
    const exceptiontable =
      this.layout === "py311"
        ? asBytes(this.readObject(depth + 1), "co_exceptiontable")
        : Buffer.alloc(0);
    this.#reportStrings([
      name,
      qualname ?? "",
      filename,
      ...names,
      ...(varnames ?? []),
      ...(freevars ?? []),
      ...(cellvars ?? []),
      ...(localsplusnames ?? []),
      localspluskinds_hex ?? "",
    ]);
    const node: CodeNode = {
      name,
      qualname,
      filename,
      firstlineno,
      argcount,
      posonlyargcount,
      kwonlyargcount,
      nlocals,
      stacksize,
      flags,
      names,
      varnames,
      freevars,
      cellvars,
      localsplusnames,
      localspluskinds_hex,
      bytecode: { bytes: bytecode.length, sha256: sha256(bytecode) },
      consts: [],
      nested: [],
    };
    this.#codeEquality.set(node, {
      consts,
      linetable: `${linetable.length}:${sha256(linetable)}`,
      exceptiontable: `${exceptiontable.length}:${sha256(exceptiontable)}`,
    });
    this.#codes.push(node);
    return node;
  }

  /**
   * Const summaries and nested relationships follow Python's object graph.
   * Building them while `code()` is still inside an ancestor container caches
   * that container before its items exist.
   */
  finishCodes(): void {
    for (const node of this.#codes) {
      const material =
        this.#codeEquality.get(node) ??
        malformed("Code object is missing decoded constants.");
      node.consts = this.#summarizeConsts(material.consts);
      node.nested = this.#nestedCodeObjects(material.consts);
    }
  }

  /**
   * A code-constant summary copies the child's name. Charge every copy the
   * report will carry before that summary is built.
   */
  #summarizeConsts(items: readonly MarshalValue[]): PythonPycConst[] {
    const summaries: PythonPycConst[] = [];
    for (const item of items) {
      if (item.kind === "code") this.#reportStrings([item.value.name]);
      summaries.push(summarizeConst(item));
    }
    return summaries;
  }

  /**
   * Strings reached through TYPE_REF are one marshal object but a full copy
   * in the report, so bound what the report carries, not the stream.
   */
  #reportStrings(values: readonly string[]): void {
    for (const value of values)
      this.#reportedStringBytes += Buffer.byteLength(value, "utf8");
    if (this.#reportedStringBytes > this.input.max_file_bytes)
      throw new PythonPycResourceFailure(
        "string",
        { max_file_bytes: this.input.max_file_bytes },
        `Reported code-object strings exceed max_file_bytes ${this.input.max_file_bytes}.`,
      );
  }

  /**
   * A consts tuple shared through TYPE_REF is one decode but a full copy in
   * every code object. Stop before those copies are built.
   */
  #chargeConsts(count: number): void {
    this.#chargeReported(count, "constants");
  }

  /**
   * A name table shared through TYPE_REF is one marshal object and a full
   * copy in every code object. Empty strings add no report bytes, so charge
   * each copied entry before `strings()` builds the array the report keeps.
   */
  #nameTable(items: readonly MarshalValue[]): string[] {
    this.#chargeReported(items.length, "names");
    return strings(items);
  }

  /**
   * Charge copies the report will carry. Nested relationships are separate
   * from the immediate const tuple, which misses a shared container repeated
   * by many owners. Name-table copies are separate from the byte budget,
   * which misses an empty string repeated by many owners.
   */
  #chargeReported(
    count: number,
    what: "constants" | "names" | "nested code relationships",
  ): void {
    this.#reportedConsts += count;
    if (this.#reportedConsts > this.input.max_objects)
      throw new PythonPycResourceFailure(
        "objects",
        { max_objects: this.input.max_objects },
        `Reported ${what} exceed max_objects ${this.input.max_objects}.`,
      );
  }

  /**
   * Distinct code objects reachable from this constant tuple. Containers are
   * walked iteratively, and a wrapper keeps the child's array instead of
   * copying every descendant. Charge the owner's reported relationships
   * before the array the report keeps is built.
   */
  #nestedCodeObjects(items: readonly MarshalValue[]): CodeNode[] {
    const seen = new Set<CodeNode>();
    const ordered: CodeNode[] = [];
    const add = (code: CodeNode): void => {
      if (seen.has(code)) return;
      seen.add(code);
      ordered.push(code);
    };
    for (const item of items) {
      if (item.kind === "code") {
        add(item.value);
        continue;
      }
      for (const code of this.#reachableCodes(item)) add(code);
    }
    this.#chargeReported(ordered.length, "nested code relationships");
    return ordered;
  }

  #reachableCodes(value: MarshalValue): readonly CodeNode[] {
    if (value.kind === "code") return [value.value];
    if (!isContainer(value)) return EMPTY_CODES;
    const cached = this.#nestedCache.get(value);
    if (cached !== undefined) return cached;
    const added: { value: MarshalValue; retained: number }[] = [];
    if (this.#memoizeNested(value, added)) {
      for (const entry of added) {
        this.#nestedCache.delete(entry.value);
        this.#nestedRetained -= entry.retained;
      }
      const walked = this.#walkNested(value);
      this.#nestedCache.set(value, walked);
      return walked;
    }
    return this.#nestedCache.get(value) ?? EMPTY_CODES;
  }

  /**
   * Post-order walk on an explicit stack. Each frame keeps its child list,
   * so a dictionary is not copied once per entry. A back edge returns before
   * any cyclic container is cached; the caller then scans that component once.
   */
  #memoizeNested(
    root: MarshalValue,
    added: { value: MarshalValue; retained: number }[],
  ): boolean {
    const gray = new Set<MarshalValue>();
    const frames: {
      value: MarshalValue;
      index: number;
      /** Built once. A dictionary child list is not copied on every step. */
      children: readonly MarshalValue[];
    }[] = [{ value: root, index: 0, children: containerChildren(root) }];
    gray.add(root);
    while (frames.length > 0) {
      this.#takeNestedStep();
      const frame = frames[frames.length - 1];
      if (frame === undefined) break;
      const children = frame.children;
      if (frame.index < children.length) {
        const child = children[frame.index];
        frame.index += 1;
        if (child === undefined || !isContainer(child)) continue;
        if (this.#nestedCache.has(child)) continue;
        if (gray.has(child)) return true;
        gray.add(child);
        frames.push({
          value: child,
          index: 0,
          children: containerChildren(child),
        });
        continue;
      }
      const retainedBefore = this.#nestedRetained;
      const codes = this.#combineNested(frame.children);
      this.#nestedCache.set(frame.value, codes);
      added.push({
        value: frame.value,
        retained: this.#nestedRetained - retainedBefore,
      });
      gray.delete(frame.value);
      frames.pop();
    }
    return false;
  }

  /** One scan of a container cycle. Cached containers outside the cycle are reused. */
  #walkNested(root: MarshalValue): readonly CodeNode[] {
    const seen = new Set<CodeNode>();
    const visited = new Set<MarshalValue>();
    const stack: MarshalValue[] = [root];
    while (stack.length > 0) {
      this.#takeNestedStep();
      const current = stack.pop();
      if (current === undefined) break;
      if (current.kind === "code") {
        seen.add(current.value);
        continue;
      }
      if (!isContainer(current) || visited.has(current)) continue;
      const known =
        current === root ? undefined : this.#nestedCache.get(current);
      if (known !== undefined) {
        for (const code of known) seen.add(code);
        continue;
      }
      visited.add(current);
      const children = containerChildren(current);
      for (let index = children.length - 1; index >= 0; index -= 1) {
        const child = children[index];
        if (child !== undefined) stack.push(child);
      }
    }
    const codes = [...seen];
    this.#retainNested(codes);
    return codes;
  }

  /**
   * Share a child's array when this container adds no further code object.
   * Otherwise retain one new list and count its slots against the walk budget.
   * `children` is the list the frame already built.
   */
  #combineNested(children: readonly MarshalValue[]): readonly CodeNode[] {
    let shared: readonly CodeNode[] | undefined;
    let onlyShared = true;
    const codes: CodeNode[] = [];
    const seen = new Set<CodeNode>();
    const add = (code: CodeNode): void => {
      if (seen.has(code)) return;
      seen.add(code);
      codes.push(code);
    };
    for (const child of children) {
      if (child.kind === "code") {
        if (shared !== undefined || codes.length > 0) onlyShared = false;
        add(child.value);
        continue;
      }
      if (!isContainer(child)) continue;
      const part = this.#nestedCache.get(child);
      if (part === undefined || part.length === 0) continue;
      if (shared === undefined && codes.length === 0) shared = part;
      else onlyShared = false;
      for (const code of part) add(code);
    }
    if (codes.length === 0) return EMPTY_CODES;
    if (onlyShared && shared !== undefined && shared.length === codes.length)
      return shared;
    this.#retainNested(codes);
    return codes;
  }

  #takeNestedStep(): void {
    this.#nestedSteps += 1;
    const limit = this.input.max_objects * this.input.max_depth;
    if (this.#nestedSteps > limit)
      throw new PythonPycResourceFailure(
        "objects",
        {
          max_objects: this.input.max_objects,
          max_depth: this.input.max_depth,
        },
        `Nested code walk exceeds ${limit} steps (max_objects * max_depth).`,
      );
  }

  #retainNested(codes: readonly CodeNode[]): void {
    this.#nestedRetained += codes.length;
    const limit = this.input.max_objects * this.input.max_depth;
    if (this.#nestedRetained > limit)
      throw new PythonPycResourceFailure(
        "objects",
        {
          max_objects: this.input.max_objects,
          max_depth: this.input.max_depth,
        },
        `Nested code walk retains more than ${limit} code references.`,
      );
  }

  readRef(): MarshalValue {
    const index = this.u32();
    const value = this.#intern[index];
    if (value === undefined)
      return malformed(
        `TYPE_REF index ${index} is missing or still being constructed.`,
      );
    return value;
  }

  #publish(slot: number | undefined, value: MarshalValue): void {
    if (slot !== undefined) this.#intern[slot] = value;
  }

  /**
   * CPython registers tuples and lists before reading items, so a container
   * can refer to itself. The same object is what later TYPE_REF entries see.
   */
  sequenceValue(
    kind: "tuple" | "list",
    count: number,
    depth: number,
    slot: number | undefined,
  ): MarshalValue {
    const items: MarshalValue[] = [];
    const value: MarshalValue =
      kind === "tuple" ? { kind: "tuple", items } : { kind: "list", items };
    this.#publish(slot, value);
    for (let index = 0; index < count; index += 1) {
      const item = this.readObject(depth + 1);
      if (item.kind === "null")
        malformed("NULL object in marshal data for sequence.");
      items.push(item);
    }
    return value;
  }

  /**
   * Sets are registered before their members. Frozensets are registered after,
   * matching CPython, so a reference inside a frozenset body is still invalid.
   * Equal members count once; unhashable members fail the walk.
   */
  setValue(
    kind: "set" | "frozenset",
    count: number,
    depth: number,
    slot: number | undefined,
  ): MarshalValue {
    const items: MarshalValue[] = [];
    const value: MarshalValue =
      kind === "set" ? { kind: "set", items } : { kind: "frozenset", items };
    if (kind === "set") this.#publish(slot, value);
    for (let index = 0; index < count; index += 1) {
      const item = this.readObject(depth + 1);
      if (item.kind === "null")
        malformed("NULL object in marshal data for set.");
      items.push(item);
    }
    const unique = this.#unique(items, "set member");
    items.length = 0;
    for (const item of unique) items.push(item);
    return value;
  }

  /**
   * Dictionaries are registered before their pairs. A null key or a null value
   * ends the dict; a null value does not become an entry. Equal keys keep the
   * first key object and replace only its value. An unhashable key fails the
   * walk.
   */
  dict(depth: number, slot: number | undefined): MarshalValue {
    const items: [MarshalValue, MarshalValue][] = [];
    const value: MarshalValue = { kind: "dict", items };
    this.#publish(slot, value);
    const indexes = new Map<string, number>();
    for (;;) {
      const key = this.readObject(depth + 1);
      if (key.kind === "null") return value;
      const mapped = this.readObject(depth + 1);
      if (mapped.kind === "null") return value;
      const hash = this.#hashKey(key, "dictionary key");
      const existing = indexes.get(hash);
      if (existing === undefined) {
        indexes.set(hash, items.length);
        items.push([key, mapped]);
      } else {
        const kept =
          items[existing] ?? malformed("Dictionary entry is missing.");
        kept[1] = mapped;
      }
    }
  }

  #unique(items: readonly MarshalValue[], role: string): MarshalValue[] {
    const seen = new Map<string, MarshalValue>();
    for (const item of items) {
      const key = this.#hashKey(item, role);
      if (!seen.has(key)) seen.set(key, item);
    }
    return [...seen.values()];
  }

  /**
   * CPython rejects code whose argument counts do not fit the local table.
   * 3.11+ counts `CO_FAST_LOCAL` entries (`co_varnames is too small`).
   * 3.7–3.10 count `co_varnames` the same way, including `*args` and `**kwargs`.
   */
  #rejectUndersizedLocals(
    argcount: number,
    kwonlyargcount: number,
    flags: number,
    locals: number,
  ): void {
    let plain = locals - argcount - kwonlyargcount;
    if ((flags & CO_VARARGS) !== 0) plain -= 1;
    if ((flags & CO_VARKEYWORDS) !== 0) plain -= 1;
    if (plain < 0) malformed("co_varnames is too small.");
  }

  #hashKey(value: MarshalValue, role: string): string {
    return this.#equalityKey(value, "hash") ?? malformed(`Unhashable ${role}.`);
  }

  /**
   * Python equality for hash-based containers and for code-constant identity.
   * Tuple and frozenset chains use an explicit stack: a `TYPE_REF` chain is
   * not stream nesting, and `max_depth` must not turn a loadable file into
   * a stack overflow. Expanded size is checked per container so a shared
   * child that is stored twice still cannot double without bound.
   */
  #equalityKey(value: MarshalValue, mode: KeyMode): string | null {
    const resolved = this.#resolveKey(value, mode);
    return resolved === null ? null : resolved.token;
  }

  #resolveKey(value: MarshalValue, mode: KeyMode): ResolvedKey | null {
    const cached = this.#cachedKey(value, mode);
    if (cached !== undefined) return cached;
    const frames: KeyFrame[] = [
      { value, index: 0, parts: [], expanded: [], failed: false },
    ];
    const gray = new Set<MarshalValue>();
    gray.add(value);
    while (frames.length > 0) {
      this.#takeKeyStep();
      const frame = frames[frames.length - 1];
      if (frame === undefined) break;
      const items =
        frame.value.kind === "tuple" || frame.value.kind === "frozenset"
          ? frame.value.items
          : undefined;
      if (items === undefined) {
        const key = this.#leafKey(frame.value, mode);
        this.#storeKey(frame.value, mode, key, gray);
        frames.pop();
        this.#recordChild(frames, key);
        continue;
      }
      if (frame.failed || frame.index >= items.length) {
        const key = frame.failed
          ? null
          : this.#containerKey(frame.value, frame.parts, frame.expanded);
        this.#storeKey(frame.value, mode, key, gray);
        frames.pop();
        this.#recordChild(frames, key);
        continue;
      }
      const child = items[frame.index];
      frame.index += 1;
      if (child === undefined) continue;
      const known = this.#cachedKey(child, mode);
      if (known !== undefined) {
        if (known === null) frame.failed = true;
        else this.#pushPart(frame, known);
        continue;
      }
      if (gray.has(child)) {
        if (mode === "hash") frame.failed = true;
        else this.#pushPart(frame, this.#identityKey(child));
        continue;
      }
      gray.add(child);
      frames.push({
        value: child,
        index: 0,
        parts: [],
        expanded: [],
        failed: false,
      });
    }
    const resolved = this.#cachedKey(value, mode);
    return resolved === undefined ? null : resolved;
  }

  #cachedKey(
    value: MarshalValue,
    mode: KeyMode,
  ): ResolvedKey | null | undefined {
    return mode === "hash"
      ? this.#keyCache.get(value)
      : this.#constCache.get(value);
  }

  #storeKey(
    value: MarshalValue,
    mode: KeyMode,
    key: ResolvedKey | null,
    gray: Set<MarshalValue>,
  ): void {
    if (mode === "hash") this.#keyCache.set(value, key);
    else if (key !== null) this.#constCache.set(value, key);
    gray.delete(value);
  }

  #recordChild(frames: KeyFrame[], key: ResolvedKey | null): void {
    const parent = frames[frames.length - 1];
    if (parent === undefined) return;
    if (key === null) {
      parent.failed = true;
      return;
    }
    this.#pushPart(parent, key);
  }

  #pushPart(frame: KeyFrame, key: ResolvedKey): void {
    frame.parts.push(key.token);
    frame.expanded.push(key.expanded);
  }

  /**
   * One step per explicit-stack iteration. The cap follows decoded objects,
   * not `max_depth`, because a reference chain is shallow in the stream.
   */
  #takeKeyStep(): void {
    this.#keySteps += 1;
    const limit = this.input.max_objects * 8;
    if (this.#keySteps > limit)
      throw new PythonPycResourceFailure(
        "objects",
        { max_objects: this.input.max_objects },
        `Marshal key walk exceeds ${limit} steps (max_objects * 8).`,
      );
  }

  #leafKey(value: MarshalValue, mode: KeyMode): ResolvedKey | null {
    const descriptor = this.#leafDescriptor(value, mode);
    if (descriptor === null) return null;
    return {
      token: this.#internToken(descriptor),
      expanded: descriptor.length,
    };
  }

  #leafDescriptor(value: MarshalValue, mode: KeyMode): string | null {
    switch (value.kind) {
      case "none":
        return "N";
      case "ellipsis":
        return "E";
      case "stopiter":
        return "S";
      case "bool":
        if (mode === "hash") return value.value ? "i1" : "i0";
        return value.value ? "B1" : "B0";
      case "int":
        if (value.bits > this.input.max_string_bytes * 3)
          this.#keyTooLarge(value.bits);
        return this.#boundedJoin("i", [value.value.toString()], "");
      case "float":
        if (Number.isNaN(value.value)) return this.#identity(value);
        if (mode === "const") return this.#constFloat(value.value);
        return this.#numberKey(value.value);
      case "complex":
        if (Number.isNaN(value.real) || Number.isNaN(value.imag))
          return this.#identity(value);
        if (mode === "hash" && value.imag === 0)
          return this.#numberKey(value.real);
        if (mode === "hash")
          return this.#boundedJoin(
            "c",
            [this.#numberKey(value.real), this.#numberKey(value.imag)],
            ":",
          );
        return this.#boundedJoin(
          "c",
          [
            this.#signedZero(value.real),
            this.#signedZero(value.imag),
            this.#numberKey(value.real),
            this.#numberKey(value.imag),
          ],
          ":",
        );
      case "str":
        return this.#stringKey(value.raw);
      case "bytes": {
        this.#reserveKey(value.value.length * 2 + 8);
        const prefix = mode === "hash" ? "b" : "y";
        return `${prefix}${value.value.length}:${value.value.toString("hex")}`;
      }
      case "code":
        return this.#codeKey(value.value);
      default:
        return mode === "const" ? this.#identity(value) : null;
    }
  }

  #constFloat(value: number): string {
    if (Object.is(value, -0)) return "z-0";
    if (Object.is(value, 0)) return "z0";
    return `z${this.#numberKey(value)}`;
  }

  #containerKey(
    value: MarshalValue,
    parts: readonly string[],
    childExpanded: readonly number[],
  ): ResolvedKey {
    const expanded = this.#expandedContainerSize(childExpanded);
    if (expanded > this.input.max_string_bytes) this.#keyTooLarge(expanded);
    const ordered = value.kind === "frozenset" ? [...parts].sort() : parts;
    const kind = value.kind === "tuple" ? "T" : "F";
    const descriptor = `${kind}:${ordered.join(",")}`;
    // Charge the descriptor that is stored, not the unmaterialized expansion.
    // A linear reference chain stays within the file budget; repeating one
    // child still fails above when `expanded` passes max_string_bytes.
    this.#reserveKey(descriptor.length);
    return { token: this.#internToken(descriptor), expanded };
  }

  /** Size of the old length-prefixed container key, without building it. */
  #expandedContainerSize(childExpanded: readonly number[]): number {
    let size = 1;
    if (childExpanded.length > 0) size += childExpanded.length - 1;
    for (const child of childExpanded) size += String(child).length + 1 + child;
    return size;
  }

  #internToken(descriptor: string): string {
    const existing = this.#keyTokens.get(descriptor);
    if (existing !== undefined) return existing;
    this.#keyTokenNext += 1;
    const token = `k${this.#keyTokenNext.toString(36)}`;
    this.#keyTokens.set(descriptor, token);
    return token;
  }

  #identityKey(value: MarshalValue): ResolvedKey {
    const token = this.#identity(value);
    return { token, expanded: token.length };
  }

  /**
   * CPython code equality ignores filename, qualname, and stack size. Consts
   * use the constant-key rules, which keep -0 distinct from 0.
   */
  #codeKey(node: CodeNode): string {
    const existing = this.#codeKeys.get(node);
    if (existing !== undefined && existing !== "") return existing;
    if (existing === "") return `cycle${this.#codeIdentity(node)}`;
    this.#codeKeys.set(node, "");
    const material =
      this.#codeEquality.get(node) ??
      malformed("Code object is missing equality material.");
    const nameTables = [
      this.#nameTableKey(node.names),
      ...(node.localsplusnames !== null
        ? [this.#nameTableKey(node.localsplusnames)]
        : [
            this.#nameTableKey(node.varnames ?? []),
            this.#nameTableKey(node.freevars ?? []),
            this.#nameTableKey(node.cellvars ?? []),
          ]),
    ];
    const fields = [
      node.name,
      String(node.argcount),
      String(node.posonlyargcount ?? 0),
      String(node.kwonlyargcount),
      node.nlocals === null ? "" : String(node.nlocals),
      String(node.flags),
      String(node.firstlineno),
      `${node.bytecode.bytes}:${node.bytecode.sha256}`,
      this.#constKeyItems(material.consts),
      ...nameTables,
      // CPython started including line and exception tables in code equality
      // with the 3.11 code layout; earlier layouts still retain their digests
      // in CodeEquality, but those fields do not affect container identity.
      ...(this.layout === "py311"
        ? [material.linetable, material.exceptiontable]
        : []),
    ];
    const key = this.#boundedJoin(
      "",
      fields.map((field) => `${field.length}:${field}`),
      ",",
    );
    this.#codeKeys.set(node, key);
    return key;
  }

  #codeIdentity(node: CodeNode): number {
    const existing = this.#codeIdentities.get(node);
    if (existing !== undefined) return existing;
    this.#identityNext += 1;
    this.#codeIdentities.set(node, this.#identityNext);
    return this.#identityNext;
  }

  /**
   * One name table, with its length and each entry's boundary. Joining on a
   * bare NUL makes `()` and `("",)` the same key, and the same join hides
   * which of `co_varnames`, `co_freevars`, and `co_cellvars` held each name.
   */
  #nameTableKey(items: readonly string[]): string {
    const entries = items.map((item) => `${item.length}:${item}`).join(",");
    return `${items.length}[${entries}]`;
  }

  /** Constant key used when two code objects compare their co_consts. */
  #constKeyItems(items: readonly MarshalValue[]): string {
    const parts: string[] = [];
    for (const item of items) {
      const key = this.#resolveKey(item, "const");
      const token = key === null ? this.#identity(item) : key.token;
      parts.push(`${token.length}:${token}`);
    }
    return this.#boundedJoin("k", parts, ",");
  }

  /** Python string identity is the marshal bytes, not the JavaScript string. */
  #stringKey(raw: Buffer): string {
    this.#reserveKey(raw.length * 2 + 8);
    return `s${raw.length}:${raw.toString("hex")}`;
  }

  #signedZero(value: number): string {
    if (Object.is(value, -0)) return "-";
    if (Object.is(value, 0)) return "+";
    return "n";
  }

  /**
   * Join a key only after its size fits the string budget and the running
   * key-byte budget. Caching the result does not make an oversized key safe.
   */
  #boundedJoin(
    prefix: string,
    parts: readonly string[],
    separator: string,
  ): string {
    let size = prefix.length;
    if (parts.length > 0) size += (parts.length - 1) * separator.length;
    for (const part of parts) size += part.length;
    this.#reserveKey(size);
    return prefix + parts.join(separator);
  }

  #reserveKey(size: number): void {
    if (size > this.input.max_string_bytes) this.#keyTooLarge(size);
    this.#keyBytes += size;
    if (this.#keyBytes > this.input.max_file_bytes)
      throw new PythonPycResourceFailure(
        "string",
        { max_file_bytes: this.input.max_file_bytes },
        `Marshal key expansion exceeds max_file_bytes ${this.input.max_file_bytes}.`,
      );
  }

  #keyTooLarge(size: number): never {
    throw new PythonPycResourceFailure(
      "string",
      { max_string_bytes: this.input.max_string_bytes },
      `Marshal key of ${size} bytes exceeds max_string_bytes ${this.input.max_string_bytes}.`,
    );
  }

  #numberKey(value: number): string {
    if (value === 0) return "i0";
    if (Number.isFinite(value) && Number.isInteger(value))
      return `i${BigInt(value).toString()}`;
    const bytes = Buffer.alloc(8);
    bytes.writeDoubleLE(value);
    return `d${bytes.toString("hex")}`;
  }

  #identity(value: MarshalValue): string {
    const existing = this.#identities.get(value);
    if (existing !== undefined) return `u${existing}`;
    this.#identityNext += 1;
    this.#identities.set(value, this.#identityNext);
    return `u${this.#identityNext}`;
  }

  /**
   * CPython stores a long as 15-bit digits, least significant first. Charge
   * the digits against the string budget (they never use another one), reject
   * the digits CPython rejects, and build the value in one linear pass.
   */
  long(): { value: bigint; bits: number } {
    const count = this.i32();
    const size = Math.abs(count);
    if (size * 2 > this.input.max_string_bytes)
      throw new PythonPycResourceFailure(
        "string",
        { max_string_bytes: this.input.max_string_bytes },
        `Marshal long of ${size * 2} digit bytes exceeds max_string_bytes ${this.input.max_string_bytes}.`,
      );
    if (this.offset > this.bytes.length - size * 2)
      malformed("Marshal long overruns the selected file.");
    const nibbles: number[] = [];
    let carry = 0;
    let carryBits = 0;
    let digit = 0;
    for (let index = 0; index < size; index += 1) {
      digit = this.u16();
      if (digit >= 0x8000) malformed("Marshal long digit is out of range.");
      carry |= digit << carryBits;
      carryBits += 15;
      while (carryBits >= 4) {
        nibbles.push(carry & 0xf);
        carry >>>= 4;
        carryBits -= 4;
      }
    }
    if (size > 0 && digit === 0) malformed("Marshal long is not normalized.");
    if (carryBits > 0) nibbles.push(carry);
    let bits = 0;
    if (size > 0) {
      let top = digit;
      while (top > 0) {
        top >>>= 1;
        bits += 1;
      }
      bits += (size - 1) * 15;
    }
    if (nibbles.length === 0) return { value: 0n, bits: 0 };
    const magnitude = BigInt(
      `0x${nibbles
        .reverse()
        .map((nibble) => nibble.toString(16))
        .join("")}`,
    );
    return { value: count < 0 ? -magnitude : magnitude, bits };
  }

  /** Marshal version 0 writes a one-byte length and a CPython float literal. */
  textFloat(): number {
    const bytes = this.blob(this.u8());
    if (bytes.some((byte) => byte > 0x7f))
      malformed("Marshal float text is not ASCII.");
    return (
      parseMarshalFloat(bytes.toString("latin1")) ??
      malformed("Marshal float text is not a CPython float.")
    );
  }

  utf8(length: number, ascii: boolean): MarshalValue {
    const bytes = this.blob(length);
    if (ascii) {
      if (bytes.some((byte) => byte > 0x7f))
        malformed("Marshal ASCII string contains a non-ASCII byte.");
      return {
        kind: "str",
        value: bytes.toString("latin1"),
        bytes: length,
        raw: bytes,
      };
    }
    return {
      kind: "str",
      value: decodeSurrogatePass(bytes),
      bytes: length,
      raw: bytes,
    };
  }

  blob(length: number): Buffer {
    if (length > this.input.max_string_bytes)
      throw new PythonPycResourceFailure(
        "string",
        { max_string_bytes: this.input.max_string_bytes },
        `Marshal string of ${length} bytes exceeds max_string_bytes ${this.input.max_string_bytes}.`,
      );
    if (this.offset > this.bytes.length - length)
      malformed("Marshal string overruns the selected file.");
    const value = this.bytes.subarray(this.offset, this.offset + length);
    this.offset += length;
    return Buffer.from(value);
  }

  u8(): number {
    if (this.offset >= this.bytes.length)
      malformed("Marshal stream ended inside an object.");
    const value = this.bytes[this.offset];
    this.offset += 1;
    if (value === undefined)
      return malformed("Marshal stream ended inside an object.");
    return value;
  }

  u16(): number {
    if (this.offset > this.bytes.length - 2)
      malformed("Marshal stream ended inside a 16-bit field.");
    const value = this.bytes.readUInt16LE(this.offset);
    this.offset += 2;
    return value;
  }

  u32(): number {
    if (this.offset > this.bytes.length - 4)
      malformed("Marshal stream ended inside a 32-bit field.");
    const value = this.bytes.readUInt32LE(this.offset);
    this.offset += 4;
    return value;
  }

  i32(): number {
    if (this.offset > this.bytes.length - 4)
      malformed("Marshal stream ended inside a 32-bit field.");
    const value = this.bytes.readInt32LE(this.offset);
    this.offset += 4;
    return value;
  }

  /** CPython `r_long` int fields must not be negative in a code object. */
  nonNegativeInt32(field: string): number {
    const value = this.i32();
    if (value < 0) malformed(`${field} is negative.`);
    return value;
  }

  f64(): number {
    if (this.offset > this.bytes.length - 8)
      malformed("Marshal stream ended inside a float.");
    const value = this.bytes.readDoubleLE(this.offset);
    this.offset += 8;
    return value;
  }
}

/**
 * Decode marshal UTF-8 the way CPython does (`surrogatepass`): encoded lone
 * surrogates are kept, every other ill-formed sequence is rejected.
 */
const decodeSurrogatePass = (bytes: Buffer): string => {
  const units: number[] = [];
  let text = "";
  for (let index = 0; index < bytes.length;) {
    const lead = bytes[index] ?? 0;
    let codePoint: number;
    let width: number;
    if (lead < 0x80) {
      codePoint = lead;
      width = 1;
    } else if (lead >= 0xc2 && lead <= 0xdf) {
      codePoint = lead & 0x1f;
      width = 2;
    } else if (lead >= 0xe0 && lead <= 0xef) {
      codePoint = lead & 0x0f;
      width = 3;
    } else if (lead >= 0xf0 && lead <= 0xf4) {
      codePoint = lead & 0x07;
      width = 4;
    } else {
      return malformed("Marshal string is not valid UTF-8.");
    }
    if (index + width > bytes.length)
      return malformed("Marshal string is not valid UTF-8.");
    for (let offset = 1; offset < width; offset += 1) {
      const next = bytes[index + offset] ?? 0;
      if ((next & 0xc0) !== 0x80)
        return malformed("Marshal string is not valid UTF-8.");
      codePoint = (codePoint << 6) | (next & 0x3f);
    }
    if (
      (width === 3 && codePoint < 0x800) ||
      (width === 4 && (codePoint < 0x10000 || codePoint > 0x10ffff))
    )
      return malformed("Marshal string is not valid UTF-8.");
    units.push(codePoint);
    index += width;
    if (units.length === 4096) {
      text += codePointsToString(units);
      units.length = 0;
    }
  }
  return text + codePointsToString(units);
};

const codePointsToString = (codePoints: readonly number[]): string =>
  codePoints
    .map((codePoint) =>
      codePoint >= 0xd800 && codePoint <= 0xdfff
        ? String.fromCharCode(codePoint)
        : String.fromCodePoint(codePoint),
    )
    .join("");

/** JSON has no non-finite numbers; keep CPython's repr for them. */
const jsonFloat = (value: number): number | "inf" | "-inf" | "nan" =>
  Number.isNaN(value)
    ? "nan"
    : value === Infinity
      ? "inf"
      : value === -Infinity
        ? "-inf"
        : value;

/**
 * Emit each distinct code object once, the top-level object first. A child
 * shared through TYPE_REF is listed once and referenced by index, so the
 * report never repeats a subtree.
 */
const flattenCode = (root: CodeNode): PythonPycCodeObject[] => {
  const indexes = new Map<CodeNode, number>();
  const order: CodeNode[] = [];
  const visit = (node: CodeNode): void => {
    if (indexes.has(node)) return;
    indexes.set(node, order.length);
    order.push(node);
    for (const child of node.nested) visit(child);
  };
  visit(root);
  return order.map(({ nested, ...code }) => ({
    ...code,
    nested_code: nested.map(
      (child) =>
        indexes.get(child) ?? malformed("Nested code object was not listed."),
    ),
  }));
};

const asBytes = (value: MarshalValue, field: string): Buffer => {
  if (value.kind !== "bytes")
    return malformed(`${field} is not marshal bytes.`);
  return value.value;
};

const asString = (value: MarshalValue, field: string): string => {
  if (value.kind !== "str")
    return malformed(`${field} is not a marshal string.`);
  return value.value;
};

const asTuple = (
  value: MarshalValue,
  field: string,
): readonly MarshalValue[] => {
  if (value.kind !== "tuple")
    return malformed(`${field} is not a marshal tuple.`);
  return value.items;
};

const strings = (items: readonly MarshalValue[]): string[] =>
  items.map((item, index) => {
    if (item.kind !== "str")
      return malformed(`Tuple item ${index} is not a marshal string.`);
    return item.value;
  });

const constSummaries = new WeakMap<MarshalValue, PythonPycConst>();

/** One summary per decoded object, so TYPE_REF does not repeat the work. */
const summarizeConst = (value: MarshalValue): PythonPycConst => {
  const cached = constSummaries.get(value);
  if (cached !== undefined) return cached;
  const summary = buildConstSummary(value);
  constSummaries.set(value, summary);
  return summary;
};

const buildConstSummary = (value: MarshalValue): PythonPycConst => {
  switch (value.kind) {
    case "none":
      return { kind: "none" };
    case "bool":
      return { kind: "bool", value: value.value };
    case "int":
      return {
        kind: "int",
        bits: value.bits,
        value:
          value.bits <= PYTHON_PYC_INT_PREVIEW_BITS
            ? value.value.toString()
            : null,
      };
    case "float":
      return { kind: "float", value: jsonFloat(value.value) };
    case "str": {
      const truncated = value.value.length > PYTHON_PYC_CONST_PREVIEW_LIMIT;
      return {
        kind: "str",
        bytes: value.bytes,
        truncated,
        value: truncated
          ? value.value.slice(0, PYTHON_PYC_CONST_PREVIEW_LIMIT)
          : value.value,
      };
    }
    case "bytes":
      return {
        kind: "bytes",
        bytes: value.value.length,
        sha256: sha256(value.value),
      };
    case "code":
      return {
        kind: "code",
        name: value.value.name,
        firstlineno: value.value.firstlineno,
      };
    case "tuple":
      return { kind: "tuple", length: value.items.length };
    case "list":
      return { kind: "list", length: value.items.length };
    case "set":
      return { kind: "set", length: value.items.length };
    case "frozenset":
      return { kind: "frozenset", length: value.items.length };
    case "dict":
      return { kind: "dict", length: value.items.length };
    case "ellipsis":
      return { kind: "ellipsis" };
    case "complex":
      return {
        kind: "complex",
        real: jsonFloat(value.real),
        imag: jsonFloat(value.imag),
      };
    default:
      return { kind: "other", type: value.kind };
  }
};

const smallBitLength = (value: bigint): number => {
  if (value === 0n) return 0;
  const magnitude = value < 0n ? -value : value;
  return magnitude.toString(2).length;
};

/**
 * CPython's `PyOS_string_to_double` grammar for marshal version 0: an optional
 * sign, then `inf`/`infinity`, `nan`, or a decimal literal. No hex floats and
 * no surrounding whitespace.
 */
const MARSHAL_FLOAT =
  /^([+-]?)(?:(inf(?:inity)?)|(nan)|((?:\d+\.\d*|\.\d+|\d+)(?:[eE][+-]?\d+)?))$/i;

const parseMarshalFloat = (text: string): number | undefined => {
  const match = MARSHAL_FLOAT.exec(text);
  if (match === null) return undefined;
  if (match[2] !== undefined) return match[1] === "-" ? -Infinity : Infinity;
  if (match[3] !== undefined) return Number.NaN;
  const value = Number(text);
  return Number.isNaN(value) ? undefined : value;
};