diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index f37c0f5885..78b026a28a 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -688,7 +688,8 @@ A whole-flash dump opens as `esp-flash-image`. `root_format_metadata` holds proof of ciphertext. `written_offset`/`written_end` bound the bytes that are not erased: an app slot whose facts say `erased` can still hold an older image, and a limitation says so. For a dump read from another flash address, call -`open_binary` with `format: "esp-flash-image"` and `flash_offset`; a +`open_binary` with `format: "esp-flash-image"` and `flash_offset`; an +`esp-image` whose `flash_dump` is set names that offset in a next probe; a relocated table can be named with `partition_table_offset`. No table found means unknown, often flash encryption. Extraction writes `nvs_keys` as ordinary data. See diff --git a/docs/esp-flash-image.md b/docs/esp-flash-image.md index b3d8643143..dac3cc7997 100644 --- a/docs/esp-flash-image.md +++ b/docs/esp-flash-image.md @@ -45,7 +45,10 @@ refuses two kinds of dump before looking for a table: bootloader at the start of a dump read from `0x1000`. The dump's origin is not the flash address REA assumed, given or default, and a table found there would place every partition at the wrong file offset; pass the - `flash_offset` where the dump starts. + `flash_offset` where the dump starts. When a partition table sits where + the image's own ROM offset puts `0x8000`, the refusal names that offset. + Without a selected format such a dump opens as its bootloader image, whose + [ESP image](esp-image.md) facts say how to reopen it. - an ESP8266 image at a ROM offset. ESP8266 dumps can hold the same table format, and they keep the [ESP image](esp-image.md) reader's ESP8266 refusal. @@ -281,7 +284,8 @@ otatool's own OTA logic runs unchanged. The lane then checks: selection is `no-otadata` with the factory app; - the relocated table, found by scanning and named with `partition_table_offset`, and the dump written from `0x1000`, read with - `flash_offset` and, without it, left an ESP image; + `flash_offset` and, without it, left an ESP image whose `flash_dump` + names `flash_offset` `0x1000`; - an app with a flipped byte beside valid partitions, and a truncated dump, read with complete coverage and its missing partitions outside the dump. diff --git a/docs/esp-image.md b/docs/esp-image.md index 3ff17c366d..82595eb747 100644 --- a/docs/esp-image.md +++ b/docs/esp-image.md @@ -214,6 +214,18 @@ bootloader is at offset 0, such as ESP32-C3 or ESP32-S3, is recognized as that bootloader image, and the partition table, apps and data follow as undescribed bytes. +A dump read from a later flash address, such as an ESP32 dump read from +`0x1000` with `read-flash 0x1000 …` or `merge-bin --target-offset 0x1000`, +opens as its bootloader image. REA says so when the file shows it: the image +at file offset 0 is not an app, its chip's ROM loads the bootloader from a +later address (`0x1000` for ESP32 and S2, `0x2000` for C5, H4, P4 and S31), +and a partition table sits where that origin puts `0x8000` (file offset +`0x7000` for `0x1000`). Then `flash_dump` records `flash_offset` and +`table_file_offset`, a limitation says to reopen the file with +`format: "esp-flash-image"` and that `flash_offset`, and `inspect_artifact` +lists the same `open_binary` call as a next probe. `flash_dump` is null for +every other image. + ## Limits An image has at most 16 segments and 3 signature blocks. Decoding reads each diff --git a/scripts/verify/firmware/esp-flash.mjs b/scripts/verify/firmware/esp-flash.mjs index f5273ff095..fdfb0690d4 100644 --- a/scripts/verify/firmware/esp-flash.mjs +++ b/scripts/verify/firmware/esp-flash.mjs @@ -339,6 +339,11 @@ const esp32Origin = async (report) => { "esp-image", "no origin, no layout", ); + // The table at file offset 0x7000 shows where the dump was read from. + assert.deepEqual(guessed.root.flash_dump, { + flash_offset: 0x1000, + table_file_offset: 0x7000, + }); const flags = [ "--target-format", "esp-flash-image", diff --git a/src/artifacts/EspImageArtifactReader.ts b/src/artifacts/EspImageArtifactReader.ts index a4337bd415..351874d6cd 100644 --- a/src/artifacts/EspImageArtifactReader.ts +++ b/src/artifacts/EspImageArtifactReader.ts @@ -1,12 +1,21 @@ import { Readable, Transform, pipeline } from "node:stream"; import type { EspImageMetadata } from "../domain/artifactFormatMetadata.js"; +import { espChipForId } from "../domain/firmware/espChips.js"; +import { + espFlashDumpOrigin, + type EspFlashDumpOrigin, +} from "../domain/firmware/espFlashLocation.js"; import { createEspImageDecoder, type EspImageFailure, } from "../domain/firmware/espImage.js"; import type { EspImage } from "../domain/firmware/espImageFacts.js"; -import { ESP_SIGNATURE_BLOCK_BYTES } from "../domain/firmware/espImageStructures.js"; +import { + ESP_SIGNATURE_BLOCK_BYTES, + offsetText, +} from "../domain/firmware/espImageStructures.js"; +import { readEspFlashRange } from "./EspFlashRegionReads.js"; import { ArtifactReaderFailure, copyArtifactEntry, @@ -202,12 +211,47 @@ export class EspImageArtifactReader implements ArtifactReader { } satisfies ArtifactEntry; this.#entries.set(entry.adapterKey, entry); } - this.#metadata = image.metadata; - if (image.coverage === "unknown" || image.limitations.length > 0) - this.#coverage = { - status: image.coverage, - limitations: [...image.limitations], - }; + const dump = await this.#flashDump(image, signal); + this.#metadata = { + ...image.metadata, + flash_dump: + dump === undefined + ? null + : { + flash_offset: dump.flashOffset, + table_file_offset: dump.tableFileOffset, + }, + }; + const limitations = [ + ...image.limitations, + ...(dump === undefined ? [] : [flashDumpLimitation(image, dump)]), + ]; + if (image.coverage === "unknown" || limitations.length > 0) + this.#coverage = { status: image.coverage, limitations }; + } + + /** + * Whether this image is the bootloader at the start of a flash dump read + * from a later flash address. An app carries `esp_app_desc_t` and is not. + */ + async #flashDump( + image: EspImage, + signal?: AbortSignal, + ): Promise { + const chip = espChipForId(image.metadata.header.chip_id); + if ( + chip === undefined || + chip.romBootloaderOffset === 0 || + image.metadata.descriptor?.kind === "app" + ) + return undefined; + const { handle, initial } = await this.#source.read(signal); + const context = { handle, fileSize: initial.size, signal }; + return espFlashDumpOrigin( + (position, length) => readEspFlashRange(context, position, length), + initial.size, + { chip, fileOffset: 0 }, + ); } /** Stream the carrier once, stopping when later bytes cannot matter. */ @@ -264,6 +308,12 @@ export class EspImageArtifactReader implements ArtifactReader { } } +const flashDumpLimitation = ( + image: EspImage, + dump: EspFlashDumpOrigin, +): string => + `This file looks like a flash dump read from flash ${offsetText(dump.flashOffset)}, not one ESP image: its ${image.metadata.chip?.target ?? "ESP"} image at file offset 0 is where the ROM loads the bootloader from ${offsetText(dump.flashOffset)}, and a partition table sits at file offset ${offsetText(dump.tableFileOffset)}, which is flash 0x8000 from that origin. Reopen it with format esp-flash-image and flash_offset ${offsetText(dump.flashOffset)}.`; + const decodeFailure = (failure: EspImageFailure): ArtifactReaderFailure => failure.kind === "unsupported" ? new ArtifactReaderFailure("unavailable", failure.message) diff --git a/src/domain/artifactFormatMetadata.ts b/src/domain/artifactFormatMetadata.ts index d227e7bfd3..a9e58af19a 100644 --- a/src/domain/artifactFormatMetadata.ts +++ b/src/domain/artifactFormatMetadata.ts @@ -216,6 +216,17 @@ export const espImageMetadataSchema = z.strictObject({ espBootloaderDescriptorSchema, ]) .nullable(), + /** + * Set when the file is a flash dump read from a later flash address: this + * image is not an app, its chip's ROM loads the bootloader from + * `flash_offset`, and a partition table sits at `table_file_offset`, where + * that origin puts the default `0x8000`. Reopen the file as + * `esp-flash-image` with that `flash_offset`. Null otherwise, and in the + * nested facts of a flash layout. + */ + flash_dump: z + .strictObject({ flash_offset: uint32, table_file_offset: offset }) + .nullable(), }); export type EspImageMetadata = z.infer; diff --git a/src/domain/artifactInspection.ts b/src/domain/artifactInspection.ts index c0d99083a3..148dffbcbc 100644 --- a/src/domain/artifactInspection.ts +++ b/src/domain/artifactInspection.ts @@ -317,11 +317,31 @@ const probesFor = ( }, ] : []; + if (format === "esp-image") return espFlashDumpProbes(inventory); return format === "directory" ? directoryProbes(inventory) : probesForFormat(format); }; +// An ESP image that its reader found at the start of a flash dump read from +// a later address is better read as that dump. +const espFlashDumpProbes = ( + inventory: ArtifactInventoryResult, +): ArtifactInspectionResult["next_probes"] => { + const root = inventory.occurrences.find( + ({ logical_path }) => logical_path === ".", + )?.format_metadata; + if (root?.kind !== "esp-image" || root.flash_dump === null) return []; + const offset = `0x${root.flash_dump.flash_offset.toString(16)}`; + return [ + { + operation: "open_binary", + rationale: `Reopen the file with format esp-flash-image and flash_offset ${offset}, then inspect_artifact: it is a flash dump read from flash ${offset}, with a partition table at file offset 0x${root.flash_dump.table_file_offset.toString(16)}.`, + authority: "artifact-bytes", + }, + ]; +}; + // An application bundle directory is only a container, so its probes follow // the formats it holds: a bundle of Mach-O executables is not a JavaScript tree. // Formats are ordered so a repeated operation keeps its most general rationale. diff --git a/src/domain/firmware/espFlashBootloader.ts b/src/domain/firmware/espFlashBootloader.ts index 35e1af771d..f28275719b 100644 --- a/src/domain/firmware/espFlashBootloader.ts +++ b/src/domain/firmware/espFlashBootloader.ts @@ -37,6 +37,10 @@ export interface EspBootloaderSearch { readonly limitations: readonly string[]; /** A known chip's image sits at another chip's ROM offset. */ readonly misplaced: boolean; + /** The first such image, which may be the bootloader of a later origin. */ + readonly misplacedImage: + | { readonly chip: EspChip; readonly fileOffset: number } + | undefined; /** An ESP8266 image sits at a ROM offset. */ readonly esp8266: boolean; } @@ -51,7 +55,7 @@ export const locateEspBootloader = ( probes: readonly EspBootloaderProbe[], ): EspBootloaderSearch => { const limitations: string[] = []; - let misplaced = false; + let misplacedImage: EspBootloaderSearch["misplacedImage"]; let esp8266 = false; for (const probe of probes) { const recognized = recognizeEspImage(probe.prefix, probe.available); @@ -68,16 +72,24 @@ export const locateEspBootloader = ( }, limitations: [], misplaced: false, + misplacedImage: undefined, esp8266: false, }; - if (chip !== undefined) misplaced = true; + if (chip !== undefined) + misplacedImage ??= { chip, fileOffset: probe.fileOffset }; limitations.push( chip === undefined ? `An ESP image at flash ${offsetText(probe.flashAddress)} has chip_id 0x${chipId.toString(16).padStart(4, "0")}, which is not in the chip table; it is not read as the bootloader.` : `An ${chip.target} image is at flash ${offsetText(probe.flashAddress)}, but its ROM loads the bootloader from ${offsetText(chip.romBootloaderOffset)}; the dump may not start at flash ${offsetText(probe.flashAddress - probe.fileOffset)}.`, ); } - return { location: undefined, limitations, misplaced, esp8266 }; + return { + location: undefined, + limitations, + misplaced: misplacedImage !== undefined, + misplacedImage, + esp8266, + }; }; /** diff --git a/src/domain/firmware/espFlashLocation.test.ts b/src/domain/firmware/espFlashLocation.test.ts index e83cd0b831..d345e73991 100644 --- a/src/domain/firmware/espFlashLocation.test.ts +++ b/src/domain/firmware/espFlashLocation.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from "vitest"; +import { espChipForId } from "./espChips.js"; import { + espFlashDumpOrigin, locateEspFlashLayout, type EspFlashSelection, } from "./espFlashLocation.js"; @@ -209,6 +211,39 @@ describe("ESP flash layout origins", () => { }, ); + it("names the origin a misplaced bootloader and the table at its 0x8000 show", async () => { + // Read from 0x1000: the bootloader at file 0, the table at file 0x7000. + const dump = fromSecondSector(0x8000); + const reader = (bytes: Uint8Array) => async (at: number, length: number) => + bytes.subarray(at, at + length); + const esp32 = espChipForId(0x0000)!; + expect( + await espFlashDumpOrigin(reader(dump), dump.length, { + chip: esp32, + fileOffset: 0, + }), + ).toEqual({ flashOffset: 0x1000, tableFileOffset: 0x7000 }); + expect(await locate(dump)).toEqual({ + ok: false, + error: + "An esp32 image is at flash 0x0, but its ROM loads the bootloader from 0x1000; the dump may not start at flash 0x0. A partition table at file offset 0x7000 shows the dump was read from flash 0x1000; pass flash_offset 0x1000", + }); + // No table there, or a chip whose ROM loads from 0x0, shows no origin. + const relocated = fromSecondSector(0x9000); + expect( + await espFlashDumpOrigin(reader(relocated), relocated.length, { + chip: esp32, + fileOffset: 0, + }), + ).toBeUndefined(); + expect( + await espFlashDumpOrigin(reader(dump), dump.length, { + chip: espChipForId(0x0005)!, + fileOffset: 0, + }), + ).toBeUndefined(); + }); + it("finds the table after the bootloader once the origin is right", async () => { const located = await locate(fromSecondSector(0x9000), { flashOffset: 0x1000, diff --git a/src/domain/firmware/espFlashLocation.ts b/src/domain/firmware/espFlashLocation.ts index dd3f4c6bf9..4ea5dda296 100644 --- a/src/domain/firmware/espFlashLocation.ts +++ b/src/domain/firmware/espFlashLocation.ts @@ -10,6 +10,7 @@ import type { EspFlashOrigin, EspPartitionTableMethod, } from "./espFlashLayout.js"; +import type { EspChip } from "./espChips.js"; import { ESP8266_REFUSAL } from "./espImage.js"; import { offsetText } from "./espImageStructures.js"; import { @@ -56,6 +57,36 @@ const tableAt = async ( await read(fileOffset, ESP_PARTITION_TABLE_MAX_BYTES), ); +/** Where a dump read from a later flash address starts, as its bytes show. */ +export interface EspFlashDumpOrigin { + /** The flash address of the file's first byte. */ + readonly flashOffset: number; + /** The file offset of the partition table at that origin's `0x8000`. */ + readonly tableFileOffset: number; +} + +/** + * An image of a chip whose ROM loads the bootloader from flash `R`, at file + * offset `F`, is the bootloader of a dump read from flash `R - F` when a + * partition table sits where that origin puts the default table, at file + * offset `0x8000 - (R - F)`. An ESP32 dump read from `0x1000`, as + * `read-flash 0x1000 …` or `merge-bin --target-offset 0x1000` writes it, + * has its bootloader at file offset 0 and its table at `0x7000`. + */ +export const espFlashDumpOrigin = async ( + read: EspFlashRead, + fileSize: number, + image: { readonly chip: EspChip; readonly fileOffset: number }, +): Promise => { + const flashOffset = image.chip.romBootloaderOffset - image.fileOffset; + const tableFileOffset = ESP_PARTITION_TABLE_DEFAULT_OFFSET - flashOffset; + if (flashOffset <= 0 || tableFileOffset < 0 || tableFileOffset >= fileSize) + return undefined; + return (await tableAt(read, tableFileOffset)).ok + ? { flashOffset, tableFileOffset } + : undefined; +}; + /** * The first table whose MD5 verifies at a flash sector boundary from the * file offset `start`; requiring the MD5 keeps arbitrary bytes that begin @@ -169,10 +200,19 @@ export const locateEspFlashLayout = async ( return err(ESP8266_REFUSAL); // A table found at an origin the bootloader contradicts would place every // partition at the wrong file offset. - if (bootloader.location === undefined && bootloader.misplaced) + if (bootloader.location === undefined && bootloader.misplaced) { + const implied = + bootloader.misplacedImage === undefined + ? undefined + : await espFlashDumpOrigin(read, fileSize, bootloader.misplacedImage); return err( - `${bootloader.limitations.join(" ")} Pass the flash_offset where the dump starts, or partition_table_offset`, + `${bootloader.limitations.join(" ")} ${ + implied === undefined + ? "Pass the flash_offset where the dump starts, or partition_table_offset" + : `A partition table at file offset ${offsetText(implied.tableFileOffset)} shows the dump was read from flash ${offsetText(implied.flashOffset)}; pass flash_offset ${offsetText(implied.flashOffset)}` + }`, ); + } const imageEnd = bootloader.location === undefined ? 0 diff --git a/src/domain/firmware/espImage.test.ts b/src/domain/firmware/espImage.test.ts index 80c751c078..d269f5fd22 100644 --- a/src/domain/firmware/espImage.test.ts +++ b/src/domain/firmware/espImage.test.ts @@ -150,6 +150,7 @@ describe("ESP image decoding", () => { max_efuse_blk_rev_full: 199, mmu_page_size: 16, }, + flash_dump: null, }); expect( decoded.segments.map(({ dataOffset, availableLength, metadata }) => [ diff --git a/src/domain/firmware/espImageFacts.ts b/src/domain/firmware/espImageFacts.ts index e8c9aaf8fc..608ac6ef10 100644 --- a/src/domain/firmware/espImageFacts.ts +++ b/src/domain/firmware/espImageFacts.ts @@ -142,6 +142,8 @@ export const deriveEspImage = (observed: EspImageObservations): EspImage => { trailing.blocks.length > 0 ? (observed.sectorOffset ?? null) : null, undescribed_trailing: trailing.undescribed, descriptor, + // The image's bytes alone cannot place it in a dump; the reader can. + flash_dump: null, }, segments, signatureBlocks: trailing.blocks, diff --git a/tests/acceptance/espFlashImageCli.test.ts b/tests/acceptance/espFlashImageCli.test.ts index 392169914b..ca93822483 100644 --- a/tests/acceptance/espFlashImageCli.test.ts +++ b/tests/acceptance/espFlashImageCli.test.ts @@ -117,6 +117,32 @@ cliTest( expect(unpaired.stdout + unpaired.stderr).toContain( "flash_offset and partition_table_offset apply only with format=esp-flash-image", ); + // Without the format, the bootloader is read as one ESP image, and the + // table at file offset 0x7000 says how to reopen the dump. + const automatic = await cli.run({ + arguments: ["inspect-artifact", source, "--json"], + }); + expect(automatic.exitCode, automatic.stdout + automatic.stderr).toBe(0); + const guessed = rootMetadata(automatic.json); + expect(guessed.graph.manifest.root_format).toBe("esp-image"); + expect(guessed.root).toMatchObject({ + flash_dump: { flash_offset: 0x1000, table_file_offset: 0x7000 }, + }); + expect( + artifactInspectionResultSchema.parse( + parseEvidence(automatic.json).normalized_result, + ).next_probes, + ).toEqual([ + { + operation: "open_binary", + rationale: + "Reopen the file with format esp-flash-image and flash_offset 0x1000, then inspect_artifact: it is a flash dump read from flash 0x1000, with a partition table at file offset 0x7000.", + authority: "artifact-bytes", + }, + ]); + expect(guessed.graph.limitations).toContain( + "This file looks like a flash dump read from flash 0x1000, not one ESP image: its esp32 image at file offset 0 is where the ROM loads the bootloader from 0x1000, and a partition table sits at file offset 0x7000, which is flash 0x8000 from that origin. Reopen it with format esp-flash-image and flash_offset 0x1000.", + ); const inspected = await cli.run({ arguments: [ "inspect-artifact",