fix(sqlite): preserve snapshot admission failure semantics

Refuse oversized database and WAL inputs before acquisition with exact observed bytes and export guidance. Keep initial path failures separate from file-set changes and retain cleanup and partial observations on late growth.

Refs #4208, #4188, #4141, #4127

Co-authored-by: hima <[email protected]>
Co-authored-by: Royce <[email protected]>
This commit is contained in:
committed 2026-10-16 03:49:44 +08:00
1 parent 85eac7b92b
commit bc51d0ea5d
7 files changed
+538 -24

No files matched your search

+14
View File
@@ -105,6 +105,20 @@ overall native heap and total resident memory are not capped. Row limits do not
override these byte and resource limits. Resource-limit failures do not return
an incomplete schema as successful complete evidence.
If the combined database and WAL exceed the snapshot budget, the refusal is a
`resource_constraint` and reports both the configured byte limit and the exact
observed combined size as a decimal string. The remediation directs callers to
create a smaller consistent offline snapshot containing only relevant data
through the application's supported export workflow. A generic backup or copy
does not guarantee a smaller result; REA does not truncate or modify the
selected source.
On hosts that cannot address the derived WAL or rollback-journal path for a
long selected database name, REA reports that the SQLite capability is
unavailable. It does not assume an inaccessible sidecar is absent. Use the
application's supported consistent offline export or copy workflow to place
the complete known file set at a shorter path.
MCP also applies its configured response budget to the complete Evidence
envelope. If delivery exceeds that budget, it reports a transport constraint
with guidance for exporting retained Evidence, requesting a matching larger
+1 -1
View File
@@ -17,7 +17,7 @@ export const SOURCE_TEST_PROJECTS = [
group: "source",
include: [
"src/*.test.ts",
"src/{apktool,jeb,adb,artifacts,browser,dotnet,electron,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts",
"src/{apktool,jeb,adb,artifacts,browser,dotnet,electron,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,sqlite,wasm,windows}/**/*.test.ts",
],
},
{
+7 -2
View File
@@ -79,7 +79,7 @@ export const sqliteDatabaseFailure = (
return new AnalysisInputError(operation, { cause }, [
{
path: ["path"],
reason: cause.reason === "limit" ? "out_of_range" : "invalid_format",
reason: "invalid_format",
message: cause.message,
},
]);
@@ -110,7 +110,12 @@ export const sqliteDatabaseFailure = (
"The selected SQLite database is held open by another process without read sharing. Close that application or inspect a copy, then retry.",
},
);
if (cause.code === "ENOENT" || cause.code === "ENOTDIR")
if (
cause.code === "ENOENT" ||
cause.code === "ENOTDIR" ||
cause.code === "ELOOP" ||
cause.code === "ENAMETOOLONG"
)
return new AnalysisInputError(operation, { cause }, [
{
path: ["path"],
+110 -20
View File
@@ -3,7 +3,12 @@ import { constants, type BigIntStats } from "node:fs";
import { join } from "node:path";
import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
import { AnalysisInputError } from "../domain/analysisErrorCore.js";
import {
AnalysisCapabilityUnavailableError,
AnalysisInputError,
AnalysisResourceConstraintError,
} from "../domain/analysisErrorCore.js";
import { SQLITE_PROVIDER_IDENTITY } from "../domain/sqlite/sqliteDatabase.js";
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
import {
readStableArtifact,
@@ -15,6 +20,49 @@ const fileSystem: StableArtifactFileSystem = {
lstat: (path) => lstat(path, { bigint: true }),
open,
};
/** Probe a generated sibling only after the caller-selected database is observed. */
const optionalSidecarStat = async (
path: string,
databasePath: string,
observedDatabase: BigIntStats,
member: "WAL" | "rollback journal",
selectedFileSystem: StableArtifactFileSystem,
): Promise<BigIntStats | null> => {
let observedSidecar: BigIntStats | null;
try {
observedSidecar = await optionalStat(path, selectedFileSystem);
} catch (cause: unknown) {
if (
cause instanceof Error &&
"code" in cause &&
cause.code === "ENAMETOOLONG"
) {
const currentDatabase = await duringAcquisition(databasePath, () =>
optionalStat(databasePath, selectedFileSystem),
);
if (!sameState(observedDatabase, currentDatabase))
throw new ArtifactReaderFailure(
"integrity",
`SQLite snapshot file set changed before sidecar inspection: ${databasePath}`,
{ cause },
);
const remedy =
"Use the application's supported consistent offline export or copy workflow to move the complete known database file set to a shorter path. Do not rename a live database or omit its WAL or journal.";
throw new AnalysisCapabilityUnavailableError(
SQLITE_PROVIDER_IDENTITY.id,
"inspect_sqlite_database",
`Cannot safely address the derived SQLite ${member} path (${String(cause.code)}): ${path}`,
{
cause,
userMessage: `The derived SQLite ${member} path is too long to inspect safely (ENAMETOOLONG). ${remedy}`,
},
);
}
throw cause;
}
return observedSidecar;
};
const optionalStat = async (
path: string,
selectedFileSystem: StableArtifactFileSystem,
@@ -50,13 +98,35 @@ const duringAcquisition = async <T>(
if (
cause instanceof Error &&
"code" in cause &&
["ENOENT", "ENOTDIR", "ELOOP"].includes(String(cause.code))
["ENOENT", "ENOTDIR", "ELOOP", "ENAMETOOLONG"].includes(
String(cause.code),
)
)
throw new ArtifactReaderFailure(
"integrity",
`SQLite snapshot file set changed during acquisition: ${path}`,
{ cause },
);
if (cause instanceof ArtifactReaderFailure && cause.reason === "limit") {
const changed = new ArtifactReaderFailure(
"integrity",
`SQLite snapshot file set exceeded its observed size during acquisition: ${path}`,
{
cause,
...(cause.partialObservation === undefined
? {}
: { partialObservation: cause.partialObservation }),
},
cause.details,
);
if (cause.cleanup !== undefined)
throw ArtifactReaderFailure.withCleanup(
changed,
cause.cleanup,
cause.partialObservation,
);
throw changed;
}
throw cause;
}
};
@@ -186,30 +256,50 @@ export const captureSqliteDatabaseSnapshot = async (
selectedFileSystem: StableArtifactFileSystem = fileSystem,
) => {
signal?.throwIfAborted();
const paths = [path, `${path}-wal`, `${path}-journal`];
const before = await Promise.all(
paths.map((path) => optionalStat(path, selectedFileSystem)),
);
const database = before[0];
const wal = before[1];
if (database === undefined || wal === undefined || before[2] === undefined)
throw new Error("Incomplete database file-set observation");
const walPath = `${path}-wal`;
const journalPath = `${path}-journal`;
const paths = [path, walPath, journalPath] as const;
const database = await optionalStat(path, selectedFileSystem);
if (database === null) throw missingDatabase(path);
if (before[2] !== null)
const wal = await optionalSidecarStat(
walPath,
path,
database,
"WAL",
selectedFileSystem,
);
const journal = await optionalSidecarStat(
journalPath,
path,
database,
"rollback journal",
selectedFileSystem,
);
const before = [database, wal, journal] as const;
const selectedSize = database.size + (wal?.size ?? 0n);
if (selectedSize > BigInt(SQLITE_DATABASE_LIMITS.inputBytes))
throw new AnalysisResourceConstraintError(
"inspect_sqlite_database",
"file-size",
`Selected SQLite database and WAL total ${selectedSize.toString()} bytes, exceeding the combined ${String(SQLITE_DATABASE_LIMITS.inputBytes)}-byte offline snapshot budget: ${path}`,
{
sqlite_snapshot_input_bytes: SQLITE_DATABASE_LIMITS.inputBytes,
observed_sqlite_snapshot_input_bytes: selectedSize.toString(),
},
{
remediationAction:
"Use the application's supported export workflow to create a consistent, smaller offline SQLite snapshot containing only the relevant data, then inspect that snapshot. REA will not truncate or modify the selected source.",
},
);
if (journal !== null)
await verifyInactiveJournal(
`${path}-journal`,
before[2],
journal,
selectedFileSystem,
resourceScope,
"initial",
signal,
);
const selectedSize = database.size + (wal?.size ?? 0n);
if (selectedSize > BigInt(SQLITE_DATABASE_LIMITS.inputBytes))
throw new ArtifactReaderFailure(
"limit",
`Selected SQLite database and WAL exceed the combined ${String(SQLITE_DATABASE_LIMITS.inputBytes)}-byte snapshot budget: ${path}`,
);
const snapshotPath = join(root, "database.snapshot");
const captured = await readObservedArtifact(
path,
@@ -252,10 +342,10 @@ export const captureSqliteDatabaseSnapshot = async (
}
// Identity alone does not establish an inactive journal when timestamps are
// coarse. Re-read the header after copying, using the same validity owner.
if (before[2] !== null)
if (journal !== null)
await verifyInactiveJournal(
`${path}-journal`,
before[2],
journal,
selectedFileSystem,
resourceScope,
"after-copy",
@@ -0,0 +1,194 @@
import { lstat, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, it } from "vitest";
import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
import type { StableArtifactFileSystem } from "../artifacts/readStableArtifact.js";
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
import type { AnalysisPartialObservation } from "../domain/analysisErrorBase.js";
import { captureSqliteDatabaseSnapshot } from "./SqliteDatabaseSnapshot.js";
import { SQLITE_DATABASE_LIMITS } from "./SqliteDatabaseLimits.js";
import { sqliteDatabaseFailure } from "./SqliteDatabaseFailures.js";
it("reports filesystem sizes above the safe-integer range exactly before opening files", async () => {
const databaseBytes = BigInt(Number.MAX_SAFE_INTEGER) + 1n;
const walBytes = 17n;
const workspace = await mkdtemp(join(tmpdir(), "rea-sqlite-budget-"));
const databasePath = join(workspace, "selected.db");
const walPath = `${databasePath}-wal`;
const journalPath = `${databasePath}-journal`;
const privateRoot = join(workspace, "private");
await writeFile(databasePath, "tiny fixture");
const stat = await lstat(databasePath, { bigint: true });
const open = async (): Promise<never> => {
throw new Error("An over-budget source must be refused before open");
};
const fileSystem: StableArtifactFileSystem = {
lstat: async (path) => {
if (path === journalPath)
throw Object.assign(new Error("journal absent"), { code: "ENOENT" });
const size =
path === databasePath
? databaseBytes
: path === walPath
? walBytes
: stat.size;
return Object.assign(
Object.create(Object.getPrototypeOf(stat)) as typeof stat,
stat,
{ size },
);
},
open,
};
const resourceScope = new ArtifactResourceScope();
try {
const failure = await captureSqliteDatabaseSnapshot(
databasePath,
privateRoot,
resourceScope,
undefined,
fileSystem,
).then(
() => {
throw new Error("Expected the oversized snapshot to be refused");
},
(cause: unknown) => cause,
);
const projected = projectAnalysisError(
sqliteDatabaseFailure(failure, databasePath, "artifact-read"),
);
expect(failure).toMatchObject({
_tag: "AnalysisResourceConstraintError",
operation: "inspect_sqlite_database",
resource: "file-size",
reportedLimits: {
sqlite_snapshot_input_bytes: SQLITE_DATABASE_LIMITS.inputBytes,
observed_sqlite_snapshot_input_bytes: (
databaseBytes + walBytes
).toString(),
},
remediationAction: expect.stringContaining(
"smaller offline SQLite snapshot",
),
});
expect(projected).toMatchObject({
code: "resource_constraint",
category: "resource_constraint",
retryable: false,
details: {
reported_limits: {
sqlite_snapshot_input_bytes: SQLITE_DATABASE_LIMITS.inputBytes,
observed_sqlite_snapshot_input_bytes: (
databaseBytes + walBytes
).toString(),
},
},
remediation: {
action: expect.stringContaining("supported export workflow"),
},
});
await expect(readdir(privateRoot)).rejects.toMatchObject({
code: "ENOENT",
});
} finally {
await resourceScope.close();
await rm(workspace, { force: true, recursive: true });
}
});
it.each([
{ label: "incomplete cleanup", retainCleanup: true },
{ label: "partial observation without cleanup", retainCleanup: false },
])(
"preserves $label when a late read-limit failure becomes changed evidence",
async ({ retainCleanup }) => {
const workspace = await mkdtemp(join(tmpdir(), "rea-sqlite-read-cleanup-"));
const databasePath = join(workspace, "selected.db");
await writeFile(databasePath, "tiny fixture");
const stat = await lstat(databasePath, { bigint: true });
const cleanup = {
reason: "fixture file-handle close failed",
resources: [databasePath],
};
const partialObservation = {
operation: "address_to_file_offset",
address: "0x1000",
provider_file_offset: 16,
provider_source_path: null,
provider_image_header_hex: null,
original_file_offset: {
available: false,
reason: "fixture observation",
},
} satisfies AnalysisPartialObservation;
const details = {
logicalPath: "selected.db",
declaredSha256: null,
calculatedSha256: null,
unpacked: false,
};
const primary = new ArtifactReaderFailure(
"limit",
"Selected artifact grew beyond its byte limit",
{ partialObservation },
details,
);
const limitFailure = retainCleanup
? ArtifactReaderFailure.withCleanup(primary, cleanup)
: primary;
const fileSystem: StableArtifactFileSystem = {
lstat: async (path) => {
if (path === `${databasePath}-journal`)
throw Object.assign(new Error("journal absent"), { code: "ENOENT" });
return stat;
},
open: async () => {
throw limitFailure;
},
};
const resourceScope = new ArtifactResourceScope();
try {
const failure = await captureSqliteDatabaseSnapshot(
databasePath,
join(workspace, "private"),
resourceScope,
undefined,
fileSystem,
).then(
() => {
throw new Error("Expected the changed snapshot to be rejected");
},
(cause: unknown) => cause,
);
expect(failure).toMatchObject({ reason: "integrity" });
if (retainCleanup)
expect(failure).toMatchObject({
cleanup,
cause: expect.any(ArtifactReaderFailure),
});
expect(failure).toMatchObject({ partialObservation, details });
const projectedFailure = sqliteDatabaseFailure(
failure,
databasePath,
"artifact-read",
);
if (retainCleanup) {
expect(projectedFailure).toBeInstanceOf(ProviderCleanupError);
expect(projectedFailure).toMatchObject({
cleanupIncomplete: true,
cleanupResources: cleanup.resources,
});
}
} finally {
await resourceScope.close();
await rm(workspace, { force: true, recursive: true });
}
},
);
+168 -1
View File
@@ -1,5 +1,13 @@
import { createHash } from "node:crypto";
import { readFile, stat, writeFile } from "node:fs/promises";
import {
lstat,
readFile,
rename,
stat,
symlink,
truncate,
writeFile,
} from "node:fs/promises";
import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { expect, onTestFinished } from "vitest";
@@ -20,6 +28,165 @@ import { cliTest } from "../support/cli/cliFixture.js";
const digest = (bytes: Buffer): string =>
createHash("sha256").update(bytes).digest("hex");
for (const expectedErrno of ["ELOOP", "ENAMETOOLONG"] as const) {
cliTest.skipIf(expectedErrno === "ELOOP" && process.platform === "win32")(
`reports an initial ${expectedErrno} path failure through CLI and MCP`,
async ({ cli }) => {
const root = await createTestTempDirectory("rea-sqlite-path-failure-");
const path =
expectedErrno === "ELOOP"
? join(root, "cycle-a", "selected.db")
: join(root, "a".repeat(300), "selected.db");
if (expectedErrno === "ELOOP") {
await symlink("cycle-b", join(root, "cycle-a"));
await symlink("cycle-a", join(root, "cycle-b"));
}
const result = await cli.run({
arguments: ["inspect-sqlite-database", path, "--json"],
environment: { REA_LOG_LEVEL: "silent" },
});
expect(result.exitCode).toBe(1);
const expectedError = {
code: "invalid_request",
details: {
issues: [
{
path: ["path"],
reason: "invalid_value",
message: expect.stringContaining(expectedErrno),
},
],
},
};
expect(result.json).toMatchObject(expectedError);
const { call } = await connectLocalToolsMcp();
const response = await call("inspect_sqlite_database", { path });
expect(parseMcpToolError(response).error).toMatchObject(expectedError);
},
);
}
cliTest(
"reports the combined database and WAL byte budget through CLI and MCP",
async ({ cli }) => {
const root = await createTestTempDirectory("rea-sqlite-size-budget-");
const path = join(root, "selected.db");
const walPath = `${path}-wal`;
const maximum = 256 * 1024 * 1024;
const databaseBytes = maximum / 2;
const walBytes = databaseBytes + 1;
await writeFile(path, "");
await writeFile(walPath, "");
await truncate(path, databaseBytes);
await truncate(walPath, walBytes);
const before = await Promise.all(
[path, walPath].map(async (filePath) => ({
path: filePath,
state: await lstat(filePath, { bigint: true }),
})),
);
const cliResult = await cli.run({
arguments: ["inspect-sqlite-database", path, "--json"],
environment: { REA_LOG_LEVEL: "silent" },
});
const { call } = await connectLocalToolsMcp();
const mcpResponse = await call("inspect_sqlite_database", { path });
const mcpError = parseMcpToolError(mcpResponse).error;
const expectedError = {
code: "resource_constraint",
category: "resource_constraint",
retryable: false,
details: {
reported_limits: {
sqlite_snapshot_input_bytes: maximum,
observed_sqlite_snapshot_input_bytes: String(maximum + 1),
},
},
remediation: {
action: expect.stringContaining("supported export workflow"),
},
};
expect(cliResult.exitCode).toBe(1);
expect(cliResult.json).toMatchObject(expectedError);
expect(mcpError).toMatchObject(expectedError);
const after = await Promise.all(
before.map(async ({ path: filePath }) => ({
path: filePath,
state: await lstat(filePath, { bigint: true }),
})),
);
for (const [index, original] of before.entries()) {
const current = after[index];
expect(current?.state).toMatchObject({
dev: original.state.dev,
ino: original.state.ino,
mode: original.state.mode,
size: original.state.size,
mtimeNs: original.state.mtimeNs,
ctimeNs: original.state.ctimeNs,
});
}
},
);
cliTest.skipIf(process.platform === "win32" || !SQLITE_NATIVE_LIMITS_AVAILABLE)(
"reports an unavailable capability when a closed database sidecar path is too long",
async ({ cli }) => {
const root = await createTestTempDirectory("rea-sqlite-long-basename-");
const path = join(root, "x".repeat(255));
const originalPath = join(root, "selected.db");
const database = new DatabaseSync(originalPath);
try {
database.exec(
"CREATE TABLE selected(value TEXT); INSERT INTO selected VALUES ('retained');",
);
} finally {
database.close();
}
await rename(originalPath, path);
const selectedBefore = await lstat(path, { bigint: true });
await expect(lstat(`${path}-wal`)).rejects.toMatchObject({
code: "ENAMETOOLONG",
});
await expect(lstat(`${path}-journal`)).rejects.toMatchObject({
code: "ENAMETOOLONG",
});
const cliResult = await cli.run({
arguments: ["inspect-sqlite-database", path, "--json"],
environment: { REA_LOG_LEVEL: "silent" },
});
const { call } = await connectLocalToolsMcp();
const mcpResult = await call("inspect_sqlite_database", { path });
const expectedError = {
code: "capability_unavailable",
category: "unsupported_provider",
retryable: false,
message: expect.stringContaining("supported consistent offline export"),
details: {
provider_id: "sqlite",
operation: "inspect_sqlite_database",
reason: expect.stringContaining("ENAMETOOLONG"),
},
remediation: {
action: expect.stringContaining("supported consistent offline export"),
},
};
expect(cliResult.exitCode).toBe(1);
expect(cliResult.json).toMatchObject(expectedError);
expect(parseMcpToolError(mcpResult).error).toMatchObject(expectedError);
const selectedAfter = await lstat(path, { bigint: true });
expect(selectedAfter).toMatchObject({
dev: selectedBefore.dev,
ino: selectedBefore.ino,
mode: selectedBefore.mode,
size: selectedBefore.size,
mtimeNs: selectedBefore.mtimeNs,
ctimeNs: selectedBefore.ctimeNs,
});
},
);
cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
"bounds default context and projects exact columns without evaluating unrelated cells",
async ({ cli }) => {
@@ -121,6 +121,50 @@ it.runIf(process.platform !== "win32").each([
},
);
it.each(["ELOOP", "ENAMETOOLONG"] as const)(
"treats %s during file acquisition as changed evidence, not an invalid input path",
async (code) => {
const { database, capture } = await fixture();
const fileSystem: StableArtifactFileSystem = {
lstat: (path) => lstat(path, { bigint: true }),
open: async (path, flags) => {
if (path === database.path)
throw Object.assign(
new Error(`${code}: path changed after observation`),
{ code },
);
return open(path, flags);
},
};
await expect(capture(fileSystem)).rejects.toMatchObject({
reason: "integrity",
message: expect.stringContaining(database.path),
});
},
);
it("retains changed-evidence semantics when a sidecar path failure follows database removal", async () => {
const { database, capture } = await fixture();
const fileSystem: StableArtifactFileSystem = {
lstat: async (path) => {
if (path === database.walPath) {
await unlink(database.path);
throw Object.assign(new Error("derived sidecar path is too long"), {
code: "ENAMETOOLONG",
});
}
return lstat(path, { bigint: true });
},
open,
};
await expect(capture(fileSystem)).rejects.toMatchObject({
reason: "integrity",
message: expect.stringContaining(database.path),
});
});
it("refuses an active journal even before SQLite writes its magic header", async () => {
const { database, capture } = await fixture();
const writer = new DatabaseSync(database.path);