feat(artifacts): inspect and extract PyInstaller archives

This commit is contained in:
Matt Van Horn authored and GitHub committed 2026-10-21 12:48:29 +08:00
1 parent 22d5a40a2e
commit afa5da745e
63 files changed
+7254 -114

No files matched your search

+31
View File
@@ -4,13 +4,44 @@ import {
ArtifactPathRegistry,
annotateEnglishUnicodeCaseCollisions,
ENGLISH_UNICODE_CASE_INVENTORY_LIMITATION,
normalizeArtifactPath,
} from "./ArtifactPaths.js";
const NOTE =
"a case-insensitive destination cannot store both. This comparison is not a filesystem observation.";
it.each(["drive", "escaped", "normalization", "prefix", "collision"])(
"bounds path failure diagnostics at the owning boundary: %s",
(kind) => {
const name = "x".repeat(1024 * 1024);
const registry = new ArtifactPathRegistry();
const fail = () => {
if (kind === "drive") normalizeArtifactPath("C:" + name);
else if (kind === "escaped")
normalizeArtifactPath("\0".repeat(name.length));
else if (kind === "normalization")
normalizeArtifactPath(name + "/../child");
else {
registry.add(name, "file");
registry.add(kind === "prefix" ? name + "/child" : name, "directory");
}
};
const failure = (() => {
try {
fail();
} catch (cause: unknown) {
return cause;
}
return undefined;
})();
expect(failure).toMatchObject({ reason: "path" });
expect((failure as Error).message.length).toBeLessThan(1024);
},
);
const annotate = (paths: readonly string[]) => {
const occurrences = paths.map((logical_path) => ({
entry_kind: "file" as const,
logical_path,
limitations: [] as string[],
}));
+30 -10
View File
@@ -2,10 +2,25 @@ import { posix } from "node:path";
import { logicalPathNfc } from "../domain/artifactIdentity.js";
import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
import { ArtifactReaderFailure } from "./ArtifactReader.js";
import { ArtifactReaderFailure, type ArtifactEntry } from "./ArtifactReader.js";
const DRIVE_PREFIX = /^[A-Za-z]:/u;
// Bound diagnostic JSON expansion to 768 characters before formatting. This is
// a presentation budget, never a limit on accepted paths or retained facts.
const PATH_DIAGNOSTIC_CODE_UNITS = 128;
const pathDiagnostic = (path: string): string =>
path.length <= PATH_DIAGNOSTIC_CODE_UNITS
? JSON.stringify(path)
: `${JSON.stringify(path.slice(0, PATH_DIAGNOSTIC_CODE_UNITS))}… (${String(path.length)} UTF-16 code units)`;
type DestinationEntryKind = Exclude<ArtifactEntry["kind"], "metadata">;
/** Declarations have query identities, but never claim a destination path. */
export const hasDestinationPath = (
kind: ArtifactEntry["kind"],
): kind is DestinationEntryKind => kind !== "metadata";
/**
* English Unicode case folding used only to describe logical names a
* case-insensitive destination might collapse. It is not a filesystem case table.
@@ -25,7 +40,7 @@ export const normalizeArtifactPath = (input: string): string => {
if (input.includes("\0") || input.startsWith("/") || DRIVE_PREFIX.test(input))
throw new ArtifactReaderFailure(
"path",
`Artifact path is absolute or unsafe: ${JSON.stringify(input)}`,
`Artifact path is absolute or unsafe: ${pathDiagnostic(input)}`,
);
const normalized = logicalPathNfc(input).replace(/\/+$/u, "");
const parts = normalized.split("/");
@@ -36,7 +51,7 @@ export const normalizeArtifactPath = (input: string): string => {
)
throw new ArtifactReaderFailure(
"path",
`Artifact path is not normalized: ${JSON.stringify(input)}`,
`Artifact path is not normalized: ${pathDiagnostic(input)}`,
);
return normalized;
};
@@ -45,14 +60,15 @@ export const normalizeArtifactPath = (input: string): string => {
export class ArtifactPathRegistry {
readonly #root: PathTrieNode = { kind: undefined, children: new Map() };
add(path: string, kind: "file" | "directory" | "symlink" | "slice"): void {
add(path: string, kind: ArtifactEntry["kind"]): void {
if (!hasDestinationPath(kind)) return;
const parts = path.split("/");
let node = this.#root;
for (const [index, part] of parts.entries()) {
if (node.kind !== undefined && node.kind !== "directory")
throw new ArtifactReaderFailure(
"path",
`Artifact prefix conflict: ${path}`,
`Artifact prefix conflict: ${pathDiagnostic(path)}`,
);
let child = node.children.get(part);
if (child === undefined) {
@@ -64,12 +80,12 @@ export class ArtifactPathRegistry {
if (node.kind !== undefined && node.kind !== kind)
throw new ArtifactReaderFailure(
"path",
`Artifact path collision: ${path}`,
`Artifact path collision: ${pathDiagnostic(path)}`,
);
if (kind !== "directory" && node.children.size > 0)
throw new ArtifactReaderFailure(
"path",
`Artifact prefix conflict: ${path}`,
`Artifact prefix conflict: ${pathDiagnostic(path)}`,
);
}
}
@@ -78,7 +94,7 @@ export class ArtifactPathRegistry {
}
interface PathTrieNode {
kind: "file" | "directory" | "symlink" | "slice" | undefined;
kind: DestinationEntryKind | undefined;
readonly children: Map<string, PathTrieNode>;
}
@@ -103,15 +119,19 @@ const englishCaseFold = (part: string): string =>
*/
export const annotateEnglishUnicodeCaseCollisions = (
occurrences: readonly {
readonly entry_kind: ArtifactEntry["kind"];
readonly logical_path: string;
limitations: string[];
}[],
): string | undefined => {
const destinations = occurrences.filter(({ entry_kind }) =>
hasDestinationPath(entry_kind),
);
const collisions = englishUnicodeCaseCollisions(
occurrences.map(({ logical_path }) => logical_path),
destinations.map(({ logical_path }) => logical_path),
);
if (collisions.size === 0) return undefined;
for (const occurrence of occurrences)
for (const occurrence of destinations)
for (const collision of collisions.get(occurrence.logical_path) ?? [])
occurrence.limitations.push(
caseCollisionLimitation(occurrence.logical_path, collision),
+2 -1
View File
@@ -33,7 +33,7 @@ export interface ArtifactReaderFailureOptions extends ErrorOptions {
/** Archive-neutral entry metadata. Reader adapters never choose output paths. */
export interface ArtifactEntry {
readonly path: string;
readonly kind: "file" | "directory" | "symlink" | "slice";
readonly kind: "file" | "directory" | "symlink" | "slice" | "metadata";
readonly declaredSize: number | null;
readonly compressedSize: number | null;
readonly executable: boolean;
@@ -167,6 +167,7 @@ export interface ArtifactReader {
| ZipPackageFormat
| "asar"
| "godot-pck"
| "pyinstaller"
| "file";
entries(
signal?: AbortSignal,
+608
View File
@@ -0,0 +1,608 @@
import type { Readable } from "node:stream";
import { finished } from "node:stream/promises";
import { setImmediate } from "node:timers/promises";
import { jsonValueByteLength } from "../domain/jsonValueByteLength.js";
import type { JsonValue } from "../domain/jsonValue.js";
import { normalizeArtifactPath } from "./ArtifactPaths.js";
import { lstat, type FileHandle } from "node:fs/promises";
import type { PyInstallerMetadata } from "../domain/artifactFormatMetadata.js";
import {
cookieMetadata,
scriptPythonMagic,
} from "./pyinstaller/PyInstallerMetadata.js";
import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js";
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
import {
RegularFileAdmissionFailure,
openRegularFile,
sameRegularFileState,
type StableRegularFileDescriptor,
} from "../filesystem/RegularFile.js";
import {
ArtifactReaderFailure,
sameArtifactEntry,
type ArtifactEntry,
type ArtifactReader,
type ArtifactReaderTraversal,
type ArtifactTraversalOptions,
} from "./ArtifactReader.js";
import { abortIfNeeded, hashReadable } from "./ArtifactHash.js";
import { readArtifactFileChunks } from "./readArtifactFileChunks.js";
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import { artifactIntegrityDetails } from "../domain/artifactIntegrity.js";
import { PyInstallerPyzStore } from "./pyinstaller/PyInstallerPyzStore.js";
import { pyInstallerMemberStream } from "./pyinstaller/PyInstallerMemberStream.js";
import {
findPyInstallerCookie,
readCArchiveToc,
type CArchiveEntry,
type PyInstallerCookie,
} from "./pyinstaller/CArchiveLayout.js";
import {
hasEncryptionMarker,
projectPyzContainer,
projectPyzMember,
projectStoredMember,
sealMember,
type PyInstallerMember,
type PyzImage,
} from "./pyinstaller/CArchiveMembers.js";
import { readPyzArchive } from "./pyinstaller/PyzArchive.js";
import type { PyInstallerFailure } from "./pyinstaller/PyInstallerFailure.js";
import { pyInstallerFileAdmissionFailure } from "./pyinstaller/PyInstallerFileAdmission.js";
const MAX_PARSED_FACT_BYTES = 32 * 1024 * 1024;
const MAX_RETAINED_ENTRY_BYTES = 64 * 1024 * 1024;
const RECORD_OWNERSHIP_RESERVATION = 256;
// Conservative allowance for registry/case trie nodes, maps and traversal state.
// Repeated prefixes are charged again rather than underestimating retained work.
const PATH_COMPONENT_RESERVATION = 1024;
interface LoadedArchive {
readonly members: readonly PyInstallerMember[];
readonly traversal: ArtifactReaderTraversal;
}
/**
* Read-only inventory of a PyInstaller CArchive appended to a native executable.
* Encrypted PYZ entries are reported and not decoded. Code members are not decompiled.
*/
export class PyInstallerArtifactReader implements ArtifactReader {
readonly format = "pyinstaller" as const;
#source: StableRegularFileDescriptor | undefined;
#opening: Promise<FileHandle> | undefined;
#closed = false;
readonly #scope = new ArtifactResourceScope();
readonly #cleanup = new ArtifactResourceScope();
readonly #lifecycle = new AbortController();
#boundSha256: string | undefined;
#loaded: Promise<LoadedArchive> | undefined;
#budget = 0;
#charged = 0;
#parsedFactBytes = 0;
#retainedEntryBytes = 0;
#metadata: PyInstallerMetadata | undefined;
readonly #pyzStore = new PyInstallerPyzStore();
readonly #members = new Map<string, PyInstallerMember>();
readonly #openedPyzSizes = new Map<string, number>();
constructor(
private readonly path: string,
source?: StableRegularFileDescriptor,
) {
this.#source = source;
this.#cleanup.retain({
kind: "lease",
resource: "PyInstaller private PYZ images",
lease: { release: () => this.#pyzStore.close() },
});
}
formatMetadata(): PyInstallerMetadata | undefined {
return this.#metadata === undefined ? undefined : { ...this.#metadata };
}
async prepareContainer(
expectedSha256: string,
signal?: AbortSignal,
): Promise<void> {
const active = this.#signal(signal);
await this.#scope.run(async () => {
const handle = await this.#openHandle(active);
await this.#verifySource();
const observed =
this.#boundSha256 ??
(
await hashReadable(
readArtifactFileChunks(handle, { signal: active }),
active,
)
).sha256;
await this.#verifySource();
abortIfNeeded(active);
if (observed !== expectedSha256)
throw new ArtifactReaderFailure(
"integrity",
"PyInstaller carrier changed after inventory",
undefined,
artifactIntegrityDetails(
".",
{ algorithm: "sha256", digest: expectedSha256 },
observed,
false,
),
);
this.#boundSha256 = observed;
});
}
async *entries(
signal?: AbortSignal,
options?: ArtifactTraversalOptions,
): AsyncIterable<ArtifactEntry> {
const active = this.#signal(signal);
const loaded = await this.#scope.run(async () => {
const value = await this.#load(active);
await this.#verifySource();
this.#members.clear();
return value;
});
options?.onCoverage?.(loaded.traversal);
for (const member of loaded.members) {
abortIfNeeded(active);
const sealed = sealMember(member);
this.#members.set(sealed.entry.adapterKey, sealed);
yield sealed.entry;
}
}
async open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
const active = this.#signal(signal);
return this.#scope.run(async () => {
const stored = this.#members.get(entry.adapterKey);
if (stored === undefined || !sameArtifactEntry(entry, stored.entry))
throw new ArtifactReaderFailure(
"integrity",
`PyInstaller entry was not produced by this reader: ${entry.path}`,
);
if (entry.kind !== "file")
throw new ArtifactReaderFailure(
"format",
`PyInstaller entry is not a regular file: ${entry.path}`,
);
if (entry.encrypted)
throw new ArtifactReaderFailure(
"unavailable",
`PYZ entry is encrypted and was not decoded: ${entry.path}`,
);
const read = stored.read;
if (read === undefined)
throw new ArtifactReaderFailure(
"format",
`PyInstaller entry has no readable bytes: ${entry.path}`,
);
await this.#verifySource();
const knownSize =
read.kind === "pyz"
? (read.uncompressedSize ??
this.#openedPyzSizes.get(entry.adapterKey))
: undefined;
if (read.kind === "pyz" && knownSize === undefined)
this.#charge(read.header?.length ?? 0);
const parts = await this.#pyzStore.memberSource(
read,
this.#source?.handle,
active,
);
if (active.aborted) {
parts.source.destroy();
await finished(parts.source, { cleanup: true }).catch(() => undefined);
abortIfNeeded(active);
}
// Keep one close attempt in this read. Failed owners stay in the store
// for reader.close() to retry, without losing the original failure.
let closeAttempt: Promise<void> | undefined;
const closeSource = (primary?: unknown) =>
(closeAttempt ??= parts.close?.(primary));
const stream = pyInstallerMemberStream(parts.source, {
sourceBytes:
read.kind === "carchive" ? read.source.compressedSize : read.length,
compressed:
read.kind === "pyz"
? read.compressed
: read.source.compressionFlag === 1,
header: read.header,
expectedBytes:
read.kind === "carchive" ? read.source.uncompressedSize : knownSize,
maxBytes:
read.kind === "carchive"
? read.source.uncompressedSize
: (knownSize ?? this.#budget - this.#charged),
signal: active,
...(read.kind === "pyz" && knownSize === undefined
? { charge: (bytes: number) => this.#charge(bytes) }
: {}),
complete: async (bytes) => {
try {
await parts.verify?.();
await this.#verifySource();
abortIfNeeded(active);
} catch (cause: unknown) {
await closeSource(cause);
throw cause;
}
await closeSource();
abortIfNeeded(active);
if (read.kind === "pyz" && knownSize === undefined)
this.#openedPyzSizes.set(entry.adapterKey, bytes);
},
});
void this.#scope
.run(async () => {
let primary: unknown;
try {
await stream.completion;
} catch (cause: unknown) {
primary = cause;
}
await closeSource(primary);
if (primary !== undefined) throw primary;
})
.catch((cause: unknown) =>
stream.output.destroy(
cause instanceof Error
? cause
: new Error("PyInstaller stream failed", { cause }),
),
);
return stream.output;
});
}
provenance(): readonly [] {
return [];
}
async close(): Promise<void> {
this.#closed = true;
const closing = new ArtifactReaderFailure(
"unavailable",
"PyInstaller reader closed",
);
this.#lifecycle.abort(closing);
await this.#scope.close();
await this.#cleanup.close();
this.#source = undefined;
this.#closed = true;
this.#loaded = undefined;
this.#members.clear();
this.#openedPyzSizes.clear();
}
#signal(signal?: AbortSignal): AbortSignal {
abortIfNeeded(signal);
if (this.#closed)
throw new ArtifactReaderFailure(
"unavailable",
"PyInstaller archive is closed",
);
return signal === undefined
? this.#lifecycle.signal
: AbortSignal.any([signal, this.#lifecycle.signal]);
}
async #load(signal?: AbortSignal): Promise<LoadedArchive> {
this.#loaded ??= this.#parse(signal);
return this.#loaded;
}
async #parse(signal?: AbortSignal): Promise<LoadedArchive> {
const handle = await this.#openHandle(signal);
await this.#verifySource();
const size = (await handle.stat()).size;
const cookie = await this.#cookie(handle, size, signal);
this.#budget = artifactExpansionBudgetBytes(cookie.packageLength);
const toc = await this.#toc(handle, cookie, signal);
for (const entry of toc) {
this.#charge(entry.uncompressedSize);
this.#reserveMetadata({ ...entry }, false);
}
const images = await this.#pyzImages(handle, cookie, toc, signal);
const magic = scriptPythonMagic(
cookie,
[...images.values()].map((image) => image.archive),
);
const encrypted = hasEncryptionMarker(toc);
const members = await this.#project(
toc,
cookie,
images,
magic,
encrypted,
signal,
);
for (const member of members)
if (member.read?.kind === "carchive")
this.#charge(member.read.header?.length ?? 0);
await this.#verifySource();
this.#metadata = cookieMetadata(cookie, toc, encrypted);
let opaqueImages = 0;
for (const image of images.values())
if (image.archive.kind === "opaque") opaqueImages++;
return {
members,
traversal: {
status: opaqueImages === 0 ? "complete" : "unknown",
limitations:
opaqueImages === 0
? []
: [
`${String(opaqueImages)} PYZ archives remain opaque; module traversal is unknown. See opaque PYZ occurrences for source magic and per-archive limitations.`,
],
},
};
}
async #openHandle(signal?: AbortSignal): Promise<FileHandle> {
abortIfNeeded(signal);
if (this.#closed)
throw new ArtifactReaderFailure(
"unavailable",
"PyInstaller archive is closed",
);
if (this.#source !== undefined) return this.#source.handle;
// Admission belongs to the reader. A departing caller cannot cancel a peer
// or replace a late descriptor owner; every caller joins the same admission.
this.#opening ??= this.#admit();
let handle: FileHandle;
try {
handle = await this.#opening;
} catch (cause: unknown) {
if (
cause instanceof ArtifactReaderFailure &&
cause.reason === "cancelled" &&
signal?.aborted
)
throw new ArtifactReaderFailure("cancelled", cause.message, {
cause: signal.reason,
cleanup: cause.cleanup,
cleanupLeases: cause.cleanupLeases,
});
throw cause;
}
abortIfNeeded(signal);
return handle;
}
async #admit(): Promise<FileHandle> {
try {
const handle = await openRegularFile(this.path, {
symlinks: "reject",
signal: this.#lifecycle.signal,
});
this.#retainCarrier(new OwnedFileHandle(handle));
this.#source = { handle, initial: await handle.stat() };
return handle;
} catch (cause: unknown) {
if (cause instanceof RegularFileAdmissionFailure)
this.#retainCarrier(cause.owner);
throw pyInstallerFileAdmissionFailure(this.path, cause);
}
}
#retainCarrier(handle: OwnedFileHandle): void {
this.#cleanup.retain({ kind: "file-handle", resource: this.path, handle });
}
async #cookie(
handle: FileHandle,
size: number,
signal?: AbortSignal,
): Promise<PyInstallerCookie> {
const found = await findPyInstallerCookie(handle, size, signal);
if (found === undefined)
throw new ArtifactReaderFailure(
"format",
"PyInstaller CArchive cookie was not found",
);
if (!found.ok) throw failure(found.error);
return found.value;
}
async #toc(
handle: FileHandle,
cookie: PyInstallerCookie,
signal?: AbortSignal,
): Promise<readonly CArchiveEntry[]> {
const toc = await readCArchiveToc(handle, cookie, signal);
if (!toc.ok) throw failure(toc.error);
return toc.value;
}
async #pyzImages(
handle: FileHandle,
cookie: PyInstallerCookie,
toc: readonly CArchiveEntry[],
signal: AbortSignal | undefined,
): Promise<ReadonlyMap<CArchiveEntry, PyzImage>> {
const images = new Map<CArchiveEntry, PyzImage>();
// Admit the complete filesystem footprint before the first private write.
this.#pyzStore.reserveImages(
toc.reduce(
(count, entry) =>
count +
(entry.typeCode === "z" && entry.compressionFlag === 1 ? 1 : 0),
0,
),
);
for (const entry of toc) {
if (entry.typeCode !== "z") continue;
abortIfNeeded(signal);
const spooled =
entry.compressionFlag === 0
? undefined
: await this.#pyzStore.spool(
handle,
cookie.archiveStart,
entry,
signal,
);
const source = spooled ?? {
handle,
start: cookie.archiveStart + entry.offset,
};
const parsed = await readPyzArchive(
source.handle,
source.start,
entry.uncompressedSize,
signal,
);
if (!parsed.ok) throw failure(parsed.error);
if (parsed.value.kind === "entries")
for (const item of parsed.value.entries)
this.#reserveMetadata({ ...item, container: entry.name }, false);
if (spooled !== undefined) await this.#pyzStore.release(spooled.handle);
images.set(entry, {
archive: parsed.value,
handle: source.handle,
start: source.start,
});
}
return images;
}
async #project(
toc: readonly CArchiveEntry[],
cookie: PyInstallerCookie,
images: ReadonlyMap<CArchiveEntry, PyzImage>,
magic: Buffer | undefined,
encrypted: boolean,
signal: AbortSignal | undefined,
): Promise<PyInstallerMember[]> {
const members: PyInstallerMember[] = [];
const append = (member: PyInstallerMember): void => {
this.#reservePathComponents(member.entry.path);
const entry = {
...member.entry,
path: normalizeArtifactPath(member.entry.path),
};
this.#reserveMetadata(
{
path: entry.path,
adapter_key: entry.adapterKey,
format_metadata: entry.formatMetadata ?? null,
limitations: [...entry.limitations],
},
true,
);
members.push({ ...member, entry });
};
for (const entry of toc) {
if (members.length % 256 === 0) await setImmediate();
abortIfNeeded(signal);
const image = entry.typeCode === "z" ? images.get(entry) : undefined;
if (image === undefined) {
append(projectStoredMember(entry, cookie.archiveStart, magic));
continue;
}
append(projectPyzContainer(entry, cookie.archiveStart, image, encrypted));
if (image.archive.kind !== "entries") continue;
for (const item of image.archive.entries) {
if (members.length % 256 === 0) await setImmediate();
abortIfNeeded(signal);
append(
projectPyzMember(
entry,
item,
encrypted,
image.archive.pythonMagic,
item.namespace || encrypted
? undefined
: {
handle: image.handle,
offset: image.start + item.offset,
length: item.length,
},
),
);
}
}
return members;
}
async #verifySource(): Promise<void> {
const source = this.#source;
if (source === undefined)
throw new ArtifactReaderFailure(
"unavailable",
"PyInstaller archive is closed",
);
const [opened, currentPath] = await Promise.all([
source.handle.stat(),
lstat(this.path),
]);
if (
!sameRegularFileState(source.initial, opened) ||
!sameRegularFileState(source.initial, currentPath)
)
throw new ArtifactReaderFailure(
"integrity",
`PyInstaller source changed during reading: ${this.path}`,
);
}
#reservePathComponents(path: string): void {
const remaining = MAX_RETAINED_ENTRY_BYTES - this.#retainedEntryBytes;
// Case annotations can carry the path and two prefixes at every component.
// Reserve escaped text growth as well as registry/case trie nodes and maps.
const componentCost =
PATH_COMPONENT_RESERVATION + 6 * jsonValueByteLength(path);
let reservation = componentCost;
for (
let index = 0;
index < path.length && reservation <= remaining;
index++
)
if (path.charCodeAt(index) === 47) reservation += componentCost;
if (reservation > remaining)
throw new ArtifactReaderFailure(
"limit",
"PyInstaller paths exceed their retained namespace budget",
{ boundary: "memory" },
);
this.#retainedEntryBytes += reservation;
}
#reserveMetadata(value: JsonValue, projected: boolean): void {
const used = projected ? this.#retainedEntryBytes : this.#parsedFactBytes;
const limit = projected ? MAX_RETAINED_ENTRY_BYTES : MAX_PARSED_FACT_BYTES;
const remaining = limit - used - RECORD_OWNERSHIP_RESERVATION;
const bytes = remaining < 0 ? 0 : jsonValueByteLength(value, remaining);
if (remaining < 0 || bytes > remaining)
throw new ArtifactReaderFailure(
"limit",
"PyInstaller directory facts exceed their retained metadata budget",
{ boundary: "memory" },
);
if (projected)
this.#retainedEntryBytes += bytes + RECORD_OWNERSHIP_RESERVATION;
else this.#parsedFactBytes += bytes + RECORD_OWNERSHIP_RESERVATION;
}
#charge(bytes: number): void {
if (
!Number.isSafeInteger(bytes) ||
bytes < 0 ||
this.#charged + bytes > this.#budget
)
throw new ArtifactReaderFailure(
"limit",
`PyInstaller members declare ${String(this.#charged + Math.max(0, bytes))} uncompressed bytes, exceeding the decompression budget`,
);
this.#charged += bytes;
}
}
const failure = (error: PyInstallerFailure): ArtifactReaderFailure =>
new ArtifactReaderFailure(error.reason, error.message);
@@ -0,0 +1,54 @@
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect, it, vi } from "vitest";
import { ArtifactReaderFailure } from "../ArtifactReader.js";
import { ArtifactResourceScope } from "../ArtifactResourceScope.js";
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { extractArtifact } from "./ArtifactExtraction.js";
vi.mock("../pyinstaller/CArchiveLayout.js", async (importOriginal) => ({
...(await importOriginal<object>()),
containsPyInstallerArchive: () =>
Promise.reject(new ArtifactReaderFailure("format", "detection failed")),
}));
it("keeps the detection error primary and reports a failed detection close", async () => {
const directory = await createTestTempDirectory("rea-extraction-detect-");
const path = join(directory, "target.elf");
await writeFile(path, Buffer.from("not an archive"));
const resourceScope = new ArtifactResourceScope();
const originalClose = OwnedFileHandle.prototype.close;
const closeFailure = new Error("detection close failed");
let attempts = 0;
const close = vi
.spyOn(OwnedFileHandle.prototype, "close")
.mockImplementation(async function (this: OwnedFileHandle) {
attempts += 1;
if (attempts === 1) throw closeFailure;
await originalClose.call(this);
});
try {
await expect(
extractArtifact({
inputPath: path,
inputFormat: "elf",
outputRoot: join(directory, "out"),
environment: process.env,
integrityPolicy: "fail",
resourceScope,
}),
).rejects.toMatchObject({
reason: "format",
message: "detection failed",
cleanup: { reason: closeFailure.message },
});
await resourceScope.close();
expect(attempts).toBe(2);
} finally {
close.mockRestore();
await resourceScope.close().catch(() => undefined);
}
});
+24 -10
View File
@@ -23,6 +23,8 @@ import { SafeOutputTreeCreationFailure } from "../SafeOutputTreeCreationFailure.
import { ZipArtifactReader } from "../ZipArtifactReader.js";
import { isZipPackageFormat } from "../../domain/zipPackageFormat.js";
import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js";
import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js";
import { containsPyInstallerArchive } from "../pyinstaller/CArchiveLayout.js";
import {
artifactExtractionResultSchema,
type ArtifactExtractionResult,
@@ -244,6 +246,7 @@ const materializeSelection = async ({
sourcePath,
input,
inventory.manifest.root_format,
signal,
);
const readerOwner: ArtifactResourceOwner = {
kind: "reader" as const,
@@ -314,14 +317,15 @@ const writeSelectedEntries = async ({
for await (const entry of reader.entries(signal)) {
const path = normalizeArtifactPath(entry.path);
registry.add(path, entry.kind);
// Logical declarations may share a query path with a real file. They do
// not consume that file's selection or participate in materialization.
if (entry.kind !== "file" && entry.kind !== "slice") continue;
const selectedItem = byPath.get(path);
if (selectedItem === undefined) {
if (entry.kind === "file" || entry.kind === "slice")
throw new ArtifactReaderFailure(
"integrity",
`Regular artifact entry is missing from inventory: ${path}`,
);
continue;
throw new ArtifactReaderFailure(
"integrity",
`Regular artifact entry is missing from inventory: ${path}`,
);
}
preflight(entry);
const stream = await reader.open(entry, signal);
@@ -444,10 +448,17 @@ const requireExtractableFormat = async (
path,
input.resourceScope,
signal,
async (handle) =>
locateGodotPck(handle, (await handle.stat()).size, signal),
async (handle) => {
const size = (await handle.stat()).size;
if (await locateGodotPck(handle, size, signal)) return true;
if (format !== "elf" && format !== "pe") return false;
abortIfNeeded(signal);
const packaged = await containsPyInstallerArchive(handle, size, signal);
abortIfNeeded(signal);
return packaged;
},
);
if (pack === undefined)
if (!pack)
throw new AnalysisUnsupportedTargetError(
"extract_artifact",
input.inputPath,
@@ -464,10 +475,13 @@ const createReader = async (
"inputPath" | "inputFormat" | "environment" | "resourceScope"
>,
containerFormat: ArtifactGraphManifest["root_format"],
signal?: AbortSignal,
): Promise<ArtifactReader> => {
const format = input.inputFormat;
if (containerFormat === "godot-pck") return new GodotPckArtifactReader(path);
if (await requireExtractableFormat(path, input))
if (containerFormat === "pyinstaller")
return new PyInstallerArtifactReader(path);
if (await requireExtractableFormat(path, input, signal))
return new DirectoryArtifactReader(path);
if (format === "asar") return new AsarArtifactReader(path);
if (isZipPackageFormat(format)) return new ZipArtifactReader(path, format);
@@ -10,7 +10,7 @@ import {
createArtifactEdges,
classifyArtifactContent,
createOccurrence,
indexOccurrencesByPath,
indexDestinationOccurrencesByPath,
resolveOccurrenceParents,
} from "./ArtifactGraphConstruction.js";
import { createArtifactClassificationProbe } from "../ArtifactClassificationProbe.js";
@@ -42,7 +42,7 @@ describe("occurrence parent ownership", () => {
resolveOccurrenceParents(
occurrences,
indexOccurrencesByPath(occurrences),
indexDestinationOccurrencesByPath(occurrences),
new Set(),
);
@@ -57,7 +57,7 @@ describe("occurrence parent ownership", () => {
const resolution = resolveOccurrenceParents(
occurrences,
indexOccurrencesByPath(occurrences),
indexDestinationOccurrencesByPath(occurrences),
new Set(),
);
@@ -1,6 +1,7 @@
import { hasGodotPckHeader } from "../../domain/godotPckFormat.js";
import type { ArtifactClassificationEvidence } from "../ArtifactClassificationProbe.js";
import { ArtifactReaderFailure } from "../ArtifactReader.js";
import { hasDestinationPath } from "../ArtifactPaths.js";
import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";
import { isJavaScriptSourcePath } from "../../domain/javascript/javascriptArtifactFiles.js";
import { digestCanonicalValue } from "../../domain/canonicalDigest.js";
@@ -56,12 +57,13 @@ export interface MutableOccurrence {
limitations: string[];
}
/** Index every occurrence by canonical logical path without choosing an owner. */
export const indexOccurrencesByPath = (
/** Index destination paths without mixing in declaration query names or choosing an owner. */
export const indexDestinationOccurrencesByPath = (
occurrences: readonly MutableOccurrence[],
): ReadonlyMap<string, readonly MutableOccurrence[]> => {
const byPath = new Map<string, MutableOccurrence[]>();
for (const occurrence of occurrences) {
if (!hasDestinationPath(occurrence.entry_kind)) continue;
const matches = byPath.get(occurrence.logical_path);
if (matches === undefined)
byPath.set(occurrence.logical_path, [occurrence]);
@@ -82,16 +84,18 @@ export const createOccurrence = (
duplicateOrdinal = 0,
): MutableOccurrence => {
const role =
entry.kind === "directory"
? {
kind: path.toLowerCase().endsWith(".framework")
? ("framework" as const)
: ("container" as const),
format: "directory" as const,
}
: containerSuffixFamily(path) === undefined
? classifyArtifactPath(path)
: ({ kind: "unknown", format: "unknown" } as const);
entry.kind === "metadata"
? { kind: "package-metadata" as const, format: "unknown" as const }
: entry.kind === "directory"
? {
kind: path.toLowerCase().endsWith(".framework")
? ("framework" as const)
: ("container" as const),
format: "directory" as const,
}
: containerSuffixFamily(path) === undefined
? classifyArtifactPath(path)
: ({ kind: "unknown", format: "unknown" } as const);
return {
occurrence_id: `occ_${digestCanonicalValue(
duplicateOrdinal === 0
@@ -263,7 +267,7 @@ export const rootOccurrenceFor = (
artifact_kind:
metadata.format === "directory" ||
isZipPackageFormat(metadata.format) ||
["asar", "godot-pck", "dmg", "pkg"].includes(metadata.format)
["asar", "godot-pck", "dmg", "pkg", "pyinstaller"].includes(metadata.format)
? "container"
: artifactRoleForFormat(metadata.path, metadata.format).kind,
artifact_format: metadata.format,
@@ -316,7 +320,7 @@ export const createArtifactEdges = (
producer?: ArtifactCommand,
): { readonly edges: ArtifactEdge[]; readonly limitations: string[] } => {
const byId = new Map(occurrences.map((item) => [item.occurrence_id, item]));
const byPath = indexOccurrencesByPath(occurrences);
const byPath = indexDestinationOccurrencesByPath(occurrences);
const edges: ArtifactEdge[] = [];
const limitations: string[] = [];
for (const occurrence of occurrences) {
@@ -364,11 +368,14 @@ export const createArtifactEdges = (
relation:
occurrence.entry_kind === "slice"
? ("slice-of" as const)
: isSourceMap
? mappedSource === undefined
? ("contains" as const)
: ("maps-source" as const)
: relationFor(occurrence.logical_path),
: occurrence.format_metadata?.kind === "pyinstaller-entry" &&
occurrence.format_metadata.bytecode_header === "derived"
? ("derived-from" as const)
: isSourceMap
? mappedSource === undefined
? ("contains" as const)
: ("maps-source" as const)
: relationFor(occurrence.logical_path),
occurrence_id: occurrence.occurrence_id,
};
edges.push({
@@ -428,6 +435,9 @@ export const resolveOccurrenceParents = (
const limitations = new Set<string>();
const unknownParentOccurrences = new Set<MutableOccurrence>();
for (const occurrence of occurrences) {
// Query paths do not imply filesystem containment. The scanner records
// the declaring container for metadata instead.
if (occurrence.entry_kind === "metadata") continue;
const path = occurrence.logical_path;
let branch = root;
let parent: MutableOccurrence | undefined;
+14 -2
View File
@@ -5,6 +5,7 @@ import type { Stats } from "node:fs";
import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js";
import type { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
import { containsPyInstallerArchive } from "../pyinstaller/CArchiveLayout.js";
import { classifyArtifactContent } from "./ArtifactGraphConstruction.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import { createArtifactClassificationProbe } from "../ArtifactClassificationProbe.js";
@@ -114,7 +115,8 @@ export const classifyAndHashRootForInventory = async (
const retainSource =
isZipPackageFormat(format) ||
format === "mach-o-universal" ||
format === "godot-pck";
format === "godot-pck" ||
format === "pyinstaller";
outcome = {
kind: "completed",
value: {
@@ -178,7 +180,7 @@ const classifyRootFormat = async (
evidence.fatSliceHeaders,
signal,
);
return classifyArtifactContent(
const format = classifyArtifactContent(
path,
{
...evidence,
@@ -186,6 +188,16 @@ const classifyRootFormat = async (
},
archiveBytes,
).format;
if (["elf", "pe", "mach-o", "mach-o-universal"].includes(format)) {
const packaged = await containsPyInstallerArchive(
handle,
archiveBytes,
signal,
);
abortIfNeeded(signal);
if (packaged) return "pyinstaller";
}
return format;
};
/** Read exact FAT header windows missed by the bounded prefix probe. */
+3
View File
@@ -7,6 +7,7 @@ import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js";
import { GodotPckArtifactReader } from "../GodotPckArtifactReader.js";
import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js";
import { NativeDmgArtifactReader } from "../NativeDmgArtifactReader.js";
import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js";
import { ZipArtifactReader } from "../ZipArtifactReader.js";
import type {
ArtifactInventoryCoverage,
@@ -31,6 +32,8 @@ export const createReader = (
return new DirectoryArtifactReader(path);
case "asar":
return new AsarArtifactReader(path);
case "pyinstaller":
return new PyInstallerArtifactReader(path, rootSource);
case "godot-pck":
return new GodotPckArtifactReader(path, rootSource);
case "mach-o-universal":
+6 -3
View File
@@ -26,7 +26,7 @@ import {
createArtifactNode,
createOccurrence,
noteUnreadContainerSuffix,
indexOccurrencesByPath,
indexDestinationOccurrencesByPath,
resolveOccurrenceParents,
type MutableOccurrence,
} from "./ArtifactGraphConstruction.js";
@@ -107,7 +107,7 @@ export const scanReader = async (
await visitArtifactEntries(context, reader, "");
// Archive directories may appear after their children. Resolve containment
// against the complete index before directory identities are materialized.
const byPath = indexOccurrencesByPath(occurrences);
const byPath = indexDestinationOccurrencesByPath(occurrences);
const parentResolution = resolveOccurrenceParents(
occurrences,
byPath,
@@ -167,11 +167,13 @@ const visitArtifactEntries = async (
const stack: Array<{
readonly reader: ArtifactReader;
readonly prefix: string;
readonly containerId: string | null;
readonly iterator: AsyncIterator<ArtifactEntry>;
}> = [
{
reader: currentReader,
prefix,
containerId: null,
iterator: inventoryEntries(context, currentReader, prefix),
},
];
@@ -202,7 +204,7 @@ const visitArtifactEntries = async (
const occurrence = createOccurrence(
entry,
logicalPath,
null,
entry.kind === "metadata" ? frame.containerId : null,
duplicateOrdinal,
);
let digested:
@@ -254,6 +256,7 @@ const visitArtifactEntries = async (
stack.push({
reader: nested,
prefix: logicalPath,
containerId: occurrence.occurrence_id,
iterator: inventoryEntries(context, nested, logicalPath),
});
}
@@ -0,0 +1,211 @@
import { deflateSync } from "node:zlib";
import { PYINSTALLER_COOKIE_MAGIC } from "./CArchiveLayout.js";
/** Final CPython 3.9 bytecode magic, little-endian with the CR LF trailer. */
export const PYTHON_39_MAGIC = Buffer.from("610d0d0a", "hex");
const ELF_PREFIX = (() => {
const bytes = Buffer.alloc(64);
bytes.writeUInt32LE(0x464c457f, 0);
bytes[4] = 2;
bytes[5] = 1;
bytes[6] = 1;
bytes.writeUInt16LE(2, 16);
bytes.writeUInt16LE(62, 18);
bytes.writeUInt32LE(1, 20);
bytes.writeUInt16LE(64, 52);
return bytes;
})();
/** One CArchive member supplied as uncompressed bytes. */
export interface CArchiveFixtureMember {
readonly name: string;
readonly typeCode: string;
readonly payload: Buffer;
readonly compress: boolean;
readonly compressedSuffix?: Buffer;
readonly declaredUncompressedSize?: number;
}
/** Build a tiny ELF with an appended CArchive, optionally followed by a trailer. */
export const cArchiveExecutable = (input: {
readonly layout: "2.0" | "2.1";
readonly pythonVersion: number;
readonly pythonLibrary?: string;
readonly members: readonly CArchiveFixtureMember[];
readonly trailer?: Buffer;
readonly tocBytes?: Buffer;
}): Buffer => {
const payloads: Buffer[] = [];
const entries: Buffer[] = [];
let dataLength = 0;
for (const member of input.members) {
const stored = member.compress
? Buffer.concat([
deflateSync(member.payload),
member.compressedSuffix ?? Buffer.alloc(0),
])
: member.payload;
entries.push(
tocEntry({
name: member.name,
typeCode: member.typeCode,
offset: dataLength,
compressedSize: stored.length,
uncompressedSize:
member.declaredUncompressedSize ?? member.payload.length,
flag: member.compress ? 1 : 0,
}),
);
payloads.push(stored);
dataLength += stored.length;
}
const toc = input.tocBytes ?? Buffer.concat(entries);
const cookie = cookieBytes({
layout: input.layout,
pythonVersion: input.pythonVersion,
pythonLibrary: input.pythonLibrary,
tocOffset: dataLength,
tocLength: toc.length,
});
return Buffer.concat([
ELF_PREFIX,
...payloads,
toc,
cookie,
input.trailer ?? Buffer.alloc(0),
]);
};
/** Cookie whose package range starts before the file. */
export const cookieRangeOutsideFile = (): Buffer => {
const cookie = Buffer.alloc(88);
PYINSTALLER_COOKIE_MAGIC.copy(cookie, 0);
cookie.writeUInt32BE(200, 8);
cookie.writeUInt32BE(80, 12);
cookie.writeUInt32BE(32, 16);
cookie.writeUInt32BE(314, 20);
return Buffer.concat([ELF_PREFIX, cookie]);
};
/** PYZ image. Module bodies are zlib-compressed. Namespace entries have no body. */
export const pyzArchive = (input: {
readonly magic: Buffer;
readonly modules: readonly PyzFixtureModule[];
readonly toc?: Buffer;
}): Buffer => {
const blobs: Buffer[] = [];
const entries: Buffer[] = [];
let offset = 12;
for (const module of input.modules) {
const compressed =
module.typeCode === 3 && module.legacyNamespacePayload !== true
? Buffer.alloc(0)
: Buffer.concat([
deflateSync(module.body),
module.compressedSuffix ?? Buffer.alloc(0),
]);
entries.push(
marshalPair(module.name, module.typeCode, offset, compressed.length),
);
if (compressed.length > 0) {
blobs.push(compressed);
offset += compressed.length;
}
}
const toc = input.toc ?? marshalList(entries);
const header = Buffer.alloc(12);
Buffer.from("PYZ\0").copy(header, 0);
input.magic.copy(header, 4, 0, 4);
header.writeUInt32BE(
12 + blobs.reduce((sum, blob) => sum + blob.length, 0),
8,
);
return Buffer.concat([header, ...blobs, toc]);
};
/** Type 3 normally stores no body; older producers can write a namespace payload. */
export interface PyzFixtureModule {
readonly legacyNamespacePayload?: boolean;
readonly compressedSuffix?: Buffer;
readonly name: string;
readonly typeCode: number;
readonly body: Buffer;
}
const tocEntry = (input: {
readonly name: string;
readonly typeCode: string;
readonly offset: number;
readonly compressedSize: number;
readonly uncompressedSize: number;
readonly flag: number;
}): Buffer => {
const name = Buffer.from(input.name, "utf8");
const length = Math.ceil((18 + name.length + 1) / 16) * 16;
const bytes = Buffer.alloc(length);
bytes.writeUInt32BE(length, 0);
bytes.writeUInt32BE(input.offset, 4);
bytes.writeUInt32BE(input.compressedSize, 8);
bytes.writeUInt32BE(input.uncompressedSize, 12);
bytes[16] = input.flag;
bytes[17] = input.typeCode.charCodeAt(0);
name.copy(bytes, 18);
return bytes;
};
const cookieBytes = (input: {
readonly layout: "2.0" | "2.1";
readonly pythonVersion: number;
readonly pythonLibrary: string | undefined;
readonly tocOffset: number;
readonly tocLength: number;
}): Buffer => {
const size = input.layout === "2.1" ? 88 : 24;
const bytes = Buffer.alloc(size);
PYINSTALLER_COOKIE_MAGIC.copy(bytes, 0);
bytes.writeUInt32BE(input.tocOffset + input.tocLength + size, 8);
bytes.writeUInt32BE(input.tocOffset, 12);
bytes.writeUInt32BE(input.tocLength, 16);
bytes.writeUInt32BE(input.pythonVersion, 20);
if (input.pythonLibrary !== undefined)
Buffer.from(input.pythonLibrary, "utf8").copy(bytes, 24);
return bytes;
};
const marshalList = (items: readonly Buffer[]): Buffer => {
const count = Buffer.alloc(4);
count.writeInt32LE(items.length, 0);
return Buffer.concat([Buffer.from([0x5b]), count, ...items]);
};
const marshalPair = (
name: string,
typeCode: number,
offset: number,
length: number,
): Buffer =>
smallTuple([
marshalString(name),
smallTuple([int32(typeCode), int32(offset), int32(length)]),
]);
const smallTuple = (items: readonly Buffer[]): Buffer =>
Buffer.concat([Buffer.from([0x29, items.length]), ...items]);
const marshalString = (text: string): Buffer => {
const body = Buffer.from(text, "utf8");
if (body.length <= 255 && body.every((byte) => byte < 0x80))
return Buffer.concat([Buffer.from([0x7a, body.length]), body]);
const length = Buffer.alloc(4);
length.writeUInt32LE(body.length);
return Buffer.concat([Buffer.from([0x75]), length, body]);
};
const int32 = (value: number): Buffer => {
const bytes = Buffer.alloc(5);
bytes[0] = 0x69;
bytes.writeInt32LE(value, 1);
return bytes;
};
+389
View File
@@ -0,0 +1,389 @@
import type { FileHandle } from "node:fs/promises";
import { readPyInstallerBytes } from "./PyInstallerBytes.js";
import { err, ok, type Result } from "../../domain/result.js";
import {
pyInstallerFailure,
type PyInstallerFailure,
} from "./PyInstallerFailure.js";
/** Magic at the start of a CArchive cookie. */
export const PYINSTALLER_COOKIE_MAGIC = Buffer.from("MEI\x0c\x0b\x0a\x0b\x0e");
/** PyInstaller 2.1+ cookie size, including the 64-byte library name. */
export const PYINSTALLER_COOKIE_V21_BYTES = 88;
/** PyInstaller 2.0 cookie size, without the library name. */
export const PYINSTALLER_COOKIE_V20_BYTES = 24;
/**
* How far before EOF the cookie may sit. macOS code signatures are appended
* after the cookie, and a large binary's signature hashes are larger than the
* bootloader's own 8 KiB scan chunk, so detection uses one bounded tail.
*/
export const PYINSTALLER_COOKIE_SEARCH_BYTES = 16 * 1024 * 1024;
const MAX_TOC_BYTES = 32 * 1024 * 1024;
const MAX_TOC_ENTRIES = 100_000;
const TOC_HEADER_BYTES = 18;
/** Observed CArchive cookie. Offsets are relative to the archive start. */
export interface PyInstallerCookie {
readonly layout: "2.0" | "2.1";
readonly archiveStart: number;
readonly packageLength: number;
readonly tocOffset: number;
readonly tocLength: number;
readonly pythonVersion: number;
readonly pythonLibrary: string | null;
readonly cookieOffset: number;
}
/** One CArchive table entry. */
export interface CArchiveEntry {
readonly index: number;
readonly tocRecordOffset: number;
readonly tocRecordLength: number;
readonly name: string;
readonly typeCode: string;
readonly offset: number;
readonly compressedSize: number;
readonly uncompressedSize: number;
readonly compressionFlag: number;
}
/** Runtime options and external dependencies are declarations, never local blobs. */
export const isCArchiveDeclaration = (
entry: CArchiveEntry,
): entry is CArchiveEntry & { readonly typeCode: "o" | "d" } =>
entry.typeCode === "o" || entry.typeCode === "d";
/** True when a native executable has a structurally valid CArchive cookie. */
export const containsPyInstallerArchive = async (
handle: FileHandle,
fileSize: number,
signal?: AbortSignal,
): Promise<boolean> => {
const found = await findPyInstallerCookie(handle, fileSize, signal);
return found?.ok === true;
};
/** Locate a structurally valid cookie. Invalid magic copies are skipped. */
export const findPyInstallerCookie = async (
handle: FileHandle,
fileSize: number,
signal?: AbortSignal,
): Promise<Result<PyInstallerCookie, PyInstallerFailure> | undefined> => {
if (
!Number.isSafeInteger(fileSize) ||
fileSize < PYINSTALLER_COOKIE_V20_BYTES
)
return undefined;
const tailSize = Math.min(fileSize, PYINSTALLER_COOKIE_SEARCH_BYTES);
const read = await readPyInstallerBytes(handle, {
position: fileSize - tailSize,
length: tailSize,
label: "CArchive tail",
signal,
});
if (!read.ok) return read;
const observed = read.value;
let failure: PyInstallerFailure | undefined;
let from = observed.length;
while (from >= PYINSTALLER_COOKIE_MAGIC.length) {
const relative = observed
.subarray(0, from)
.lastIndexOf(PYINSTALLER_COOKIE_MAGIC);
if (relative < 0) break;
const cookieOffset = fileSize - tailSize + relative;
const parsed = parseCookieAt(observed, relative, cookieOffset, fileSize);
if (parsed.ok) return parsed;
failure = parsed.error;
from = relative;
}
return failure === undefined ? undefined : err(failure);
};
/** Read and parse the TOC that a validated cookie names. */
export const readCArchiveToc = async (
handle: FileHandle,
cookie: PyInstallerCookie,
signal?: AbortSignal,
): Promise<Result<readonly CArchiveEntry[], PyInstallerFailure>> => {
if (cookie.tocLength > MAX_TOC_BYTES)
return err(
pyInstallerFailure(
"limit",
"CArchive table of contents exceeds its bound",
),
);
const position = cookie.archiveStart + cookie.tocOffset;
const read = await readPyInstallerBytes(handle, {
position,
length: cookie.tocLength,
label: "CArchive table of contents",
signal,
});
if (!read.ok) return read;
const parsed = parseCArchiveToc(read.value);
if (!parsed.ok) return parsed;
const valid = validateCArchiveMembers(parsed.value, cookie.tocOffset);
return valid.ok ? parsed : valid;
};
/** Parse TOC bytes. Each entry is a big-endian header plus a NUL-padded name. */
export const parseCArchiveToc = (
bytes: Buffer,
): Result<readonly CArchiveEntry[], PyInstallerFailure> => {
const entries: CArchiveEntry[] = [];
let offset = 0;
while (offset < bytes.length) {
if (entries.length >= MAX_TOC_ENTRIES)
return err(
pyInstallerFailure(
"limit",
"CArchive table of contents exceeds its entry bound",
),
);
if (offset + TOC_HEADER_BYTES > bytes.length)
return err(
pyInstallerFailure(
"format",
"CArchive table of contents entry is truncated",
),
);
const entryLength = bytes.readUInt32BE(offset);
if (entryLength < TOC_HEADER_BYTES || offset + entryLength > bytes.length)
return err(
pyInstallerFailure(
"format",
"CArchive table of contents entry extends outside the table",
),
);
const nameBytes = bytes.subarray(
offset + TOC_HEADER_BYTES,
offset + entryLength,
);
const name = tocName(nameBytes);
if (!name.ok) return name;
const compressionFlag = bytes[offset + 16] ?? 0;
if (compressionFlag !== 0 && compressionFlag !== 1)
return err(
pyInstallerFailure(
"format",
`CArchive compression flag ${String(compressionFlag)} is not stored or zlib`,
),
);
const typeCode = bytes[offset + 17] ?? 0;
if (typeCode < 0x20 || typeCode > 0x7e)
return err(
pyInstallerFailure(
"format",
"CArchive type code is not a printable character",
),
);
entries.push({
index: entries.length,
tocRecordOffset: offset,
tocRecordLength: entryLength,
name: name.value,
typeCode: String.fromCharCode(typeCode),
offset: bytes.readUInt32BE(offset + 4),
compressedSize: bytes.readUInt32BE(offset + 8),
uncompressedSize: bytes.readUInt32BE(offset + 12),
compressionFlag,
});
offset += entryLength;
}
return ok(entries);
};
const parseCookieAt = (
tail: Buffer,
relative: number,
cookieOffset: number,
fileSize: number,
): Result<PyInstallerCookie, PyInstallerFailure> => {
const modern = parseCookie(tail, relative, cookieOffset, fileSize, "2.1");
if (modern.ok) return modern;
const legacy = parseCookie(tail, relative, cookieOffset, fileSize, "2.0");
return legacy.ok ? legacy : modern;
};
const parseCookie = (
tail: Buffer,
relative: number,
cookieOffset: number,
fileSize: number,
layout: "2.0" | "2.1",
): Result<PyInstallerCookie, PyInstallerFailure> => {
const cookieBytes =
layout === "2.1"
? PYINSTALLER_COOKIE_V21_BYTES
: PYINSTALLER_COOKIE_V20_BYTES;
if (relative + cookieBytes > tail.length)
return err(
pyInstallerFailure(
"format",
"CArchive cookie extends past the end of the file",
),
);
const view = tail.subarray(relative, relative + cookieBytes);
const packageLength = view.readUInt32BE(8);
const tocOffset = view.readUInt32BE(12);
const tocLength = view.readUInt32BE(16);
const pythonVersion = view.readUInt32BE(20);
if (
!safeSum(tocOffset, tocLength, cookieBytes) ||
packageLength !== tocOffset + tocLength + cookieBytes
)
return err(
pyInstallerFailure(
"format",
"CArchive cookie does not describe a table of contents inside the package",
),
);
const archiveStart = cookieOffset + cookieBytes - packageLength;
if (archiveStart < 0 || archiveStart + packageLength > fileSize)
return err(
pyInstallerFailure("format", "CArchive cookie range is outside the file"),
);
const library =
layout === "2.1" ? libraryName(view.subarray(24, 88)) : ok(null);
if (!library.ok) return library;
return ok({
layout,
archiveStart,
packageLength,
tocOffset,
tocLength,
pythonVersion,
pythonLibrary: library.value,
cookieOffset,
});
};
const libraryName = (
bytes: Buffer,
): Result<string | null, PyInstallerFailure> => {
const end = bytes.indexOf(0);
const text = end < 0 ? bytes : bytes.subarray(0, end);
if (end >= 0 && bytes.subarray(end).some((byte) => byte !== 0))
return err(
pyInstallerFailure(
"format",
"CArchive Python library name is not NUL-padded",
),
);
if (text.length === 0) return ok(null);
try {
return ok(
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text),
);
} catch {
return err(
pyInstallerFailure(
"format",
"CArchive Python library name is not valid UTF-8",
),
);
}
};
const tocName = (bytes: Buffer): Result<string, PyInstallerFailure> => {
const end = bytes.indexOf(0);
const text = end < 0 ? bytes : bytes.subarray(0, end);
if (text.length === 0)
return err(pyInstallerFailure("format", "CArchive entry name is empty"));
try {
return ok(
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text),
);
} catch {
return err(
pyInstallerFailure("format", "CArchive entry name is not valid UTF-8"),
);
}
};
const safeSum = (left: number, middle: number, right: number): boolean =>
Number.isSafeInteger(left) &&
Number.isSafeInteger(middle) &&
Number.isSafeInteger(right) &&
left >= 0 &&
middle >= 0 &&
right >= 0 &&
left + middle + right <= Number.MAX_SAFE_INTEGER;
/** Source directory semantics are validated once before any consumer projects paths. */
const validateCArchiveMembers = (
entries: readonly CArchiveEntry[],
tocOffset: number,
): Result<null, PyInstallerFailure> => {
const names = new Set<string>();
for (const entry of entries) {
if (isCArchiveDeclaration(entry)) {
if (
entry.compressedSize !== 0 ||
entry.uncompressedSize !== 0 ||
entry.compressionFlag !== 0
)
return err(
pyInstallerFailure(
"integrity",
`CArchive declaration at table index ${String(entry.index)} has local payload or compression`,
),
);
} else {
if (names.has(entry.name))
return err(
pyInstallerFailure(
"integrity",
`Duplicate CArchive member name at table index ${String(entry.index)}`,
),
);
names.add(entry.name);
}
if (
entry.compressionFlag === 0 &&
entry.compressedSize !== entry.uncompressedSize
)
return err(
pyInstallerFailure(
"integrity",
`Stored CArchive member size disagrees with its table entry at table index ${String(entry.index)}`,
),
);
if (entry.offset > tocOffset)
return err(
pyInstallerFailure(
"format",
`CArchive member offset is outside the archive at table index ${String(entry.index)}`,
),
);
}
const ranges = entries
.filter((entry) => entry.compressedSize > 0)
.map((entry) => ({
index: entry.index,
start: entry.offset,
end: entry.offset + entry.compressedSize,
}))
.sort((left, right) => left.start - right.start);
let occupied = 0;
for (const range of ranges) {
if (
!Number.isSafeInteger(range.end) ||
range.end > tocOffset ||
range.start < occupied
)
return err(
pyInstallerFailure(
"integrity",
`CArchive member extends outside the archive or overlaps another member at table index ${String(range.index)}`,
),
);
occupied = range.end;
}
return ok(null);
};
@@ -0,0 +1,340 @@
import type { FileHandle } from "node:fs/promises";
import { copyArtifactEntry, type ArtifactEntry } from "../ArtifactReader.js";
import { isCArchiveDeclaration, type CArchiveEntry } from "./CArchiveLayout.js";
import type { PyzArchive, PyzEntry } from "./PyzArchive.js";
import {
RECONSTRUCTED_PYTHON_HEADER_LIMITATION,
reconstructedPycHeader,
} from "./PythonBytecode.js";
import {
cArchiveEntryMetadata,
pyzEntryMetadata,
pyzEntryEncrypted,
} from "./PyInstallerMetadata.js";
import type { PyInstallerEntryMetadata } from "../../domain/artifactFormatMetadata.js";
const CRYPTO_KEY_NAME = "pyimod00_crypto_key";
/** Parsed facts and the descriptor identity used to locate a retained image. */
export interface PyzImage {
readonly archive: PyzArchive;
readonly handle: FileHandle;
readonly start: number;
}
/** Stable entry descriptions and bounded read plans; expanded member bytes are never retained. */
export interface PyInstallerMember {
readonly entry: ArtifactEntry;
readonly read:
| {
readonly kind: "carchive";
readonly archiveStart: number;
readonly source: CArchiveEntry;
readonly header: Buffer | undefined;
}
| {
readonly kind: "pyz";
readonly handle: FileHandle;
readonly offset: number;
readonly length: number;
readonly compressed: boolean;
readonly uncompressedSize: number | undefined;
readonly header: Buffer | undefined;
}
| undefined;
}
/** The producer's key is an importable, nonempty Python module, not a data filename. */
export const hasEncryptionMarker = (
entries: readonly CArchiveEntry[],
): boolean =>
entries.some(
(entry) =>
entry.name === CRYPTO_KEY_NAME &&
(entry.typeCode === "m" || entry.typeCode === "M") &&
entry.uncompressedSize > 0,
);
export const projectStoredMember = (
entry: CArchiveEntry,
archiveStart: number,
pythonMagic: Buffer | undefined,
): PyInstallerMember => {
if (isCArchiveDeclaration(entry))
return {
entry: fileEntry({
path: `__pyinstaller__/${entry.typeCode === "o" ? "runtime-options" : "dependencies"}/${String(entry.index)}`,
kind: "metadata",
declaredSize: entry.uncompressedSize,
compressedSize: entry.compressedSize,
byteOffset: null,
limitations:
entry.typeCode === "d"
? [
"Multipackage dependency bytes belong to another executable and remain unresolved.",
]
: [],
adapterKey: `carchive:${String(entry.index)}:${entry.name}`,
formatMetadata: cArchiveEntryMetadata(entry),
}),
read: undefined,
};
const header =
pythonMagic !== undefined && isBytecodeType(entry.typeCode)
? reconstructedPycHeader(pythonMagic)
: undefined;
const path = canonicalMemberPath(entry.name);
const symlink = entry.typeCode === "n";
const limitations: string[] = [];
if (symlink) limitations.push("PyInstaller symlink target was not read.");
if (header !== undefined)
limitations.push(RECONSTRUCTED_PYTHON_HEADER_LIMITATION);
const declaredSize = entry.uncompressedSize;
return {
entry: fileEntry({
path,
kind: symlink ? "symlink" : "file",
declaredSize,
compressedSize: entry.compressedSize,
byteOffset: sourceByteOffset(entry, archiveStart),
limitations,
adapterKey: `carchive:${String(entry.index)}:${entry.name}`,
formatMetadata: cArchiveEntryMetadata(entry, header),
}),
read: {
kind: "carchive",
archiveStart,
source: entry,
header,
},
};
};
export const projectPyzContainer = (
entry: CArchiveEntry,
archiveStart: number,
image: PyzImage,
encrypted: boolean,
): PyInstallerMember => {
const { archive } = image;
const path = canonicalMemberPath(entry.name);
const byteOffset = sourceByteOffset(entry, archiveStart);
const limitations: string[] = [];
if (archive.kind === "opaque")
return {
entry: fileEntry({
path,
declaredSize: entry.uncompressedSize,
compressedSize: entry.compressedSize,
byteOffset,
// Legacy encryption applies to inner payloads, never the PYZ image.
encrypted: false,
limitations: [
...limitations,
archive.limitation,
...(encrypted
? ["PYZ entries are encrypted and were not decoded."]
: []),
],
adapterKey: `carchive:${String(entry.index)}:${entry.name}`,
formatMetadata: cArchiveEntryMetadata(
entry,
undefined,
archive,
encrypted,
),
}),
read:
entry.compressionFlag === 1
? {
kind: "pyz",
handle: image.handle,
offset: image.start,
length: entry.uncompressedSize,
compressed: false,
uncompressedSize: entry.uncompressedSize,
header: undefined,
}
: {
kind: "carchive",
archiveStart,
source: entry,
header: undefined,
},
};
return {
entry: {
path,
kind: "directory",
declaredSize: entry.uncompressedSize,
compressedSize: entry.compressedSize,
executable: false,
encrypted: false,
byteOffset,
declaredIntegrity: null,
unpacked: false,
limitations: [
...limitations,
`PyInstaller PYZ sha256 ${archive.sha256}.`,
...(encrypted
? ["PYZ entries are encrypted and were not decoded."]
: []),
],
adapterKey: `carchive:${String(entry.index)}:${entry.name}`,
formatMetadata: cArchiveEntryMetadata(
entry,
undefined,
archive,
encrypted,
),
},
read: undefined,
};
};
export const projectPyzMember = (
container: CArchiveEntry,
item: PyzEntry,
encryptionMarker: boolean,
pythonMagic: Buffer | undefined,
range:
| {
readonly handle: FileHandle;
readonly offset: number;
readonly length: number;
}
| undefined,
): PyInstallerMember => {
const path = pyzMemberPath(canonicalMemberPath(container.name), item);
const encrypted = pyzEntryEncrypted(item, encryptionMarker);
if (item.namespace)
return {
entry: {
path: path,
kind: "directory",
declaredSize: 0,
compressedSize: item.length,
executable: false,
encrypted,
byteOffset: null,
declaredIntegrity: null,
unpacked: false,
limitations: [
"Namespace package has no importable bytecode; its declared payload is not materialized.",
...(encrypted
? [
"PYZ namespace payload is declared encrypted and is not materialized.",
]
: []),
],
adapterKey: `pyz:${String(container.index)}:${String(item.index)}:${path}`,
formatMetadata: pyzEntryMetadata(
container,
item,
undefined,
encryptionMarker,
),
},
read: undefined,
};
const header =
!encrypted &&
pythonMagic !== undefined &&
(item.typeCode === 0 || item.typeCode === 1)
? reconstructedPycHeader(pythonMagic)
: undefined;
const limitations: string[] = [];
if (encrypted)
limitations.push("PYZ entry is encrypted and was not decoded.");
if (header !== undefined)
limitations.push(RECONSTRUCTED_PYTHON_HEADER_LIMITATION);
limitations.push(
"PYZ tables declare compressed lengths; the uncompressed size is unknown until decoding.",
);
return {
entry: fileEntry({
path: path,
declaredSize: null,
compressedSize: item.length,
byteOffset: null,
encrypted,
limitations,
adapterKey: `pyz:${String(container.index)}:${String(item.index)}:${path}`,
formatMetadata: pyzEntryMetadata(
container,
item,
header,
encryptionMarker,
),
}),
read:
range === undefined
? undefined
: {
kind: "pyz",
...range,
compressed: true,
uncompressedSize: undefined,
header,
},
};
};
/**
* CArchive and PYZ data names treat `\` as a separator. The logical path is
* canonical; the stored spelling is kept when that conversion changes it.
* `source_location` is an offset plus declared size, so a compressed member
* publishes a file span only when that size is the physical extent. Otherwise
* the span stays unknown, as ZIP does for compressed entries.
*/
const canonicalMemberPath = (storedName: string): string =>
storedName.replaceAll("\\", "/");
const sourceByteOffset = (
entry: CArchiveEntry,
archiveStart: number,
): number | null =>
entry.compressionFlag === 0 ? archiveStart + entry.offset : null;
const pyzMemberPath = (container: string, item: PyzEntry): string => {
if (item.typeCode === 2)
return `${container}/${canonicalMemberPath(item.name)}`;
const dotted = item.name.replaceAll(".", "/");
if (item.namespace) return `${container}/${dotted}`;
if (item.package) return `${container}/${dotted}/__init__.pyc`;
return `${container}/${dotted}.pyc`;
};
const isBytecodeType = (typeCode: string): boolean =>
typeCode === "s" || typeCode === "m" || typeCode === "M";
const fileEntry = (input: {
readonly path: string;
readonly declaredSize: number | null;
readonly compressedSize: number;
readonly byteOffset: number | null;
readonly limitations: readonly string[];
readonly adapterKey: string;
readonly formatMetadata: PyInstallerEntryMetadata;
readonly encrypted?: boolean;
readonly kind?: "file" | "symlink" | "metadata";
}): ArtifactEntry => ({
path: input.path,
kind: input.kind ?? "file",
declaredSize: input.declaredSize,
compressedSize: input.compressedSize,
executable: false,
encrypted: input.encrypted === true,
byteOffset: input.byteOffset,
declaredIntegrity: null,
unpacked: false,
limitations: input.limitations,
adapterKey: input.adapterKey,
formatMetadata: input.formatMetadata,
});
/** Stable entry stored for a later open(). */
export const sealMember = (member: PyInstallerMember): PyInstallerMember => ({
...member,
entry: copyArtifactEntry(member.entry),
});
@@ -0,0 +1,62 @@
import { expect, it } from "vitest";
import { readMarshalSubset } from "./MarshalSubset.js";
const bytes = (hex: string): Buffer => Buffer.from(hex, "hex");
it("reads a Python 3.9 reference-flagged PYZ table", () => {
const parsed = readMarshalSubset(
bytes(
"db04000000a902da0568656c6c6fa903e900000000e90a000000e914000000a902da03706b67a903e901000000e91e000000e928000000a902fa0c6a736f6e2e6465636f646572a9037204000000e964000000e932000000a902da026e73a903e90300000072040000007204000000",
),
);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value).toEqual([
["hello", [0, 10, 20]],
["pkg", [1, 30, 40]],
["json.decoder", [0, 100, 50]],
["ns", [3, 0, 0]],
]);
});
it("resolves a shared string through a reference reserved before the tuple", () => {
const parsed = readMarshalSubset(
bytes(
"db020000002902da06736861726564a903e900000000e901000000e90200000029027201000000a9037204000000e903000000e904000000",
),
);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value).toEqual([
["shared", [0, 1, 2]],
["shared", [1, 3, 4]],
]);
});
it("reads a long integer that fits in a safe integer", () => {
const parsed = readMarshalSubset(
bytes(
"5b01000000a902da03626967a903e900000000ec03000000000000000004e905000000",
),
);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value).toEqual([["big", [0, 2 ** 40, 5]]]);
});
it("reads a negative 32-bit integer", () => {
const parsed = readMarshalSubset(
bytes("5b01000000a902da016ea903e900000000e9ffffffffe901000000"),
);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value).toEqual([["n", [0, -1, 1]]]);
});
it("rejects a marshal type outside the subset", () => {
const parsed = readMarshalSubset(Buffer.from([0x4e]));
expect(parsed.ok).toBe(false);
if (parsed.ok) return;
expect(parsed.error.reason).toBe("format");
});
+282
View File
@@ -0,0 +1,282 @@
import { err, ok, type Result } from "../../domain/result.js";
import {
pyInstallerFailure,
type PyInstallerFailure,
} from "./PyInstallerFailure.js";
const MAX_DEPTH = 16;
const MAX_NODES = 100_000;
const MAX_SEQUENCE = 100_000;
const MAX_STRING_BYTES = 65_536;
const FLAG_REF = 0x80;
/** Values the PYZ table of contents is allowed to contain. */
export type MarshalValue =
| number
| string
| Uint8Array
| readonly MarshalValue[];
interface MarshalCursor {
readonly bytes: Buffer;
offset: number;
readonly refs: Array<MarshalValue | undefined>;
depth: number;
nodes: number;
}
/**
* Read one marshal value using only list, tuple, str, int, and bytes.
* Container slots are reserved before their elements so later references match
* CPython. Any other type fails the whole value.
*/
export const readMarshalSubset = (
bytes: Buffer,
): Result<MarshalValue, PyInstallerFailure> => {
const cursor: MarshalCursor = {
bytes,
offset: 0,
refs: [],
depth: 0,
nodes: 0,
};
const value = readObject(cursor);
if (!value.ok) return value;
if (cursor.offset !== bytes.length)
return err(
pyInstallerFailure(
"format",
"PYZ marshal value did not consume its table of contents",
),
);
return value;
};
const readObject = (
cursor: MarshalCursor,
): Result<MarshalValue, PyInstallerFailure> => {
if (cursor.depth > MAX_DEPTH || cursor.nodes >= MAX_NODES)
return err(
pyInstallerFailure(
"limit",
"PYZ marshal value exceeds the depth or object bound",
),
);
cursor.nodes += 1;
const code = readByte(cursor);
if (!code.ok) return code;
const flag = (code.value & FLAG_REF) !== 0;
const type = code.value & ~FLAG_REF;
if (type === 0x69) return storeScalar(cursor, flag, readInt32(cursor));
if (type === 0x6c) return storeScalar(cursor, flag, readLong(cursor));
if (type === 0x7a || type === 0x5a)
return storeScalar(cursor, flag, readShortAscii(cursor));
if (type === 0x61 || type === 0x41)
return storeScalar(cursor, flag, readCountedAscii(cursor));
if (type === 0x75) return storeScalar(cursor, flag, readUtf8(cursor));
if (type === 0x73) return storeScalar(cursor, flag, readByteString(cursor));
if (type === 0x28) return readSequence(cursor, flag, readInt32(cursor));
if (type === 0x29) return readSequence(cursor, flag, readByte(cursor));
if (type === 0x5b) return readSequence(cursor, flag, readInt32(cursor));
if (type === 0x72) return readReference(cursor, flag);
return err(
pyInstallerFailure(
"format",
"PYZ marshal value uses a type outside the supported subset",
),
);
};
const storeScalar = (
cursor: MarshalCursor,
flag: boolean,
value: Result<MarshalValue, PyInstallerFailure>,
): Result<MarshalValue, PyInstallerFailure> => {
if (!value.ok) return value;
if (flag) cursor.refs.push(value.value);
return value;
};
const readSequence = (
cursor: MarshalCursor,
flag: boolean,
count: Result<number, PyInstallerFailure>,
): Result<MarshalValue, PyInstallerFailure> => {
if (!count.ok) return count;
if (count.value < 0 || count.value > MAX_SEQUENCE)
return err(
pyInstallerFailure(
"limit",
"PYZ marshal sequence exceeds the supported count",
),
);
const slot = flag ? cursor.refs.length : undefined;
if (slot !== undefined) cursor.refs.push(undefined);
const items: MarshalValue[] = [];
cursor.depth += 1;
for (let index = 0; index < count.value; index += 1) {
const item = readObject(cursor);
if (!item.ok) return item;
items.push(item.value);
}
cursor.depth -= 1;
if (slot !== undefined) cursor.refs[slot] = items;
return ok(items);
};
const readReference = (
cursor: MarshalCursor,
flag: boolean,
): Result<MarshalValue, PyInstallerFailure> => {
const index = readInt32(cursor);
if (!index.ok) return index;
const value = cursor.refs[index.value];
if (value === undefined)
return err(
pyInstallerFailure(
"format",
"PYZ marshal reference does not point at a completed value",
),
);
if (flag) cursor.refs.push(value);
return ok(value);
};
const readShortAscii = (
cursor: MarshalCursor,
): Result<string, PyInstallerFailure> => {
const length = readByte(cursor);
if (!length.ok) return length;
return readAsciiBytes(cursor, length.value);
};
const readCountedAscii = (
cursor: MarshalCursor,
): Result<string, PyInstallerFailure> => {
const length = readInt32(cursor);
if (!length.ok) return length;
return readAsciiBytes(cursor, length.value);
};
const readAsciiBytes = (
cursor: MarshalCursor,
length: number,
): Result<string, PyInstallerFailure> => {
const bytes = readBytes(cursor, length);
if (!bytes.ok) return bytes;
if (bytes.value.some((byte) => byte > 0x7f))
return err(
pyInstallerFailure(
"format",
"PYZ marshal ASCII string contains a non-ASCII byte",
),
);
return ok(bytes.value.toString("ascii"));
};
const readUtf8 = (
cursor: MarshalCursor,
): Result<string, PyInstallerFailure> => {
const length = readInt32(cursor);
if (!length.ok) return length;
const bytes = readBytes(cursor, length.value);
if (!bytes.ok) return bytes;
try {
return ok(
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes.value,
),
);
} catch {
return err(
pyInstallerFailure("format", "PYZ marshal string is not valid UTF-8"),
);
}
};
const readByteString = (
cursor: MarshalCursor,
): Result<Uint8Array, PyInstallerFailure> => {
const length = readInt32(cursor);
if (!length.ok) return length;
const bytes = readBytes(cursor, length.value);
if (!bytes.ok) return bytes;
return ok(bytes.value);
};
const readLong = (
cursor: MarshalCursor,
): Result<number, PyInstallerFailure> => {
const digits = readInt32(cursor);
if (!digits.ok) return digits;
const count = Math.abs(digits.value);
if (count > 8)
return err(
pyInstallerFailure("limit", "PYZ marshal integer exceeds the safe range"),
);
let magnitude = 0n;
for (let index = 0; index < count; index += 1) {
const digit = readUint16(cursor);
if (!digit.ok) return digit;
if (digit.value > 0x7fff)
return err(
pyInstallerFailure("format", "PYZ marshal long digit is out of range"),
);
magnitude += BigInt(digit.value) << BigInt(15 * index);
}
if (magnitude > BigInt(Number.MAX_SAFE_INTEGER))
return err(
pyInstallerFailure("limit", "PYZ marshal integer exceeds the safe range"),
);
const signed = digits.value < 0 ? -Number(magnitude) : Number(magnitude);
return ok(signed);
};
const readBytes = (
cursor: MarshalCursor,
length: number,
): Result<Buffer, PyInstallerFailure> => {
if (!Number.isSafeInteger(length) || length < 0 || length > MAX_STRING_BYTES)
return err(
pyInstallerFailure(
"limit",
"PYZ marshal string exceeds the supported length",
),
);
const end = cursor.offset + length;
if (end > cursor.bytes.length)
return err(pyInstallerFailure("format", "PYZ marshal value is truncated"));
const bytes = cursor.bytes.subarray(cursor.offset, end);
cursor.offset = end;
return ok(bytes);
};
const readByte = (
cursor: MarshalCursor,
): Result<number, PyInstallerFailure> => {
if (cursor.offset >= cursor.bytes.length)
return err(pyInstallerFailure("format", "PYZ marshal value is truncated"));
const value = cursor.bytes[cursor.offset] ?? 0;
cursor.offset += 1;
return ok(value);
};
const readInt32 = (
cursor: MarshalCursor,
): Result<number, PyInstallerFailure> => {
if (cursor.offset + 4 > cursor.bytes.length)
return err(pyInstallerFailure("format", "PYZ marshal value is truncated"));
const value = cursor.bytes.readInt32LE(cursor.offset);
cursor.offset += 4;
return ok(value);
};
const readUint16 = (
cursor: MarshalCursor,
): Result<number, PyInstallerFailure> => {
if (cursor.offset + 2 > cursor.bytes.length)
return err(pyInstallerFailure("format", "PYZ marshal value is truncated"));
const value = cursor.bytes.readUInt16LE(cursor.offset);
cursor.offset += 2;
return ok(value);
};
@@ -0,0 +1,139 @@
import type { FileHandle } from "node:fs/promises";
import { expect, it, vi } from "vitest";
import { withPyInstallerReader } from "../../../tests/fixtures/pyinstallerReader.js";
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
import type { ArtifactEntry } from "../ArtifactReader.js";
import { PyInstallerPyzStore } from "./PyInstallerPyzStore.js";
import {
cArchiveExecutable,
pyzArchive,
PYTHON_39_MAGIC,
} from "./CArchive.fixture.js";
const compressedImages = (count: number, modules = false) =>
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
members: Array.from({ length: count }, (_, index) => ({
name: `PYZ-${index}.pyz`,
typeCode: "z",
compress: true,
payload: pyzArchive({
magic: PYTHON_39_MAGIC,
modules: modules
? [{ name: "app", typeCode: 0, body: Buffer.from("body") }]
: [],
}),
})),
});
it("refuses excess private images before creating any temporary file", async () => {
const spool = vi
.spyOn(PyInstallerPyzStore.prototype, "spool")
.mockRejectedValue(new Error("unexpected private file creation"));
try {
await withPyInstallerReader(compressedImages(129), async (reader) => {
await expect(
reader.entries()[Symbol.asyncIterator]().next(),
).rejects.toMatchObject({
reason: "limit",
boundary: "capacity",
});
expect(spool).not.toHaveBeenCalled();
});
} finally {
spool.mockRestore();
}
});
it("accepts the private-image capacity and retires all image owners", async () => {
const spool = vi.spyOn(PyInstallerPyzStore.prototype, "spool");
try {
await withPyInstallerReader(compressedImages(128), async (reader, root) => {
const listed: ArtifactEntry[] = [];
for await (const entry of reader.entries()) listed.push(entry);
expect(listed).toHaveLength(128);
expect(spool).toHaveBeenCalledTimes(128);
await reader.close();
expect(root.fd).toBeGreaterThanOrEqual(0);
});
} finally {
spool.mockRestore();
}
});
it.each([false, true])(
"preserves private-image admission cancellation (failed cleanup: %s)",
async (failClose) => {
await withPyInstallerReader(
compressedImages(1, true),
async (reader, root) => {
let selected: ArtifactEntry | undefined;
for await (const entry of reader.entries())
if (entry.path.endsWith("app.pyc")) selected = entry;
if (selected === undefined) throw new Error("Missing child");
let entered!: () => void, release!: () => void;
const started = new Promise<void>((resolve) => {
entered = resolve;
});
const gate = new Promise<void>((resolve) => {
release = resolve;
});
const prototype = Object.getPrototypeOf(root) as FileHandle;
const nativeStat = prototype.stat,
nativeClose = OwnedFileHandle.prototype.close;
let admitted: FileHandle | undefined,
retained: OwnedFileHandle | undefined,
attempts = 0;
const cancelled = new Error("cancel private-image admission"),
cleanup = new Error("admission cleanup EIO");
const signal = new AbortController();
const stat = vi
.spyOn(prototype, "stat")
.mockImplementation(async function (
this: FileHandle,
...args: Parameters<FileHandle["stat"]>
) {
if (this !== root && admitted === undefined) {
admitted = this;
entered();
await gate;
}
return Reflect.apply(nativeStat, this, args);
});
const close = vi
.spyOn(OwnedFileHandle.prototype, "close")
.mockImplementation(async function (this: OwnedFileHandle) {
if (this.handle === admitted) {
retained = this;
attempts++;
if (failClose && attempts === 1) throw cleanup;
}
await nativeClose.call(this);
});
const opening = reader.open(selected, signal.signal);
void opening.catch(() => undefined);
try {
await started;
signal.abort(cancelled);
release();
await expect(opening).rejects.toMatchObject({
reason: "cancelled",
cancellation: { reason: cancelled },
...(failClose ? { cleanup: { reason: cleanup.message } } : {}),
});
if (failClose) expect(retained?.handle.fd).toBeGreaterThanOrEqual(0);
await reader.close();
expect(attempts).toBe(failClose ? 2 : 1);
expect(admitted?.fd).toBe(-1);
expect(root.fd).toBeGreaterThanOrEqual(0);
} finally {
release();
await opening.catch(() => undefined);
stat.mockRestore();
close.mockRestore();
}
},
);
},
);
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,673 @@
import { withPyInstallerReader } from "../../../tests/fixtures/pyinstallerReader.js";
import { join } from "node:path";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { createHash } from "node:crypto";
import { writeFile, type FileHandle } from "node:fs/promises";
import { buffer } from "node:stream/consumers";
import { deflateSync } from "node:zlib";
import { setImmediate } from "node:timers/promises";
import { expect, it, vi } from "vitest";
import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js";
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
import { hashReadable } from "../ArtifactHash.js";
import {
ArtifactReaderFailure,
type ArtifactEntry,
} from "../ArtifactReader.js";
import {
cArchiveExecutable,
pyzArchive,
PYTHON_39_MAGIC,
} from "./CArchive.fixture.js";
const hash = (bytes: Buffer): string =>
createHash("sha256").update(bytes).digest("hex");
const archive = (body: Buffer) =>
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
members: [{ name: "data", typeCode: "x", payload: body, compress: false }],
});
const members = async (reader: PyInstallerArtifactReader) => {
const entries: ArtifactEntry[] = [];
for await (const entry of reader.entries()) entries.push(entry);
return entries;
};
const deferred = () => {
let resolve: () => void = () => undefined;
const promise = new Promise<void>((done) => {
resolve = done;
});
return { promise, resolve };
};
it.each([false, true])(
"settles independent private and carrier closes and retries only failed owners (carrier failure: %s)",
async (failCarrier) => {
const path = join(await createTestTempDirectory("rea-pyi-cleanup-"), "app");
await writeFile(
path,
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
members: ["one", "two"].map((name) => ({
name: `${name}.pyz`,
typeCode: "z",
compress: true,
payload: pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [{ name: "app", typeCode: 0, body: Buffer.from(name) }],
}),
})),
}),
);
const reader = new PyInstallerArtifactReader(path);
const entries = (await members(reader)).filter((entry) =>
entry.path.endsWith("app.pyc"),
);
expect(entries).toHaveLength(2);
const nativeClose = OwnedFileHandle.prototype.close;
const privateOwners = new Set<OwnedFileHandle>();
const attempts: OwnedFileHandle[] = [];
let collecting = true;
let failing = true;
const close = vi
.spyOn(OwnedFileHandle.prototype, "close")
.mockImplementation(async function (this: OwnedFileHandle) {
attempts.push(this);
if (collecting) privateOwners.add(this);
if (failing && (privateOwners.has(this) || failCarrier))
throw new Error(
privateOwners.has(this) ? "private close EIO" : "carrier close EIO",
);
await nativeClose.call(this);
});
try {
for (const entry of entries)
await expect(buffer(await reader.open(entry))).rejects.toMatchObject({
cleanup: { reason: "private close EIO" },
});
expect(privateOwners.size).toBe(2);
collecting = false;
attempts.length = 0;
const failure = await reader.close().then(
() => undefined,
(cause: unknown) => cause,
);
expect(attempts).toHaveLength(3);
expect(new Set(attempts).size).toBe(3);
expect(failure).toMatchObject({
cleanup: {
resources: expect.arrayContaining(failCarrier ? [path] : []),
},
});
expect(
(failure as ArtifactReaderFailure).cleanup?.resources,
).toHaveLength(failCarrier ? 3 : 2);
expect((failure as ArtifactReaderFailure).cleanup?.reason).toContain(
"private close EIO",
);
if (failCarrier)
expect((failure as ArtifactReaderFailure).cleanup?.reason).toContain(
"carrier close EIO",
);
const carrier = attempts.find((owner) => !privateOwners.has(owner))!;
expect(carrier.handle.fd >= 0).toBe(failCarrier);
for (const owner of privateOwners)
expect(owner.handle.fd).toBeGreaterThanOrEqual(0);
failing = false;
attempts.length = 0;
await reader.close();
expect(attempts).toHaveLength(failCarrier ? 3 : 2);
for (const owner of [...privateOwners, carrier])
expect(owner.handle.fd).toBe(-1);
attempts.length = 0;
await reader.close();
expect(attempts).toEqual([]);
} finally {
close.mockRestore();
await reader.close();
}
},
);
it("binds materialization to the inventoried digest and checks cached carrier freshness", async () => {
const bytes = archive(Buffer.from("first"));
await withPyInstallerReader(bytes, async (reader, _handle, path) => {
await expect(reader.prepareContainer("0".repeat(64))).rejects.toMatchObject(
{ reason: "integrity" },
);
await reader.prepareContainer(hash(bytes));
await reader.prepareContainer(hash(bytes));
await writeFile(path, archive(Buffer.from("other")));
await expect(reader.prepareContainer(hash(bytes))).rejects.toMatchObject({
reason: "integrity",
});
});
});
it("bounds descriptor reads while hashing a selected stored member", async () => {
const payload = Buffer.alloc(2 * 1024 * 1024, 0x61);
await withPyInstallerReader(archive(payload), async (reader, handle) => {
const entry = (await members(reader))[0]!;
const read = handle.read.bind(handle);
let maximum = 0;
const spy = vi
.spyOn(handle, "read")
.mockImplementation((...args: unknown[]) => {
if (typeof args[2] === "number") maximum = Math.max(maximum, args[2]);
return Reflect.apply(read, handle, args);
});
try {
const observed = await hashReadable(await reader.open(entry));
expect(observed.sha256).toBe(hash(payload));
expect(observed.bytes).toBe(payload.length);
expect(maximum).toBeGreaterThan(0);
expect(maximum).toBeLessThanOrEqual(64 * 1024);
} finally {
spy.mockRestore();
}
});
});
it.each([false, true])(
"joins pending native reads during cancellation/close (I/O failure: %s)",
async (fail) => {
await withPyInstallerReader(
archive(Buffer.alloc(256 * 1024, 0x61)),
async (reader, handle) => {
const entry = (await members(reader))[0]!;
const entered = deferred(),
release = deferred();
const read = handle.read.bind(handle);
const io = Object.assign(new Error("native read failed"), {
code: "EIO",
});
const caller = new Error("caller stopped reading");
const controller = new AbortController();
let held = false;
const spy = vi
.spyOn(handle, "read")
.mockImplementation(async (...args: unknown[]) => {
if (!held) {
held = true;
entered.resolve();
await release.promise;
if (fail) throw io;
}
return Reflect.apply(read, handle, args);
});
const consumed = reader
.open(entry, controller.signal)
.then((stream) => buffer(stream));
void consumed.catch(() => undefined);
let closing: Promise<void> | undefined;
try {
await entered.promise;
controller.abort(caller);
let closed = false;
closing = reader.close().then(() => {
closed = true;
});
await setImmediate();
expect(closed).toBe(false);
expect(handle.fd).toBeGreaterThanOrEqual(0);
release.resolve();
await expect(consumed).rejects.toMatchObject({
reason: fail ? "io" : "cancelled",
cause: fail ? io : caller,
});
await closing;
expect(handle.fd).toBeGreaterThanOrEqual(0);
} finally {
release.resolve();
await Promise.allSettled([
consumed,
...(closing === undefined ? [] : [closing]),
]);
spy.mockRestore();
}
},
);
},
);
it("preserves BOM characters in length-delimited UTF-8 fields", async () => {
const bytes = cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
pythonLibrary: "\ufefflibpython.so",
members: [
{
name: "\ufeffasset",
typeCode: "x",
payload: Buffer.from("data"),
compress: false,
},
{
name: "PYZ.pyz",
typeCode: "z",
compress: false,
payload: pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [
{ name: "\ufeffmodule", typeCode: 0, body: Buffer.from("module") },
],
}),
},
],
});
await withPyInstallerReader(bytes, async (reader) => {
const entries = await members(reader);
expect(reader.formatMetadata()).toMatchObject({
python_library: "\ufefflibpython.so",
});
expect(entries.map((entry) => entry.path)).toContain("\ufeffasset");
expect(
entries.find((entry) => entry.path === "\ufeffasset")?.formatMetadata,
).toMatchObject({ source_name: "\ufeffasset" });
expect(
entries.find((entry) => entry.path === "PYZ.pyz/\ufeffmodule.pyc")
?.formatMetadata,
).toMatchObject({ source_name: "\ufeffmodule" });
});
});
it("keeps runtime options as declarations with literal names and distinct source indices", async () => {
const option = "pyi-runtime-tmpdir /tmp/../chosen";
await withPyInstallerReader(
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
members: [
{
name: option,
typeCode: "o",
payload: Buffer.alloc(0),
compress: false,
},
{
name: option,
typeCode: "o",
payload: Buffer.alloc(0),
compress: false,
},
{
name: "data",
typeCode: "x",
payload: Buffer.from("data"),
compress: false,
},
],
}),
async (reader) => {
const entries = await members(reader);
const options = entries.filter((entry) => entry.kind === "metadata");
expect(options.map((entry) => entry.path)).toEqual([
"__pyinstaller__/runtime-options/0",
"__pyinstaller__/runtime-options/1",
]);
expect(options.map((entry) => entry.formatMetadata)).toMatchObject([
{ source_name: option, index: 0 },
{ source_name: option, index: 1 },
]);
expect(reader.formatMetadata()).toMatchObject({
runtime_option_count: 2,
entry_count: 3,
});
await expect(reader.open(options[0]!)).rejects.toMatchObject({
reason: "format",
});
},
);
});
it("rejects metadata expansion before retaining every repeated long container path", async () => {
const image = pyzArchive({
magic: PYTHON_39_MAGIC,
modules: Array.from({ length: 3000 }, (_, index) => ({
name: `module${String(index)}`,
typeCode: 0,
body: Buffer.from("x"),
})),
});
await withPyInstallerReader(
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
members: [
{
name: "a".repeat(40000),
typeCode: "z",
payload: image,
compress: false,
},
],
}),
async (reader) => {
let failure: unknown;
try {
for await (const entry of reader.entries()) {
void entry;
}
} catch (cause: unknown) {
failure = cause;
}
expect(failure).toBeInstanceOf(ArtifactReaderFailure);
expect(failure).toMatchObject({ reason: "limit", boundary: "memory" });
},
);
});
it.each([false, true])(
"validates compressed member bounds through the stream (truncated: %s)",
async (truncated) => {
const payload = Buffer.from("complete module payload");
const encoded = deflateSync(payload);
const bytes = cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
members: [
{
name: "data",
typeCode: "x",
payload,
compress: true,
declaredUncompressedSize: truncated ? payload.length : 4,
},
],
});
if (truncated)
bytes.writeUInt32BE(encoded.length - 2, 64 + encoded.length + 8);
await withPyInstallerReader(bytes, async (reader) => {
const entry = (await members(reader))[0]!;
await expect(buffer(await reader.open(entry))).rejects.toMatchObject({
reason: truncated ? "format" : "limit",
});
});
},
);
it("joins a late private-file close and retries its failed owner after cancellation", async () => {
const bytes = cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
members: [
{
name: "PYZ.pyz",
typeCode: "z",
payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }),
compress: true,
},
],
});
await withPyInstallerReader(bytes, async (reader, borrowed) => {
const entered = deferred(),
release = deferred();
const nativeClose = OwnedFileHandle.prototype.close;
const failure = new Error("private final close failed");
let owner: OwnedFileHandle | undefined;
let attempts = 0;
const spy = vi
.spyOn(OwnedFileHandle.prototype, "close")
.mockImplementation(async function (this: OwnedFileHandle) {
if (owner === undefined) {
owner = this;
entered.resolve();
await release.promise;
throw failure;
}
if (this === owner) attempts += 1;
await nativeClose.call(this);
});
const signal = new AbortController();
const listing = reader
.entries(signal.signal)
[Symbol.asyncIterator]()
.next()
.catch((cause: unknown) => cause);
try {
await entered.promise;
let settled = false;
const closing = reader.close().then(() => {
settled = true;
});
signal.abort(new Error("cancel during final cleanup"));
await setImmediate();
expect(settled).toBe(false);
expect(owner?.handle.fd).toBeGreaterThanOrEqual(0);
release.resolve();
expect(await listing).toBe(failure);
await closing;
expect(attempts).toBe(1);
expect(owner?.handle.fd).toBe(-1);
expect(borrowed.fd).toBeGreaterThanOrEqual(0);
} finally {
release.resolve();
spy.mockRestore();
await listing;
await reader.close();
}
});
});
for (const cancel of [false, true]) {
it.each([false, true])(
`awaits private member close before completion (cancel: ${cancel}, close failure: %s)`,
async (failClose) => {
const bytes = cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
members: [
{
name: "PYZ.pyz",
typeCode: "z",
compress: true,
payload: pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [
{ name: "app", typeCode: 0, body: Buffer.from("body") },
],
}),
},
],
});
await withPyInstallerReader(bytes, async (reader, borrowed) => {
const entry = (await members(reader)).find((item) =>
item.path.endsWith("app.pyc"),
)!;
const entered = deferred(),
release = deferred();
const signal = new AbortController();
const failure = new Error("private member final close failed");
const nativeClose = OwnedFileHandle.prototype.close;
let owner: OwnedFileHandle | undefined;
let attempts = 0;
const spy = vi
.spyOn(OwnedFileHandle.prototype, "close")
.mockImplementation(async function (this: OwnedFileHandle) {
if (owner === undefined) {
owner = this;
attempts += 1;
entered.resolve();
await release.promise;
if (failClose) throw failure;
} else if (this === owner) attempts += 1;
await nativeClose.call(this);
});
const output = await reader.open(entry, signal.signal);
const reading = buffer(output).then(
(value) => ({ ok: true as const, value }),
(cause: unknown) => ({ ok: false as const, cause }),
);
try {
await entered.promise;
expect(output.readableEnded).toBe(false);
expect(owner?.handle.fd).toBeGreaterThanOrEqual(0);
if (cancel) {
signal.abort(new Error("cancel during member cleanup"));
await setImmediate();
expect(output.destroyed).toBe(false);
}
release.resolve();
const result = await reading;
if (failClose || cancel) {
expect(result.ok).toBe(false);
if (result.ok) throw new Error("Expected read failure");
expect(result.cause).toMatchObject(
failClose
? { reason: "io", message: failure.message }
: { reason: "cancelled" },
);
if (failClose) expect(owner?.handle.fd).toBeGreaterThanOrEqual(0);
} else {
expect(result.ok).toBe(true);
if (!result.ok) throw result.cause;
expect(result.value.subarray(16)).toEqual(Buffer.from("body"));
}
await reader.close();
expect(attempts).toBe(failClose ? 2 : 1);
expect(owner?.handle.fd).toBe(-1);
expect(borrowed.fd).toBeGreaterThanOrEqual(0);
} finally {
release.resolve();
await reading;
spy.mockRestore();
await reader.close();
}
});
},
);
}
it("preserves a private-image read failure when its descriptor close also fails", async () => {
const bytes = cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
members: [
{
name: "PYZ.pyz",
typeCode: "z",
payload: pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [{ name: "app", typeCode: 0, body: Buffer.from("body") }],
}),
compress: true,
},
],
});
await withPyInstallerReader(bytes, async (reader, borrowed) => {
const entry = (await members(reader)).find((item) =>
item.path.endsWith("app.pyc"),
)!;
const nativeClose = OwnedFileHandle.prototype.close;
const nativeRead = borrowed.read;
const io = Object.assign(new Error("private read EIO"), { code: "EIO" });
let failedOwner: OwnedFileHandle | undefined;
const read = vi
.spyOn(Object.getPrototypeOf(borrowed), "read")
.mockImplementation(function (this: FileHandle, ...args: unknown[]) {
return this === borrowed
? Reflect.apply(nativeRead, this, args)
: Promise.reject(io);
});
const close = vi
.spyOn(OwnedFileHandle.prototype, "close")
.mockImplementation(async function (this: OwnedFileHandle) {
if (this.handle !== borrowed) {
failedOwner = this;
throw new Error("private close EIO");
}
await nativeClose.call(this);
});
try {
await expect(buffer(await reader.open(entry))).rejects.toMatchObject({
reason: "io",
cause: io,
});
await expect(reader.close()).rejects.toMatchObject({
cleanup: { reason: "private close EIO" },
});
expect(failedOwner?.handle.fd).toBeGreaterThanOrEqual(0);
} finally {
read.mockRestore();
close.mockRestore();
await reader.close();
}
expect(failedOwner?.handle.fd).toBe(-1);
expect(borrowed.fd).toBeGreaterThanOrEqual(0);
});
});
it.each([false, true])(
"shares owned carrier admission across callers and joins late admission (owner closes: %s)",
async (ownerCloses) => {
const bytes = archive(Buffer.from("shared admission"));
await withPyInstallerReader(
bytes,
async (_borrowedReader, borrowed, path) => {
const reader = new PyInstallerArtifactReader(path);
const entered = deferred(),
release = deferred();
const handles = new Set<FileHandle>();
const stat = borrowed.stat;
const spy = vi
.spyOn(Object.getPrototypeOf(borrowed), "stat")
.mockImplementation(async function (
this: FileHandle,
...args: unknown[]
) {
if (this !== borrowed) {
handles.add(this);
entered.resolve();
await release.promise;
}
return Reflect.apply(stat, this, args);
});
const caller = new AbortController(),
reason = new Error("one admission waiter leaves");
const first = reader
.prepareContainer(hash(bytes), caller.signal)
.catch((cause: unknown) => cause);
const peer = reader
.prepareContainer(hash(bytes))
.catch((cause: unknown) => cause);
let closing: Promise<void> | undefined;
try {
await entered.promise;
await setImmediate();
expect(handles.size).toBe(1);
caller.abort(reason);
if (ownerCloses) {
let closed = false;
closing = reader.close().then(() => {
closed = true;
});
await setImmediate();
expect(closed).toBe(false);
}
release.resolve();
expect(await first).toMatchObject({
reason: "cancelled",
cause: reason,
});
if (ownerCloses) {
expect(await peer).toMatchObject({ reason: "cancelled" });
await closing;
} else expect(await peer).toBeUndefined();
await reader.close();
for (const handle of handles) expect(handle.fd).toBe(-1);
} finally {
release.resolve();
await Promise.allSettled([first, peer]);
spy.mockRestore();
await reader.close();
await closing;
for (const handle of handles) {
await handle.close();
}
}
},
);
},
);
@@ -0,0 +1,60 @@
import type { FileHandle } from "node:fs/promises";
import { err, ok, type Result } from "../../domain/result.js";
import { ArtifactReaderFailure } from "../ArtifactReader.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import {
pyInstallerFailure,
type PyInstallerFailure,
} from "./PyInstallerFailure.js";
/** Fill one already-budgeted format range; only zero bytes establish EOF. */
export const readPyInstallerBytes = async (
handle: Pick<FileHandle, "read">,
input: {
readonly position: number;
readonly length: number;
readonly label: string;
readonly signal?: AbortSignal | undefined;
},
): Promise<Result<Buffer, PyInstallerFailure>> => {
abortIfNeeded(input.signal);
if (
!Number.isSafeInteger(input.position) ||
!Number.isSafeInteger(input.length) ||
input.position < 0 ||
input.length < 0 ||
!Number.isSafeInteger(input.position + input.length)
)
return err(
pyInstallerFailure(
"format",
`${input.label} has an invalid source range`,
),
);
const bytes = Buffer.alloc(input.length);
let offset = 0;
while (offset < bytes.length) {
abortIfNeeded(input.signal);
let count: number;
try {
const read = await handle.read(
bytes,
offset,
bytes.length - offset,
input.position + offset,
);
count = read.bytesRead;
} catch (cause: unknown) {
throw new ArtifactReaderFailure(
"io",
`${input.label} could not be read`,
{ cause },
);
}
abortIfNeeded(input.signal);
if (count === 0)
return err(pyInstallerFailure("format", `${input.label} is truncated`));
offset += count;
}
return ok(bytes);
};
@@ -0,0 +1,10 @@
/** Typed failure from a pure PyInstaller parser before the reader adapts it. */
export interface PyInstallerFailure {
readonly reason: "format" | "integrity" | "limit";
readonly message: string;
}
export const pyInstallerFailure = (
reason: PyInstallerFailure["reason"],
message: string,
): PyInstallerFailure => ({ reason, message });
@@ -0,0 +1,36 @@
import {
NonRegularFileReadError,
RegularFileAdmissionFailure,
RegularFileOperationCancelled,
} from "../../filesystem/RegularFile.js";
import { ArtifactReaderFailure } from "../ArtifactReader.js";
/** Both carrier and private-image admission preserve cancellation and cleanup. */
export const pyInstallerFileAdmissionFailure = (
path: string,
cause: unknown,
): ArtifactReaderFailure => {
if (cause instanceof ArtifactReaderFailure) return cause;
if (cause instanceof RegularFileOperationCancelled)
return new ArtifactReaderFailure(
"cancelled",
"PyInstaller file admission cancelled",
{ cause: cause.reason },
);
if (cause instanceof RegularFileAdmissionFailure)
return ArtifactReaderFailure.withCleanup(
pyInstallerFileAdmissionFailure(path, cause.cause),
ArtifactReaderFailure.cleanupObservation(cause.cleanupCause, path),
);
if (cause instanceof NonRegularFileReadError)
return new ArtifactReaderFailure(
"format",
`PyInstaller source is not a regular file: ${path}`,
{ cause },
);
return new ArtifactReaderFailure(
"io",
"PyInstaller archive could not be opened",
{ cause },
);
};
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,147 @@
import { Readable, Transform } from "node:stream";
import { pipeline } from "node:stream/promises";
import { createInflate } from "node:zlib";
import { ArtifactReaderFailure } from "../ArtifactReader.js";
import { abortIfNeeded } from "../ArtifactHash.js";
interface MemberStreamOptions {
readonly sourceBytes: number;
readonly compressed: boolean;
readonly header: Buffer | undefined;
readonly expectedBytes: number | undefined;
readonly maxBytes: number;
readonly signal: AbortSignal;
readonly charge?: (bytes: number) => void;
readonly complete?: (bytes: number) => Promise<void>;
}
/** Backpressure bounds plaintext independently of the untrusted declared size. */
export const pyInstallerMemberStream = (
source: Readable,
options: MemberStreamOptions,
): { readonly output: Readable; readonly completion: Promise<void> } => {
let bytes = 0;
let headerSent = false;
let completing = false;
const prependHeader = (): void => {
if (!headerSent && options.header !== undefined)
output.push(options.header);
headerSent = true;
};
const output = new Transform({
transform(chunk: Buffer, _encoding, callback) {
try {
abortIfNeeded(options.signal);
bytes += chunk.length;
if (bytes > options.maxBytes)
throw new ArtifactReaderFailure(
"limit",
"PyInstaller member exceeds its plaintext budget",
);
options.charge?.(chunk.length);
prependHeader();
callback(null, chunk);
} catch (cause: unknown) {
callback(asError(cause));
}
},
flush(callback) {
completing = true;
void (async () => {
abortIfNeeded(options.signal);
if (
inflate !== undefined &&
inflate.bytesWritten !== options.sourceBytes
)
throw new ArtifactReaderFailure(
"integrity",
"Compressed PyInstaller member did not consume its complete declared source span",
);
if (
options.expectedBytes !== undefined &&
bytes !== options.expectedBytes
)
throw new ArtifactReaderFailure(
"integrity",
"PyInstaller member size disagrees with its declaration or previous observation",
);
await options.complete?.(bytes);
abortIfNeeded(options.signal);
prependHeader();
})().then(
() => {
completing = false;
try {
abortIfNeeded(options.signal);
} catch (cause: unknown) {
callback(asError(cause));
return;
}
callback();
},
(cause: unknown) => {
completing = false;
callback(asError(cause));
},
);
},
});
// A consumer can retire its reader before attaching its stream error handler.
output.on("error", () => undefined);
const onAbort = (): void => {
// Verification and private-file cleanup own their accepted native I/O.
if (completing) return;
// The descriptor source owns cancellation until its native read settles.
// Retiring it downstream earlier would hide a native I/O rejection.
if (!source.readableEnded && !source.destroyed) return;
output.destroy(
new ArtifactReaderFailure(
"cancelled",
"PyInstaller member read cancelled",
{ cause: options.signal.reason },
),
);
};
options.signal.addEventListener("abort", onAbort, { once: true });
const onSourceFailure = (cause: unknown): void => {
output.destroy(
cause instanceof ArtifactReaderFailure
? cause
: new ArtifactReaderFailure(
"io",
"PyInstaller member source could not be read",
{ cause },
),
);
};
source.on("error", onSourceFailure);
const inflate = options.compressed ? createInflate() : undefined;
inflate?.on("error", (cause: unknown) =>
output.destroy(
new ArtifactReaderFailure(
"format",
"Compressed PyInstaller member is not a complete zlib stream",
{ cause },
),
),
);
const completion = (
inflate === undefined
? pipeline(source, output)
: pipeline(source, inflate, output)
).finally(() => {
options.signal.removeEventListener("abort", onAbort);
source.removeListener("error", onSourceFailure);
});
// Preserve rejection for the owning scope without an early unhandled promise.
void completion.catch(() => undefined);
if (options.signal.aborted) onAbort();
return { output, completion };
};
const asError = (cause: unknown): Error =>
cause instanceof Error
? cause
: new ArtifactReaderFailure("format", "PyInstaller member read failed", {
cause,
});
@@ -0,0 +1,179 @@
import type {
PyInstallerMetadata,
PyInstallerEntryMetadata,
} from "../../domain/artifactFormatMetadata.js";
import type { CArchiveEntry, PyInstallerCookie } from "./CArchiveLayout.js";
import type { PyzArchive, PyzEntry } from "./PyzArchive.js";
import { pythonVersionForMagic } from "./PythonBytecode.js";
const pythonVersion = (value: number) =>
value >= 100
? { major: Math.floor(value / 100), minor: value % 100 }
: value >= 20
? { major: Math.floor(value / 10), minor: value % 10 }
: undefined;
export const cookieMetadata = (
cookie: PyInstallerCookie,
entries: readonly CArchiveEntry[],
encryptionMarker: boolean,
): PyInstallerMetadata => {
const version = pythonVersion(cookie.pythonVersion);
return {
kind: "pyinstaller",
layout_version: cookie.layout,
archive_start: cookie.archiveStart,
archive_size: cookie.packageLength,
cookie_offset: cookie.cookieOffset,
toc_offset: cookie.tocOffset,
toc_size: cookie.tocLength,
entry_count: entries.length,
runtime_option_count: entries.filter((entry) => entry.typeCode === "o")
.length,
external_dependency_count: entries.filter((entry) => entry.typeCode === "d")
.length,
python_version_raw: cookie.pythonVersion,
python_major: version?.major ?? null,
python_minor: version?.minor ?? null,
python_library: cookie.pythonLibrary,
encryption_marker_present: encryptionMarker,
};
};
/** A sibling PYZ header is usable for scripts only when every image and the cookie agree. */
export const scriptPythonMagic = (
cookie: PyInstallerCookie,
archives: Iterable<PyzArchive>,
): Buffer | undefined => {
const version = pythonVersion(cookie.pythonVersion);
if (version === undefined) return undefined;
let selected: Buffer | undefined;
for (const archive of archives) {
const magic = archive.pythonMagic;
if (
pythonVersionForMagic(magic) !==
`${String(version.major)}.${String(version.minor)}` ||
(selected !== undefined && !selected.equals(magic))
)
return undefined;
selected = magic;
}
return selected;
};
const entryMetadata = (
source: Pick<
PyInstallerEntryMetadata,
| "container"
| "container_name"
| "index"
| "source_name"
| "type_code"
| "relative_offset"
| "compression_flag"
>,
bytecode: boolean,
header: Buffer | undefined,
): PyInstallerEntryMetadata => ({
kind: "pyinstaller-entry",
...source,
bytecode_header: bytecode
? header === undefined
? "unknown"
: "derived"
: "not-bytecode",
bytecode_magic: header?.subarray(0, 4).toString("hex") ?? null,
pyz_status: "not-pyz",
directory_record_offset: null,
directory_record_size: null,
pyz_sha256: null,
pyz_python_magic: null,
pyz_python_version: null,
pyz_entry_count: null,
encryption_source: null,
});
export const cArchiveEntryMetadata = (
entry: CArchiveEntry,
header?: Buffer,
archive?: PyzArchive,
encryptionMarker = false,
): PyInstallerEntryMetadata => ({
...entryMetadata(
{
container: "carchive",
container_name: null,
index: entry.index,
source_name: entry.name,
type_code: entry.typeCode,
relative_offset: entry.offset,
compression_flag: entry.compressionFlag === 1 ? 1 : 0,
},
["s", "m", "M"].includes(entry.typeCode),
header,
),
...(entry.typeCode === "d" ? dependencyMetadata(entry.name) : {}),
pyz_status:
archive?.kind === "entries"
? "decoded"
: archive === undefined
? "not-pyz"
: "opaque",
directory_record_offset: entry.tocRecordOffset,
directory_record_size: entry.tocRecordLength,
pyz_sha256: archive?.sha256 ?? null,
pyz_python_magic: archive?.pythonMagic.toString("hex") ?? null,
pyz_python_version:
archive === undefined
? null
: (pythonVersionForMagic(archive.pythonMagic) ?? null),
pyz_entry_count: archive?.kind === "entries" ? archive.entries.length : null,
encryption_source:
archive !== undefined && encryptionMarker ? "carchive-marker" : null,
});
/** The bootloader splits at the first colon; neither part is a local read grant. */
const dependencyMetadata = (
source: string,
): Pick<
PyInstallerEntryMetadata,
"dependency_status" | "dependency_archive" | "dependency_member"
> => {
const colon = source.indexOf(":");
const valid = colon > 0 && colon < source.length - 1;
return {
dependency_status: valid ? "unresolved" : "invalid-reference",
dependency_archive: valid ? source.slice(0, colon) : null,
dependency_member: valid ? source.slice(colon + 1) : null,
};
};
/** Legacy writers encrypt nonempty namespace payloads as well as module payloads. */
export const pyzEntryEncrypted = (
entry: PyzEntry,
encryptionMarker: boolean,
): boolean => encryptionMarker && entry.length > 0;
export const pyzEntryMetadata = (
container: CArchiveEntry,
entry: PyzEntry,
header: Buffer | undefined,
encryptionMarker: boolean,
): PyInstallerEntryMetadata => ({
...entryMetadata(
{
container: "pyz",
container_name: container.name,
index: entry.index,
source_name: entry.name,
type_code: entry.typeCode,
relative_offset: entry.offset,
compression_flag: entry.compressionFlag,
},
entry.typeCode === 0 || entry.typeCode === 1,
header,
),
encryption_source: pyzEntryEncrypted(entry, encryptionMarker)
? "carchive-marker"
: null,
});
@@ -0,0 +1,255 @@
import type { FileHandle } from "node:fs/promises";
import { lstat, mkdtemp, open, rm } from "node:fs/promises";
import type { Stats } from "node:fs";
import { Readable } from "node:stream";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
import {
openRegularFile,
RegularFileAdmissionFailure,
sameRegularFileState,
} from "../../filesystem/RegularFile.js";
import { ArtifactReaderFailure } from "../ArtifactReader.js";
import {
ArtifactResourceScope,
type ArtifactResourceOwner,
} from "../ArtifactResourceScope.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import { readArtifactFileChunks } from "../readArtifactFileChunks.js";
import { pyInstallerMemberStream } from "./PyInstallerMemberStream.js";
import type { CArchiveEntry } from "./CArchiveLayout.js";
import type { PyInstallerMember } from "./CArchiveMembers.js";
import { pyInstallerFileAdmissionFailure } from "./PyInstallerFileAdmission.js";
// Each outer-compressed image retains one inode and its identity until close.
// This is a per-reader filesystem capacity budget, independent of byte budgets.
const MAX_PRIVATE_PYZ_IMAGES = 128;
interface StoredImage {
readonly path: string;
readonly initial: Stats;
}
export interface PyInstallerMemberSource {
readonly source: Readable;
readonly verify?: () => Promise<void>;
readonly close?: (primary?: unknown) => Promise<void>;
}
/** The reader's operation scope drains work before retiring these private files. */
export class PyInstallerPyzStore {
#root: string | undefined;
#serial = 0;
readonly #owners = new Map<
FileHandle,
Extract<ArtifactResourceOwner, { kind: "file-handle" }>
>();
readonly #cleanup = new ArtifactResourceScope();
readonly #images = new Map<FileHandle, StoredImage>();
reserveImages(count: number): void {
if (count > MAX_PRIVATE_PYZ_IMAGES)
throw new ArtifactReaderFailure(
"limit",
"PyInstaller compressed PYZ images exceed the 128-file private storage budget",
{ boundary: "capacity" },
);
}
async spool(
handle: FileHandle,
archiveStart: number,
entry: CArchiveEntry,
signal?: AbortSignal,
): Promise<{ handle: FileHandle; start: number }> {
abortIfNeeded(signal);
if (entry.compressedSize === 0)
throw new ArtifactReaderFailure(
"format",
"Compressed PYZ image has no zlib bytes",
);
this.#root ??= await mkdtemp(join(tmpdir(), "rea-pyz-"));
const path = join(this.#root, String(this.#serial++));
const writer = await open(path, "wx", 0o600);
this.#retain(path, new OwnedFileHandle(writer));
abortIfNeeded(signal);
const source = readArtifactFileChunks(handle, {
start: archiveStart + entry.offset,
end: archiveStart + entry.offset + entry.compressedSize - 1,
signal,
});
const stream = pyInstallerMemberStream(source, {
sourceBytes: entry.compressedSize,
compressed: true,
header: undefined,
expectedBytes: entry.uncompressedSize,
maxBytes: entry.uncompressedSize,
signal: signal ?? new AbortController().signal,
});
const copied = this.#writeImage(writer, stream.output, signal);
const outcomes = await Promise.allSettled([stream.completion, copied]);
// A failed native write can retire the upstream stream. Preserve that I/O
// failure instead of replacing it with a secondary premature-close error.
const copy = outcomes[1];
if (copy?.status === "rejected") throw copy.reason;
const decoded = outcomes[0];
if (decoded?.status === "rejected") throw decoded.reason;
await this.release(writer);
abortIfNeeded(signal);
const opened = await this.#admit(path, signal);
this.#images.set(opened, { path, initial: await opened.stat() });
return { handle: opened, start: 0 };
}
async release(handle: FileHandle): Promise<void> {
const owner = this.#owners.get(handle);
if (owner === undefined) return;
const outcome = await this.#cleanup.release(owner);
if (outcome.kind === "failed") throw outcome.cause;
this.#owners.delete(handle);
}
async memberSource(
read: NonNullable<PyInstallerMember["read"]>,
root: FileHandle | undefined,
signal: AbortSignal,
): Promise<PyInstallerMemberSource> {
let handle = read.kind === "carchive" ? root : read.handle;
if (handle === undefined)
throw new ArtifactReaderFailure(
"unavailable",
"PyInstaller archive is closed",
);
const image = read.kind === "pyz" ? this.#images.get(handle) : undefined;
let verify: (() => Promise<void>) | undefined;
let close: ((primary?: unknown) => Promise<void>) | undefined;
if (image !== undefined) {
handle = await this.#admit(image.path, signal);
const opened = handle;
verify = () => this.#verifyImage(opened, image);
close = async (primary) => {
try {
await this.release(opened);
} catch (cause: unknown) {
throw ArtifactReaderFailure.withCleanup(
primary ?? cause,
ArtifactReaderFailure.cleanupObservation(cause, image.path),
);
}
};
await verify();
}
abortIfNeeded(signal);
const start =
read.kind === "carchive"
? read.archiveStart + read.source.offset
: read.offset;
const length =
read.kind === "carchive" ? read.source.compressedSize : read.length;
const source =
length === 0
? Readable.from([])
: readArtifactFileChunks(handle, {
start,
end: start + length - 1,
signal,
});
source.on("error", () => undefined);
return {
source,
...(verify === undefined ? {} : { verify }),
...(close === undefined ? {} : { close }),
};
}
async close(): Promise<void> {
await this.#cleanup.close();
this.#owners.clear();
if (this.#root !== undefined) {
try {
await rm(this.#root, { recursive: true, force: true });
} catch (cause: unknown) {
throw ArtifactReaderFailure.withCleanup(
cause,
ArtifactReaderFailure.cleanupObservation(cause, this.#root),
);
}
this.#root = undefined;
}
this.#images.clear();
}
async #admit(path: string, signal?: AbortSignal): Promise<FileHandle> {
try {
const handle = await openRegularFile(path, {
symlinks: "reject",
signal,
});
this.#retain(path, new OwnedFileHandle(handle));
return handle;
} catch (cause: unknown) {
if (cause instanceof RegularFileAdmissionFailure)
this.#retain(path, cause.owner);
throw pyInstallerFileAdmissionFailure(path, cause);
}
}
#retain(path: string, handle: OwnedFileHandle): void {
const owner = { kind: "file-handle" as const, resource: path, handle };
this.#owners.set(handle.handle, owner);
this.#cleanup.retain(owner);
}
async #writeImage(
handle: FileHandle,
source: Readable,
signal?: AbortSignal,
): Promise<void> {
let position = 0;
for await (const chunk of source as AsyncIterable<Buffer>) {
// Stop writing after cancellation; drain only already-produced bytes so
// the descriptor source can report its in-flight I/O outcome truthfully.
if (signal?.aborted) continue;
let offset = 0;
try {
while (offset < chunk.length) {
const { bytesWritten } = await handle.write(
chunk,
offset,
chunk.length - offset,
position,
);
if (bytesWritten === 0)
throw new ArtifactReaderFailure(
"io",
"Private PYZ write made no progress",
);
offset += bytesWritten;
position += bytesWritten;
if (signal?.aborted) break;
}
} catch (cause: unknown) {
const failure =
cause instanceof ArtifactReaderFailure
? cause
: new ArtifactReaderFailure(
"io",
"Private PYZ image could not be written",
{ cause },
);
source.destroy(failure);
throw failure;
}
}
abortIfNeeded(signal);
}
async #verifyImage(handle: FileHandle, image: StoredImage): Promise<void> {
const states = await Promise.all([handle.stat(), lstat(image.path)]);
if (states.some((state) => !sameRegularFileState(image.initial, state)))
throw new ArtifactReaderFailure(
"integrity",
"Private PYZ image changed after parsing",
);
}
}
@@ -0,0 +1,33 @@
/**
* Final CPython bytecode magics for the minor versions whose 16-byte `.pyc`
* header this reader can reconstruct. Other magics leave the PYZ opaque.
* Each value is the four-byte little-endian magic, including the trailing CR LF.
*/
const PYTHON_BYTECODE_MAGICS: Readonly<Record<string, string>> = {
"420d0d0a": "3.7",
"550d0d0a": "3.8",
"610d0d0a": "3.9",
"6f0d0d0a": "3.10",
a70d0d0a: "3.11",
cb0d0d0a: "3.12",
f30d0d0a: "3.13",
"2b0e0d0a": "3.14",
};
/** Limitation recorded when a member's digest covers a derived `.pyc` header. */
export const RECONSTRUCTED_PYTHON_HEADER_LIMITATION =
"Python bytecode header was reconstructed from the archive Python magic; this digest covers the derived file.";
/** Dotted Python version for a supported bytecode magic, when one is known. */
export const pythonVersionForMagic = (magic: Buffer): string | undefined =>
PYTHON_BYTECODE_MAGICS[magic.toString("hex")];
/**
* 16-byte `.pyc` header used since Python 3.7: magic, zero flags, zero
* timestamp, and zero source size. The flags word selects the timestamp form.
*/
export const reconstructedPycHeader = (magic: Buffer): Buffer => {
const header = Buffer.alloc(16);
magic.copy(header, 0, 0, 4);
return header;
};
@@ -0,0 +1,137 @@
import { open, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect, it } from "vitest";
import { readPyzArchive, MAX_PYZ_TABLE_BYTES } from "./PyzArchive.js";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { PYTHON_39_MAGIC, pyzArchive } from "./CArchive.fixture.js";
it("lists modules from a supported PYZ and leaves an unknown magic opaque", async () => {
const body = Buffer.from("module");
const parsed = await readImage(
pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [{ name: "json.decoder", typeCode: 0, body }],
}),
);
expect(parsed.ok).toBe(true);
if (!parsed.ok || parsed.value.kind !== "entries") return;
expect(parsed.value.pythonVersion).toBe("3.9");
expect(parsed.value.entries).toMatchObject([
{ name: "json.decoder", typeCode: 0, package: false },
]);
const opaque = await readImage(
pyzArchive({
magic: Buffer.alloc(4),
modules: [{ name: "json.decoder", typeCode: 0, body }],
}),
);
expect(opaque.ok).toBe(true);
if (!opaque.ok || opaque.value.kind !== "opaque") return;
expect(opaque.value.pythonMagic).toEqual(Buffer.alloc(4));
expect(opaque.value.limitation).toContain("left opaque");
});
it("leaves a PYZ opaque when the table uses an unsupported marshal type", async () => {
const image = pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [],
toc: Buffer.from([0x4e]),
});
const parsed = await readImage(image);
expect(parsed.ok).toBe(true);
if (!parsed.ok || parsed.value.kind !== "opaque") return;
expect(parsed.value.limitation).toContain("left opaque");
expect(parsed.value.pythonMagic).toEqual(PYTHON_39_MAGIC);
});
it("reports an entry that extends outside the PYZ as an integrity error", async () => {
const image = pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [{ name: "missing", typeCode: 0, body: Buffer.from("x") }],
});
image.writeInt32LE(100_000, image.length - 4);
const parsed = await readImage(image);
expect(parsed.ok).toBe(false);
if (parsed.ok) return;
expect(parsed.error.reason).toBe("integrity");
});
it("rejects an oversized sparse PYZ table before allocating its bytes", async () => {
const root = await createTestTempDirectory("rea-pyz-table-budget-");
const path = join(root, "image.pyz");
const header = Buffer.alloc(12);
header.write("PYZ\0");
PYTHON_39_MAGIC.copy(header, 4);
header.writeUInt32BE(12, 8);
await writeFile(path, header);
const handle = await open(path, "r+");
try {
const size = 12 + MAX_PYZ_TABLE_BYTES + 1;
await handle.truncate(size);
expect(await readPyzArchive(handle, 0, size)).toMatchObject({
ok: false,
error: { reason: "limit" },
});
} finally {
await handle.close();
}
});
it("rejects a PYZ table whose back-references repeat one long row", async () => {
// One namespace row with a 16 KiB name, flagged for reference, then 999
// back-references to it. The table is about 21 KiB but would expand to
// 16 MiB of entry names.
const name = Buffer.alloc(16 * 1024, 0x61);
const refs = 1_000;
const row = Buffer.concat([
Buffer.from([0x28 | 0x80]),
int32(2),
Buffer.from([0x61]),
int32(name.length),
name,
Buffer.from([0x29, 3, 0x69]),
int32(3),
Buffer.from([0x69]),
int32(0),
Buffer.from([0x69]),
int32(0),
]);
const reference = Buffer.concat([Buffer.from([0x72]), int32(0)]);
const table = Buffer.concat([
Buffer.from([0x5b]),
int32(refs),
row,
...Array.from({ length: refs - 1 }, () => reference),
]);
const header = Buffer.alloc(12);
header.write("PYZ\0");
PYTHON_39_MAGIC.copy(header, 4);
header.writeUInt32BE(12, 8);
const image = Buffer.concat([header, table]);
expect(image.length).toBeLessThan(32 * 1024);
const parsed = await readImage(image);
expect(parsed).toMatchObject({ ok: false, error: { reason: "limit" } });
if (parsed.ok) return;
expect(parsed.error.message).toContain("expand past");
});
const int32 = (value: number): Buffer => {
const bytes = Buffer.alloc(4);
bytes.writeInt32LE(value);
return bytes;
};
const readImage = async (bytes: Buffer) => {
const root = await createTestTempDirectory("rea-pyz-file-");
const path = join(root, "image.pyz");
await writeFile(path, bytes);
const handle = await open(path, "r");
try {
return await readPyzArchive(handle, 0, bytes.length);
} finally {
await handle.close();
}
};
+266
View File
@@ -0,0 +1,266 @@
import type { FileHandle } from "node:fs/promises";
import { hashReadable } from "../ArtifactHash.js";
import { readArtifactFileChunks } from "../readArtifactFileChunks.js";
import { readPyInstallerBytes } from "./PyInstallerBytes.js";
import { err, ok, type Result } from "../../domain/result.js";
import { readMarshalSubset, type MarshalValue } from "./MarshalSubset.js";
import {
pyInstallerFailure,
type PyInstallerFailure,
} from "./PyInstallerFailure.js";
import { pythonVersionForMagic } from "./PythonBytecode.js";
const PYZ_MAGIC = Buffer.from("PYZ\0");
const PYZ_HEADER_BYTES = 12;
/** One PYZ table entry. Type codes follow the archive's module, package, data, and namespace values. */
export interface PyzEntry {
readonly index: number;
readonly name: string;
readonly typeCode: number;
readonly offset: number;
readonly length: number;
/** PYZ compresses every nonempty payload, independently of its logical module type. */
readonly compressionFlag: 0 | 1;
readonly package: boolean;
readonly namespace: boolean;
}
export type PyzArchive =
| {
readonly kind: "entries";
readonly pythonMagic: Buffer;
readonly pythonVersion: string;
readonly sha256: string;
readonly entries: readonly PyzEntry[];
}
| {
readonly kind: "opaque";
readonly pythonMagic: Buffer;
readonly sha256: string;
readonly limitation: string;
};
/** Bound the transient marshal table allocation independently of image expansion. */
export const MAX_PYZ_TABLE_BYTES = 16 * 1024 * 1024;
/** Parse only the header and table in memory; hash the complete file range in chunks. */
export const readPyzArchive = async (
handle: FileHandle,
start: number,
size: number,
signal?: AbortSignal,
): Promise<Result<PyzArchive, PyInstallerFailure>> => {
if (size < PYZ_HEADER_BYTES)
return err(
pyInstallerFailure("format", "PYZ archive magic does not match"),
);
const headerRead = await readPyInstallerBytes(handle, {
position: start,
length: PYZ_HEADER_BYTES,
label: "PYZ header",
signal,
});
if (!headerRead.ok) return headerRead;
const header = headerRead.value;
const tocOffset =
header.length === PYZ_HEADER_BYTES ? header.readUInt32BE(8) : size;
const validOffset = tocOffset >= PYZ_HEADER_BYTES && tocOffset <= size;
const knownMagic = pythonVersionForMagic(header.subarray(4, 8)) !== undefined;
const tableSize = validOffset && knownMagic ? size - tocOffset : 0;
if (tableSize > MAX_PYZ_TABLE_BYTES)
return err(
pyInstallerFailure(
"limit",
"PYZ table exceeds the 16 MiB in-memory table budget",
),
);
const read = await readPyInstallerBytes(handle, {
position: start + tocOffset,
length: tableSize,
label: "PYZ table",
signal,
});
if (!read.ok) return read;
const table = read.value;
const digest = await hashReadable(
readArtifactFileChunks(handle, {
start,
end: start + size - 1,
...(signal === undefined ? {} : { signal }),
}),
signal,
);
if (digest.bytes !== size)
return err(
pyInstallerFailure("format", "PYZ image extends outside the file"),
);
return parsePyzParts(header, table, size, digest.sha256);
};
const parsePyzParts = (
header: Buffer,
table: Buffer,
size: number,
sha256: string,
): Result<PyzArchive, PyInstallerFailure> => {
if (
header.length < PYZ_HEADER_BYTES ||
!header.subarray(0, 4).equals(PYZ_MAGIC)
)
return err(
pyInstallerFailure("format", "PYZ archive magic does not match"),
);
const pythonMagic = Buffer.from(header.subarray(4, 8));
const pythonVersion = pythonVersionForMagic(pythonMagic);
if (pythonVersion === undefined)
return ok({
kind: "opaque",
pythonMagic,
sha256,
limitation: `PYZ Python magic ${pythonMagic.toString("hex")} is not in the supported set; the member was left opaque.`,
});
const tocOffset = header.readUInt32BE(8);
if (tocOffset < PYZ_HEADER_BYTES || tocOffset > size)
return err(
pyInstallerFailure(
"format",
"PYZ table of contents is outside the archive",
),
);
const marshalled = readMarshalSubset(table);
if (!marshalled.ok) {
if (marshalled.error.reason !== "format") return marshalled;
return ok({
kind: "opaque",
pythonMagic,
sha256,
limitation: `PYZ table of contents could not be parsed (${marshalled.error.message}); the member was left opaque.`,
});
}
const entries = pyzEntries(marshalled.value, tocOffset, table.length);
if (!entries.ok)
return entries.error.reason === "format"
? ok({
kind: "opaque",
pythonMagic,
sha256,
limitation: `${entries.error.message}; the member was left opaque.`,
})
: entries;
return ok({
kind: "entries",
pythonMagic,
pythonVersion,
sha256,
entries: entries.value,
});
};
/**
* Marshal back-references let one stored row appear many times, so a small
* table can name far more bytes than it holds. Each genuine entry stores its
* own name inline, so the expanded names can never exceed the table itself.
*/
const pyzEntries = (
value: MarshalValue,
payloadEnd: number,
tableBytes: number,
): Result<readonly PyzEntry[], PyInstallerFailure> => {
if (!Array.isArray(value))
return err(
pyInstallerFailure("format", "PYZ table of contents is not a list"),
);
const entries: PyzEntry[] = [];
let nameBytes = 0;
for (const item of value) {
const entry = onePyzEntry(item, payloadEnd, entries.length);
if (!entry.ok) return entry;
nameBytes += Buffer.byteLength(entry.value.name, "utf8");
if (nameBytes > tableBytes)
return err(
pyInstallerFailure(
"limit",
`PYZ table entry names expand past the ${String(tableBytes)}-byte table that stores them`,
),
);
entries.push(entry.value);
}
return ok(entries);
};
const onePyzEntry = (
item: MarshalValue,
payloadEnd: number,
index: number,
): Result<PyzEntry, PyInstallerFailure> => {
if (!Array.isArray(item) || item.length !== 2)
return err(
pyInstallerFailure(
"format",
"PYZ table entry is not a name and position pair",
),
);
const [name, position] = item;
if (
typeof name !== "string" ||
!Array.isArray(position) ||
position.length !== 3
)
return err(
pyInstallerFailure(
"format",
"PYZ table entry is not a name, type, offset, and length",
),
);
const [typeCode, offset, length] = position;
if (
!Number.isSafeInteger(typeCode) ||
!Number.isSafeInteger(offset) ||
!Number.isSafeInteger(length)
)
return err(
pyInstallerFailure(
"format",
"PYZ table entry positions are not integers",
),
);
if (typeCode < 0 || typeCode > 3)
return err(
pyInstallerFailure(
"format",
`PYZ table entry type ${String(typeCode)} is not a known module, package, data, or namespace code`,
),
);
if (typeCode !== 3 && length === 0)
return err(
pyInstallerFailure(
"format",
`PYZ entry at table index ${String(index)} has no compressed bytes`,
),
);
if (
length < 0 ||
offset < 0 ||
offset > payloadEnd ||
length > payloadEnd - offset ||
(length > 0 && offset < PYZ_HEADER_BYTES)
)
return err(
pyInstallerFailure(
"integrity",
`PYZ entry at table index ${String(index)} extends outside the archive payload region`,
),
);
return ok({
index,
name,
typeCode,
offset,
length,
compressionFlag: length === 0 ? 0 : 1,
package: typeCode === 1 || typeCode === 3,
namespace: typeCode === 3,
});
};
+1
View File
@@ -90,6 +90,7 @@ const subjectFormatSchema = z.enum([
"unknown",
"javascript-bundle",
"entitlements",
"pyinstaller",
]);
// Snapshot entries retain the same raw-only architecture boundary as targets.
const subjectSchema = z.discriminatedUnion("format", [
@@ -51,6 +51,9 @@ export const artifactInventoryViewRequestSchema = z.discriminatedUnion(
collection: z.literal("occurrences"),
occurrence_ids: z.array(prefixedDigestSchema("occ")).optional(),
logical_paths: z.array(z.string()).optional(),
entry_kinds: z
.array(artifactOccurrenceSchema.shape.entry_kind)
.optional(),
fields: z.array(artifactOccurrenceSchema.keyof()).optional(),
}),
z.strictObject({
@@ -104,6 +107,7 @@ export const artifactInventorySummarySchema = z.strictObject({
directory: z.number().int().nonnegative(),
symlink: z.number().int().nonnegative(),
slice: z.number().int().nonnegative(),
metadata: z.number().int().nonnegative().exactOptional(),
}),
source_complete: z.boolean(),
inventory_evidence_ids: z.array(prefixedDigestSchema("ev")).min(1),
@@ -140,6 +144,7 @@ interface InventorySource {
index?: {
byId: ReadonlyMap<string, number>;
byPath: ReadonlyMap<string, readonly number[]>;
byKind: ReadonlyMap<ArtifactOccurrence["entry_kind"], readonly number[]>;
};
counts?: InventoryCounts;
}
@@ -241,7 +246,9 @@ export const projectArtifactInventoryView = (
?.format_metadata ?? null;
const rootMetadata = source.rootMetadata;
const includeCoverage =
view.kind === "inventory" || inventory.manifest.root_format === "godot-pck";
view.kind === "inventory" ||
inventory.manifest.root_format === "godot-pck" ||
inventory.manifest.root_format === "pyinstaller";
const sourceCoverage = includeCoverage
? { source_coverage: inventory.coverage }
: {};
@@ -309,7 +316,15 @@ export const projectArtifactInventoryView = (
edges: inventory.edges.length,
integrity_contradictions: inventory.integrityContradictions.length,
},
observed_entry_kind_counts: counts.kinds,
observed_entry_kind_counts: {
file: counts.kinds.file,
directory: counts.kinds.directory,
symlink: counts.kinds.symlink,
slice: counts.kinds.slice,
...(counts.kinds.metadata === 0
? {}
: { metadata: counts.kinds.metadata }),
},
...(includeCoverage
? { available_hash_status_counts: counts.hashes }
: {}),
@@ -406,7 +421,7 @@ const nodePage = (
const inventoryCounts = (source: InventorySource): InventoryCounts => {
if (source.counts !== undefined) return source.counts;
const kinds = { file: 0, directory: 0, symlink: 0, slice: 0 };
const kinds = { file: 0, directory: 0, symlink: 0, slice: 0, metadata: 0 };
const hashes = {
verified: 0,
mismatched: 0,
@@ -435,27 +450,43 @@ const occurrenceIndices = (
source: InventorySource,
view: OccurrenceView,
): number[] | undefined => {
if (view.occurrence_ids === undefined && view.logical_paths === undefined)
if (
view.occurrence_ids === undefined &&
view.logical_paths === undefined &&
view.entry_kinds === undefined
)
return undefined;
if (view.occurrence_ids?.length === 0 || view.logical_paths?.length === 0)
if (
view.occurrence_ids?.length === 0 ||
view.logical_paths?.length === 0 ||
view.entry_kinds?.length === 0
)
return [];
if (source.index === undefined) {
const byId = new Map<string, number>();
const byPath = new Map<string, number[]>();
const byKind = new Map<ArtifactOccurrence["entry_kind"], number[]>();
for (const [index, row] of source.inventory.occurrences.entries()) {
byId.set(row.occurrence_id, index);
const existing = byPath.get(row.logical_path);
if (existing === undefined) byPath.set(row.logical_path, [index]);
else existing.push(index);
const kind = byKind.get(row.entry_kind);
if (kind === undefined) byKind.set(row.entry_kind, [index]);
else kind.push(index);
}
source.index = { byId, byPath };
source.index = { byId, byPath, byKind };
}
const lookup = source.index;
const paths =
view.logical_paths === undefined ? undefined : new Set(view.logical_paths);
const selected =
view.occurrence_ids === undefined
? [...(paths ?? [])].flatMap((path) => lookup.byPath.get(path) ?? [])
? paths === undefined
? [...new Set(view.entry_kinds)].flatMap(
(kind) => lookup.byKind.get(kind) ?? [],
)
: [...paths].flatMap((path) => lookup.byPath.get(path) ?? [])
: [...new Set(view.occurrence_ids)].flatMap((id) => {
const index = lookup.byId.get(id);
return index === undefined ||
@@ -464,7 +495,15 @@ const occurrenceIndices = (
? []
: [index];
});
return selected.sort((left, right) => left - right);
const kinds =
view.entry_kinds === undefined ? undefined : new Set(view.entry_kinds);
return selected
.filter(
(index) =>
kinds === undefined ||
kinds.has(source.inventory.occurrences[index]!.entry_kind),
)
.sort((left, right) => left - right);
};
const nodeIndices = (
+61
View File
@@ -29,9 +29,70 @@ export const godotPckEntryMetadataSchema = z.strictObject({
flags: uint32,
md5_status: z.enum(["declared", "all-zero"]),
});
/** Source cookie fields and derived directory counts without Python execution. */
export const pyInstallerMetadataSchema = z.strictObject({
kind: z.literal("pyinstaller"),
layout_version: z.enum(["2.0", "2.1"]),
archive_start: offset,
archive_size: offset,
cookie_offset: offset,
toc_offset: offset,
toc_size: offset,
entry_count: offset,
runtime_option_count: offset,
external_dependency_count: offset,
python_version_raw: uint32,
python_major: uint32.nullable(),
python_minor: uint32.nullable(),
python_library: z.string().nullable(),
encryption_marker_present: z.boolean(),
});
export type PyInstallerMetadata = z.infer<typeof pyInstallerMetadataSchema>;
/** Primitive facts preserve source spelling and keep metadata copies independent. */
export const pyInstallerEntryMetadataSchema = z.strictObject({
kind: z.literal("pyinstaller-entry"),
container: z.enum(["carchive", "pyz"]),
container_name: z.string().nullable(),
index: offset,
source_name: z.string(),
type_code: z.union([z.string().length(1), uint32]),
relative_offset: offset,
directory_record_offset: offset.nullable(),
directory_record_size: offset.nullable(),
compression_flag: z.union([z.literal(0), z.literal(1)]),
bytecode_header: z.enum(["not-bytecode", "unknown", "derived"]),
bytecode_magic: z
.string()
.regex(/^[0-9a-f]{8}$/u)
.nullable(),
pyz_status: z.enum(["not-pyz", "decoded", "opaque"]),
pyz_sha256: z
.string()
.regex(/^[0-9a-f]{64}$/u)
.nullable(),
pyz_python_magic: z
.string()
.regex(/^[0-9a-f]{8}$/u)
.nullable(),
pyz_python_version: z.string().nullable(),
pyz_entry_count: offset.nullable(),
encryption_source: z.literal("carchive-marker").nullable(),
dependency_status: z
.enum(["unresolved", "invalid-reference"])
.exactOptional(),
dependency_archive: z.string().nullable().exactOptional(),
dependency_member: z.string().nullable().exactOptional(),
});
export type PyInstallerEntryMetadata = z.infer<
typeof pyInstallerEntryMetadataSchema
>;
export const artifactFormatMetadataSchema = z.discriminatedUnion("kind", [
godotPckMetadataSchema,
godotPckEntryMetadataSchema,
pyInstallerMetadataSchema,
pyInstallerEntryMetadataSchema,
]);
export type ArtifactFormatMetadata = z.infer<
typeof artifactFormatMetadataSchema
+2 -1
View File
@@ -83,6 +83,7 @@ const artifactFormatSchema = z.enum([
"entitlements",
"directory",
"file",
"pyinstaller",
"unknown",
]);
@@ -138,7 +139,7 @@ export const artifactOccurrenceSchema = z.object({
artifact_id: artifactIdSchema.nullable(),
parent_occurrence_id: occurrenceIdSchema.nullable(),
logical_path: boundedRelativePathSchema,
entry_kind: z.enum(["file", "directory", "symlink", "slice"]),
entry_kind: z.enum(["file", "directory", "symlink", "slice", "metadata"]),
artifact_kind: artifactKindSchema,
artifact_format: artifactFormatSchema,
declared_size: z
+13 -12
View File
@@ -348,19 +348,20 @@ const orderOccurrences = (
.flatMap(([path, group]) => {
if (group.length < 2) return group;
const ordinalById = new Map<string, number>();
const first = group[0];
if (first === undefined)
throw new TypeError("Duplicate occurrence group is empty");
const kinds = new Set(group.map(({ entry_kind }) => entry_kind));
for (let ordinal = 0; ordinal < group.length; ordinal++) {
ordinalById.set(
occurrenceIdForLocation({
rootArtifactId,
logicalPath: path,
entryKind: first.entry_kind,
...(ordinal === 0 ? {} : { duplicateOrdinal: ordinal }),
}),
ordinal,
);
// Entry kinds come from a fixed schema enum, so work stays linear in
// the group size without changing or guessing producer identities.
for (const entryKind of kinds)
ordinalById.set(
occurrenceIdForLocation({
rootArtifactId,
logicalPath: path,
entryKind,
...(ordinal === 0 ? {} : { duplicateOrdinal: ordinal }),
}),
ordinal,
);
}
return group.sort(
(left, right) =>
+1
View File
@@ -70,6 +70,7 @@ const subjectFormatSchema = z.enum([
"mach-o-universal",
"javascript-bundle",
"entitlements",
"pyinstaller",
]);
const subjectBaseSchema = z.strictObject({
name: z.string().min(1),