feat(artifacts): inspect and extract PyInstaller archives
This commit is contained in:
1 parent
22d5a40a2e
commit
afa5da745e
63 files changed
+7254
-114
No files matched your search
@@ -4,13 +4,44 @@ import {
|
||||
ArtifactPathRegistry,
|
||||
annotateEnglishUnicodeCaseCollisions,
|
||||
ENGLISH_UNICODE_CASE_INVENTORY_LIMITATION,
|
||||
normalizeArtifactPath,
|
||||
} from "./ArtifactPaths.js";
|
||||
|
||||
const NOTE =
|
||||
"a case-insensitive destination cannot store both. This comparison is not a filesystem observation.";
|
||||
|
||||
it.each(["drive", "escaped", "normalization", "prefix", "collision"])(
|
||||
"bounds path failure diagnostics at the owning boundary: %s",
|
||||
(kind) => {
|
||||
const name = "x".repeat(1024 * 1024);
|
||||
const registry = new ArtifactPathRegistry();
|
||||
const fail = () => {
|
||||
if (kind === "drive") normalizeArtifactPath("C:" + name);
|
||||
else if (kind === "escaped")
|
||||
normalizeArtifactPath("\0".repeat(name.length));
|
||||
else if (kind === "normalization")
|
||||
normalizeArtifactPath(name + "/../child");
|
||||
else {
|
||||
registry.add(name, "file");
|
||||
registry.add(kind === "prefix" ? name + "/child" : name, "directory");
|
||||
}
|
||||
};
|
||||
const failure = (() => {
|
||||
try {
|
||||
fail();
|
||||
} catch (cause: unknown) {
|
||||
return cause;
|
||||
}
|
||||
return undefined;
|
||||
})();
|
||||
expect(failure).toMatchObject({ reason: "path" });
|
||||
expect((failure as Error).message.length).toBeLessThan(1024);
|
||||
},
|
||||
);
|
||||
|
||||
const annotate = (paths: readonly string[]) => {
|
||||
const occurrences = paths.map((logical_path) => ({
|
||||
entry_kind: "file" as const,
|
||||
logical_path,
|
||||
limitations: [] as string[],
|
||||
}));
|
||||
|
||||
@@ -2,10 +2,25 @@ import { posix } from "node:path";
|
||||
|
||||
import { logicalPathNfc } from "../domain/artifactIdentity.js";
|
||||
import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
import { ArtifactReaderFailure, type ArtifactEntry } from "./ArtifactReader.js";
|
||||
|
||||
const DRIVE_PREFIX = /^[A-Za-z]:/u;
|
||||
|
||||
// Bound diagnostic JSON expansion to 768 characters before formatting. This is
|
||||
// a presentation budget, never a limit on accepted paths or retained facts.
|
||||
const PATH_DIAGNOSTIC_CODE_UNITS = 128;
|
||||
const pathDiagnostic = (path: string): string =>
|
||||
path.length <= PATH_DIAGNOSTIC_CODE_UNITS
|
||||
? JSON.stringify(path)
|
||||
: `${JSON.stringify(path.slice(0, PATH_DIAGNOSTIC_CODE_UNITS))}… (${String(path.length)} UTF-16 code units)`;
|
||||
|
||||
type DestinationEntryKind = Exclude<ArtifactEntry["kind"], "metadata">;
|
||||
|
||||
/** Declarations have query identities, but never claim a destination path. */
|
||||
export const hasDestinationPath = (
|
||||
kind: ArtifactEntry["kind"],
|
||||
): kind is DestinationEntryKind => kind !== "metadata";
|
||||
|
||||
/**
|
||||
* English Unicode case folding used only to describe logical names a
|
||||
* case-insensitive destination might collapse. It is not a filesystem case table.
|
||||
@@ -25,7 +40,7 @@ export const normalizeArtifactPath = (input: string): string => {
|
||||
if (input.includes("\0") || input.startsWith("/") || DRIVE_PREFIX.test(input))
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Artifact path is absolute or unsafe: ${JSON.stringify(input)}`,
|
||||
`Artifact path is absolute or unsafe: ${pathDiagnostic(input)}`,
|
||||
);
|
||||
const normalized = logicalPathNfc(input).replace(/\/+$/u, "");
|
||||
const parts = normalized.split("/");
|
||||
@@ -36,7 +51,7 @@ export const normalizeArtifactPath = (input: string): string => {
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Artifact path is not normalized: ${JSON.stringify(input)}`,
|
||||
`Artifact path is not normalized: ${pathDiagnostic(input)}`,
|
||||
);
|
||||
return normalized;
|
||||
};
|
||||
@@ -45,14 +60,15 @@ export const normalizeArtifactPath = (input: string): string => {
|
||||
export class ArtifactPathRegistry {
|
||||
readonly #root: PathTrieNode = { kind: undefined, children: new Map() };
|
||||
|
||||
add(path: string, kind: "file" | "directory" | "symlink" | "slice"): void {
|
||||
add(path: string, kind: ArtifactEntry["kind"]): void {
|
||||
if (!hasDestinationPath(kind)) return;
|
||||
const parts = path.split("/");
|
||||
let node = this.#root;
|
||||
for (const [index, part] of parts.entries()) {
|
||||
if (node.kind !== undefined && node.kind !== "directory")
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Artifact prefix conflict: ${path}`,
|
||||
`Artifact prefix conflict: ${pathDiagnostic(path)}`,
|
||||
);
|
||||
let child = node.children.get(part);
|
||||
if (child === undefined) {
|
||||
@@ -64,12 +80,12 @@ export class ArtifactPathRegistry {
|
||||
if (node.kind !== undefined && node.kind !== kind)
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Artifact path collision: ${path}`,
|
||||
`Artifact path collision: ${pathDiagnostic(path)}`,
|
||||
);
|
||||
if (kind !== "directory" && node.children.size > 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Artifact prefix conflict: ${path}`,
|
||||
`Artifact prefix conflict: ${pathDiagnostic(path)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -78,7 +94,7 @@ export class ArtifactPathRegistry {
|
||||
}
|
||||
|
||||
interface PathTrieNode {
|
||||
kind: "file" | "directory" | "symlink" | "slice" | undefined;
|
||||
kind: DestinationEntryKind | undefined;
|
||||
readonly children: Map<string, PathTrieNode>;
|
||||
}
|
||||
|
||||
@@ -103,15 +119,19 @@ const englishCaseFold = (part: string): string =>
|
||||
*/
|
||||
export const annotateEnglishUnicodeCaseCollisions = (
|
||||
occurrences: readonly {
|
||||
readonly entry_kind: ArtifactEntry["kind"];
|
||||
readonly logical_path: string;
|
||||
limitations: string[];
|
||||
}[],
|
||||
): string | undefined => {
|
||||
const destinations = occurrences.filter(({ entry_kind }) =>
|
||||
hasDestinationPath(entry_kind),
|
||||
);
|
||||
const collisions = englishUnicodeCaseCollisions(
|
||||
occurrences.map(({ logical_path }) => logical_path),
|
||||
destinations.map(({ logical_path }) => logical_path),
|
||||
);
|
||||
if (collisions.size === 0) return undefined;
|
||||
for (const occurrence of occurrences)
|
||||
for (const occurrence of destinations)
|
||||
for (const collision of collisions.get(occurrence.logical_path) ?? [])
|
||||
occurrence.limitations.push(
|
||||
caseCollisionLimitation(occurrence.logical_path, collision),
|
||||
|
||||
@@ -33,7 +33,7 @@ export interface ArtifactReaderFailureOptions extends ErrorOptions {
|
||||
/** Archive-neutral entry metadata. Reader adapters never choose output paths. */
|
||||
export interface ArtifactEntry {
|
||||
readonly path: string;
|
||||
readonly kind: "file" | "directory" | "symlink" | "slice";
|
||||
readonly kind: "file" | "directory" | "symlink" | "slice" | "metadata";
|
||||
readonly declaredSize: number | null;
|
||||
readonly compressedSize: number | null;
|
||||
readonly executable: boolean;
|
||||
@@ -167,6 +167,7 @@ export interface ArtifactReader {
|
||||
| ZipPackageFormat
|
||||
| "asar"
|
||||
| "godot-pck"
|
||||
| "pyinstaller"
|
||||
| "file";
|
||||
entries(
|
||||
signal?: AbortSignal,
|
||||
|
||||
@@ -0,0 +1,608 @@
|
||||
import type { Readable } from "node:stream";
|
||||
import { finished } from "node:stream/promises";
|
||||
import { setImmediate } from "node:timers/promises";
|
||||
import { jsonValueByteLength } from "../domain/jsonValueByteLength.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { normalizeArtifactPath } from "./ArtifactPaths.js";
|
||||
import { lstat, type FileHandle } from "node:fs/promises";
|
||||
|
||||
import type { PyInstallerMetadata } from "../domain/artifactFormatMetadata.js";
|
||||
import {
|
||||
cookieMetadata,
|
||||
scriptPythonMagic,
|
||||
} from "./pyinstaller/PyInstallerMetadata.js";
|
||||
import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js";
|
||||
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
import {
|
||||
RegularFileAdmissionFailure,
|
||||
openRegularFile,
|
||||
sameRegularFileState,
|
||||
type StableRegularFileDescriptor,
|
||||
} from "../filesystem/RegularFile.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
sameArtifactEntry,
|
||||
type ArtifactEntry,
|
||||
type ArtifactReader,
|
||||
type ArtifactReaderTraversal,
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { abortIfNeeded, hashReadable } from "./ArtifactHash.js";
|
||||
import { readArtifactFileChunks } from "./readArtifactFileChunks.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import { artifactIntegrityDetails } from "../domain/artifactIntegrity.js";
|
||||
import { PyInstallerPyzStore } from "./pyinstaller/PyInstallerPyzStore.js";
|
||||
import { pyInstallerMemberStream } from "./pyinstaller/PyInstallerMemberStream.js";
|
||||
import {
|
||||
findPyInstallerCookie,
|
||||
readCArchiveToc,
|
||||
type CArchiveEntry,
|
||||
type PyInstallerCookie,
|
||||
} from "./pyinstaller/CArchiveLayout.js";
|
||||
import {
|
||||
hasEncryptionMarker,
|
||||
projectPyzContainer,
|
||||
projectPyzMember,
|
||||
projectStoredMember,
|
||||
sealMember,
|
||||
type PyInstallerMember,
|
||||
type PyzImage,
|
||||
} from "./pyinstaller/CArchiveMembers.js";
|
||||
import { readPyzArchive } from "./pyinstaller/PyzArchive.js";
|
||||
import type { PyInstallerFailure } from "./pyinstaller/PyInstallerFailure.js";
|
||||
import { pyInstallerFileAdmissionFailure } from "./pyinstaller/PyInstallerFileAdmission.js";
|
||||
|
||||
const MAX_PARSED_FACT_BYTES = 32 * 1024 * 1024;
|
||||
const MAX_RETAINED_ENTRY_BYTES = 64 * 1024 * 1024;
|
||||
const RECORD_OWNERSHIP_RESERVATION = 256;
|
||||
// Conservative allowance for registry/case trie nodes, maps and traversal state.
|
||||
// Repeated prefixes are charged again rather than underestimating retained work.
|
||||
const PATH_COMPONENT_RESERVATION = 1024;
|
||||
|
||||
interface LoadedArchive {
|
||||
readonly members: readonly PyInstallerMember[];
|
||||
readonly traversal: ArtifactReaderTraversal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read-only inventory of a PyInstaller CArchive appended to a native executable.
|
||||
* Encrypted PYZ entries are reported and not decoded. Code members are not decompiled.
|
||||
*/
|
||||
export class PyInstallerArtifactReader implements ArtifactReader {
|
||||
readonly format = "pyinstaller" as const;
|
||||
#source: StableRegularFileDescriptor | undefined;
|
||||
#opening: Promise<FileHandle> | undefined;
|
||||
#closed = false;
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
readonly #cleanup = new ArtifactResourceScope();
|
||||
readonly #lifecycle = new AbortController();
|
||||
#boundSha256: string | undefined;
|
||||
#loaded: Promise<LoadedArchive> | undefined;
|
||||
#budget = 0;
|
||||
#charged = 0;
|
||||
#parsedFactBytes = 0;
|
||||
#retainedEntryBytes = 0;
|
||||
#metadata: PyInstallerMetadata | undefined;
|
||||
readonly #pyzStore = new PyInstallerPyzStore();
|
||||
readonly #members = new Map<string, PyInstallerMember>();
|
||||
readonly #openedPyzSizes = new Map<string, number>();
|
||||
|
||||
constructor(
|
||||
private readonly path: string,
|
||||
source?: StableRegularFileDescriptor,
|
||||
) {
|
||||
this.#source = source;
|
||||
this.#cleanup.retain({
|
||||
kind: "lease",
|
||||
resource: "PyInstaller private PYZ images",
|
||||
lease: { release: () => this.#pyzStore.close() },
|
||||
});
|
||||
}
|
||||
|
||||
formatMetadata(): PyInstallerMetadata | undefined {
|
||||
return this.#metadata === undefined ? undefined : { ...this.#metadata };
|
||||
}
|
||||
|
||||
async prepareContainer(
|
||||
expectedSha256: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
const active = this.#signal(signal);
|
||||
await this.#scope.run(async () => {
|
||||
const handle = await this.#openHandle(active);
|
||||
await this.#verifySource();
|
||||
const observed =
|
||||
this.#boundSha256 ??
|
||||
(
|
||||
await hashReadable(
|
||||
readArtifactFileChunks(handle, { signal: active }),
|
||||
active,
|
||||
)
|
||||
).sha256;
|
||||
await this.#verifySource();
|
||||
abortIfNeeded(active);
|
||||
if (observed !== expectedSha256)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"PyInstaller carrier changed after inventory",
|
||||
undefined,
|
||||
artifactIntegrityDetails(
|
||||
".",
|
||||
{ algorithm: "sha256", digest: expectedSha256 },
|
||||
observed,
|
||||
false,
|
||||
),
|
||||
);
|
||||
this.#boundSha256 = observed;
|
||||
});
|
||||
}
|
||||
|
||||
async *entries(
|
||||
signal?: AbortSignal,
|
||||
options?: ArtifactTraversalOptions,
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
const active = this.#signal(signal);
|
||||
const loaded = await this.#scope.run(async () => {
|
||||
const value = await this.#load(active);
|
||||
await this.#verifySource();
|
||||
this.#members.clear();
|
||||
return value;
|
||||
});
|
||||
options?.onCoverage?.(loaded.traversal);
|
||||
for (const member of loaded.members) {
|
||||
abortIfNeeded(active);
|
||||
const sealed = sealMember(member);
|
||||
this.#members.set(sealed.entry.adapterKey, sealed);
|
||||
yield sealed.entry;
|
||||
}
|
||||
}
|
||||
|
||||
async open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
|
||||
const active = this.#signal(signal);
|
||||
return this.#scope.run(async () => {
|
||||
const stored = this.#members.get(entry.adapterKey);
|
||||
if (stored === undefined || !sameArtifactEntry(entry, stored.entry))
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`PyInstaller entry was not produced by this reader: ${entry.path}`,
|
||||
);
|
||||
if (entry.kind !== "file")
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`PyInstaller entry is not a regular file: ${entry.path}`,
|
||||
);
|
||||
if (entry.encrypted)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
`PYZ entry is encrypted and was not decoded: ${entry.path}`,
|
||||
);
|
||||
const read = stored.read;
|
||||
if (read === undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`PyInstaller entry has no readable bytes: ${entry.path}`,
|
||||
);
|
||||
await this.#verifySource();
|
||||
const knownSize =
|
||||
read.kind === "pyz"
|
||||
? (read.uncompressedSize ??
|
||||
this.#openedPyzSizes.get(entry.adapterKey))
|
||||
: undefined;
|
||||
if (read.kind === "pyz" && knownSize === undefined)
|
||||
this.#charge(read.header?.length ?? 0);
|
||||
const parts = await this.#pyzStore.memberSource(
|
||||
read,
|
||||
this.#source?.handle,
|
||||
active,
|
||||
);
|
||||
if (active.aborted) {
|
||||
parts.source.destroy();
|
||||
await finished(parts.source, { cleanup: true }).catch(() => undefined);
|
||||
abortIfNeeded(active);
|
||||
}
|
||||
// Keep one close attempt in this read. Failed owners stay in the store
|
||||
// for reader.close() to retry, without losing the original failure.
|
||||
let closeAttempt: Promise<void> | undefined;
|
||||
const closeSource = (primary?: unknown) =>
|
||||
(closeAttempt ??= parts.close?.(primary));
|
||||
const stream = pyInstallerMemberStream(parts.source, {
|
||||
sourceBytes:
|
||||
read.kind === "carchive" ? read.source.compressedSize : read.length,
|
||||
compressed:
|
||||
read.kind === "pyz"
|
||||
? read.compressed
|
||||
: read.source.compressionFlag === 1,
|
||||
header: read.header,
|
||||
expectedBytes:
|
||||
read.kind === "carchive" ? read.source.uncompressedSize : knownSize,
|
||||
maxBytes:
|
||||
read.kind === "carchive"
|
||||
? read.source.uncompressedSize
|
||||
: (knownSize ?? this.#budget - this.#charged),
|
||||
signal: active,
|
||||
...(read.kind === "pyz" && knownSize === undefined
|
||||
? { charge: (bytes: number) => this.#charge(bytes) }
|
||||
: {}),
|
||||
complete: async (bytes) => {
|
||||
try {
|
||||
await parts.verify?.();
|
||||
await this.#verifySource();
|
||||
abortIfNeeded(active);
|
||||
} catch (cause: unknown) {
|
||||
await closeSource(cause);
|
||||
throw cause;
|
||||
}
|
||||
await closeSource();
|
||||
abortIfNeeded(active);
|
||||
if (read.kind === "pyz" && knownSize === undefined)
|
||||
this.#openedPyzSizes.set(entry.adapterKey, bytes);
|
||||
},
|
||||
});
|
||||
void this.#scope
|
||||
.run(async () => {
|
||||
let primary: unknown;
|
||||
try {
|
||||
await stream.completion;
|
||||
} catch (cause: unknown) {
|
||||
primary = cause;
|
||||
}
|
||||
await closeSource(primary);
|
||||
if (primary !== undefined) throw primary;
|
||||
})
|
||||
.catch((cause: unknown) =>
|
||||
stream.output.destroy(
|
||||
cause instanceof Error
|
||||
? cause
|
||||
: new Error("PyInstaller stream failed", { cause }),
|
||||
),
|
||||
);
|
||||
return stream.output;
|
||||
});
|
||||
}
|
||||
|
||||
provenance(): readonly [] {
|
||||
return [];
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.#closed = true;
|
||||
const closing = new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"PyInstaller reader closed",
|
||||
);
|
||||
this.#lifecycle.abort(closing);
|
||||
await this.#scope.close();
|
||||
await this.#cleanup.close();
|
||||
this.#source = undefined;
|
||||
this.#closed = true;
|
||||
this.#loaded = undefined;
|
||||
this.#members.clear();
|
||||
this.#openedPyzSizes.clear();
|
||||
}
|
||||
|
||||
#signal(signal?: AbortSignal): AbortSignal {
|
||||
abortIfNeeded(signal);
|
||||
if (this.#closed)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"PyInstaller archive is closed",
|
||||
);
|
||||
return signal === undefined
|
||||
? this.#lifecycle.signal
|
||||
: AbortSignal.any([signal, this.#lifecycle.signal]);
|
||||
}
|
||||
|
||||
async #load(signal?: AbortSignal): Promise<LoadedArchive> {
|
||||
this.#loaded ??= this.#parse(signal);
|
||||
return this.#loaded;
|
||||
}
|
||||
|
||||
async #parse(signal?: AbortSignal): Promise<LoadedArchive> {
|
||||
const handle = await this.#openHandle(signal);
|
||||
await this.#verifySource();
|
||||
const size = (await handle.stat()).size;
|
||||
const cookie = await this.#cookie(handle, size, signal);
|
||||
this.#budget = artifactExpansionBudgetBytes(cookie.packageLength);
|
||||
const toc = await this.#toc(handle, cookie, signal);
|
||||
for (const entry of toc) {
|
||||
this.#charge(entry.uncompressedSize);
|
||||
this.#reserveMetadata({ ...entry }, false);
|
||||
}
|
||||
const images = await this.#pyzImages(handle, cookie, toc, signal);
|
||||
const magic = scriptPythonMagic(
|
||||
cookie,
|
||||
[...images.values()].map((image) => image.archive),
|
||||
);
|
||||
const encrypted = hasEncryptionMarker(toc);
|
||||
const members = await this.#project(
|
||||
toc,
|
||||
cookie,
|
||||
images,
|
||||
magic,
|
||||
encrypted,
|
||||
signal,
|
||||
);
|
||||
for (const member of members)
|
||||
if (member.read?.kind === "carchive")
|
||||
this.#charge(member.read.header?.length ?? 0);
|
||||
await this.#verifySource();
|
||||
this.#metadata = cookieMetadata(cookie, toc, encrypted);
|
||||
let opaqueImages = 0;
|
||||
for (const image of images.values())
|
||||
if (image.archive.kind === "opaque") opaqueImages++;
|
||||
return {
|
||||
members,
|
||||
traversal: {
|
||||
status: opaqueImages === 0 ? "complete" : "unknown",
|
||||
limitations:
|
||||
opaqueImages === 0
|
||||
? []
|
||||
: [
|
||||
`${String(opaqueImages)} PYZ archives remain opaque; module traversal is unknown. See opaque PYZ occurrences for source magic and per-archive limitations.`,
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async #openHandle(signal?: AbortSignal): Promise<FileHandle> {
|
||||
abortIfNeeded(signal);
|
||||
if (this.#closed)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"PyInstaller archive is closed",
|
||||
);
|
||||
if (this.#source !== undefined) return this.#source.handle;
|
||||
// Admission belongs to the reader. A departing caller cannot cancel a peer
|
||||
// or replace a late descriptor owner; every caller joins the same admission.
|
||||
this.#opening ??= this.#admit();
|
||||
let handle: FileHandle;
|
||||
try {
|
||||
handle = await this.#opening;
|
||||
} catch (cause: unknown) {
|
||||
if (
|
||||
cause instanceof ArtifactReaderFailure &&
|
||||
cause.reason === "cancelled" &&
|
||||
signal?.aborted
|
||||
)
|
||||
throw new ArtifactReaderFailure("cancelled", cause.message, {
|
||||
cause: signal.reason,
|
||||
cleanup: cause.cleanup,
|
||||
cleanupLeases: cause.cleanupLeases,
|
||||
});
|
||||
throw cause;
|
||||
}
|
||||
abortIfNeeded(signal);
|
||||
return handle;
|
||||
}
|
||||
|
||||
async #admit(): Promise<FileHandle> {
|
||||
try {
|
||||
const handle = await openRegularFile(this.path, {
|
||||
symlinks: "reject",
|
||||
signal: this.#lifecycle.signal,
|
||||
});
|
||||
this.#retainCarrier(new OwnedFileHandle(handle));
|
||||
this.#source = { handle, initial: await handle.stat() };
|
||||
return handle;
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof RegularFileAdmissionFailure)
|
||||
this.#retainCarrier(cause.owner);
|
||||
throw pyInstallerFileAdmissionFailure(this.path, cause);
|
||||
}
|
||||
}
|
||||
|
||||
#retainCarrier(handle: OwnedFileHandle): void {
|
||||
this.#cleanup.retain({ kind: "file-handle", resource: this.path, handle });
|
||||
}
|
||||
|
||||
async #cookie(
|
||||
handle: FileHandle,
|
||||
size: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<PyInstallerCookie> {
|
||||
const found = await findPyInstallerCookie(handle, size, signal);
|
||||
if (found === undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"PyInstaller CArchive cookie was not found",
|
||||
);
|
||||
if (!found.ok) throw failure(found.error);
|
||||
return found.value;
|
||||
}
|
||||
|
||||
async #toc(
|
||||
handle: FileHandle,
|
||||
cookie: PyInstallerCookie,
|
||||
signal?: AbortSignal,
|
||||
): Promise<readonly CArchiveEntry[]> {
|
||||
const toc = await readCArchiveToc(handle, cookie, signal);
|
||||
if (!toc.ok) throw failure(toc.error);
|
||||
return toc.value;
|
||||
}
|
||||
|
||||
async #pyzImages(
|
||||
handle: FileHandle,
|
||||
cookie: PyInstallerCookie,
|
||||
toc: readonly CArchiveEntry[],
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<ReadonlyMap<CArchiveEntry, PyzImage>> {
|
||||
const images = new Map<CArchiveEntry, PyzImage>();
|
||||
// Admit the complete filesystem footprint before the first private write.
|
||||
this.#pyzStore.reserveImages(
|
||||
toc.reduce(
|
||||
(count, entry) =>
|
||||
count +
|
||||
(entry.typeCode === "z" && entry.compressionFlag === 1 ? 1 : 0),
|
||||
0,
|
||||
),
|
||||
);
|
||||
for (const entry of toc) {
|
||||
if (entry.typeCode !== "z") continue;
|
||||
abortIfNeeded(signal);
|
||||
const spooled =
|
||||
entry.compressionFlag === 0
|
||||
? undefined
|
||||
: await this.#pyzStore.spool(
|
||||
handle,
|
||||
cookie.archiveStart,
|
||||
entry,
|
||||
signal,
|
||||
);
|
||||
const source = spooled ?? {
|
||||
handle,
|
||||
start: cookie.archiveStart + entry.offset,
|
||||
};
|
||||
const parsed = await readPyzArchive(
|
||||
source.handle,
|
||||
source.start,
|
||||
entry.uncompressedSize,
|
||||
signal,
|
||||
);
|
||||
if (!parsed.ok) throw failure(parsed.error);
|
||||
if (parsed.value.kind === "entries")
|
||||
for (const item of parsed.value.entries)
|
||||
this.#reserveMetadata({ ...item, container: entry.name }, false);
|
||||
if (spooled !== undefined) await this.#pyzStore.release(spooled.handle);
|
||||
images.set(entry, {
|
||||
archive: parsed.value,
|
||||
handle: source.handle,
|
||||
start: source.start,
|
||||
});
|
||||
}
|
||||
return images;
|
||||
}
|
||||
|
||||
async #project(
|
||||
toc: readonly CArchiveEntry[],
|
||||
cookie: PyInstallerCookie,
|
||||
images: ReadonlyMap<CArchiveEntry, PyzImage>,
|
||||
magic: Buffer | undefined,
|
||||
encrypted: boolean,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<PyInstallerMember[]> {
|
||||
const members: PyInstallerMember[] = [];
|
||||
const append = (member: PyInstallerMember): void => {
|
||||
this.#reservePathComponents(member.entry.path);
|
||||
const entry = {
|
||||
...member.entry,
|
||||
path: normalizeArtifactPath(member.entry.path),
|
||||
};
|
||||
this.#reserveMetadata(
|
||||
{
|
||||
path: entry.path,
|
||||
adapter_key: entry.adapterKey,
|
||||
format_metadata: entry.formatMetadata ?? null,
|
||||
limitations: [...entry.limitations],
|
||||
},
|
||||
true,
|
||||
);
|
||||
members.push({ ...member, entry });
|
||||
};
|
||||
for (const entry of toc) {
|
||||
if (members.length % 256 === 0) await setImmediate();
|
||||
abortIfNeeded(signal);
|
||||
const image = entry.typeCode === "z" ? images.get(entry) : undefined;
|
||||
if (image === undefined) {
|
||||
append(projectStoredMember(entry, cookie.archiveStart, magic));
|
||||
continue;
|
||||
}
|
||||
append(projectPyzContainer(entry, cookie.archiveStart, image, encrypted));
|
||||
if (image.archive.kind !== "entries") continue;
|
||||
for (const item of image.archive.entries) {
|
||||
if (members.length % 256 === 0) await setImmediate();
|
||||
abortIfNeeded(signal);
|
||||
append(
|
||||
projectPyzMember(
|
||||
entry,
|
||||
item,
|
||||
encrypted,
|
||||
image.archive.pythonMagic,
|
||||
item.namespace || encrypted
|
||||
? undefined
|
||||
: {
|
||||
handle: image.handle,
|
||||
offset: image.start + item.offset,
|
||||
length: item.length,
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
return members;
|
||||
}
|
||||
|
||||
async #verifySource(): Promise<void> {
|
||||
const source = this.#source;
|
||||
if (source === undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"PyInstaller archive is closed",
|
||||
);
|
||||
const [opened, currentPath] = await Promise.all([
|
||||
source.handle.stat(),
|
||||
lstat(this.path),
|
||||
]);
|
||||
if (
|
||||
!sameRegularFileState(source.initial, opened) ||
|
||||
!sameRegularFileState(source.initial, currentPath)
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`PyInstaller source changed during reading: ${this.path}`,
|
||||
);
|
||||
}
|
||||
|
||||
#reservePathComponents(path: string): void {
|
||||
const remaining = MAX_RETAINED_ENTRY_BYTES - this.#retainedEntryBytes;
|
||||
// Case annotations can carry the path and two prefixes at every component.
|
||||
// Reserve escaped text growth as well as registry/case trie nodes and maps.
|
||||
const componentCost =
|
||||
PATH_COMPONENT_RESERVATION + 6 * jsonValueByteLength(path);
|
||||
let reservation = componentCost;
|
||||
for (
|
||||
let index = 0;
|
||||
index < path.length && reservation <= remaining;
|
||||
index++
|
||||
)
|
||||
if (path.charCodeAt(index) === 47) reservation += componentCost;
|
||||
if (reservation > remaining)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
"PyInstaller paths exceed their retained namespace budget",
|
||||
{ boundary: "memory" },
|
||||
);
|
||||
this.#retainedEntryBytes += reservation;
|
||||
}
|
||||
|
||||
#reserveMetadata(value: JsonValue, projected: boolean): void {
|
||||
const used = projected ? this.#retainedEntryBytes : this.#parsedFactBytes;
|
||||
const limit = projected ? MAX_RETAINED_ENTRY_BYTES : MAX_PARSED_FACT_BYTES;
|
||||
const remaining = limit - used - RECORD_OWNERSHIP_RESERVATION;
|
||||
const bytes = remaining < 0 ? 0 : jsonValueByteLength(value, remaining);
|
||||
if (remaining < 0 || bytes > remaining)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
"PyInstaller directory facts exceed their retained metadata budget",
|
||||
{ boundary: "memory" },
|
||||
);
|
||||
if (projected)
|
||||
this.#retainedEntryBytes += bytes + RECORD_OWNERSHIP_RESERVATION;
|
||||
else this.#parsedFactBytes += bytes + RECORD_OWNERSHIP_RESERVATION;
|
||||
}
|
||||
|
||||
#charge(bytes: number): void {
|
||||
if (
|
||||
!Number.isSafeInteger(bytes) ||
|
||||
bytes < 0 ||
|
||||
this.#charged + bytes > this.#budget
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
`PyInstaller members declare ${String(this.#charged + Math.max(0, bytes))} uncompressed bytes, exceeding the decompression budget`,
|
||||
);
|
||||
this.#charged += bytes;
|
||||
}
|
||||
}
|
||||
|
||||
const failure = (error: PyInstallerFailure): ArtifactReaderFailure =>
|
||||
new ArtifactReaderFailure(error.reason, error.message);
|
||||
@@ -0,0 +1,54 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { expect, it, vi } from "vitest";
|
||||
|
||||
import { ArtifactReaderFailure } from "../ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "../ArtifactResourceScope.js";
|
||||
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import { extractArtifact } from "./ArtifactExtraction.js";
|
||||
|
||||
vi.mock("../pyinstaller/CArchiveLayout.js", async (importOriginal) => ({
|
||||
...(await importOriginal<object>()),
|
||||
containsPyInstallerArchive: () =>
|
||||
Promise.reject(new ArtifactReaderFailure("format", "detection failed")),
|
||||
}));
|
||||
|
||||
it("keeps the detection error primary and reports a failed detection close", async () => {
|
||||
const directory = await createTestTempDirectory("rea-extraction-detect-");
|
||||
const path = join(directory, "target.elf");
|
||||
await writeFile(path, Buffer.from("not an archive"));
|
||||
const resourceScope = new ArtifactResourceScope();
|
||||
const originalClose = OwnedFileHandle.prototype.close;
|
||||
const closeFailure = new Error("detection close failed");
|
||||
let attempts = 0;
|
||||
const close = vi
|
||||
.spyOn(OwnedFileHandle.prototype, "close")
|
||||
.mockImplementation(async function (this: OwnedFileHandle) {
|
||||
attempts += 1;
|
||||
if (attempts === 1) throw closeFailure;
|
||||
await originalClose.call(this);
|
||||
});
|
||||
try {
|
||||
await expect(
|
||||
extractArtifact({
|
||||
inputPath: path,
|
||||
inputFormat: "elf",
|
||||
outputRoot: join(directory, "out"),
|
||||
environment: process.env,
|
||||
integrityPolicy: "fail",
|
||||
resourceScope,
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: "detection failed",
|
||||
cleanup: { reason: closeFailure.message },
|
||||
});
|
||||
await resourceScope.close();
|
||||
expect(attempts).toBe(2);
|
||||
} finally {
|
||||
close.mockRestore();
|
||||
await resourceScope.close().catch(() => undefined);
|
||||
}
|
||||
});
|
||||
@@ -23,6 +23,8 @@ import { SafeOutputTreeCreationFailure } from "../SafeOutputTreeCreationFailure.
|
||||
import { ZipArtifactReader } from "../ZipArtifactReader.js";
|
||||
import { isZipPackageFormat } from "../../domain/zipPackageFormat.js";
|
||||
import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js";
|
||||
import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js";
|
||||
import { containsPyInstallerArchive } from "../pyinstaller/CArchiveLayout.js";
|
||||
import {
|
||||
artifactExtractionResultSchema,
|
||||
type ArtifactExtractionResult,
|
||||
@@ -244,6 +246,7 @@ const materializeSelection = async ({
|
||||
sourcePath,
|
||||
input,
|
||||
inventory.manifest.root_format,
|
||||
signal,
|
||||
);
|
||||
const readerOwner: ArtifactResourceOwner = {
|
||||
kind: "reader" as const,
|
||||
@@ -314,14 +317,15 @@ const writeSelectedEntries = async ({
|
||||
for await (const entry of reader.entries(signal)) {
|
||||
const path = normalizeArtifactPath(entry.path);
|
||||
registry.add(path, entry.kind);
|
||||
// Logical declarations may share a query path with a real file. They do
|
||||
// not consume that file's selection or participate in materialization.
|
||||
if (entry.kind !== "file" && entry.kind !== "slice") continue;
|
||||
const selectedItem = byPath.get(path);
|
||||
if (selectedItem === undefined) {
|
||||
if (entry.kind === "file" || entry.kind === "slice")
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Regular artifact entry is missing from inventory: ${path}`,
|
||||
);
|
||||
continue;
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Regular artifact entry is missing from inventory: ${path}`,
|
||||
);
|
||||
}
|
||||
preflight(entry);
|
||||
const stream = await reader.open(entry, signal);
|
||||
@@ -444,10 +448,17 @@ const requireExtractableFormat = async (
|
||||
path,
|
||||
input.resourceScope,
|
||||
signal,
|
||||
async (handle) =>
|
||||
locateGodotPck(handle, (await handle.stat()).size, signal),
|
||||
async (handle) => {
|
||||
const size = (await handle.stat()).size;
|
||||
if (await locateGodotPck(handle, size, signal)) return true;
|
||||
if (format !== "elf" && format !== "pe") return false;
|
||||
abortIfNeeded(signal);
|
||||
const packaged = await containsPyInstallerArchive(handle, size, signal);
|
||||
abortIfNeeded(signal);
|
||||
return packaged;
|
||||
},
|
||||
);
|
||||
if (pack === undefined)
|
||||
if (!pack)
|
||||
throw new AnalysisUnsupportedTargetError(
|
||||
"extract_artifact",
|
||||
input.inputPath,
|
||||
@@ -464,10 +475,13 @@ const createReader = async (
|
||||
"inputPath" | "inputFormat" | "environment" | "resourceScope"
|
||||
>,
|
||||
containerFormat: ArtifactGraphManifest["root_format"],
|
||||
signal?: AbortSignal,
|
||||
): Promise<ArtifactReader> => {
|
||||
const format = input.inputFormat;
|
||||
if (containerFormat === "godot-pck") return new GodotPckArtifactReader(path);
|
||||
if (await requireExtractableFormat(path, input))
|
||||
if (containerFormat === "pyinstaller")
|
||||
return new PyInstallerArtifactReader(path);
|
||||
if (await requireExtractableFormat(path, input, signal))
|
||||
return new DirectoryArtifactReader(path);
|
||||
if (format === "asar") return new AsarArtifactReader(path);
|
||||
if (isZipPackageFormat(format)) return new ZipArtifactReader(path, format);
|
||||
|
||||
@@ -10,7 +10,7 @@ import {
|
||||
createArtifactEdges,
|
||||
classifyArtifactContent,
|
||||
createOccurrence,
|
||||
indexOccurrencesByPath,
|
||||
indexDestinationOccurrencesByPath,
|
||||
resolveOccurrenceParents,
|
||||
} from "./ArtifactGraphConstruction.js";
|
||||
import { createArtifactClassificationProbe } from "../ArtifactClassificationProbe.js";
|
||||
@@ -42,7 +42,7 @@ describe("occurrence parent ownership", () => {
|
||||
|
||||
resolveOccurrenceParents(
|
||||
occurrences,
|
||||
indexOccurrencesByPath(occurrences),
|
||||
indexDestinationOccurrencesByPath(occurrences),
|
||||
new Set(),
|
||||
);
|
||||
|
||||
@@ -57,7 +57,7 @@ describe("occurrence parent ownership", () => {
|
||||
|
||||
const resolution = resolveOccurrenceParents(
|
||||
occurrences,
|
||||
indexOccurrencesByPath(occurrences),
|
||||
indexDestinationOccurrencesByPath(occurrences),
|
||||
new Set(),
|
||||
);
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { hasGodotPckHeader } from "../../domain/godotPckFormat.js";
|
||||
import type { ArtifactClassificationEvidence } from "../ArtifactClassificationProbe.js";
|
||||
import { ArtifactReaderFailure } from "../ArtifactReader.js";
|
||||
import { hasDestinationPath } from "../ArtifactPaths.js";
|
||||
import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";
|
||||
import { isJavaScriptSourcePath } from "../../domain/javascript/javascriptArtifactFiles.js";
|
||||
import { digestCanonicalValue } from "../../domain/canonicalDigest.js";
|
||||
@@ -56,12 +57,13 @@ export interface MutableOccurrence {
|
||||
limitations: string[];
|
||||
}
|
||||
|
||||
/** Index every occurrence by canonical logical path without choosing an owner. */
|
||||
export const indexOccurrencesByPath = (
|
||||
/** Index destination paths without mixing in declaration query names or choosing an owner. */
|
||||
export const indexDestinationOccurrencesByPath = (
|
||||
occurrences: readonly MutableOccurrence[],
|
||||
): ReadonlyMap<string, readonly MutableOccurrence[]> => {
|
||||
const byPath = new Map<string, MutableOccurrence[]>();
|
||||
for (const occurrence of occurrences) {
|
||||
if (!hasDestinationPath(occurrence.entry_kind)) continue;
|
||||
const matches = byPath.get(occurrence.logical_path);
|
||||
if (matches === undefined)
|
||||
byPath.set(occurrence.logical_path, [occurrence]);
|
||||
@@ -82,16 +84,18 @@ export const createOccurrence = (
|
||||
duplicateOrdinal = 0,
|
||||
): MutableOccurrence => {
|
||||
const role =
|
||||
entry.kind === "directory"
|
||||
? {
|
||||
kind: path.toLowerCase().endsWith(".framework")
|
||||
? ("framework" as const)
|
||||
: ("container" as const),
|
||||
format: "directory" as const,
|
||||
}
|
||||
: containerSuffixFamily(path) === undefined
|
||||
? classifyArtifactPath(path)
|
||||
: ({ kind: "unknown", format: "unknown" } as const);
|
||||
entry.kind === "metadata"
|
||||
? { kind: "package-metadata" as const, format: "unknown" as const }
|
||||
: entry.kind === "directory"
|
||||
? {
|
||||
kind: path.toLowerCase().endsWith(".framework")
|
||||
? ("framework" as const)
|
||||
: ("container" as const),
|
||||
format: "directory" as const,
|
||||
}
|
||||
: containerSuffixFamily(path) === undefined
|
||||
? classifyArtifactPath(path)
|
||||
: ({ kind: "unknown", format: "unknown" } as const);
|
||||
return {
|
||||
occurrence_id: `occ_${digestCanonicalValue(
|
||||
duplicateOrdinal === 0
|
||||
@@ -263,7 +267,7 @@ export const rootOccurrenceFor = (
|
||||
artifact_kind:
|
||||
metadata.format === "directory" ||
|
||||
isZipPackageFormat(metadata.format) ||
|
||||
["asar", "godot-pck", "dmg", "pkg"].includes(metadata.format)
|
||||
["asar", "godot-pck", "dmg", "pkg", "pyinstaller"].includes(metadata.format)
|
||||
? "container"
|
||||
: artifactRoleForFormat(metadata.path, metadata.format).kind,
|
||||
artifact_format: metadata.format,
|
||||
@@ -316,7 +320,7 @@ export const createArtifactEdges = (
|
||||
producer?: ArtifactCommand,
|
||||
): { readonly edges: ArtifactEdge[]; readonly limitations: string[] } => {
|
||||
const byId = new Map(occurrences.map((item) => [item.occurrence_id, item]));
|
||||
const byPath = indexOccurrencesByPath(occurrences);
|
||||
const byPath = indexDestinationOccurrencesByPath(occurrences);
|
||||
const edges: ArtifactEdge[] = [];
|
||||
const limitations: string[] = [];
|
||||
for (const occurrence of occurrences) {
|
||||
@@ -364,11 +368,14 @@ export const createArtifactEdges = (
|
||||
relation:
|
||||
occurrence.entry_kind === "slice"
|
||||
? ("slice-of" as const)
|
||||
: isSourceMap
|
||||
? mappedSource === undefined
|
||||
? ("contains" as const)
|
||||
: ("maps-source" as const)
|
||||
: relationFor(occurrence.logical_path),
|
||||
: occurrence.format_metadata?.kind === "pyinstaller-entry" &&
|
||||
occurrence.format_metadata.bytecode_header === "derived"
|
||||
? ("derived-from" as const)
|
||||
: isSourceMap
|
||||
? mappedSource === undefined
|
||||
? ("contains" as const)
|
||||
: ("maps-source" as const)
|
||||
: relationFor(occurrence.logical_path),
|
||||
occurrence_id: occurrence.occurrence_id,
|
||||
};
|
||||
edges.push({
|
||||
@@ -428,6 +435,9 @@ export const resolveOccurrenceParents = (
|
||||
const limitations = new Set<string>();
|
||||
const unknownParentOccurrences = new Set<MutableOccurrence>();
|
||||
for (const occurrence of occurrences) {
|
||||
// Query paths do not imply filesystem containment. The scanner records
|
||||
// the declaring container for metadata instead.
|
||||
if (occurrence.entry_kind === "metadata") continue;
|
||||
const path = occurrence.logical_path;
|
||||
let branch = root;
|
||||
let parent: MutableOccurrence | undefined;
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { Stats } from "node:fs";
|
||||
import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js";
|
||||
import type { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
|
||||
|
||||
import { containsPyInstallerArchive } from "../pyinstaller/CArchiveLayout.js";
|
||||
import { classifyArtifactContent } from "./ArtifactGraphConstruction.js";
|
||||
import { abortIfNeeded } from "../ArtifactHash.js";
|
||||
import { createArtifactClassificationProbe } from "../ArtifactClassificationProbe.js";
|
||||
@@ -114,7 +115,8 @@ export const classifyAndHashRootForInventory = async (
|
||||
const retainSource =
|
||||
isZipPackageFormat(format) ||
|
||||
format === "mach-o-universal" ||
|
||||
format === "godot-pck";
|
||||
format === "godot-pck" ||
|
||||
format === "pyinstaller";
|
||||
outcome = {
|
||||
kind: "completed",
|
||||
value: {
|
||||
@@ -178,7 +180,7 @@ const classifyRootFormat = async (
|
||||
evidence.fatSliceHeaders,
|
||||
signal,
|
||||
);
|
||||
return classifyArtifactContent(
|
||||
const format = classifyArtifactContent(
|
||||
path,
|
||||
{
|
||||
...evidence,
|
||||
@@ -186,6 +188,16 @@ const classifyRootFormat = async (
|
||||
},
|
||||
archiveBytes,
|
||||
).format;
|
||||
if (["elf", "pe", "mach-o", "mach-o-universal"].includes(format)) {
|
||||
const packaged = await containsPyInstallerArchive(
|
||||
handle,
|
||||
archiveBytes,
|
||||
signal,
|
||||
);
|
||||
abortIfNeeded(signal);
|
||||
if (packaged) return "pyinstaller";
|
||||
}
|
||||
return format;
|
||||
};
|
||||
|
||||
/** Read exact FAT header windows missed by the bounded prefix probe. */
|
||||
|
||||
@@ -7,6 +7,7 @@ import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js";
|
||||
import { GodotPckArtifactReader } from "../GodotPckArtifactReader.js";
|
||||
import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js";
|
||||
import { NativeDmgArtifactReader } from "../NativeDmgArtifactReader.js";
|
||||
import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js";
|
||||
import { ZipArtifactReader } from "../ZipArtifactReader.js";
|
||||
import type {
|
||||
ArtifactInventoryCoverage,
|
||||
@@ -31,6 +32,8 @@ export const createReader = (
|
||||
return new DirectoryArtifactReader(path);
|
||||
case "asar":
|
||||
return new AsarArtifactReader(path);
|
||||
case "pyinstaller":
|
||||
return new PyInstallerArtifactReader(path, rootSource);
|
||||
case "godot-pck":
|
||||
return new GodotPckArtifactReader(path, rootSource);
|
||||
case "mach-o-universal":
|
||||
|
||||
@@ -26,7 +26,7 @@ import {
|
||||
createArtifactNode,
|
||||
createOccurrence,
|
||||
noteUnreadContainerSuffix,
|
||||
indexOccurrencesByPath,
|
||||
indexDestinationOccurrencesByPath,
|
||||
resolveOccurrenceParents,
|
||||
type MutableOccurrence,
|
||||
} from "./ArtifactGraphConstruction.js";
|
||||
@@ -107,7 +107,7 @@ export const scanReader = async (
|
||||
await visitArtifactEntries(context, reader, "");
|
||||
// Archive directories may appear after their children. Resolve containment
|
||||
// against the complete index before directory identities are materialized.
|
||||
const byPath = indexOccurrencesByPath(occurrences);
|
||||
const byPath = indexDestinationOccurrencesByPath(occurrences);
|
||||
const parentResolution = resolveOccurrenceParents(
|
||||
occurrences,
|
||||
byPath,
|
||||
@@ -167,11 +167,13 @@ const visitArtifactEntries = async (
|
||||
const stack: Array<{
|
||||
readonly reader: ArtifactReader;
|
||||
readonly prefix: string;
|
||||
readonly containerId: string | null;
|
||||
readonly iterator: AsyncIterator<ArtifactEntry>;
|
||||
}> = [
|
||||
{
|
||||
reader: currentReader,
|
||||
prefix,
|
||||
containerId: null,
|
||||
iterator: inventoryEntries(context, currentReader, prefix),
|
||||
},
|
||||
];
|
||||
@@ -202,7 +204,7 @@ const visitArtifactEntries = async (
|
||||
const occurrence = createOccurrence(
|
||||
entry,
|
||||
logicalPath,
|
||||
null,
|
||||
entry.kind === "metadata" ? frame.containerId : null,
|
||||
duplicateOrdinal,
|
||||
);
|
||||
let digested:
|
||||
@@ -254,6 +256,7 @@ const visitArtifactEntries = async (
|
||||
stack.push({
|
||||
reader: nested,
|
||||
prefix: logicalPath,
|
||||
containerId: occurrence.occurrence_id,
|
||||
iterator: inventoryEntries(context, nested, logicalPath),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
import { deflateSync } from "node:zlib";
|
||||
|
||||
import { PYINSTALLER_COOKIE_MAGIC } from "./CArchiveLayout.js";
|
||||
|
||||
/** Final CPython 3.9 bytecode magic, little-endian with the CR LF trailer. */
|
||||
export const PYTHON_39_MAGIC = Buffer.from("610d0d0a", "hex");
|
||||
|
||||
const ELF_PREFIX = (() => {
|
||||
const bytes = Buffer.alloc(64);
|
||||
bytes.writeUInt32LE(0x464c457f, 0);
|
||||
bytes[4] = 2;
|
||||
bytes[5] = 1;
|
||||
bytes[6] = 1;
|
||||
bytes.writeUInt16LE(2, 16);
|
||||
bytes.writeUInt16LE(62, 18);
|
||||
bytes.writeUInt32LE(1, 20);
|
||||
bytes.writeUInt16LE(64, 52);
|
||||
return bytes;
|
||||
})();
|
||||
|
||||
/** One CArchive member supplied as uncompressed bytes. */
|
||||
export interface CArchiveFixtureMember {
|
||||
readonly name: string;
|
||||
readonly typeCode: string;
|
||||
readonly payload: Buffer;
|
||||
readonly compress: boolean;
|
||||
readonly compressedSuffix?: Buffer;
|
||||
readonly declaredUncompressedSize?: number;
|
||||
}
|
||||
|
||||
/** Build a tiny ELF with an appended CArchive, optionally followed by a trailer. */
|
||||
export const cArchiveExecutable = (input: {
|
||||
readonly layout: "2.0" | "2.1";
|
||||
readonly pythonVersion: number;
|
||||
readonly pythonLibrary?: string;
|
||||
readonly members: readonly CArchiveFixtureMember[];
|
||||
readonly trailer?: Buffer;
|
||||
readonly tocBytes?: Buffer;
|
||||
}): Buffer => {
|
||||
const payloads: Buffer[] = [];
|
||||
const entries: Buffer[] = [];
|
||||
let dataLength = 0;
|
||||
for (const member of input.members) {
|
||||
const stored = member.compress
|
||||
? Buffer.concat([
|
||||
deflateSync(member.payload),
|
||||
member.compressedSuffix ?? Buffer.alloc(0),
|
||||
])
|
||||
: member.payload;
|
||||
entries.push(
|
||||
tocEntry({
|
||||
name: member.name,
|
||||
typeCode: member.typeCode,
|
||||
offset: dataLength,
|
||||
compressedSize: stored.length,
|
||||
uncompressedSize:
|
||||
member.declaredUncompressedSize ?? member.payload.length,
|
||||
flag: member.compress ? 1 : 0,
|
||||
}),
|
||||
);
|
||||
payloads.push(stored);
|
||||
dataLength += stored.length;
|
||||
}
|
||||
const toc = input.tocBytes ?? Buffer.concat(entries);
|
||||
const cookie = cookieBytes({
|
||||
layout: input.layout,
|
||||
pythonVersion: input.pythonVersion,
|
||||
pythonLibrary: input.pythonLibrary,
|
||||
tocOffset: dataLength,
|
||||
tocLength: toc.length,
|
||||
});
|
||||
return Buffer.concat([
|
||||
ELF_PREFIX,
|
||||
...payloads,
|
||||
toc,
|
||||
cookie,
|
||||
input.trailer ?? Buffer.alloc(0),
|
||||
]);
|
||||
};
|
||||
|
||||
/** Cookie whose package range starts before the file. */
|
||||
export const cookieRangeOutsideFile = (): Buffer => {
|
||||
const cookie = Buffer.alloc(88);
|
||||
PYINSTALLER_COOKIE_MAGIC.copy(cookie, 0);
|
||||
cookie.writeUInt32BE(200, 8);
|
||||
cookie.writeUInt32BE(80, 12);
|
||||
cookie.writeUInt32BE(32, 16);
|
||||
cookie.writeUInt32BE(314, 20);
|
||||
return Buffer.concat([ELF_PREFIX, cookie]);
|
||||
};
|
||||
|
||||
/** PYZ image. Module bodies are zlib-compressed. Namespace entries have no body. */
|
||||
export const pyzArchive = (input: {
|
||||
readonly magic: Buffer;
|
||||
readonly modules: readonly PyzFixtureModule[];
|
||||
readonly toc?: Buffer;
|
||||
}): Buffer => {
|
||||
const blobs: Buffer[] = [];
|
||||
const entries: Buffer[] = [];
|
||||
let offset = 12;
|
||||
for (const module of input.modules) {
|
||||
const compressed =
|
||||
module.typeCode === 3 && module.legacyNamespacePayload !== true
|
||||
? Buffer.alloc(0)
|
||||
: Buffer.concat([
|
||||
deflateSync(module.body),
|
||||
module.compressedSuffix ?? Buffer.alloc(0),
|
||||
]);
|
||||
entries.push(
|
||||
marshalPair(module.name, module.typeCode, offset, compressed.length),
|
||||
);
|
||||
if (compressed.length > 0) {
|
||||
blobs.push(compressed);
|
||||
offset += compressed.length;
|
||||
}
|
||||
}
|
||||
const toc = input.toc ?? marshalList(entries);
|
||||
const header = Buffer.alloc(12);
|
||||
Buffer.from("PYZ\0").copy(header, 0);
|
||||
input.magic.copy(header, 4, 0, 4);
|
||||
header.writeUInt32BE(
|
||||
12 + blobs.reduce((sum, blob) => sum + blob.length, 0),
|
||||
8,
|
||||
);
|
||||
return Buffer.concat([header, ...blobs, toc]);
|
||||
};
|
||||
|
||||
/** Type 3 normally stores no body; older producers can write a namespace payload. */
|
||||
export interface PyzFixtureModule {
|
||||
readonly legacyNamespacePayload?: boolean;
|
||||
readonly compressedSuffix?: Buffer;
|
||||
readonly name: string;
|
||||
readonly typeCode: number;
|
||||
readonly body: Buffer;
|
||||
}
|
||||
|
||||
const tocEntry = (input: {
|
||||
readonly name: string;
|
||||
readonly typeCode: string;
|
||||
readonly offset: number;
|
||||
readonly compressedSize: number;
|
||||
readonly uncompressedSize: number;
|
||||
readonly flag: number;
|
||||
}): Buffer => {
|
||||
const name = Buffer.from(input.name, "utf8");
|
||||
const length = Math.ceil((18 + name.length + 1) / 16) * 16;
|
||||
const bytes = Buffer.alloc(length);
|
||||
bytes.writeUInt32BE(length, 0);
|
||||
bytes.writeUInt32BE(input.offset, 4);
|
||||
bytes.writeUInt32BE(input.compressedSize, 8);
|
||||
bytes.writeUInt32BE(input.uncompressedSize, 12);
|
||||
bytes[16] = input.flag;
|
||||
bytes[17] = input.typeCode.charCodeAt(0);
|
||||
name.copy(bytes, 18);
|
||||
return bytes;
|
||||
};
|
||||
|
||||
const cookieBytes = (input: {
|
||||
readonly layout: "2.0" | "2.1";
|
||||
readonly pythonVersion: number;
|
||||
readonly pythonLibrary: string | undefined;
|
||||
readonly tocOffset: number;
|
||||
readonly tocLength: number;
|
||||
}): Buffer => {
|
||||
const size = input.layout === "2.1" ? 88 : 24;
|
||||
const bytes = Buffer.alloc(size);
|
||||
PYINSTALLER_COOKIE_MAGIC.copy(bytes, 0);
|
||||
bytes.writeUInt32BE(input.tocOffset + input.tocLength + size, 8);
|
||||
bytes.writeUInt32BE(input.tocOffset, 12);
|
||||
bytes.writeUInt32BE(input.tocLength, 16);
|
||||
bytes.writeUInt32BE(input.pythonVersion, 20);
|
||||
if (input.pythonLibrary !== undefined)
|
||||
Buffer.from(input.pythonLibrary, "utf8").copy(bytes, 24);
|
||||
return bytes;
|
||||
};
|
||||
|
||||
const marshalList = (items: readonly Buffer[]): Buffer => {
|
||||
const count = Buffer.alloc(4);
|
||||
count.writeInt32LE(items.length, 0);
|
||||
return Buffer.concat([Buffer.from([0x5b]), count, ...items]);
|
||||
};
|
||||
|
||||
const marshalPair = (
|
||||
name: string,
|
||||
typeCode: number,
|
||||
offset: number,
|
||||
length: number,
|
||||
): Buffer =>
|
||||
smallTuple([
|
||||
marshalString(name),
|
||||
smallTuple([int32(typeCode), int32(offset), int32(length)]),
|
||||
]);
|
||||
|
||||
const smallTuple = (items: readonly Buffer[]): Buffer =>
|
||||
Buffer.concat([Buffer.from([0x29, items.length]), ...items]);
|
||||
|
||||
const marshalString = (text: string): Buffer => {
|
||||
const body = Buffer.from(text, "utf8");
|
||||
if (body.length <= 255 && body.every((byte) => byte < 0x80))
|
||||
return Buffer.concat([Buffer.from([0x7a, body.length]), body]);
|
||||
const length = Buffer.alloc(4);
|
||||
length.writeUInt32LE(body.length);
|
||||
return Buffer.concat([Buffer.from([0x75]), length, body]);
|
||||
};
|
||||
|
||||
const int32 = (value: number): Buffer => {
|
||||
const bytes = Buffer.alloc(5);
|
||||
bytes[0] = 0x69;
|
||||
bytes.writeInt32LE(value, 1);
|
||||
return bytes;
|
||||
};
|
||||
@@ -0,0 +1,389 @@
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
import { readPyInstallerBytes } from "./PyInstallerBytes.js";
|
||||
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import {
|
||||
pyInstallerFailure,
|
||||
type PyInstallerFailure,
|
||||
} from "./PyInstallerFailure.js";
|
||||
|
||||
/** Magic at the start of a CArchive cookie. */
|
||||
export const PYINSTALLER_COOKIE_MAGIC = Buffer.from("MEI\x0c\x0b\x0a\x0b\x0e");
|
||||
|
||||
/** PyInstaller 2.1+ cookie size, including the 64-byte library name. */
|
||||
export const PYINSTALLER_COOKIE_V21_BYTES = 88;
|
||||
|
||||
/** PyInstaller 2.0 cookie size, without the library name. */
|
||||
export const PYINSTALLER_COOKIE_V20_BYTES = 24;
|
||||
|
||||
/**
|
||||
* How far before EOF the cookie may sit. macOS code signatures are appended
|
||||
* after the cookie, and a large binary's signature hashes are larger than the
|
||||
* bootloader's own 8 KiB scan chunk, so detection uses one bounded tail.
|
||||
*/
|
||||
export const PYINSTALLER_COOKIE_SEARCH_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
const MAX_TOC_BYTES = 32 * 1024 * 1024;
|
||||
const MAX_TOC_ENTRIES = 100_000;
|
||||
const TOC_HEADER_BYTES = 18;
|
||||
|
||||
/** Observed CArchive cookie. Offsets are relative to the archive start. */
|
||||
export interface PyInstallerCookie {
|
||||
readonly layout: "2.0" | "2.1";
|
||||
readonly archiveStart: number;
|
||||
readonly packageLength: number;
|
||||
readonly tocOffset: number;
|
||||
readonly tocLength: number;
|
||||
readonly pythonVersion: number;
|
||||
readonly pythonLibrary: string | null;
|
||||
readonly cookieOffset: number;
|
||||
}
|
||||
|
||||
/** One CArchive table entry. */
|
||||
export interface CArchiveEntry {
|
||||
readonly index: number;
|
||||
readonly tocRecordOffset: number;
|
||||
readonly tocRecordLength: number;
|
||||
readonly name: string;
|
||||
readonly typeCode: string;
|
||||
readonly offset: number;
|
||||
readonly compressedSize: number;
|
||||
readonly uncompressedSize: number;
|
||||
readonly compressionFlag: number;
|
||||
}
|
||||
|
||||
/** Runtime options and external dependencies are declarations, never local blobs. */
|
||||
export const isCArchiveDeclaration = (
|
||||
entry: CArchiveEntry,
|
||||
): entry is CArchiveEntry & { readonly typeCode: "o" | "d" } =>
|
||||
entry.typeCode === "o" || entry.typeCode === "d";
|
||||
|
||||
/** True when a native executable has a structurally valid CArchive cookie. */
|
||||
export const containsPyInstallerArchive = async (
|
||||
handle: FileHandle,
|
||||
fileSize: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<boolean> => {
|
||||
const found = await findPyInstallerCookie(handle, fileSize, signal);
|
||||
return found?.ok === true;
|
||||
};
|
||||
|
||||
/** Locate a structurally valid cookie. Invalid magic copies are skipped. */
|
||||
export const findPyInstallerCookie = async (
|
||||
handle: FileHandle,
|
||||
fileSize: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Result<PyInstallerCookie, PyInstallerFailure> | undefined> => {
|
||||
if (
|
||||
!Number.isSafeInteger(fileSize) ||
|
||||
fileSize < PYINSTALLER_COOKIE_V20_BYTES
|
||||
)
|
||||
return undefined;
|
||||
const tailSize = Math.min(fileSize, PYINSTALLER_COOKIE_SEARCH_BYTES);
|
||||
const read = await readPyInstallerBytes(handle, {
|
||||
position: fileSize - tailSize,
|
||||
length: tailSize,
|
||||
label: "CArchive tail",
|
||||
signal,
|
||||
});
|
||||
if (!read.ok) return read;
|
||||
const observed = read.value;
|
||||
let failure: PyInstallerFailure | undefined;
|
||||
let from = observed.length;
|
||||
while (from >= PYINSTALLER_COOKIE_MAGIC.length) {
|
||||
const relative = observed
|
||||
.subarray(0, from)
|
||||
.lastIndexOf(PYINSTALLER_COOKIE_MAGIC);
|
||||
if (relative < 0) break;
|
||||
const cookieOffset = fileSize - tailSize + relative;
|
||||
const parsed = parseCookieAt(observed, relative, cookieOffset, fileSize);
|
||||
if (parsed.ok) return parsed;
|
||||
failure = parsed.error;
|
||||
from = relative;
|
||||
}
|
||||
return failure === undefined ? undefined : err(failure);
|
||||
};
|
||||
|
||||
/** Read and parse the TOC that a validated cookie names. */
|
||||
export const readCArchiveToc = async (
|
||||
handle: FileHandle,
|
||||
cookie: PyInstallerCookie,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Result<readonly CArchiveEntry[], PyInstallerFailure>> => {
|
||||
if (cookie.tocLength > MAX_TOC_BYTES)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"limit",
|
||||
"CArchive table of contents exceeds its bound",
|
||||
),
|
||||
);
|
||||
const position = cookie.archiveStart + cookie.tocOffset;
|
||||
const read = await readPyInstallerBytes(handle, {
|
||||
position,
|
||||
length: cookie.tocLength,
|
||||
label: "CArchive table of contents",
|
||||
signal,
|
||||
});
|
||||
if (!read.ok) return read;
|
||||
const parsed = parseCArchiveToc(read.value);
|
||||
if (!parsed.ok) return parsed;
|
||||
const valid = validateCArchiveMembers(parsed.value, cookie.tocOffset);
|
||||
return valid.ok ? parsed : valid;
|
||||
};
|
||||
|
||||
/** Parse TOC bytes. Each entry is a big-endian header plus a NUL-padded name. */
|
||||
export const parseCArchiveToc = (
|
||||
bytes: Buffer,
|
||||
): Result<readonly CArchiveEntry[], PyInstallerFailure> => {
|
||||
const entries: CArchiveEntry[] = [];
|
||||
let offset = 0;
|
||||
while (offset < bytes.length) {
|
||||
if (entries.length >= MAX_TOC_ENTRIES)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"limit",
|
||||
"CArchive table of contents exceeds its entry bound",
|
||||
),
|
||||
);
|
||||
if (offset + TOC_HEADER_BYTES > bytes.length)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"CArchive table of contents entry is truncated",
|
||||
),
|
||||
);
|
||||
const entryLength = bytes.readUInt32BE(offset);
|
||||
if (entryLength < TOC_HEADER_BYTES || offset + entryLength > bytes.length)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"CArchive table of contents entry extends outside the table",
|
||||
),
|
||||
);
|
||||
const nameBytes = bytes.subarray(
|
||||
offset + TOC_HEADER_BYTES,
|
||||
offset + entryLength,
|
||||
);
|
||||
const name = tocName(nameBytes);
|
||||
if (!name.ok) return name;
|
||||
const compressionFlag = bytes[offset + 16] ?? 0;
|
||||
if (compressionFlag !== 0 && compressionFlag !== 1)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
`CArchive compression flag ${String(compressionFlag)} is not stored or zlib`,
|
||||
),
|
||||
);
|
||||
const typeCode = bytes[offset + 17] ?? 0;
|
||||
if (typeCode < 0x20 || typeCode > 0x7e)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"CArchive type code is not a printable character",
|
||||
),
|
||||
);
|
||||
entries.push({
|
||||
index: entries.length,
|
||||
tocRecordOffset: offset,
|
||||
tocRecordLength: entryLength,
|
||||
name: name.value,
|
||||
typeCode: String.fromCharCode(typeCode),
|
||||
offset: bytes.readUInt32BE(offset + 4),
|
||||
compressedSize: bytes.readUInt32BE(offset + 8),
|
||||
uncompressedSize: bytes.readUInt32BE(offset + 12),
|
||||
compressionFlag,
|
||||
});
|
||||
offset += entryLength;
|
||||
}
|
||||
return ok(entries);
|
||||
};
|
||||
|
||||
const parseCookieAt = (
|
||||
tail: Buffer,
|
||||
relative: number,
|
||||
cookieOffset: number,
|
||||
fileSize: number,
|
||||
): Result<PyInstallerCookie, PyInstallerFailure> => {
|
||||
const modern = parseCookie(tail, relative, cookieOffset, fileSize, "2.1");
|
||||
if (modern.ok) return modern;
|
||||
const legacy = parseCookie(tail, relative, cookieOffset, fileSize, "2.0");
|
||||
return legacy.ok ? legacy : modern;
|
||||
};
|
||||
|
||||
const parseCookie = (
|
||||
tail: Buffer,
|
||||
relative: number,
|
||||
cookieOffset: number,
|
||||
fileSize: number,
|
||||
layout: "2.0" | "2.1",
|
||||
): Result<PyInstallerCookie, PyInstallerFailure> => {
|
||||
const cookieBytes =
|
||||
layout === "2.1"
|
||||
? PYINSTALLER_COOKIE_V21_BYTES
|
||||
: PYINSTALLER_COOKIE_V20_BYTES;
|
||||
if (relative + cookieBytes > tail.length)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"CArchive cookie extends past the end of the file",
|
||||
),
|
||||
);
|
||||
const view = tail.subarray(relative, relative + cookieBytes);
|
||||
const packageLength = view.readUInt32BE(8);
|
||||
const tocOffset = view.readUInt32BE(12);
|
||||
const tocLength = view.readUInt32BE(16);
|
||||
const pythonVersion = view.readUInt32BE(20);
|
||||
if (
|
||||
!safeSum(tocOffset, tocLength, cookieBytes) ||
|
||||
packageLength !== tocOffset + tocLength + cookieBytes
|
||||
)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"CArchive cookie does not describe a table of contents inside the package",
|
||||
),
|
||||
);
|
||||
const archiveStart = cookieOffset + cookieBytes - packageLength;
|
||||
if (archiveStart < 0 || archiveStart + packageLength > fileSize)
|
||||
return err(
|
||||
pyInstallerFailure("format", "CArchive cookie range is outside the file"),
|
||||
);
|
||||
const library =
|
||||
layout === "2.1" ? libraryName(view.subarray(24, 88)) : ok(null);
|
||||
if (!library.ok) return library;
|
||||
return ok({
|
||||
layout,
|
||||
archiveStart,
|
||||
packageLength,
|
||||
tocOffset,
|
||||
tocLength,
|
||||
pythonVersion,
|
||||
pythonLibrary: library.value,
|
||||
cookieOffset,
|
||||
});
|
||||
};
|
||||
|
||||
const libraryName = (
|
||||
bytes: Buffer,
|
||||
): Result<string | null, PyInstallerFailure> => {
|
||||
const end = bytes.indexOf(0);
|
||||
const text = end < 0 ? bytes : bytes.subarray(0, end);
|
||||
if (end >= 0 && bytes.subarray(end).some((byte) => byte !== 0))
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"CArchive Python library name is not NUL-padded",
|
||||
),
|
||||
);
|
||||
if (text.length === 0) return ok(null);
|
||||
try {
|
||||
return ok(
|
||||
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text),
|
||||
);
|
||||
} catch {
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"CArchive Python library name is not valid UTF-8",
|
||||
),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const tocName = (bytes: Buffer): Result<string, PyInstallerFailure> => {
|
||||
const end = bytes.indexOf(0);
|
||||
const text = end < 0 ? bytes : bytes.subarray(0, end);
|
||||
if (text.length === 0)
|
||||
return err(pyInstallerFailure("format", "CArchive entry name is empty"));
|
||||
try {
|
||||
return ok(
|
||||
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text),
|
||||
);
|
||||
} catch {
|
||||
return err(
|
||||
pyInstallerFailure("format", "CArchive entry name is not valid UTF-8"),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const safeSum = (left: number, middle: number, right: number): boolean =>
|
||||
Number.isSafeInteger(left) &&
|
||||
Number.isSafeInteger(middle) &&
|
||||
Number.isSafeInteger(right) &&
|
||||
left >= 0 &&
|
||||
middle >= 0 &&
|
||||
right >= 0 &&
|
||||
left + middle + right <= Number.MAX_SAFE_INTEGER;
|
||||
|
||||
/** Source directory semantics are validated once before any consumer projects paths. */
|
||||
const validateCArchiveMembers = (
|
||||
entries: readonly CArchiveEntry[],
|
||||
tocOffset: number,
|
||||
): Result<null, PyInstallerFailure> => {
|
||||
const names = new Set<string>();
|
||||
for (const entry of entries) {
|
||||
if (isCArchiveDeclaration(entry)) {
|
||||
if (
|
||||
entry.compressedSize !== 0 ||
|
||||
entry.uncompressedSize !== 0 ||
|
||||
entry.compressionFlag !== 0
|
||||
)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"integrity",
|
||||
`CArchive declaration at table index ${String(entry.index)} has local payload or compression`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
if (names.has(entry.name))
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"integrity",
|
||||
`Duplicate CArchive member name at table index ${String(entry.index)}`,
|
||||
),
|
||||
);
|
||||
names.add(entry.name);
|
||||
}
|
||||
if (
|
||||
entry.compressionFlag === 0 &&
|
||||
entry.compressedSize !== entry.uncompressedSize
|
||||
)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"integrity",
|
||||
`Stored CArchive member size disagrees with its table entry at table index ${String(entry.index)}`,
|
||||
),
|
||||
);
|
||||
if (entry.offset > tocOffset)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
`CArchive member offset is outside the archive at table index ${String(entry.index)}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
const ranges = entries
|
||||
.filter((entry) => entry.compressedSize > 0)
|
||||
.map((entry) => ({
|
||||
index: entry.index,
|
||||
start: entry.offset,
|
||||
end: entry.offset + entry.compressedSize,
|
||||
}))
|
||||
.sort((left, right) => left.start - right.start);
|
||||
let occupied = 0;
|
||||
for (const range of ranges) {
|
||||
if (
|
||||
!Number.isSafeInteger(range.end) ||
|
||||
range.end > tocOffset ||
|
||||
range.start < occupied
|
||||
)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"integrity",
|
||||
`CArchive member extends outside the archive or overlaps another member at table index ${String(range.index)}`,
|
||||
),
|
||||
);
|
||||
occupied = range.end;
|
||||
}
|
||||
return ok(null);
|
||||
};
|
||||
@@ -0,0 +1,340 @@
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
import { copyArtifactEntry, type ArtifactEntry } from "../ArtifactReader.js";
|
||||
import { isCArchiveDeclaration, type CArchiveEntry } from "./CArchiveLayout.js";
|
||||
import type { PyzArchive, PyzEntry } from "./PyzArchive.js";
|
||||
import {
|
||||
RECONSTRUCTED_PYTHON_HEADER_LIMITATION,
|
||||
reconstructedPycHeader,
|
||||
} from "./PythonBytecode.js";
|
||||
import {
|
||||
cArchiveEntryMetadata,
|
||||
pyzEntryMetadata,
|
||||
pyzEntryEncrypted,
|
||||
} from "./PyInstallerMetadata.js";
|
||||
import type { PyInstallerEntryMetadata } from "../../domain/artifactFormatMetadata.js";
|
||||
|
||||
const CRYPTO_KEY_NAME = "pyimod00_crypto_key";
|
||||
|
||||
/** Parsed facts and the descriptor identity used to locate a retained image. */
|
||||
export interface PyzImage {
|
||||
readonly archive: PyzArchive;
|
||||
readonly handle: FileHandle;
|
||||
readonly start: number;
|
||||
}
|
||||
|
||||
/** Stable entry descriptions and bounded read plans; expanded member bytes are never retained. */
|
||||
export interface PyInstallerMember {
|
||||
readonly entry: ArtifactEntry;
|
||||
readonly read:
|
||||
| {
|
||||
readonly kind: "carchive";
|
||||
readonly archiveStart: number;
|
||||
readonly source: CArchiveEntry;
|
||||
readonly header: Buffer | undefined;
|
||||
}
|
||||
| {
|
||||
readonly kind: "pyz";
|
||||
readonly handle: FileHandle;
|
||||
readonly offset: number;
|
||||
readonly length: number;
|
||||
readonly compressed: boolean;
|
||||
readonly uncompressedSize: number | undefined;
|
||||
readonly header: Buffer | undefined;
|
||||
}
|
||||
| undefined;
|
||||
}
|
||||
|
||||
/** The producer's key is an importable, nonempty Python module, not a data filename. */
|
||||
export const hasEncryptionMarker = (
|
||||
entries: readonly CArchiveEntry[],
|
||||
): boolean =>
|
||||
entries.some(
|
||||
(entry) =>
|
||||
entry.name === CRYPTO_KEY_NAME &&
|
||||
(entry.typeCode === "m" || entry.typeCode === "M") &&
|
||||
entry.uncompressedSize > 0,
|
||||
);
|
||||
|
||||
export const projectStoredMember = (
|
||||
entry: CArchiveEntry,
|
||||
archiveStart: number,
|
||||
pythonMagic: Buffer | undefined,
|
||||
): PyInstallerMember => {
|
||||
if (isCArchiveDeclaration(entry))
|
||||
return {
|
||||
entry: fileEntry({
|
||||
path: `__pyinstaller__/${entry.typeCode === "o" ? "runtime-options" : "dependencies"}/${String(entry.index)}`,
|
||||
kind: "metadata",
|
||||
declaredSize: entry.uncompressedSize,
|
||||
compressedSize: entry.compressedSize,
|
||||
byteOffset: null,
|
||||
limitations:
|
||||
entry.typeCode === "d"
|
||||
? [
|
||||
"Multipackage dependency bytes belong to another executable and remain unresolved.",
|
||||
]
|
||||
: [],
|
||||
adapterKey: `carchive:${String(entry.index)}:${entry.name}`,
|
||||
formatMetadata: cArchiveEntryMetadata(entry),
|
||||
}),
|
||||
read: undefined,
|
||||
};
|
||||
const header =
|
||||
pythonMagic !== undefined && isBytecodeType(entry.typeCode)
|
||||
? reconstructedPycHeader(pythonMagic)
|
||||
: undefined;
|
||||
const path = canonicalMemberPath(entry.name);
|
||||
const symlink = entry.typeCode === "n";
|
||||
const limitations: string[] = [];
|
||||
if (symlink) limitations.push("PyInstaller symlink target was not read.");
|
||||
if (header !== undefined)
|
||||
limitations.push(RECONSTRUCTED_PYTHON_HEADER_LIMITATION);
|
||||
const declaredSize = entry.uncompressedSize;
|
||||
return {
|
||||
entry: fileEntry({
|
||||
path,
|
||||
kind: symlink ? "symlink" : "file",
|
||||
declaredSize,
|
||||
compressedSize: entry.compressedSize,
|
||||
byteOffset: sourceByteOffset(entry, archiveStart),
|
||||
limitations,
|
||||
adapterKey: `carchive:${String(entry.index)}:${entry.name}`,
|
||||
formatMetadata: cArchiveEntryMetadata(entry, header),
|
||||
}),
|
||||
read: {
|
||||
kind: "carchive",
|
||||
archiveStart,
|
||||
source: entry,
|
||||
header,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export const projectPyzContainer = (
|
||||
entry: CArchiveEntry,
|
||||
archiveStart: number,
|
||||
image: PyzImage,
|
||||
encrypted: boolean,
|
||||
): PyInstallerMember => {
|
||||
const { archive } = image;
|
||||
const path = canonicalMemberPath(entry.name);
|
||||
const byteOffset = sourceByteOffset(entry, archiveStart);
|
||||
const limitations: string[] = [];
|
||||
if (archive.kind === "opaque")
|
||||
return {
|
||||
entry: fileEntry({
|
||||
path,
|
||||
declaredSize: entry.uncompressedSize,
|
||||
compressedSize: entry.compressedSize,
|
||||
byteOffset,
|
||||
// Legacy encryption applies to inner payloads, never the PYZ image.
|
||||
encrypted: false,
|
||||
limitations: [
|
||||
...limitations,
|
||||
archive.limitation,
|
||||
...(encrypted
|
||||
? ["PYZ entries are encrypted and were not decoded."]
|
||||
: []),
|
||||
],
|
||||
adapterKey: `carchive:${String(entry.index)}:${entry.name}`,
|
||||
formatMetadata: cArchiveEntryMetadata(
|
||||
entry,
|
||||
undefined,
|
||||
archive,
|
||||
encrypted,
|
||||
),
|
||||
}),
|
||||
read:
|
||||
entry.compressionFlag === 1
|
||||
? {
|
||||
kind: "pyz",
|
||||
handle: image.handle,
|
||||
offset: image.start,
|
||||
length: entry.uncompressedSize,
|
||||
compressed: false,
|
||||
uncompressedSize: entry.uncompressedSize,
|
||||
header: undefined,
|
||||
}
|
||||
: {
|
||||
kind: "carchive",
|
||||
archiveStart,
|
||||
source: entry,
|
||||
header: undefined,
|
||||
},
|
||||
};
|
||||
return {
|
||||
entry: {
|
||||
path,
|
||||
kind: "directory",
|
||||
declaredSize: entry.uncompressedSize,
|
||||
compressedSize: entry.compressedSize,
|
||||
executable: false,
|
||||
encrypted: false,
|
||||
byteOffset,
|
||||
declaredIntegrity: null,
|
||||
unpacked: false,
|
||||
limitations: [
|
||||
...limitations,
|
||||
`PyInstaller PYZ sha256 ${archive.sha256}.`,
|
||||
...(encrypted
|
||||
? ["PYZ entries are encrypted and were not decoded."]
|
||||
: []),
|
||||
],
|
||||
adapterKey: `carchive:${String(entry.index)}:${entry.name}`,
|
||||
formatMetadata: cArchiveEntryMetadata(
|
||||
entry,
|
||||
undefined,
|
||||
archive,
|
||||
encrypted,
|
||||
),
|
||||
},
|
||||
read: undefined,
|
||||
};
|
||||
};
|
||||
|
||||
export const projectPyzMember = (
|
||||
container: CArchiveEntry,
|
||||
item: PyzEntry,
|
||||
encryptionMarker: boolean,
|
||||
pythonMagic: Buffer | undefined,
|
||||
range:
|
||||
| {
|
||||
readonly handle: FileHandle;
|
||||
readonly offset: number;
|
||||
readonly length: number;
|
||||
}
|
||||
| undefined,
|
||||
): PyInstallerMember => {
|
||||
const path = pyzMemberPath(canonicalMemberPath(container.name), item);
|
||||
const encrypted = pyzEntryEncrypted(item, encryptionMarker);
|
||||
if (item.namespace)
|
||||
return {
|
||||
entry: {
|
||||
path: path,
|
||||
kind: "directory",
|
||||
declaredSize: 0,
|
||||
compressedSize: item.length,
|
||||
executable: false,
|
||||
encrypted,
|
||||
byteOffset: null,
|
||||
declaredIntegrity: null,
|
||||
unpacked: false,
|
||||
limitations: [
|
||||
"Namespace package has no importable bytecode; its declared payload is not materialized.",
|
||||
...(encrypted
|
||||
? [
|
||||
"PYZ namespace payload is declared encrypted and is not materialized.",
|
||||
]
|
||||
: []),
|
||||
],
|
||||
adapterKey: `pyz:${String(container.index)}:${String(item.index)}:${path}`,
|
||||
formatMetadata: pyzEntryMetadata(
|
||||
container,
|
||||
item,
|
||||
undefined,
|
||||
encryptionMarker,
|
||||
),
|
||||
},
|
||||
read: undefined,
|
||||
};
|
||||
const header =
|
||||
!encrypted &&
|
||||
pythonMagic !== undefined &&
|
||||
(item.typeCode === 0 || item.typeCode === 1)
|
||||
? reconstructedPycHeader(pythonMagic)
|
||||
: undefined;
|
||||
const limitations: string[] = [];
|
||||
if (encrypted)
|
||||
limitations.push("PYZ entry is encrypted and was not decoded.");
|
||||
if (header !== undefined)
|
||||
limitations.push(RECONSTRUCTED_PYTHON_HEADER_LIMITATION);
|
||||
limitations.push(
|
||||
"PYZ tables declare compressed lengths; the uncompressed size is unknown until decoding.",
|
||||
);
|
||||
return {
|
||||
entry: fileEntry({
|
||||
path: path,
|
||||
declaredSize: null,
|
||||
compressedSize: item.length,
|
||||
byteOffset: null,
|
||||
encrypted,
|
||||
limitations,
|
||||
adapterKey: `pyz:${String(container.index)}:${String(item.index)}:${path}`,
|
||||
formatMetadata: pyzEntryMetadata(
|
||||
container,
|
||||
item,
|
||||
header,
|
||||
encryptionMarker,
|
||||
),
|
||||
}),
|
||||
read:
|
||||
range === undefined
|
||||
? undefined
|
||||
: {
|
||||
kind: "pyz",
|
||||
...range,
|
||||
compressed: true,
|
||||
uncompressedSize: undefined,
|
||||
header,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* CArchive and PYZ data names treat `\` as a separator. The logical path is
|
||||
* canonical; the stored spelling is kept when that conversion changes it.
|
||||
* `source_location` is an offset plus declared size, so a compressed member
|
||||
* publishes a file span only when that size is the physical extent. Otherwise
|
||||
* the span stays unknown, as ZIP does for compressed entries.
|
||||
*/
|
||||
const canonicalMemberPath = (storedName: string): string =>
|
||||
storedName.replaceAll("\\", "/");
|
||||
|
||||
const sourceByteOffset = (
|
||||
entry: CArchiveEntry,
|
||||
archiveStart: number,
|
||||
): number | null =>
|
||||
entry.compressionFlag === 0 ? archiveStart + entry.offset : null;
|
||||
|
||||
const pyzMemberPath = (container: string, item: PyzEntry): string => {
|
||||
if (item.typeCode === 2)
|
||||
return `${container}/${canonicalMemberPath(item.name)}`;
|
||||
const dotted = item.name.replaceAll(".", "/");
|
||||
if (item.namespace) return `${container}/${dotted}`;
|
||||
if (item.package) return `${container}/${dotted}/__init__.pyc`;
|
||||
return `${container}/${dotted}.pyc`;
|
||||
};
|
||||
|
||||
const isBytecodeType = (typeCode: string): boolean =>
|
||||
typeCode === "s" || typeCode === "m" || typeCode === "M";
|
||||
|
||||
const fileEntry = (input: {
|
||||
readonly path: string;
|
||||
readonly declaredSize: number | null;
|
||||
readonly compressedSize: number;
|
||||
readonly byteOffset: number | null;
|
||||
readonly limitations: readonly string[];
|
||||
readonly adapterKey: string;
|
||||
readonly formatMetadata: PyInstallerEntryMetadata;
|
||||
readonly encrypted?: boolean;
|
||||
readonly kind?: "file" | "symlink" | "metadata";
|
||||
}): ArtifactEntry => ({
|
||||
path: input.path,
|
||||
kind: input.kind ?? "file",
|
||||
declaredSize: input.declaredSize,
|
||||
compressedSize: input.compressedSize,
|
||||
executable: false,
|
||||
encrypted: input.encrypted === true,
|
||||
byteOffset: input.byteOffset,
|
||||
declaredIntegrity: null,
|
||||
unpacked: false,
|
||||
limitations: input.limitations,
|
||||
adapterKey: input.adapterKey,
|
||||
formatMetadata: input.formatMetadata,
|
||||
});
|
||||
|
||||
/** Stable entry stored for a later open(). */
|
||||
export const sealMember = (member: PyInstallerMember): PyInstallerMember => ({
|
||||
...member,
|
||||
entry: copyArtifactEntry(member.entry),
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { readMarshalSubset } from "./MarshalSubset.js";
|
||||
|
||||
const bytes = (hex: string): Buffer => Buffer.from(hex, "hex");
|
||||
|
||||
it("reads a Python 3.9 reference-flagged PYZ table", () => {
|
||||
const parsed = readMarshalSubset(
|
||||
bytes(
|
||||
"db04000000a902da0568656c6c6fa903e900000000e90a000000e914000000a902da03706b67a903e901000000e91e000000e928000000a902fa0c6a736f6e2e6465636f646572a9037204000000e964000000e932000000a902da026e73a903e90300000072040000007204000000",
|
||||
),
|
||||
);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value).toEqual([
|
||||
["hello", [0, 10, 20]],
|
||||
["pkg", [1, 30, 40]],
|
||||
["json.decoder", [0, 100, 50]],
|
||||
["ns", [3, 0, 0]],
|
||||
]);
|
||||
});
|
||||
|
||||
it("resolves a shared string through a reference reserved before the tuple", () => {
|
||||
const parsed = readMarshalSubset(
|
||||
bytes(
|
||||
"db020000002902da06736861726564a903e900000000e901000000e90200000029027201000000a9037204000000e903000000e904000000",
|
||||
),
|
||||
);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value).toEqual([
|
||||
["shared", [0, 1, 2]],
|
||||
["shared", [1, 3, 4]],
|
||||
]);
|
||||
});
|
||||
|
||||
it("reads a long integer that fits in a safe integer", () => {
|
||||
const parsed = readMarshalSubset(
|
||||
bytes(
|
||||
"5b01000000a902da03626967a903e900000000ec03000000000000000004e905000000",
|
||||
),
|
||||
);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value).toEqual([["big", [0, 2 ** 40, 5]]]);
|
||||
});
|
||||
|
||||
it("reads a negative 32-bit integer", () => {
|
||||
const parsed = readMarshalSubset(
|
||||
bytes("5b01000000a902da016ea903e900000000e9ffffffffe901000000"),
|
||||
);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value).toEqual([["n", [0, -1, 1]]]);
|
||||
});
|
||||
|
||||
it("rejects a marshal type outside the subset", () => {
|
||||
const parsed = readMarshalSubset(Buffer.from([0x4e]));
|
||||
expect(parsed.ok).toBe(false);
|
||||
if (parsed.ok) return;
|
||||
expect(parsed.error.reason).toBe("format");
|
||||
});
|
||||
@@ -0,0 +1,282 @@
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import {
|
||||
pyInstallerFailure,
|
||||
type PyInstallerFailure,
|
||||
} from "./PyInstallerFailure.js";
|
||||
|
||||
const MAX_DEPTH = 16;
|
||||
const MAX_NODES = 100_000;
|
||||
const MAX_SEQUENCE = 100_000;
|
||||
const MAX_STRING_BYTES = 65_536;
|
||||
const FLAG_REF = 0x80;
|
||||
|
||||
/** Values the PYZ table of contents is allowed to contain. */
|
||||
export type MarshalValue =
|
||||
| number
|
||||
| string
|
||||
| Uint8Array
|
||||
| readonly MarshalValue[];
|
||||
|
||||
interface MarshalCursor {
|
||||
readonly bytes: Buffer;
|
||||
offset: number;
|
||||
readonly refs: Array<MarshalValue | undefined>;
|
||||
depth: number;
|
||||
nodes: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read one marshal value using only list, tuple, str, int, and bytes.
|
||||
* Container slots are reserved before their elements so later references match
|
||||
* CPython. Any other type fails the whole value.
|
||||
*/
|
||||
export const readMarshalSubset = (
|
||||
bytes: Buffer,
|
||||
): Result<MarshalValue, PyInstallerFailure> => {
|
||||
const cursor: MarshalCursor = {
|
||||
bytes,
|
||||
offset: 0,
|
||||
refs: [],
|
||||
depth: 0,
|
||||
nodes: 0,
|
||||
};
|
||||
const value = readObject(cursor);
|
||||
if (!value.ok) return value;
|
||||
if (cursor.offset !== bytes.length)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"PYZ marshal value did not consume its table of contents",
|
||||
),
|
||||
);
|
||||
return value;
|
||||
};
|
||||
|
||||
const readObject = (
|
||||
cursor: MarshalCursor,
|
||||
): Result<MarshalValue, PyInstallerFailure> => {
|
||||
if (cursor.depth > MAX_DEPTH || cursor.nodes >= MAX_NODES)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"limit",
|
||||
"PYZ marshal value exceeds the depth or object bound",
|
||||
),
|
||||
);
|
||||
cursor.nodes += 1;
|
||||
const code = readByte(cursor);
|
||||
if (!code.ok) return code;
|
||||
const flag = (code.value & FLAG_REF) !== 0;
|
||||
const type = code.value & ~FLAG_REF;
|
||||
if (type === 0x69) return storeScalar(cursor, flag, readInt32(cursor));
|
||||
if (type === 0x6c) return storeScalar(cursor, flag, readLong(cursor));
|
||||
if (type === 0x7a || type === 0x5a)
|
||||
return storeScalar(cursor, flag, readShortAscii(cursor));
|
||||
if (type === 0x61 || type === 0x41)
|
||||
return storeScalar(cursor, flag, readCountedAscii(cursor));
|
||||
if (type === 0x75) return storeScalar(cursor, flag, readUtf8(cursor));
|
||||
if (type === 0x73) return storeScalar(cursor, flag, readByteString(cursor));
|
||||
if (type === 0x28) return readSequence(cursor, flag, readInt32(cursor));
|
||||
if (type === 0x29) return readSequence(cursor, flag, readByte(cursor));
|
||||
if (type === 0x5b) return readSequence(cursor, flag, readInt32(cursor));
|
||||
if (type === 0x72) return readReference(cursor, flag);
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"PYZ marshal value uses a type outside the supported subset",
|
||||
),
|
||||
);
|
||||
};
|
||||
|
||||
const storeScalar = (
|
||||
cursor: MarshalCursor,
|
||||
flag: boolean,
|
||||
value: Result<MarshalValue, PyInstallerFailure>,
|
||||
): Result<MarshalValue, PyInstallerFailure> => {
|
||||
if (!value.ok) return value;
|
||||
if (flag) cursor.refs.push(value.value);
|
||||
return value;
|
||||
};
|
||||
|
||||
const readSequence = (
|
||||
cursor: MarshalCursor,
|
||||
flag: boolean,
|
||||
count: Result<number, PyInstallerFailure>,
|
||||
): Result<MarshalValue, PyInstallerFailure> => {
|
||||
if (!count.ok) return count;
|
||||
if (count.value < 0 || count.value > MAX_SEQUENCE)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"limit",
|
||||
"PYZ marshal sequence exceeds the supported count",
|
||||
),
|
||||
);
|
||||
const slot = flag ? cursor.refs.length : undefined;
|
||||
if (slot !== undefined) cursor.refs.push(undefined);
|
||||
const items: MarshalValue[] = [];
|
||||
cursor.depth += 1;
|
||||
for (let index = 0; index < count.value; index += 1) {
|
||||
const item = readObject(cursor);
|
||||
if (!item.ok) return item;
|
||||
items.push(item.value);
|
||||
}
|
||||
cursor.depth -= 1;
|
||||
if (slot !== undefined) cursor.refs[slot] = items;
|
||||
return ok(items);
|
||||
};
|
||||
|
||||
const readReference = (
|
||||
cursor: MarshalCursor,
|
||||
flag: boolean,
|
||||
): Result<MarshalValue, PyInstallerFailure> => {
|
||||
const index = readInt32(cursor);
|
||||
if (!index.ok) return index;
|
||||
const value = cursor.refs[index.value];
|
||||
if (value === undefined)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"PYZ marshal reference does not point at a completed value",
|
||||
),
|
||||
);
|
||||
if (flag) cursor.refs.push(value);
|
||||
return ok(value);
|
||||
};
|
||||
|
||||
const readShortAscii = (
|
||||
cursor: MarshalCursor,
|
||||
): Result<string, PyInstallerFailure> => {
|
||||
const length = readByte(cursor);
|
||||
if (!length.ok) return length;
|
||||
return readAsciiBytes(cursor, length.value);
|
||||
};
|
||||
|
||||
const readCountedAscii = (
|
||||
cursor: MarshalCursor,
|
||||
): Result<string, PyInstallerFailure> => {
|
||||
const length = readInt32(cursor);
|
||||
if (!length.ok) return length;
|
||||
return readAsciiBytes(cursor, length.value);
|
||||
};
|
||||
|
||||
const readAsciiBytes = (
|
||||
cursor: MarshalCursor,
|
||||
length: number,
|
||||
): Result<string, PyInstallerFailure> => {
|
||||
const bytes = readBytes(cursor, length);
|
||||
if (!bytes.ok) return bytes;
|
||||
if (bytes.value.some((byte) => byte > 0x7f))
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"PYZ marshal ASCII string contains a non-ASCII byte",
|
||||
),
|
||||
);
|
||||
return ok(bytes.value.toString("ascii"));
|
||||
};
|
||||
|
||||
const readUtf8 = (
|
||||
cursor: MarshalCursor,
|
||||
): Result<string, PyInstallerFailure> => {
|
||||
const length = readInt32(cursor);
|
||||
if (!length.ok) return length;
|
||||
const bytes = readBytes(cursor, length.value);
|
||||
if (!bytes.ok) return bytes;
|
||||
try {
|
||||
return ok(
|
||||
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
bytes.value,
|
||||
),
|
||||
);
|
||||
} catch {
|
||||
return err(
|
||||
pyInstallerFailure("format", "PYZ marshal string is not valid UTF-8"),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const readByteString = (
|
||||
cursor: MarshalCursor,
|
||||
): Result<Uint8Array, PyInstallerFailure> => {
|
||||
const length = readInt32(cursor);
|
||||
if (!length.ok) return length;
|
||||
const bytes = readBytes(cursor, length.value);
|
||||
if (!bytes.ok) return bytes;
|
||||
return ok(bytes.value);
|
||||
};
|
||||
|
||||
const readLong = (
|
||||
cursor: MarshalCursor,
|
||||
): Result<number, PyInstallerFailure> => {
|
||||
const digits = readInt32(cursor);
|
||||
if (!digits.ok) return digits;
|
||||
const count = Math.abs(digits.value);
|
||||
if (count > 8)
|
||||
return err(
|
||||
pyInstallerFailure("limit", "PYZ marshal integer exceeds the safe range"),
|
||||
);
|
||||
let magnitude = 0n;
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
const digit = readUint16(cursor);
|
||||
if (!digit.ok) return digit;
|
||||
if (digit.value > 0x7fff)
|
||||
return err(
|
||||
pyInstallerFailure("format", "PYZ marshal long digit is out of range"),
|
||||
);
|
||||
magnitude += BigInt(digit.value) << BigInt(15 * index);
|
||||
}
|
||||
if (magnitude > BigInt(Number.MAX_SAFE_INTEGER))
|
||||
return err(
|
||||
pyInstallerFailure("limit", "PYZ marshal integer exceeds the safe range"),
|
||||
);
|
||||
const signed = digits.value < 0 ? -Number(magnitude) : Number(magnitude);
|
||||
return ok(signed);
|
||||
};
|
||||
|
||||
const readBytes = (
|
||||
cursor: MarshalCursor,
|
||||
length: number,
|
||||
): Result<Buffer, PyInstallerFailure> => {
|
||||
if (!Number.isSafeInteger(length) || length < 0 || length > MAX_STRING_BYTES)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"limit",
|
||||
"PYZ marshal string exceeds the supported length",
|
||||
),
|
||||
);
|
||||
const end = cursor.offset + length;
|
||||
if (end > cursor.bytes.length)
|
||||
return err(pyInstallerFailure("format", "PYZ marshal value is truncated"));
|
||||
const bytes = cursor.bytes.subarray(cursor.offset, end);
|
||||
cursor.offset = end;
|
||||
return ok(bytes);
|
||||
};
|
||||
|
||||
const readByte = (
|
||||
cursor: MarshalCursor,
|
||||
): Result<number, PyInstallerFailure> => {
|
||||
if (cursor.offset >= cursor.bytes.length)
|
||||
return err(pyInstallerFailure("format", "PYZ marshal value is truncated"));
|
||||
const value = cursor.bytes[cursor.offset] ?? 0;
|
||||
cursor.offset += 1;
|
||||
return ok(value);
|
||||
};
|
||||
|
||||
const readInt32 = (
|
||||
cursor: MarshalCursor,
|
||||
): Result<number, PyInstallerFailure> => {
|
||||
if (cursor.offset + 4 > cursor.bytes.length)
|
||||
return err(pyInstallerFailure("format", "PYZ marshal value is truncated"));
|
||||
const value = cursor.bytes.readInt32LE(cursor.offset);
|
||||
cursor.offset += 4;
|
||||
return ok(value);
|
||||
};
|
||||
|
||||
const readUint16 = (
|
||||
cursor: MarshalCursor,
|
||||
): Result<number, PyInstallerFailure> => {
|
||||
if (cursor.offset + 2 > cursor.bytes.length)
|
||||
return err(pyInstallerFailure("format", "PYZ marshal value is truncated"));
|
||||
const value = cursor.bytes.readUInt16LE(cursor.offset);
|
||||
cursor.offset += 2;
|
||||
return ok(value);
|
||||
};
|
||||
@@ -0,0 +1,139 @@
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { withPyInstallerReader } from "../../../tests/fixtures/pyinstallerReader.js";
|
||||
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
|
||||
import type { ArtifactEntry } from "../ArtifactReader.js";
|
||||
import { PyInstallerPyzStore } from "./PyInstallerPyzStore.js";
|
||||
import {
|
||||
cArchiveExecutable,
|
||||
pyzArchive,
|
||||
PYTHON_39_MAGIC,
|
||||
} from "./CArchive.fixture.js";
|
||||
|
||||
const compressedImages = (count: number, modules = false) =>
|
||||
cArchiveExecutable({
|
||||
layout: "2.1",
|
||||
pythonVersion: 309,
|
||||
members: Array.from({ length: count }, (_, index) => ({
|
||||
name: `PYZ-${index}.pyz`,
|
||||
typeCode: "z",
|
||||
compress: true,
|
||||
payload: pyzArchive({
|
||||
magic: PYTHON_39_MAGIC,
|
||||
modules: modules
|
||||
? [{ name: "app", typeCode: 0, body: Buffer.from("body") }]
|
||||
: [],
|
||||
}),
|
||||
})),
|
||||
});
|
||||
|
||||
it("refuses excess private images before creating any temporary file", async () => {
|
||||
const spool = vi
|
||||
.spyOn(PyInstallerPyzStore.prototype, "spool")
|
||||
.mockRejectedValue(new Error("unexpected private file creation"));
|
||||
try {
|
||||
await withPyInstallerReader(compressedImages(129), async (reader) => {
|
||||
await expect(
|
||||
reader.entries()[Symbol.asyncIterator]().next(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
boundary: "capacity",
|
||||
});
|
||||
expect(spool).not.toHaveBeenCalled();
|
||||
});
|
||||
} finally {
|
||||
spool.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts the private-image capacity and retires all image owners", async () => {
|
||||
const spool = vi.spyOn(PyInstallerPyzStore.prototype, "spool");
|
||||
try {
|
||||
await withPyInstallerReader(compressedImages(128), async (reader, root) => {
|
||||
const listed: ArtifactEntry[] = [];
|
||||
for await (const entry of reader.entries()) listed.push(entry);
|
||||
expect(listed).toHaveLength(128);
|
||||
expect(spool).toHaveBeenCalledTimes(128);
|
||||
await reader.close();
|
||||
expect(root.fd).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
} finally {
|
||||
spool.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"preserves private-image admission cancellation (failed cleanup: %s)",
|
||||
async (failClose) => {
|
||||
await withPyInstallerReader(
|
||||
compressedImages(1, true),
|
||||
async (reader, root) => {
|
||||
let selected: ArtifactEntry | undefined;
|
||||
for await (const entry of reader.entries())
|
||||
if (entry.path.endsWith("app.pyc")) selected = entry;
|
||||
if (selected === undefined) throw new Error("Missing child");
|
||||
let entered!: () => void, release!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
entered = resolve;
|
||||
});
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const prototype = Object.getPrototypeOf(root) as FileHandle;
|
||||
const nativeStat = prototype.stat,
|
||||
nativeClose = OwnedFileHandle.prototype.close;
|
||||
let admitted: FileHandle | undefined,
|
||||
retained: OwnedFileHandle | undefined,
|
||||
attempts = 0;
|
||||
const cancelled = new Error("cancel private-image admission"),
|
||||
cleanup = new Error("admission cleanup EIO");
|
||||
const signal = new AbortController();
|
||||
const stat = vi
|
||||
.spyOn(prototype, "stat")
|
||||
.mockImplementation(async function (
|
||||
this: FileHandle,
|
||||
...args: Parameters<FileHandle["stat"]>
|
||||
) {
|
||||
if (this !== root && admitted === undefined) {
|
||||
admitted = this;
|
||||
entered();
|
||||
await gate;
|
||||
}
|
||||
return Reflect.apply(nativeStat, this, args);
|
||||
});
|
||||
const close = vi
|
||||
.spyOn(OwnedFileHandle.prototype, "close")
|
||||
.mockImplementation(async function (this: OwnedFileHandle) {
|
||||
if (this.handle === admitted) {
|
||||
retained = this;
|
||||
attempts++;
|
||||
if (failClose && attempts === 1) throw cleanup;
|
||||
}
|
||||
await nativeClose.call(this);
|
||||
});
|
||||
const opening = reader.open(selected, signal.signal);
|
||||
void opening.catch(() => undefined);
|
||||
try {
|
||||
await started;
|
||||
signal.abort(cancelled);
|
||||
release();
|
||||
await expect(opening).rejects.toMatchObject({
|
||||
reason: "cancelled",
|
||||
cancellation: { reason: cancelled },
|
||||
...(failClose ? { cleanup: { reason: cleanup.message } } : {}),
|
||||
});
|
||||
if (failClose) expect(retained?.handle.fd).toBeGreaterThanOrEqual(0);
|
||||
await reader.close();
|
||||
expect(attempts).toBe(failClose ? 2 : 1);
|
||||
expect(admitted?.fd).toBe(-1);
|
||||
expect(root.fd).toBeGreaterThanOrEqual(0);
|
||||
} finally {
|
||||
release();
|
||||
await opening.catch(() => undefined);
|
||||
stat.mockRestore();
|
||||
close.mockRestore();
|
||||
}
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,673 @@
|
||||
import { withPyInstallerReader } from "../../../tests/fixtures/pyinstallerReader.js";
|
||||
import { join } from "node:path";
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import { createHash } from "node:crypto";
|
||||
import { writeFile, type FileHandle } from "node:fs/promises";
|
||||
import { buffer } from "node:stream/consumers";
|
||||
import { deflateSync } from "node:zlib";
|
||||
import { setImmediate } from "node:timers/promises";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js";
|
||||
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
|
||||
import { hashReadable } from "../ArtifactHash.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
} from "../ArtifactReader.js";
|
||||
import {
|
||||
cArchiveExecutable,
|
||||
pyzArchive,
|
||||
PYTHON_39_MAGIC,
|
||||
} from "./CArchive.fixture.js";
|
||||
|
||||
const hash = (bytes: Buffer): string =>
|
||||
createHash("sha256").update(bytes).digest("hex");
|
||||
const archive = (body: Buffer) =>
|
||||
cArchiveExecutable({
|
||||
layout: "2.1",
|
||||
pythonVersion: 309,
|
||||
members: [{ name: "data", typeCode: "x", payload: body, compress: false }],
|
||||
});
|
||||
const members = async (reader: PyInstallerArtifactReader) => {
|
||||
const entries: ArtifactEntry[] = [];
|
||||
for await (const entry of reader.entries()) entries.push(entry);
|
||||
return entries;
|
||||
};
|
||||
const deferred = () => {
|
||||
let resolve: () => void = () => undefined;
|
||||
const promise = new Promise<void>((done) => {
|
||||
resolve = done;
|
||||
});
|
||||
return { promise, resolve };
|
||||
};
|
||||
|
||||
it.each([false, true])(
|
||||
"settles independent private and carrier closes and retries only failed owners (carrier failure: %s)",
|
||||
async (failCarrier) => {
|
||||
const path = join(await createTestTempDirectory("rea-pyi-cleanup-"), "app");
|
||||
await writeFile(
|
||||
path,
|
||||
cArchiveExecutable({
|
||||
layout: "2.1",
|
||||
pythonVersion: 309,
|
||||
members: ["one", "two"].map((name) => ({
|
||||
name: `${name}.pyz`,
|
||||
typeCode: "z",
|
||||
compress: true,
|
||||
payload: pyzArchive({
|
||||
magic: PYTHON_39_MAGIC,
|
||||
modules: [{ name: "app", typeCode: 0, body: Buffer.from(name) }],
|
||||
}),
|
||||
})),
|
||||
}),
|
||||
);
|
||||
const reader = new PyInstallerArtifactReader(path);
|
||||
const entries = (await members(reader)).filter((entry) =>
|
||||
entry.path.endsWith("app.pyc"),
|
||||
);
|
||||
expect(entries).toHaveLength(2);
|
||||
const nativeClose = OwnedFileHandle.prototype.close;
|
||||
const privateOwners = new Set<OwnedFileHandle>();
|
||||
const attempts: OwnedFileHandle[] = [];
|
||||
let collecting = true;
|
||||
let failing = true;
|
||||
const close = vi
|
||||
.spyOn(OwnedFileHandle.prototype, "close")
|
||||
.mockImplementation(async function (this: OwnedFileHandle) {
|
||||
attempts.push(this);
|
||||
if (collecting) privateOwners.add(this);
|
||||
if (failing && (privateOwners.has(this) || failCarrier))
|
||||
throw new Error(
|
||||
privateOwners.has(this) ? "private close EIO" : "carrier close EIO",
|
||||
);
|
||||
await nativeClose.call(this);
|
||||
});
|
||||
try {
|
||||
for (const entry of entries)
|
||||
await expect(buffer(await reader.open(entry))).rejects.toMatchObject({
|
||||
cleanup: { reason: "private close EIO" },
|
||||
});
|
||||
expect(privateOwners.size).toBe(2);
|
||||
collecting = false;
|
||||
attempts.length = 0;
|
||||
const failure = await reader.close().then(
|
||||
() => undefined,
|
||||
(cause: unknown) => cause,
|
||||
);
|
||||
expect(attempts).toHaveLength(3);
|
||||
expect(new Set(attempts).size).toBe(3);
|
||||
expect(failure).toMatchObject({
|
||||
cleanup: {
|
||||
resources: expect.arrayContaining(failCarrier ? [path] : []),
|
||||
},
|
||||
});
|
||||
expect(
|
||||
(failure as ArtifactReaderFailure).cleanup?.resources,
|
||||
).toHaveLength(failCarrier ? 3 : 2);
|
||||
expect((failure as ArtifactReaderFailure).cleanup?.reason).toContain(
|
||||
"private close EIO",
|
||||
);
|
||||
if (failCarrier)
|
||||
expect((failure as ArtifactReaderFailure).cleanup?.reason).toContain(
|
||||
"carrier close EIO",
|
||||
);
|
||||
const carrier = attempts.find((owner) => !privateOwners.has(owner))!;
|
||||
expect(carrier.handle.fd >= 0).toBe(failCarrier);
|
||||
for (const owner of privateOwners)
|
||||
expect(owner.handle.fd).toBeGreaterThanOrEqual(0);
|
||||
failing = false;
|
||||
attempts.length = 0;
|
||||
await reader.close();
|
||||
expect(attempts).toHaveLength(failCarrier ? 3 : 2);
|
||||
for (const owner of [...privateOwners, carrier])
|
||||
expect(owner.handle.fd).toBe(-1);
|
||||
attempts.length = 0;
|
||||
await reader.close();
|
||||
expect(attempts).toEqual([]);
|
||||
} finally {
|
||||
close.mockRestore();
|
||||
await reader.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("binds materialization to the inventoried digest and checks cached carrier freshness", async () => {
|
||||
const bytes = archive(Buffer.from("first"));
|
||||
await withPyInstallerReader(bytes, async (reader, _handle, path) => {
|
||||
await expect(reader.prepareContainer("0".repeat(64))).rejects.toMatchObject(
|
||||
{ reason: "integrity" },
|
||||
);
|
||||
await reader.prepareContainer(hash(bytes));
|
||||
await reader.prepareContainer(hash(bytes));
|
||||
await writeFile(path, archive(Buffer.from("other")));
|
||||
await expect(reader.prepareContainer(hash(bytes))).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("bounds descriptor reads while hashing a selected stored member", async () => {
|
||||
const payload = Buffer.alloc(2 * 1024 * 1024, 0x61);
|
||||
await withPyInstallerReader(archive(payload), async (reader, handle) => {
|
||||
const entry = (await members(reader))[0]!;
|
||||
const read = handle.read.bind(handle);
|
||||
let maximum = 0;
|
||||
const spy = vi
|
||||
.spyOn(handle, "read")
|
||||
.mockImplementation((...args: unknown[]) => {
|
||||
if (typeof args[2] === "number") maximum = Math.max(maximum, args[2]);
|
||||
return Reflect.apply(read, handle, args);
|
||||
});
|
||||
try {
|
||||
const observed = await hashReadable(await reader.open(entry));
|
||||
expect(observed.sha256).toBe(hash(payload));
|
||||
expect(observed.bytes).toBe(payload.length);
|
||||
expect(maximum).toBeGreaterThan(0);
|
||||
expect(maximum).toBeLessThanOrEqual(64 * 1024);
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"joins pending native reads during cancellation/close (I/O failure: %s)",
|
||||
async (fail) => {
|
||||
await withPyInstallerReader(
|
||||
archive(Buffer.alloc(256 * 1024, 0x61)),
|
||||
async (reader, handle) => {
|
||||
const entry = (await members(reader))[0]!;
|
||||
const entered = deferred(),
|
||||
release = deferred();
|
||||
const read = handle.read.bind(handle);
|
||||
const io = Object.assign(new Error("native read failed"), {
|
||||
code: "EIO",
|
||||
});
|
||||
const caller = new Error("caller stopped reading");
|
||||
const controller = new AbortController();
|
||||
let held = false;
|
||||
const spy = vi
|
||||
.spyOn(handle, "read")
|
||||
.mockImplementation(async (...args: unknown[]) => {
|
||||
if (!held) {
|
||||
held = true;
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
if (fail) throw io;
|
||||
}
|
||||
return Reflect.apply(read, handle, args);
|
||||
});
|
||||
const consumed = reader
|
||||
.open(entry, controller.signal)
|
||||
.then((stream) => buffer(stream));
|
||||
void consumed.catch(() => undefined);
|
||||
let closing: Promise<void> | undefined;
|
||||
try {
|
||||
await entered.promise;
|
||||
controller.abort(caller);
|
||||
let closed = false;
|
||||
closing = reader.close().then(() => {
|
||||
closed = true;
|
||||
});
|
||||
await setImmediate();
|
||||
expect(closed).toBe(false);
|
||||
expect(handle.fd).toBeGreaterThanOrEqual(0);
|
||||
release.resolve();
|
||||
await expect(consumed).rejects.toMatchObject({
|
||||
reason: fail ? "io" : "cancelled",
|
||||
cause: fail ? io : caller,
|
||||
});
|
||||
await closing;
|
||||
expect(handle.fd).toBeGreaterThanOrEqual(0);
|
||||
} finally {
|
||||
release.resolve();
|
||||
await Promise.allSettled([
|
||||
consumed,
|
||||
...(closing === undefined ? [] : [closing]),
|
||||
]);
|
||||
spy.mockRestore();
|
||||
}
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves BOM characters in length-delimited UTF-8 fields", async () => {
|
||||
const bytes = cArchiveExecutable({
|
||||
layout: "2.1",
|
||||
pythonVersion: 309,
|
||||
pythonLibrary: "\ufefflibpython.so",
|
||||
members: [
|
||||
{
|
||||
name: "\ufeffasset",
|
||||
typeCode: "x",
|
||||
payload: Buffer.from("data"),
|
||||
compress: false,
|
||||
},
|
||||
{
|
||||
name: "PYZ.pyz",
|
||||
typeCode: "z",
|
||||
compress: false,
|
||||
payload: pyzArchive({
|
||||
magic: PYTHON_39_MAGIC,
|
||||
modules: [
|
||||
{ name: "\ufeffmodule", typeCode: 0, body: Buffer.from("module") },
|
||||
],
|
||||
}),
|
||||
},
|
||||
],
|
||||
});
|
||||
await withPyInstallerReader(bytes, async (reader) => {
|
||||
const entries = await members(reader);
|
||||
expect(reader.formatMetadata()).toMatchObject({
|
||||
python_library: "\ufefflibpython.so",
|
||||
});
|
||||
expect(entries.map((entry) => entry.path)).toContain("\ufeffasset");
|
||||
expect(
|
||||
entries.find((entry) => entry.path === "\ufeffasset")?.formatMetadata,
|
||||
).toMatchObject({ source_name: "\ufeffasset" });
|
||||
expect(
|
||||
entries.find((entry) => entry.path === "PYZ.pyz/\ufeffmodule.pyc")
|
||||
?.formatMetadata,
|
||||
).toMatchObject({ source_name: "\ufeffmodule" });
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps runtime options as declarations with literal names and distinct source indices", async () => {
|
||||
const option = "pyi-runtime-tmpdir /tmp/../chosen";
|
||||
await withPyInstallerReader(
|
||||
cArchiveExecutable({
|
||||
layout: "2.1",
|
||||
pythonVersion: 309,
|
||||
members: [
|
||||
{
|
||||
name: option,
|
||||
typeCode: "o",
|
||||
payload: Buffer.alloc(0),
|
||||
compress: false,
|
||||
},
|
||||
{
|
||||
name: option,
|
||||
typeCode: "o",
|
||||
payload: Buffer.alloc(0),
|
||||
compress: false,
|
||||
},
|
||||
{
|
||||
name: "data",
|
||||
typeCode: "x",
|
||||
payload: Buffer.from("data"),
|
||||
compress: false,
|
||||
},
|
||||
],
|
||||
}),
|
||||
async (reader) => {
|
||||
const entries = await members(reader);
|
||||
const options = entries.filter((entry) => entry.kind === "metadata");
|
||||
expect(options.map((entry) => entry.path)).toEqual([
|
||||
"__pyinstaller__/runtime-options/0",
|
||||
"__pyinstaller__/runtime-options/1",
|
||||
]);
|
||||
expect(options.map((entry) => entry.formatMetadata)).toMatchObject([
|
||||
{ source_name: option, index: 0 },
|
||||
{ source_name: option, index: 1 },
|
||||
]);
|
||||
expect(reader.formatMetadata()).toMatchObject({
|
||||
runtime_option_count: 2,
|
||||
entry_count: 3,
|
||||
});
|
||||
await expect(reader.open(options[0]!)).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects metadata expansion before retaining every repeated long container path", async () => {
|
||||
const image = pyzArchive({
|
||||
magic: PYTHON_39_MAGIC,
|
||||
modules: Array.from({ length: 3000 }, (_, index) => ({
|
||||
name: `module${String(index)}`,
|
||||
typeCode: 0,
|
||||
body: Buffer.from("x"),
|
||||
})),
|
||||
});
|
||||
await withPyInstallerReader(
|
||||
cArchiveExecutable({
|
||||
layout: "2.1",
|
||||
pythonVersion: 309,
|
||||
members: [
|
||||
{
|
||||
name: "a".repeat(40000),
|
||||
typeCode: "z",
|
||||
payload: image,
|
||||
compress: false,
|
||||
},
|
||||
],
|
||||
}),
|
||||
async (reader) => {
|
||||
let failure: unknown;
|
||||
try {
|
||||
for await (const entry of reader.entries()) {
|
||||
void entry;
|
||||
}
|
||||
} catch (cause: unknown) {
|
||||
failure = cause;
|
||||
}
|
||||
expect(failure).toBeInstanceOf(ArtifactReaderFailure);
|
||||
expect(failure).toMatchObject({ reason: "limit", boundary: "memory" });
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"validates compressed member bounds through the stream (truncated: %s)",
|
||||
async (truncated) => {
|
||||
const payload = Buffer.from("complete module payload");
|
||||
const encoded = deflateSync(payload);
|
||||
const bytes = cArchiveExecutable({
|
||||
layout: "2.1",
|
||||
pythonVersion: 309,
|
||||
members: [
|
||||
{
|
||||
name: "data",
|
||||
typeCode: "x",
|
||||
payload,
|
||||
compress: true,
|
||||
declaredUncompressedSize: truncated ? payload.length : 4,
|
||||
},
|
||||
],
|
||||
});
|
||||
if (truncated)
|
||||
bytes.writeUInt32BE(encoded.length - 2, 64 + encoded.length + 8);
|
||||
await withPyInstallerReader(bytes, async (reader) => {
|
||||
const entry = (await members(reader))[0]!;
|
||||
await expect(buffer(await reader.open(entry))).rejects.toMatchObject({
|
||||
reason: truncated ? "format" : "limit",
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("joins a late private-file close and retries its failed owner after cancellation", async () => {
|
||||
const bytes = cArchiveExecutable({
|
||||
layout: "2.1",
|
||||
pythonVersion: 309,
|
||||
members: [
|
||||
{
|
||||
name: "PYZ.pyz",
|
||||
typeCode: "z",
|
||||
payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }),
|
||||
compress: true,
|
||||
},
|
||||
],
|
||||
});
|
||||
await withPyInstallerReader(bytes, async (reader, borrowed) => {
|
||||
const entered = deferred(),
|
||||
release = deferred();
|
||||
const nativeClose = OwnedFileHandle.prototype.close;
|
||||
const failure = new Error("private final close failed");
|
||||
let owner: OwnedFileHandle | undefined;
|
||||
let attempts = 0;
|
||||
const spy = vi
|
||||
.spyOn(OwnedFileHandle.prototype, "close")
|
||||
.mockImplementation(async function (this: OwnedFileHandle) {
|
||||
if (owner === undefined) {
|
||||
owner = this;
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
throw failure;
|
||||
}
|
||||
if (this === owner) attempts += 1;
|
||||
await nativeClose.call(this);
|
||||
});
|
||||
const signal = new AbortController();
|
||||
const listing = reader
|
||||
.entries(signal.signal)
|
||||
[Symbol.asyncIterator]()
|
||||
.next()
|
||||
.catch((cause: unknown) => cause);
|
||||
try {
|
||||
await entered.promise;
|
||||
let settled = false;
|
||||
const closing = reader.close().then(() => {
|
||||
settled = true;
|
||||
});
|
||||
signal.abort(new Error("cancel during final cleanup"));
|
||||
await setImmediate();
|
||||
expect(settled).toBe(false);
|
||||
expect(owner?.handle.fd).toBeGreaterThanOrEqual(0);
|
||||
release.resolve();
|
||||
expect(await listing).toBe(failure);
|
||||
await closing;
|
||||
expect(attempts).toBe(1);
|
||||
expect(owner?.handle.fd).toBe(-1);
|
||||
expect(borrowed.fd).toBeGreaterThanOrEqual(0);
|
||||
} finally {
|
||||
release.resolve();
|
||||
spy.mockRestore();
|
||||
await listing;
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
for (const cancel of [false, true]) {
|
||||
it.each([false, true])(
|
||||
`awaits private member close before completion (cancel: ${cancel}, close failure: %s)`,
|
||||
async (failClose) => {
|
||||
const bytes = cArchiveExecutable({
|
||||
layout: "2.1",
|
||||
pythonVersion: 309,
|
||||
members: [
|
||||
{
|
||||
name: "PYZ.pyz",
|
||||
typeCode: "z",
|
||||
compress: true,
|
||||
payload: pyzArchive({
|
||||
magic: PYTHON_39_MAGIC,
|
||||
modules: [
|
||||
{ name: "app", typeCode: 0, body: Buffer.from("body") },
|
||||
],
|
||||
}),
|
||||
},
|
||||
],
|
||||
});
|
||||
await withPyInstallerReader(bytes, async (reader, borrowed) => {
|
||||
const entry = (await members(reader)).find((item) =>
|
||||
item.path.endsWith("app.pyc"),
|
||||
)!;
|
||||
const entered = deferred(),
|
||||
release = deferred();
|
||||
const signal = new AbortController();
|
||||
const failure = new Error("private member final close failed");
|
||||
const nativeClose = OwnedFileHandle.prototype.close;
|
||||
let owner: OwnedFileHandle | undefined;
|
||||
let attempts = 0;
|
||||
const spy = vi
|
||||
.spyOn(OwnedFileHandle.prototype, "close")
|
||||
.mockImplementation(async function (this: OwnedFileHandle) {
|
||||
if (owner === undefined) {
|
||||
owner = this;
|
||||
attempts += 1;
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
if (failClose) throw failure;
|
||||
} else if (this === owner) attempts += 1;
|
||||
await nativeClose.call(this);
|
||||
});
|
||||
const output = await reader.open(entry, signal.signal);
|
||||
const reading = buffer(output).then(
|
||||
(value) => ({ ok: true as const, value }),
|
||||
(cause: unknown) => ({ ok: false as const, cause }),
|
||||
);
|
||||
try {
|
||||
await entered.promise;
|
||||
expect(output.readableEnded).toBe(false);
|
||||
expect(owner?.handle.fd).toBeGreaterThanOrEqual(0);
|
||||
if (cancel) {
|
||||
signal.abort(new Error("cancel during member cleanup"));
|
||||
await setImmediate();
|
||||
expect(output.destroyed).toBe(false);
|
||||
}
|
||||
release.resolve();
|
||||
const result = await reading;
|
||||
if (failClose || cancel) {
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) throw new Error("Expected read failure");
|
||||
expect(result.cause).toMatchObject(
|
||||
failClose
|
||||
? { reason: "io", message: failure.message }
|
||||
: { reason: "cancelled" },
|
||||
);
|
||||
if (failClose) expect(owner?.handle.fd).toBeGreaterThanOrEqual(0);
|
||||
} else {
|
||||
expect(result.ok).toBe(true);
|
||||
if (!result.ok) throw result.cause;
|
||||
expect(result.value.subarray(16)).toEqual(Buffer.from("body"));
|
||||
}
|
||||
await reader.close();
|
||||
expect(attempts).toBe(failClose ? 2 : 1);
|
||||
expect(owner?.handle.fd).toBe(-1);
|
||||
expect(borrowed.fd).toBeGreaterThanOrEqual(0);
|
||||
} finally {
|
||||
release.resolve();
|
||||
await reading;
|
||||
spy.mockRestore();
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
it("preserves a private-image read failure when its descriptor close also fails", async () => {
|
||||
const bytes = cArchiveExecutable({
|
||||
layout: "2.1",
|
||||
pythonVersion: 309,
|
||||
members: [
|
||||
{
|
||||
name: "PYZ.pyz",
|
||||
typeCode: "z",
|
||||
payload: pyzArchive({
|
||||
magic: PYTHON_39_MAGIC,
|
||||
modules: [{ name: "app", typeCode: 0, body: Buffer.from("body") }],
|
||||
}),
|
||||
compress: true,
|
||||
},
|
||||
],
|
||||
});
|
||||
await withPyInstallerReader(bytes, async (reader, borrowed) => {
|
||||
const entry = (await members(reader)).find((item) =>
|
||||
item.path.endsWith("app.pyc"),
|
||||
)!;
|
||||
const nativeClose = OwnedFileHandle.prototype.close;
|
||||
const nativeRead = borrowed.read;
|
||||
const io = Object.assign(new Error("private read EIO"), { code: "EIO" });
|
||||
let failedOwner: OwnedFileHandle | undefined;
|
||||
const read = vi
|
||||
.spyOn(Object.getPrototypeOf(borrowed), "read")
|
||||
.mockImplementation(function (this: FileHandle, ...args: unknown[]) {
|
||||
return this === borrowed
|
||||
? Reflect.apply(nativeRead, this, args)
|
||||
: Promise.reject(io);
|
||||
});
|
||||
const close = vi
|
||||
.spyOn(OwnedFileHandle.prototype, "close")
|
||||
.mockImplementation(async function (this: OwnedFileHandle) {
|
||||
if (this.handle !== borrowed) {
|
||||
failedOwner = this;
|
||||
throw new Error("private close EIO");
|
||||
}
|
||||
await nativeClose.call(this);
|
||||
});
|
||||
try {
|
||||
await expect(buffer(await reader.open(entry))).rejects.toMatchObject({
|
||||
reason: "io",
|
||||
cause: io,
|
||||
});
|
||||
await expect(reader.close()).rejects.toMatchObject({
|
||||
cleanup: { reason: "private close EIO" },
|
||||
});
|
||||
expect(failedOwner?.handle.fd).toBeGreaterThanOrEqual(0);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
close.mockRestore();
|
||||
await reader.close();
|
||||
}
|
||||
expect(failedOwner?.handle.fd).toBe(-1);
|
||||
expect(borrowed.fd).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"shares owned carrier admission across callers and joins late admission (owner closes: %s)",
|
||||
async (ownerCloses) => {
|
||||
const bytes = archive(Buffer.from("shared admission"));
|
||||
await withPyInstallerReader(
|
||||
bytes,
|
||||
async (_borrowedReader, borrowed, path) => {
|
||||
const reader = new PyInstallerArtifactReader(path);
|
||||
const entered = deferred(),
|
||||
release = deferred();
|
||||
const handles = new Set<FileHandle>();
|
||||
const stat = borrowed.stat;
|
||||
const spy = vi
|
||||
.spyOn(Object.getPrototypeOf(borrowed), "stat")
|
||||
.mockImplementation(async function (
|
||||
this: FileHandle,
|
||||
...args: unknown[]
|
||||
) {
|
||||
if (this !== borrowed) {
|
||||
handles.add(this);
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
}
|
||||
return Reflect.apply(stat, this, args);
|
||||
});
|
||||
const caller = new AbortController(),
|
||||
reason = new Error("one admission waiter leaves");
|
||||
const first = reader
|
||||
.prepareContainer(hash(bytes), caller.signal)
|
||||
.catch((cause: unknown) => cause);
|
||||
const peer = reader
|
||||
.prepareContainer(hash(bytes))
|
||||
.catch((cause: unknown) => cause);
|
||||
let closing: Promise<void> | undefined;
|
||||
try {
|
||||
await entered.promise;
|
||||
await setImmediate();
|
||||
expect(handles.size).toBe(1);
|
||||
caller.abort(reason);
|
||||
if (ownerCloses) {
|
||||
let closed = false;
|
||||
closing = reader.close().then(() => {
|
||||
closed = true;
|
||||
});
|
||||
await setImmediate();
|
||||
expect(closed).toBe(false);
|
||||
}
|
||||
release.resolve();
|
||||
expect(await first).toMatchObject({
|
||||
reason: "cancelled",
|
||||
cause: reason,
|
||||
});
|
||||
if (ownerCloses) {
|
||||
expect(await peer).toMatchObject({ reason: "cancelled" });
|
||||
await closing;
|
||||
} else expect(await peer).toBeUndefined();
|
||||
await reader.close();
|
||||
for (const handle of handles) expect(handle.fd).toBe(-1);
|
||||
} finally {
|
||||
release.resolve();
|
||||
await Promise.allSettled([first, peer]);
|
||||
spy.mockRestore();
|
||||
await reader.close();
|
||||
await closing;
|
||||
for (const handle of handles) {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,60 @@
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import { ArtifactReaderFailure } from "../ArtifactReader.js";
|
||||
import { abortIfNeeded } from "../ArtifactHash.js";
|
||||
import {
|
||||
pyInstallerFailure,
|
||||
type PyInstallerFailure,
|
||||
} from "./PyInstallerFailure.js";
|
||||
|
||||
/** Fill one already-budgeted format range; only zero bytes establish EOF. */
|
||||
export const readPyInstallerBytes = async (
|
||||
handle: Pick<FileHandle, "read">,
|
||||
input: {
|
||||
readonly position: number;
|
||||
readonly length: number;
|
||||
readonly label: string;
|
||||
readonly signal?: AbortSignal | undefined;
|
||||
},
|
||||
): Promise<Result<Buffer, PyInstallerFailure>> => {
|
||||
abortIfNeeded(input.signal);
|
||||
if (
|
||||
!Number.isSafeInteger(input.position) ||
|
||||
!Number.isSafeInteger(input.length) ||
|
||||
input.position < 0 ||
|
||||
input.length < 0 ||
|
||||
!Number.isSafeInteger(input.position + input.length)
|
||||
)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
`${input.label} has an invalid source range`,
|
||||
),
|
||||
);
|
||||
const bytes = Buffer.alloc(input.length);
|
||||
let offset = 0;
|
||||
while (offset < bytes.length) {
|
||||
abortIfNeeded(input.signal);
|
||||
let count: number;
|
||||
try {
|
||||
const read = await handle.read(
|
||||
bytes,
|
||||
offset,
|
||||
bytes.length - offset,
|
||||
input.position + offset,
|
||||
);
|
||||
count = read.bytesRead;
|
||||
} catch (cause: unknown) {
|
||||
throw new ArtifactReaderFailure(
|
||||
"io",
|
||||
`${input.label} could not be read`,
|
||||
{ cause },
|
||||
);
|
||||
}
|
||||
abortIfNeeded(input.signal);
|
||||
if (count === 0)
|
||||
return err(pyInstallerFailure("format", `${input.label} is truncated`));
|
||||
offset += count;
|
||||
}
|
||||
return ok(bytes);
|
||||
};
|
||||
@@ -0,0 +1,10 @@
|
||||
/** Typed failure from a pure PyInstaller parser before the reader adapts it. */
|
||||
export interface PyInstallerFailure {
|
||||
readonly reason: "format" | "integrity" | "limit";
|
||||
readonly message: string;
|
||||
}
|
||||
|
||||
export const pyInstallerFailure = (
|
||||
reason: PyInstallerFailure["reason"],
|
||||
message: string,
|
||||
): PyInstallerFailure => ({ reason, message });
|
||||
@@ -0,0 +1,36 @@
|
||||
import {
|
||||
NonRegularFileReadError,
|
||||
RegularFileAdmissionFailure,
|
||||
RegularFileOperationCancelled,
|
||||
} from "../../filesystem/RegularFile.js";
|
||||
import { ArtifactReaderFailure } from "../ArtifactReader.js";
|
||||
|
||||
/** Both carrier and private-image admission preserve cancellation and cleanup. */
|
||||
export const pyInstallerFileAdmissionFailure = (
|
||||
path: string,
|
||||
cause: unknown,
|
||||
): ArtifactReaderFailure => {
|
||||
if (cause instanceof ArtifactReaderFailure) return cause;
|
||||
if (cause instanceof RegularFileOperationCancelled)
|
||||
return new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
"PyInstaller file admission cancelled",
|
||||
{ cause: cause.reason },
|
||||
);
|
||||
if (cause instanceof RegularFileAdmissionFailure)
|
||||
return ArtifactReaderFailure.withCleanup(
|
||||
pyInstallerFileAdmissionFailure(path, cause.cause),
|
||||
ArtifactReaderFailure.cleanupObservation(cause.cleanupCause, path),
|
||||
);
|
||||
if (cause instanceof NonRegularFileReadError)
|
||||
return new ArtifactReaderFailure(
|
||||
"format",
|
||||
`PyInstaller source is not a regular file: ${path}`,
|
||||
{ cause },
|
||||
);
|
||||
return new ArtifactReaderFailure(
|
||||
"io",
|
||||
"PyInstaller archive could not be opened",
|
||||
{ cause },
|
||||
);
|
||||
};
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,147 @@
|
||||
import { Readable, Transform } from "node:stream";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { createInflate } from "node:zlib";
|
||||
import { ArtifactReaderFailure } from "../ArtifactReader.js";
|
||||
import { abortIfNeeded } from "../ArtifactHash.js";
|
||||
|
||||
interface MemberStreamOptions {
|
||||
readonly sourceBytes: number;
|
||||
readonly compressed: boolean;
|
||||
readonly header: Buffer | undefined;
|
||||
readonly expectedBytes: number | undefined;
|
||||
readonly maxBytes: number;
|
||||
readonly signal: AbortSignal;
|
||||
readonly charge?: (bytes: number) => void;
|
||||
readonly complete?: (bytes: number) => Promise<void>;
|
||||
}
|
||||
|
||||
/** Backpressure bounds plaintext independently of the untrusted declared size. */
|
||||
export const pyInstallerMemberStream = (
|
||||
source: Readable,
|
||||
options: MemberStreamOptions,
|
||||
): { readonly output: Readable; readonly completion: Promise<void> } => {
|
||||
let bytes = 0;
|
||||
let headerSent = false;
|
||||
let completing = false;
|
||||
const prependHeader = (): void => {
|
||||
if (!headerSent && options.header !== undefined)
|
||||
output.push(options.header);
|
||||
headerSent = true;
|
||||
};
|
||||
const output = new Transform({
|
||||
transform(chunk: Buffer, _encoding, callback) {
|
||||
try {
|
||||
abortIfNeeded(options.signal);
|
||||
bytes += chunk.length;
|
||||
if (bytes > options.maxBytes)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
"PyInstaller member exceeds its plaintext budget",
|
||||
);
|
||||
options.charge?.(chunk.length);
|
||||
prependHeader();
|
||||
callback(null, chunk);
|
||||
} catch (cause: unknown) {
|
||||
callback(asError(cause));
|
||||
}
|
||||
},
|
||||
flush(callback) {
|
||||
completing = true;
|
||||
void (async () => {
|
||||
abortIfNeeded(options.signal);
|
||||
if (
|
||||
inflate !== undefined &&
|
||||
inflate.bytesWritten !== options.sourceBytes
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"Compressed PyInstaller member did not consume its complete declared source span",
|
||||
);
|
||||
if (
|
||||
options.expectedBytes !== undefined &&
|
||||
bytes !== options.expectedBytes
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"PyInstaller member size disagrees with its declaration or previous observation",
|
||||
);
|
||||
await options.complete?.(bytes);
|
||||
abortIfNeeded(options.signal);
|
||||
prependHeader();
|
||||
})().then(
|
||||
() => {
|
||||
completing = false;
|
||||
try {
|
||||
abortIfNeeded(options.signal);
|
||||
} catch (cause: unknown) {
|
||||
callback(asError(cause));
|
||||
return;
|
||||
}
|
||||
callback();
|
||||
},
|
||||
(cause: unknown) => {
|
||||
completing = false;
|
||||
callback(asError(cause));
|
||||
},
|
||||
);
|
||||
},
|
||||
});
|
||||
// A consumer can retire its reader before attaching its stream error handler.
|
||||
output.on("error", () => undefined);
|
||||
const onAbort = (): void => {
|
||||
// Verification and private-file cleanup own their accepted native I/O.
|
||||
if (completing) return;
|
||||
// The descriptor source owns cancellation until its native read settles.
|
||||
// Retiring it downstream earlier would hide a native I/O rejection.
|
||||
if (!source.readableEnded && !source.destroyed) return;
|
||||
output.destroy(
|
||||
new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
"PyInstaller member read cancelled",
|
||||
{ cause: options.signal.reason },
|
||||
),
|
||||
);
|
||||
};
|
||||
options.signal.addEventListener("abort", onAbort, { once: true });
|
||||
const onSourceFailure = (cause: unknown): void => {
|
||||
output.destroy(
|
||||
cause instanceof ArtifactReaderFailure
|
||||
? cause
|
||||
: new ArtifactReaderFailure(
|
||||
"io",
|
||||
"PyInstaller member source could not be read",
|
||||
{ cause },
|
||||
),
|
||||
);
|
||||
};
|
||||
source.on("error", onSourceFailure);
|
||||
const inflate = options.compressed ? createInflate() : undefined;
|
||||
inflate?.on("error", (cause: unknown) =>
|
||||
output.destroy(
|
||||
new ArtifactReaderFailure(
|
||||
"format",
|
||||
"Compressed PyInstaller member is not a complete zlib stream",
|
||||
{ cause },
|
||||
),
|
||||
),
|
||||
);
|
||||
const completion = (
|
||||
inflate === undefined
|
||||
? pipeline(source, output)
|
||||
: pipeline(source, inflate, output)
|
||||
).finally(() => {
|
||||
options.signal.removeEventListener("abort", onAbort);
|
||||
source.removeListener("error", onSourceFailure);
|
||||
});
|
||||
// Preserve rejection for the owning scope without an early unhandled promise.
|
||||
void completion.catch(() => undefined);
|
||||
if (options.signal.aborted) onAbort();
|
||||
return { output, completion };
|
||||
};
|
||||
|
||||
const asError = (cause: unknown): Error =>
|
||||
cause instanceof Error
|
||||
? cause
|
||||
: new ArtifactReaderFailure("format", "PyInstaller member read failed", {
|
||||
cause,
|
||||
});
|
||||
@@ -0,0 +1,179 @@
|
||||
import type {
|
||||
PyInstallerMetadata,
|
||||
PyInstallerEntryMetadata,
|
||||
} from "../../domain/artifactFormatMetadata.js";
|
||||
import type { CArchiveEntry, PyInstallerCookie } from "./CArchiveLayout.js";
|
||||
import type { PyzArchive, PyzEntry } from "./PyzArchive.js";
|
||||
import { pythonVersionForMagic } from "./PythonBytecode.js";
|
||||
|
||||
const pythonVersion = (value: number) =>
|
||||
value >= 100
|
||||
? { major: Math.floor(value / 100), minor: value % 100 }
|
||||
: value >= 20
|
||||
? { major: Math.floor(value / 10), minor: value % 10 }
|
||||
: undefined;
|
||||
|
||||
export const cookieMetadata = (
|
||||
cookie: PyInstallerCookie,
|
||||
entries: readonly CArchiveEntry[],
|
||||
encryptionMarker: boolean,
|
||||
): PyInstallerMetadata => {
|
||||
const version = pythonVersion(cookie.pythonVersion);
|
||||
return {
|
||||
kind: "pyinstaller",
|
||||
layout_version: cookie.layout,
|
||||
archive_start: cookie.archiveStart,
|
||||
archive_size: cookie.packageLength,
|
||||
cookie_offset: cookie.cookieOffset,
|
||||
toc_offset: cookie.tocOffset,
|
||||
toc_size: cookie.tocLength,
|
||||
entry_count: entries.length,
|
||||
runtime_option_count: entries.filter((entry) => entry.typeCode === "o")
|
||||
.length,
|
||||
external_dependency_count: entries.filter((entry) => entry.typeCode === "d")
|
||||
.length,
|
||||
python_version_raw: cookie.pythonVersion,
|
||||
python_major: version?.major ?? null,
|
||||
python_minor: version?.minor ?? null,
|
||||
python_library: cookie.pythonLibrary,
|
||||
encryption_marker_present: encryptionMarker,
|
||||
};
|
||||
};
|
||||
|
||||
/** A sibling PYZ header is usable for scripts only when every image and the cookie agree. */
|
||||
export const scriptPythonMagic = (
|
||||
cookie: PyInstallerCookie,
|
||||
archives: Iterable<PyzArchive>,
|
||||
): Buffer | undefined => {
|
||||
const version = pythonVersion(cookie.pythonVersion);
|
||||
if (version === undefined) return undefined;
|
||||
let selected: Buffer | undefined;
|
||||
for (const archive of archives) {
|
||||
const magic = archive.pythonMagic;
|
||||
if (
|
||||
pythonVersionForMagic(magic) !==
|
||||
`${String(version.major)}.${String(version.minor)}` ||
|
||||
(selected !== undefined && !selected.equals(magic))
|
||||
)
|
||||
return undefined;
|
||||
selected = magic;
|
||||
}
|
||||
return selected;
|
||||
};
|
||||
|
||||
const entryMetadata = (
|
||||
source: Pick<
|
||||
PyInstallerEntryMetadata,
|
||||
| "container"
|
||||
| "container_name"
|
||||
| "index"
|
||||
| "source_name"
|
||||
| "type_code"
|
||||
| "relative_offset"
|
||||
| "compression_flag"
|
||||
>,
|
||||
bytecode: boolean,
|
||||
header: Buffer | undefined,
|
||||
): PyInstallerEntryMetadata => ({
|
||||
kind: "pyinstaller-entry",
|
||||
...source,
|
||||
bytecode_header: bytecode
|
||||
? header === undefined
|
||||
? "unknown"
|
||||
: "derived"
|
||||
: "not-bytecode",
|
||||
bytecode_magic: header?.subarray(0, 4).toString("hex") ?? null,
|
||||
pyz_status: "not-pyz",
|
||||
directory_record_offset: null,
|
||||
directory_record_size: null,
|
||||
pyz_sha256: null,
|
||||
pyz_python_magic: null,
|
||||
pyz_python_version: null,
|
||||
pyz_entry_count: null,
|
||||
encryption_source: null,
|
||||
});
|
||||
|
||||
export const cArchiveEntryMetadata = (
|
||||
entry: CArchiveEntry,
|
||||
header?: Buffer,
|
||||
archive?: PyzArchive,
|
||||
encryptionMarker = false,
|
||||
): PyInstallerEntryMetadata => ({
|
||||
...entryMetadata(
|
||||
{
|
||||
container: "carchive",
|
||||
container_name: null,
|
||||
index: entry.index,
|
||||
source_name: entry.name,
|
||||
type_code: entry.typeCode,
|
||||
relative_offset: entry.offset,
|
||||
compression_flag: entry.compressionFlag === 1 ? 1 : 0,
|
||||
},
|
||||
["s", "m", "M"].includes(entry.typeCode),
|
||||
header,
|
||||
),
|
||||
...(entry.typeCode === "d" ? dependencyMetadata(entry.name) : {}),
|
||||
pyz_status:
|
||||
archive?.kind === "entries"
|
||||
? "decoded"
|
||||
: archive === undefined
|
||||
? "not-pyz"
|
||||
: "opaque",
|
||||
directory_record_offset: entry.tocRecordOffset,
|
||||
directory_record_size: entry.tocRecordLength,
|
||||
pyz_sha256: archive?.sha256 ?? null,
|
||||
pyz_python_magic: archive?.pythonMagic.toString("hex") ?? null,
|
||||
pyz_python_version:
|
||||
archive === undefined
|
||||
? null
|
||||
: (pythonVersionForMagic(archive.pythonMagic) ?? null),
|
||||
pyz_entry_count: archive?.kind === "entries" ? archive.entries.length : null,
|
||||
encryption_source:
|
||||
archive !== undefined && encryptionMarker ? "carchive-marker" : null,
|
||||
});
|
||||
|
||||
/** The bootloader splits at the first colon; neither part is a local read grant. */
|
||||
const dependencyMetadata = (
|
||||
source: string,
|
||||
): Pick<
|
||||
PyInstallerEntryMetadata,
|
||||
"dependency_status" | "dependency_archive" | "dependency_member"
|
||||
> => {
|
||||
const colon = source.indexOf(":");
|
||||
const valid = colon > 0 && colon < source.length - 1;
|
||||
return {
|
||||
dependency_status: valid ? "unresolved" : "invalid-reference",
|
||||
dependency_archive: valid ? source.slice(0, colon) : null,
|
||||
dependency_member: valid ? source.slice(colon + 1) : null,
|
||||
};
|
||||
};
|
||||
|
||||
/** Legacy writers encrypt nonempty namespace payloads as well as module payloads. */
|
||||
export const pyzEntryEncrypted = (
|
||||
entry: PyzEntry,
|
||||
encryptionMarker: boolean,
|
||||
): boolean => encryptionMarker && entry.length > 0;
|
||||
|
||||
export const pyzEntryMetadata = (
|
||||
container: CArchiveEntry,
|
||||
entry: PyzEntry,
|
||||
header: Buffer | undefined,
|
||||
encryptionMarker: boolean,
|
||||
): PyInstallerEntryMetadata => ({
|
||||
...entryMetadata(
|
||||
{
|
||||
container: "pyz",
|
||||
container_name: container.name,
|
||||
index: entry.index,
|
||||
source_name: entry.name,
|
||||
type_code: entry.typeCode,
|
||||
relative_offset: entry.offset,
|
||||
compression_flag: entry.compressionFlag,
|
||||
},
|
||||
entry.typeCode === 0 || entry.typeCode === 1,
|
||||
header,
|
||||
),
|
||||
encryption_source: pyzEntryEncrypted(entry, encryptionMarker)
|
||||
? "carchive-marker"
|
||||
: null,
|
||||
});
|
||||
@@ -0,0 +1,255 @@
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
import { lstat, mkdtemp, open, rm } from "node:fs/promises";
|
||||
import type { Stats } from "node:fs";
|
||||
import { Readable } from "node:stream";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
|
||||
import {
|
||||
openRegularFile,
|
||||
RegularFileAdmissionFailure,
|
||||
sameRegularFileState,
|
||||
} from "../../filesystem/RegularFile.js";
|
||||
import { ArtifactReaderFailure } from "../ArtifactReader.js";
|
||||
import {
|
||||
ArtifactResourceScope,
|
||||
type ArtifactResourceOwner,
|
||||
} from "../ArtifactResourceScope.js";
|
||||
import { abortIfNeeded } from "../ArtifactHash.js";
|
||||
import { readArtifactFileChunks } from "../readArtifactFileChunks.js";
|
||||
import { pyInstallerMemberStream } from "./PyInstallerMemberStream.js";
|
||||
import type { CArchiveEntry } from "./CArchiveLayout.js";
|
||||
import type { PyInstallerMember } from "./CArchiveMembers.js";
|
||||
import { pyInstallerFileAdmissionFailure } from "./PyInstallerFileAdmission.js";
|
||||
|
||||
// Each outer-compressed image retains one inode and its identity until close.
|
||||
// This is a per-reader filesystem capacity budget, independent of byte budgets.
|
||||
const MAX_PRIVATE_PYZ_IMAGES = 128;
|
||||
|
||||
interface StoredImage {
|
||||
readonly path: string;
|
||||
readonly initial: Stats;
|
||||
}
|
||||
export interface PyInstallerMemberSource {
|
||||
readonly source: Readable;
|
||||
readonly verify?: () => Promise<void>;
|
||||
readonly close?: (primary?: unknown) => Promise<void>;
|
||||
}
|
||||
|
||||
/** The reader's operation scope drains work before retiring these private files. */
|
||||
export class PyInstallerPyzStore {
|
||||
#root: string | undefined;
|
||||
#serial = 0;
|
||||
readonly #owners = new Map<
|
||||
FileHandle,
|
||||
Extract<ArtifactResourceOwner, { kind: "file-handle" }>
|
||||
>();
|
||||
readonly #cleanup = new ArtifactResourceScope();
|
||||
readonly #images = new Map<FileHandle, StoredImage>();
|
||||
|
||||
reserveImages(count: number): void {
|
||||
if (count > MAX_PRIVATE_PYZ_IMAGES)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
"PyInstaller compressed PYZ images exceed the 128-file private storage budget",
|
||||
{ boundary: "capacity" },
|
||||
);
|
||||
}
|
||||
|
||||
async spool(
|
||||
handle: FileHandle,
|
||||
archiveStart: number,
|
||||
entry: CArchiveEntry,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ handle: FileHandle; start: number }> {
|
||||
abortIfNeeded(signal);
|
||||
if (entry.compressedSize === 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"Compressed PYZ image has no zlib bytes",
|
||||
);
|
||||
this.#root ??= await mkdtemp(join(tmpdir(), "rea-pyz-"));
|
||||
const path = join(this.#root, String(this.#serial++));
|
||||
const writer = await open(path, "wx", 0o600);
|
||||
this.#retain(path, new OwnedFileHandle(writer));
|
||||
abortIfNeeded(signal);
|
||||
const source = readArtifactFileChunks(handle, {
|
||||
start: archiveStart + entry.offset,
|
||||
end: archiveStart + entry.offset + entry.compressedSize - 1,
|
||||
signal,
|
||||
});
|
||||
const stream = pyInstallerMemberStream(source, {
|
||||
sourceBytes: entry.compressedSize,
|
||||
compressed: true,
|
||||
header: undefined,
|
||||
expectedBytes: entry.uncompressedSize,
|
||||
maxBytes: entry.uncompressedSize,
|
||||
signal: signal ?? new AbortController().signal,
|
||||
});
|
||||
const copied = this.#writeImage(writer, stream.output, signal);
|
||||
const outcomes = await Promise.allSettled([stream.completion, copied]);
|
||||
// A failed native write can retire the upstream stream. Preserve that I/O
|
||||
// failure instead of replacing it with a secondary premature-close error.
|
||||
const copy = outcomes[1];
|
||||
if (copy?.status === "rejected") throw copy.reason;
|
||||
const decoded = outcomes[0];
|
||||
if (decoded?.status === "rejected") throw decoded.reason;
|
||||
await this.release(writer);
|
||||
abortIfNeeded(signal);
|
||||
const opened = await this.#admit(path, signal);
|
||||
this.#images.set(opened, { path, initial: await opened.stat() });
|
||||
return { handle: opened, start: 0 };
|
||||
}
|
||||
|
||||
async release(handle: FileHandle): Promise<void> {
|
||||
const owner = this.#owners.get(handle);
|
||||
if (owner === undefined) return;
|
||||
const outcome = await this.#cleanup.release(owner);
|
||||
if (outcome.kind === "failed") throw outcome.cause;
|
||||
this.#owners.delete(handle);
|
||||
}
|
||||
|
||||
async memberSource(
|
||||
read: NonNullable<PyInstallerMember["read"]>,
|
||||
root: FileHandle | undefined,
|
||||
signal: AbortSignal,
|
||||
): Promise<PyInstallerMemberSource> {
|
||||
let handle = read.kind === "carchive" ? root : read.handle;
|
||||
if (handle === undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"PyInstaller archive is closed",
|
||||
);
|
||||
const image = read.kind === "pyz" ? this.#images.get(handle) : undefined;
|
||||
let verify: (() => Promise<void>) | undefined;
|
||||
let close: ((primary?: unknown) => Promise<void>) | undefined;
|
||||
if (image !== undefined) {
|
||||
handle = await this.#admit(image.path, signal);
|
||||
const opened = handle;
|
||||
verify = () => this.#verifyImage(opened, image);
|
||||
close = async (primary) => {
|
||||
try {
|
||||
await this.release(opened);
|
||||
} catch (cause: unknown) {
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
primary ?? cause,
|
||||
ArtifactReaderFailure.cleanupObservation(cause, image.path),
|
||||
);
|
||||
}
|
||||
};
|
||||
await verify();
|
||||
}
|
||||
abortIfNeeded(signal);
|
||||
const start =
|
||||
read.kind === "carchive"
|
||||
? read.archiveStart + read.source.offset
|
||||
: read.offset;
|
||||
const length =
|
||||
read.kind === "carchive" ? read.source.compressedSize : read.length;
|
||||
const source =
|
||||
length === 0
|
||||
? Readable.from([])
|
||||
: readArtifactFileChunks(handle, {
|
||||
start,
|
||||
end: start + length - 1,
|
||||
signal,
|
||||
});
|
||||
source.on("error", () => undefined);
|
||||
return {
|
||||
source,
|
||||
...(verify === undefined ? {} : { verify }),
|
||||
...(close === undefined ? {} : { close }),
|
||||
};
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.#cleanup.close();
|
||||
this.#owners.clear();
|
||||
if (this.#root !== undefined) {
|
||||
try {
|
||||
await rm(this.#root, { recursive: true, force: true });
|
||||
} catch (cause: unknown) {
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
cause,
|
||||
ArtifactReaderFailure.cleanupObservation(cause, this.#root),
|
||||
);
|
||||
}
|
||||
this.#root = undefined;
|
||||
}
|
||||
this.#images.clear();
|
||||
}
|
||||
|
||||
async #admit(path: string, signal?: AbortSignal): Promise<FileHandle> {
|
||||
try {
|
||||
const handle = await openRegularFile(path, {
|
||||
symlinks: "reject",
|
||||
signal,
|
||||
});
|
||||
this.#retain(path, new OwnedFileHandle(handle));
|
||||
return handle;
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof RegularFileAdmissionFailure)
|
||||
this.#retain(path, cause.owner);
|
||||
throw pyInstallerFileAdmissionFailure(path, cause);
|
||||
}
|
||||
}
|
||||
|
||||
#retain(path: string, handle: OwnedFileHandle): void {
|
||||
const owner = { kind: "file-handle" as const, resource: path, handle };
|
||||
this.#owners.set(handle.handle, owner);
|
||||
this.#cleanup.retain(owner);
|
||||
}
|
||||
|
||||
async #writeImage(
|
||||
handle: FileHandle,
|
||||
source: Readable,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
let position = 0;
|
||||
for await (const chunk of source as AsyncIterable<Buffer>) {
|
||||
// Stop writing after cancellation; drain only already-produced bytes so
|
||||
// the descriptor source can report its in-flight I/O outcome truthfully.
|
||||
if (signal?.aborted) continue;
|
||||
let offset = 0;
|
||||
try {
|
||||
while (offset < chunk.length) {
|
||||
const { bytesWritten } = await handle.write(
|
||||
chunk,
|
||||
offset,
|
||||
chunk.length - offset,
|
||||
position,
|
||||
);
|
||||
if (bytesWritten === 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"io",
|
||||
"Private PYZ write made no progress",
|
||||
);
|
||||
offset += bytesWritten;
|
||||
position += bytesWritten;
|
||||
if (signal?.aborted) break;
|
||||
}
|
||||
} catch (cause: unknown) {
|
||||
const failure =
|
||||
cause instanceof ArtifactReaderFailure
|
||||
? cause
|
||||
: new ArtifactReaderFailure(
|
||||
"io",
|
||||
"Private PYZ image could not be written",
|
||||
{ cause },
|
||||
);
|
||||
source.destroy(failure);
|
||||
throw failure;
|
||||
}
|
||||
}
|
||||
abortIfNeeded(signal);
|
||||
}
|
||||
|
||||
async #verifyImage(handle: FileHandle, image: StoredImage): Promise<void> {
|
||||
const states = await Promise.all([handle.stat(), lstat(image.path)]);
|
||||
if (states.some((state) => !sameRegularFileState(image.initial, state)))
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"Private PYZ image changed after parsing",
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
/**
|
||||
* Final CPython bytecode magics for the minor versions whose 16-byte `.pyc`
|
||||
* header this reader can reconstruct. Other magics leave the PYZ opaque.
|
||||
* Each value is the four-byte little-endian magic, including the trailing CR LF.
|
||||
*/
|
||||
const PYTHON_BYTECODE_MAGICS: Readonly<Record<string, string>> = {
|
||||
"420d0d0a": "3.7",
|
||||
"550d0d0a": "3.8",
|
||||
"610d0d0a": "3.9",
|
||||
"6f0d0d0a": "3.10",
|
||||
a70d0d0a: "3.11",
|
||||
cb0d0d0a: "3.12",
|
||||
f30d0d0a: "3.13",
|
||||
"2b0e0d0a": "3.14",
|
||||
};
|
||||
|
||||
/** Limitation recorded when a member's digest covers a derived `.pyc` header. */
|
||||
export const RECONSTRUCTED_PYTHON_HEADER_LIMITATION =
|
||||
"Python bytecode header was reconstructed from the archive Python magic; this digest covers the derived file.";
|
||||
|
||||
/** Dotted Python version for a supported bytecode magic, when one is known. */
|
||||
export const pythonVersionForMagic = (magic: Buffer): string | undefined =>
|
||||
PYTHON_BYTECODE_MAGICS[magic.toString("hex")];
|
||||
|
||||
/**
|
||||
* 16-byte `.pyc` header used since Python 3.7: magic, zero flags, zero
|
||||
* timestamp, and zero source size. The flags word selects the timestamp form.
|
||||
*/
|
||||
export const reconstructedPycHeader = (magic: Buffer): Buffer => {
|
||||
const header = Buffer.alloc(16);
|
||||
magic.copy(header, 0, 0, 4);
|
||||
return header;
|
||||
};
|
||||
@@ -0,0 +1,137 @@
|
||||
import { open, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { readPyzArchive, MAX_PYZ_TABLE_BYTES } from "./PyzArchive.js";
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import { PYTHON_39_MAGIC, pyzArchive } from "./CArchive.fixture.js";
|
||||
|
||||
it("lists modules from a supported PYZ and leaves an unknown magic opaque", async () => {
|
||||
const body = Buffer.from("module");
|
||||
const parsed = await readImage(
|
||||
pyzArchive({
|
||||
magic: PYTHON_39_MAGIC,
|
||||
modules: [{ name: "json.decoder", typeCode: 0, body }],
|
||||
}),
|
||||
);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok || parsed.value.kind !== "entries") return;
|
||||
expect(parsed.value.pythonVersion).toBe("3.9");
|
||||
expect(parsed.value.entries).toMatchObject([
|
||||
{ name: "json.decoder", typeCode: 0, package: false },
|
||||
]);
|
||||
|
||||
const opaque = await readImage(
|
||||
pyzArchive({
|
||||
magic: Buffer.alloc(4),
|
||||
modules: [{ name: "json.decoder", typeCode: 0, body }],
|
||||
}),
|
||||
);
|
||||
expect(opaque.ok).toBe(true);
|
||||
if (!opaque.ok || opaque.value.kind !== "opaque") return;
|
||||
expect(opaque.value.pythonMagic).toEqual(Buffer.alloc(4));
|
||||
expect(opaque.value.limitation).toContain("left opaque");
|
||||
});
|
||||
|
||||
it("leaves a PYZ opaque when the table uses an unsupported marshal type", async () => {
|
||||
const image = pyzArchive({
|
||||
magic: PYTHON_39_MAGIC,
|
||||
modules: [],
|
||||
toc: Buffer.from([0x4e]),
|
||||
});
|
||||
const parsed = await readImage(image);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok || parsed.value.kind !== "opaque") return;
|
||||
expect(parsed.value.limitation).toContain("left opaque");
|
||||
expect(parsed.value.pythonMagic).toEqual(PYTHON_39_MAGIC);
|
||||
});
|
||||
|
||||
it("reports an entry that extends outside the PYZ as an integrity error", async () => {
|
||||
const image = pyzArchive({
|
||||
magic: PYTHON_39_MAGIC,
|
||||
modules: [{ name: "missing", typeCode: 0, body: Buffer.from("x") }],
|
||||
});
|
||||
image.writeInt32LE(100_000, image.length - 4);
|
||||
const parsed = await readImage(image);
|
||||
expect(parsed.ok).toBe(false);
|
||||
if (parsed.ok) return;
|
||||
expect(parsed.error.reason).toBe("integrity");
|
||||
});
|
||||
|
||||
it("rejects an oversized sparse PYZ table before allocating its bytes", async () => {
|
||||
const root = await createTestTempDirectory("rea-pyz-table-budget-");
|
||||
const path = join(root, "image.pyz");
|
||||
const header = Buffer.alloc(12);
|
||||
header.write("PYZ\0");
|
||||
PYTHON_39_MAGIC.copy(header, 4);
|
||||
header.writeUInt32BE(12, 8);
|
||||
await writeFile(path, header);
|
||||
const handle = await open(path, "r+");
|
||||
try {
|
||||
const size = 12 + MAX_PYZ_TABLE_BYTES + 1;
|
||||
await handle.truncate(size);
|
||||
expect(await readPyzArchive(handle, 0, size)).toMatchObject({
|
||||
ok: false,
|
||||
error: { reason: "limit" },
|
||||
});
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a PYZ table whose back-references repeat one long row", async () => {
|
||||
// One namespace row with a 16 KiB name, flagged for reference, then 999
|
||||
// back-references to it. The table is about 21 KiB but would expand to
|
||||
// 16 MiB of entry names.
|
||||
const name = Buffer.alloc(16 * 1024, 0x61);
|
||||
const refs = 1_000;
|
||||
const row = Buffer.concat([
|
||||
Buffer.from([0x28 | 0x80]),
|
||||
int32(2),
|
||||
Buffer.from([0x61]),
|
||||
int32(name.length),
|
||||
name,
|
||||
Buffer.from([0x29, 3, 0x69]),
|
||||
int32(3),
|
||||
Buffer.from([0x69]),
|
||||
int32(0),
|
||||
Buffer.from([0x69]),
|
||||
int32(0),
|
||||
]);
|
||||
const reference = Buffer.concat([Buffer.from([0x72]), int32(0)]);
|
||||
const table = Buffer.concat([
|
||||
Buffer.from([0x5b]),
|
||||
int32(refs),
|
||||
row,
|
||||
...Array.from({ length: refs - 1 }, () => reference),
|
||||
]);
|
||||
const header = Buffer.alloc(12);
|
||||
header.write("PYZ\0");
|
||||
PYTHON_39_MAGIC.copy(header, 4);
|
||||
header.writeUInt32BE(12, 8);
|
||||
const image = Buffer.concat([header, table]);
|
||||
expect(image.length).toBeLessThan(32 * 1024);
|
||||
|
||||
const parsed = await readImage(image);
|
||||
expect(parsed).toMatchObject({ ok: false, error: { reason: "limit" } });
|
||||
if (parsed.ok) return;
|
||||
expect(parsed.error.message).toContain("expand past");
|
||||
});
|
||||
|
||||
const int32 = (value: number): Buffer => {
|
||||
const bytes = Buffer.alloc(4);
|
||||
bytes.writeInt32LE(value);
|
||||
return bytes;
|
||||
};
|
||||
|
||||
const readImage = async (bytes: Buffer) => {
|
||||
const root = await createTestTempDirectory("rea-pyz-file-");
|
||||
const path = join(root, "image.pyz");
|
||||
await writeFile(path, bytes);
|
||||
const handle = await open(path, "r");
|
||||
try {
|
||||
return await readPyzArchive(handle, 0, bytes.length);
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,266 @@
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
import { hashReadable } from "../ArtifactHash.js";
|
||||
import { readArtifactFileChunks } from "../readArtifactFileChunks.js";
|
||||
import { readPyInstallerBytes } from "./PyInstallerBytes.js";
|
||||
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import { readMarshalSubset, type MarshalValue } from "./MarshalSubset.js";
|
||||
import {
|
||||
pyInstallerFailure,
|
||||
type PyInstallerFailure,
|
||||
} from "./PyInstallerFailure.js";
|
||||
import { pythonVersionForMagic } from "./PythonBytecode.js";
|
||||
|
||||
const PYZ_MAGIC = Buffer.from("PYZ\0");
|
||||
const PYZ_HEADER_BYTES = 12;
|
||||
|
||||
/** One PYZ table entry. Type codes follow the archive's module, package, data, and namespace values. */
|
||||
export interface PyzEntry {
|
||||
readonly index: number;
|
||||
readonly name: string;
|
||||
readonly typeCode: number;
|
||||
readonly offset: number;
|
||||
readonly length: number;
|
||||
/** PYZ compresses every nonempty payload, independently of its logical module type. */
|
||||
readonly compressionFlag: 0 | 1;
|
||||
readonly package: boolean;
|
||||
readonly namespace: boolean;
|
||||
}
|
||||
|
||||
export type PyzArchive =
|
||||
| {
|
||||
readonly kind: "entries";
|
||||
readonly pythonMagic: Buffer;
|
||||
readonly pythonVersion: string;
|
||||
readonly sha256: string;
|
||||
readonly entries: readonly PyzEntry[];
|
||||
}
|
||||
| {
|
||||
readonly kind: "opaque";
|
||||
readonly pythonMagic: Buffer;
|
||||
readonly sha256: string;
|
||||
readonly limitation: string;
|
||||
};
|
||||
|
||||
/** Bound the transient marshal table allocation independently of image expansion. */
|
||||
export const MAX_PYZ_TABLE_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
/** Parse only the header and table in memory; hash the complete file range in chunks. */
|
||||
export const readPyzArchive = async (
|
||||
handle: FileHandle,
|
||||
start: number,
|
||||
size: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Result<PyzArchive, PyInstallerFailure>> => {
|
||||
if (size < PYZ_HEADER_BYTES)
|
||||
return err(
|
||||
pyInstallerFailure("format", "PYZ archive magic does not match"),
|
||||
);
|
||||
const headerRead = await readPyInstallerBytes(handle, {
|
||||
position: start,
|
||||
length: PYZ_HEADER_BYTES,
|
||||
label: "PYZ header",
|
||||
signal,
|
||||
});
|
||||
if (!headerRead.ok) return headerRead;
|
||||
const header = headerRead.value;
|
||||
const tocOffset =
|
||||
header.length === PYZ_HEADER_BYTES ? header.readUInt32BE(8) : size;
|
||||
const validOffset = tocOffset >= PYZ_HEADER_BYTES && tocOffset <= size;
|
||||
const knownMagic = pythonVersionForMagic(header.subarray(4, 8)) !== undefined;
|
||||
const tableSize = validOffset && knownMagic ? size - tocOffset : 0;
|
||||
if (tableSize > MAX_PYZ_TABLE_BYTES)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"limit",
|
||||
"PYZ table exceeds the 16 MiB in-memory table budget",
|
||||
),
|
||||
);
|
||||
const read = await readPyInstallerBytes(handle, {
|
||||
position: start + tocOffset,
|
||||
length: tableSize,
|
||||
label: "PYZ table",
|
||||
signal,
|
||||
});
|
||||
if (!read.ok) return read;
|
||||
const table = read.value;
|
||||
const digest = await hashReadable(
|
||||
readArtifactFileChunks(handle, {
|
||||
start,
|
||||
end: start + size - 1,
|
||||
...(signal === undefined ? {} : { signal }),
|
||||
}),
|
||||
signal,
|
||||
);
|
||||
if (digest.bytes !== size)
|
||||
return err(
|
||||
pyInstallerFailure("format", "PYZ image extends outside the file"),
|
||||
);
|
||||
return parsePyzParts(header, table, size, digest.sha256);
|
||||
};
|
||||
|
||||
const parsePyzParts = (
|
||||
header: Buffer,
|
||||
table: Buffer,
|
||||
size: number,
|
||||
sha256: string,
|
||||
): Result<PyzArchive, PyInstallerFailure> => {
|
||||
if (
|
||||
header.length < PYZ_HEADER_BYTES ||
|
||||
!header.subarray(0, 4).equals(PYZ_MAGIC)
|
||||
)
|
||||
return err(
|
||||
pyInstallerFailure("format", "PYZ archive magic does not match"),
|
||||
);
|
||||
const pythonMagic = Buffer.from(header.subarray(4, 8));
|
||||
const pythonVersion = pythonVersionForMagic(pythonMagic);
|
||||
if (pythonVersion === undefined)
|
||||
return ok({
|
||||
kind: "opaque",
|
||||
pythonMagic,
|
||||
sha256,
|
||||
limitation: `PYZ Python magic ${pythonMagic.toString("hex")} is not in the supported set; the member was left opaque.`,
|
||||
});
|
||||
const tocOffset = header.readUInt32BE(8);
|
||||
if (tocOffset < PYZ_HEADER_BYTES || tocOffset > size)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"PYZ table of contents is outside the archive",
|
||||
),
|
||||
);
|
||||
const marshalled = readMarshalSubset(table);
|
||||
if (!marshalled.ok) {
|
||||
if (marshalled.error.reason !== "format") return marshalled;
|
||||
return ok({
|
||||
kind: "opaque",
|
||||
pythonMagic,
|
||||
sha256,
|
||||
limitation: `PYZ table of contents could not be parsed (${marshalled.error.message}); the member was left opaque.`,
|
||||
});
|
||||
}
|
||||
const entries = pyzEntries(marshalled.value, tocOffset, table.length);
|
||||
if (!entries.ok)
|
||||
return entries.error.reason === "format"
|
||||
? ok({
|
||||
kind: "opaque",
|
||||
pythonMagic,
|
||||
sha256,
|
||||
limitation: `${entries.error.message}; the member was left opaque.`,
|
||||
})
|
||||
: entries;
|
||||
return ok({
|
||||
kind: "entries",
|
||||
pythonMagic,
|
||||
pythonVersion,
|
||||
sha256,
|
||||
entries: entries.value,
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Marshal back-references let one stored row appear many times, so a small
|
||||
* table can name far more bytes than it holds. Each genuine entry stores its
|
||||
* own name inline, so the expanded names can never exceed the table itself.
|
||||
*/
|
||||
const pyzEntries = (
|
||||
value: MarshalValue,
|
||||
payloadEnd: number,
|
||||
tableBytes: number,
|
||||
): Result<readonly PyzEntry[], PyInstallerFailure> => {
|
||||
if (!Array.isArray(value))
|
||||
return err(
|
||||
pyInstallerFailure("format", "PYZ table of contents is not a list"),
|
||||
);
|
||||
const entries: PyzEntry[] = [];
|
||||
let nameBytes = 0;
|
||||
for (const item of value) {
|
||||
const entry = onePyzEntry(item, payloadEnd, entries.length);
|
||||
if (!entry.ok) return entry;
|
||||
nameBytes += Buffer.byteLength(entry.value.name, "utf8");
|
||||
if (nameBytes > tableBytes)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"limit",
|
||||
`PYZ table entry names expand past the ${String(tableBytes)}-byte table that stores them`,
|
||||
),
|
||||
);
|
||||
entries.push(entry.value);
|
||||
}
|
||||
return ok(entries);
|
||||
};
|
||||
|
||||
const onePyzEntry = (
|
||||
item: MarshalValue,
|
||||
payloadEnd: number,
|
||||
index: number,
|
||||
): Result<PyzEntry, PyInstallerFailure> => {
|
||||
if (!Array.isArray(item) || item.length !== 2)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"PYZ table entry is not a name and position pair",
|
||||
),
|
||||
);
|
||||
const [name, position] = item;
|
||||
if (
|
||||
typeof name !== "string" ||
|
||||
!Array.isArray(position) ||
|
||||
position.length !== 3
|
||||
)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"PYZ table entry is not a name, type, offset, and length",
|
||||
),
|
||||
);
|
||||
const [typeCode, offset, length] = position;
|
||||
if (
|
||||
!Number.isSafeInteger(typeCode) ||
|
||||
!Number.isSafeInteger(offset) ||
|
||||
!Number.isSafeInteger(length)
|
||||
)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"PYZ table entry positions are not integers",
|
||||
),
|
||||
);
|
||||
if (typeCode < 0 || typeCode > 3)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
`PYZ table entry type ${String(typeCode)} is not a known module, package, data, or namespace code`,
|
||||
),
|
||||
);
|
||||
if (typeCode !== 3 && length === 0)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
`PYZ entry at table index ${String(index)} has no compressed bytes`,
|
||||
),
|
||||
);
|
||||
if (
|
||||
length < 0 ||
|
||||
offset < 0 ||
|
||||
offset > payloadEnd ||
|
||||
length > payloadEnd - offset ||
|
||||
(length > 0 && offset < PYZ_HEADER_BYTES)
|
||||
)
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"integrity",
|
||||
`PYZ entry at table index ${String(index)} extends outside the archive payload region`,
|
||||
),
|
||||
);
|
||||
return ok({
|
||||
index,
|
||||
name,
|
||||
typeCode,
|
||||
offset,
|
||||
length,
|
||||
compressionFlag: length === 0 ? 0 : 1,
|
||||
package: typeCode === 1 || typeCode === 3,
|
||||
namespace: typeCode === 3,
|
||||
});
|
||||
};
|
||||
@@ -90,6 +90,7 @@ const subjectFormatSchema = z.enum([
|
||||
"unknown",
|
||||
"javascript-bundle",
|
||||
"entitlements",
|
||||
"pyinstaller",
|
||||
]);
|
||||
// Snapshot entries retain the same raw-only architecture boundary as targets.
|
||||
const subjectSchema = z.discriminatedUnion("format", [
|
||||
|
||||
@@ -51,6 +51,9 @@ export const artifactInventoryViewRequestSchema = z.discriminatedUnion(
|
||||
collection: z.literal("occurrences"),
|
||||
occurrence_ids: z.array(prefixedDigestSchema("occ")).optional(),
|
||||
logical_paths: z.array(z.string()).optional(),
|
||||
entry_kinds: z
|
||||
.array(artifactOccurrenceSchema.shape.entry_kind)
|
||||
.optional(),
|
||||
fields: z.array(artifactOccurrenceSchema.keyof()).optional(),
|
||||
}),
|
||||
z.strictObject({
|
||||
@@ -104,6 +107,7 @@ export const artifactInventorySummarySchema = z.strictObject({
|
||||
directory: z.number().int().nonnegative(),
|
||||
symlink: z.number().int().nonnegative(),
|
||||
slice: z.number().int().nonnegative(),
|
||||
metadata: z.number().int().nonnegative().exactOptional(),
|
||||
}),
|
||||
source_complete: z.boolean(),
|
||||
inventory_evidence_ids: z.array(prefixedDigestSchema("ev")).min(1),
|
||||
@@ -140,6 +144,7 @@ interface InventorySource {
|
||||
index?: {
|
||||
byId: ReadonlyMap<string, number>;
|
||||
byPath: ReadonlyMap<string, readonly number[]>;
|
||||
byKind: ReadonlyMap<ArtifactOccurrence["entry_kind"], readonly number[]>;
|
||||
};
|
||||
counts?: InventoryCounts;
|
||||
}
|
||||
@@ -241,7 +246,9 @@ export const projectArtifactInventoryView = (
|
||||
?.format_metadata ?? null;
|
||||
const rootMetadata = source.rootMetadata;
|
||||
const includeCoverage =
|
||||
view.kind === "inventory" || inventory.manifest.root_format === "godot-pck";
|
||||
view.kind === "inventory" ||
|
||||
inventory.manifest.root_format === "godot-pck" ||
|
||||
inventory.manifest.root_format === "pyinstaller";
|
||||
const sourceCoverage = includeCoverage
|
||||
? { source_coverage: inventory.coverage }
|
||||
: {};
|
||||
@@ -309,7 +316,15 @@ export const projectArtifactInventoryView = (
|
||||
edges: inventory.edges.length,
|
||||
integrity_contradictions: inventory.integrityContradictions.length,
|
||||
},
|
||||
observed_entry_kind_counts: counts.kinds,
|
||||
observed_entry_kind_counts: {
|
||||
file: counts.kinds.file,
|
||||
directory: counts.kinds.directory,
|
||||
symlink: counts.kinds.symlink,
|
||||
slice: counts.kinds.slice,
|
||||
...(counts.kinds.metadata === 0
|
||||
? {}
|
||||
: { metadata: counts.kinds.metadata }),
|
||||
},
|
||||
...(includeCoverage
|
||||
? { available_hash_status_counts: counts.hashes }
|
||||
: {}),
|
||||
@@ -406,7 +421,7 @@ const nodePage = (
|
||||
|
||||
const inventoryCounts = (source: InventorySource): InventoryCounts => {
|
||||
if (source.counts !== undefined) return source.counts;
|
||||
const kinds = { file: 0, directory: 0, symlink: 0, slice: 0 };
|
||||
const kinds = { file: 0, directory: 0, symlink: 0, slice: 0, metadata: 0 };
|
||||
const hashes = {
|
||||
verified: 0,
|
||||
mismatched: 0,
|
||||
@@ -435,27 +450,43 @@ const occurrenceIndices = (
|
||||
source: InventorySource,
|
||||
view: OccurrenceView,
|
||||
): number[] | undefined => {
|
||||
if (view.occurrence_ids === undefined && view.logical_paths === undefined)
|
||||
if (
|
||||
view.occurrence_ids === undefined &&
|
||||
view.logical_paths === undefined &&
|
||||
view.entry_kinds === undefined
|
||||
)
|
||||
return undefined;
|
||||
if (view.occurrence_ids?.length === 0 || view.logical_paths?.length === 0)
|
||||
if (
|
||||
view.occurrence_ids?.length === 0 ||
|
||||
view.logical_paths?.length === 0 ||
|
||||
view.entry_kinds?.length === 0
|
||||
)
|
||||
return [];
|
||||
if (source.index === undefined) {
|
||||
const byId = new Map<string, number>();
|
||||
const byPath = new Map<string, number[]>();
|
||||
const byKind = new Map<ArtifactOccurrence["entry_kind"], number[]>();
|
||||
for (const [index, row] of source.inventory.occurrences.entries()) {
|
||||
byId.set(row.occurrence_id, index);
|
||||
const existing = byPath.get(row.logical_path);
|
||||
if (existing === undefined) byPath.set(row.logical_path, [index]);
|
||||
else existing.push(index);
|
||||
const kind = byKind.get(row.entry_kind);
|
||||
if (kind === undefined) byKind.set(row.entry_kind, [index]);
|
||||
else kind.push(index);
|
||||
}
|
||||
source.index = { byId, byPath };
|
||||
source.index = { byId, byPath, byKind };
|
||||
}
|
||||
const lookup = source.index;
|
||||
const paths =
|
||||
view.logical_paths === undefined ? undefined : new Set(view.logical_paths);
|
||||
const selected =
|
||||
view.occurrence_ids === undefined
|
||||
? [...(paths ?? [])].flatMap((path) => lookup.byPath.get(path) ?? [])
|
||||
? paths === undefined
|
||||
? [...new Set(view.entry_kinds)].flatMap(
|
||||
(kind) => lookup.byKind.get(kind) ?? [],
|
||||
)
|
||||
: [...paths].flatMap((path) => lookup.byPath.get(path) ?? [])
|
||||
: [...new Set(view.occurrence_ids)].flatMap((id) => {
|
||||
const index = lookup.byId.get(id);
|
||||
return index === undefined ||
|
||||
@@ -464,7 +495,15 @@ const occurrenceIndices = (
|
||||
? []
|
||||
: [index];
|
||||
});
|
||||
return selected.sort((left, right) => left - right);
|
||||
const kinds =
|
||||
view.entry_kinds === undefined ? undefined : new Set(view.entry_kinds);
|
||||
return selected
|
||||
.filter(
|
||||
(index) =>
|
||||
kinds === undefined ||
|
||||
kinds.has(source.inventory.occurrences[index]!.entry_kind),
|
||||
)
|
||||
.sort((left, right) => left - right);
|
||||
};
|
||||
|
||||
const nodeIndices = (
|
||||
|
||||
@@ -29,9 +29,70 @@ export const godotPckEntryMetadataSchema = z.strictObject({
|
||||
flags: uint32,
|
||||
md5_status: z.enum(["declared", "all-zero"]),
|
||||
});
|
||||
|
||||
/** Source cookie fields and derived directory counts without Python execution. */
|
||||
export const pyInstallerMetadataSchema = z.strictObject({
|
||||
kind: z.literal("pyinstaller"),
|
||||
layout_version: z.enum(["2.0", "2.1"]),
|
||||
archive_start: offset,
|
||||
archive_size: offset,
|
||||
cookie_offset: offset,
|
||||
toc_offset: offset,
|
||||
toc_size: offset,
|
||||
entry_count: offset,
|
||||
runtime_option_count: offset,
|
||||
external_dependency_count: offset,
|
||||
python_version_raw: uint32,
|
||||
python_major: uint32.nullable(),
|
||||
python_minor: uint32.nullable(),
|
||||
python_library: z.string().nullable(),
|
||||
encryption_marker_present: z.boolean(),
|
||||
});
|
||||
export type PyInstallerMetadata = z.infer<typeof pyInstallerMetadataSchema>;
|
||||
|
||||
/** Primitive facts preserve source spelling and keep metadata copies independent. */
|
||||
export const pyInstallerEntryMetadataSchema = z.strictObject({
|
||||
kind: z.literal("pyinstaller-entry"),
|
||||
container: z.enum(["carchive", "pyz"]),
|
||||
container_name: z.string().nullable(),
|
||||
index: offset,
|
||||
source_name: z.string(),
|
||||
type_code: z.union([z.string().length(1), uint32]),
|
||||
relative_offset: offset,
|
||||
directory_record_offset: offset.nullable(),
|
||||
directory_record_size: offset.nullable(),
|
||||
compression_flag: z.union([z.literal(0), z.literal(1)]),
|
||||
bytecode_header: z.enum(["not-bytecode", "unknown", "derived"]),
|
||||
bytecode_magic: z
|
||||
.string()
|
||||
.regex(/^[0-9a-f]{8}$/u)
|
||||
.nullable(),
|
||||
pyz_status: z.enum(["not-pyz", "decoded", "opaque"]),
|
||||
pyz_sha256: z
|
||||
.string()
|
||||
.regex(/^[0-9a-f]{64}$/u)
|
||||
.nullable(),
|
||||
pyz_python_magic: z
|
||||
.string()
|
||||
.regex(/^[0-9a-f]{8}$/u)
|
||||
.nullable(),
|
||||
pyz_python_version: z.string().nullable(),
|
||||
pyz_entry_count: offset.nullable(),
|
||||
encryption_source: z.literal("carchive-marker").nullable(),
|
||||
dependency_status: z
|
||||
.enum(["unresolved", "invalid-reference"])
|
||||
.exactOptional(),
|
||||
dependency_archive: z.string().nullable().exactOptional(),
|
||||
dependency_member: z.string().nullable().exactOptional(),
|
||||
});
|
||||
export type PyInstallerEntryMetadata = z.infer<
|
||||
typeof pyInstallerEntryMetadataSchema
|
||||
>;
|
||||
export const artifactFormatMetadataSchema = z.discriminatedUnion("kind", [
|
||||
godotPckMetadataSchema,
|
||||
godotPckEntryMetadataSchema,
|
||||
pyInstallerMetadataSchema,
|
||||
pyInstallerEntryMetadataSchema,
|
||||
]);
|
||||
export type ArtifactFormatMetadata = z.infer<
|
||||
typeof artifactFormatMetadataSchema
|
||||
|
||||
@@ -83,6 +83,7 @@ const artifactFormatSchema = z.enum([
|
||||
"entitlements",
|
||||
"directory",
|
||||
"file",
|
||||
"pyinstaller",
|
||||
"unknown",
|
||||
]);
|
||||
|
||||
@@ -138,7 +139,7 @@ export const artifactOccurrenceSchema = z.object({
|
||||
artifact_id: artifactIdSchema.nullable(),
|
||||
parent_occurrence_id: occurrenceIdSchema.nullable(),
|
||||
logical_path: boundedRelativePathSchema,
|
||||
entry_kind: z.enum(["file", "directory", "symlink", "slice"]),
|
||||
entry_kind: z.enum(["file", "directory", "symlink", "slice", "metadata"]),
|
||||
artifact_kind: artifactKindSchema,
|
||||
artifact_format: artifactFormatSchema,
|
||||
declared_size: z
|
||||
|
||||
@@ -348,19 +348,20 @@ const orderOccurrences = (
|
||||
.flatMap(([path, group]) => {
|
||||
if (group.length < 2) return group;
|
||||
const ordinalById = new Map<string, number>();
|
||||
const first = group[0];
|
||||
if (first === undefined)
|
||||
throw new TypeError("Duplicate occurrence group is empty");
|
||||
const kinds = new Set(group.map(({ entry_kind }) => entry_kind));
|
||||
for (let ordinal = 0; ordinal < group.length; ordinal++) {
|
||||
ordinalById.set(
|
||||
occurrenceIdForLocation({
|
||||
rootArtifactId,
|
||||
logicalPath: path,
|
||||
entryKind: first.entry_kind,
|
||||
...(ordinal === 0 ? {} : { duplicateOrdinal: ordinal }),
|
||||
}),
|
||||
ordinal,
|
||||
);
|
||||
// Entry kinds come from a fixed schema enum, so work stays linear in
|
||||
// the group size without changing or guessing producer identities.
|
||||
for (const entryKind of kinds)
|
||||
ordinalById.set(
|
||||
occurrenceIdForLocation({
|
||||
rootArtifactId,
|
||||
logicalPath: path,
|
||||
entryKind,
|
||||
...(ordinal === 0 ? {} : { duplicateOrdinal: ordinal }),
|
||||
}),
|
||||
ordinal,
|
||||
);
|
||||
}
|
||||
return group.sort(
|
||||
(left, right) =>
|
||||
|
||||
@@ -70,6 +70,7 @@ const subjectFormatSchema = z.enum([
|
||||
"mach-o-universal",
|
||||
"javascript-bundle",
|
||||
"entitlements",
|
||||
"pyinstaller",
|
||||
]);
|
||||
const subjectBaseSchema = z.strictObject({
|
||||
name: z.string().min(1),
|
||||
|
||||
Reference in new issue
Block a user