Anuj Singhandmorluto
0363e5a8e1
* fix(reference): join internal symlink targets with the inventory NFC normalizer checkSymlinks resolved an internal symlink target with bare posix.normalize and looked the result up in the inventoried path set. A target that a filesystem reports in NFD (macOS readlink commonly does) joined to an NFD spelling while the inventory holds NFC, so validation reported "Internal symlink target must resolve to an inventoried entry" for a target the inventory actually contains. Route the join through normalizeJoinedLogicalPath, the same inventory identity normalizer #1513 applied to the runtime, HTML/module, and import joins from #1488. The feature-seed NFKC query fold stays a query. Refs #1488 * fix(reference): preserve source spelling during symlink lookup --------- Co-authored-by: morluto <[email protected]>
550 lines
16 KiB
TypeScript
550 lines
16 KiB
TypeScript
import { posix } from "node:path";
|
|
|
|
import { z } from "zod";
|
|
|
|
import { canonicalJson } from "./comparisonSemantics.js";
|
|
import { digestCanonicalValue } from "./canonicalDigest.js";
|
|
import { normalizeJoinedLogicalPath } from "./artifactIdentity.js";
|
|
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
|
|
import { digestSchema } from "./../domain/digests.js";
|
|
import { prefixedDigestSchema } from "./../domain/digests.js";
|
|
const boundedTextSchema = z.string().min(1);
|
|
const relativePathSchema = z
|
|
.string()
|
|
.min(1)
|
|
.refine(
|
|
(path) =>
|
|
!path.startsWith("/") &&
|
|
!path.includes("\\") &&
|
|
path
|
|
.split("/")
|
|
.every((part) => part !== "" && part !== "." && part !== ".."),
|
|
"Expected a normalized relative POSIX path without traversal",
|
|
);
|
|
|
|
const classificationSchema = z.enum([
|
|
"source",
|
|
"test",
|
|
"config",
|
|
"manifest",
|
|
"generated",
|
|
"vendor",
|
|
"documentation",
|
|
"unknown",
|
|
]);
|
|
|
|
const entryBaseShape = {
|
|
path: relativePathSchema,
|
|
classifications: z.array(classificationSchema).min(1),
|
|
limitations: z.array(boundedTextSchema),
|
|
};
|
|
|
|
const sourceFileBaseShape = {
|
|
...entryBaseShape,
|
|
kind: z.literal("file"),
|
|
language: z.string().min(1).nullable(),
|
|
};
|
|
|
|
const sourceFileSchema = z.union([
|
|
z.strictObject({
|
|
...sourceFileBaseShape,
|
|
sha256: digestSchema,
|
|
size: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER),
|
|
content_state: z.literal("hashed"),
|
|
}),
|
|
z.strictObject({
|
|
...sourceFileBaseShape,
|
|
sha256: z.null(),
|
|
size: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).nullable(),
|
|
content_state: z.enum([
|
|
"redacted-secret",
|
|
"excluded",
|
|
"unreadable",
|
|
"unknown",
|
|
]),
|
|
}),
|
|
]);
|
|
|
|
const sourceDirectorySchema = z.strictObject({
|
|
...entryBaseShape,
|
|
kind: z.literal("directory"),
|
|
tree_state: z.enum([
|
|
"enumerated",
|
|
"partial",
|
|
"excluded",
|
|
"unreadable",
|
|
"unknown",
|
|
]),
|
|
});
|
|
|
|
const symlinkTargetSchema = z
|
|
.string()
|
|
.min(1)
|
|
.refine(
|
|
(target) =>
|
|
isPortableAbsoluteSymlinkTarget(target) ||
|
|
(!target.startsWith("/") &&
|
|
!target.includes("\\") &&
|
|
posix.normalize(target) === target),
|
|
"Expected a normalized POSIX symlink target",
|
|
);
|
|
|
|
const isPortableAbsoluteSymlinkTarget = (target: string): boolean =>
|
|
target.startsWith("/") ||
|
|
/^[A-Za-z]:[\\/]/u.test(target) ||
|
|
/^\\\\[^\\/]+[\\/][^\\/]+/u.test(target);
|
|
|
|
const sourceSymlinkSchema = z.union([
|
|
z.strictObject({
|
|
...entryBaseShape,
|
|
kind: z.literal("symlink"),
|
|
target: symlinkTargetSchema,
|
|
target_state: z.enum(["internal", "external", "missing"]),
|
|
}),
|
|
z.strictObject({
|
|
...entryBaseShape,
|
|
kind: z.literal("symlink"),
|
|
target: z.null(),
|
|
target_state: z.enum(["unreadable", "unknown"]),
|
|
}),
|
|
]);
|
|
|
|
const sourceEntrySchema = z.union([
|
|
sourceFileSchema,
|
|
sourceDirectorySchema,
|
|
sourceSymlinkSchema,
|
|
]);
|
|
|
|
const sourceRelationshipSchema = z.strictObject({
|
|
from_path: relativePathSchema,
|
|
to: z.string().min(1),
|
|
kind: z.enum(["imports", "requires", "references", "declares-module"]),
|
|
resolution: z.enum(["internal", "external", "unresolved", "unknown"]),
|
|
parse_state: z.enum(["parsed", "partial", "unknown"]),
|
|
});
|
|
|
|
const parseFailureSchema = z.strictObject({
|
|
path: relativePathSchema,
|
|
parser: z.string().min(1),
|
|
reason: boundedTextSchema,
|
|
});
|
|
|
|
const exclusionSchema = z.strictObject({
|
|
path: relativePathSchema,
|
|
reason: z.enum([
|
|
"configured-secret",
|
|
"symlink-escape",
|
|
"unreadable",
|
|
"caller-excluded",
|
|
]),
|
|
});
|
|
|
|
const vcsSchema = z
|
|
.discriminatedUnion("kind", [
|
|
z.strictObject({
|
|
kind: z.literal("git"),
|
|
head: z.string().regex(/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/u),
|
|
dirty: z.boolean().nullable(),
|
|
}),
|
|
z.strictObject({
|
|
kind: z.literal("none"),
|
|
head: z.null(),
|
|
dirty: z.null(),
|
|
}),
|
|
z.strictObject({
|
|
kind: z.literal("unknown"),
|
|
head: z.null(),
|
|
dirty: z.null(),
|
|
}),
|
|
])
|
|
.nullable();
|
|
|
|
const graphShape = {
|
|
schema: z.literal("HistoricalSourceGraph"),
|
|
authority: z.literal("historical-reference"),
|
|
root_alias: z.literal("$REFERENCE_ROOT"),
|
|
root_sha256: digestSchema,
|
|
inventory_state: z.enum(["complete", "partial", "unknown"]),
|
|
entries: z.array(sourceEntrySchema),
|
|
relationships: z.array(sourceRelationshipSchema),
|
|
parse_failures: z.array(parseFailureSchema),
|
|
exclusions: z.array(exclusionSchema),
|
|
languages: z.array(z.string().min(1)),
|
|
manifests: z.array(relativePathSchema),
|
|
vcs: vcsSchema,
|
|
provenance: z.strictObject({
|
|
importer: z.string().min(1),
|
|
importer_version: z.string().min(1).nullable(),
|
|
caller: z
|
|
.string()
|
|
.min(1)
|
|
.regex(/^[\w .:@\x2f+\x2d]+$/u),
|
|
}),
|
|
limitations: z.array(boundedTextSchema),
|
|
};
|
|
|
|
const historicalSourceGraphStructureSchema = z
|
|
.strictObject(graphShape)
|
|
.superRefine(checkGraphInvariants);
|
|
|
|
export const historicalSourceGraphSchema =
|
|
historicalSourceGraphStructureSchema.superRefine((graph, context) => {
|
|
if (
|
|
computeRootSha256(graph.entries, graph.exclusions) !== graph.root_sha256
|
|
)
|
|
context.addIssue({
|
|
code: "custom",
|
|
message: "Historical source root commitment does not match entries",
|
|
path: ["root_sha256"],
|
|
});
|
|
});
|
|
|
|
const historicalSourceGraphInputSchema = z.strictObject({
|
|
...graphShape,
|
|
root_sha256: z.never().optional(),
|
|
});
|
|
|
|
const historicalSourceManifestBaseSchema = z.strictObject({
|
|
schema: z.literal("HistoricalSourceManifest"),
|
|
authority: z.literal("historical-reference"),
|
|
manifest_id: prefixedDigestSchema("hsm"),
|
|
graph_sha256: digestSchema,
|
|
root_sha256: digestSchema,
|
|
inventory_state: z.enum(["complete", "partial", "unknown"]),
|
|
entry_count: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER),
|
|
relationship_count: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER),
|
|
});
|
|
|
|
export type HistoricalSourceGraph = z.infer<typeof historicalSourceGraphSchema>;
|
|
export type HistoricalSourceGraphInput = z.infer<
|
|
typeof historicalSourceGraphInputSchema
|
|
>;
|
|
export type HistoricalSourceManifest = z.infer<
|
|
typeof historicalSourceManifestBaseSchema
|
|
>;
|
|
|
|
/** Stable identity used to sort and deduplicate source parse failures. */
|
|
export const historicalSourceParseFailureKey = (
|
|
failure: HistoricalSourceGraphInput["parse_failures"][number],
|
|
): string => `${failure.path}\u0000${failure.parser}\u0000${failure.reason}`;
|
|
|
|
const relationshipKey = (
|
|
relationship: z.infer<typeof sourceRelationshipSchema>,
|
|
): string =>
|
|
`${relationship.from_path}\u0000${relationship.to}\u0000${relationship.kind}\u0000${relationship.resolution}\u0000${relationship.parse_state}`;
|
|
|
|
const checkSortedUnique = (
|
|
values: readonly string[],
|
|
field: string,
|
|
context: z.RefinementCtx,
|
|
): void => {
|
|
for (let index = 1; index < values.length; index += 1)
|
|
if (
|
|
compareUnicodeCodePoints(values[index - 1] ?? "", values[index] ?? "") >=
|
|
0
|
|
) {
|
|
context.addIssue({
|
|
code: "custom",
|
|
message: `${field} must be sorted and unique by Unicode code point`,
|
|
path: [field, index],
|
|
});
|
|
return;
|
|
}
|
|
};
|
|
|
|
/** Derive the canonical language index from inventoried source files. */
|
|
export const historicalSourceLanguages = (
|
|
entries: readonly z.infer<typeof sourceEntrySchema>[],
|
|
): string[] =>
|
|
[
|
|
...new Set(
|
|
entries.flatMap((entry) =>
|
|
entry.kind === "file" && entry.language !== null
|
|
? [entry.language]
|
|
: [],
|
|
),
|
|
),
|
|
].sort(compareUnicodeCodePoints);
|
|
|
|
/** Derive the canonical manifest path index from inventoried source files. */
|
|
export const historicalSourceManifests = (
|
|
entries: readonly z.infer<typeof sourceEntrySchema>[],
|
|
): string[] =>
|
|
entries
|
|
.flatMap((entry) =>
|
|
entry.kind === "file" && entry.classifications.includes("manifest")
|
|
? [entry.path]
|
|
: [],
|
|
)
|
|
.sort(compareUnicodeCodePoints);
|
|
|
|
type GraphStructure = z.infer<typeof historicalSourceGraphStructureSchema>;
|
|
|
|
const checkRelationships = (
|
|
graph: GraphStructure,
|
|
paths: ReadonlySet<string>,
|
|
context: z.RefinementCtx,
|
|
): void => {
|
|
const files = new Set(
|
|
graph.entries.flatMap((entry) =>
|
|
entry.kind === "file" ? [entry.path] : [],
|
|
),
|
|
);
|
|
for (const [index, relationship] of graph.relationships.entries()) {
|
|
if (!files.has(relationship.from_path))
|
|
context.addIssue({
|
|
code: "custom",
|
|
message: "Relationship source must name an inventoried file",
|
|
path: ["relationships", index, "from_path"],
|
|
});
|
|
if (relationship.resolution !== "internal") continue;
|
|
const parsed = relativePathSchema.safeParse(relationship.to);
|
|
if (!parsed.success || !paths.has(parsed.data))
|
|
context.addIssue({
|
|
code: "custom",
|
|
message:
|
|
"Internal relationship target must be normalized and inventoried",
|
|
path: ["relationships", index, "to"],
|
|
});
|
|
}
|
|
};
|
|
|
|
const checkSymlinks = (
|
|
graph: GraphStructure,
|
|
paths: ReadonlySet<string>,
|
|
context: z.RefinementCtx,
|
|
): void => {
|
|
const normalizedPathCounts = new Map<string, number>();
|
|
for (const path of paths) {
|
|
const identity = normalizeJoinedLogicalPath(path);
|
|
normalizedPathCounts.set(
|
|
identity,
|
|
(normalizedPathCounts.get(identity) ?? 0) + 1,
|
|
);
|
|
}
|
|
for (const [index, entry] of graph.entries.entries()) {
|
|
if (entry.kind !== "symlink") continue;
|
|
if (entry.target === null) continue;
|
|
const absolute = isPortableAbsoluteSymlinkTarget(entry.target);
|
|
if (
|
|
(entry.target_state === "external" && !absolute) ||
|
|
(entry.target_state === "internal" && absolute)
|
|
)
|
|
context.addIssue({
|
|
code: "custom",
|
|
message: "Symlink target path does not match its target state",
|
|
path: ["entries", index, "target"],
|
|
});
|
|
if (entry.target_state !== "internal") continue;
|
|
const resolved = posix.normalize(
|
|
posix.join(posix.dirname(entry.path), entry.target),
|
|
);
|
|
const normalized = normalizeJoinedLogicalPath(resolved);
|
|
if (
|
|
resolved.startsWith("../") ||
|
|
resolved === ".." ||
|
|
(!paths.has(resolved) && normalizedPathCounts.get(normalized) !== 1)
|
|
)
|
|
context.addIssue({
|
|
code: "custom",
|
|
message: "Internal symlink target must resolve to an inventoried entry",
|
|
path: ["entries", index, "target"],
|
|
});
|
|
}
|
|
};
|
|
|
|
const checkCompleteInventory = (
|
|
graph: GraphStructure,
|
|
context: z.RefinementCtx,
|
|
): void => {
|
|
if (graph.inventory_state !== "complete") return;
|
|
const partialEntry = graph.entries.some(
|
|
(entry) =>
|
|
entry.limitations.length > 0 ||
|
|
(entry.kind === "file" && entry.content_state !== "hashed") ||
|
|
(entry.kind === "directory" && entry.tree_state !== "enumerated") ||
|
|
(entry.kind === "symlink" && entry.target_state !== "internal"),
|
|
);
|
|
const partialRelationship = graph.relationships.some(
|
|
({ parse_state, resolution }) =>
|
|
parse_state !== "parsed" ||
|
|
["unresolved", "unknown"].includes(resolution),
|
|
);
|
|
if (
|
|
!partialEntry &&
|
|
!partialRelationship &&
|
|
graph.parse_failures.length === 0 &&
|
|
graph.exclusions.length === 0 &&
|
|
graph.limitations.length === 0
|
|
)
|
|
return;
|
|
context.addIssue({
|
|
code: "custom",
|
|
message:
|
|
"Complete inventory cannot contain partial states, exclusions, failures, or limitations",
|
|
path: ["inventory_state"],
|
|
});
|
|
};
|
|
|
|
function checkGraphInvariants(
|
|
graph: GraphStructure,
|
|
context: z.RefinementCtx,
|
|
): void {
|
|
checkSortedUnique(
|
|
graph.entries.map(({ path }) => path),
|
|
"entries",
|
|
context,
|
|
);
|
|
for (const [index, entry] of graph.entries.entries())
|
|
checkSortedUnique(
|
|
entry.classifications,
|
|
`entries.${index}.classifications`,
|
|
context,
|
|
);
|
|
checkSortedUnique(graph.languages, "languages", context);
|
|
checkSortedUnique(graph.manifests, "manifests", context);
|
|
checkSortedUnique(
|
|
graph.relationships.map(relationshipKey),
|
|
"relationships",
|
|
context,
|
|
);
|
|
checkSortedUnique(
|
|
graph.parse_failures.map(historicalSourceParseFailureKey),
|
|
"parse_failures",
|
|
context,
|
|
);
|
|
checkSortedUnique(
|
|
graph.exclusions.map(({ path, reason }) => `${path}\u0000${reason}`),
|
|
"exclusions",
|
|
context,
|
|
);
|
|
|
|
if (
|
|
canonicalJson(graph.languages, "Reference source graph") !==
|
|
canonicalJson(
|
|
historicalSourceLanguages(graph.entries),
|
|
"Reference source graph",
|
|
)
|
|
)
|
|
context.addIssue({
|
|
code: "custom",
|
|
message: "languages must be derived from file entries",
|
|
path: ["languages"],
|
|
});
|
|
if (
|
|
canonicalJson(graph.manifests, "Reference source graph") !==
|
|
canonicalJson(
|
|
historicalSourceManifests(graph.entries),
|
|
"Reference source graph",
|
|
)
|
|
)
|
|
context.addIssue({
|
|
code: "custom",
|
|
message: "manifests must be derived from classified file entries",
|
|
path: ["manifests"],
|
|
});
|
|
|
|
const paths = new Set(graph.entries.map(({ path }) => path));
|
|
checkRelationships(graph, paths, context);
|
|
checkSymlinks(graph, paths, context);
|
|
checkCompleteInventory(graph, context);
|
|
}
|
|
|
|
const rootCommitment = (
|
|
entries: readonly z.infer<typeof sourceEntrySchema>[],
|
|
exclusions: readonly z.infer<typeof exclusionSchema>[],
|
|
): unknown => ({
|
|
entries: entries.map((entry) => {
|
|
switch (entry.kind) {
|
|
case "file":
|
|
return {
|
|
kind: entry.kind,
|
|
path: entry.path,
|
|
sha256: entry.sha256,
|
|
size: entry.size,
|
|
content_state: entry.content_state,
|
|
};
|
|
case "directory":
|
|
return {
|
|
kind: entry.kind,
|
|
path: entry.path,
|
|
tree_state: entry.tree_state,
|
|
};
|
|
case "symlink":
|
|
return {
|
|
kind: entry.kind,
|
|
path: entry.path,
|
|
target: entry.target,
|
|
target_state: entry.target_state,
|
|
};
|
|
}
|
|
}),
|
|
exclusions,
|
|
});
|
|
|
|
const computeRootSha256 = (
|
|
entries: readonly z.infer<typeof sourceEntrySchema>[],
|
|
exclusions: readonly z.infer<typeof exclusionSchema>[],
|
|
): string =>
|
|
digestCanonicalValue(
|
|
rootCommitment(entries, exclusions),
|
|
"Reference source graph",
|
|
);
|
|
|
|
/** Build and internally commit a normalized historical source graph. */
|
|
export const createHistoricalSourceGraph = (
|
|
input: unknown,
|
|
): HistoricalSourceGraph => {
|
|
const parsed = historicalSourceGraphInputSchema.parse(input);
|
|
return historicalSourceGraphSchema.parse({
|
|
...parsed,
|
|
root_sha256: computeRootSha256(parsed.entries, parsed.exclusions),
|
|
});
|
|
};
|
|
|
|
/** Parse a graph and reject stale or caller-invented root commitments. */
|
|
export const parseHistoricalSourceGraph = (
|
|
input: unknown,
|
|
): HistoricalSourceGraph => historicalSourceGraphSchema.parse(input);
|
|
|
|
/** Compute a deterministic graph commitment containing no absolute root path. */
|
|
export const computeHistoricalSourceGraphSha256 = (input: unknown): string =>
|
|
digestCanonicalValue(
|
|
parseHistoricalSourceGraph(input),
|
|
"Reference source graph",
|
|
);
|
|
|
|
/** Build a deterministic, relocation-independent manifest for a source graph. */
|
|
export const createHistoricalSourceManifest = (
|
|
input: unknown,
|
|
): HistoricalSourceManifest => {
|
|
const graph = parseHistoricalSourceGraph(input);
|
|
const semantic = {
|
|
schema: "HistoricalSourceManifest" as const,
|
|
authority: "historical-reference" as const,
|
|
graph_sha256: computeHistoricalSourceGraphSha256(graph),
|
|
root_sha256: graph.root_sha256,
|
|
inventory_state: graph.inventory_state,
|
|
entry_count: graph.entries.length,
|
|
relationship_count: graph.relationships.length,
|
|
};
|
|
return historicalSourceManifestBaseSchema.parse({
|
|
...semantic,
|
|
manifest_id: `hsm_${digestCanonicalValue(semantic, "Reference source graph")}`,
|
|
});
|
|
};
|
|
|
|
/** Parse and verify a historical source manifest's semantic identifier. */
|
|
export const parseHistoricalSourceManifest = (
|
|
input: unknown,
|
|
): HistoricalSourceManifest => {
|
|
const manifest = historicalSourceManifestBaseSchema.parse(input);
|
|
const { manifest_id: manifestId, ...semantic } = manifest;
|
|
if (
|
|
`hsm_${digestCanonicalValue(semantic, "Reference source graph")}` !==
|
|
manifestId
|
|
)
|
|
throw new TypeError("Historical source manifest identifier does not match");
|
|
return manifest;
|
|
};
|